Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into experimental

Dockerfile

@@ -0,0 +1,9 @@
+FROM node:12
+WORKDIR /usr/src/app
+COPY package.json .
+RUN npm install
+COPY index.js .
+COPY src src
+COPY entrypoint.sh .
+EXPOSE 8080
+CMD ./entrypoint.sh

config.json.example

@@ -8,6 +8,7 @@
     "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
     "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
     "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
+    "proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash)
     "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For"
     "db": "./databases/sponsorTimes.db",
     "privateDB": "./databases/private.db",

entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+echo 'Entrypoint script'
+cd /usr/src/app
+cp /etc/sponsorblock/config.json . || cat <<EOF > config.json
+{
+    "port": 8080,
+    "globalSalt": "[CHANGE THIS]",
+    "adminUserID": "[CHANGE THIS]",
+    "youtubeAPIKey": null,
+    "discordReportChannelWebhookURL": null, 
+    "discordFirstTimeSubmissionsWebhookURL": null, 
+    "discordAutoModWebhookURL": null,
+    "proxySubmission": null,
+    "behindProxy": "X-Forwarded-For",
+    "db": "./databases/sponsorTimes.db",
+    "privateDB": "./databases/private.db",
+    "createDatabaseIfNotExist": true,
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    "mode": "development",
+    "readOnly": false
+}
+EOF
+node index.js

index.js

@@ -1,5 +1,6 @@
 var config = require('./src/config.js');
 var createServer = require('./src/app.js');
+const logger = require('./src/utils/logger.js');
 var server = createServer(() => {
-  console.log("Server started.");
+  logger.info("Server started on port " + config.port + ".");
-});
+});

src/databases/Mysql.js

@@ -1,8 +1,10 @@
 var MysqlInterface = require('sync-mysql');
+var config = require('../config.js');
+const logger = require('../utils/logger.js');
 
 class Mysql {
-  constructor(config) {
+  constructor(msConfig) {
-    this.connection = new MysqlInterface(config);
+    this.connection = new MysqlInterface(msConfig);
   }
 
   exec(query) {
@@ -10,7 +12,7 @@ class Mysql {
   }
 
   prepare (type, query, params) {
-    (config.mode === "development") && console.log("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params);
+    logger.debug("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params);
     if (type === 'get') {
       return this.connection.query(query, params)[0];
     } else if (type === 'run') {
@@ -18,7 +20,7 @@ class Mysql {
     } else if (type === 'all') {
       return this.connection.query(query, params);
     } else {
-      console.log('returning undefined...')
+      logger.warn('returning undefined...');
       return undefined;
     }
   }

src/databases/Sqlite.js

@@ -1,4 +1,6 @@
 const { db } = require("./databases");
+var config = require('../config.js');
+const logger = require('../utils/logger.js');
 
 class Sqlite {
   constructor(connection) {
@@ -17,8 +19,8 @@ class Sqlite {
     } else if (type === 'all') {
       return this.connection.prepare(query).all(...params); 
     } else {
-      (config.mode === "development") && console.log('returning undefined...')
+      logger.debug('sqlite query: returning undefined')
-      (config.mode === "development") && console.log("prepare:        type: " + type + ", query: " + query + ", params: " + params);
+      logger.debug("prepare:        type: " + type + ", query: " + query + ", params: " + params);
       return undefined;
     }
   }

src/middleware/logger.js

@@ -1,7 +1,6 @@
-var fs = require('fs');
+const log = require('../utils/logger.js'); // log not logger to not interfere with function name
-var config = require('../config.js');
 
 module.exports = function logger (req, res, next) {
-  (config.mode === "development") && console.log('Request recieved: ' + req.url);
+  log.info('Request recieved: ' + req.url);
   next();
 }

src/routes/getSkipSegments.js

@@ -5,6 +5,7 @@ var databases = require('../databases/databases.js');
 var db = databases.db;
 var privateDB = databases.privateDB;
 
+var logger = require('../utils/logger.js');
 var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 
@@ -166,7 +167,7 @@ function handleGetSegments(req, res) {
 
     return segments;
   } catch (error) {
-    console.error(error);
+    logger.error(error);
     res.sendStatus(500);
 
     return false;

src/routes/getUsername.js

@@ -1,6 +1,7 @@
 var db = require('../databases/databases.js').db;
 
 var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
 
 module.exports = function getUsername (req, res) {
     let userID = req.query.userID;
@@ -28,7 +29,7 @@ module.exports = function getUsername (req, res) {
             });
         }
     } catch (err) {
-        console.log(err);
+        logger.error(err);
         res.sendStatus(500);
 
         return;

src/routes/getViewsForUser.js

@@ -1,6 +1,6 @@
 var db = require('../databases/databases.js').db;
 var getHash = require('../utils/getHash.js');
-
+var logger = require('../utils/logger.js');
 module.exports = function getViewsForUser(req, res) {
   let userID = req.query.userID;
 
@@ -25,7 +25,7 @@ module.exports = function getViewsForUser(req, res) {
           res.sendStatus(404);
       }
   } catch (err) {
-      console.log(err);
+      logger.error(err);
       res.sendStatus(500);
 
       return;

src/routes/oldSubmitSponsorTimes.js

@@ -1,5 +1,3 @@
-var config = require('../config.js');
-
 var postSkipSegments = require('./postSkipSegments.js');
 
 module.exports = async function submitSponsorTimes(req, res) {

src/routes/postSkipSegments.js

@@ -4,6 +4,7 @@ var databases = require('../databases/databases.js');
 var db = databases.db;
 var privateDB = databases.privateDB;
 var YouTubeAPI = require('../utils/youtubeAPI.js');
+var logger = require('../utils/logger.js');
 var request = require('request');
 var isoDurations = require('iso8601-duration');
 
@@ -25,7 +26,7 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) {
                 id: videoID
             }, function (err, data) {
                 if (err || data.items.length === 0) {
-                    err && console.log(err);
+                    err && logger.error(err);
                     return;
                 }
 
@@ -52,13 +53,13 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) {
                     }
                     }, (err, res) => {
                         if (err) {
-                            console.log("Failed to send first time submission Discord hook.");
+                            logger.error("Failed to send first time submission Discord hook.");
-                            console.log(JSON.stringify(err));
+                            logger.error(JSON.stringify(err));
-                            console.log("\n");
+                            logger.error("\n");
                         } else if (res && res.statusCode >= 400) {
-                            console.log("Error sending first time submission Discord hook");
+                            logger.error("Error sending first time submission Discord hook");
-                            console.log(JSON.stringify(res));
+                            logger.error(JSON.stringify(res));
-                            console.log("\n");
+                            logger.error("\n");
                         }
                 });
             });
@@ -69,9 +70,13 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) {
 // submission: {videoID, startTime, endTime}
 // callback:  function(reject: "String containing reason the submission was rejected")
 // returns: string when an error, false otherwise
+
+// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return 
+//   false for a pass - it was confusing and lead to this bug - any use of this function in 
+//   the future could have the same problem. 
 async function autoModerateSubmission(submission, callback) {
     // Get the video information from the youtube API
-    if (config.youtubeAPI !== null) {
+    if (config.youtubeAPIKey !== null) {
         let {err, data} = await new Promise((resolve, reject) => {
             YouTubeAPI.videos.list({
                 part: "contentDetails",
@@ -101,15 +106,31 @@ async function autoModerateSubmission(submission, callback) {
         }
          
     } else {
-        console.log("Skipped YouTube API");
+        logger.debug("Skipped YouTube API");
 
         // Can't moderate the submission without calling the youtube API
         // so allow by default.
-        return;
+        return false;
     }
 }
 
+function proxySubmission(req) {
+    request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => {
+        if (config.mode === 'development') {
+            if (!err) {
+                logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
+            } else {
+                logger.debug("Proxy Submission: Failed to make call");
+            }
+        }
+    });
+}
+
 module.exports = async function postSkipSegments(req, res) {
+    if (config.proxySubmission) {
+        proxySubmission(req);
+    }
+
     let videoID = req.query.videoID || req.body.videoID;
     let userID = req.query.userID || req.body.userID;
 
@@ -244,7 +265,7 @@ module.exports = async function postSkipSegments(req, res) {
             } catch (err) {
                 //a DB change probably occurred
                 res.sendStatus(502);
-                console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + 
+                logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + 
                     segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err);
                 
                 return;
@@ -254,7 +275,7 @@ module.exports = async function postSkipSegments(req, res) {
             sendDiscordNotification(userID, videoID, UUID, segmentInfo);
         }
     } catch (err) {
-        console.error(err);
+        logger.error(err);
 
         res.sendStatus(500);
 

src/routes/setUsername.js

@@ -3,6 +3,7 @@ var config = require('../config.js');
 
 var db = require('../databases/databases.js').db;
 var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
 
 
 module.exports = function setUsername(req, res) {
@@ -45,7 +46,7 @@ module.exports = function setUsername(req, res) {
 
       res.sendStatus(200);
   } catch (err) {
-      console.log(err);
+      logger.error(err);
       res.sendStatus(500);
 
       return;

src/routes/voteOnSponsorTime.js

@@ -11,6 +11,7 @@ var db = databases.db;
 var privateDB = databases.privateDB;
 var YouTubeAPI = require('../utils/youtubeAPI.js');
 var request = require('request');
+const logger = require('../utils/logger.js');
 
 function categoryVote(UUID, userID, isVIP, category, hashedIP, res) {
     // Check if they've already made a vote
@@ -96,6 +97,9 @@ async function voteOnSponsorTime(req, res) {
     //check if this user is on the vip list
     let isVIP = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [nonAnonUserID]).userCount > 0;
 
+    //check if user voting on own submission
+    let isOwnSubmission = db.prepare("get", "SELECT UUID as submissionCount FROM sponsorTimes where userID = ? AND UUID = ?", [nonAnonUserID, UUID]) !== undefined;
+        
     if (type === undefined && category !== undefined) {
         return categoryVote(UUID, userID, isVIP, category, hashedIP, res);
     }
@@ -165,13 +169,13 @@ async function voteOnSponsorTime(req, res) {
         let row = db.prepare('get', "SELECT votes, views FROM sponsorTimes WHERE UUID = ?", [UUID]);
 
         if (voteTypeEnum === voteTypes.normal) {
-            if (isVIP && incrementAmount < 0) {
+            if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
                 //this user is a vip and a downvote
                 incrementAmount = - (row.votes + 2 - oldIncrementAmount);
                 type = incrementAmount;
             }
         } else if (voteTypeEnum == voteTypes.incorrect) {
-            if (isVIP) {
+            if (isVIP || isOwnSubmission) {
                 //this user is a vip and a downvote
                 incrementAmount = 500 * incrementAmount;
                 type = incrementAmount < 0 ? 12 : 13;
@@ -203,7 +207,7 @@ async function voteOnSponsorTime(req, res) {
                         id: submissionInfoRow.videoID
                     }, function (err, data) {
                         if (err || data.items.length === 0) {
-                            err && console.log(err);
+                            err && logger.error(err);
                             return;
                         }
                         
@@ -232,13 +236,13 @@ async function voteOnSponsorTime(req, res) {
                             }
                         }, (err, res) => {
                             if (err) {
-                                console.log("Failed to send reported submission Discord hook.");
+                                logger.error("Failed to send reported submission Discord hook.");
-                                console.log(JSON.stringify(err));
+                                logger.error(JSON.stringify(err));
-                                console.log("\n");
+                                logger.error("\n");
                             } else if (res && res.statusCode >= 400) {
-                                console.log("Error sending reported submission Discord hook");
+                                logger.error("Error sending reported submission Discord hook");
-                                console.log(JSON.stringify(res));
+                                logger.error(JSON.stringify(res));
-                                console.log("\n");
+                                logger.error("\n");
                             }
                         });
                     });
@@ -299,7 +303,7 @@ async function voteOnSponsorTime(req, res) {
 
         res.sendStatus(200);
     } catch (err) {
-        console.error(err);
+        logger.error(err);
 
         res.status(500).json({error: 'Internal error creating segment vote'});
     }
@@ -310,4 +314,4 @@ module.exports = {
     endpoint: function (req, res) {
        voteOnSponsorTime(req, res);
     },
-  };
+  };

src/utils/logger.js

@@ -0,0 +1,36 @@
+const config = require('../config.js');
+
+const levels = {
+  ERROR: "ERROR",
+  WARN: "WARN",
+  INFO: "INFO",
+  DEBUG: "DEBUG"
+};
+
+const settings = {
+  ERROR: true,
+  WARN: true,
+  INFO: false,
+  DEBUG: false
+};
+
+if (config.mode === 'development') {
+  settings.INFO = true;
+  settings.DEBUG = true;
+}
+
+function log(level, string) {
+  if (!!settings[level]) {
+    if (level.length === 4) {level = level + " "}; // ensure logs are aligned
+    console.log(level + " " + new Date().toISOString() + " : " + string);
+  }
+}
+
+module.exports = {
+  levels,
+  log,
+  error: (string) => {log(levels.ERROR, string)},
+  warn: (string) => {log(levels.WARN, string)},
+  info: (string) => {log(levels.INFO, string)},
+  debug: (string) => {log(levels.DEBUG, string)},
+};

test/cases/voteOnSponsorTime.js

@@ -20,6 +20,8 @@ describe('voteOnSponsorTime', () => {
     db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0)");
     db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0)");
     db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)");
 
     db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
     privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");
@@ -151,6 +153,42 @@ describe('voteOnSponsorTime', () => {
     });
   });
 
+  it('should be able to completely downvote your own segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=own-submission-id&UUID=own-submission-uuid&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["own-submission-uuid"]);
+          if (row.votes <= -2) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 500 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('should not be able to completely downvote somebody elses segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=not-own-submission-uuid&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["not-own-submission-uuid"]);
+          if (row.votes === 499) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 500 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
   it('Should be able to vote for a category and it should immediately change (for now)', (done) => {
     request.get(utils.getbaseURL() 
      + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=intro", null,