Merge pull request #114 from Joe-Dowd/no-segments

Mark video as containing no segments

databases/_upgrade_sponsorTimes_2.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+/* Add new table: noSegments */
+CREATE TABLE "noSegments" (
+	"videoID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL
+);
+
+/* Add version to config */
+UPDATE config SET value = 2 WHERE key = 'version';
+
+COMMIT;

src/app.js

@@ -22,6 +22,8 @@ var getViewsForUser = require('./routes/getViewsForUser.js');
 var getTopUsers = require('./routes/getTopUsers.js');
 var getTotalStats = require('./routes/getTotalStats.js');
 var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js');
+var postNoSegments = require('./routes/postNoSegments.js');
+var getIsUserVIP = require('./routes/getIsUserVIP.js');
 
 // Old Routes
 var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js');
@@ -89,6 +91,13 @@ app.get('/api/getTotalStats', getTotalStats);
 //send out a formatted time saved total
 app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
 
+//submit video containing no segments
+app.post('/api/noSegments', postNoSegments);
+
+//get if user is a vip
+app.get('/api/isUserVIP', getIsUserVIP);
+
+
 app.get('/database.db', function (req, res) {
     res.sendFile("./databases/sponsorTimes.db", { root: "./" });
 });

src/databases/Mysql.js

@@ -1,5 +1,4 @@
 var MysqlInterface = require('sync-mysql');
-var config = require('../config.js');
 const logger = require('../utils/logger.js');
 
 class Mysql {

src/databases/databases.js

@@ -3,7 +3,8 @@ var Sqlite3 = require('better-sqlite3');
 var fs = require('fs');
 var path = require('path');
 var Sqlite = require('./Sqlite.js')
-var Mysql = require('./Mysql.js')
+var Mysql = require('./Mysql.js');
+const logger = require('../utils/logger.js');
 
 let options = {
   readonly: config.readOnly,
@@ -60,12 +61,16 @@ if (config.mysql) {
     let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
     let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
 
-    let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql";
+    let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
+    logger.debug('db update: trying ' + path);
     while (fs.existsSync(path)) {
+      logger.debug('db update: updating ' + path);
       db.exec(fs.readFileSync(path).toString());
 
       versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
-      path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql";
+      path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
+      logger.debug('db update: trying ' + path);
     }
+    logger.debug('db update: no file ' + path);
   }
 }

src/routes/getIsUserVIP.js

@@ -0,0 +1,31 @@
+var db = require('../databases/databases.js').db;
+
+var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+
+module.exports = (req, res) => {
+    let userID = req.query.userID;
+
+    if (userID == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    try {
+        let vipState = isUserVIP(userID);
+        res.status(200).json({
+            hashedUserID: userID,
+            vip: vipState
+        });
+    } catch (err) {
+        logger.error(err);
+        res.sendStatus(500);
+
+        return;
+    }
+}

src/routes/getSkipSegments.js

@@ -1,4 +1,3 @@
-var fs = require('fs');
 var config = require('../config.js');
 
 var databases = require('../databases/databases.js');

src/routes/postNoSegments.js

@@ -0,0 +1,74 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+module.exports = (req, res) => {
+    // Collect user input data
+    let videoID = req.body.videoID;
+    let userID = req.body.userID;
+    let categories = req.body.categories;
+
+    // Check input data is valid
+    if (!videoID 
+        || !userID 
+        || !categories 
+        || !Array.isArray(categories) 
+        || categories.length === 0
+    ) {
+        res.status(400).json({
+            message: 'Bad Format'
+        });
+        return;
+    }
+
+    // Check if user is VIP
+    userID = getHash(userID);
+    let userIsVIP = isUserVIP(userID);
+
+    if (!userIsVIP) {
+        res.status(403).json({
+            message: 'Must be a VIP to mark videos.'
+        });
+        return;
+    }
+
+    // Get existing no segment markers
+    let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]);
+    if (!noSegmentList || noSegmentList.length === 0) {
+        noSegmentList = [];
+    } else {
+        noSegmentList = noSegmentList.map((obj) => {
+            return obj.category;
+        });
+    }
+
+    // get user categories not already submitted that match accepted format
+    let categoriesToMark = categories.filter((category) => {
+        return !!category.match(/^[_a-zA-Z]+$/);
+    }).filter((category) => {
+        return noSegmentList.indexOf(category) === -1;
+    });
+
+    // remove any duplicates
+    categoriesToMark = categoriesToMark.filter((category, index) => {
+        return categoriesToMark.indexOf(category) === index;
+    });
+
+    // create database entry
+    categoriesToMark.forEach((category) => {
+        try {
+            db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]);
+        } catch (err) {
+            logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'");
+            logger.error(err);
+            res.status(500).json({
+                message: "Internal Server Error: Could not write marker to the database."
+            });
+        }
+    });
+
+    res.status(200).json({
+        submitted: categoriesToMark
+    }); 
+};

src/routes/postSkipSegments.js

@@ -148,9 +148,9 @@ function proxySubmission(req) {
     request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => {
         if (config.mode === 'development') {
             if (!err) {
-                logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
+                logger.debug('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
             } else {
-                logger.debug("Proxy Submission: Failed to make call");
+                logger.error("Proxy Submission: Failed to make call");
             }
         }
     });
@@ -187,6 +187,7 @@ module.exports = async function postSkipSegments(req, res) {
     //hash the ip 5000 times so no one can get it from the database
     let hashedIP = getHash(getIP(req) + config.globalSalt);
 
+    let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
     // Check if all submissions are correct
     for (let i = 0; i < segments.length; i++) {
         if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@@ -195,6 +196,18 @@ module.exports = async function postSkipSegments(req, res) {
             return;
         }
 
+        // Reject segemnt if it's in the no segments list
+        if (noSegmentList.indexOf(segments[i].category) !== -1) {
+            // TODO: Do something about the fradulent submission
+            logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
+            res.status(403).send(
+              "Request rejected by auto moderator: This video has been reported as not containing any segments with the category '"
+              + segments[i].category + "'. If you believe this is incorrect, contact someone on Discord."
+            );
+            return;
+        }
+        
+
         let startTime = parseFloat(segments[i].segment[0]);
         let endTime = parseFloat(segments[i].segment[1]);
 

src/routes/voteOnSponsorTime.js

@@ -5,7 +5,7 @@ var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 var getFormattedTime = require('../utils/getFormattedTime.js');
 var isUserTrustworthy = require('../utils/isUserTrustworthy.js');
-const {getVoteAuthor, getVoteAuthorRaw, dispatchEvent} = require('../utils/webhookUtils.js');
+const { getVoteAuthor, getVoteAuthorRaw, dispatchEvent } = require('../utils/webhookUtils.js');
 
 var databases = require('../databases/databases.js');
 var db = databases.db;

src/utils/isUserVIP.js

@@ -0,0 +1,8 @@
+const databases = require('../databases/databases.js');
+const db = databases.db;
+
+module.exports = (userID) => {
+    return db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0;
+}
+
+

src/utils/logger.js

@@ -46,7 +46,7 @@ if (config.mode === 'development') {
   settings.INFO = true;
   settings.DEBUG = true;
 } else if (config.mode === 'test') {
-    settings.WARN = false;
+  settings.WARN = false;
 }
 
 function log(level, string) {

test.js

@@ -9,6 +9,7 @@ if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB);
 
 var createServer = require('./src/app.js');
 var createMockServer = require('./test/mocks.js');
+const logger = require('./src/utils/logger.js');
 
 // Instantiate a Mocha instance.
 var mocha = new Mocha();
@@ -27,9 +28,9 @@ fs.readdirSync(testDir).filter(function(file) {
 });
 
 var mockServer = createMockServer(() => {
-  console.log("Started mock HTTP Server");
+  logger.info("Started mock HTTP Server");
   var server = createServer(() => {
-    console.log("Started main HTTP server");
+    logger.info("Started main HTTP server");
     // Run the tests.
     mocha.run(function(failures) {
       mockServer.close();

test/cases/getIsUserVIP.js

@@ -0,0 +1,57 @@
+var request = require('request');
+var utils = require('../utils.js');
+var db = require('../../src/databases/databases.js').db;
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getIsUserVIP', () => {
+  before(() => {
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("supertestman") + "')");
+  }); 
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP?userID=supertestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+
+  it('Should get a 400 if no userID', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+  it('Should say a VIP is a VIP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP?userID=supertestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else {
+          if (JSON.parse(body).vip === true) done(); // pass
+          else done("Result was non-vip when should have been vip");
+        }
+      });
+  });
+
+  it('Should say a normal user is not a VIP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP?userID=regulartestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else {
+          if (JSON.parse(body).vip === false) done(); // pass
+          else done("Result was vip when should have been non-vip");
+        }
+      });
+  });
+});

test/cases/noSegmentRecords.js

@@ -0,0 +1,408 @@
+var request = require('request');
+
+var utils = require('../utils.js');
+const getHash = require('../../src/utils/getHash.js');
+
+var databases = require('../../src/databases/databases.js');
+const logger = require('../../src/utils/logger.js');
+var db = databases.db;
+
+describe('noSegmentRecords', () => {
+  before(() => {
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')");
+  });
+
+  it('Should update the database version when starting the application', (done) => {
+    let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    if (version > 1) done();
+    else done('Version isn\'t greater that 1. Version is ' + version);
+  });
+
+  it('Should be able to submit categories not in video (http response)', (done) => {
+    let json = {
+      videoID: 'no-segments-video-id',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'outro',
+        'shilling',
+        'shilling',
+        'shil ling',
+        '',
+        'intro'
+      ]
+    };
+
+    let expected = {
+      submitted: [
+        'outro',
+        'shilling'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          if (JSON.stringify(body) === JSON.stringify(expected)) {
+            done();
+          } else {
+            done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(body));
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categories not in video (sql check)', (done) => {
+    let json = {
+      videoID: 'no-segments-video-id-1',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'outro',
+        'shilling',
+        'shilling',
+        'shil ling',
+        '',
+        'intro'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['no-segments-video-id-1']);
+          if (result.length !== 4) {
+            console.log(result);
+            done("Expected 4 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categories with _ in the category', (done) => {
+    let json = {
+      videoID: 'underscore',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'word_word',
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['underscore']);
+          if (result.length !== 1) {
+            console.log(result);
+            done("Expected 1 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categories with upper and lower case in the category', (done) => {
+    let json = {
+      videoID: 'bothCases',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'wordWord',
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['bothCases']);
+          if (result.length !== 1) {
+            console.log(result);
+            done("Expected 1 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should not be able to submit categories with $ in the category', (done) => {
+    let json = {
+      videoID: 'specialChar',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'word&word',
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['specialChar']);
+          if (result.length !== 0) {
+            console.log(result);
+            done("Expected 0 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for missing params', (done) => {
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json: {}}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no categories', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: []
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no userID', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: null,
+      categories: ['sponsor']
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no videoID', (done) => {
+    let json = {
+      videoID: null,
+      userID: 'test',
+      categories: ['sponsor']
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 object categories)', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: {}
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 bad format categories', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: 'sponsor'
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+  
+  it('Should return 403 if user is not VIP', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: [
+        'sponsor'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 403) {
+          done();
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  /*
+   * Submission tests in this file do not check database records, only status codes.
+   * To test the submission code properly see ./test/cases/postSkipSegments.js
+   */
+
+  it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "sponsor"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 403) {
+        done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+  it('Should not be able to submit segments to a video where any of the submissions with a no-segment record', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "sponsor"
+         },{
+          segment: [50, 60],
+          category: "intro"
+        }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 403) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+
+  it('Should  be able to submit a segment to a video with a different no-segment record', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "intro"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 200) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+  it('Should be able to submit a segment to a video with no no-segment records', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "normalVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "intro"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 200) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+});