mreged no-segments

databases/_upgrade_sponsorTimes_2.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+/* Add new table: noSegments */
+CREATE TABLE "noSegments" (
+	"videoID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL
+);
+
+/* Add version to config */
+UPDATE config SET value = 2 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_3.sql

@@ -1,7 +1,7 @@
 BEGIN TRANSACTION;
 
 /* Add hash field */
-CREATE TABLE "sqlb_temp_table_1" (
+CREATE TABLE "sqlb_temp_table_3" (
 	"videoID"	TEXT NOT NULL,
 	"startTime"	REAL NOT NULL,
 	"endTime"	REAL NOT NULL,
@@ -12,15 +12,15 @@ CREATE TABLE "sqlb_temp_table_1" (
 	"timeSubmitted"	INTEGER NOT NULL,
 	"views"	INTEGER NOT NULL,
 	"category"	TEXT NOT NULL DEFAULT "sponsor",
-	"hashedVideoID"	TEXT NOT NULL,
+	"shadowHidden"	INTEGER NOT NULL,
-	"shadowHidden"	INTEGER NOT NULL
+	"hashedVideoID"	TEXT NOT NULL 
 );
-INSERT INTO sqlb_temp_table_1 SELECT *, sha256(videoID) FROM sponsorTimes;
+INSERT INTO sqlb_temp_table_3 SELECT *, sha256(videoID) FROM sponsorTimes;
 
 DROP TABLE sponsorTimes;
-ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
+ALTER TABLE sqlb_temp_table_3 RENAME TO "sponsorTimes";
 
 /* Bump version in config */
-UPDATE config SET value = 2 WHERE key = "version";
+UPDATE config SET value = 3 WHERE key = "version";
 
 COMMIT;

src/app.js

@@ -23,6 +23,8 @@ var getViewsForUser = require('./routes/getViewsForUser.js');
 var getTopUsers = require('./routes/getTopUsers.js');
 var getTotalStats = require('./routes/getTotalStats.js');
 var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js');
+var postNoSegments = require('./routes/postNoSegments.js');
+var getIsUserVIP = require('./routes/getIsUserVIP.js');
 
 // Old Routes
 var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js');
@@ -93,6 +95,13 @@ app.get('/api/getTotalStats', getTotalStats);
 //send out a formatted time saved total
 app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
 
+//submit video containing no segments
+app.post('/api/postNoSegments', postNoSegments);
+
+//get if user is a vip
+app.get('/api/getIsUserVIP', getIsUserVIP);
+
+
 app.get('/database.db', function (req, res) {
     res.sendFile("./databases/sponsorTimes.db", { root: "./" });
 });

src/databases/Mysql.js

@@ -1,5 +1,4 @@
 var MysqlInterface = require('sync-mysql');
-var config = require('../config.js');
 const logger = require('../utils/logger.js');
 
 class Mysql {

src/databases/databases.js

@@ -3,7 +3,8 @@ var Sqlite3 = require('better-sqlite3');
 var fs = require('fs');
 var path = require('path');
 var Sqlite = require('./Sqlite.js')
-var Mysql = require('./Mysql.js')
+var Mysql = require('./Mysql.js');
+const logger = require('../utils/logger.js');
 
 let options = {
   readonly: config.readOnly,
@@ -33,6 +34,10 @@ if (config.mysql) {
   }
 
   if (!config.readOnly) {
+    db.function("sha256", function (string) {
+      return require('crypto').createHash("sha256").update(string).digest("hex");
+    });
+    
     // Upgrade database if required
     ugradeDB(db, "sponsorTimes");
     ugradeDB(privateDB, "private")
@@ -61,11 +66,15 @@ if (config.mysql) {
     let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
 
     let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
+    logger.debug('db update: trying ' + path);
     while (fs.existsSync(path)) {
+      logger.debug('db update: updating ' + path);
       db.exec(fs.readFileSync(path).toString());
 
       versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
       path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
+      logger.debug('db update: trying ' + path);
     }
+    logger.debug('db update: no file ' + path);
   }
 }

src/routes/getIsUserVIP.js

@@ -0,0 +1,31 @@
+var db = require('../databases/databases.js').db;
+
+var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+
+module.exports = function getUsername (req, res) {
+    let userID = req.query.userID;
+
+    if (userID == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    try {
+        let vipState = isUserVIP(userID);
+        res.status(200).json({
+            hashedUserID: userID,
+            vip: vipState
+        });
+    } catch (err) {
+        logger.error(err);
+        res.sendStatus(500);
+
+        return;
+    }
+}

src/routes/getSkipSegments.js

@@ -1,4 +1,3 @@
-var fs = require('fs');
 var config = require('../config.js');
 
 var databases = require('../databases/databases.js');

src/routes/postNoSegments.js

@@ -0,0 +1,78 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+module.exports = (req, res) => {
+    // Collect user input data
+    let videoID = req.body.videoID;
+    let userID = req.body.userID;
+    let categorys = req.body.categorys;
+
+    // Check input data is valid
+    if (!videoID 
+        || !userID 
+        || !categorys 
+        || !Array.isArray(categorys) 
+        || categorys.length === 0
+    ) {
+        res.status(400).json({
+            status: 400,
+            message: 'Bad Format'
+        });
+        return;
+    }
+
+    // Check if user is VIP
+    userID = getHash(userID);
+    let userIsVIP = isUserVIP(userID);
+
+    if (!userIsVIP) {
+        res.status(403).json({
+            status: 403,
+            message: 'Must be a VIP to mark videos.'
+        });
+        return;
+    }
+
+    // Get existing no segment markers
+    let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]);
+    if (!noSegmentList || noSegmentList.length === 0) {
+        noSegmentList = [];
+    } else {
+        noSegmentList = noSegmentList.map((obj) => {
+            return obj.category;
+        });
+    }
+
+    // get user categorys not already submitted that match accepted format
+    let categorysToMark = categorys.filter((category) => {
+        return !!category.match(/^[a-zA-Z]+$/);
+    }).filter((category) => {
+        return noSegmentList.indexOf(category) === -1;
+    });
+
+    // remove any duplicates
+    categorysToMark = categorysToMark.filter((category, index) => {
+        return categorysToMark.indexOf(category) === index;
+    });
+
+    // create database entry
+    categorysToMark.forEach((category) => {
+        try {
+            db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]);
+        } catch (err) {
+            logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'");
+            logger.error(err);
+            res.status(500).json({
+                status: 500,
+                message: "Internal Server Error: Could not write marker to the database."
+            });
+        }
+    });
+
+    res.status(200).json({
+        status: 200,
+        submitted: categorysToMark
+    }); 
+};

src/routes/postSkipSegments.js

@@ -148,9 +148,9 @@ function proxySubmission(req) {
     request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => {
         if (config.mode === 'development') {
             if (!err) {
-                logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
+                logger.debug('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
             } else {
-                logger.debug("Proxy Submission: Failed to make call");
+                logger.error("Proxy Submission: Failed to make call");
             }
         }
     });
@@ -187,6 +187,7 @@ module.exports = async function postSkipSegments(req, res) {
     //hash the ip 5000 times so no one can get it from the database
     let hashedIP = getHash(getIP(req) + config.globalSalt);
 
+    let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
     // Check if all submissions are correct
     for (let i = 0; i < segments.length; i++) {
         if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@@ -195,6 +196,18 @@ module.exports = async function postSkipSegments(req, res) {
             return;
         }
 
+        // Reject segemnt if it's in the no segments list
+        if (noSegmentList.indexOf(segments[i].category) !== -1) {
+            // TODO: Do something about the fradulent submission
+            logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
+            res.status(403).send(
+              "Request rejected by auto moderator: This video has been reported as not containing any segments with the category '"
+              + segments[i].category + "'. If you believe this is incorrect, contact someone on Discord."
+            );
+            return;
+        }
+        
+
         let startTime = parseFloat(segments[i].segment[0]);
         let endTime = parseFloat(segments[i].segment[1]);
 

src/routes/voteOnSponsorTime.js

@@ -5,7 +5,7 @@ var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 var getFormattedTime = require('../utils/getFormattedTime.js');
 var isUserTrustworthy = require('../utils/isUserTrustworthy.js');
-const {getVoteAuthor, getVoteAuthorRaw, dispatchEvent} = require('../utils/webhookUtils.js');
+const { getVoteAuthor, getVoteAuthorRaw, dispatchEvent } = require('../utils/webhookUtils.js');
 
 var databases = require('../databases/databases.js');
 var db = databases.db;

src/utils/isUserVIP.js

@@ -0,0 +1,8 @@
+const databases = require('../databases/databases.js');
+const db = databases.db;
+
+module.exports = (userID) => {
+    return db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0;
+}
+
+

test.js

@@ -9,6 +9,7 @@ if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB);
 
 var createServer = require('./src/app.js');
 var createMockServer = require('./test/mocks.js');
+const logger = require('./src/utils/logger.js');
 
 // Instantiate a Mocha instance.
 var mocha = new Mocha();
@@ -27,9 +28,9 @@ fs.readdirSync(testDir).filter(function(file) {
 });
 
 var mockServer = createMockServer(() => {
-  console.log("Started mock HTTP Server");
+  logger.info("Started mock HTTP Server");
   var server = createServer(() => {
-    console.log("Started main HTTP server");
+    logger.info("Started main HTTP server");
     // Run the tests.
     mocha.run(function(failures) {
       mockServer.close();

test/cases/getIsUserVIP.js

@@ -0,0 +1,57 @@
+var request = require('request');
+var utils = require('../utils.js');
+var db = require('../../src/databases/databases.js').db;
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getSavedTimeForUser', () => {
+  before(() => {
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("supertestman") + "')");
+  }); 
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getIsUserVIP?userID=supertestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+
+  it('Should get a 400 if no userID', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getIsUserVIP", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+  it('Should say a VIP is a VIP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getIsUserVIP?userID=supertestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else {
+          if (JSON.parse(body).vip === true) done(); // pass
+          else done("Result was non-vip when should have been vip");
+        }
+      });
+  });
+
+  it('Should say a normal user is not a VIP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getIsUserVIP?userID=regulartestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else {
+          if (JSON.parse(body).vip === false) done(); // pass
+          else done("Result was vip when should have been non-vip");
+        }
+      });
+  });
+});

test/cases/noSegmentRecords.js

@@ -0,0 +1,325 @@
+var request = require('request');
+
+var utils = require('../utils.js');
+const getHash = require('../../src/utils/getHash.js');
+
+var databases = require('../../src/databases/databases.js');
+const logger = require('../../src/utils/logger.js');
+var db = databases.db;
+
+describe('noSegmentRecords', () => {
+  before(() => {
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')");
+  });
+
+  it('Should update the database version when starting the application', (done) => {
+    let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    if (version > 1) done();
+    else done('Version isn\'t greater that 1. Version is ' + version);
+  });
+
+  it('Should be able to submit categorys not in video (http response)', (done) => {
+    let json = {
+      videoID: 'no-segments-video-id',
+      userID: 'VIPUser-noSegments',
+      categorys: [
+        'outro',
+        'shilling',
+        'shilling',
+        'shil ling',
+        '',
+        'intro'
+      ]
+    };
+
+    let expected = {
+      status: 200,
+      submitted: [
+        'outro',
+        'shilling'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          if (JSON.stringify(body) === JSON.stringify(expected)) {
+            done();
+          } else {
+            done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(body));
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categorys not in video (sql check)', (done) => {
+    let json = {
+      videoID: 'no-segments-video-id-1',
+      userID: 'VIPUser-noSegments',
+      categorys: [
+        'outro',
+        'shilling',
+        'shilling',
+        'shil ling',
+        '',
+        'intro'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['no-segments-video-id-1']);
+          if (result.length !== 4) {
+            console.log(result);
+            done("Expected 4 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for missing params', (done) => {
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json: {}}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no categorys', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categorys: []
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no userID', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: null,
+      categorys: ['sponsor']
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no videoID', (done) => {
+    let json = {
+      videoID: null,
+      userID: 'test',
+      categorys: ['sponsor']
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 object categorys)', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categorys: {}
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 bad format categorys', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categorys: 'sponsor'
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+  
+  it('Should return 403 if user is not VIP', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categorys: [
+        'sponsor'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/postNoSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 403) {
+          done();
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  /*
+   * Submission tests in this file do not check database records, only status codes.
+   * To test the submission code properly see ./test/cases/postSkipSegments.js
+   */
+
+  it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "sponsor"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 403) {
+        done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+  it('Should not be able to submit segments to a video where any of the submissions with a no-segment record', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "sponsor"
+         },{
+          segment: [50, 60],
+          category: "intro"
+        }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 403) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+
+  it('Should  be able to submit a segment to a video with a different no-segment record', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "intro"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 200) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+  it('Should be able to submit a segment to a video with no no-segment records', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "normalVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "intro"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 200) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+});

test/cases/voteOnSponsorTime.js

@@ -5,7 +5,7 @@ const getHash = require('../../src/utils/getHash.js');
 
 describe('voteOnSponsorTime', () => {
   before(() => {
-    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
     db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
     db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
     db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest')+"')");