Merge pull request #127 from Joe-Dowd/query-by-hash-prefix

Query by hash prefix

databases/_upgrade_private_1.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* for testing the db upgrade, don't remove because it looks empty */
+
+/* Add version to config */
+INSERT INTO config (key, value) VALUES("version", 1);
+
+COMMIT;

databases/_upgrade_sponsorTimes_3.sql

@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+CREATE TABLE "sqlb_temp_table_3" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT "sponsor",
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID"	TEXT NOT NULL 
+);
+
+/* hash upgade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */
+INSERT INTO sqlb_temp_table_3 SELECT *, sha256(videoID) FROM sponsorTimes;
+
+DROP TABLE sponsorTimes;
+ALTER TABLE sqlb_temp_table_3 RENAME TO "sponsorTimes";
+
+/* Bump version in config */
+UPDATE config SET value = 3 WHERE key = "version";
+
+COMMIT;

src/app.js

@@ -11,6 +11,7 @@ const userCounter = require('./middleware/userCounter.js');
 // Routes
 var getSkipSegments = require('./routes/getSkipSegments.js').endpoint;
 var postSkipSegments = require('./routes/postSkipSegments.js');
+var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js');
 var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js');
 var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js');
 var setUsername = require('./routes/setUsername.js');
@@ -53,6 +54,9 @@ app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
 app.get('/api/skipSegments', getSkipSegments);
 app.post('/api/skipSegments', postSkipSegments);
 
+// add the privacy protecting skip segments functions
+app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash);
+
 //voting endpoint
 app.get('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);
 app.post('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);

src/databases/databases.js

@@ -5,6 +5,7 @@ var path = require('path');
 var Sqlite = require('./Sqlite.js')
 var Mysql = require('./Mysql.js');
 const logger = require('../utils/logger.js');
+const getHash = require('../utils/getHash.js');
 
 let options = {
   readonly: config.readOnly,
@@ -34,6 +35,10 @@ if (config.mysql) {
   }
 
   if (!config.readOnly) {
+    db.function("sha256", (string) => {
+      return getHash(string, 1);
+    });
+    
     // Upgrade database if required
     ugradeDB(db, "sponsorTimes");
     ugradeDB(privateDB, "private")

src/routes/getSkipSegments.js

@@ -8,45 +8,101 @@ var logger = require('../utils/logger.js');
 var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 
+function cleanGetSegments(videoID, categories) {
+    let userHashedIP, shadowHiddenSegments;
+
+    let segments = [];
+
+    try {
+        for (const category of categories) {
+            const categorySegments = db
+                .prepare(
+                    'all',
+                    'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
+                    [videoID, category]
+                )
+                .filter(segment => {
+                    if (segment.votes < -1) {
+                        return false; //too untrustworthy, just ignore it
+                    }
+
+                    //check if shadowHidden
+                    //this means it is hidden to everyone but the original ip that submitted it
+                    if (segment.shadowHidden != 1) {
+                        return true;
+                    }
+
+                    if (shadowHiddenSegments === undefined) {
+                        shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
+                    }
+
+                    //if this isn't their ip, don't send it to them
+                    return shadowHiddenSegments.some(shadowHiddenSegment => {
+                        if (userHashedIP === undefined) {
+                            //hash the IP only if it's strictly necessary
+                            userHashedIP = getHash(getIP(req) + config.globalSalt);
+                        }
+                        return shadowHiddenSegment.hashedIP === userHashedIP;
+                    });
+                });
+
+            chooseSegments(categorySegments).forEach(chosenSegment => {
+                segments.push({
+                    category,
+                    segment: [chosenSegment.startTime, chosenSegment.endTime],
+                    UUID: chosenSegment.UUID,
+                });
+            });
+        }
+
+        return segments;
+    } catch (err) {
+        if (err) {
+            logger.error(err);
+            return null;
+        }
+    }
+}
+
 //gets a weighted random choice from the choices array based on their `votes` property.
 //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
 //choices are unique
 function getWeightedRandomChoice(choices, amountOfChoices) {
-  //trivial case: no need to go through the whole process
+    //trivial case: no need to go through the whole process
-  if (amountOfChoices >= choices.length) {
+    if (amountOfChoices >= choices.length) {
-    return choices;
+        return choices;
-  }
-
-  //assign a weight to each choice
-  let totalWeight = 0;
-  choices = choices.map(choice => {
-    //The 3 makes -2 the minimum votes before being ignored completely
-    //https://www.desmos.com/calculator/c1duhfrmts
-    //this can be changed if this system increases in popularity.
-    const weight = Math.exp((choice.votes + 3), 0.85);
-    totalWeight += weight;
-
-    return { ...choice, weight };
-  });
-
-  //iterate and find amountOfChoices choices
-  const chosen = [];
-  while (amountOfChoices-- > 0) {
-    //weighted random draw of one element of choices
-    const randomNumber = Math.random() * totalWeight;
-    let stackWeight = choices[0].weight;
-    let i = 0;
-    while (stackWeight < randomNumber) {
-      stackWeight += choices[++i].weight;
     }
 
-    //add it to the chosen ones and remove it from the choices before the next iteration
+    //assign a weight to each choice
-    chosen.push(choices[i]);
+    let totalWeight = 0;
-    totalWeight -= choices[i].weight;
+    choices = choices.map(choice => {
-    choices.splice(i, 1);
+        //The 3 makes -2 the minimum votes before being ignored completely
-  }
+        //https://www.desmos.com/calculator/c1duhfrmts
+        //this can be changed if this system increases in popularity.
+        const weight = Math.exp((choice.votes + 3), 0.85);
+        totalWeight += weight;
 
-  return chosen;
+        return { ...choice, weight };
+    });
+
+    //iterate and find amountOfChoices choices
+    const chosen = [];
+    while (amountOfChoices-- > 0) {
+        //weighted random draw of one element of choices
+        const randomNumber = Math.random() * totalWeight;
+        let stackWeight = choices[0].weight;
+        let i = 0;
+        while (stackWeight < randomNumber) {
+            stackWeight += choices[++i].weight;
+        }
+
+        //add it to the chosen ones and remove it from the choices before the next iteration
+        chosen.push(choices[i]);
+        totalWeight -= choices[i].weight;
+        choices.splice(i, 1);
+    }
+
+    return chosen;
 }
 
 //This function will find segments that are contained inside of eachother, called similar segments
@@ -54,34 +110,34 @@ function getWeightedRandomChoice(choices, amountOfChoices) {
 //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
 //Segments with less than -1 votes are already ignored before this function is called
 function chooseSegments(segments) {
-  //Create groups of segments that are similar to eachother
+    //Create groups of segments that are similar to eachother
-  //Segments must be sorted by their startTime so that we can build groups chronologically:
+    //Segments must be sorted by their startTime so that we can build groups chronologically:
-  //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
+    //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
-  //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
+    //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
-  //   inside that group (because they're sorted) so we can create a new one
+    //   inside that group (because they're sorted) so we can create a new one
-  const similarSegmentsGroups = [];
+    const similarSegmentsGroups = [];
-  let currentGroup;
+    let currentGroup;
-  let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
+    let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
-  segments.forEach(segment => {
+    segments.forEach(segment => {
-    if (segment.startTime > cursor) {
+        if (segment.startTime > cursor) {
-      currentGroup = { segments: [], votes: 0 };
+            currentGroup = { segments: [], votes: 0 };
-      similarSegmentsGroups.push(currentGroup);
+            similarSegmentsGroups.push(currentGroup);
-    }
+        }
 
-    currentGroup.segments.push(segment);
+        currentGroup.segments.push(segment);
-    //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
+        //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
-    if (segment.votes > 0) {
+        if (segment.votes > 0) {
-      currentGroup.votes += segment.votes;
+            currentGroup.votes += segment.votes;
-    }
+        }
 
-    cursor = Math.max(cursor, segment.endTime);
+        cursor = Math.max(cursor, segment.endTime);
-  });
+    });
 
-  //if there are too many groups, find the best 8
+    //if there are too many groups, find the best 8
-  return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
+    return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
-    //randomly choose 1 good segment per group and return them
+        //randomly choose 1 good segment per group and return them
-    group => getWeightedRandomChoice(group.segments, 1)[0]
+        group => getWeightedRandomChoice(group.segments, 1)[0]
-  );
+    );
 }
 
 /**
@@ -95,91 +151,39 @@ function chooseSegments(segments) {
  * @returns
  */
 function handleGetSegments(req, res) {
-  const videoID = req.query.videoID;
+    const videoID = req.query.videoID;
-  // Default to sponsor
+    // Default to sponsor
-  // If using params instead of JSON, only one category can be pulled
+    // If using params instead of JSON, only one category can be pulled
-  const categories = req.query.categories
+    const categories = req.query.categories
-    ? JSON.parse(req.query.categories)
+        ? JSON.parse(req.query.categories)
-    : req.query.category
+        : req.query.category
-    ? [req.query.category]
+        ? [req.query.category]
-    : ['sponsor'];
+        : ['sponsor'];
 
-  /**
+    let segments = cleanGetSegments(videoID, categories);
-   * @type {Array<{
-   *                 segment: number[],
-   *                 category: string,
-   *                 UUID: string
-   *              }>
-   *       }
-   */
-  const segments = [];
 
-  let userHashedIP, shadowHiddenSegments;
+    if (segments === null || segments === undefined) {
-
+        res.sendStatus(500);
-  try {
+        return false;
-    for (const category of categories) {
-      const categorySegments = db
-        .prepare(
-          'all',
-          'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
-          [videoID, category]
-        )
-        .filter(segment => {
-          if (segment.votes < -1) {
-            return false; //too untrustworthy, just ignore it
-          }
-
-          //check if shadowHidden
-          //this means it is hidden to everyone but the original ip that submitted it
-          if (segment.shadowHidden != 1) {
-            return true;
-          }
-
-          if (shadowHiddenSegments === undefined) {
-            shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
-          }
-
-          //if this isn't their ip, don't send it to them
-          return shadowHiddenSegments.some(shadowHiddenSegment => {
-            if (userHashedIP === undefined) {
-              //hash the IP only if it's strictly necessary
-              userHashedIP = getHash(getIP(req) + config.globalSalt);
-            }
-            return shadowHiddenSegment.hashedIP === userHashedIP;
-          });
-        });
-
-      chooseSegments(categorySegments).forEach(chosenSegment => {
-        segments.push({
-          category,
-          segment: [chosenSegment.startTime, chosenSegment.endTime],
-          UUID: chosenSegment.UUID,
-        });
-      });
     }
 
     if (segments.length == 0) {
-      res.sendStatus(404);
+        res.sendStatus(404);
-      return false;
+        return false;
     }
 
     return segments;
-  } catch (error) {
-    logger.error(error);
-    res.sendStatus(500);
-
-    return false;
-  }
 }
 
 module.exports = {
-  handleGetSegments,
+    handleGetSegments,
-  endpoint: function (req, res) {
+    cleanGetSegments,
-    let segments = handleGetSegments(req, res);
+    endpoint: function (req, res) {
+        let segments = handleGetSegments(req, res);
 
-    if (segments) {
+        if (segments) {
-      //send result
+            //send result
-      res.send(segments);
+            res.send(segments);
-    }
+        }
-  },
+    },
 };

src/routes/getSkipSegmentsByHash.js

@@ -0,0 +1,37 @@
+const hashPrefixTester = require('../utils/hashPrefixTester.js');
+const getSegments = require('./getSkipSegments.js').cleanGetSegments;
+
+const databases = require('../databases/databases.js');
+const logger = require('../utils/logger.js');
+const db = databases.db;
+
+module.exports = async function (req, res) {
+    let hashPrefix = req.params.prefix;
+    if (!hashPrefixTester(req.params.prefix)) {
+        res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+        return;
+    }
+
+    const categories = req.query.categories
+    ? JSON.parse(req.query.categories)
+    : req.query.category
+    ? [req.query.category]
+    : ['sponsor'];
+
+    // Get all video id's that match hash prefix
+    const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
+    if (videoIds.length === 0) {
+        res.sendStatus(404);
+        return;
+    }
+
+    let segments = videoIds.map((video) => {
+        return {
+            videoID: video.videoID,
+            hash: video.hashedVideoID,
+            segments: getSegments(video.videoID, categories)
+        };
+    });
+
+    res.status(200).json(segments);
+}

src/routes/postSkipSegments.js

@@ -302,9 +302,11 @@ module.exports = async function postSkipSegments(req, res) {
 
             try {
                 db.prepare('run', "INSERT INTO sponsorTimes " + 
-                    "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" +
+                    "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
-                    "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [videoID, segmentInfo.segment[0], 
+                    "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
-                    segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned]);
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1)
+                    ]
+                );
             
                 //add to private db as well
                 privateDB.prepare('run', "INSERT INTO sponsorTimes VALUES(?, ?, ?)", [videoID, hashedIP, timeSubmitted]);

src/routes/voteOnSponsorTime.js

@@ -1,4 +1,3 @@
-var fs = require('fs');
 var config = require('../config.js');
 
 var getHash = require('../utils/getHash.js');
@@ -370,8 +369,6 @@ async function voteOnSponsorTime(req, res) {
 }
 
 module.exports = {
-    voteOnSponsorTime,
+  voteOnSponsorTime,
-    endpoint: function (req, res) {
+  endpoint: voteOnSponsorTime
-       voteOnSponsorTime(req, res);
+};
-    },
-  };

src/utils/hashPrefixTester.js

@@ -0,0 +1,10 @@
+const config = require('../config.js');
+
+const minimumPrefix = config.minimumPrefix || '3';
+const maximumPrefix = config.maximumPrefix || '32'; // Half the hash.
+
+const prefixChecker = new RegExp('^[\\da-f]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i');
+
+module.exports = (prefix) => {
+    return prefixChecker.test(prefix);
+};

test/cases/dbUpgrade.js

@@ -0,0 +1,12 @@
+const databases = require('../../src/databases/databases.js');
+const db = databases.db;
+const privateDB = databases.privateDB;
+
+describe('dbUpgrade', () => {
+  it('Should update the database version when starting the application', (done) => {
+    let dbVersion = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    let privateVersion = privateDB.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    if (dbVersion >= 1 && privateVersion >= 1) done();
+    else done('Versions are not at least 1. db is ' + dbVersion + ', private is ' + privateVersion);
+  });
+});

test/cases/getHash.js

@@ -13,6 +13,10 @@ describe('getHash', () => {
     assert.equal(getHash("test"), "2f327ef967ade1ebf4319163f7debbda9cc17bb0c8c834b00b30ca1cf1c256ee");
   });
 
+  it ('Should be able to output the same has the DB upgrade script will output', () => {
+    assert.equal(getHash("vid", 1), "1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663");
+  });
+
   it ('Should take a variable number of passes', () => {
     assert.equal(getHash("test", 1), "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08");
     assert.equal(getHash("test", 2), "7b3d979ca8330a94fa7e9e1b466d8b99e0bcdea1ec90596c0dcc8d7ef6b4300c");

test/cases/getSavedTimeForUser.js

@@ -5,8 +5,8 @@ var getHash = require('../../src/utils/getHash.js');
 
 describe('getSavedTimeForUser', () => {
   before(() => {
-    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
-    db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0, '" + getHash('getSavedTimeForUser', 0) + "')");
   }); 
 
   it('Should be able to get a 200', (done) => {

test/cases/getSegmentsByHash.js

@@ -0,0 +1,195 @@
+var request = require('request');
+var db = require('../../src/databases/databases.js').db;
+var utils = require('../utils.js');
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getSegmentsByHash', () => {
+  before(() => {
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    db.exec(startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+    db.exec(startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+    db.exec(startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+    db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+  }); 
+
+  it('Should update the database version when starting the application', (done) => {
+    let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    if (version > 2) done();
+    else done('Version isn\'t greater than 2. Version is ' + version);
+  });
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          done();
+        } // pass
+      });
+  });
+
+  it('Should be able to get a 200 with empty segments for video but no matching categories', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/3272f?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          if (JSON.parse(body) && JSON.parse(body).length > 0 && JSON.parse(body)[0].segments.length === 0) {
+            done(); // pass
+          } else {
+            done("response had segments");
+          }
+        }
+      });
+  });
+
+  it('Should be able to get a 404 if no videos', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/11111?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 400 prefix too short', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/11?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 400 prefix too long', (done) => {
+    let prefix = new Array(50).join('1');
+    if (prefix.length <= 32) { // default value, config can change this
+      done('failed to generate a long enough string for the test ' + prefix.length);
+      return;
+    }
+
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/'+prefix+'?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should not return 400 prefix in range', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/11111?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 400) done("prefix length 5 gave 400 " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 404 for no hash', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 404) done("expected 404, got " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 500 for bad format categories', (done) => { // should probably be 400
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/?categories=shilling', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 500) done("expected 500 got " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should be able to get multiple videos', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          body = JSON.parse(body);
+          if (body.length !== 2) done("expected 2 video, got " + body.length);
+          else if (body[0].segments.length !== 2) done("expected 2 segments for first video, got " + body[0].segments.length);
+          else if (body[1].segments.length !== 1) done("expected 1 segment for second video, got " + body[1].segments.length);
+          else done();
+        }
+      });
+  });
+
+  it('Should be able to get 200 for no categories (default sponsor)', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/fdaf', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          body = JSON.parse(body);
+          if (body.length !== 2) done("expected 2 videos, got " + body.length);
+          else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
+          else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
+          else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+          else done();
+        }
+      });
+  });
+
+  it('Should be able to post a segment and get it using endpoint', (done) => {
+    let testID = 'abc123goodVideo';
+    request.post(utils.getbaseURL() 
+      + "/api/postVideoSponsorTimes", {
+        json: {
+          userID: "test",
+          videoID: testID,
+          segments: [{
+            segment: [13, 17],
+            category: "sponsor"
+          }]
+        }
+      }, 
+      (err, res, body) => {
+        if (err) done('(post) ' + err);
+        else if (res.statusCode === 200) {
+          request.get(utils.getbaseURL() 
+            + '/api/skipSegments/'+getHash(testID, 1).substring(0,3), null, 
+              (err, res, body) => {
+                if (err) done("(get) Couldn't call endpoint");
+                else if (res.statusCode !== 200) done("(get) non 200 status code, was " + res.statusCode);
+                else {
+                  body = JSON.parse(body);
+                  if (body.length !== 1) done("(get) expected 1 video, got " + body.length);
+                  else if (body[0].segments.length !== 1) done("(get) expected 1 segments for first video, got " + body[0].segments.length);
+                  else if (body[0].segments[0].category !== 'sponsor') done("(get) segment should be sponsor, was "+body[0].segments[0].category);
+                  else done();
+                }
+              });
+        } else {
+          done("(post) non 200 status code, was " + res.statusCode);
+        }
+      }
+    );
+  });
+});

test/cases/getSkipSegments.js

@@ -1,31 +1,19 @@
 var request = require('request');
 var db = require('../../src/databases/databases.js').db;
 var utils = require('../utils.js');
+var getHash = require('../../src/utils/getHash.js');
 
-/*
- *CREATE TABLE IF NOT EXISTS "sponsorTimes" (
-	"videoID"	TEXT NOT NULL,
-	"startTime"	REAL NOT NULL,
-	"endTime"	REAL NOT NULL,
-	"votes"	INTEGER NOT NULL,
-	"UUID"	TEXT NOT NULL UNIQUE,
-	"userID"	TEXT NOT NULL,
-	"timeSubmitted"	INTEGER NOT NULL,
-	"views"	INTEGER NOT NULL,
-	"shadowHidden"	INTEGER NOT NULL
-);
- */
 
 describe('getSkipSegments', () => {
   before(() => {
-    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
-    db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 0) + "')");
-    db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 0) + "')");
-    db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 0) + "')");
-    db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')");
-    db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')");
-    db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')");
-    db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')");
   }); 
   
 

test/cases/noSegmentRecords.js

@@ -22,7 +22,7 @@ describe('noSegmentRecords', () => {
   it('Should update the database version when starting the application', (done) => {
     let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
     if (version > 1) done();
-    else done('Version isn\'t greater that 1. Version is ' + version);
+    else done('Version isn\'t greater than 1. Version is ' + version);
   });
 
   it('Should be able to submit categories not in video (http response)', (done) => {

test/cases/oldGetSponsorTime.js

@@ -1,6 +1,7 @@
 var request = require('request');
 var db = require('../../src/databases/databases.js').db;
 var utils = require('../utils.js');
+var getHash = require('../../src/utils/getHash.js');
 
 
 /*
@@ -19,9 +20,9 @@ var utils = require('../utils.js');
 
 describe('getVideoSponsorTime (Old get method)', () => {
   before(() => {
-    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
-    db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 1) + "')");
-    db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 1) + "')");
   }); 
 
   it('Should be able to get a time', (done) => {

test/cases/voteOnSponsorTime.js

@@ -5,23 +5,23 @@ const getHash = require('../../src/utils/getHash.js');
 
 describe('voteOnSponsorTime', () => {
   before(() => {
-    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
-    db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest', 1)+"')");
-    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest2', 1)+"')");
-    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0)");
+    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest2', 1)+"')");
-    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0)");
+    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest2', 1)+"')");
-    db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest3', 1)+"')");
-    db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest,test', 1)+"')");
-    db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-test3', 1)+"')");
-    db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-test3', 1)+"')");
-    db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')");
-    db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0)");
+    db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')");
-    db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter', 1)+"')");
-    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')");
-    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')");
-    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')");
-    db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('own-submission-video', 1)+"')");
-    db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)");
+    db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('not-own-submission-video', 1)+"')");
 
     db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
     privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");