Merge pull request #4 from Joe-Dowd/query-by-hash-prefix

Query by hash prefix

src/app.js

@@ -11,6 +11,7 @@ const userCounter = require('./middleware/userCounter.js');
 // Routes
 var getSkipSegments = require('./routes/getSkipSegments.js').endpoint;
 var postSkipSegments = require('./routes/postSkipSegments.js');
+var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js');
 var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js');
 var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js');
 var setUsername = require('./routes/setUsername.js');
@@ -53,6 +54,9 @@ app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
 app.get('/api/skipSegments', getSkipSegments);
 app.post('/api/skipSegments', postSkipSegments);
 
+// add the privacy protecting skip segments functions
+app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash);
+
 //voting endpoint
 app.get('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);
 app.post('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);

src/databases/databases.js

@@ -5,6 +5,7 @@ var path = require('path');
 var Sqlite = require('./Sqlite.js')
 var Mysql = require('./Mysql.js');
 const logger = require('../utils/logger.js');
+const getHash = require('../utils/getHash.js');
 
 let options = {
   readonly: config.readOnly,
@@ -34,6 +35,10 @@ if (config.mysql) {
   }
 
   if (!config.readOnly) {
+    db.function("sha256", (string) => {
+      return getHash(string, 1);
+    });
+    
     // Upgrade database if required
     ugradeDB(db, "sponsorTimes");
     ugradeDB(privateDB, "private")

src/routes/getSkipSegments.js

@@ -8,6 +8,62 @@ var logger = require('../utils/logger.js');
 var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 
+function cleanGetSegments(videoID, categories) {
+  let userHashedIP, shadowHiddenSegments;
+
+  let segments = [];
+
+  try {
+    for (const category of categories) {
+      const categorySegments = db
+        .prepare(
+          'all',
+          'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
+          [videoID, category]
+        )
+        .filter(segment => {
+          if (segment.votes < -1) {
+            return false; //too untrustworthy, just ignore it
+          }
+
+          //check if shadowHidden
+          //this means it is hidden to everyone but the original ip that submitted it
+          if (segment.shadowHidden != 1) {
+            return true;
+          }
+
+          if (shadowHiddenSegments === undefined) {
+            shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
+          }
+
+          //if this isn't their ip, don't send it to them
+          return shadowHiddenSegments.some(shadowHiddenSegment => {
+            if (userHashedIP === undefined) {
+              //hash the IP only if it's strictly necessary
+              userHashedIP = getHash(getIP(req) + config.globalSalt);
+            }
+            return shadowHiddenSegment.hashedIP === userHashedIP;
+          });
+        });
+
+      chooseSegments(categorySegments).forEach(chosenSegment => {
+        segments.push({
+          category,
+          segment: [chosenSegment.startTime, chosenSegment.endTime],
+          UUID: chosenSegment.UUID,
+        });
+      });
+    }
+
+    return segments;
+  } catch (err) {
+    if (err) {
+      logger.error('j 2 Query failed');
+      return undefined;
+    }
+  }
+}
+
 //gets a weighted random choice from the choices array based on their `votes` property.
 //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
 //choices are unique
@@ -104,58 +160,11 @@ function handleGetSegments(req, res) {
     ? [req.query.category]
     : ['sponsor'];
 
-  /**
-   * @type {Array<{
-   *                 segment: number[],
-   *                 category: string,
-   *                 UUID: string
-   *              }>
-   *       }
-   */
-  const segments = [];
+  let segments = cleanGetSegments(videoID, categories);
 
-  let userHashedIP, shadowHiddenSegments;
-
-  try {
-    for (const category of categories) {
-      const categorySegments = db
-        .prepare(
-          'all',
-          'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
-          [videoID, category]
-        )
-        .filter(segment => {
-          if (segment.votes < -1) {
-            return false; //too untrustworthy, just ignore it
-          }
-
-          //check if shadowHidden
-          //this means it is hidden to everyone but the original ip that submitted it
-          if (segment.shadowHidden != 1) {
-            return true;
-          }
-
-          if (shadowHiddenSegments === undefined) {
-            shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
-          }
-
-          //if this isn't their ip, don't send it to them
-          return shadowHiddenSegments.some(shadowHiddenSegment => {
-            if (userHashedIP === undefined) {
-              //hash the IP only if it's strictly necessary
-              userHashedIP = getHash(getIP(req) + config.globalSalt);
-            }
-            return shadowHiddenSegment.hashedIP === userHashedIP;
-          });
-        });
-
-      chooseSegments(categorySegments).forEach(chosenSegment => {
-        segments.push({
-          category,
-          segment: [chosenSegment.startTime, chosenSegment.endTime],
-          UUID: chosenSegment.UUID,
-        });
-      });
+    if (segments === undefined) {
+      res.sendStatus(500);
+      return false;
     }
 
     if (segments.length == 0) {
@@ -164,16 +173,11 @@ function handleGetSegments(req, res) {
     }
 
     return segments;
-  } catch (error) {
-    logger.error(error);
-    res.sendStatus(500);
-
-    return false;
-  }
 }
 
 module.exports = {
   handleGetSegments,
+  cleanGetSegments,
   endpoint: function (req, res) {
     let segments = handleGetSegments(req, res);
 

src/routes/getSkipSegmentsByHash.js

@@ -0,0 +1,37 @@
+const hashPrefixTester = require('../utils/hashPrefixTester.js');
+const getSegments = require('./getSkipSegments.js').cleanGetSegments;
+
+const databases = require('../databases/databases.js');
+const logger = require('../utils/logger.js');
+const db = databases.db;
+
+module.exports = async function (req, res) {
+    let hashPrefix = req.params.prefix;
+    if (!hashPrefixTester(req.params.prefix)) {
+        res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+        return;
+    }
+
+    const categories = req.query.categories
+    ? JSON.parse(req.query.categories)
+    : req.query.category
+    ? [req.query.category]
+    : ['sponsor'];
+
+    // Get all video id's that match hash prefix
+    const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
+    if (videoIds.length === 0) {
+        res.sendStatus(404);
+        return;
+    }
+
+    let segments = videoIds.map((video) => {
+        return {
+            videoID: video.videoID,
+            hash: video.hashedVideoID,
+            segments: getSegments(video.videoID, categories)
+        };
+    });
+
+    res.status(200).json(segments);
+}

src/routes/postSkipSegments.js

@@ -302,9 +302,11 @@ module.exports = async function postSkipSegments(req, res) {
 
             try {
                 db.prepare('run', "INSERT INTO sponsorTimes " + 
-                    "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" +
-                    "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [videoID, segmentInfo.segment[0], 
-                    segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned]);
+                    "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
+                    "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1)
+                    ]
+                );
             
                 //add to private db as well
                 privateDB.prepare('run', "INSERT INTO sponsorTimes VALUES(?, ?, ?)", [videoID, hashedIP, timeSubmitted]);

src/routes/voteOnSponsorTime.js

@@ -1,4 +1,3 @@
-var fs = require('fs');
 var config = require('../config.js');
 
 var getHash = require('../utils/getHash.js');
@@ -370,8 +369,6 @@ async function voteOnSponsorTime(req, res) {
 }
 
 module.exports = {
-    voteOnSponsorTime,
-    endpoint: function (req, res) {
-       voteOnSponsorTime(req, res);
-    },
-  };
+  voteOnSponsorTime,
+  endpoint: voteOnSponsorTime
+};

src/utils/hashPrefixTester.js

@@ -0,0 +1,10 @@
+const config = require('../config.js');
+
+const minimumPrefix = config.minimumPrefix || '3';
+const maximumPrefix = config.maximumPrefix || '32'; // Half the hash.
+
+const prefixChecker = new RegExp('^[\\da-f]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i');
+
+module.exports = (prefix) => {
+    return prefixChecker.test(prefix);
+};