Merge branch 'master' into 147-add-cache-for-gettopusers

src/app.js

@@ -3,8 +3,11 @@ var express = require('express');
 var app = express();
 var config = require('./config.js');
 var redis = require('./utils/redis.js');
+const getIP = require('./utils/getIP.js');
+const getHash = require('./utils/getHash.js');
 
 // Middleware 
+const rateLimitMiddleware = require('./middleware/requestRateLimit.js');
 var corsMiddleware = require('./middleware/cors.js');
 var loggerMiddleware = require('./middleware/logger.js');
 const userCounter = require('./middleware/userCounter.js');
@@ -24,13 +27,24 @@ var getViewsForUser = require('./routes/getViewsForUser.js');
 var getTopUsers = require('./routes/getTopUsers.js');
 var getTotalStats = require('./routes/getTotalStats.js');
 var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js');
+var getUserInfo = require('./routes/getUserInfo.js');
 var postNoSegments = require('./routes/postNoSegments.js');
 var getIsUserVIP = require('./routes/getIsUserVIP.js');
+var warnUser = require('./routes/postWarning.js');
+var postSegmentShift = require('./routes/postSegmentShift.js');
 
 // Old Routes
 var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js');
 var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js');
 
+// Rate limit endpoint lists
+let voteEndpoints = [voteOnSponsorTime.endpoint];
+let viewEndpoints = [viewedVideoSponsorTime];
+if (config.rateLimit) {
+    if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote));
+    if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
+}
+
 //setup CORS correctly
 app.use(corsMiddleware);
 app.use(loggerMiddleware);
@@ -59,12 +73,12 @@ app.post('/api/skipSegments', postSkipSegments);
 app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash);
 
 //voting endpoint
-app.get('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);
-app.post('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);
+app.get('/api/voteOnSponsorTime', ...voteEndpoints);
+app.post('/api/voteOnSponsorTime', ...voteEndpoints);
 
-//Endpoint when a sponsorTime is used up
-app.get('/api/viewedVideoSponsorTime', viewedVideoSponsorTime);
-app.post('/api/viewedVideoSponsorTime', viewedVideoSponsorTime);
+//Endpoint when a submission is skipped
+app.get('/api/viewedVideoSponsorTime', ...viewEndpoints);
+app.post('/api/viewedVideoSponsorTime', ...viewEndpoints);
 
 //To set your username for the stats view
 app.post('/api/setUsername', setUsername);
@@ -93,6 +107,8 @@ app.get('/api/getTopUsers', getTopUsers);
 //send the total submissions, total views and total minutes saved
 app.get('/api/getTotalStats', getTotalStats);
 
+app.get('/api/getUserInfo', getUserInfo);
+
 //send out a formatted time saved total
 app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
 
@@ -102,6 +118,11 @@ app.post('/api/noSegments', postNoSegments);
 //get if user is a vip
 app.get('/api/isUserVIP', getIsUserVIP);
 
+//sent user a warning
+app.post('/api/warnUser', warnUser);
+
+//get if user is a vip
+app.post('/api/segmentShift', postSegmentShift);
 
 app.get('/database.db', function (req, res) {
     res.sendFile("./databases/sponsorTimes.db", { root: "./" });

src/config.js

@@ -20,7 +20,9 @@ addDefaults(config, {
   "privateDBSchema": "./databases/_private.db.sql",
   "readOnly": false,
   "webhooks": [],
-  "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"]
+  "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+  "maxNumberOfActiveWarnings": 3,
+  "hoursAfterWarningExpires": 24
 })
 
 module.exports = config;

src/middleware/requestRateLimit.js

@@ -0,0 +1,18 @@
+const getIP = require('../utils/getIP.js');
+const getHash = require('../utils/getHash.js');
+const rateLimit = require('express-rate-limit');
+
+module.exports = (limitConfig) => rateLimit({
+  windowMs: limitConfig.windowMs,
+  max: limitConfig.max,
+  message: limitConfig.message,
+  statusCode: limitConfig.statusCode,
+  headers: false,
+  keyGenerator: (req /*, res*/) => {
+    return getHash(getIP(req), 1);
+  },
+  skip: (/*req, res*/) => {
+    // skip rate limit if running in test mode
+    return process.env.npm_lifecycle_script === 'node test.js';
+  }
+});

src/routes/getSkipSegmentsByHash.js

@@ -20,10 +20,6 @@ module.exports = async function (req, res) {
 
     // Get all video id's that match hash prefix
     const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
-    if (videoIds.length === 0) {
-        res.sendStatus(404);
-        return;
-    }
 
     let segments = videoIds.map((video) => {
         return {
@@ -33,5 +29,5 @@ module.exports = async function (req, res) {
         };
     });
 
-    res.status(200).json(segments);
+    res.status((segments.length === 0) ? 404 : 200).json(segments);
 }

src/routes/getUserInfo.js

@@ -0,0 +1,82 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+
+function dbGetSubmittedSegmentSummary (userID) {
+  try {
+    let row = db.prepare("get", "SELECT SUM(((endTime - startTime) / 60) * views) as minutesSaved, count(*) as segmentCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]);
+    if (row.minutesSaved != null) {
+      return {
+        minutesSaved: row.minutesSaved,
+        segmentCount: row.segmentCount,
+      };
+    } else {
+      return {
+        minutesSaved: 0,
+        segmentCount: 0,
+      };
+    }
+  } catch (err) {
+    return false;
+  }
+}
+
+function dbGetUsername (userID) {
+  try {
+    let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]);
+    if (row !== undefined) {
+      return row.userName;
+    } else {
+      //no username yet, just send back the userID
+      return userID;
+    }
+  } catch (err) {
+    return false;
+  }
+}
+
+function dbGetViewsForUser (userID) {
+  try {
+    let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]);
+    //increase the view count by one
+    if (row.viewCount != null) {
+      return row.viewCount;
+    } else {
+      return 0;
+    }
+  } catch (err) {
+    return false;
+  }
+}
+
+function dbGetWarningsForUser (userID) {
+  try {
+    let rows = db.prepare('all', "SELECT * FROM warnings WHERE userID = ?", [userID]);
+    return rows.length;
+  } catch (err) {
+    logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0') ;
+    return 0;
+  }
+}
+
+module.exports = function getUserInfo (req, res) {
+  let userID = req.query.userID;
+
+  if (userID == undefined) {
+    //invalid request
+    res.status(400).send('Parameters are not valid');
+    return;
+  }
+
+  //hash the userID
+  userID = getHash(userID);
+  
+  const segmentsSummary = dbGetSubmittedSegmentSummary(userID);
+  res.send({
+    userID,
+    userName: dbGetUsername(userID),
+    minutesSaved: segmentsSummary.minutesSaved,
+    segmentCount: segmentsSummary.segmentCount,
+    viewCount: dbGetViewsForUser(userID),
+    warnings: dbGetWarningsForUser(userID)
+  });
+}

src/routes/postSegmentShift.js

@@ -0,0 +1,101 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+const ACTION_NONE = Symbol('none');
+const ACTION_UPDATE = Symbol('update');
+const ACTION_REMOVE = Symbol('remove');
+
+function shiftSegment(segment, shift) {
+  if (segment.startTime >= segment.endTime) return {action: ACTION_NONE, segment};
+  if (shift.startTime >= shift.endTime) return {action: ACTION_NONE, segment};
+  const duration = shift.endTime - shift.startTime;
+  if (shift.endTime < segment.startTime) {
+    // Scenario #1 cut before segment
+    segment.startTime -= duration;
+    segment.endTime -= duration;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (shift.startTime > segment.endTime) {
+    // Scenario #2 cut after segment
+    return {action: ACTION_NONE, segment};
+  }
+  if (segment.startTime < shift.startTime && segment.endTime > shift.endTime) {
+    // Scenario #3 cut inside segment
+    segment.endTime -= duration;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (segment.startTime >= shift.startTime && segment.endTime > shift.endTime) {
+    // Scenario #4 cut overlap startTime
+    segment.startTime = shift.startTime;
+    segment.endTime -= duration;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (segment.startTime < shift.startTime && segment.endTime <= shift.endTime) {
+    // Scenario #5 cut overlap endTime
+    segment.endTime = shift.startTime;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (segment.startTime >= shift.startTime && segment.endTime <= shift.endTime) {
+    // Scenario #6 cut overlap startTime and endTime
+    return {action: ACTION_REMOVE, segment};
+  }
+  return {action: ACTION_NONE, segment};
+}
+
+module.exports = (req, res) => {
+  // Collect user input data
+  const videoID = req.body.videoID;
+  const startTime = req.body.startTime;
+  const endTime = req.body.endTime;
+  let userID = req.body.userID;
+
+  // Check input data is valid
+  if (!videoID
+    || !userID
+    || !startTime
+    || !endTime
+  ) {
+    res.status(400).json({
+      message: 'Bad Format'
+    });
+    return;
+  }
+
+  // Check if user is VIP
+  userID = getHash(userID);
+  const userIsVIP = isUserVIP(userID);
+
+  if (!userIsVIP) {
+    res.status(403).json({
+      message: 'Must be a VIP to perform this action.'
+    });
+    return;
+  }
+  
+  try {
+    const segments = db.prepare('all', 'SELECT startTime, endTime, UUID FROM sponsorTimes WHERE videoID = ?', [videoID]);
+    const shift = {
+      startTime,
+      endTime,
+    };
+    segments.forEach(segment => {
+      const result = shiftSegment(segment, shift);
+        switch (result.action) {
+          case ACTION_UPDATE:
+            db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]);
+          break;
+          case ACTION_REMOVE:
+            db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ?, votes = -2 WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]);
+          break;
+        }
+    });
+  }
+  catch(err) {
+    logger.error(err);
+    res.sendStatus(500);
+  }
+  
+  res.sendStatus(200);
+};

src/routes/postSkipSegments.js

@@ -5,6 +5,7 @@ const db = databases.db;
 const privateDB = databases.privateDB;
 const YouTubeAPI = require('../utils/youtubeAPI.js');
 const logger = require('../utils/logger.js');
+const getSubmissionUUID = require('../utils/getSubmissionUUID.js');
 const request = require('request');
 const isoDurations = require('iso8601-duration');
 const fetch = require('node-fetch');
@@ -201,8 +202,7 @@ async function autoModerateSubmission(submission) {
                           startTime = parseFloat(segments[i].segment[0]);
                           endTime = parseFloat(segments[i].segment[1]);
 
-                          let UUID = getHash("v2-categories" + submission.videoID + startTime +
-                              endTime  + segments[i].category + submission.userID, 1);
+                          const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
                           // Send to Discord
                           // Note, if this is too spammy. Consider sending all the segments as one Webhook
                           sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
@@ -269,6 +269,16 @@ module.exports = async function postSkipSegments(req, res) {
 
     //hash the ip 5000 times so no one can get it from the database
     let hashedIP = getHash(getIP(req) + config.globalSalt);
+    
+    const MILLISECONDS_IN_HOUR = 3600000;
+    const now = Date.now();
+    let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?",
+      [userID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))]
+    ).count;
+    
+    if (warningsCount >= config.maxNumberOfActiveWarnings) {
+      return res.status(403).send('Submission blocked. Too many active warnings!');
+    }
 
     let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
     
@@ -400,8 +410,7 @@ module.exports = async function postSkipSegments(req, res) {
             //this can just be a hash of the data
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
-            let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] +
-                segmentInfo.segment[1]  + segmentInfo.category + userID, 1);
+            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
 
             try {
                 db.prepare('run', "INSERT INTO sponsorTimes " + 

src/routes/postWarning.js

@@ -0,0 +1,24 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+module.exports = (req, res) => {
+    // Collect user input data
+    let issuerUserID = getHash(req.body.issuerUserID);
+    let userID = getHash(req.body.userID);
+    let issueTime = new Date().getTime();
+
+    // Ensure user is a VIP
+    if (!isUserVIP(issuerUserID)) {
+        logger.debug("Permission violation: User " + issuerUserID + " attempted to warn user " + userID + "."); // maybe warn?
+        res.status(403).json({"message": "Not a VIP"});
+        return;
+    }
+
+    db.prepare('run', 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES (?, ?, ?)', [userID, issueTime, issuerUserID]);
+    res.status(200).json({
+        message: "Warning issued to user '" + userID + "'."
+    });
+
+};

src/routes/voteOnSponsorTime.js

@@ -247,6 +247,16 @@ async function voteOnSponsorTime(req, res) {
             return;
         }
     }
+    
+    const MILLISECONDS_IN_HOUR = 3600000;
+    const now = Date.now();
+    let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?",
+      [nonAnonUserID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))]
+    ).count;
+    
+    if (warningsCount >= config.maxNumberOfActiveWarnings) {
+      return res.status(403).send('Vote blocked. Too many active warnings!');
+    }
 
     let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;
 

src/utils/getSubmissionUUID.js

@@ -0,0 +1,7 @@
+const getHash = require('./getHash.js');
+
+module.exports = function getSubmissionUUID(videoID, category, userID,
+  startTime, endTime) {
+  return getHash('v2-categories' + videoID + startTime + endTime + category +
+    userID, 1);
+};

src/utils/youtubeAPI.js

@@ -19,6 +19,11 @@ if (config.mode === "test") {
 
     // YouTubeAPI.videos.list wrapper with cacheing
     exportObject.listVideos = (videoID, part, callback) => {
+        if (videoID.length !== 11 || videoID.includes(".")) {
+            callback("Invalid video ID");
+            return;
+        }
+
         let redisKey = "youtube.video." + videoID + "." + part;
         redis.get(redisKey, (getErr, result) => {
             if (getErr || !result) {