From 43db13ab3fa30a69770e422d3f9e2794007a018a Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 14 Sep 2020 09:17:35 +0200 Subject: [PATCH 01/17] Added getUserInfo endpoint This endpoint provides the following data about the user: * userID * userName * minutesSaved * segmentCount * viewCount --- src/app.js | 3 ++ src/routes/getUserInfo.js | 71 +++++++++++++++++++++++++++ test/cases/getUserInfo.js | 101 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 src/routes/getUserInfo.js create mode 100644 test/cases/getUserInfo.js diff --git a/src/app.js b/src/app.js index c70115d..c6ab3b5 100644 --- a/src/app.js +++ b/src/app.js @@ -22,6 +22,7 @@ var getViewsForUser = require('./routes/getViewsForUser.js'); var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); +var getUserInfo = require('./routes/getUserInfo.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -86,6 +87,8 @@ app.get('/api/getTopUsers', getTopUsers); //send the total submissions, total views and total minutes saved app.get('/api/getTotalStats', getTotalStats); +app.get('/api/getUserInfo', getUserInfo); + //send out a formatted time saved total app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); diff --git a/src/routes/getUserInfo.js b/src/routes/getUserInfo.js new file mode 100644 index 0000000..8479129 --- /dev/null +++ b/src/routes/getUserInfo.js @@ -0,0 +1,71 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); + +function dbGetSubmittedSegmentSummary (userID) { + try { + let row = db.prepare("get", "SELECT SUM(((endTime - startTime) / 60) * views) as minutesSaved, count(*) as segmentCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]); + if (row.minutesSaved != null) { + return { + minutesSaved: row.minutesSaved, + segmentCount: row.segmentCount, + }; + } else { + return { + minutesSaved: 0, + segmentCount: 0, + }; + } + } catch (err) { + return false; + } +} + +function dbGetUsername (userID) { + try { + let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]); + if (row !== undefined) { + return row.userName; + } else { + //no username yet, just send back the userID + return userID; + } + } catch (err) { + return false; + } +} + +function dbGetViewsForUser (userID) { + try { + let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]); + //increase the view count by one + if (row.viewCount != null) { + return row.viewCount; + } else { + return 0; + } + } catch (err) { + return false; + } +} + +module.exports = function getUserInfo (req, res) { + let userID = req.query.userID; + + if (userID == undefined) { + //invalid request + res.status(400).send('Parameters are not valid'); + return; + } + + //hash the userID + userID = getHash(userID); + + const segmentsSummary = dbGetSubmittedSegmentSummary(userID); + res.send({ + userID, + userName: dbGetUsername(userID), + minutesSaved: segmentsSummary.minutesSaved, + segmentCount: segmentsSummary.segmentCount, + viewCount: dbGetViewsForUser(userID), + }); +} diff --git a/test/cases/getUserInfo.js b/test/cases/getUserInfo.js new file mode 100644 index 0000000..499ae3c --- /dev/null +++ b/test/cases/getUserInfo.js @@ -0,0 +1,101 @@ +var request = require('request'); +var utils = require('../utils.js'); +var db = require('../../src/databases/databases.js').db; +var getHash = require('../../src/utils/getHash.js'); + +describe('getUserInfo', () => { + before(() => { + let startOfUserNamesQuery = "INSERT INTO userNames (userID, userName) VALUES"; + db.exec(startOfUserNamesQuery + "('" + getHash("getuserinfo_user_01") + "', 'Username user 01')"); + let startOfSponsorTimesQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000001', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000002', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -1, 'uuid000003', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -2, 'uuid000004', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)"); + db.exec(startOfSponsorTimesQuery + "('xzzzxxyyy', 1, 11, -5, 'uuid000005', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)"); + db.exec(startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + }); + + it('Should be able to get a 200', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_user_01', null, + (err, res, body) => { + if (err) { + done('couldn\'t call endpoint'); + } else { + if (res.statusCode !== 200) { + done('non 200'); + } else { + done(); // pass + } + } + }); + }); + + it('Should be able to get a 400 (No userID parameter)', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo', null, + (err, res, body) => { + if (err) { + done('couldn\'t call endpoint'); + } else { + if (res.statusCode !== 400) { + done('non 400'); + } else { + done(); // pass + } + } + }); + }); + + it('Should return info', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_user_01', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.userName !== 'Username user 01') { + return done('Returned incorrect userName "' + data.userName + '"'); + } + if (data.minutesSaved !== 5) { + return done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); + } + if (data.viewCount !== 30) { + return done('Returned incorrect viewCount "' + data.viewCount + '"'); + } + if (data.segmentCount !== 3) { + return done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } + done(); // pass + } + } + }); + }); + + it('Should return userID for userName (No userName set)', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_user_02', null, + (err, res, body) => { + if (err) { + done('couldn\'t call endpoint'); + } else { + if (res.statusCode !== 200) { + done('non 200'); + } else { + const data = JSON.parse(body); + if (data.userName !== 'c2a28fd225e88f74945794ae85aef96001d4a1aaa1022c656f0dd48ac0a3ea0f') { + return done('Did not return userID for userName'); + } + done(); // pass + } + } + }); + }); +}); From 075cb9d5f2a28d39aa9f558be343000925e1d1c1 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 15 Sep 2020 23:56:10 +0100 Subject: [PATCH 02/17] added warning system Co-authored-by: Ajay Ramachandran --- databases/_upgrade_sponsorTimes_4.sql | 12 ++++ package-lock.json | 5 ++ src/app.js | 4 ++ src/routes/getUserInfo.js | 11 ++++ src/routes/postWarning.js | 24 ++++++++ src/routes/voteOnSponsorTime.js | 3 + test/cases/getUserInfo.js | 84 ++++++++++++++++++++++++--- test/cases/postWarning.js | 53 +++++++++++++++++ 8 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_4.sql create mode 100644 src/routes/postWarning.js create mode 100644 test/cases/postWarning.js diff --git a/databases/_upgrade_sponsorTimes_4.sql b/databases/_upgrade_sponsorTimes_4.sql new file mode 100644 index 0000000..91e51e2 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_4.sql @@ -0,0 +1,12 @@ +BEGIN TRANSACTION; + +/* Create warnings table */ +CREATE TABLE "warnings" ( + userID TEXT NOT NULL, + issueTime INTEGER NOT NULL, + issuerUserID TEXT NOT NULL +); + +UPDATE config SET value = 4 WHERE key = "version"; + +COMMIT; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0878451..5f08ab3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1866,6 +1866,11 @@ "semver": "^5.7.0" } }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "node-forge": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", diff --git a/src/app.js b/src/app.js index 787f32e..7e14923 100644 --- a/src/app.js +++ b/src/app.js @@ -27,6 +27,7 @@ var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var getUserInfo = require('./routes/getUserInfo.js'); var postNoSegments = require('./routes/postNoSegments.js'); var getIsUserVIP = require('./routes/getIsUserVIP.js'); +var warnUser = require('./routes/postWarning.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -105,6 +106,9 @@ app.post('/api/noSegments', postNoSegments); //get if user is a vip app.get('/api/isUserVIP', getIsUserVIP); +//sent user a warning +app.post('/api/warnUser', warnUser); + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); diff --git a/src/routes/getUserInfo.js b/src/routes/getUserInfo.js index 8479129..de17999 100644 --- a/src/routes/getUserInfo.js +++ b/src/routes/getUserInfo.js @@ -48,6 +48,16 @@ function dbGetViewsForUser (userID) { } } +function dbGetWarningsForUser (userID) { + try { + let rows = db.prepare('all', "SELECT * FROM warnings WHERE userID = ?", [userID]); + return rows.length; + } catch (err) { + logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0') ; + return 0; + } +} + module.exports = function getUserInfo (req, res) { let userID = req.query.userID; @@ -67,5 +77,6 @@ module.exports = function getUserInfo (req, res) { minutesSaved: segmentsSummary.minutesSaved, segmentCount: segmentsSummary.segmentCount, viewCount: dbGetViewsForUser(userID), + warnings: dbGetWarningsForUser(userID) }); } diff --git a/src/routes/postWarning.js b/src/routes/postWarning.js new file mode 100644 index 0000000..797d307 --- /dev/null +++ b/src/routes/postWarning.js @@ -0,0 +1,24 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); + +module.exports = (req, res) => { + // Collect user input data + let issuerUserID = getHash(req.body.issuerUserID); + let userID = getHash(req.body.userID); + let issueTime = new Date().getTime(); + + // Ensure user is a VIP + if (!isUserVIP(issuerUserID)) { + logger.debug("Permission violation: User " + issuerUserID + " attempted to warn user " + userID + "."); // maybe warn? + res.status(403).json({"message": "Not a VIP"}); + return; + } + + db.prepare('run', 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES (?, ?, ?)', [userID, issueTime, issuerUserID]); + res.status(200).json({ + message: "Warning issued to user '" + userID + "'." + }); + +}; \ No newline at end of file diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index e74dd3e..e8c4edc 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -270,6 +270,9 @@ async function voteOnSponsorTime(req, res) { } else if (votesRow.type === 2) { //extra downvote oldIncrementAmount = -4; + } else if (votesRow.type === 20) { + //undo/cancel vote + oldIncrementAmount = 0; } else if (votesRow.type < 0) { //vip downvote oldIncrementAmount = votesRow.type; diff --git a/test/cases/getUserInfo.js b/test/cases/getUserInfo.js index 499ae3c..f04c02f 100644 --- a/test/cases/getUserInfo.js +++ b/test/cases/getUserInfo.js @@ -16,6 +16,11 @@ describe('getUserInfo', () => { db.exec(startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 0)"); db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + + + db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_0') + "', 10, 'getuserinfo_vip')"); + db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_1') + "', 10, 'getuserinfo_vip')"); + db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_1') + "', 10, 'getuserinfo_vip')"); }); it('Should be able to get a 200', (done) => { @@ -26,7 +31,7 @@ describe('getUserInfo', () => { done('couldn\'t call endpoint'); } else { if (res.statusCode !== 200) { - done('non 200'); + done('non 200 (' + res.statusCode + ')'); } else { done(); // pass } @@ -62,18 +67,79 @@ describe('getUserInfo', () => { } else { const data = JSON.parse(body); if (data.userName !== 'Username user 01') { - return done('Returned incorrect userName "' + data.userName + '"'); + done('Returned incorrect userName "' + data.userName + '"'); + } else if (data.minutesSaved !== 5) { + done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); + } else if (data.viewCount !== 30) { + done('Returned incorrect viewCount "' + data.viewCount + '"'); + } else if (data.segmentCount !== 3) { + done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } else { + done(); // pass } - if (data.minutesSaved !== 5) { - return done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); + } + } + }); + }); + + it('Should get warning data', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_warning_0', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.warnings !== 1) { + done('wrong number of warnings: ' + data.warnings + ', not ' + 1); + } else { + done(); // pass } - if (data.viewCount !== 30) { - return done('Returned incorrect viewCount "' + data.viewCount + '"'); + } + } + }); + }); + + it('Should get multiple warnings', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_warning_1', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.warnings !== 2) { + done('wrong number of warnings: ' + data.warnings + ', not ' + 2); + } else { + done(); // pass } - if (data.segmentCount !== 3) { - return done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } + } + }); + }); + + it('Should not get warnings if noe', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_warning_2', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.warnings !== 0) { + done('wrong number of warnings: ' + data.warnings + ', not ' + 0); + } else { + done(); // pass } - done(); // pass } } }); diff --git a/test/cases/postWarning.js b/test/cases/postWarning.js new file mode 100644 index 0000000..7661bf5 --- /dev/null +++ b/test/cases/postWarning.js @@ -0,0 +1,53 @@ +var request = require('request'); +var utils = require('../utils.js'); +var db = require('../../src/databases/databases.js').db; +var getHash = require('../../src/utils/getHash.js'); + +describe('postWarning', () => { + before(() => { + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("warning-vip") + "')"); + }); + + it('Should update the database version when starting the application', (done) => { + let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (version > 3) done(); + else done('Version isn\'t greater than 3. Version is ' + version); + }); + + it('Should be able to create warning if vip (exp 200)', (done) => { + let json = { + issuerUserID: 'warning-vip', + userID: 'warning-0' + }; + + request.post(utils.getbaseURL() + + "/api/warnUser", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done(); + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + it('Should not be able to create warning if vip (exp 403)', (done) => { + let json = { + issuerUserID: 'warning-not-vip', + userID: 'warning-1' + }; + + request.post(utils.getbaseURL() + + "/api/warnUser", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); +}); \ No newline at end of file From 3c79c0f7a840ac8ec8800dac066c7d7decaae409 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 16 Sep 2020 13:48:54 -0400 Subject: [PATCH 03/17] Remove extra test case --- test/cases/postWarning.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/cases/postWarning.js b/test/cases/postWarning.js index 7661bf5..e5e8586 100644 --- a/test/cases/postWarning.js +++ b/test/cases/postWarning.js @@ -8,12 +8,6 @@ describe('postWarning', () => { db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("warning-vip") + "')"); }); - it('Should update the database version when starting the application', (done) => { - let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; - if (version > 3) done(); - else done('Version isn\'t greater than 3. Version is ' + version); - }); - it('Should be able to create warning if vip (exp 200)', (done) => { let json = { issuerUserID: 'warning-vip', @@ -50,4 +44,4 @@ describe('postWarning', () => { } }); }); -}); \ No newline at end of file +}); From 36f654f41c549d71eb32f44e894beab879d9efae Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Wed, 16 Sep 2020 22:40:11 +0200 Subject: [PATCH 04/17] Blocking users with too many active warnings from submitting votes and submissions --- config.json.example | 4 ++- src/config.js | 4 ++- src/routes/postSkipSegments.js | 10 ++++++ src/routes/voteOnSponsorTime.js | 10 ++++++ test.json | 4 ++- test/cases/postSkipSegments.js | 64 +++++++++++++++++++++++++++++++++ test/cases/voteOnSponsorTime.js | 32 +++++++++++++++++ 7 files changed, 125 insertions(+), 3 deletions(-) diff --git a/config.json.example b/config.json.example index a2eee23..f3a2413 100644 --- a/config.json.example +++ b/config.json.example @@ -22,5 +22,7 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire + "hoursAfterWarningExpire": 24 } diff --git a/src/config.js b/src/config.js index d85b930..b3f30d3 100644 --- a/src/config.js +++ b/src/config.js @@ -20,7 +20,9 @@ addDefaults(config, { "privateDBSchema": "./databases/_private.db.sql", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + "maxNumberOfActiveWarnings": 3, + "hoursAfterWarningExpires": 24 }) module.exports = config; diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 820e0cf..4961e92 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -269,6 +269,16 @@ module.exports = async function postSkipSegments(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(getIP(req) + config.globalSalt); + + const MILLISECONDS_IN_HOUR = 3600000; + const now = Date.now(); + let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?", + [userID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))] + ).count; + + if (warningsCount >= config.maxNumberOfActiveWarnings) { + return res.status(403).send('Submission blocked. Too many active warnings!'); + } let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category }); diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index e8c4edc..0e6de00 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -235,6 +235,16 @@ async function voteOnSponsorTime(req, res) { return; } } + + const MILLISECONDS_IN_HOUR = 3600000; + const now = Date.now(); + let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?", + [nonAnonUserID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))] + ).count; + + if (warningsCount >= config.maxNumberOfActiveWarnings) { + return res.status(403).send('Vote blocked. Too many active warnings!'); + } let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect; diff --git a/test.json b/test.json index 8905d83..c620be4 100644 --- a/test.json +++ b/test.json @@ -49,5 +49,7 @@ ] } ], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + "maxNumberOfActiveWarnings": 3, + "hoursAfterWarningExpires": 24 } diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index a2e2c97..b8966b1 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -1,5 +1,7 @@ var assert = require('assert'); var request = require('request'); +var config = require('../../src/config.js'); +var getHash = require('../../src/utils/getHash.js'); var utils = require('../utils.js'); @@ -7,6 +9,24 @@ var databases = require('../../src/databases/databases.js'); var db = databases.db; describe('postSkipSegments', () => { + before(() => { + const now = Date.now(); + const warnVip01Hash = getHash("warn-vip01"); + const warnUser01Hash = getHash("warn-user01"); + const warnUser02Hash = getHash("warn-user02"); + const MILLISECONDS_IN_HOUR = 3600000; + const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires; + const startOfWarningQuery = 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES'; + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-1000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-2000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-3601000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 1000)) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 2000)) + "', '" + warnVip01Hash + "')"); + }); + it('Should be able to submit a single time (Params method)', (done) => { request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, @@ -130,6 +150,50 @@ describe('postSkipSegments', () => { }); }); + it('Should be rejected if user has to many active warnings', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "warn-user01", + videoID: "dQw4w9WgXcF", + segments: [{ + segment: [0, 10], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); // success + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be accepted if user has some active warnings', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "warn-user02", + videoID: "dQw4w9WgXcF", + segments: [{ + segment: [50, 60], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done(); // success + } else { + done("Status code was " + res.statusCode + " " + body); + } + }); + }); + it('Should be allowed if youtube thinks duration is 0', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null, diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index 2e94a0b..084d0f9 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -1,11 +1,19 @@ const request = require('request'); +const config = require('../../src/config.js'); const { db, privateDB } = require('../../src/databases/databases.js'); const utils = require('../utils.js'); const getHash = require('../../src/utils/getHash.js'); describe('voteOnSponsorTime', () => { before(() => { + const now = Date.now(); + const warnVip01Hash = getHash("warn-vip01"); + const warnUser01Hash = getHash("warn-voteuser01"); + const warnUser02Hash = getHash("warn-voteuser02"); + const MILLISECONDS_IN_HOUR = 3600000; + const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires; let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + const startOfWarningQuery = 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES'; db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')"); db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest2', 1) + "')"); @@ -25,6 +33,17 @@ describe('voteOnSponsorTime', () => { db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('not-own-submission-video', 1) + "')"); db.exec(startOfQuery + "('incorrect-category', 1, 11, 500, 'incorrect-category', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category', 1) + "')"); db.exec(startOfQuery + "('incorrect-category-change', 1, 11, 500, 'incorrect-category-change', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category-change', 1) + "')"); + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'warnvote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')"); + + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-1000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-2000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-3601000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 1000)) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 2000)) + "', '" + warnVip01Hash + "')"); + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); @@ -332,4 +351,17 @@ describe('voteOnSponsorTime', () => { }); }); + it('Should not be able to upvote a segment (Too many warning)', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=warn-voteuser01&UUID=warnvote-uuid-0&type=1", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); // success + } else { + done("Status code was " + res.statusCode); + } + }); + }); + }); \ No newline at end of file From 4cea3c2a3b07c01c2b36dd91cc029e698cbff213 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Thu, 1 Oct 2020 00:51:58 +0100 Subject: [PATCH 05/17] Added [] return rather than 404 for no segments with matching hash --- src/routes/getSkipSegmentsByHash.js | 4 ++-- test/cases/getSegmentsByHash.js | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 23a9167..fe25cc4 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -20,10 +20,10 @@ module.exports = async function (req, res) { // Get all video id's that match hash prefix const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']); - if (videoIds.length === 0) { + /*if (videoIds.length === 0) { res.sendStatus(404); return; - } + }*/ let segments = videoIds.map((video) => { return { diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 2704b61..c063345 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -40,7 +40,7 @@ describe('getSegmentsByHash', () => { }); }); - it('Should be able to get a 404 if no videos', (done) => { + /*it('Should be able to get a 404 if no videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11111?categories=["shilling"]', null, (err, res, body) => { @@ -50,8 +50,23 @@ describe('getSegmentsByHash', () => { done(); // pass } }); + });*/ + + it('Should be able to get an emptry array if no videos', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11111?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + if (JSON.parse(body).length === 0) done(); // pass + else done("non empty array returned"); + + } + }); }); + it('Should return 400 prefix too short', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11?categories=["shilling"]', null, From fe91d13ff3f7ca5f771f323f2ab192f73f08905e Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Thu, 1 Oct 2020 00:53:13 +0100 Subject: [PATCH 06/17] removed old functionality comments --- src/routes/getSkipSegmentsByHash.js | 4 ---- test/cases/getSegmentsByHash.js | 13 ------------- 2 files changed, 17 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index fe25cc4..19dab5a 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -20,10 +20,6 @@ module.exports = async function (req, res) { // Get all video id's that match hash prefix const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']); - /*if (videoIds.length === 0) { - res.sendStatus(404); - return; - }*/ let segments = videoIds.map((video) => { return { diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index c063345..fab4d77 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -40,18 +40,6 @@ describe('getSegmentsByHash', () => { }); }); - /*it('Should be able to get a 404 if no videos', (done) => { - request.get(utils.getbaseURL() - + '/api/skipSegments/11111?categories=["shilling"]', null, - (err, res, body) => { - if (err) done("Couldn't call endpoint"); - else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode); - else { - done(); // pass - } - }); - });*/ - it('Should be able to get an emptry array if no videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11111?categories=["shilling"]', null, @@ -66,7 +54,6 @@ describe('getSegmentsByHash', () => { }); }); - it('Should return 400 prefix too short', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11?categories=["shilling"]', null, From 8e33fdf49f6954db690b3a61bd11d64e4df6bb69 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 4 Oct 2020 21:37:35 +0200 Subject: [PATCH 07/17] Added segmentShift See #151 --- src/app.js | 4 + src/routes/postSegmentShift.js | 101 ++++++++++++ test/cases/segmentShift.js | 273 +++++++++++++++++++++++++++++++++ 3 files changed, 378 insertions(+) create mode 100644 src/routes/postSegmentShift.js create mode 100644 test/cases/segmentShift.js diff --git a/src/app.js b/src/app.js index a102a76..854031a 100644 --- a/src/app.js +++ b/src/app.js @@ -26,6 +26,7 @@ var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var postNoSegments = require('./routes/postNoSegments.js'); var getIsUserVIP = require('./routes/getIsUserVIP.js'); +var postSegmentShift = require('./routes/postSegmentShift.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -102,6 +103,9 @@ app.post('/api/noSegments', postNoSegments); //get if user is a vip app.get('/api/isUserVIP', getIsUserVIP); +//get if user is a vip +app.post('/api/segmentShift', postSegmentShift); + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); diff --git a/src/routes/postSegmentShift.js b/src/routes/postSegmentShift.js new file mode 100644 index 0000000..fd8c568 --- /dev/null +++ b/src/routes/postSegmentShift.js @@ -0,0 +1,101 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); + +const ACTION_NONE = Symbol('none'); +const ACTION_UPDATE = Symbol('update'); +const ACTION_REMOVE = Symbol('remove'); + +function shiftSegment(segment, shift) { + if (segment.startTime >= segment.endTime) return {action: ACTION_NONE, segment}; + if (shift.startTime >= shift.endTime) return {action: ACTION_NONE, segment}; + const duration = shift.endTime - shift.startTime; + if (shift.endTime < segment.startTime) { + // Scenario #1 cut before segment + segment.startTime -= duration; + segment.endTime -= duration; + return {action: ACTION_UPDATE, segment}; + } + if (shift.startTime > segment.endTime) { + // Scenario #2 cut after segment + return {action: ACTION_NONE, segment}; + } + if (segment.startTime < shift.startTime && segment.endTime > shift.endTime) { + // Scenario #3 cut inside segment + segment.endTime -= duration; + return {action: ACTION_UPDATE, segment}; + } + if (segment.startTime >= shift.startTime && segment.endTime > shift.endTime) { + // Scenario #4 cut overlap startTime + segment.startTime = shift.startTime; + segment.endTime -= duration; + return {action: ACTION_UPDATE, segment}; + } + if (segment.startTime < shift.startTime && segment.endTime <= shift.endTime) { + // Scenario #5 cut overlap endTime + segment.endTime = shift.startTime; + return {action: ACTION_UPDATE, segment}; + } + if (segment.startTime >= shift.startTime && segment.endTime <= shift.endTime) { + // Scenario #6 cut overlap startTime and endTime + return {action: ACTION_REMOVE, segment}; + } + return {action: ACTION_NONE, segment}; +} + +module.exports = (req, res) => { + // Collect user input data + const videoID = req.body.videoID; + const startTime = req.body.startTime; + const endTime = req.body.endTime; + let userID = req.body.userID; + + // Check input data is valid + if (!videoID + || !userID + || !startTime + || !endTime + ) { + res.status(400).json({ + message: 'Bad Format' + }); + return; + } + + // Check if user is VIP + userID = getHash(userID); + const userIsVIP = isUserVIP(userID); + + if (!userIsVIP) { + res.status(403).json({ + message: 'Must be a VIP to perform this action.' + }); + return; + } + + try { + const segments = db.prepare('all', 'SELECT startTime, endTime, UUID FROM sponsorTimes WHERE videoID = ?', [videoID]); + const shift = { + startTime, + endTime, + }; + segments.forEach(segment => { + const result = shiftSegment(segment, shift); + switch (result.action) { + case ACTION_UPDATE: + db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]); + break; + case ACTION_REMOVE: + db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ?, votes = -2 WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]); + break; + } + }); + } + catch(err) { + logger.error(err); + res.sendStatus(500); + } + + res.sendStatus(200); +}; diff --git a/test/cases/segmentShift.js b/test/cases/segmentShift.js new file mode 100644 index 0000000..e1011e2 --- /dev/null +++ b/test/cases/segmentShift.js @@ -0,0 +1,273 @@ +const request = require('request'); +const utils = require('../utils.js'); +const { db } = require('../../src/databases/databases.js'); +const getHash = require('../../src/utils/getHash.js'); + +function dbSponsorTimesAdd(db, videoID, startTime, endTime, UUID, category) { + const votes = 0, + userID = 0, + timeSubmitted = 0, + views = 0, + shadowHidden = 0, + hashedVideoID = `hash_${UUID}`; + db.exec(`INSERT INTO + sponsorTimes (videoID, startTime, endTime, votes, UUID, + userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) + VALUES + ('${videoID}', ${startTime}, ${endTime}, ${votes}, '${UUID}', + '${userID}', ${timeSubmitted}, ${views}, '${category}', ${shadowHidden}, '${hashedVideoID}') + `); +} + +function dbSponsorTimesSetByUUID(db, UUID, startTime, endTime) { + db.prepare('run', `UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?`, [startTime, endTime, UUID]); +} + +function dbSponsorTimesCompareExpect(db, expect) { + for (let i=0, len=expect.length; i { + if (err) return done(err); + return done(res.statusCode === 403 ? undefined : res.statusCode); + }); + }); + + it('Shift is outside segments', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 20, + endTime: 30, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 50, + endTime: 80, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 30, + endTime: 35, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is inside segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 65, + endTime: 75, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 60, + endTime: 80, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 40, + endTime: 45, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is overlaping startTime of segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 32, + endTime: 42, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 50, + endTime: 80, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 32, + endTime: 35, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is overlaping endTime of segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 85, + endTime: 95, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 60, + endTime: 85, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 40, + endTime: 45, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is overlaping segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 35, + endTime: 55, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 40, + endTime: 70, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 40, + endTime: 45, + removed: true, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 100, + endTime: 120, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + +}); From cce20319ada29e6fb70287f83ed51df05dd92b3b Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 6 Oct 2020 15:59:32 +0200 Subject: [PATCH 08/17] Submission UUID generation moved to function See issue #139 --- src/routes/postSkipSegments.js | 7 +++---- src/utils/getSubmissionUUID.js | 7 +++++++ test/cases/getSubmissionUUID.js | 8 ++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 src/utils/getSubmissionUUID.js create mode 100644 test/cases/getSubmissionUUID.js diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e47bb09..40c8287 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -5,6 +5,7 @@ const db = databases.db; const privateDB = databases.privateDB; const YouTubeAPI = require('../utils/youtubeAPI.js'); const logger = require('../utils/logger.js'); +const getSubmissionUUID = require('../utils/getSubmissionUUID.js'); const request = require('request'); const isoDurations = require('iso8601-duration'); const fetch = require('node-fetch'); @@ -201,8 +202,7 @@ async function autoModerateSubmission(submission) { startTime = parseFloat(segments[i].segment[0]); endTime = parseFloat(segments[i].segment[1]); - let UUID = getHash("v2-categories" + submission.videoID + startTime + - endTime + segments[i].category + submission.userID, 1); + const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime); // Send to Discord // Note, if this is too spammy. Consider sending all the segments as one Webhook sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); @@ -400,8 +400,7 @@ module.exports = async function postSkipSegments(req, res) { //this can just be a hash of the data //it's better than generating an actual UUID like what was used before //also better for duplication checking - let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] + - segmentInfo.segment[1] + segmentInfo.category + userID, 1); + const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]); try { db.prepare('run', "INSERT INTO sponsorTimes " + diff --git a/src/utils/getSubmissionUUID.js b/src/utils/getSubmissionUUID.js new file mode 100644 index 0000000..4142be1 --- /dev/null +++ b/src/utils/getSubmissionUUID.js @@ -0,0 +1,7 @@ +const getHash = require('./getHash.js'); + +module.exports = function getSubmissionUUID(videoID, category, userID, + startTime, endTime) { + return getHash('v2-categories' + videoID + startTime + endTime + category + + userID, 1); +}; diff --git a/test/cases/getSubmissionUUID.js b/test/cases/getSubmissionUUID.js new file mode 100644 index 0000000..75f23f2 --- /dev/null +++ b/test/cases/getSubmissionUUID.js @@ -0,0 +1,8 @@ +const getSubmissionUUID = require('../../src/utils/getSubmissionUUID.js'); +const assert = require('assert'); + +describe('getSubmissionUUID', () => { + it('Should return the hashed value', () => { + assert.equal(getSubmissionUUID('video001', 'sponsor', 'testuser001', 13.33337, 42.000001), '1d33d7016aa6482849019bd906d75c08fe6b815e64e823146df35f66c35612dd'); + }); +}); From 62916f6a7ec63c74823c68a41e67bf0e46fa96a9 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 6 Oct 2020 23:22:48 +0100 Subject: [PATCH 09/17] reverted to 404 sith empty array --- src/routes/getSkipSegmentsByHash.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 19dab5a..28131b6 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -29,5 +29,5 @@ module.exports = async function (req, res) { }; }); - res.status(200).json(segments); + res.status((segments.length === 0) ? 404 : 200).json(segments); } \ No newline at end of file From b244b1e1addf8ece82158957a48d9697b816ad10 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 6 Oct 2020 23:25:16 +0100 Subject: [PATCH 10/17] reverted test to 404 --- test/cases/getSegmentsByHash.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index fab4d77..ed4f819 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -40,16 +40,15 @@ describe('getSegmentsByHash', () => { }); }); - it('Should be able to get an emptry array if no videos', (done) => { + it('Should be able to get an empty array if no videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11111?categories=["shilling"]', null, (err, res, body) => { if (err) done("Couldn't call endpoint"); - else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode); else { - if (JSON.parse(body).length === 0) done(); // pass + if (JSON.parse(body).length === 0 && body === '[]') done(); // pass else done("non empty array returned"); - } }); }); From 58097f0d60a7f604c0fc75c4e44b4ce2cbbc5366 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 11 Oct 2020 16:17:17 +0200 Subject: [PATCH 11/17] Add vote rate limit --- config.json.example | 9 ++++++++- package-lock.json | 10 ++++++++++ package.json | 5 +++-- src/app.js | 7 +++++-- src/middleware/voteRateLimit.js | 18 ++++++++++++++++++ test.json | 9 ++++++++- 6 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 src/middleware/voteRateLimit.js diff --git a/config.json.example b/config.json.example index a2eee23..82d30fe 100644 --- a/config.json.example +++ b/config.json.example @@ -22,5 +22,12 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "rateLimit": { + "vote": { + "windowMs": 900000, // 15 minutes + "max": 20, // 20 requests in 15min time window + "message": "Too many votes, please try again later" + } + } } diff --git a/package-lock.json b/package-lock.json index 0878451..c608133 100644 --- a/package-lock.json +++ b/package-lock.json @@ -785,6 +785,11 @@ } } }, + "express-rate-limit": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.1.3.tgz", + "integrity": "sha512-TINcxve5510pXj4n9/1AMupkj3iWxl3JuZaWhCdYDlZeoCPqweGZrxbrlqTCFb1CT5wli7s8e2SH/Qz2c9GorA==" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -1866,6 +1871,11 @@ "semver": "^5.7.0" } }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "node-forge": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", diff --git a/package.json b/package.json index 4a44856..d3112d2 100644 --- a/package.json +++ b/package.json @@ -14,13 +14,14 @@ "dependencies": { "better-sqlite3": "^5.4.3", "express": "^4.17.1", + "express-rate-limit": "^5.1.3", "http": "0.0.0", "iso8601-duration": "^1.2.0", + "node-fetch": "^2.6.0", "redis": "^3.0.2", "sync-mysql": "^3.0.1", "uuid": "^3.3.2", - "youtube-api": "^2.0.10", - "node-fetch": "^2.6.0" + "youtube-api": "^2.0.10" }, "devDependencies": { "mocha": "^7.1.1", diff --git a/src/app.js b/src/app.js index a102a76..0f12117 100644 --- a/src/app.js +++ b/src/app.js @@ -3,8 +3,11 @@ var express = require('express'); var app = express(); var config = require('./config.js'); var redis = require('./utils/redis.js'); +const getIP = require('./utils/getIP.js'); +const getHash = require('./utils/getHash.js'); // Middleware +const voteRateLimitMiddleware = require('./middleware/voteRateLimit.js'); var corsMiddleware = require('./middleware/cors.js'); var loggerMiddleware = require('./middleware/logger.js'); const userCounter = require('./middleware/userCounter.js'); @@ -59,8 +62,8 @@ app.post('/api/skipSegments', postSkipSegments); app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash); //voting endpoint -app.get('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint); -app.post('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint); +app.get('/api/voteOnSponsorTime', voteRateLimitMiddleware, voteOnSponsorTime.endpoint); +app.post('/api/voteOnSponsorTime', voteRateLimitMiddleware, voteOnSponsorTime.endpoint); //Endpoint when a sponsorTime is used up app.get('/api/viewedVideoSponsorTime', viewedVideoSponsorTime); diff --git a/src/middleware/voteRateLimit.js b/src/middleware/voteRateLimit.js new file mode 100644 index 0000000..50120ff --- /dev/null +++ b/src/middleware/voteRateLimit.js @@ -0,0 +1,18 @@ +const config = require('../config.js'); +const getIP = require('../utils/getIP.js'); +const getHash = require('../utils/getHash.js'); +const rateLimit = require('express-rate-limit'); + +module.exports = rateLimit({ + windowMs: config.rateLimit.vote.windowMs, + max: config.rateLimit.vote.max, + message: config.rateLimit.vote.message, + headers: false, + keyGenerator: (req /*, res*/) => { + return getHash(req.ip, 1); + }, + skip: (/*req, res*/) => { + // skip rate limit if running in test mode + return process.env.npm_lifecycle_script === 'node test.js'; + } +}); diff --git a/test.json b/test.json index 8905d83..23ac900 100644 --- a/test.json +++ b/test.json @@ -49,5 +49,12 @@ ] } ], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + "rateLimit": { + "vote": { + "windowMs": 900000, + "max": 20, + "message": "Too many votes, please try again later" + } + } } From 73b7332639815427d90086cc8d393f1fbb6ce6ad Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Oct 2020 12:42:02 -0400 Subject: [PATCH 12/17] Add vote rate limit to views as well --- src/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.js b/src/app.js index 0f12117..1aba966 100644 --- a/src/app.js +++ b/src/app.js @@ -66,8 +66,8 @@ app.get('/api/voteOnSponsorTime', voteRateLimitMiddleware, voteOnSponsorTime.end app.post('/api/voteOnSponsorTime', voteRateLimitMiddleware, voteOnSponsorTime.endpoint); //Endpoint when a sponsorTime is used up -app.get('/api/viewedVideoSponsorTime', viewedVideoSponsorTime); -app.post('/api/viewedVideoSponsorTime', viewedVideoSponsorTime); +app.get('/api/viewedVideoSponsorTime', voteRateLimitMiddleware, viewedVideoSponsorTime); +app.post('/api/viewedVideoSponsorTime', voteRateLimitMiddleware, viewedVideoSponsorTime); //To set your username for the stats view app.post('/api/setUsername', setUsername); From a5f5f72346c64a2aaeb6f0a4a3ffc8846be42a03 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Oct 2020 13:07:57 -0400 Subject: [PATCH 13/17] Setup different ratelimit for views and votes --- config.json.example | 8 +++++++- src/app.js | 20 +++++++++++++------ .../{voteRateLimit.js => requestRateLimit.js} | 12 +++++------ test.json | 16 ++++++++++----- 4 files changed, 38 insertions(+), 18 deletions(-) rename src/middleware/{voteRateLimit.js => requestRateLimit.js} (61%) diff --git a/config.json.example b/config.json.example index 82d30fe..9ff06fb 100644 --- a/config.json.example +++ b/config.json.example @@ -27,7 +27,13 @@ "vote": { "windowMs": 900000, // 15 minutes "max": 20, // 20 requests in 15min time window - "message": "Too many votes, please try again later" + "message": "Too many votes, please try again later", + "statusCode": 200 + }, + "view": { + "windowMs": 900000, // 15 minutes + "max": 20, // 20 requests in 15min time window + "statusCode": 200 } } } diff --git a/src/app.js b/src/app.js index 1aba966..0933f93 100644 --- a/src/app.js +++ b/src/app.js @@ -7,7 +7,7 @@ const getIP = require('./utils/getIP.js'); const getHash = require('./utils/getHash.js'); // Middleware -const voteRateLimitMiddleware = require('./middleware/voteRateLimit.js'); +const rateLimitMiddleware = require('./middleware/requestRateLimit.js'); var corsMiddleware = require('./middleware/cors.js'); var loggerMiddleware = require('./middleware/logger.js'); const userCounter = require('./middleware/userCounter.js'); @@ -34,6 +34,14 @@ var getIsUserVIP = require('./routes/getIsUserVIP.js'); var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); +// Rate limit endpoint lists +let voteEndpoints = [voteOnSponsorTime.endpoint]; +let viewEndpoints = [viewedVideoSponsorTime]; +if (config.rateLimit) { + // if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote)); + if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view)); +} + //setup CORS correctly app.use(corsMiddleware); app.use(loggerMiddleware); @@ -62,12 +70,12 @@ app.post('/api/skipSegments', postSkipSegments); app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash); //voting endpoint -app.get('/api/voteOnSponsorTime', voteRateLimitMiddleware, voteOnSponsorTime.endpoint); -app.post('/api/voteOnSponsorTime', voteRateLimitMiddleware, voteOnSponsorTime.endpoint); +app.get('/api/voteOnSponsorTime', ...voteEndpoints); +app.post('/api/voteOnSponsorTime', ...voteEndpoints); -//Endpoint when a sponsorTime is used up -app.get('/api/viewedVideoSponsorTime', voteRateLimitMiddleware, viewedVideoSponsorTime); -app.post('/api/viewedVideoSponsorTime', voteRateLimitMiddleware, viewedVideoSponsorTime); +//Endpoint when a submission is skipped +app.get('/api/viewedVideoSponsorTime', ...viewEndpoints); +app.post('/api/viewedVideoSponsorTime', ...viewEndpoints); //To set your username for the stats view app.post('/api/setUsername', setUsername); diff --git a/src/middleware/voteRateLimit.js b/src/middleware/requestRateLimit.js similarity index 61% rename from src/middleware/voteRateLimit.js rename to src/middleware/requestRateLimit.js index 50120ff..b66a0aa 100644 --- a/src/middleware/voteRateLimit.js +++ b/src/middleware/requestRateLimit.js @@ -1,15 +1,15 @@ -const config = require('../config.js'); const getIP = require('../utils/getIP.js'); const getHash = require('../utils/getHash.js'); const rateLimit = require('express-rate-limit'); -module.exports = rateLimit({ - windowMs: config.rateLimit.vote.windowMs, - max: config.rateLimit.vote.max, - message: config.rateLimit.vote.message, +module.exports = (limitConfig) => rateLimit({ + windowMs: limitConfig.windowMs, + max: limitConfig.max, + message: limitConfig.message, + statusCode: limitConfig.statusCode, headers: false, keyGenerator: (req /*, res*/) => { - return getHash(req.ip, 1); + return getHash(getIP(req), 1); }, skip: (/*req, res*/) => { // skip rate limit if running in test mode diff --git a/test.json b/test.json index 23ac900..15d06f8 100644 --- a/test.json +++ b/test.json @@ -51,10 +51,16 @@ ], "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], "rateLimit": { - "vote": { - "windowMs": 900000, - "max": 20, - "message": "Too many votes, please try again later" + "vote": { + "windowMs": 900000, + "max": 20, + "message": "Too many votes, please try again later", + "statusCode": 200 + }, + "view": { + "windowMs": 900000, + "max": 20, + "statusCode": 200 + } } - } } From 9aba0560a1ae7491d6e9c5dad4a9431164cfe5a2 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Oct 2020 13:13:16 -0400 Subject: [PATCH 14/17] Fix indentation --- test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.json b/test.json index 15d06f8..0ed40f0 100644 --- a/test.json +++ b/test.json @@ -62,5 +62,5 @@ "max": 20, "statusCode": 200 } - } + } } From d7763b688db4c33d2b6ece587cc6a70a83fcc108 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Oct 2020 13:14:52 -0400 Subject: [PATCH 15/17] Fix example error codes --- config.json.example | 2 +- test.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 9ff06fb..5f3c199 100644 --- a/config.json.example +++ b/config.json.example @@ -28,7 +28,7 @@ "windowMs": 900000, // 15 minutes "max": 20, // 20 requests in 15min time window "message": "Too many votes, please try again later", - "statusCode": 200 + "statusCode": 429 }, "view": { "windowMs": 900000, // 15 minutes diff --git a/test.json b/test.json index 0ed40f0..d8b1e7b 100644 --- a/test.json +++ b/test.json @@ -55,7 +55,7 @@ "windowMs": 900000, "max": 20, "message": "Too many votes, please try again later", - "statusCode": 200 + "statusCode": 429 }, "view": { "windowMs": 900000, From 11eb742540d1cc17ad213c62ce1d7542d0a093fe Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Oct 2020 13:29:54 -0400 Subject: [PATCH 16/17] Uncomment code --- src/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.js b/src/app.js index 0933f93..df03333 100644 --- a/src/app.js +++ b/src/app.js @@ -38,7 +38,7 @@ var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); let voteEndpoints = [voteOnSponsorTime.endpoint]; let viewEndpoints = [viewedVideoSponsorTime]; if (config.rateLimit) { - // if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote)); + if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote)); if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view)); } From e0df3d4208897770c90bfb9c6249e8d5db047064 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Oct 2020 14:11:20 -0400 Subject: [PATCH 17/17] Don't allow dots in videoID --- src/utils/youtubeAPI.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index a053eb1..1deb3b2 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -19,6 +19,11 @@ if (config.mode === "test") { // YouTubeAPI.videos.list wrapper with cacheing exportObject.listVideos = (videoID, part, callback) => { + if (videoID.length !== 11 || videoID.includes(".")) { + callback("Invalid video ID"); + return; + } + let redisKey = "youtube.video." + videoID + "." + part; redis.get(redisKey, (getErr, result) => { if (getErr || !result) {