From 43db13ab3fa30a69770e422d3f9e2794007a018a Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 14 Sep 2020 09:17:35 +0200 Subject: [PATCH 01/18] Added getUserInfo endpoint This endpoint provides the following data about the user: * userID * userName * minutesSaved * segmentCount * viewCount --- src/app.js | 3 ++ src/routes/getUserInfo.js | 71 +++++++++++++++++++++++++++ test/cases/getUserInfo.js | 101 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 src/routes/getUserInfo.js create mode 100644 test/cases/getUserInfo.js diff --git a/src/app.js b/src/app.js index c70115d..c6ab3b5 100644 --- a/src/app.js +++ b/src/app.js @@ -22,6 +22,7 @@ var getViewsForUser = require('./routes/getViewsForUser.js'); var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); +var getUserInfo = require('./routes/getUserInfo.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -86,6 +87,8 @@ app.get('/api/getTopUsers', getTopUsers); //send the total submissions, total views and total minutes saved app.get('/api/getTotalStats', getTotalStats); +app.get('/api/getUserInfo', getUserInfo); + //send out a formatted time saved total app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); diff --git a/src/routes/getUserInfo.js b/src/routes/getUserInfo.js new file mode 100644 index 0000000..8479129 --- /dev/null +++ b/src/routes/getUserInfo.js @@ -0,0 +1,71 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); + +function dbGetSubmittedSegmentSummary (userID) { + try { + let row = db.prepare("get", "SELECT SUM(((endTime - startTime) / 60) * views) as minutesSaved, count(*) as segmentCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]); + if (row.minutesSaved != null) { + return { + minutesSaved: row.minutesSaved, + segmentCount: row.segmentCount, + }; + } else { + return { + minutesSaved: 0, + segmentCount: 0, + }; + } + } catch (err) { + return false; + } +} + +function dbGetUsername (userID) { + try { + let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]); + if (row !== undefined) { + return row.userName; + } else { + //no username yet, just send back the userID + return userID; + } + } catch (err) { + return false; + } +} + +function dbGetViewsForUser (userID) { + try { + let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]); + //increase the view count by one + if (row.viewCount != null) { + return row.viewCount; + } else { + return 0; + } + } catch (err) { + return false; + } +} + +module.exports = function getUserInfo (req, res) { + let userID = req.query.userID; + + if (userID == undefined) { + //invalid request + res.status(400).send('Parameters are not valid'); + return; + } + + //hash the userID + userID = getHash(userID); + + const segmentsSummary = dbGetSubmittedSegmentSummary(userID); + res.send({ + userID, + userName: dbGetUsername(userID), + minutesSaved: segmentsSummary.minutesSaved, + segmentCount: segmentsSummary.segmentCount, + viewCount: dbGetViewsForUser(userID), + }); +} diff --git a/test/cases/getUserInfo.js b/test/cases/getUserInfo.js new file mode 100644 index 0000000..499ae3c --- /dev/null +++ b/test/cases/getUserInfo.js @@ -0,0 +1,101 @@ +var request = require('request'); +var utils = require('../utils.js'); +var db = require('../../src/databases/databases.js').db; +var getHash = require('../../src/utils/getHash.js'); + +describe('getUserInfo', () => { + before(() => { + let startOfUserNamesQuery = "INSERT INTO userNames (userID, userName) VALUES"; + db.exec(startOfUserNamesQuery + "('" + getHash("getuserinfo_user_01") + "', 'Username user 01')"); + let startOfSponsorTimesQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000001', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000002', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -1, 'uuid000003', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -2, 'uuid000004', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)"); + db.exec(startOfSponsorTimesQuery + "('xzzzxxyyy', 1, 11, -5, 'uuid000005', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)"); + db.exec(startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 0)"); + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + }); + + it('Should be able to get a 200', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_user_01', null, + (err, res, body) => { + if (err) { + done('couldn\'t call endpoint'); + } else { + if (res.statusCode !== 200) { + done('non 200'); + } else { + done(); // pass + } + } + }); + }); + + it('Should be able to get a 400 (No userID parameter)', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo', null, + (err, res, body) => { + if (err) { + done('couldn\'t call endpoint'); + } else { + if (res.statusCode !== 400) { + done('non 400'); + } else { + done(); // pass + } + } + }); + }); + + it('Should return info', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_user_01', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.userName !== 'Username user 01') { + return done('Returned incorrect userName "' + data.userName + '"'); + } + if (data.minutesSaved !== 5) { + return done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); + } + if (data.viewCount !== 30) { + return done('Returned incorrect viewCount "' + data.viewCount + '"'); + } + if (data.segmentCount !== 3) { + return done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } + done(); // pass + } + } + }); + }); + + it('Should return userID for userName (No userName set)', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_user_02', null, + (err, res, body) => { + if (err) { + done('couldn\'t call endpoint'); + } else { + if (res.statusCode !== 200) { + done('non 200'); + } else { + const data = JSON.parse(body); + if (data.userName !== 'c2a28fd225e88f74945794ae85aef96001d4a1aaa1022c656f0dd48ac0a3ea0f') { + return done('Did not return userID for userName'); + } + done(); // pass + } + } + }); + }); +}); From 075cb9d5f2a28d39aa9f558be343000925e1d1c1 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 15 Sep 2020 23:56:10 +0100 Subject: [PATCH 02/18] added warning system Co-authored-by: Ajay Ramachandran --- databases/_upgrade_sponsorTimes_4.sql | 12 ++++ package-lock.json | 5 ++ src/app.js | 4 ++ src/routes/getUserInfo.js | 11 ++++ src/routes/postWarning.js | 24 ++++++++ src/routes/voteOnSponsorTime.js | 3 + test/cases/getUserInfo.js | 84 ++++++++++++++++++++++++--- test/cases/postWarning.js | 53 +++++++++++++++++ 8 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_4.sql create mode 100644 src/routes/postWarning.js create mode 100644 test/cases/postWarning.js diff --git a/databases/_upgrade_sponsorTimes_4.sql b/databases/_upgrade_sponsorTimes_4.sql new file mode 100644 index 0000000..91e51e2 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_4.sql @@ -0,0 +1,12 @@ +BEGIN TRANSACTION; + +/* Create warnings table */ +CREATE TABLE "warnings" ( + userID TEXT NOT NULL, + issueTime INTEGER NOT NULL, + issuerUserID TEXT NOT NULL +); + +UPDATE config SET value = 4 WHERE key = "version"; + +COMMIT; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0878451..5f08ab3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1866,6 +1866,11 @@ "semver": "^5.7.0" } }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "node-forge": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", diff --git a/src/app.js b/src/app.js index 787f32e..7e14923 100644 --- a/src/app.js +++ b/src/app.js @@ -27,6 +27,7 @@ var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var getUserInfo = require('./routes/getUserInfo.js'); var postNoSegments = require('./routes/postNoSegments.js'); var getIsUserVIP = require('./routes/getIsUserVIP.js'); +var warnUser = require('./routes/postWarning.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -105,6 +106,9 @@ app.post('/api/noSegments', postNoSegments); //get if user is a vip app.get('/api/isUserVIP', getIsUserVIP); +//sent user a warning +app.post('/api/warnUser', warnUser); + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); diff --git a/src/routes/getUserInfo.js b/src/routes/getUserInfo.js index 8479129..de17999 100644 --- a/src/routes/getUserInfo.js +++ b/src/routes/getUserInfo.js @@ -48,6 +48,16 @@ function dbGetViewsForUser (userID) { } } +function dbGetWarningsForUser (userID) { + try { + let rows = db.prepare('all', "SELECT * FROM warnings WHERE userID = ?", [userID]); + return rows.length; + } catch (err) { + logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0') ; + return 0; + } +} + module.exports = function getUserInfo (req, res) { let userID = req.query.userID; @@ -67,5 +77,6 @@ module.exports = function getUserInfo (req, res) { minutesSaved: segmentsSummary.minutesSaved, segmentCount: segmentsSummary.segmentCount, viewCount: dbGetViewsForUser(userID), + warnings: dbGetWarningsForUser(userID) }); } diff --git a/src/routes/postWarning.js b/src/routes/postWarning.js new file mode 100644 index 0000000..797d307 --- /dev/null +++ b/src/routes/postWarning.js @@ -0,0 +1,24 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); + +module.exports = (req, res) => { + // Collect user input data + let issuerUserID = getHash(req.body.issuerUserID); + let userID = getHash(req.body.userID); + let issueTime = new Date().getTime(); + + // Ensure user is a VIP + if (!isUserVIP(issuerUserID)) { + logger.debug("Permission violation: User " + issuerUserID + " attempted to warn user " + userID + "."); // maybe warn? + res.status(403).json({"message": "Not a VIP"}); + return; + } + + db.prepare('run', 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES (?, ?, ?)', [userID, issueTime, issuerUserID]); + res.status(200).json({ + message: "Warning issued to user '" + userID + "'." + }); + +}; \ No newline at end of file diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index e74dd3e..e8c4edc 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -270,6 +270,9 @@ async function voteOnSponsorTime(req, res) { } else if (votesRow.type === 2) { //extra downvote oldIncrementAmount = -4; + } else if (votesRow.type === 20) { + //undo/cancel vote + oldIncrementAmount = 0; } else if (votesRow.type < 0) { //vip downvote oldIncrementAmount = votesRow.type; diff --git a/test/cases/getUserInfo.js b/test/cases/getUserInfo.js index 499ae3c..f04c02f 100644 --- a/test/cases/getUserInfo.js +++ b/test/cases/getUserInfo.js @@ -16,6 +16,11 @@ describe('getUserInfo', () => { db.exec(startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 0)"); db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + + + db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_0') + "', 10, 'getuserinfo_vip')"); + db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_1') + "', 10, 'getuserinfo_vip')"); + db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_1') + "', 10, 'getuserinfo_vip')"); }); it('Should be able to get a 200', (done) => { @@ -26,7 +31,7 @@ describe('getUserInfo', () => { done('couldn\'t call endpoint'); } else { if (res.statusCode !== 200) { - done('non 200'); + done('non 200 (' + res.statusCode + ')'); } else { done(); // pass } @@ -62,18 +67,79 @@ describe('getUserInfo', () => { } else { const data = JSON.parse(body); if (data.userName !== 'Username user 01') { - return done('Returned incorrect userName "' + data.userName + '"'); + done('Returned incorrect userName "' + data.userName + '"'); + } else if (data.minutesSaved !== 5) { + done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); + } else if (data.viewCount !== 30) { + done('Returned incorrect viewCount "' + data.viewCount + '"'); + } else if (data.segmentCount !== 3) { + done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } else { + done(); // pass } - if (data.minutesSaved !== 5) { - return done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); + } + } + }); + }); + + it('Should get warning data', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_warning_0', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.warnings !== 1) { + done('wrong number of warnings: ' + data.warnings + ', not ' + 1); + } else { + done(); // pass } - if (data.viewCount !== 30) { - return done('Returned incorrect viewCount "' + data.viewCount + '"'); + } + } + }); + }); + + it('Should get multiple warnings', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_warning_1', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.warnings !== 2) { + done('wrong number of warnings: ' + data.warnings + ', not ' + 2); + } else { + done(); // pass } - if (data.segmentCount !== 3) { - return done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } + } + }); + }); + + it('Should not get warnings if noe', (done) => { + request.get(utils.getbaseURL() + + '/api/getUserInfo?userID=getuserinfo_warning_2', null, + (err, res, body) => { + if (err) { + done("couldn't call endpoint"); + } else { + if (res.statusCode !== 200) { + done("non 200"); + } else { + const data = JSON.parse(body); + if (data.warnings !== 0) { + done('wrong number of warnings: ' + data.warnings + ', not ' + 0); + } else { + done(); // pass } - done(); // pass } } }); diff --git a/test/cases/postWarning.js b/test/cases/postWarning.js new file mode 100644 index 0000000..7661bf5 --- /dev/null +++ b/test/cases/postWarning.js @@ -0,0 +1,53 @@ +var request = require('request'); +var utils = require('../utils.js'); +var db = require('../../src/databases/databases.js').db; +var getHash = require('../../src/utils/getHash.js'); + +describe('postWarning', () => { + before(() => { + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("warning-vip") + "')"); + }); + + it('Should update the database version when starting the application', (done) => { + let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (version > 3) done(); + else done('Version isn\'t greater than 3. Version is ' + version); + }); + + it('Should be able to create warning if vip (exp 200)', (done) => { + let json = { + issuerUserID: 'warning-vip', + userID: 'warning-0' + }; + + request.post(utils.getbaseURL() + + "/api/warnUser", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done(); + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + it('Should not be able to create warning if vip (exp 403)', (done) => { + let json = { + issuerUserID: 'warning-not-vip', + userID: 'warning-1' + }; + + request.post(utils.getbaseURL() + + "/api/warnUser", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); +}); \ No newline at end of file From 3c79c0f7a840ac8ec8800dac066c7d7decaae409 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 16 Sep 2020 13:48:54 -0400 Subject: [PATCH 03/18] Remove extra test case --- test/cases/postWarning.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/cases/postWarning.js b/test/cases/postWarning.js index 7661bf5..e5e8586 100644 --- a/test/cases/postWarning.js +++ b/test/cases/postWarning.js @@ -8,12 +8,6 @@ describe('postWarning', () => { db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("warning-vip") + "')"); }); - it('Should update the database version when starting the application', (done) => { - let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; - if (version > 3) done(); - else done('Version isn\'t greater than 3. Version is ' + version); - }); - it('Should be able to create warning if vip (exp 200)', (done) => { let json = { issuerUserID: 'warning-vip', @@ -50,4 +44,4 @@ describe('postWarning', () => { } }); }); -}); \ No newline at end of file +}); From 36f654f41c549d71eb32f44e894beab879d9efae Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Wed, 16 Sep 2020 22:40:11 +0200 Subject: [PATCH 04/18] Blocking users with too many active warnings from submitting votes and submissions --- config.json.example | 4 ++- src/config.js | 4 ++- src/routes/postSkipSegments.js | 10 ++++++ src/routes/voteOnSponsorTime.js | 10 ++++++ test.json | 4 ++- test/cases/postSkipSegments.js | 64 +++++++++++++++++++++++++++++++++ test/cases/voteOnSponsorTime.js | 32 +++++++++++++++++ 7 files changed, 125 insertions(+), 3 deletions(-) diff --git a/config.json.example b/config.json.example index a2eee23..f3a2413 100644 --- a/config.json.example +++ b/config.json.example @@ -22,5 +22,7 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire + "hoursAfterWarningExpire": 24 } diff --git a/src/config.js b/src/config.js index d85b930..b3f30d3 100644 --- a/src/config.js +++ b/src/config.js @@ -20,7 +20,9 @@ addDefaults(config, { "privateDBSchema": "./databases/_private.db.sql", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + "maxNumberOfActiveWarnings": 3, + "hoursAfterWarningExpires": 24 }) module.exports = config; diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 820e0cf..4961e92 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -269,6 +269,16 @@ module.exports = async function postSkipSegments(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(getIP(req) + config.globalSalt); + + const MILLISECONDS_IN_HOUR = 3600000; + const now = Date.now(); + let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?", + [userID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))] + ).count; + + if (warningsCount >= config.maxNumberOfActiveWarnings) { + return res.status(403).send('Submission blocked. Too many active warnings!'); + } let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category }); diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index e8c4edc..0e6de00 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -235,6 +235,16 @@ async function voteOnSponsorTime(req, res) { return; } } + + const MILLISECONDS_IN_HOUR = 3600000; + const now = Date.now(); + let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?", + [nonAnonUserID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))] + ).count; + + if (warningsCount >= config.maxNumberOfActiveWarnings) { + return res.status(403).send('Vote blocked. Too many active warnings!'); + } let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect; diff --git a/test.json b/test.json index 8905d83..c620be4 100644 --- a/test.json +++ b/test.json @@ -49,5 +49,7 @@ ] } ], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + "maxNumberOfActiveWarnings": 3, + "hoursAfterWarningExpires": 24 } diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index a2e2c97..b8966b1 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -1,5 +1,7 @@ var assert = require('assert'); var request = require('request'); +var config = require('../../src/config.js'); +var getHash = require('../../src/utils/getHash.js'); var utils = require('../utils.js'); @@ -7,6 +9,24 @@ var databases = require('../../src/databases/databases.js'); var db = databases.db; describe('postSkipSegments', () => { + before(() => { + const now = Date.now(); + const warnVip01Hash = getHash("warn-vip01"); + const warnUser01Hash = getHash("warn-user01"); + const warnUser02Hash = getHash("warn-user02"); + const MILLISECONDS_IN_HOUR = 3600000; + const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires; + const startOfWarningQuery = 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES'; + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-1000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-2000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-3601000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 1000)) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 2000)) + "', '" + warnVip01Hash + "')"); + }); + it('Should be able to submit a single time (Params method)', (done) => { request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, @@ -130,6 +150,50 @@ describe('postSkipSegments', () => { }); }); + it('Should be rejected if user has to many active warnings', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "warn-user01", + videoID: "dQw4w9WgXcF", + segments: [{ + segment: [0, 10], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); // success + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be accepted if user has some active warnings', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "warn-user02", + videoID: "dQw4w9WgXcF", + segments: [{ + segment: [50, 60], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done(); // success + } else { + done("Status code was " + res.statusCode + " " + body); + } + }); + }); + it('Should be allowed if youtube thinks duration is 0', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null, diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index 2e94a0b..084d0f9 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -1,11 +1,19 @@ const request = require('request'); +const config = require('../../src/config.js'); const { db, privateDB } = require('../../src/databases/databases.js'); const utils = require('../utils.js'); const getHash = require('../../src/utils/getHash.js'); describe('voteOnSponsorTime', () => { before(() => { + const now = Date.now(); + const warnVip01Hash = getHash("warn-vip01"); + const warnUser01Hash = getHash("warn-voteuser01"); + const warnUser02Hash = getHash("warn-voteuser02"); + const MILLISECONDS_IN_HOUR = 3600000; + const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires; let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + const startOfWarningQuery = 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES'; db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')"); db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest2', 1) + "')"); @@ -25,6 +33,17 @@ describe('voteOnSponsorTime', () => { db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('not-own-submission-video', 1) + "')"); db.exec(startOfQuery + "('incorrect-category', 1, 11, 500, 'incorrect-category', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category', 1) + "')"); db.exec(startOfQuery + "('incorrect-category-change', 1, 11, 500, 'incorrect-category-change', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category-change', 1) + "')"); + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'warnvote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')"); + + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-1000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-2000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-3601000) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 1000)) + "', '" + warnVip01Hash + "')"); + db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 2000)) + "', '" + warnVip01Hash + "')"); + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); @@ -332,4 +351,17 @@ describe('voteOnSponsorTime', () => { }); }); + it('Should not be able to upvote a segment (Too many warning)', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=warn-voteuser01&UUID=warnvote-uuid-0&type=1", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); // success + } else { + done("Status code was " + res.statusCode); + } + }); + }); + }); \ No newline at end of file From 4cea3c2a3b07c01c2b36dd91cc029e698cbff213 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Thu, 1 Oct 2020 00:51:58 +0100 Subject: [PATCH 05/18] Added [] return rather than 404 for no segments with matching hash --- src/routes/getSkipSegmentsByHash.js | 4 ++-- test/cases/getSegmentsByHash.js | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 23a9167..fe25cc4 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -20,10 +20,10 @@ module.exports = async function (req, res) { // Get all video id's that match hash prefix const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']); - if (videoIds.length === 0) { + /*if (videoIds.length === 0) { res.sendStatus(404); return; - } + }*/ let segments = videoIds.map((video) => { return { diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 2704b61..c063345 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -40,7 +40,7 @@ describe('getSegmentsByHash', () => { }); }); - it('Should be able to get a 404 if no videos', (done) => { + /*it('Should be able to get a 404 if no videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11111?categories=["shilling"]', null, (err, res, body) => { @@ -50,8 +50,23 @@ describe('getSegmentsByHash', () => { done(); // pass } }); + });*/ + + it('Should be able to get an emptry array if no videos', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11111?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + if (JSON.parse(body).length === 0) done(); // pass + else done("non empty array returned"); + + } + }); }); + it('Should return 400 prefix too short', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11?categories=["shilling"]', null, From fe91d13ff3f7ca5f771f323f2ab192f73f08905e Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Thu, 1 Oct 2020 00:53:13 +0100 Subject: [PATCH 06/18] removed old functionality comments --- src/routes/getSkipSegmentsByHash.js | 4 ---- test/cases/getSegmentsByHash.js | 13 ------------- 2 files changed, 17 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index fe25cc4..19dab5a 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -20,10 +20,6 @@ module.exports = async function (req, res) { // Get all video id's that match hash prefix const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']); - /*if (videoIds.length === 0) { - res.sendStatus(404); - return; - }*/ let segments = videoIds.map((video) => { return { diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index c063345..fab4d77 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -40,18 +40,6 @@ describe('getSegmentsByHash', () => { }); }); - /*it('Should be able to get a 404 if no videos', (done) => { - request.get(utils.getbaseURL() - + '/api/skipSegments/11111?categories=["shilling"]', null, - (err, res, body) => { - if (err) done("Couldn't call endpoint"); - else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode); - else { - done(); // pass - } - }); - });*/ - it('Should be able to get an emptry array if no videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11111?categories=["shilling"]', null, @@ -66,7 +54,6 @@ describe('getSegmentsByHash', () => { }); }); - it('Should return 400 prefix too short', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11?categories=["shilling"]', null, From 8e33fdf49f6954db690b3a61bd11d64e4df6bb69 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 4 Oct 2020 21:37:35 +0200 Subject: [PATCH 07/18] Added segmentShift See #151 --- src/app.js | 4 + src/routes/postSegmentShift.js | 101 ++++++++++++ test/cases/segmentShift.js | 273 +++++++++++++++++++++++++++++++++ 3 files changed, 378 insertions(+) create mode 100644 src/routes/postSegmentShift.js create mode 100644 test/cases/segmentShift.js diff --git a/src/app.js b/src/app.js index a102a76..854031a 100644 --- a/src/app.js +++ b/src/app.js @@ -26,6 +26,7 @@ var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var postNoSegments = require('./routes/postNoSegments.js'); var getIsUserVIP = require('./routes/getIsUserVIP.js'); +var postSegmentShift = require('./routes/postSegmentShift.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -102,6 +103,9 @@ app.post('/api/noSegments', postNoSegments); //get if user is a vip app.get('/api/isUserVIP', getIsUserVIP); +//get if user is a vip +app.post('/api/segmentShift', postSegmentShift); + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); diff --git a/src/routes/postSegmentShift.js b/src/routes/postSegmentShift.js new file mode 100644 index 0000000..fd8c568 --- /dev/null +++ b/src/routes/postSegmentShift.js @@ -0,0 +1,101 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); + +const ACTION_NONE = Symbol('none'); +const ACTION_UPDATE = Symbol('update'); +const ACTION_REMOVE = Symbol('remove'); + +function shiftSegment(segment, shift) { + if (segment.startTime >= segment.endTime) return {action: ACTION_NONE, segment}; + if (shift.startTime >= shift.endTime) return {action: ACTION_NONE, segment}; + const duration = shift.endTime - shift.startTime; + if (shift.endTime < segment.startTime) { + // Scenario #1 cut before segment + segment.startTime -= duration; + segment.endTime -= duration; + return {action: ACTION_UPDATE, segment}; + } + if (shift.startTime > segment.endTime) { + // Scenario #2 cut after segment + return {action: ACTION_NONE, segment}; + } + if (segment.startTime < shift.startTime && segment.endTime > shift.endTime) { + // Scenario #3 cut inside segment + segment.endTime -= duration; + return {action: ACTION_UPDATE, segment}; + } + if (segment.startTime >= shift.startTime && segment.endTime > shift.endTime) { + // Scenario #4 cut overlap startTime + segment.startTime = shift.startTime; + segment.endTime -= duration; + return {action: ACTION_UPDATE, segment}; + } + if (segment.startTime < shift.startTime && segment.endTime <= shift.endTime) { + // Scenario #5 cut overlap endTime + segment.endTime = shift.startTime; + return {action: ACTION_UPDATE, segment}; + } + if (segment.startTime >= shift.startTime && segment.endTime <= shift.endTime) { + // Scenario #6 cut overlap startTime and endTime + return {action: ACTION_REMOVE, segment}; + } + return {action: ACTION_NONE, segment}; +} + +module.exports = (req, res) => { + // Collect user input data + const videoID = req.body.videoID; + const startTime = req.body.startTime; + const endTime = req.body.endTime; + let userID = req.body.userID; + + // Check input data is valid + if (!videoID + || !userID + || !startTime + || !endTime + ) { + res.status(400).json({ + message: 'Bad Format' + }); + return; + } + + // Check if user is VIP + userID = getHash(userID); + const userIsVIP = isUserVIP(userID); + + if (!userIsVIP) { + res.status(403).json({ + message: 'Must be a VIP to perform this action.' + }); + return; + } + + try { + const segments = db.prepare('all', 'SELECT startTime, endTime, UUID FROM sponsorTimes WHERE videoID = ?', [videoID]); + const shift = { + startTime, + endTime, + }; + segments.forEach(segment => { + const result = shiftSegment(segment, shift); + switch (result.action) { + case ACTION_UPDATE: + db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]); + break; + case ACTION_REMOVE: + db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ?, votes = -2 WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]); + break; + } + }); + } + catch(err) { + logger.error(err); + res.sendStatus(500); + } + + res.sendStatus(200); +}; diff --git a/test/cases/segmentShift.js b/test/cases/segmentShift.js new file mode 100644 index 0000000..e1011e2 --- /dev/null +++ b/test/cases/segmentShift.js @@ -0,0 +1,273 @@ +const request = require('request'); +const utils = require('../utils.js'); +const { db } = require('../../src/databases/databases.js'); +const getHash = require('../../src/utils/getHash.js'); + +function dbSponsorTimesAdd(db, videoID, startTime, endTime, UUID, category) { + const votes = 0, + userID = 0, + timeSubmitted = 0, + views = 0, + shadowHidden = 0, + hashedVideoID = `hash_${UUID}`; + db.exec(`INSERT INTO + sponsorTimes (videoID, startTime, endTime, votes, UUID, + userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) + VALUES + ('${videoID}', ${startTime}, ${endTime}, ${votes}, '${UUID}', + '${userID}', ${timeSubmitted}, ${views}, '${category}', ${shadowHidden}, '${hashedVideoID}') + `); +} + +function dbSponsorTimesSetByUUID(db, UUID, startTime, endTime) { + db.prepare('run', `UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?`, [startTime, endTime, UUID]); +} + +function dbSponsorTimesCompareExpect(db, expect) { + for (let i=0, len=expect.length; i { + if (err) return done(err); + return done(res.statusCode === 403 ? undefined : res.statusCode); + }); + }); + + it('Shift is outside segments', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 20, + endTime: 30, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 50, + endTime: 80, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 30, + endTime: 35, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is inside segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 65, + endTime: 75, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 60, + endTime: 80, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 40, + endTime: 45, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is overlaping startTime of segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 32, + endTime: 42, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 50, + endTime: 80, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 32, + endTime: 35, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is overlaping endTime of segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 85, + endTime: 95, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 60, + endTime: 85, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 40, + endTime: 45, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 110, + endTime: 130, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + it('Shift is overlaping segment', function(done) { + request.post(`${baseURL}/api/segmentShift`, { + json: { + videoID: 'vsegshift01', + userID: privateVipUserID, + startTime: 35, + endTime: 55, + } + }, (err, res, body) => { + if (err) return done(err); + if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`); + const expect = [ + { + UUID: 'vsegshifttest01uuid01', + startTime: 0, + endTime: 10, + }, + { + UUID: 'vsegshifttest01uuid02', + startTime: 40, + endTime: 70, + }, + { + UUID: 'vsegshifttest01uuid03', + startTime: 40, + endTime: 45, + removed: true, + }, + { + UUID: 'vsegshifttest01uuid04', + startTime: 100, + endTime: 120, + }, + ]; + done(dbSponsorTimesCompareExpect(db, expect)); + }); + }); + + +}); From 25b91af8bc6b3db9a82ccebff02bcdcd31923b99 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 6 Oct 2020 23:25:11 +0200 Subject: [PATCH 08/18] Add cache for getTopUsers See #147 --- config.json.example | 3 +- src/routes/getTopUsers.js | 133 ++++++++++++++++++--------------- src/utils/createMemoryCache.js | 42 +++++++++++ 3 files changed, 117 insertions(+), 61 deletions(-) create mode 100644 src/utils/createMemoryCache.js diff --git a/config.json.example b/config.json.example index a2eee23..d6f9204 100644 --- a/config.json.example +++ b/config.json.example @@ -22,5 +22,6 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "getTopUsersCacheTimeMinutes": 5 // cacheTime for getTopUsers result in minutes } diff --git a/src/routes/getTopUsers.js b/src/routes/getTopUsers.js index 22b8ca9..b78dad5 100644 --- a/src/routes/getTopUsers.js +++ b/src/routes/getTopUsers.js @@ -1,6 +1,68 @@ var db = require('../databases/databases.js').db; +const logger = require('../utils/logger.js'); +const createMemoryCache = require('../utils/createMemoryCache.js'); +const config = require('../config.js'); -module.exports = function getTopUsers (req, res) { +const MILLISECONDS_IN_MINUTE = 60000; +const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); + +function generateTopUsersStats(sortBy, categoryStatsEnabled = false) { + return new Promise((resolve, reject) => { + const userNames = []; + const viewCounts = []; + const totalSubmissions = []; + const minutesSaved = []; + const categoryStats = categoryStatsEnabled ? [] : undefined; + + let additionalFields = ''; + if (categoryStatsEnabled) { + additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " + + "SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " + + "SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " + + "SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " + + "SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " + + "SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, "; + } + + const rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + + "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " + + "SUM(votes) as userVotes, " + + additionalFields + + "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " + + "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " + + "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " + + "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " + + "ORDER BY " + sortBy + " DESC LIMIT 100", []); + + for (let i = 0; i < rows.length; i++) { + userNames[i] = rows[i].userName; + + viewCounts[i] = rows[i].viewCount; + totalSubmissions[i] = rows[i].totalSubmissions; + minutesSaved[i] = rows[i].minutesSaved; + if (categoryStatsEnabled) { + categoryStats[i] = [ + rows[i].categorySponsor, + rows[i].categorySumIntro, + rows[i].categorySumOutro, + rows[i].categorySumInteraction, + rows[i].categorySelfpromo, + rows[i].categoryMusicOfftopic, + ]; + } + } + + resolve({ + userNames, + viewCounts, + totalSubmissions, + minutesSaved, + categoryStats + }); + }); +} + +module.exports = async function getTopUsers (req, res) { let sortType = req.query.sortType; let categoryStatsEnabled = req.query.categoryStats; @@ -9,71 +71,22 @@ module.exports = function getTopUsers (req, res) { res.sendStatus(400); return; } - + //setup which sort type to use - let sortBy = ""; + let sortBy = ''; if (sortType == 0) { - sortBy = "minutesSaved"; + sortBy = 'minutesSaved'; } else if (sortType == 1) { - sortBy = "viewCount"; + sortBy = 'viewCount'; } else if (sortType == 2) { - sortBy = "totalSubmissions"; + sortBy = 'totalSubmissions'; } else { - //invalid request - res.sendStatus(400); - return; + //invalid request + return res.sendStatus(400); } - - let userNames = []; - let viewCounts = []; - let totalSubmissions = []; - let minutesSaved = []; - let categoryStats = categoryStatsEnabled ? [] : undefined; - let additionalFields = ''; - if (categoryStatsEnabled) { - additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " + - "SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " + - "SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " + - "SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " + - "SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " + - "SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, "; - } - - let rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + - "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " + - "SUM(votes) as userVotes, " + - additionalFields + - "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " + - "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " + - "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " + - "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " + - "ORDER BY " + sortBy + " DESC LIMIT 100", []); + const stats = await getTopUsersWithCache(sortBy, categoryStatsEnabled); - for (let i = 0; i < rows.length; i++) { - userNames[i] = rows[i].userName; - - viewCounts[i] = rows[i].viewCount; - totalSubmissions[i] = rows[i].totalSubmissions; - minutesSaved[i] = rows[i].minutesSaved; - if (categoryStatsEnabled) { - categoryStats[i] = [ - rows[i].categorySponsor, - rows[i].categorySumIntro, - rows[i].categorySumOutro, - rows[i].categorySumInteraction, - rows[i].categorySelfpromo, - rows[i].categoryMusicOfftopic, - ]; - } - } - //send this result - res.send({ - userNames, - viewCounts, - totalSubmissions, - minutesSaved, - categoryStats - }); -} \ No newline at end of file + res.send(stats); +} diff --git a/src/utils/createMemoryCache.js b/src/utils/createMemoryCache.js new file mode 100644 index 0000000..0f089ce --- /dev/null +++ b/src/utils/createMemoryCache.js @@ -0,0 +1,42 @@ +module.exports = function createMemoryCache(memoryFn, cacheTimeMs) { + // holds the promise results + const cache = new Map; + // holds the promises that are not fulfilled + const promiseMemory = new Map; + return function (...args) { + // create cacheKey by joining arguments as string + const cacheKey = args.join('.'); + // check if promising is already running + if (promiseMemory.has(cacheKey)) { + return promiseMemory.get(cacheKey); + } + else { + // check if result is in cache + if (cache.has(cacheKey)) { + const cacheItem = cache.get(cacheKey); + const now = Date.now(); + // check if cache is valid + if (!(cacheItem.cacheTime + cacheTimeMs < now)) { + return Promise.resolve(cacheItem.result); + } + } + // create new promise + const promise = new Promise(async (resolve, reject) => { + resolve((await memoryFn(...args))); + }); + // store promise reference until fulfilled + promiseMemory.set(cacheKey, promise); + return promise.then(result => { + // store promise result in cache + cache.set(cacheKey, { + result, + cacheTime: Date.now(), + }); + // remove fulfilled promise from memory + promiseMemory.delete(cacheKey); + // return promise result + return result; + }); + } + }; +}; From 62916f6a7ec63c74823c68a41e67bf0e46fa96a9 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 6 Oct 2020 23:22:48 +0100 Subject: [PATCH 09/18] reverted to 404 sith empty array --- src/routes/getSkipSegmentsByHash.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 19dab5a..28131b6 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -29,5 +29,5 @@ module.exports = async function (req, res) { }; }); - res.status(200).json(segments); + res.status((segments.length === 0) ? 404 : 200).json(segments); } \ No newline at end of file From b244b1e1addf8ece82158957a48d9697b816ad10 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 6 Oct 2020 23:25:16 +0100 Subject: [PATCH 10/18] reverted test to 404 --- test/cases/getSegmentsByHash.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index fab4d77..ed4f819 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -40,16 +40,15 @@ describe('getSegmentsByHash', () => { }); }); - it('Should be able to get an emptry array if no videos', (done) => { + it('Should be able to get an empty array if no videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/11111?categories=["shilling"]', null, (err, res, body) => { if (err) done("Couldn't call endpoint"); - else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode); else { - if (JSON.parse(body).length === 0) done(); // pass + if (JSON.parse(body).length === 0 && body === '[]') done(); // pass else done("non empty array returned"); - } }); }); From fb7ff50febbb45ddb2cdcfc8d7a7c60d786f9078 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Fri, 9 Oct 2020 08:58:15 +0200 Subject: [PATCH 11/18] Update src/utils/createMemoryCache.js Co-authored-by: Ajay Ramachandran --- src/utils/createMemoryCache.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/createMemoryCache.js b/src/utils/createMemoryCache.js index 0f089ce..77053ae 100644 --- a/src/utils/createMemoryCache.js +++ b/src/utils/createMemoryCache.js @@ -1,8 +1,8 @@ module.exports = function createMemoryCache(memoryFn, cacheTimeMs) { // holds the promise results - const cache = new Map; + const cache = new Map(); // holds the promises that are not fulfilled - const promiseMemory = new Map; + const promiseMemory = new Map(); return function (...args) { // create cacheKey by joining arguments as string const cacheKey = args.join('.'); From 41dc16453eadfd8240e5565acaa597df174d5eee Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Fri, 9 Oct 2020 08:58:31 +0200 Subject: [PATCH 12/18] Update src/utils/createMemoryCache.js Co-authored-by: Ajay Ramachandran --- src/utils/createMemoryCache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/createMemoryCache.js b/src/utils/createMemoryCache.js index 77053ae..ec4402d 100644 --- a/src/utils/createMemoryCache.js +++ b/src/utils/createMemoryCache.js @@ -3,7 +3,7 @@ module.exports = function createMemoryCache(memoryFn, cacheTimeMs) { const cache = new Map(); // holds the promises that are not fulfilled const promiseMemory = new Map(); - return function (...args) { + return (...args) => { // create cacheKey by joining arguments as string const cacheKey = args.join('.'); // check if promising is already running From 85dd187cb0b59750a6cd31030b9b7e155cfc2cc7 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 00:25:05 -0500 Subject: [PATCH 13/18] reject submissions if total length is more than 80 precent of the video --- src/routes/postSkipSegments.js | 61 ++++++++++++++ test/cases/postSkipSegments.js | 141 +++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 40c8287..de55938 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -396,6 +396,40 @@ module.exports = async function postSkipSegments(req, res) { startingVotes = 10000; } + if (config.youtubeAPIKey !== null) { + let {err, data} = await new Promise((resolve, reject) => { + YouTubeAPI.listVideos(videoID, "contentDetails,snippet", (err, data) => resolve({err, data})); + }); + + //get all segments for this video and user + let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ?", [userID, videoID]); + let allSegmentTimes = []; + if (allSubmittedByUser !== undefined) + { + for (const segmentInfo of allSubmittedByUser) { + allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]) + } + } + for (let i = 0; i < segments.length; i++) { + let startTime = parseFloat(segments[i].segment[0]); + let endTime = parseFloat(segments[i].segment[1]); + allSegmentTimes.push([startTime, endTime]); + } + const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function(a, b) { return a[0]-b[0] || a[1]-b[1] })); + + let videoDuration = data.items[0].contentDetails.duration; + videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); + if (videoDuration != 0) { + let allSegmentDuration = 0; + allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); + if (allSegmentDuration > (videoDuration/100)*80) { + // Reject submission if all segments combine are over 80% of the video + res.status(400).send("Total length of your submitted segments are over 80% of the video."); + return; + } + } + } + for (const segmentInfo of segments) { //this can just be a hash of the data //it's better than generating an actual UUID like what was used before @@ -437,3 +471,30 @@ module.exports = async function postSkipSegments(req, res) { sendWebhooks(userID, videoID, UUIDs[i], segments[i]); } } + +// Takes an array of arrays: +// ex) +// [ +// [3, 40], +// [50, 70], +// [60, 80], +// [100, 150] +// ] +// => transforms to combining overlapping segments +// [ +// [3, 40], +// [50, 80], +// [100, 150] +// ] +function mergeTimeSegments(ranges) { + var result = [], last; + + ranges.forEach(function (r) { + if (!last || r[0] > last[1]) + result.push(last = r); + else if (r[1] > last[1]) + last[1] = r[1]; + }); + + return result; +} \ No newline at end of file diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index a2e2c97..af87a9d 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -5,8 +5,16 @@ var utils = require('../utils.js'); var databases = require('../../src/databases/databases.js'); var db = databases.db; +const getHash = require('../../src/utils/getHash.js'); describe('postSkipSegments', () => { + before(() => { + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + + db.exec(startOfQuery + "('80percent_video', 0, 1000, 0, '80percent-uuid-0', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); + db.exec(startOfQuery + "('80percent_video', 1001, 1005, 0, '80percent-uuid-1', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); + }); + it('Should be able to submit a single time (Params method)', (done) => { request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, @@ -90,6 +98,139 @@ describe('postSkipSegments', () => { }); }).timeout(5000); + it('Should allow multiple times if total is under 80% of video(JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "L_jWHffIx5E", + segments: [{ + segment: [3, 3000], + category: "sponsor" + },{ + segment: [3002, 3050], + category: "intro" + },{ + segment: [45, 100], + category: "interaction" + },{ + segment: [99, 170], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["L_jWHffIx5E"]); + let success = true; + if (rows.length === 4) { + for (const row of rows) { + if ((row.startTime !== 3 || row.endTime !== 3000 || row.category !== "sponsor") && + (row.startTime !== 3002 || row.endTime !== 3050 || row.category !== "intro") && + (row.startTime !== 45 || row.endTime !== 100 || row.category !== "interaction") && + (row.startTime !== 99 || row.endTime !== 170 || row.category !== "sponsor")) { + success = false; + break; + } + } + } + + if (success) done(); + else done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows)); + } else { + done("Status code was " + res.statusCode); + } + }); + }).timeout(5000); + + it('Should reject multiple times if total is over 80% of video (JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "n9rIGdXnSJc", + segments: [{ + segment: [0, 2000], + category: "interaction" + },{ + segment: [3000, 4000], + category: "sponsor" + },{ + segment: [1500, 2750], + category: "sponsor" + },{ + segment: [4050, 4750], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["n9rIGdXnSJc"]); + let success = true; + if (rows.length === 4) { + for (const row of rows) { + if ((row.startTime === 0 || row.endTime === 2000 || row.category === "interaction") || + (row.startTime === 3000 || row.endTime === 4000 || row.category === "sponsor") || + (row.startTime === 1500 || row.endTime === 2750 || row.category === "sponsor") || + (row.startTime === 4050 || row.endTime === 4750 || row.category === "intro")) { + success = false; + break; + } + } + } + + if (success) done(); + else + done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows)); + } else { + done("Status code was " + res.statusCode); + } + }); + }).timeout(5000); + + it('Should reject multiple times if total is over 80% of video including previosuly submitted times(JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "80percent_video", + segments: [{ + segment: [2000, 4000], + category: "sponsor" + },{ + segment: [1500, 2750], + category: "sponsor" + },{ + segment: [4050, 4750], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["80percent_video"]); + let success = true && rows.length == 2; + for (const row of rows) { + if ((row.startTime === 2000 || row.endTime === 4000 || row.category === "sponsor") || + (row.startTime === 1500 || row.endTime === 2750 || row.category === "sponsor") || + (row.startTime === 4050 || row.endTime === 4750 || row.category === "sponsor")) { + success = false; + break; + } + } + if (success) done(); + else + done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows)); + } else { + done("Status code was " + res.statusCode); + } + }); + }).timeout(5000); + it('Should be accepted if a non-sponsor is less than 1 second', (done) => { request.post(utils.getbaseURL() + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=intro", null, From 2825cb63fb69fbe9a566f9ab1a6be3756460ac37 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 00:33:00 -0500 Subject: [PATCH 14/18] ignore segments with less than -1 votes --- src/routes/postSkipSegments.js | 2 +- test/cases/postSkipSegments.js | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index de55938..27bfdde 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -402,7 +402,7 @@ module.exports = async function postSkipSegments(req, res) { }); //get all segments for this video and user - let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ?", [userID, videoID]); + let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ? and votes > -1", [userID, videoID]); let allSegmentTimes = []; if (allSubmittedByUser !== undefined) { diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index af87a9d..eb8e287 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -13,6 +13,7 @@ describe('postSkipSegments', () => { db.exec(startOfQuery + "('80percent_video', 0, 1000, 0, '80percent-uuid-0', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); db.exec(startOfQuery + "('80percent_video', 1001, 1005, 0, '80percent-uuid-1', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); + db.exec(startOfQuery + "('80percent_video', 0, 5000, -2, '80percent-uuid-2', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); }); it('Should be able to submit a single time (Params method)', (done) => { @@ -122,7 +123,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["L_jWHffIx5E"]); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["L_jWHffIx5E"]); let success = true; if (rows.length === 4) { for (const row of rows) { @@ -168,7 +169,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { - let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["n9rIGdXnSJc"]); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["n9rIGdXnSJc"]); let success = true; if (rows.length === 4) { for (const row of rows) { @@ -212,7 +213,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { - let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["80percent_video"]); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["80percent_video"]); let success = true && rows.length == 2; for (const row of rows) { if ((row.startTime === 2000 || row.endTime === 4000 || row.category === "sponsor") || From bcf90e8094f8933637168392545a57f8b55718a8 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 00:43:29 -0500 Subject: [PATCH 15/18] fix formatting and add more comments --- src/routes/postSkipSegments.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 27bfdde..092b7f8 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -404,23 +404,28 @@ module.exports = async function postSkipSegments(req, res) { //get all segments for this video and user let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ? and votes > -1", [userID, videoID]); let allSegmentTimes = []; - if (allSubmittedByUser !== undefined) - { + if (allSubmittedByUser !== undefined) { + //add segments the user has previously submitted for (const segmentInfo of allSubmittedByUser) { allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]) } } + + //add segments they are trying to add in this submission for (let i = 0; i < segments.length; i++) { let startTime = parseFloat(segments[i].segment[0]); let endTime = parseFloat(segments[i].segment[1]); allSegmentTimes.push([startTime, endTime]); } + + //merge all the times into non-overlapping arrays const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function(a, b) { return a[0]-b[0] || a[1]-b[1] })); let videoDuration = data.items[0].contentDetails.duration; videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); if (videoDuration != 0) { let allSegmentDuration = 0; + //sum all segment times together allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); if (allSegmentDuration > (videoDuration/100)*80) { // Reject submission if all segments combine are over 80% of the video From 2426b515122cd8bde8e4a2a1b16a1298e2399bc9 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Wed, 14 Oct 2020 20:18:31 +0100 Subject: [PATCH 16/18] added delete nosegment endpoint --- src/app.js | 3 ++ src/routes/deleteNoSegments.js | 43 +++++++++++++++++++++++++ test/cases/noSegmentRecords.js | 58 ++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 src/routes/deleteNoSegments.js diff --git a/src/app.js b/src/app.js index a102a76..14e9dbd 100644 --- a/src/app.js +++ b/src/app.js @@ -25,6 +25,7 @@ var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var postNoSegments = require('./routes/postNoSegments.js'); +var deleteNoSegments = require('./routes/deleteNoSegments.js'); var getIsUserVIP = require('./routes/getIsUserVIP.js'); // Old Routes @@ -99,6 +100,8 @@ app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); //submit video containing no segments app.post('/api/noSegments', postNoSegments); +app.delete('/api/noSegments', deleteNoSegments); + //get if user is a vip app.get('/api/isUserVIP', getIsUserVIP); diff --git a/src/routes/deleteNoSegments.js b/src/routes/deleteNoSegments.js new file mode 100644 index 0000000..adb44ba --- /dev/null +++ b/src/routes/deleteNoSegments.js @@ -0,0 +1,43 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); + +module.exports = (req, res) => { + // Collect user input data + let videoID = req.body.videoID; + let userID = req.body.userID; + let categories = req.body.categories; + + // Check input data is valid + if (!videoID + || !userID + || !categories + || !Array.isArray(categories) + || categories.length === 0 + ) { + res.status(400).json({ + message: 'Bad Format' + }); + return; + } + + // Check if user is VIP + userID = getHash(userID); + let userIsVIP = isUserVIP(userID); + + if (!userIsVIP) { + res.status(403).json({ + message: 'Must be a VIP to mark videos.' + }); + return; + } + + db.prepare("all", 'SELECT * FROM noSegments WHERE videoID = ?', [videoID]).filter((entry) => { + return (categories.indexOf(entry.category) !== -1); + }).forEach((entry) => { + db.prepare('run', 'DELETE FROM noSegments WHERE videoID = ? AND category = ?', [videoID, entry.category]); + }); + + res.status(200).json({message: 'Removed no segments entrys for video ' + videoID}); +}; \ No newline at end of file diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index f922d12..4167688 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -17,6 +17,11 @@ describe('noSegmentRecords', () => { db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')"); db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')"); db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')"); + + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'delete-record', 'sponsor')"); + + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'sponsor')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'intro')"); }); it('Should update the database version when starting the application', (done) => { @@ -309,6 +314,59 @@ describe('noSegmentRecords', () => { }); }); + it('Should be able to delete a noSegment record', (done) => { + let json = { + videoID: 'delete-record', + userID: 'VIPUser-noSegments', + categories: [ + 'sponsor' + ] + }; + + request.delete(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['delete-record']); + if (result.length === 0) { + done(); + } else { + done("Didn't delete record"); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to delete one noSegment record without removing another', (done) => { + let json = { + videoID: 'delete-record-1', + userID: 'VIPUser-noSegments', + categories: [ + 'sponsor' + ] + }; + + request.delete(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['delete-record-1']); + if (result.length === 1) { + done(); + } else { + done("Didn't delete record"); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + /* * Submission tests in this file do not check database records, only status codes. * To test the submission code properly see ./test/cases/postSkipSegments.js From 830ef7e0dccccc0dc8285279e290db9007617c55 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Wed, 14 Oct 2020 20:52:01 +0100 Subject: [PATCH 17/18] made youtube api cache have conatcnt part value --- src/routes/postSkipSegments.js | 6 +++--- src/routes/voteOnSponsorTime.js | 2 +- src/utils/youtubeAPI.js | 5 +++-- test/youtubeMock.js | 3 +-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 092b7f8..4da9466 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -50,7 +50,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { if (config.youtubeAPIKey !== null) { let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]); - YouTubeAPI.listVideos(videoID, "snippet", (err, data) => { + YouTubeAPI.listVideos(videoID, (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; @@ -154,7 +154,7 @@ async function autoModerateSubmission(submission) { // Get the video information from the youtube API if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { - YouTubeAPI.listVideos(submission.videoID, "contentDetails,snippet", (err, data) => resolve({err, data})); + YouTubeAPI.listVideos(submission.videoID, (err, data) => resolve({err, data})); }); if (err) { @@ -398,7 +398,7 @@ module.exports = async function postSkipSegments(req, res) { if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { - YouTubeAPI.listVideos(videoID, "contentDetails,snippet", (err, data) => resolve({err, data})); + YouTubeAPI.listVideos(videoID, (err, data) => resolve({err, data})); }); //get all segments for this video and user diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 59a6789..d2a9f49 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -49,7 +49,7 @@ function sendWebhooks(voteData) { } if (config.youtubeAPIKey !== null) { - YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => { + YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index 1deb3b2..96e718e 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -18,13 +18,14 @@ if (config.mode === "test") { exportObject = YouTubeAPI; // YouTubeAPI.videos.list wrapper with cacheing - exportObject.listVideos = (videoID, part, callback) => { + exportObject.listVideos = (videoID, callback) => { + let part = 'contentDetails,snippet'; if (videoID.length !== 11 || videoID.includes(".")) { callback("Invalid video ID"); return; } - let redisKey = "youtube.video." + videoID + "." + part; + let redisKey = "youtube.video." + videoID; redis.get(redisKey, (getErr, result) => { if (getErr || !result) { logger.debug("redis: no cache for video information: " + videoID); diff --git a/test/youtubeMock.js b/test/youtubeMock.js index cc722d9..7e15f64 100644 --- a/test/youtubeMock.js +++ b/test/youtubeMock.js @@ -8,9 +8,8 @@ YouTubeAPI.videos.list({ // https://developers.google.com/youtube/v3/docs/videos const YouTubeAPI = { - listVideos: (id, part, callback) => { + listVideos: (id, callback) => { YouTubeAPI.videos.list({ - part: part, id: id }, callback); }, From 064d3ad5d587723120367195446724c6f60248c1 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 18:40:55 -0500 Subject: [PATCH 18/18] fix test imports --- test/cases/postSkipSegments.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index 4fefef2..0a37aaa 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -7,7 +7,6 @@ var utils = require('../utils.js'); var databases = require('../../src/databases/databases.js'); var db = databases.db; -const getHash = require('../../src/utils/getHash.js'); describe('postSkipSegments', () => { before(() => {