From fee0c92280ab404235325480478453568081d92d Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 18 Apr 2020 13:07:38 -0400 Subject: [PATCH 01/41] Ignore .DS_Store --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e329b30..3362a8d 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,7 @@ test/databases/sponsorTimes.db-wal test/databases/private.db # Config files -config.json \ No newline at end of file +config.json + +# Mac files +.DS_Store \ No newline at end of file From 35fa384a5b25631ce1f8e573b4b4efb93d49266f Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sun, 19 Apr 2020 12:10:39 -0400 Subject: [PATCH 02/41] Added automod check with NB --- package.json | 3 +- src/routes/postSkipSegments.js | 48 ++++++++++++++++++++-------- test/cases/postSkipSegments.js | 58 ++++++++++++++++++++-------------- 3 files changed, 70 insertions(+), 39 deletions(-) diff --git a/package.json b/package.json index e162a6b..b8f62a8 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "http": "0.0.0", "iso8601-duration": "^1.2.0", "uuid": "^3.3.2", - "youtube-api": "^2.0.10" + "youtube-api": "^2.0.10", + "node-fetch": "^2.6.0" }, "devDependencies": { "mocha": "^7.1.1", diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index d8fe2a3..93ac5a6 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -10,6 +10,7 @@ var isoDurations = require('iso8601-duration'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); var getFormattedTime = require('../utils/getFormattedTime.js'); +const fetch = require('node-fetch'); // TODO: might need to be a util //returns true if the user is considered trustworthy @@ -21,8 +22,8 @@ async function isUserTrustworthy(userID) { if (totalSubmissionsRow.totalSubmissions > 5) { //check if they have a high downvote ratio let downvotedSubmissionsRow = db.prepare("SELECT count(*) as downvotedSubmissions FROM sponsorTimes WHERE userID = ? AND (votes < 0 OR shadowHidden > 0)").get(userID); - - return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || + + return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions); } @@ -48,14 +49,14 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { let startTime = parseFloat(segmentInfo.segment[0]); let endTime = parseFloat(segmentInfo.segment[1]); - + request.post(config.discordFirstTimeSubmissionsWebhookURL, { json: { "embeds": [{ "title": data.items[0].snippet.title, "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), "description": "Submission ID: " + UUID + - "\n\nTimestamp: " + + "\n\nTimestamp: " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + "\n\nCategory: " + segmentInfo.category, "color": 10813440, @@ -110,11 +111,30 @@ async function autoModerateSubmission(submission, callback) { if ((submission.endTime - submission.startTime) > (duration/100)*80) { return "Sponsor segment is over 80% of the video."; } else { - return false; + let overlap = false; + nb_predictions = fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID).then().then(); + for (nb_seg in nb_predictions.sponsorSegments){ + let head = 0; + let tail = 0; + // If there's an overlap, find the percentage of overlap. + if (submission.startTime <= nb_seg[1] && nb_seg[0] <= submission.endTime){ + head = Math.max(submission.startTime, nb_seg[0]); + tail = Math.min(submission.endTime, nb_seg[1]); + } + if ((tail-head)/(nb_seg[1]-nb_seg[0]) > 0.65){ + overlap = true; + break; + } + } + if (overlap){ + return "Sponsor segment has passed checks."; + } else{ + return "Sponsor segment doesn't have at least 65% match."; + } } } } - + } else { console.log("Skipped YouTube API"); @@ -195,7 +215,7 @@ module.exports = async function postSkipSegments(req, res) { //check to see if this ip has submitted too many sponsors today let rateLimitCheckRow = privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday]); - + if (rateLimitCheckRow.count >= 10) { //too many sponsors for the same video from the same ip address res.sendStatus(429); @@ -205,7 +225,7 @@ module.exports = async function postSkipSegments(req, res) { //check to see if the user has already submitted sponsors for this video let duplicateCheckRow = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID]); - + if (duplicateCheckRow.count >= 8) { //too many sponsors for the same video from the same user res.sendStatus(429); @@ -233,24 +253,24 @@ module.exports = async function postSkipSegments(req, res) { //this can just be a hash of the data //it's better than generating an actual UUID like what was used before //also better for duplication checking - let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] + + let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] + segmentInfo.segment[1] + segmentInfo.category + userID, 1); try { - db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], + db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); - + //add to private db as well privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); } catch (err) { //a DB change probably occurred res.sendStatus(502); - console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + + console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err); - + return; } - + // Discord notification sendDiscordNotification(userID, videoID, UUID, segmentInfo); } diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index 05c7374..c8f3c39 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -8,8 +8,8 @@ var db = databases.db; describe('postSkipSegments', () => { it('Should be able to submit a single time (Params method)', (done) => { - request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -26,7 +26,7 @@ describe('postSkipSegments', () => { }); it('Should be able to submit a single time (JSON method)', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", @@ -36,7 +36,7 @@ describe('postSkipSegments', () => { category: "sponsor" }] } - }, + }, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -53,7 +53,7 @@ describe('postSkipSegments', () => { }); it('Should be able to submit multiple times (JSON method)', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", @@ -66,7 +66,7 @@ describe('postSkipSegments', () => { category: "intro" }] } - }, + }, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -91,18 +91,28 @@ describe('postSkipSegments', () => { }); it('Should be rejected if over 80% of the video', (done) => { - request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=qqwerty&startTime=30&endTime=1000000&userID=testing", null, + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=qqwerty&startTime=30&endTime=1000000&userID=testing", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode === 403) done(); // pass else done("non 403 status code: " + res.statusCode + " ("+body+")"); }); }); - + + it('Should be rejected if there\'s not at least 65% overlap with NB' , (done) => { + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 403) done(); // pass + else done("non 403 status code: " + res.statusCode + " ("+body+")"); + }); + }); + it('Should be rejected if not a valid videoID', (done) => { - request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing", null, + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode === 403) done(); // pass @@ -111,8 +121,8 @@ describe('postSkipSegments', () => { }); it('Should return 400 for missing params (Params method)', (done) => { - request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", null, + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", null, (err, res, body) => { if (err) done(true); if (res.statusCode === 400) done(); @@ -121,7 +131,7 @@ describe('postSkipSegments', () => { }); it('Should return 400 for missing params (JSON method) 1', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", @@ -133,7 +143,7 @@ describe('postSkipSegments', () => { category: "intro" }] } - }, + }, (err, res, body) => { if (err) done(true); else if (res.statusCode === 400) done(); @@ -141,13 +151,13 @@ describe('postSkipSegments', () => { }); }); it('Should return 400 for missing params (JSON method) 2', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", videoID: "dQw4w9WgXcQ" } - }, + }, (err, res, body) => { if (err) done(true); else if (res.statusCode === 400) done(); @@ -155,7 +165,7 @@ describe('postSkipSegments', () => { }); }); it('Should return 400 for missing params (JSON method) 3', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", @@ -168,7 +178,7 @@ describe('postSkipSegments', () => { category: "intro" }] } - }, + }, (err, res, body) => { if (err) done(true); else if (res.statusCode === 400) done(); @@ -176,7 +186,7 @@ describe('postSkipSegments', () => { }); }); it('Should return 400 for missing params (JSON method) 4', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", @@ -188,7 +198,7 @@ describe('postSkipSegments', () => { category: "intro" }] } - }, + }, (err, res, body) => { if (err) done(true); else if (res.statusCode === 400) done(); @@ -196,17 +206,17 @@ describe('postSkipSegments', () => { }); }); it('Should return 400 for missing params (JSON method) 5', (done) => { - request.post(utils.getbaseURL() + request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes", { json: { userID: "test", videoID: "dQw4w9WgXcQ" } - }, + }, (err, res, body) => { if (err) done(true); else if (res.statusCode === 400) done(); else done(true); }); }); -}); \ No newline at end of file +}); From ba555c0a7cd25afb65e3e93e7979ef70be0a347c Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 25 Apr 2020 17:11:44 -0400 Subject: [PATCH 03/41] Syntax fixes to pass tests. --- src/routes/postSkipSegments.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 93ac5a6..cab50c0 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -112,8 +112,11 @@ async function autoModerateSubmission(submission, callback) { return "Sponsor segment is over 80% of the video."; } else { let overlap = false; - nb_predictions = fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID).then().then(); - for (nb_seg in nb_predictions.sponsorSegments){ + http = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); + if (http.status === 500) return false; + nb_predictions = await http.json(); + + for (const nb_seg of nb_predictions.sponsorSegments){ let head = 0; let tail = 0; // If there's an overlap, find the percentage of overlap. @@ -121,13 +124,13 @@ async function autoModerateSubmission(submission, callback) { head = Math.max(submission.startTime, nb_seg[0]); tail = Math.min(submission.endTime, nb_seg[1]); } - if ((tail-head)/(nb_seg[1]-nb_seg[0]) > 0.65){ + if ((tail-head)/(nb_seg[1]-nb_seg[0]) >= 0.65){ overlap = true; break; } } if (overlap){ - return "Sponsor segment has passed checks."; + return false; } else{ return "Sponsor segment doesn't have at least 65% match."; } From b101831a1bc6c577463a375b5ad48426261e16b1 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 25 Apr 2020 17:15:19 -0400 Subject: [PATCH 04/41] Extended timeout to test cases where necessary --- test/cases/postSkipSegments.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index c8f3c39..e524e01 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -88,7 +88,7 @@ describe('postSkipSegments', () => { done("Status code was " + res.statusCode); } }); - }); + }).timeout(5000); it('Should be rejected if over 80% of the video', (done) => { request.get(utils.getbaseURL() @@ -100,7 +100,7 @@ describe('postSkipSegments', () => { }); }); - it('Should be rejected if there\'s not at least 65% overlap with NB' , (done) => { + it('Should be *rejected* if there\'s not at least 65% overlap with NB' , (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null, (err, res, body) => { @@ -108,7 +108,17 @@ describe('postSkipSegments', () => { else if (res.statusCode === 403) done(); // pass else done("non 403 status code: " + res.statusCode + " ("+body+")"); }); - }); + }).timeout(5000); + + it('Should be *accepted* if there\'s at least 65% overlap with NB' , (done) => { + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=6&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 200) done(); // pass + else done("non 200 status code: " + res.statusCode + " ("+body+")"); + }); + }).timeout(5000); it('Should be rejected if not a valid videoID', (done) => { request.get(utils.getbaseURL() From 1026f652800934730705d0f49e27f8d77393e431 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 25 Apr 2020 17:38:21 -0400 Subject: [PATCH 05/41] Changed interval check --- src/routes/postSkipSegments.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index cab50c0..ba02b6e 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -114,19 +114,16 @@ async function autoModerateSubmission(submission, callback) { let overlap = false; http = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); if (http.status === 500) return false; - nb_predictions = await http.json(); + nb_predictions = await http.json(); for (const nb_seg of nb_predictions.sponsorSegments){ - let head = 0; - let tail = 0; - // If there's an overlap, find the percentage of overlap. - if (submission.startTime <= nb_seg[1] && nb_seg[0] <= submission.endTime){ - head = Math.max(submission.startTime, nb_seg[0]); - tail = Math.min(submission.endTime, nb_seg[1]); - } - if ((tail-head)/(nb_seg[1]-nb_seg[0]) >= 0.65){ - overlap = true; - break; + // The submission needs to be a subset of the widened NB prediction + // and at least 65% of NB's prediction. + if (nb_seg[0]-2 <= submission.startTime && submission.endTime <= nb_seg[1]+2){ + if ((submission.endTime - submission.startTime)/(nb_seg[1]-nb_seg[0] >= 0.65)){ + overlap=true; + break; + } } } if (overlap){ From ee75eea939e3fd0b1e790655bdc0f9ac2a4b822d Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 25 Apr 2020 17:38:21 -0400 Subject: [PATCH 06/41] Changed interval check --- src/routes/postSkipSegments.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index cab50c0..9c9b5a2 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -114,19 +114,16 @@ async function autoModerateSubmission(submission, callback) { let overlap = false; http = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); if (http.status === 500) return false; - nb_predictions = await http.json(); + nb_predictions = await http.json(); for (const nb_seg of nb_predictions.sponsorSegments){ - let head = 0; - let tail = 0; - // If there's an overlap, find the percentage of overlap. - if (submission.startTime <= nb_seg[1] && nb_seg[0] <= submission.endTime){ - head = Math.max(submission.startTime, nb_seg[0]); - tail = Math.min(submission.endTime, nb_seg[1]); - } - if ((tail-head)/(nb_seg[1]-nb_seg[0]) >= 0.65){ - overlap = true; - break; + // The submission needs to be a subset of the widened NB prediction + // and at least 65% of NB's prediction. + if (nb_seg[0]-2 <= submission.startTime && submission.endTime <= nb_seg[1]+2){ + if ((submission.endTime - submission.startTime)/(nb_seg[1]-nb_seg[0]) >= 0.65){ + overlap=true; + break; + } } } if (overlap){ From 9fcd23d63ac3c74649cf95e4aede9c57980356e3 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Thu, 30 Apr 2020 14:16:19 -0700 Subject: [PATCH 07/41] Let NB mod fail on any internal server error --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 9c9b5a2..dc74379 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -113,7 +113,7 @@ async function autoModerateSubmission(submission, callback) { } else { let overlap = false; http = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); - if (http.status === 500) return false; + if (http.status >= 500 && http.status < 600) return false; nb_predictions = await http.json(); for (const nb_seg of nb_predictions.sponsorSegments){ From 004f7149240733f9200b82d7952cf37ac145bd3c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 18:10:27 -0400 Subject: [PATCH 08/41] Use response.ok --- src/routes/postSkipSegments.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index dc74379..2b0114d 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -112,10 +112,10 @@ async function autoModerateSubmission(submission, callback) { return "Sponsor segment is over 80% of the video."; } else { let overlap = false; - http = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); - if (http.status >= 500 && http.status < 600) return false; + let response = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); + if (!response.ok) return false; - nb_predictions = await http.json(); + nb_predictions = await response.json(); for (const nb_seg of nb_predictions.sponsorSegments){ // The submission needs to be a subset of the widened NB prediction // and at least 65% of NB's prediction. From abee01029151654e4a8168731172d9b8ef580f27 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 18:15:17 -0400 Subject: [PATCH 09/41] Renamed variables and added spacing. --- src/routes/postSkipSegments.js | 39 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 2b0114d..e18af9b 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -108,29 +108,30 @@ async function autoModerateSubmission(submission, callback) { duration = isoDurations.toSeconds(isoDurations.parse(duration)); // Reject submission if over 80% of the video - if ((submission.endTime - submission.startTime) > (duration/100)*80) { + if ((submission.endTime - submission.startTime) > (duration/100) * 80) { return "Sponsor segment is over 80% of the video."; } else { - let overlap = false; - let response = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); - if (!response.ok) return false; + let overlap = false; - nb_predictions = await response.json(); - for (const nb_seg of nb_predictions.sponsorSegments){ - // The submission needs to be a subset of the widened NB prediction - // and at least 65% of NB's prediction. - if (nb_seg[0]-2 <= submission.startTime && submission.endTime <= nb_seg[1]+2){ - if ((submission.endTime - submission.startTime)/(nb_seg[1]-nb_seg[0]) >= 0.65){ - overlap=true; - break; - } + let response = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); + if (!response.ok) return false; + + let nbPredictions = await response.json(); + for (const nbSegment of nbPredictions.sponsorSegments) { + // The submission needs to be a subset of the widened NB prediction + // and at least 65% of NB's prediction. + if (nbSegment[0] - 2 <= submission.startTime && submission.endTime <= nbSegment[1] + 2){ + if ((submission.endTime - submission.startTime) / (nbSegment[1]-nbSegment[0]) >= 0.65){ + overlap = true; + break; + } + } + } + if (overlap) { + return false; + } else{ + return "Sponsor segment doesn't have at least 65% match."; } - } - if (overlap){ - return false; - } else{ - return "Sponsor segment doesn't have at least 65% match."; - } } } } From 8290c9e1f483845f74f37bca587bbaaa0d76f608 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 18:39:03 -0400 Subject: [PATCH 10/41] Added new NB overlapping code --- src/routes/postSkipSegments.js | 14 +++++++------- test/cases/postSkipSegments.js | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e18af9b..a41bebd 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -118,15 +118,15 @@ async function autoModerateSubmission(submission, callback) { let nbPredictions = await response.json(); for (const nbSegment of nbPredictions.sponsorSegments) { - // The submission needs to be a subset of the widened NB prediction - // and at least 65% of NB's prediction. - if (nbSegment[0] - 2 <= submission.startTime && submission.endTime <= nbSegment[1] + 2){ - if ((submission.endTime - submission.startTime) / (nbSegment[1]-nbSegment[0]) >= 0.65){ - overlap = true; - break; - } + // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds + // This calculated how off it is + let offAmount = Math.abs(nbSegment[0] - submission.startTime) + Math.abs(nbSegment[1] - submission.endTime); + if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.45 || offAmount <= 7) { + overlap = true; + break; } } + if (overlap) { return false; } else{ diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index e524e01..c6b7eec 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -100,7 +100,7 @@ describe('postSkipSegments', () => { }); }); - it('Should be *rejected* if there\'s not at least 65% overlap with NB' , (done) => { + it("Should be rejected if there's not at least 65% overlap with NB", (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null, (err, res, body) => { @@ -110,7 +110,17 @@ describe('postSkipSegments', () => { }); }).timeout(5000); - it('Should be *accepted* if there\'s at least 65% overlap with NB' , (done) => { + it("Should be accepted if only off by 5s", (done) => { + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=12&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 200) done(); // pass + else done("non 403 status code: " + res.statusCode + " ("+body+")"); + }); + }).timeout(5000); + + it("Should be accepted if there's at least 65% overlap with NB" , (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=6&userID=testing", null, (err, res, body) => { From c2b17ea7a8b828bee337724a79783206f4115b12 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 19:00:26 -0400 Subject: [PATCH 11/41] Made auto-moderate only call API once --- src/routes/postSkipSegments.js | 56 ++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 5882529..3e5b632 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -84,16 +84,15 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { } } -// submission: {videoID, startTime, endTime} // callback: function(reject: "String containing reason the submission was rejected") // returns: string when an error, false otherwise -async function autoModerateSubmission(submission, callback) { +async function autoModerateSubmission(videoID, segments) { // Get the video information from the youtube API if (config.youtubeAPI !== null) { let {err, data} = await new Promise((resolve, reject) => { YouTubeAPI.videos.list({ part: "contentDetails", - id: submission.videoID + id: videoID }, (err, data) => resolve({err, data})); }); @@ -102,39 +101,44 @@ async function autoModerateSubmission(submission, callback) { } else { // Check to see if video exists if (data.pageInfo.totalResults === 0) { - return "No video exists with id " + submission.videoID; + return "No video exists with id " + videoID; } else { let duration = data.items[0].contentDetails.duration; duration = isoDurations.toSeconds(isoDurations.parse(duration)); - if (duration == 0) { // Allow submission if the duration is 0 (bug in youtube api) return false; - } else if ((submission.endTime - submission.startTime) > (duration / 100) * 80) { - // Reject submission if over 80% of the video - return "Sponsor segment is over 80% of the video."; - } else { - let overlap = false; + } - let response = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + submission.videoID); - if (!response.ok) return false; + for (const segment of segments) { + if ((segment.segment[1] - segment.segment[0]) > (duration / 100) * 80) { + // Reject submission if over 80% of the video + return "One of your submitted segments is over 80% of the video."; + } + } + + let overlap = false; - let nbPredictions = await response.json(); - for (const nbSegment of nbPredictions.sponsorSegments) { + let response = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + videoID); + if (!response.ok) return false; + + let nbPredictions = await response.json(); + for (const nbSegment of nbPredictions.sponsorSegments) { + for (const segment of segments) { // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds // This calculated how off it is - let offAmount = Math.abs(nbSegment[0] - submission.startTime) + Math.abs(nbSegment[1] - submission.endTime); + let offAmount = Math.abs(nbSegment[0] - segment.segment[0]) + Math.abs(nbSegment[1] - segment.segment[1]); if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.45 || offAmount <= 7) { overlap = true; break; } } + } - if (overlap) { - return false; - } else{ - return "Sponsor segment doesn't have at least 65% match."; - } + if (overlap) { + return false; + } else{ + return "One of your submitted segments doesn't have at least 65% match."; } } } @@ -200,12 +204,12 @@ module.exports = async function postSkipSegments(req, res) { res.sendStatus(409); return; } - - let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime}); - if (autoModerateResult) { - res.status(403).send("Request rejected by auto moderator: " + autoModerateResult); - return; - } + } + // Auto moderator check + let autoModerateResult = await autoModerateSubmission(videoID, segments); + if (autoModerateResult) { + res.status(403).send("Request rejected by auto moderator: " + autoModerateResult); + return; } try { From a278036f1d5b523f6363143322fca538ed91767b Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 19:13:42 -0400 Subject: [PATCH 12/41] Added neural block mock --- config.json.example | 1 + src/routes/postSkipSegments.js | 4 +++- test.json | 3 ++- test/cases/postSkipSegments.js | 8 ++++---- test/mocks.js | 11 +++++++++++ 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index c548ffc..ea5bfc6 100644 --- a/config.json.example +++ b/config.json.example @@ -8,6 +8,7 @@ "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] + "neuralBlockURL": "https://ai.neuralblock.app", "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For" "db": "./databases/sponsorTimes.db", "privateDB": "./databases/private.db", diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 7a52bc3..d48737e 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -116,10 +116,12 @@ async function autoModerateSubmission(videoID, segments) { return "One of your submitted segments is over 80% of the video."; } } + + let neuralBlockURL = config.neuralBlockURL || "https://ai.neuralblock.app"; let overlap = false; - let response = await fetch("https://ai.neuralblock.app/api/getSponsorSegments?vid=" + videoID); + let response = await fetch(neuralBlockURL + "/api/getSponsorSegments?vid=" + videoID); if (!response.ok) return false; let nbPredictions = await response.json(); diff --git a/test.json b/test.json index 28373a1..86ba466 100644 --- a/test.json +++ b/test.json @@ -6,7 +6,8 @@ "youtubeAPIKey": "", "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", - "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", + "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", + "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock", "behindProxy": true, "db": "./test/databases/sponsorTimes.db", "privateDB": "./test/databases/private.db", diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index b5fae77..c3559be 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -108,7 +108,7 @@ describe('postSkipSegments', () => { else if (res.statusCode === 403) done(); // pass else done("non 403 status code: " + res.statusCode + " ("+body+")"); }); - }).timeout(5000); + }); it("Should be accepted if only off by 5s", (done) => { request.get(utils.getbaseURL() @@ -118,7 +118,7 @@ describe('postSkipSegments', () => { else if (res.statusCode === 200) done(); // pass else done("non 403 status code: " + res.statusCode + " ("+body+")"); }); - }).timeout(5000); + }); it("Should be accepted if there's at least 65% overlap with NB" , (done) => { request.get(utils.getbaseURL() @@ -128,7 +128,7 @@ describe('postSkipSegments', () => { else if (res.statusCode === 200) done(); // pass else done("non 200 status code: " + res.statusCode + " ("+body+")"); }); - }).timeout(5000); + }); it('Should be allowed if youtube thinks duration is 0', (done) => { request.get(utils.getbaseURL() @@ -138,7 +138,7 @@ describe('postSkipSegments', () => { else if (res.statusCode === 200) done(); // pass else done("non 200 status code: " + res.statusCode + " ("+body+")"); }); - }).timeout(5000); + }); it('Should be rejected if not a valid videoID', (done) => { request.get(utils.getbaseURL() diff --git a/test/mocks.js b/test/mocks.js index b58a47f..4e2ad92 100644 --- a/test/mocks.js +++ b/test/mocks.js @@ -15,6 +15,17 @@ app.post('/CompletelyIncorrectReportWebhook', (req, res) => { res.sendStatus(200); }); +app.get('/NeuralBlock/api/getSponsorSegments', (req, res) => { + if (req.query.vid === "LevkAjUE6d4") { + res.json({ + sponsorSegments: [[0.47,7.549],[264.023,317.293]] + }); + return; + } + + res.sendStatus(500); +}); + module.exports = function createMockServer(callback) { return app.listen(config.mockPort, callback); } \ No newline at end of file From eb936d579ce8adcc21cf2d8654f33d39bdb44f75 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 19:17:52 -0400 Subject: [PATCH 13/41] Don't moderate if VIP. --- src/routes/postSkipSegments.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index d48737e..f571b8c 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -181,6 +181,9 @@ module.exports = async function postSkipSegments(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(getIP(req) + config.globalSalt); + //check if this user is on the vip list + let isVIP = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(userID).userCount > 0; + // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { @@ -207,17 +210,17 @@ module.exports = async function postSkipSegments(req, res) { return; } } + // Auto moderator check - let autoModerateResult = await autoModerateSubmission(videoID, segments); - if (autoModerateResult) { - res.status(403).send("Request rejected by auto moderator: " + autoModerateResult); - return; + if (!isVIP) { + let autoModerateResult = await autoModerateSubmission(videoID, segments); + if (autoModerateResult) { + res.status(403).send("Request rejected by auto moderator: " + autoModerateResult); + return; + } } try { - //check if this user is on the vip list - let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(userID); - //get current time let timeSubmitted = Date.now(); @@ -254,7 +257,7 @@ module.exports = async function postSkipSegments(req, res) { } let startingVotes = 0; - if (vipRow.userCount > 0) { + if (isVIP) { //this user is a vip, start them at a higher approval rating startingVotes = 10; } From 5e252ed4af019ae2a3fe1e9cc7a54764decac3fb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 21:19:37 -0400 Subject: [PATCH 14/41] Added extra message to auto moderator --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index f571b8c..562e009 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -215,7 +215,7 @@ module.exports = async function postSkipSegments(req, res) { if (!isVIP) { let autoModerateResult = await autoModerateSubmission(videoID, segments); if (autoModerateResult) { - res.status(403).send("Request rejected by auto moderator: " + autoModerateResult); + res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); return; } } From 684000074e4383f774aae7616d97b2f9f65357a7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 23:40:04 -0400 Subject: [PATCH 15/41] Fix overlap percentage wrong Co-authored-by: Andrew Lee --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 562e009..639f608 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -130,7 +130,7 @@ async function autoModerateSubmission(videoID, segments) { // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds // This calculated how off it is let offAmount = Math.abs(nbSegment[0] - segment.segment[0]) + Math.abs(nbSegment[1] - segment.segment[1]); - if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.45 || offAmount <= 7) { + if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.35 || offAmount <= 7) { overlap = true; break; } From 318d816a37cf2f41cde2fd1369c6e9914dc4eccf Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 23:41:38 -0400 Subject: [PATCH 16/41] Change test case to check 7s --- test/cases/postSkipSegments.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index c3559be..ab743c7 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -110,9 +110,9 @@ describe('postSkipSegments', () => { }); }); - it("Should be accepted if only off by 5s", (done) => { + it("Should be accepted if only off by 7s", (done) => { request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=12&userID=testing", null, + + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=14.079&userID=testing", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode === 200) done(); // pass From f5eb522e39478c7d3ce0443c388b6362da4ffc56 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 3 May 2020 11:46:16 -0400 Subject: [PATCH 17/41] Removed default NB server --- src/routes/postSkipSegments.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 639f608..919a15a 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -117,7 +117,8 @@ async function autoModerateSubmission(videoID, segments) { } } - let neuralBlockURL = config.neuralBlockURL || "https://ai.neuralblock.app"; + let neuralBlockURL = config.neuralBlockURL; + if (!neuralBlockURL) return false; let overlap = false; From bf2646978a1eed187ef12cdef1e45e5f567fa912 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 3 May 2020 11:49:04 -0400 Subject: [PATCH 18/41] Removed NB URL from example config --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index ea5bfc6..104893f 100644 --- a/config.json.example +++ b/config.json.example @@ -8,7 +8,7 @@ "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] - "neuralBlockURL": "https://ai.neuralblock.app", + "neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For" "db": "./databases/sponsorTimes.db", "privateDB": "./databases/private.db", From 8e150c6862b2123f18b30c4eac03b49609df3962 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 11 May 2020 19:00:33 -0400 Subject: [PATCH 19/41] Don't check NB for non-sponsors and properly reject if any submission fails the criteria for multiple segment submissions --- src/routes/postSkipSegments.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 919a15a..f883c76 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -120,22 +120,29 @@ async function autoModerateSubmission(videoID, segments) { let neuralBlockURL = config.neuralBlockURL; if (!neuralBlockURL) return false; - let overlap = false; + let overlap = true; let response = await fetch(neuralBlockURL + "/api/getSponsorSegments?vid=" + videoID); if (!response.ok) return false; let nbPredictions = await response.json(); - for (const nbSegment of nbPredictions.sponsorSegments) { - for (const segment of segments) { + for (const segment of segments) { + if (segment.category !== "sponsor") continue; + + let thisSegmentOverlaps = false; + for (const nbSegment of nbPredictions.sponsorSegments) { // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds // This calculated how off it is let offAmount = Math.abs(nbSegment[0] - segment.segment[0]) + Math.abs(nbSegment[1] - segment.segment[1]); if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.35 || offAmount <= 7) { - overlap = true; - break; + thisSegmentOverlaps = true; } } + + if (!thisSegmentOverlaps){ + overlap = false; + break; + } } if (overlap) { From 373548d396e8a49c8ff80dc5cc8a542c0b8d27bb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 29 Aug 2020 17:39:08 -0400 Subject: [PATCH 20/41] Remove unused variable --- src/routes/postSkipSegments.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 61a8ac2..e5b2f96 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -238,9 +238,6 @@ module.exports = async function postSkipSegments(req, res) { } try { - //check if this user is on the vip list - let vipRow = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]); - //get current time let timeSubmitted = Date.now(); From c6d28d7fc527e1e564bfa94ba6d079a69b4c8ddb Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 29 Aug 2020 17:45:59 -0400 Subject: [PATCH 21/41] Apply suggestions from code review Fix syntax changes caused by bad merge Co-authored-by: Ajay Ramachandran --- src/routes/postSkipSegments.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e5b2f96..0b1f561 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -79,7 +79,7 @@ async function autoModerateSubmission(submission, callback) { let {err, data} = await new Promise((resolve, reject) => { YouTubeAPI.videos.list({ part: "contentDetails", - id: videoID + id: submission.videoID }, (err, data) => resolve({err, data})); }); @@ -88,7 +88,7 @@ async function autoModerateSubmission(submission, callback) { } else { // Check to see if video exists if (data.pageInfo.totalResults === 0) { - return "No video exists with id " + videoID; + return "No video exists with id " + submission.videoID; } else { let duration = data.items[0].contentDetails.duration; duration = isoDurations.toSeconds(isoDurations.parse(duration)); @@ -193,7 +193,7 @@ module.exports = async function postSkipSegments(req, res) { let hashedIP = getHash(getIP(req) + config.globalSalt); //check if this user is on the vip list - let isVIP = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(userID).userCount > 0; + let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0; // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { From 8fc1fe7d8e6dbc3d151ef93877a13c477c3eff9e Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 29 Aug 2020 17:48:39 -0400 Subject: [PATCH 22/41] Re-add node-fetch package --- src/routes/postSkipSegments.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 61a8ac2..d3ee529 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -13,6 +13,8 @@ var getIP = require('../utils/getIP.js'); var getFormattedTime = require('../utils/getFormattedTime.js'); var isUserTrustworthy = require('../utils/isUserTrustworthy.js') +const fetch = require('node-fetch'); + function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { //check if they are a first time user //if so, send a notification to discord From d965308346427b90cce62d04ff260957aadf9e4a Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 29 Aug 2020 18:23:04 -0400 Subject: [PATCH 23/41] Merge fixes to nb automod and api change --- src/routes/postSkipSegments.js | 56 +++++++++++----------------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 61cd81d..9d83586 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -97,48 +97,26 @@ async function autoModerateSubmission(submission, callback) { if (duration == 0) { // Allow submission if the duration is 0 (bug in youtube api) return false; - } + } else if ((submission.endTime - submission.startTime) > (duration/100)*80) { + // Reject submission if over 80% of the video + return "Sponsor segment is over 80% of the video."; + } else { + // Check NeuralBlock + let neuralBlockURL = config.neuralBlockURL; + if (!neuralBlockURL) return false; - for (const segment of segments) { - if ((segment.segment[1] - segment.segment[0]) > (duration / 100) * 80) { - // Reject submission if over 80% of the video - return "One of your submitted segments is over 80% of the video."; + let response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + + "&segments=" + submission.startTime + "," + submission.endTime); + if (!response.ok) return false; + + let nbPredictions = await response.json(); + if (nbPredictions.probabilities[0] >= 0.70){ + return false; + } else { + // Send to Discord + return "Automoderator has rejected the segment. Sending to Discord for manual review."; } } - - let neuralBlockURL = config.neuralBlockURL; - if (!neuralBlockURL) return false; - - let overlap = true; - - let response = await fetch(neuralBlockURL + "/api/getSponsorSegments?vid=" + videoID); - if (!response.ok) return false; - - let nbPredictions = await response.json(); - for (const segment of segments) { - if (segment.category !== "sponsor") continue; - - let thisSegmentOverlaps = false; - for (const nbSegment of nbPredictions.sponsorSegments) { - // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds - // This calculated how off it is - let offAmount = Math.abs(nbSegment[0] - segment.segment[0]) + Math.abs(nbSegment[1] - segment.segment[1]); - if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.35 || offAmount <= 7) { - thisSegmentOverlaps = true; - } - } - - if (!thisSegmentOverlaps){ - overlap = false; - break; - } - } - - if (overlap) { - return false; - } else{ - return "One of your submitted segments doesn't have at least 65% match."; - } } } From 0872c727d5746425142f34edd77b93908f3c4ddb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 29 Aug 2020 18:58:52 -0400 Subject: [PATCH 24/41] Fix auto moderate merge issues --- src/routes/postSkipSegments.js | 79 ++++++++++++++++------------------ 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 0b1f561..f86a2da 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -1,17 +1,19 @@ -var config = require('../config.js'); +const config = require('../config.js'); -var databases = require('../databases/databases.js'); -var db = databases.db; -var privateDB = databases.privateDB; -var YouTubeAPI = require('../utils/youtubeAPI.js'); -var logger = require('../utils/logger.js'); -var request = require('request'); -var isoDurations = require('iso8601-duration'); +const databases = require('../databases/databases.js'); +const db = databases.db; +const privateDB = databases.privateDB; +const YouTubeAPI = require('../utils/youtubeAPI.js'); +const logger = require('../utils/logger.js'); +const request = require('request'); +const isoDurations = require('iso8601-duration'); +const fetch = require('node-fetch'); + +const getHash = require('../utils/getHash.js'); +const getIP = require('../utils/getIP.js'); +const getFormattedTime = require('../utils/getFormattedTime.js'); +const isUserTrustworthy = require('../utils/isUserTrustworthy.js') -var getHash = require('../utils/getHash.js'); -var getIP = require('../utils/getIP.js'); -var getFormattedTime = require('../utils/getFormattedTime.js'); -var isUserTrustworthy = require('../utils/isUserTrustworthy.js') function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { //check if they are a first time user @@ -73,7 +75,7 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. -async function autoModerateSubmission(submission, callback) { +async function autoModerateSubmission(submission) { // Get the video information from the youtube API if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { @@ -95,13 +97,9 @@ async function autoModerateSubmission(submission, callback) { if (duration == 0) { // Allow submission if the duration is 0 (bug in youtube api) return false; - } - - for (const segment of segments) { - if ((segment.segment[1] - segment.segment[0]) > (duration / 100) * 80) { - // Reject submission if over 80% of the video - return "One of your submitted segments is over 80% of the video."; - } + } else if ((submission.endTime - submission.startTime) > (duration / 100) * 80) { + // Reject submission if over 80% of the video + return "One of your submitted segments is over 80% of the video."; } let neuralBlockURL = config.neuralBlockURL; @@ -109,27 +107,24 @@ async function autoModerateSubmission(submission, callback) { let overlap = true; - let response = await fetch(neuralBlockURL + "/api/getSponsorSegments?vid=" + videoID); + let response = await fetch(neuralBlockURL + "/api/getSponsorSegments?vid=" + submission.videoID); if (!response.ok) return false; let nbPredictions = await response.json(); - for (const segment of segments) { - if (segment.category !== "sponsor") continue; + if (submission.category !== "sponsor") return false; - let thisSegmentOverlaps = false; - for (const nbSegment of nbPredictions.sponsorSegments) { - // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds - // This calculated how off it is - let offAmount = Math.abs(nbSegment[0] - segment.segment[0]) + Math.abs(nbSegment[1] - segment.segment[1]); - if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.35 || offAmount <= 7) { - thisSegmentOverlaps = true; - } + let thisSegmentOverlaps = false; + for (const nbSegment of nbPredictions.sponsorSegments) { + // The submission needs to be similar to the NB prediction by 65% or off by less than 7 seconds + // This calculated how off it is + let offAmount = Math.abs(nbSegment[0] - submission.startTime) + Math.abs(nbSegment[1] - submission.endTime); + if (offAmount / (nbSegment[1] - nbSegment[0]) <= 0.35 || offAmount <= 7) { + thisSegmentOverlaps = true; } + } - if (!thisSegmentOverlaps){ - overlap = false; - break; - } + if (!thisSegmentOverlaps){ + overlap = false; } if (overlap) { @@ -226,14 +221,14 @@ module.exports = async function postSkipSegments(req, res) { res.sendStatus(409); return; } - } - // Auto moderator check - if (!isVIP) { - let autoModerateResult = await autoModerateSubmission(videoID, segments); - if (autoModerateResult) { - res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); - return; + // Auto moderator check + if (!isVIP) { + let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime, category: segments[i].category}); + if (autoModerateResult) { + res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); + return; + } } } From 72ce7fcb568ce779111f781005c8f67545dc8a75 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 29 Aug 2020 20:41:07 -0400 Subject: [PATCH 25/41] Update test cases with new automod api --- test/cases/postSkipSegments.js | 22 +--------------------- test/mocks.js | 12 +++++++++++- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index b31fedb..09bc164 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -120,7 +120,7 @@ describe('postSkipSegments', () => { }); }); - it("Should be rejected if there's not at least 65% overlap with NB", (done) => { + it("Should be rejected if NB's predicted probability is <70%.", (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null, (err, res, body) => { @@ -130,26 +130,6 @@ describe('postSkipSegments', () => { }); }); - it("Should be accepted if only off by 7s", (done) => { - request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=14.079&userID=testing", null, - (err, res, body) => { - if (err) done("Couldn't call endpoint"); - else if (res.statusCode === 200) done(); // pass - else done("non 403 status code: " + res.statusCode + " ("+body+")"); - }); - }); - - it("Should be accepted if there's at least 65% overlap with NB" , (done) => { - request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=0&endTime=6&userID=testing", null, - (err, res, body) => { - if (err) done("Couldn't call endpoint"); - else if (res.statusCode === 200) done(); // pass - else done("non 200 status code: " + res.statusCode + " ("+body+")"); - }); - }); - it('Should be allowed if youtube thinks duration is 0', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null, diff --git a/test/mocks.js b/test/mocks.js index 4e2ad92..0fecf75 100644 --- a/test/mocks.js +++ b/test/mocks.js @@ -15,6 +15,17 @@ app.post('/CompletelyIncorrectReportWebhook', (req, res) => { res.sendStatus(200); }); +app.get('/NeuralBlock/api/checkSponsorSegments', (req, res) => { + if (req.query.vid === "LevkAjUE6d4") { + res.json({ + probabilities: [0.69] + }); + return; + } + res.sendStatus(500); +}); + +//getSponsorSegments is no longer being used for automod app.get('/NeuralBlock/api/getSponsorSegments', (req, res) => { if (req.query.vid === "LevkAjUE6d4") { res.json({ @@ -22,7 +33,6 @@ app.get('/NeuralBlock/api/getSponsorSegments', (req, res) => { }); return; } - res.sendStatus(500); }); From e8116de4bb7fb81accb8a8e27a95baa49e21b44f Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 5 Sep 2020 16:56:23 -0400 Subject: [PATCH 26/41] Update configs for NB discord URLs --- config.json.example | 1 + test.json | 5 +++-- test/mocks.js | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config.json.example b/config.json.example index 124a790..634cc0c 100644 --- a/config.json.example +++ b/config.json.example @@ -9,6 +9,7 @@ "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] "neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app + "discordNeuralBlockRejectWebhookURL": null, //URL from discord if you would like notifications when NeuralBlock rejects a submission [optional] "proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash) "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For" "db": "./databases/sponsorTimes.db", diff --git a/test.json b/test.json index 6950605..841322e 100644 --- a/test.json +++ b/test.json @@ -4,9 +4,10 @@ "globalSalt": "testSalt", "adminUserID": "testUserId", "youtubeAPIKey": "", - "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", - "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", + "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", + "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", + "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook", "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock", "behindProxy": true, "db": "./test/databases/sponsorTimes.db", diff --git a/test/mocks.js b/test/mocks.js index ad8bceb..ddf7f19 100644 --- a/test/mocks.js +++ b/test/mocks.js @@ -16,6 +16,9 @@ app.post('/CompletelyIncorrectReportWebhook', (req, res) => { }); // Testing NeuralBlock +app.post('/NeuralBlockRejectWebhook', (req, res) => { + res.sendStatus(200); +}); app.get('/NeuralBlock/api/checkSponsorSegments', (req, res) => { if (req.query.vid === "LevkAjUE6d4") { @@ -45,4 +48,4 @@ app.post('/CustomWebhook', (req, res) => { module.exports = function createMockServer(callback) { return app.listen(config.mockPort, callback); -} \ No newline at end of file +} From 66c31090371f0cb122905ed327c1a2f1b559ee82 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 5 Sep 2020 18:07:43 -0400 Subject: [PATCH 27/41] Add Discord webhook for videos that fail NB check --- src/routes/postSkipSegments.js | 88 +++++++++++++++++++++++++++++++--- test/cases/postSkipSegments.js | 16 +++---- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e1a5953..377d81f 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -57,11 +57,11 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { err && logger.error(err); return; } - + let startTime = parseFloat(segmentInfo.segment[0]); let endTime = parseFloat(segmentInfo.segment[1]); sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {submissionStart: startTime, submissionEnd: endTime}, segmentInfo); - + // If it is a first time submission // Then send a notification to discord if (config.discordFirstTimeSubmissionsWebhookURL === null) return; @@ -71,7 +71,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { "title": data.items[0].snippet.title, "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), "description": "Submission ID: " + UUID + - "\n\nTimestamp: " + + "\n\nTimestamp: " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + "\n\nCategory: " + segmentInfo.category, "color": 10813440, @@ -98,6 +98,71 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { } } +function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, probability) { + if (config.youtubeAPIKey !== null) { + //let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " + + // "(select count(1) from sponsorTimes where userID = s.userID) count, " + + // "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " + + // "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.userId=?", + //[userID]); + let submissionCount = db.prepare('get', "SELECT COUNT(*) count FROM sponsorTimes WHERE userID=?", [userID]); + let disregardedCount = db.prepare('get', "SELECT COUNT(*) disregarded FROM sponsorTimes WHERE userID=? and votes <= -2", [userID]); + //let uName = db.prepare('get', "SELECT userName FROM userNames WHERE userID=?", [userID]); + YouTubeAPI.videos.list({ + part: "snippet", + id: videoID + }, function (err, data) { + if (err || data.items.length === 0) { + err && logger.error(err); + return; + } + //sendWebhookNotification(userID, videoID, UUID, submissionInfoRow.submissionCount, data, {submissionStart: startTime, submissionEnd: endTime}, segmentInfo); + let submittedBy = userID;//""; + // If there's no userName then just show the userID + // if (uName.userName == userID){ + // submittedBy = userID; + // } else { // else show both + // submittedBy = uName.userName + "\n " + userID; + // } + + // Send discord message + if (config.discordNeuralBlockRejectWebhookURL === null) return; + request.post(config.discordNeuralBlockRejectWebhookURL, { + json: { + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), + "description": "**Submission ID:** " + UUID + + "\n**Timestamp:** " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + + "\n**Predicted Probability:** " + probability + + "\n**Category:** " + category + + "\n**Submitted by:** "+ submittedBy + + "\n**Total User Submissions:** "+submissionCount.count + + "\n**Ignored User Submissions:** "+disregardedCount.disregarded, + "color": 10813440, + "author": { + "name": userID + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + } + }] + } + }, (err, res) => { + if (err) { + logger.error("Failed to send NeuralBlock Discord hook."); + logger.error(JSON.stringify(err)); + logger.error("\n"); + } else if (res && res.statusCode >= 400) { + logger.error("Error sending NeuralBlock Discord hook"); + logger.error(JSON.stringify(res)); + logger.error("\n"); + } + }); + }); + } +} + // callback: function(reject: "String containing reason the submission was rejected") // returns: string when an error, false otherwise @@ -142,7 +207,10 @@ async function autoModerateSubmission(submission) { if (nbPredictions.probabilities[0] >= 0.70){ return false; } else { + let UUID = getHash("v2-categories" + submission.videoID + submission.startTime + + submission.endTime + submission.category + submission.userID, 1); // Send to Discord + sendWebhooksNB(submission.userID, submission.videoID, UUID, submission.startTime, submission.endTime, submission.category, nbPredictions.probabilities[0]); return "NB disagreement."; } } @@ -204,6 +272,8 @@ module.exports = async function postSkipSegments(req, res) { //check if this user is on the vip list let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0; + let decreaseVotes = 0; + // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { @@ -238,8 +308,12 @@ module.exports = async function postSkipSegments(req, res) { // Auto moderator check if (!isVIP) { - let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime, category: segments[i].category}); - if (autoModerateResult) { + let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime, category: segments[i].category, segmentInfo: segments[i]}); + if (autoModerateResult == "NB disagreement."){ + // If NB automod rejects, the submission will start with -2 votes + decreaseVotes = -2; + } else if (autoModerateResult) { + //Normal automod behavior res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); return; } @@ -291,7 +365,7 @@ module.exports = async function postSkipSegments(req, res) { shadowBanned = 1; } - let startingVotes = 0; + let startingVotes = 0 + decreaseVotes; if (isVIP) { //this user is a vip, start them at a higher approval rating startingVotes = 10000; @@ -323,7 +397,7 @@ module.exports = async function postSkipSegments(req, res) { // Discord notification sendWebhooks(userID, videoID, UUID, segmentInfo); - + UUIDs.push(UUID); } } catch (err) { diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index 09bc164..a2e2c97 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -91,8 +91,8 @@ describe('postSkipSegments', () => { }).timeout(5000); it('Should be accepted if a non-sponsor is less than 1 second', (done) => { - request.post(utils.getbaseURL() - + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=intro", null, + request.post(utils.getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=intro", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode === 200) done(); // pass @@ -101,8 +101,8 @@ describe('postSkipSegments', () => { }); it('Should be rejected if a sponsor is less than 1 second', (done) => { - request.post(utils.getbaseURL() - + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", null, + request.post(utils.getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode === 400) done(); // pass @@ -125,14 +125,14 @@ describe('postSkipSegments', () => { + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); - else if (res.statusCode === 403) done(); // pass - else done("non 403 status code: " + res.statusCode + " ("+body+")"); + else if (res.statusCode === 200) done(); // pass + else done("non 200 status code: " + res.statusCode + " ("+body+")"); }); }); it('Should be allowed if youtube thinks duration is 0', (done) => { - request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null, + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode === 200) done(); // pass From fbd8113217db476b86cd20fbe03eae3572580d92 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sat, 5 Sep 2020 18:26:20 -0400 Subject: [PATCH 28/41] Bug fix automod params --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 377d81f..f8ea458 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -308,7 +308,7 @@ module.exports = async function postSkipSegments(req, res) { // Auto moderator check if (!isVIP) { - let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime, category: segments[i].category, segmentInfo: segments[i]}); + let autoModerateResult = await autoModerateSubmission({userID, videoID, startTime, endTime, category: segments[i].category}); if (autoModerateResult == "NB disagreement."){ // If NB automod rejects, the submission will start with -2 votes decreaseVotes = -2; From 6e8d644574fbbc849fd4c605b633b611da30542c Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sun, 6 Sep 2020 21:08:51 -0400 Subject: [PATCH 29/41] Removed unnecessary YT API call from NB Discord webhook --- src/routes/postSkipSegments.js | 111 ++++++++++++++------------------- 1 file changed, 48 insertions(+), 63 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index f8ea458..55857be 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -98,69 +98,54 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { } } -function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, probability) { - if (config.youtubeAPIKey !== null) { - //let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " + - // "(select count(1) from sponsorTimes where userID = s.userID) count, " + - // "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " + - // "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.userId=?", - //[userID]); - let submissionCount = db.prepare('get', "SELECT COUNT(*) count FROM sponsorTimes WHERE userID=?", [userID]); - let disregardedCount = db.prepare('get', "SELECT COUNT(*) disregarded FROM sponsorTimes WHERE userID=? and votes <= -2", [userID]); - //let uName = db.prepare('get', "SELECT userName FROM userNames WHERE userID=?", [userID]); - YouTubeAPI.videos.list({ - part: "snippet", - id: videoID - }, function (err, data) { - if (err || data.items.length === 0) { - err && logger.error(err); - return; - } - //sendWebhookNotification(userID, videoID, UUID, submissionInfoRow.submissionCount, data, {submissionStart: startTime, submissionEnd: endTime}, segmentInfo); - let submittedBy = userID;//""; - // If there's no userName then just show the userID - // if (uName.userName == userID){ - // submittedBy = userID; - // } else { // else show both - // submittedBy = uName.userName + "\n " + userID; - // } +function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, probability, ytData) { + //let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " + + // "(select count(1) from sponsorTimes where userID = s.userID) count, " + + // "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " + + // "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.userId=?", + //[userID]); + let submissionCount = db.prepare('get', "SELECT COUNT(*) count FROM sponsorTimes WHERE userID=?", [userID]); + let disregardedCount = db.prepare('get', "SELECT COUNT(*) disregarded FROM sponsorTimes WHERE userID=? and votes <= -2", [userID]); + //let uName = db.prepare('get', "SELECT userName FROM userNames WHERE userID=?", [userID]); + let submittedBy = userID;//""; + // If there's no userName then just show the userID + // if (uName.userName == userID){ + // submittedBy = userID; + // } else { // else show both + // submittedBy = uName.userName + "\n " + userID; + // } - // Send discord message - if (config.discordNeuralBlockRejectWebhookURL === null) return; - request.post(config.discordNeuralBlockRejectWebhookURL, { - json: { - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), - "description": "**Submission ID:** " + UUID + - "\n**Timestamp:** " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + - "\n**Predicted Probability:** " + probability + - "\n**Category:** " + category + - "\n**Submitted by:** "+ submittedBy + - "\n**Total User Submissions:** "+submissionCount.count + - "\n**Ignored User Submissions:** "+disregardedCount.disregarded, - "color": 10813440, - "author": { - "name": userID - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - } - }] + // Send discord message + if (config.discordNeuralBlockRejectWebhookURL === null) return; + request.post(config.discordNeuralBlockRejectWebhookURL, { + json: { + "embeds": [{ + "title": ytData.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), + "description": "**Submission ID:** " + UUID + + "\n**Timestamp:** " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + + "\n**Predicted Probability:** " + probability + + "\n**Category:** " + category + + "\n**Submitted by:** "+ submittedBy + + "\n**Total User Submissions:** "+submissionCount.count + + "\n**Ignored User Submissions:** "+disregardedCount.disregarded, + "color": 10813440, + "thumbnail": { + "url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "", } - }, (err, res) => { - if (err) { - logger.error("Failed to send NeuralBlock Discord hook."); - logger.error(JSON.stringify(err)); - logger.error("\n"); - } else if (res && res.statusCode >= 400) { - logger.error("Error sending NeuralBlock Discord hook"); - logger.error(JSON.stringify(res)); - logger.error("\n"); - } - }); - }); - } + }] + } + }, (err, res) => { + if (err) { + logger.error("Failed to send NeuralBlock Discord hook."); + logger.error(JSON.stringify(err)); + logger.error("\n"); + } else if (res && res.statusCode >= 400) { + logger.error("Error sending NeuralBlock Discord hook"); + logger.error(JSON.stringify(res)); + logger.error("\n"); + } + }); } // callback: function(reject: "String containing reason the submission was rejected") @@ -174,7 +159,7 @@ async function autoModerateSubmission(submission) { if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { YouTubeAPI.videos.list({ - part: "contentDetails", + part: "contentDetails,snippet", id: submission.videoID }, (err, data) => resolve({err, data})); }); @@ -210,7 +195,7 @@ async function autoModerateSubmission(submission) { let UUID = getHash("v2-categories" + submission.videoID + submission.startTime + submission.endTime + submission.category + submission.userID, 1); // Send to Discord - sendWebhooksNB(submission.userID, submission.videoID, UUID, submission.startTime, submission.endTime, submission.category, nbPredictions.probabilities[0]); + sendWebhooksNB(submission.userID, submission.videoID, UUID, submission.startTime, submission.endTime, submission.category, nbPredictions.probabilities[0], data); return "NB disagreement."; } } From 8b132c37a040b50de14cc90a79127e094480b816 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Sun, 6 Sep 2020 21:10:28 -0400 Subject: [PATCH 30/41] Disable NB automod vote decrease --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 55857be..e3f94e9 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -296,7 +296,7 @@ module.exports = async function postSkipSegments(req, res) { let autoModerateResult = await autoModerateSubmission({userID, videoID, startTime, endTime, category: segments[i].category}); if (autoModerateResult == "NB disagreement."){ // If NB automod rejects, the submission will start with -2 votes - decreaseVotes = -2; + //decreaseVotes = -2; //Disable for now } else if (autoModerateResult) { //Normal automod behavior res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); From cb386b7f8d07f0abcecac2ee4517cc7ea3b803e8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 7 Sep 2020 10:23:51 -0400 Subject: [PATCH 31/41] Don't fail when YouTube API fails --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 799711f..91ae71e 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -115,7 +115,7 @@ async function autoModerateSubmission(submission, callback) { }); if (err) { - return "Couldn't get video information."; + return false; } else { // Check to see if video exists if (data.pageInfo.totalResults === 0) { From ce3680616933d18acfaf824a5d93383ab85ac75f Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Mon, 7 Sep 2020 12:29:13 -0400 Subject: [PATCH 32/41] Added usernames to NB Discord webhooks --- src/routes/postSkipSegments.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e3f94e9..b6e10cc 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -106,14 +106,20 @@ function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, pro //[userID]); let submissionCount = db.prepare('get', "SELECT COUNT(*) count FROM sponsorTimes WHERE userID=?", [userID]); let disregardedCount = db.prepare('get', "SELECT COUNT(*) disregarded FROM sponsorTimes WHERE userID=? and votes <= -2", [userID]); - //let uName = db.prepare('get', "SELECT userName FROM userNames WHERE userID=?", [userID]); - let submittedBy = userID;//""; - // If there's no userName then just show the userID - // if (uName.userName == userID){ - // submittedBy = userID; - // } else { // else show both - // submittedBy = uName.userName + "\n " + userID; - // } + let uName = db.prepare('get', "SELECT userName FROM userNames WHERE userID=?", [userID]); + + let submittedBy = ""; + try { + // If a userName was created then show both + if (uName.userName !== userID){ + submittedBy = uName.userName + "\n " + userID; + } else { + submittedBy = userID; + } + } catch { + //Catch in case User is not in userNames table + submittedBy = userID; + } // Send discord message if (config.discordNeuralBlockRejectWebhookURL === null) return; From 137a6dc771c2494b63a7dbd21704e83ee67fa043 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Mon, 7 Sep 2020 13:16:03 -0400 Subject: [PATCH 33/41] Make db queries one request --- src/routes/postSkipSegments.js | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index b6e10cc..052d730 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -99,25 +99,17 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { } function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, probability, ytData) { - //let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " + - // "(select count(1) from sponsorTimes where userID = s.userID) count, " + - // "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " + - // "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.userId=?", - //[userID]); - let submissionCount = db.prepare('get', "SELECT COUNT(*) count FROM sponsorTimes WHERE userID=?", [userID]); - let disregardedCount = db.prepare('get', "SELECT COUNT(*) disregarded FROM sponsorTimes WHERE userID=? and votes <= -2", [userID]); - let uName = db.prepare('get', "SELECT userName FROM userNames WHERE userID=?", [userID]); + let submissionInfoRow = db.prepare('get', "SELECT " + + "(select count(1) from sponsorTimes where userID = ?) count, " + + "(select count(1) from sponsorTimes where userID = ? and votes <= -2) disregarded, " + + "coalesce((select userName FROM userNames WHERE userID = ?), ?) userName", + [userID, userID, userID, userID]); let submittedBy = ""; - try { - // If a userName was created then show both - if (uName.userName !== userID){ - submittedBy = uName.userName + "\n " + userID; - } else { - submittedBy = userID; - } - } catch { - //Catch in case User is not in userNames table + // If a userName was created then show both + if (submissionInfoRow.userName !== userID){ + submittedBy = submissionInfoRow.userName + "\n " + userID; + } else { submittedBy = userID; } @@ -133,8 +125,8 @@ function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, pro "\n**Predicted Probability:** " + probability + "\n**Category:** " + category + "\n**Submitted by:** "+ submittedBy + - "\n**Total User Submissions:** "+submissionCount.count + - "\n**Ignored User Submissions:** "+disregardedCount.disregarded, + "\n**Total User Submissions:** "+submissionInfoRow.count + + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded, "color": 10813440, "thumbnail": { "url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "", From dec0971c14e3308d786626a8355eb1b9e5a7e4b1 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Wed, 9 Sep 2020 16:40:13 +0200 Subject: [PATCH 34/41] Adding check to only allow a list of categories --- config.json.example | 3 ++- src/routes/postSkipSegments.js | 5 +++++ src/routes/voteOnSponsorTime.js | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index befb39e..337c317 100644 --- a/config.json.example +++ b/config.json.example @@ -19,5 +19,6 @@ "privateDBSchema": "./databases/_private.db.sql", "mode": "development", "readOnly": false, - "webhooks": [] + "webhooks": [], + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected } diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 91ae71e..40a1b5f 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -194,6 +194,11 @@ module.exports = async function postSkipSegments(req, res) { res.sendStatus(400); return; } + + if (!config.categoryList.includes(segments[i].category)) { + res.status("400").send("Category doesn't exist."); + return; + } let startTime = parseFloat(segments[i].segment[0]); let endTime = parseFloat(segments[i].segment[1]); diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index f3be68b..7fe4f18 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -150,6 +150,11 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { res.status("400").send("Submission doesn't exist."); return; } + + if (!config.categoryList.includes(category)) { + res.status("400").send("Category doesn't exist."); + return; + } let timeSubmitted = Date.now(); From 6485fd0f88a061b4770804410cf3d73a490f7ff1 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Wed, 9 Sep 2020 18:57:47 +0200 Subject: [PATCH 35/41] Update test.json --- test.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test.json b/test.json index 503aa5d..0c67e5c 100644 --- a/test.json +++ b/test.json @@ -46,5 +46,6 @@ "vote.down" ] } - ] + ], + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] } From a068e194e9a6df9b998c23e551401ddce0878260 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Wed, 9 Sep 2020 21:57:59 +0100 Subject: [PATCH 36/41] added test for change category --- test/cases/voteOnSponsorTime.js | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index bcbca78..615eefb 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -22,6 +22,8 @@ describe('voteOnSponsorTime', () => { db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)"); db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)"); db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('incorrect-category', 1, 11, 500, 'incorrect-category', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('incorrect-category-change', 1, 11, 500, 'incorrect-category-change', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); @@ -207,6 +209,24 @@ describe('voteOnSponsorTime', () => { }); }); + it('Should not able to change to an invalid category', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=fakecategory", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category"]); + if (row.category === "sponsor") { + done() + } else { + done("Vote did not succeed. Submission went from sponsor to " + row.category); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + it('Should be able to change your vote for a category and it should immediately change (for now)', (done) => { request.get(utils.getbaseURL() + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro", null, @@ -225,6 +245,29 @@ describe('voteOnSponsorTime', () => { }); }); + + it('Should not be able to change your vote to an invalid category', (done) => { + const vote = (inputCat, assertCat, callback) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category-change&category="+inputCat, null, + (err) => { + if (err) done(err); + else{ + let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category-change"]); + if (row.category === assertCat) { + callback(); + } else { + done("Vote did not succeed. Submission went from sponsor to " + row.category); + } + } + }); + }; + vote("sponsor", "sponsor", () => { + vote("fakeCategory", "sponsor", done); + }); + }); + + it('VIP should be able to vote for a category and it should immediately change', (done) => { request.get(utils.getbaseURL() + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&category=outro", null, From 548061ff13a2ec388692af75eea62b14d1bc8221 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 9 Sep 2020 21:32:47 -0400 Subject: [PATCH 37/41] Add error message for 400 errors --- src/routes/postSkipSegments.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 91ae71e..e219f71 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -177,7 +177,7 @@ module.exports = async function postSkipSegments(req, res) { //check if all correct inputs are here and the length is 1 second or more if (videoID == undefined || userID == undefined || segments == undefined || segments.length < 1) { //invalid request - res.sendStatus(400); + res.sendStatus(400).send("Parameters are not valid"); return; } @@ -191,7 +191,7 @@ module.exports = async function postSkipSegments(req, res) { for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { //invalid request - res.sendStatus(400); + res.sendStatus(400).send("One of your segments are invalid"); return; } @@ -201,7 +201,7 @@ module.exports = async function postSkipSegments(req, res) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request - res.sendStatus(400); + res.sendStatus(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); return; } From 4baf13f82c8eeb745ab3fc7dbd381fed35ac5aac Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 11 Sep 2020 16:35:12 -0400 Subject: [PATCH 38/41] Fix error messages in postSkipSegments --- src/routes/postSkipSegments.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e219f71..5132cd8 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -177,7 +177,7 @@ module.exports = async function postSkipSegments(req, res) { //check if all correct inputs are here and the length is 1 second or more if (videoID == undefined || userID == undefined || segments == undefined || segments.length < 1) { //invalid request - res.sendStatus(400).send("Parameters are not valid"); + res.status(400).send("Parameters are not valid"); return; } @@ -191,7 +191,7 @@ module.exports = async function postSkipSegments(req, res) { for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { //invalid request - res.sendStatus(400).send("One of your segments are invalid"); + res.status(400).send("One of your segments are invalid"); return; } @@ -201,7 +201,7 @@ module.exports = async function postSkipSegments(req, res) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request - res.sendStatus(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); + res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); return; } From 6843e22a7b30d4a772c439a901662c26ab9f2349 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Fri, 11 Sep 2020 17:08:41 -0400 Subject: [PATCH 39/41] Change automod check to call YT and NB API a single time per submission --- src/routes/postSkipSegments.js | 101 +++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 052d730..91c92f7 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -169,34 +169,59 @@ async function autoModerateSubmission(submission) { if (data.pageInfo.totalResults === 0) { return "No video exists with id " + submission.videoID; } else { - let duration = data.items[0].contentDetails.duration; - duration = isoDurations.toSeconds(isoDurations.parse(duration)); - if (duration == 0) { - // Allow submission if the duration is 0 (bug in youtube api) - return false; - } else if ((submission.endTime - submission.startTime) > (duration/100)*80) { - // Reject submission if over 80% of the video - return "One of your submitted segments is over 80% of the video."; - } else { - // Check NeuralBlock - let neuralBlockURL = config.neuralBlockURL; - if (!neuralBlockURL) return false; + let segments = submission.segments; + let nbString = ""; + for (let i = 0; i < segments.length; i++) { + let startTime = parseFloat(segments[i].segment[0]); + let endTime = parseFloat(segments[i].segment[1]); - let response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + - "&segments=" + submission.startTime + "," + submission.endTime); - if (!response.ok) return false; - - let nbPredictions = await response.json(); - if (nbPredictions.probabilities[0] >= 0.70){ - return false; + let duration = data.items[0].contentDetails.duration; + duration = isoDurations.toSeconds(isoDurations.parse(duration)); + if (duration == 0) { + // Allow submission if the duration is 0 (bug in youtube api) + return false; + } else if ((endTime - startTime) > (duration/100)*80) { + // Reject submission if over 80% of the video + return "One of your submitted segments is over 80% of the video."; } else { - let UUID = getHash("v2-categories" + submission.videoID + submission.startTime + - submission.endTime + submission.category + submission.userID, 1); - // Send to Discord - sendWebhooksNB(submission.userID, submission.videoID, UUID, submission.startTime, submission.endTime, submission.category, nbPredictions.probabilities[0], data); - return "NB disagreement."; + if (segments[i].category === "sponsor") { + //Prepare timestamps to send to NB all at once + nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";"; + } } } + // Check NeuralBlock + let neuralBlockURL = config.neuralBlockURL; + if (!neuralBlockURL) return false; + let response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + + "&segments=" + nbString.substring(0,nbString.length-1)); + if (!response.ok) return false; + + let nbPredictions = await response.json(); + nbDecision = false; + let predictionIdx = 0; //Keep track because only sponsor categories were submitted + for (let i = 0; i < segments.length; i++){ + if (segments[i].category === "sponsor"){ + if (nbPredictions.probabilities[predictionIdx] < 0.70){ + nbDecision = true; // At least one bad entry + startTime = parseFloat(segments[i].segment[0]); + endTime = parseFloat(segments[i].segment[1]); + + let UUID = getHash("v2-categories" + submission.videoID + startTime + + endTime + segments[i].category + submission.userID, 1); + // Send to Discord + // Note, if this is too spammy. Consider sending all the segments as one Webhook + sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); + } + predictionIdx++; + } + + } + if (nbDecision){ + return "NB disagreement."; + } else { + return false; + } } } @@ -288,21 +313,23 @@ module.exports = async function postSkipSegments(req, res) { res.sendStatus(409); return; } - - // Auto moderator check - if (!isVIP) { - let autoModerateResult = await autoModerateSubmission({userID, videoID, startTime, endTime, category: segments[i].category}); - if (autoModerateResult == "NB disagreement."){ - // If NB automod rejects, the submission will start with -2 votes - //decreaseVotes = -2; //Disable for now - } else if (autoModerateResult) { - //Normal automod behavior - res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); - return; - } - } } + // Auto moderator check + if (!isVIP) { + let autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category}); + if (autoModerateResult == "NB disagreement."){ + // If NB automod rejects, the submission will start with -2 votes. + // Note, if one submission is bad all submissions will be affected. + // However, this behavior is consistent with other automod functions + // already in place. + //decreaseVotes = -2; //Disable for now + } else if (autoModerateResult) { + //Normal automod behavior + res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord."); + return; + } + } // Will be filled when submitting let UUIDs = []; From a49db96d493db6d0f1edcb11c0274595a4b21a7d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 11 Sep 2020 17:30:54 -0400 Subject: [PATCH 40/41] Clarify error message --- src/routes/postSkipSegments.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 91c92f7..b24e810 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -218,7 +218,7 @@ async function autoModerateSubmission(submission) { } if (nbDecision){ - return "NB disagreement."; + return "Rejected based on NeuralBlock predictions."; } else { return false; } @@ -318,7 +318,7 @@ module.exports = async function postSkipSegments(req, res) { // Auto moderator check if (!isVIP) { let autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category}); - if (autoModerateResult == "NB disagreement."){ + if (autoModerateResult == "Rejected based on NeuralBlock predictions."){ // If NB automod rejects, the submission will start with -2 votes. // Note, if one submission is bad all submissions will be affected. // However, this behavior is consistent with other automod functions From 70a526fb4ff45a54bb86f9ba5fa19e1875a06eae Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 13 Sep 2020 18:27:02 -0400 Subject: [PATCH 41/41] Don't call webhooks twice --- src/routes/postSkipSegments.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index ab80d25..ae3822a 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -410,9 +410,6 @@ module.exports = async function postSkipSegments(req, res) { return; } - // Discord notification - sendWebhooks(userID, videoID, UUID, segmentInfo); - UUIDs.push(UUID); } } catch (err) {