From 85dd187cb0b59750a6cd31030b9b7e155cfc2cc7 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 00:25:05 -0500 Subject: [PATCH 1/5] reject submissions if total length is more than 80 precent of the video --- src/routes/postSkipSegments.js | 61 ++++++++++++++ test/cases/postSkipSegments.js | 141 +++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 40c8287..de55938 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -396,6 +396,40 @@ module.exports = async function postSkipSegments(req, res) { startingVotes = 10000; } + if (config.youtubeAPIKey !== null) { + let {err, data} = await new Promise((resolve, reject) => { + YouTubeAPI.listVideos(videoID, "contentDetails,snippet", (err, data) => resolve({err, data})); + }); + + //get all segments for this video and user + let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ?", [userID, videoID]); + let allSegmentTimes = []; + if (allSubmittedByUser !== undefined) + { + for (const segmentInfo of allSubmittedByUser) { + allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]) + } + } + for (let i = 0; i < segments.length; i++) { + let startTime = parseFloat(segments[i].segment[0]); + let endTime = parseFloat(segments[i].segment[1]); + allSegmentTimes.push([startTime, endTime]); + } + const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function(a, b) { return a[0]-b[0] || a[1]-b[1] })); + + let videoDuration = data.items[0].contentDetails.duration; + videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); + if (videoDuration != 0) { + let allSegmentDuration = 0; + allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); + if (allSegmentDuration > (videoDuration/100)*80) { + // Reject submission if all segments combine are over 80% of the video + res.status(400).send("Total length of your submitted segments are over 80% of the video."); + return; + } + } + } + for (const segmentInfo of segments) { //this can just be a hash of the data //it's better than generating an actual UUID like what was used before @@ -437,3 +471,30 @@ module.exports = async function postSkipSegments(req, res) { sendWebhooks(userID, videoID, UUIDs[i], segments[i]); } } + +// Takes an array of arrays: +// ex) +// [ +// [3, 40], +// [50, 70], +// [60, 80], +// [100, 150] +// ] +// => transforms to combining overlapping segments +// [ +// [3, 40], +// [50, 80], +// [100, 150] +// ] +function mergeTimeSegments(ranges) { + var result = [], last; + + ranges.forEach(function (r) { + if (!last || r[0] > last[1]) + result.push(last = r); + else if (r[1] > last[1]) + last[1] = r[1]; + }); + + return result; +} \ No newline at end of file diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index a2e2c97..af87a9d 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -5,8 +5,16 @@ var utils = require('../utils.js'); var databases = require('../../src/databases/databases.js'); var db = databases.db; +const getHash = require('../../src/utils/getHash.js'); describe('postSkipSegments', () => { + before(() => { + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + + db.exec(startOfQuery + "('80percent_video', 0, 1000, 0, '80percent-uuid-0', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); + db.exec(startOfQuery + "('80percent_video', 1001, 1005, 0, '80percent-uuid-1', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); + }); + it('Should be able to submit a single time (Params method)', (done) => { request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, @@ -90,6 +98,139 @@ describe('postSkipSegments', () => { }); }).timeout(5000); + it('Should allow multiple times if total is under 80% of video(JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "L_jWHffIx5E", + segments: [{ + segment: [3, 3000], + category: "sponsor" + },{ + segment: [3002, 3050], + category: "intro" + },{ + segment: [45, 100], + category: "interaction" + },{ + segment: [99, 170], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["L_jWHffIx5E"]); + let success = true; + if (rows.length === 4) { + for (const row of rows) { + if ((row.startTime !== 3 || row.endTime !== 3000 || row.category !== "sponsor") && + (row.startTime !== 3002 || row.endTime !== 3050 || row.category !== "intro") && + (row.startTime !== 45 || row.endTime !== 100 || row.category !== "interaction") && + (row.startTime !== 99 || row.endTime !== 170 || row.category !== "sponsor")) { + success = false; + break; + } + } + } + + if (success) done(); + else done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows)); + } else { + done("Status code was " + res.statusCode); + } + }); + }).timeout(5000); + + it('Should reject multiple times if total is over 80% of video (JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "n9rIGdXnSJc", + segments: [{ + segment: [0, 2000], + category: "interaction" + },{ + segment: [3000, 4000], + category: "sponsor" + },{ + segment: [1500, 2750], + category: "sponsor" + },{ + segment: [4050, 4750], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["n9rIGdXnSJc"]); + let success = true; + if (rows.length === 4) { + for (const row of rows) { + if ((row.startTime === 0 || row.endTime === 2000 || row.category === "interaction") || + (row.startTime === 3000 || row.endTime === 4000 || row.category === "sponsor") || + (row.startTime === 1500 || row.endTime === 2750 || row.category === "sponsor") || + (row.startTime === 4050 || row.endTime === 4750 || row.category === "intro")) { + success = false; + break; + } + } + } + + if (success) done(); + else + done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows)); + } else { + done("Status code was " + res.statusCode); + } + }); + }).timeout(5000); + + it('Should reject multiple times if total is over 80% of video including previosuly submitted times(JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "80percent_video", + segments: [{ + segment: [2000, 4000], + category: "sponsor" + },{ + segment: [1500, 2750], + category: "sponsor" + },{ + segment: [4050, 4750], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["80percent_video"]); + let success = true && rows.length == 2; + for (const row of rows) { + if ((row.startTime === 2000 || row.endTime === 4000 || row.category === "sponsor") || + (row.startTime === 1500 || row.endTime === 2750 || row.category === "sponsor") || + (row.startTime === 4050 || row.endTime === 4750 || row.category === "sponsor")) { + success = false; + break; + } + } + if (success) done(); + else + done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows)); + } else { + done("Status code was " + res.statusCode); + } + }); + }).timeout(5000); + it('Should be accepted if a non-sponsor is less than 1 second', (done) => { request.post(utils.getbaseURL() + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=intro", null, From 2825cb63fb69fbe9a566f9ab1a6be3756460ac37 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 00:33:00 -0500 Subject: [PATCH 2/5] ignore segments with less than -1 votes --- src/routes/postSkipSegments.js | 2 +- test/cases/postSkipSegments.js | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index de55938..27bfdde 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -402,7 +402,7 @@ module.exports = async function postSkipSegments(req, res) { }); //get all segments for this video and user - let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ?", [userID, videoID]); + let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ? and votes > -1", [userID, videoID]); let allSegmentTimes = []; if (allSubmittedByUser !== undefined) { diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index af87a9d..eb8e287 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -13,6 +13,7 @@ describe('postSkipSegments', () => { db.exec(startOfQuery + "('80percent_video', 0, 1000, 0, '80percent-uuid-0', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); db.exec(startOfQuery + "('80percent_video', 1001, 1005, 0, '80percent-uuid-1', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); + db.exec(startOfQuery + "('80percent_video', 0, 5000, -2, '80percent-uuid-2', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')"); }); it('Should be able to submit a single time (Params method)', (done) => { @@ -122,7 +123,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["L_jWHffIx5E"]); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["L_jWHffIx5E"]); let success = true; if (rows.length === 4) { for (const row of rows) { @@ -168,7 +169,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { - let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["n9rIGdXnSJc"]); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["n9rIGdXnSJc"]); let success = true; if (rows.length === 4) { for (const row of rows) { @@ -212,7 +213,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { - let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["80percent_video"]); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["80percent_video"]); let success = true && rows.length == 2; for (const row of rows) { if ((row.startTime === 2000 || row.endTime === 4000 || row.category === "sponsor") || From bcf90e8094f8933637168392545a57f8b55718a8 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 00:43:29 -0500 Subject: [PATCH 3/5] fix formatting and add more comments --- src/routes/postSkipSegments.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 27bfdde..092b7f8 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -404,23 +404,28 @@ module.exports = async function postSkipSegments(req, res) { //get all segments for this video and user let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ? and votes > -1", [userID, videoID]); let allSegmentTimes = []; - if (allSubmittedByUser !== undefined) - { + if (allSubmittedByUser !== undefined) { + //add segments the user has previously submitted for (const segmentInfo of allSubmittedByUser) { allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]) } } + + //add segments they are trying to add in this submission for (let i = 0; i < segments.length; i++) { let startTime = parseFloat(segments[i].segment[0]); let endTime = parseFloat(segments[i].segment[1]); allSegmentTimes.push([startTime, endTime]); } + + //merge all the times into non-overlapping arrays const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function(a, b) { return a[0]-b[0] || a[1]-b[1] })); let videoDuration = data.items[0].contentDetails.duration; videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); if (videoDuration != 0) { let allSegmentDuration = 0; + //sum all segment times together allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); if (allSegmentDuration > (videoDuration/100)*80) { // Reject submission if all segments combine are over 80% of the video From 830ef7e0dccccc0dc8285279e290db9007617c55 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Wed, 14 Oct 2020 20:52:01 +0100 Subject: [PATCH 4/5] made youtube api cache have conatcnt part value --- src/routes/postSkipSegments.js | 6 +++--- src/routes/voteOnSponsorTime.js | 2 +- src/utils/youtubeAPI.js | 5 +++-- test/youtubeMock.js | 3 +-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 092b7f8..4da9466 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -50,7 +50,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { if (config.youtubeAPIKey !== null) { let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]); - YouTubeAPI.listVideos(videoID, "snippet", (err, data) => { + YouTubeAPI.listVideos(videoID, (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; @@ -154,7 +154,7 @@ async function autoModerateSubmission(submission) { // Get the video information from the youtube API if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { - YouTubeAPI.listVideos(submission.videoID, "contentDetails,snippet", (err, data) => resolve({err, data})); + YouTubeAPI.listVideos(submission.videoID, (err, data) => resolve({err, data})); }); if (err) { @@ -398,7 +398,7 @@ module.exports = async function postSkipSegments(req, res) { if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { - YouTubeAPI.listVideos(videoID, "contentDetails,snippet", (err, data) => resolve({err, data})); + YouTubeAPI.listVideos(videoID, (err, data) => resolve({err, data})); }); //get all segments for this video and user diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 59a6789..d2a9f49 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -49,7 +49,7 @@ function sendWebhooks(voteData) { } if (config.youtubeAPIKey !== null) { - YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => { + YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index 1deb3b2..96e718e 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -18,13 +18,14 @@ if (config.mode === "test") { exportObject = YouTubeAPI; // YouTubeAPI.videos.list wrapper with cacheing - exportObject.listVideos = (videoID, part, callback) => { + exportObject.listVideos = (videoID, callback) => { + let part = 'contentDetails,snippet'; if (videoID.length !== 11 || videoID.includes(".")) { callback("Invalid video ID"); return; } - let redisKey = "youtube.video." + videoID + "." + part; + let redisKey = "youtube.video." + videoID; redis.get(redisKey, (getErr, result) => { if (getErr || !result) { logger.debug("redis: no cache for video information: " + videoID); diff --git a/test/youtubeMock.js b/test/youtubeMock.js index cc722d9..7e15f64 100644 --- a/test/youtubeMock.js +++ b/test/youtubeMock.js @@ -8,9 +8,8 @@ YouTubeAPI.videos.list({ // https://developers.google.com/youtube/v3/docs/videos const YouTubeAPI = { - listVideos: (id, part, callback) => { + listVideos: (id, callback) => { YouTubeAPI.videos.list({ - part: part, id: id }, callback); }, From 064d3ad5d587723120367195446724c6f60248c1 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 14 Oct 2020 18:40:55 -0500 Subject: [PATCH 5/5] fix test imports --- test/cases/postSkipSegments.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index 4fefef2..0a37aaa 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -7,7 +7,6 @@ var utils = require('../utils.js'); var databases = require('../../src/databases/databases.js'); var db = databases.db; -const getHash = require('../../src/utils/getHash.js'); describe('postSkipSegments', () => { before(() => {