From 7481b42499724985ed46189487dfb1975461c92d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 16 Apr 2020 20:14:13 -0400 Subject: [PATCH 01/15] Update README.MD --- README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.MD b/README.MD index d56f940..37d541b 100644 --- a/README.MD +++ b/README.MD @@ -8,7 +8,7 @@ This is the server backend for it This is a simple Sqlite database that will hold all the timing data. -To make sure that this project doesn't die, I have made the database publicly downloadable at https://api.sponsor.ajay.app/database.db (formerly https://sponsor.ajay.app/database.db). You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. +To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea. From 950b8ecb55412ed4dd8af3c0fb2ba1b1cd1c0832 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 19 Apr 2020 19:18:24 -0400 Subject: [PATCH 02/15] Update getDaysSavedFormatted.js --- src/routes/getDaysSavedFormatted.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/getDaysSavedFormatted.js b/src/routes/getDaysSavedFormatted.js index 7fece0c..2f0b141 100644 --- a/src/routes/getDaysSavedFormatted.js +++ b/src/routes/getDaysSavedFormatted.js @@ -1,4 +1,4 @@ -var db = require('../databases/databases.js'); +var db = require('../databases/databases.js').db; module.exports = function getDaysSavedFormatted (req, res) { let row = db.prepare("select sum((endtime - starttime) / 60 / 60 / 24 * views) as dayssaved from sponsortimes where shadowhidden != 1").get(); @@ -9,4 +9,4 @@ module.exports = function getDaysSavedFormatted (req, res) { dayssaved: row.dayssaved.tofixed(2) }); } -} \ No newline at end of file +} From 14e21a1009993a63b70950f0cec99a175c586c41 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 19 Apr 2020 19:22:38 -0400 Subject: [PATCH 03/15] Fixed wrong database being sent --- src/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.js b/src/app.js index 3c7ec8a..043ce44 100644 --- a/src/app.js +++ b/src/app.js @@ -85,7 +85,7 @@ app.get('/api/getTotalStats', getTotalStats); app.get('/api/getdayssavedformatted', getDaysSavedFormatted); app.get('/database.db', function (req, res) { - res.sendfile("./databases/sponsortimes.db", { root: __dirname }); + res.sendFile("./databases/sponsorTimes.db", { root: __dirname }); }); // Create an HTTP service. From f8340d770cd373050c75a0653d621ece03352080 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 19 Apr 2020 19:26:09 -0400 Subject: [PATCH 04/15] Fixed capitalization --- src/routes/getDaysSavedFormatted.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/getDaysSavedFormatted.js b/src/routes/getDaysSavedFormatted.js index 2f0b141..45f13d8 100644 --- a/src/routes/getDaysSavedFormatted.js +++ b/src/routes/getDaysSavedFormatted.js @@ -1,12 +1,12 @@ var db = require('../databases/databases.js').db; module.exports = function getDaysSavedFormatted (req, res) { - let row = db.prepare("select sum((endtime - starttime) / 60 / 60 / 24 * views) as dayssaved from sponsortimes where shadowhidden != 1").get(); + let row = db.prepare("SELECT SUM((endTime - startTime) / 60 / 60 / 24 * views) as daysSaved from sponsorTimes where shadowHidden != 1").get(); if (row !== undefined) { //send this result res.send({ - dayssaved: row.dayssaved.tofixed(2) + daysSaved: row.daysSaved.toFixed(2) }); } } From ad2fe00af0f333a6a42ccc559fd9a55f73647606 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 19 Apr 2020 19:39:40 -0400 Subject: [PATCH 05/15] Fixed database endpoint and getDatsSavedFormatted --- src/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.js b/src/app.js index 043ce44..4ad83ff 100644 --- a/src/app.js +++ b/src/app.js @@ -82,10 +82,10 @@ app.get('/api/getTopUsers', getTopUsers); app.get('/api/getTotalStats', getTotalStats); //send out a formatted time saved total -app.get('/api/getdayssavedformatted', getDaysSavedFormatted); +app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); app.get('/database.db', function (req, res) { - res.sendFile("./databases/sponsorTimes.db", { root: __dirname }); + res.sendFile("./databases/sponsorTimes.db", { root: "./" }); }); // Create an HTTP service. From 818c288ba03e60e7eb438e9d00bfdf8b013273fc Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 19 Apr 2020 19:47:17 -0400 Subject: [PATCH 06/15] Set production mode based on config --- src/app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app.js b/src/app.js index 4ad83ff..9561dc1 100644 --- a/src/app.js +++ b/src/app.js @@ -26,7 +26,6 @@ var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); - //setup CORS correctly app.use(corsMiddleware); app.use(loggerMiddleware); @@ -35,6 +34,9 @@ app.use(express.json()) // Setup pretty JSON if (config.mode === "development") app.set('json spaces', 2); +// Set production mode +app.set('env', config.mode || 'production'); + //add the get function app.get('/api/getVideoSponsorTimes', oldGetVideoSponsorTimes); From a3b01e4eae3a3a668e8c7aa4ac3ff855cf2230b9 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 23 Apr 2020 11:58:36 -0400 Subject: [PATCH 07/15] Fix test process not dying --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index 3a13418..a5f09b0 100644 --- a/test.js +++ b/test.js @@ -34,7 +34,7 @@ var mockServer = createMockServer(() => { mocha.run(function(failures) { mockServer.close(); server.close(); - process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures + process.exit(failures ? 1 : 0); // exit with non-zero status if there were failures }); }); }); From 546ded3bdd88dc60f134886821cf342f5fbb7ede Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 23 Apr 2020 12:48:02 -0400 Subject: [PATCH 08/15] Fixed mocks not responding properly --- test.js | 2 +- test/mocks.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test.js b/test.js index a5f09b0..45ace90 100644 --- a/test.js +++ b/test.js @@ -34,7 +34,7 @@ var mockServer = createMockServer(() => { mocha.run(function(failures) { mockServer.close(); server.close(); - process.exit(failures ? 1 : 0); // exit with non-zero status if there were failures + process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures }); }); }); diff --git a/test/mocks.js b/test/mocks.js index a124646..49060af 100644 --- a/test/mocks.js +++ b/test/mocks.js @@ -4,11 +4,11 @@ var app = express(); var config = require('../src/config.js'); app.post('/ReportChannelWebhook', (req, res) => { - res.status(200); + res.sendStatus(200); }); app.post('/FirstTimeSubmissionsWebhook', (req, res) => { - res.status(200); + res.sendStatus(200); }); module.exports = function createMockServer(callback) { From 6cd672f9e636a907af637a4f0e14647ac11f6a24 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Fri, 24 Apr 2020 18:11:09 +0100 Subject: [PATCH 09/15] allow submissions when duration is parsed as 0 --- src/routes/postSkipSegments.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index d8fe2a3..66cc4b0 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -105,9 +105,11 @@ async function autoModerateSubmission(submission, callback) { } else { let duration = data.items[0].contentDetails.duration; duration = isoDurations.toSeconds(isoDurations.parse(duration)); - - // Reject submission if over 80% of the video - if ((submission.endTime - submission.startTime) > (duration/100)*80) { + if (duration === 0) { + // Allow submission if the duration is 0 (bug in youtube api) + return false; + } else if ((submission.endTime - submission.startTime) > (duration/100)*80) { + // Reject submission if over 80% of the video return "Sponsor segment is over 80% of the video."; } else { return false; From fd397de6b4b75e6658489b0919fc38b504cb05b1 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Fri, 24 Apr 2020 18:20:40 +0100 Subject: [PATCH 10/15] allow submission if duration is 0 --- src/routes/postSkipSegments.js | 4 ++-- test/cases/postSkipSegments.js | 10 ++++++++++ test/youtubeMock.js | 25 +++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 66cc4b0..d410568 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -101,11 +101,11 @@ async function autoModerateSubmission(submission, callback) { } else { // Check to see if video exists if (data.pageInfo.totalResults === 0) { - callback("No video exists with id " + submission.videoID); + return "No video exists with id " + submission.videoID; } else { let duration = data.items[0].contentDetails.duration; duration = isoDurations.toSeconds(isoDurations.parse(duration)); - if (duration === 0) { + if (duration == 0) { // Allow submission if the duration is 0 (bug in youtube api) return false; } else if ((submission.endTime - submission.startTime) > (duration/100)*80) { diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index 05c7374..3f1492f 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -99,6 +99,16 @@ describe('postSkipSegments', () => { else done("non 403 status code: " + res.statusCode + " ("+body+")"); }); }); + + it('Should be allowed if youtube thinks duration is 0', (done) => { + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 200) done(); // pass + else done("non 200 status code: " + res.statusCode + " ("+body+")"); + }); + }); it('Should be rejected if not a valid videoID', (done) => { request.get(utils.getbaseURL() diff --git a/test/youtubeMock.js b/test/youtubeMock.js index 0a9998d..dd64898 100644 --- a/test/youtubeMock.js +++ b/test/youtubeMock.js @@ -10,14 +10,35 @@ YouTubeAPI.videos.list({ const YouTubeAPI = { videos: { list: (obj, callback) => { - if (obj.videoID === "knownWrongID") { + if (obj.id === "knownWrongID") { callback(undefined, { pageInfo: { totalResults: 0 }, items: [] }); - } else { + } if (obj.id === "noDuration") { + callback(undefined, { + pageInfo: { + totalResults: 1 + }, + items: [ + { + contentDetails: { + duration: "PT0S" + }, + snippet: { + title: "Example Title", + thumbnails: { + maxres: { + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png" + } + } + } + } + ] + }); + } else { callback(undefined, { pageInfo: { totalResults: 1 From cfa80be4c7553bfcd0d095d84207ad8a1239dc6b Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Tue, 28 Apr 2020 05:04:51 +0100 Subject: [PATCH 11/15] add user history to discord notification on segment reporting --- src/routes/voteOnSponsorTime.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index bc9a326..85d7076 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -86,7 +86,11 @@ module.exports = async function voteOnSponsorTime(req, res) { // Send discord message if (type != 1) { // Get video ID - let submissionInfoRow = db.prepare("SELECT videoID, userID, startTime, endTime FROM sponsorTimes WHERE UUID = ?").get(UUID); + let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, "+ + "(select count(1) from sponsorTimes where userID = s.userID) count, "+ + "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded "+ + "FROM sponsorTimes s inner join userNames u on s.userID = u.userID where s.UUID=?" + ).get(UUID); let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID); @@ -96,9 +100,11 @@ module.exports = async function voteOnSponsorTime(req, res) { id: submissionInfoRow.videoID }, function (err, data) { if (err || data.items.length === 0) { + console.log('pre-err'); err && console.log(err); return; } + console.log('presend' + submissionInfoRow.count); request.post(config.discordReportChannelWebhookURL, { json: { @@ -107,12 +113,15 @@ module.exports = async function voteOnSponsorTime(req, res) { "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + - " Views**\n\nSubmission ID: " + UUID + - "\n\nSubmitted by: " + submissionInfoRow.userID + "\n\nTimestamp: " + + " Views**\n\n**Submission ID:** " + UUID + + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID + + "\n\n**Total User Submissions:** "+submissionInfoRow.count + + "\n**Disregarded User Submissions:** "+submissionInfoRow.disregarded + +"\n\n**Timestamp:** " + getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), "color": 10813440, "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "") + "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 && "Report by VIP User") }, "thumbnail": { "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", @@ -167,5 +176,6 @@ module.exports = async function voteOnSponsorTime(req, res) { res.sendStatus(200); } catch (err) { console.error(err); + res.status(400).json({error: 'Think of something to return'}); } } \ No newline at end of file From f6df7e5f6f71ed26ae9dcc28dd6f012d6b549516 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Tue, 28 Apr 2020 05:07:39 +0100 Subject: [PATCH 12/15] remove testing logs --- src/routes/voteOnSponsorTime.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 85d7076..c1a1c14 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -100,11 +100,9 @@ module.exports = async function voteOnSponsorTime(req, res) { id: submissionInfoRow.videoID }, function (err, data) { if (err || data.items.length === 0) { - console.log('pre-err'); err && console.log(err); return; } - console.log('presend' + submissionInfoRow.count); request.post(config.discordReportChannelWebhookURL, { json: { From 13aee13f18a07bd3db951b5049a7173195c631e4 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Tue, 28 Apr 2020 05:10:55 +0100 Subject: [PATCH 13/15] Improved phrasing of error responses \n removed debug logs --- src/routes/voteOnSponsorTime.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index c1a1c14..d98f7ac 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -174,6 +174,6 @@ module.exports = async function voteOnSponsorTime(req, res) { res.sendStatus(200); } catch (err) { console.error(err); - res.status(400).json({error: 'Think of something to return'}); + res.status(500).json({error: 'Internal error creating segment vote'}); } } \ No newline at end of file From e9e525ec63f0d9e66b993d10dad36043b3d98c34 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Tue, 28 Apr 2020 05:18:54 +0100 Subject: [PATCH 14/15] fixed formatting --- src/routes/voteOnSponsorTime.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index d98f7ac..094565e 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -108,10 +108,10 @@ module.exports = async function voteOnSponsorTime(req, res) { json: { "embeds": [{ "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + - "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), - "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + - " Views**\n\n**Submission ID:** " + UUID + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), + "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + + " Views**\n\n**Submission ID:** " + UUID + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID + "\n\n**Total User Submissions:** "+submissionInfoRow.count + "\n**Disregarded User Submissions:** "+submissionInfoRow.disregarded From d930853edfaab704d056b30f3f7db847dbe3edca Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Wed, 29 Apr 2020 22:27:38 +0100 Subject: [PATCH 15/15] review changes --- src/routes/voteOnSponsorTime.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 094565e..dda73ac 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -114,12 +114,12 @@ module.exports = async function voteOnSponsorTime(req, res) { + " Views**\n\n**Submission ID:** " + UUID + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID + "\n\n**Total User Submissions:** "+submissionInfoRow.count - + "\n**Disregarded User Submissions:** "+submissionInfoRow.disregarded + + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded +"\n\n**Timestamp:** " + getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), "color": 10813440, "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 && "Report by VIP User") + "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "") }, "thumbnail": { "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",