From cd875bd0f97182e2c8d33472faf2bcc0c41e50d2 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Sun, 5 Jul 2020 04:37:57 +0100 Subject: [PATCH 01/20] docker --- Dockerfile | 9 +++++++++ entrypoint.sh | 4 ++++ 2 files changed, 13 insertions(+) create mode 100644 Dockerfile create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b32fe20 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM node:12 +WORKDIR /usr/src/app +COPY package.json . +RUN npm install +COPY index.js . +COPY src . +COPY entrypoint.sh . +EXPOSE 8080 +ENTRYPOINT ./entrypoint.sh \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..516cdfc --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,4 @@ +#!bin/bash +set -e +cp /etc/config.json . +node index.js \ No newline at end of file From b03daa3c710e45526a4f19429857a9ed101d8244 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Sun, 5 Jul 2020 04:40:56 +0100 Subject: [PATCH 02/20] executable entrypoint --- entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 entrypoint.sh diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 From 9a168aae456fb283b5365407a78fdbec01cb111b Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Sun, 5 Jul 2020 05:09:06 +0100 Subject: [PATCH 03/20] change mount point for config --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 516cdfc..f6d846f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,4 @@ #!bin/bash set -e -cp /etc/config.json . +cp /etc/sponsorblock/config.json . node index.js \ No newline at end of file From 8e89d2bf92cba6eecc56204bb117386134db85a5 Mon Sep 17 00:00:00 2001 From: Joe-Dowd Date: Sun, 5 Jul 2020 09:25:57 +0100 Subject: [PATCH 04/20] Added default config for docker --- Dockerfile | 4 ++-- entrypoint.sh | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index b32fe20..efca3de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ WORKDIR /usr/src/app COPY package.json . RUN npm install COPY index.js . -COPY src . +COPY src src COPY entrypoint.sh . EXPOSE 8080 -ENTRYPOINT ./entrypoint.sh \ No newline at end of file +CMD ./entrypoint.sh \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index f6d846f..5e14a9e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,39 @@ -#!bin/bash +#!/bin/bash set -e -cp /etc/sponsorblock/config.json . +echo 'Entrypoint script' +cd /usr/src/app +cp /etc/sponsorblock/config.json . || cat < config.json +{ + "port": 8080, + "mysql": { + "host": "127.0.0.1", + "port": 3306, + "database": "sponsorblock", + "user": "sponsorblock", + "password": "sponsorblock" + }, + "privateMysql": { + "host": "127.0.0.1", + "port": 3306, + "database": "sponsorblock_private", + "user": "sponsorblock", + "password": "sponsorblock" + }, + "globalSalt": "", + "adminUserID": "", + "youtubeAPIKey": "", + "discordReportChannelWebhookURL": null, + "discordFirstTimeSubmissionsWebhookURL": null, + "discordAutoModWebhookURL": null, + "behindProxy": true, + "db": null, + "privateDB": null, + "createDatabaseIfNotExist": true, + "schemaFolder": null, + "dbSchema": null, + "privateDBSchema": null, + "mode": "development", + "readOnly": false +} +EOF node index.js \ No newline at end of file From 18f5d656de8610eeb25c64c989e29b28bb548113 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 27 Jul 2020 05:51:49 +0100 Subject: [PATCH 05/20] Aded option to proxy submisissions to another server to persist. --- index.js | 2 +- src/routes/postSkipSegments.js | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 5e4b977..61acd0c 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,5 @@ var config = require('./src/config.js'); var createServer = require('./src/app.js'); var server = createServer(() => { - console.log("Server started."); + console.log("Server started on port "+config.port+"."); }); \ No newline at end of file diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index ffbe767..01e6ee9 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -69,9 +69,13 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { // submission: {videoID, startTime, endTime} // callback: function(reject: "String containing reason the submission was rejected") // returns: string when an error, false otherwise + +// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return +// false for a pass - it was confusing and lead to this bug - any use of this function in +// the furute could have the same problem. async function autoModerateSubmission(submission, callback) { // Get the video information from the youtube API - if (config.youtubeAPI !== null) { + if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { YouTubeAPI.videos.list({ part: "contentDetails", @@ -101,15 +105,30 @@ async function autoModerateSubmission(submission, callback) { } } else { - console.log("Skipped YouTube API"); + (config.mode === 'development') && console.log("Skipped YouTube API"); // Can't moderate the submission without calling the youtube API // so allow by default. - return; + return false; } } +function proxySubmission(req) { + console.log(req.body); + request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => { + if (!err) { + console.log('Proxy Sunmission: ' + result.statusCode + ' ('+result.body+')'); + } else { + console.log("Proxy Submission: Failed to make call"); + } + }); +} + module.exports = async function postSkipSegments(req, res) { + if (config.proxySubmission) { + proxySubmission(req); + } + let videoID = req.query.videoID || req.body.videoID; let userID = req.query.userID || req.body.userID; From 0e34423dd6f9300199dcc2fa571775757bd25da2 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 27 Jul 2020 05:53:16 +0100 Subject: [PATCH 06/20] added option to example config --- config.json.example | 1 + 1 file changed, 1 insertion(+) diff --git a/config.json.example b/config.json.example index c548ffc..22971d9 100644 --- a/config.json.example +++ b/config.json.example @@ -8,6 +8,7 @@ "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] + "proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash) "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For" "db": "./databases/sponsorTimes.db", "privateDB": "./databases/private.db", From 99c8d5d23885f76e84edf46925876a2c47edf123 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 27 Jul 2020 05:56:23 +0100 Subject: [PATCH 07/20] removed log and fixed log typo --- src/routes/postSkipSegments.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 01e6ee9..e020b6e 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -114,10 +114,9 @@ async function autoModerateSubmission(submission, callback) { } function proxySubmission(req) { - console.log(req.body); request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => { if (!err) { - console.log('Proxy Sunmission: ' + result.statusCode + ' ('+result.body+')'); + console.log('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); } else { console.log("Proxy Submission: Failed to make call"); } From 51fe7d55999073d032a56fd0948e488b3975acdd Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 27 Jul 2020 19:53:23 -0400 Subject: [PATCH 08/20] Changed to if statement --- src/routes/postSkipSegments.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e020b6e..22e4b6e 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -72,7 +72,7 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return // false for a pass - it was confusing and lead to this bug - any use of this function in -// the furute could have the same problem. +// the future could have the same problem. async function autoModerateSubmission(submission, callback) { // Get the video information from the youtube API if (config.youtubeAPIKey !== null) { @@ -105,7 +105,7 @@ async function autoModerateSubmission(submission, callback) { } } else { - (config.mode === 'development') && console.log("Skipped YouTube API"); + if (config.mode === 'development') console.log("Skipped YouTube API"); // Can't moderate the submission without calling the youtube API // so allow by default. @@ -115,10 +115,12 @@ async function autoModerateSubmission(submission, callback) { function proxySubmission(req) { request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => { - if (!err) { - console.log('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); - } else { - console.log("Proxy Submission: Failed to make call"); + if (config.mode === 'development') { + if (!err) { + console.log('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); + } else { + console.log("Proxy Submission: Failed to make call"); + } } }); } From 98b79b042b6c9543e7777b652b8806a177b1d44a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 27 Jul 2020 19:54:08 -0400 Subject: [PATCH 09/20] Return when proxying --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 22e4b6e..d5f4724 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -127,7 +127,7 @@ function proxySubmission(req) { module.exports = async function postSkipSegments(req, res) { if (config.proxySubmission) { - proxySubmission(req); + return proxySubmission(req); } let videoID = req.query.videoID || req.body.videoID; From cadf2640febbf3b3adc8ce28af02857a8cae89bf Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 27 Jul 2020 19:54:38 -0400 Subject: [PATCH 10/20] Add spacing --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 61acd0c..508c089 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,5 @@ var config = require('./src/config.js'); var createServer = require('./src/app.js'); var server = createServer(() => { - console.log("Server started on port "+config.port+"."); -}); \ No newline at end of file + console.log("Server started on port " + config.port + "."); +}); From 78783368664a635857ae1173c279d5c484c85c24 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 27 Jul 2020 20:53:15 -0400 Subject: [PATCH 11/20] Don't return after proxying --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index d5f4724..22e4b6e 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -127,7 +127,7 @@ function proxySubmission(req) { module.exports = async function postSkipSegments(req, res) { if (config.proxySubmission) { - return proxySubmission(req); + proxySubmission(req); } let videoID = req.query.videoID || req.body.videoID; From 5ac5c30fd658ad4f76620141a254ebcdf5745e4d Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 28 Jul 2020 04:32:12 +0100 Subject: [PATCH 12/20] import config --- src/databases/Mysql.js | 5 +++-- src/databases/Sqlite.js | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/databases/Mysql.js b/src/databases/Mysql.js index b1c8fc1..337df73 100644 --- a/src/databases/Mysql.js +++ b/src/databases/Mysql.js @@ -1,8 +1,9 @@ var MysqlInterface = require('sync-mysql'); +var config = require('../config.js'); class Mysql { - constructor(config) { - this.connection = new MysqlInterface(config); + constructor(msConfig) { + this.connection = new MysqlInterface(msConfig); } exec(query) { diff --git a/src/databases/Sqlite.js b/src/databases/Sqlite.js index c043656..52e1632 100644 --- a/src/databases/Sqlite.js +++ b/src/databases/Sqlite.js @@ -1,4 +1,5 @@ const { db } = require("./databases"); +var config = require('../config.js'); class Sqlite { constructor(connection) { From d5a720fa0c895e4f31c2474109e39bc7e92dc374 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 2 Aug 2020 11:58:11 -0400 Subject: [PATCH 13/20] Check that start time is not less than zero --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index f575e53..acecbc2 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -148,7 +148,7 @@ module.exports = async function postSkipSegments(req, res) { let endTime = parseFloat(segments[i].segment[1]); if (isNaN(startTime) || isNaN(endTime) - || startTime === Infinity || endTime === Infinity || startTime > endTime) { + || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime) { //invalid request res.sendStatus(400); return; From 25166348562f2281e0cdd77138eb6c7e8c61839a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 2 Aug 2020 12:08:44 -0400 Subject: [PATCH 14/20] Don't allow 0 ms submissions --- src/routes/postSkipSegments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index acecbc2..2aac4ea 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -148,7 +148,7 @@ module.exports = async function postSkipSegments(req, res) { let endTime = parseFloat(segments[i].segment[1]); if (isNaN(startTime) || isNaN(endTime) - || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime) { + || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request res.sendStatus(400); return; From 226197a8274bd5172de8162db947af3997585a69 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 21 Aug 2020 15:27:41 +0100 Subject: [PATCH 15/20] Added logger util and used in place of console log --- src/databases/Mysql.js | 5 ++-- src/databases/Sqlite.js | 5 ++-- src/middleware/logger.js | 5 ++-- src/routes/getSkipSegments.js | 3 ++- src/routes/getUsername.js | 3 ++- src/routes/getViewsForUser.js | 4 ++-- src/routes/oldSubmitSponsorTimes.js | 2 -- src/routes/postSkipSegments.js | 25 ++++++++++---------- src/routes/setUsername.js | 3 ++- src/routes/voteOnSponsorTime.js | 17 +++++++------- src/utils/logger.js | 36 +++++++++++++++++++++++++++++ 11 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 src/utils/logger.js diff --git a/src/databases/Mysql.js b/src/databases/Mysql.js index 337df73..40e3d76 100644 --- a/src/databases/Mysql.js +++ b/src/databases/Mysql.js @@ -1,5 +1,6 @@ var MysqlInterface = require('sync-mysql'); var config = require('../config.js'); +var logger = require('../utils/logger.js'); class Mysql { constructor(msConfig) { @@ -11,7 +12,7 @@ class Mysql { } prepare (type, query, params) { - (config.mode === "development") && console.log("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params); + logger.debug("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params); if (type === 'get') { return this.connection.query(query, params)[0]; } else if (type === 'run') { @@ -19,7 +20,7 @@ class Mysql { } else if (type === 'all') { return this.connection.query(query, params); } else { - console.log('returning undefined...') + logger.warn('returning undefined...'); return undefined; } } diff --git a/src/databases/Sqlite.js b/src/databases/Sqlite.js index 52e1632..6b71fec 100644 --- a/src/databases/Sqlite.js +++ b/src/databases/Sqlite.js @@ -1,5 +1,6 @@ const { db } = require("./databases"); var config = require('../config.js'); +const logger = require('../utils/logger.js'); class Sqlite { constructor(connection) { @@ -18,8 +19,8 @@ class Sqlite { } else if (type === 'all') { return this.connection.prepare(query).all(...params); } else { - (config.mode === "development") && console.log('returning undefined...') - (config.mode === "development") && console.log("prepare: type: " + type + ", query: " + query + ", params: " + params); + logger.debug('sqlite query: returning undefined') + logger.debug("prepare: type: " + type + ", query: " + query + ", params: " + params); return undefined; } } diff --git a/src/middleware/logger.js b/src/middleware/logger.js index 23ffb16..32e888b 100644 --- a/src/middleware/logger.js +++ b/src/middleware/logger.js @@ -1,7 +1,6 @@ -var fs = require('fs'); -var config = require('../config.js'); +const log = require('../utils/logger.js'); // log not logger to not interfere with function name module.exports = function logger (req, res, next) { - (config.mode === "development") && console.log('Request recieved: ' + req.url); + log.info('Request recieved: ' + req.url); next(); } \ No newline at end of file diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index be91056..ba23262 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -5,6 +5,7 @@ var databases = require('../databases/databases.js'); var db = databases.db; var privateDB = databases.privateDB; +var logger = require('../utils/logger.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); @@ -166,7 +167,7 @@ function handleGetSegments(req, res) { return segments; } catch (error) { - console.error(error); + logger.error(error); res.sendStatus(500); return false; diff --git a/src/routes/getUsername.js b/src/routes/getUsername.js index 3ac4bd2..d04b8ee 100644 --- a/src/routes/getUsername.js +++ b/src/routes/getUsername.js @@ -1,6 +1,7 @@ var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); +const logger = require('../utils/logger.js'); module.exports = function getUsername (req, res) { let userID = req.query.userID; @@ -28,7 +29,7 @@ module.exports = function getUsername (req, res) { }); } } catch (err) { - console.log(err); + logger.error(err); res.sendStatus(500); return; diff --git a/src/routes/getViewsForUser.js b/src/routes/getViewsForUser.js index 013a201..79d8961 100644 --- a/src/routes/getViewsForUser.js +++ b/src/routes/getViewsForUser.js @@ -1,6 +1,6 @@ var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); - +var logger = require('../utils/logger.js'); module.exports = function getViewsForUser(req, res) { let userID = req.query.userID; @@ -25,7 +25,7 @@ module.exports = function getViewsForUser(req, res) { res.sendStatus(404); } } catch (err) { - console.log(err); + logger.error(err); res.sendStatus(500); return; diff --git a/src/routes/oldSubmitSponsorTimes.js b/src/routes/oldSubmitSponsorTimes.js index b818a24..613ec9c 100644 --- a/src/routes/oldSubmitSponsorTimes.js +++ b/src/routes/oldSubmitSponsorTimes.js @@ -1,5 +1,3 @@ -var config = require('../config.js'); - var postSkipSegments = require('./postSkipSegments.js'); module.exports = async function submitSponsorTimes(req, res) { diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index eb8d940..1dbaf21 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -4,6 +4,7 @@ var databases = require('../databases/databases.js'); var db = databases.db; var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); +var logger = require('../utils/logger.js'); var request = require('request'); var isoDurations = require('iso8601-duration'); @@ -25,7 +26,7 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { id: videoID }, function (err, data) { if (err || data.items.length === 0) { - err && console.log(err); + err && logger.error(err); return; } @@ -52,13 +53,13 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { } }, (err, res) => { if (err) { - console.log("Failed to send first time submission Discord hook."); - console.log(JSON.stringify(err)); - console.log("\n"); + logger.error("Failed to send first time submission Discord hook."); + logger.error(JSON.stringify(err)); + logger.error("\n"); } else if (res && res.statusCode >= 400) { - console.log("Error sending first time submission Discord hook"); - console.log(JSON.stringify(res)); - console.log("\n"); + logger.error("Error sending first time submission Discord hook"); + logger.error(JSON.stringify(res)); + logger.error("\n"); } }); }); @@ -105,7 +106,7 @@ async function autoModerateSubmission(submission, callback) { } } else { - if (config.mode === 'development') console.log("Skipped YouTube API"); + logger.debug("Skipped YouTube API"); // Can't moderate the submission without calling the youtube API // so allow by default. @@ -117,9 +118,9 @@ function proxySubmission(req) { request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => { if (config.mode === 'development') { if (!err) { - console.log('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); + logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); } else { - console.log("Proxy Submission: Failed to make call"); + logger.debug("Proxy Submission: Failed to make call"); } } }); @@ -264,7 +265,7 @@ module.exports = async function postSkipSegments(req, res) { } catch (err) { //a DB change probably occurred res.sendStatus(502); - console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + + logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err); return; @@ -274,7 +275,7 @@ module.exports = async function postSkipSegments(req, res) { sendDiscordNotification(userID, videoID, UUID, segmentInfo); } } catch (err) { - console.error(err); + logger.error(err); res.sendStatus(500); diff --git a/src/routes/setUsername.js b/src/routes/setUsername.js index 09fcb90..cc143fb 100644 --- a/src/routes/setUsername.js +++ b/src/routes/setUsername.js @@ -3,6 +3,7 @@ var config = require('../config.js'); var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); +const logger = require('../utils/logger.js'); module.exports = function setUsername(req, res) { @@ -45,7 +46,7 @@ module.exports = function setUsername(req, res) { res.sendStatus(200); } catch (err) { - console.log(err); + logger.error(err); res.sendStatus(500); return; diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index f664d40..0e9fdf8 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -11,6 +11,7 @@ var db = databases.db; var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); var request = require('request'); +const logger = require('../utils/logger.js'); function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // Check if they've already made a vote @@ -203,7 +204,7 @@ async function voteOnSponsorTime(req, res) { id: submissionInfoRow.videoID }, function (err, data) { if (err || data.items.length === 0) { - err && console.log(err); + err && logger.error(err); return; } @@ -232,13 +233,13 @@ async function voteOnSponsorTime(req, res) { } }, (err, res) => { if (err) { - console.log("Failed to send reported submission Discord hook."); - console.log(JSON.stringify(err)); - console.log("\n"); + logger.error("Failed to send reported submission Discord hook."); + logger.error(JSON.stringify(err)); + logger.error("\n"); } else if (res && res.statusCode >= 400) { - console.log("Error sending reported submission Discord hook"); - console.log(JSON.stringify(res)); - console.log("\n"); + logger.error("Error sending reported submission Discord hook"); + logger.error(JSON.stringify(res)); + logger.error("\n"); } }); }); @@ -299,7 +300,7 @@ async function voteOnSponsorTime(req, res) { res.sendStatus(200); } catch (err) { - console.error(err); + logger.error(err); res.status(500).json({error: 'Internal error creating segment vote'}); } diff --git a/src/utils/logger.js b/src/utils/logger.js new file mode 100644 index 0000000..70fe13e --- /dev/null +++ b/src/utils/logger.js @@ -0,0 +1,36 @@ +const config = require('../config.js'); + +const levels = { + ERROR: "ERROR", + WARN: "WARN", + INFO: "INFO", + DEBUG: "DEBUG" +}; + +const settings = { + ERROR: true, + WARN: true, + INFO: false, + DEBUG: false +}; + +if (config.mode === 'development') { + settings.INFO = true; + settings.DEBUG = true; +} + +function log(level, string) { + if (!!settings[level]) { + if (level.length === 4) {level = level + " "}; // ensure logs are aligned + console.log(level + " " + new Date().toISOString() + " : " + string); + } +} + +module.exports = { + levels, + log, + error: (string) => {log(levels.ERROR, string)}, + warn: (string) => {log(levels.WARN, string)}, + info: (string) => {log(levels.INFO, string)}, + debug: (string) => {log(levels.DEBUG, string)}, +}; \ No newline at end of file From 16c68dd51d5f1066a925438e1726190987f99a3a Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 21 Aug 2020 22:32:34 +0100 Subject: [PATCH 16/20] allow removing own segments with a downvote --- index.js | 3 ++- src/databases/Mysql.js | 2 +- src/routes/voteOnSponsorTime.js | 7 ++++-- test/cases/voteOnSponsorTime.js | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 508c089..a877bab 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,6 @@ var config = require('./src/config.js'); var createServer = require('./src/app.js'); +const logger = require('./src/utils/logger.js'); var server = createServer(() => { - console.log("Server started on port " + config.port + "."); + logger.info("Server started on port " + config.port + "."); }); diff --git a/src/databases/Mysql.js b/src/databases/Mysql.js index 40e3d76..8f3d8ba 100644 --- a/src/databases/Mysql.js +++ b/src/databases/Mysql.js @@ -1,6 +1,6 @@ var MysqlInterface = require('sync-mysql'); var config = require('../config.js'); -var logger = require('../utils/logger.js'); +const logger = require('../utils/logger.js'); class Mysql { constructor(msConfig) { diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 0e9fdf8..1e55ed9 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -97,6 +97,9 @@ async function voteOnSponsorTime(req, res) { //check if this user is on the vip list let isVIP = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [nonAnonUserID]).userCount > 0; + //check if user voting on own submission + let isOwnSubmission = !!db.prepare('all', 'SELECT UUID as submissionCount FROM sponsorTimes where userID = ? AND UUID = ?', [nonAnonUserID, UUID]).length; + if (type === undefined && category !== undefined) { return categoryVote(UUID, userID, isVIP, category, hashedIP, res); } @@ -166,13 +169,13 @@ async function voteOnSponsorTime(req, res) { let row = db.prepare('get', "SELECT votes, views FROM sponsorTimes WHERE UUID = ?", [UUID]); if (voteTypeEnum === voteTypes.normal) { - if (isVIP && incrementAmount < 0) { + if ((isVIP || isOwnSubmission) && incrementAmount < 0) { //this user is a vip and a downvote incrementAmount = - (row.votes + 2 - oldIncrementAmount); type = incrementAmount; } } else if (voteTypeEnum == voteTypes.incorrect) { - if (isVIP) { + if (isVIP || isOwnSubmission) { //this user is a vip and a downvote incrementAmount = 500 * incrementAmount; type = incrementAmount < 0 ? 12 : 13; diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index 1ce4b7c..bcbca78 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -20,6 +20,8 @@ describe('voteOnSponsorTime', () => { db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0)"); db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0)"); db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); @@ -151,6 +153,42 @@ describe('voteOnSponsorTime', () => { }); }); + it('should be able to completely downvote your own segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=own-submission-id&UUID=own-submission-uuid&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["own-submission-uuid"]); + if (row.votes <= -2) { + done() + } else { + done("Vote did not succeed. Submission went from 500 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('should not be able to completely downvote somebody elses segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=not-own-submission-uuid&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["not-own-submission-uuid"]); + if (row.votes === 499) { + done() + } else { + done("Vote did not succeed. Submission went from 500 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + it('Should be able to vote for a category and it should immediately change (for now)', (done) => { request.get(utils.getbaseURL() + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=intro", null, From fba25fea0fe68d9db7ec8715fbe3c51866fd2e5d Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 20:45:31 +0100 Subject: [PATCH 17/20] Update src/routes/voteOnSponsorTime.js Co-authored-by: Ajay Ramachandran --- src/routes/voteOnSponsorTime.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 1e55ed9..3841390 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -98,7 +98,7 @@ async function voteOnSponsorTime(req, res) { let isVIP = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [nonAnonUserID]).userCount > 0; //check if user voting on own submission - let isOwnSubmission = !!db.prepare('all', 'SELECT UUID as submissionCount FROM sponsorTimes where userID = ? AND UUID = ?', [nonAnonUserID, UUID]).length; + let isOwnSubmission = db.prepare("get", "SELECT UUID as submissionCount FROM sponsorTimes where userID = ? AND UUID = ?", [nonAnonUserID, UUID]) !== undefined; if (type === undefined && category !== undefined) { return categoryVote(UUID, userID, isVIP, category, hashedIP, res); @@ -314,4 +314,4 @@ module.exports = { endpoint: function (req, res) { voteOnSponsorTime(req, res); }, - }; \ No newline at end of file + }; From 17649157c10a70b288d59abad3b9dab6b8d5cfbd Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 20:53:32 +0100 Subject: [PATCH 18/20] changed dockler default config to use sqlite --- entrypoint.sh | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 5e14a9e..6215525 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,33 +5,20 @@ cd /usr/src/app cp /etc/sponsorblock/config.json . || cat < config.json { "port": 8080, - "mysql": { - "host": "127.0.0.1", - "port": 3306, - "database": "sponsorblock", - "user": "sponsorblock", - "password": "sponsorblock" - }, - "privateMysql": { - "host": "127.0.0.1", - "port": 3306, - "database": "sponsorblock_private", - "user": "sponsorblock", - "password": "sponsorblock" - }, - "globalSalt": "", - "adminUserID": "", - "youtubeAPIKey": "", + "globalSalt": "[CHANGE THIS]", + "adminUserID": "[CHANGE THIS]", + "youtubeAPIKey": null, "discordReportChannelWebhookURL": null, "discordFirstTimeSubmissionsWebhookURL": null, "discordAutoModWebhookURL": null, - "behindProxy": true, - "db": null, - "privateDB": null, + "proxySubmission": null, + "behindProxy": "X-Forwarded-For", + "db": "./databases/sponsorTimes.db", + "privateDB": "./databases/private.db", "createDatabaseIfNotExist": true, - "schemaFolder": null, - "dbSchema": null, - "privateDBSchema": null, + "schemaFolder": "./databases", + "dbSchema": "./databases/_sponsorTimes.db.sql", + "privateDBSchema": "./databases/_private.db.sql", "mode": "development", "readOnly": false } From 9cebb8769f34368f499bb7282c0a402877b0da41 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 Aug 2020 11:34:07 -0400 Subject: [PATCH 19/20] Add author name to discord notification --- src/routes/voteOnSponsorTime.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 3841390..f2cf152 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -13,6 +13,18 @@ var YouTubeAPI = require('../utils/youtubeAPI.js'); var request = require('request'); const logger = require('../utils/logger.js'); +function getVoteAuthor(submissionCount, isVIP, isOwnSubmission) { + if (submissionCount === 0) { + return "Report by New User"; + } else if (isVIP) { + return "Report by VIP User"; + } else if (isOwnSubmission) { + return "Report by Submitter"; + } + + return ""; +} + function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // Check if they've already made a vote let previousVoteInfo = privateDB.prepare('get', "select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?", [UUID, userID]); @@ -227,7 +239,7 @@ async function voteOnSponsorTime(req, res) { getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), "color": 10813440, "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (isVIP ? "Report by VIP User" : "") + "name": getVoteAuthor(userSubmissionCountRow.submissionCount, isVIP, isOwnSubmission) }, "thumbnail": { "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", From ece475076e4577ac210f6b9f0b4922654837e7eb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 Aug 2020 12:35:57 -0400 Subject: [PATCH 20/20] Added log colors and method name --- src/middleware/logger.js | 2 +- src/utils/logger.js | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/middleware/logger.js b/src/middleware/logger.js index 32e888b..43a0c36 100644 --- a/src/middleware/logger.js +++ b/src/middleware/logger.js @@ -1,6 +1,6 @@ const log = require('../utils/logger.js'); // log not logger to not interfere with function name module.exports = function logger (req, res, next) { - log.info('Request recieved: ' + req.url); + log.info("Request recieved: " + req.method + " " + req.url); next(); } \ No newline at end of file diff --git a/src/utils/logger.js b/src/utils/logger.js index 70fe13e..dd3f62c 100644 --- a/src/utils/logger.js +++ b/src/utils/logger.js @@ -7,6 +7,34 @@ const levels = { DEBUG: "DEBUG" }; +const colors = { + Reset: "\x1b[0m", + Bright: "\x1b[1m", + Dim: "\x1b[2m", + Underscore: "\x1b[4m", + Blink: "\x1b[5m", + Reverse: "\x1b[7m", + Hidden: "\x1b[8m", + + FgBlack: "\x1b[30m", + FgRed: "\x1b[31m", + FgGreen: "\x1b[32m", + FgYellow: "\x1b[33m", + FgBlue: "\x1b[34m", + FgMagenta: "\x1b[35m", + FgCyan: "\x1b[36m", + FgWhite: "\x1b[37m", + + BgBlack: "\x1b[40m", + BgRed: "\x1b[41m", + BgGreen: "\x1b[42m", + BgYellow: "\x1b[43m", + BgBlue: "\x1b[44m", + BgMagenta: "\x1b[45m", + BgCyan: "\x1b[46m", + BgWhite: "\x1b[47m", +} + const settings = { ERROR: true, WARN: true, @@ -21,8 +49,12 @@ if (config.mode === 'development') { function log(level, string) { if (!!settings[level]) { + let color = colors.Bright; + if (level === levels.ERROR) color = colors.FgRed; + if (level === levels.WARN) color = colors.FgYellow; + if (level.length === 4) {level = level + " "}; // ensure logs are aligned - console.log(level + " " + new Date().toISOString() + " : " + string); + console.log(colors.Dim, level + " " + new Date().toISOString() + ": ", color, string, colors.Reset); } }