From d767f2ff6bf79faa113941baa9b8f701e3bb56e0 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 27 Apr 2020 23:01:51 -0400 Subject: [PATCH 01/13] Added schema upgrade system and started on new vote type --- config.json.example | 2 + databases/_sponsorTimes.db.sql | 5 + databases/_upgrade_0.sql | 25 +++ src/databases/databases.js | 7 + src/routes/voteOnSponsorTime.js | 298 ++++++++++++++++------------ test.json | 6 +- test/databases/_private.db.sql | 18 -- test/databases/_sponsorTimes.db.sql | 23 --- test/mocks.js | 4 + 9 files changed, 216 insertions(+), 172 deletions(-) create mode 100644 databases/_upgrade_0.sql delete mode 100644 test/databases/_private.db.sql delete mode 100644 test/databases/_sponsorTimes.db.sql diff --git a/config.json.example b/config.json.example index b8a9d79..ccda7e2 100644 --- a/config.json.example +++ b/config.json.example @@ -7,10 +7,12 @@ "youtubeAPIKey": null, //get this from Google Cloud Platform [optional] "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] + "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] "behindProxy": true, "db": "./databases/sponsorTimes.db", "privateDB": "./databases/private.db", "createDatabaseIfNotExist": true, //This will run on startup every time (unless readOnly is true) - so ensure "create table if not exists" is used in the schema + "schemaFolder": "./databases", "dbSchema": "./databases/_sponsorTimes.db.sql", "privateDBSchema": "./databases/_private.db.sql", "mode": "development", diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index dfca8bf..1fd0f29 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -18,6 +18,11 @@ CREATE TABLE IF NOT EXISTS "userNames" ( "userID" TEXT NOT NULL, "userName" TEXT NOT NULL ); + +CREATE TABLE IF NOT EXISTS "version" ( + "code" INTEGER NOT NULL default '0' +); + CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID); CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID); COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_0.sql b/databases/_upgrade_0.sql new file mode 100644 index 0000000..4f747bc --- /dev/null +++ b/databases/_upgrade_0.sql @@ -0,0 +1,25 @@ +BEGIN TRANSACTION; + +/* Add incorrectVotes field */ +CREATE TABLE "sqlb_temp_table_1" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT "sponsor", + "shadowHidden" INTEGER NOT NULL +); +INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes; + +DROP TABLE sponsorTimes; +ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; + +/* Increase version number */ +INSERT INTO version VALUES(1); + +COMMIT; \ No newline at end of file diff --git a/src/databases/databases.js b/src/databases/databases.js index d01d01f..8b4b407 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -15,6 +15,13 @@ if (config.createDatabaseIfNotExist && !config.readOnly) { if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString()); } +// Upgrade database if required +let versionCode = db.prepare("SELECT code FROM version").get() || 0; +let path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; +if (fs.existsSync(path)) { + db.exec(fs.readFileSync(path).toString()); +} + // Enable WAL mode checkpoint number if (!config.readOnly && config.mode === "production") { db.exec("PRAGMA journal_mode=WAL;"); diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index bc9a326..32ed465 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -11,114 +11,147 @@ var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); var request = require('request'); +function completelyIncorrectVote(req, res, params) { + + +} + module.exports = async function voteOnSponsorTime(req, res) { - let UUID = req.query.UUID; - let userID = req.query.userID; - let type = req.query.type; + let UUID = req.query.UUID; + let userID = req.query.userID; + let type = req.query.type; - if (UUID == undefined || userID == undefined || type == undefined) { - //invalid request - res.sendStatus(400); - return; - } + if (UUID == undefined || userID == undefined || type == undefined) { + //invalid request + res.sendStatus(400); + return; + } - //hash the userID - let nonAnonUserID = getHash(userID); - userID = getHash(userID + UUID); + //hash the userID + let nonAnonUserID = getHash(userID); + userID = getHash(userID + UUID); - //x-forwarded-for if this server is behind a proxy - let ip = getIP(req); + //x-forwarded-for if this server is behind a proxy + let ip = getIP(req); - //hash the ip 5000 times so no one can get it from the database - let hashedIP = getHash(ip + config.globalSalt); + //hash the ip 5000 times so no one can get it from the database + let hashedIP = getHash(ip + config.globalSalt); - try { - //check if vote has already happened - let votesRow = privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID); - - //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future - let incrementAmount = 0; - let oldIncrementAmount = 0; + let voteTypes = { + normal: 0, + incorrect: 1 + } - if (type == 1) { - //upvote - incrementAmount = 1; - } else if (type == 0) { - //downvote - incrementAmount = -1; - } else { - //unrecongnised type of vote - res.sendStatus(400); - return; - } - if (votesRow != undefined) { - if (votesRow.type == 1) { - //upvote - oldIncrementAmount = 1; - } else if (votesRow.type == 0) { - //downvote - oldIncrementAmount = -1; - } else if (votesRow.type == 2) { - //extra downvote - oldIncrementAmount = -4; - } else if (votesRow.type < 0) { - //vip downvote - oldIncrementAmount = votesRow.type; - } - } + let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect; - //check if this user is on the vip list - let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID); + try { + //check if vote has already happened + let votesRow = privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID); + + //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future + let incrementAmount = 0; + let oldIncrementAmount = 0; - //check if the increment amount should be multiplied (downvotes have more power if there have been many views) - let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID); - - if (vipRow.userCount != 0 && incrementAmount < 0) { - //this user is a vip and a downvote - incrementAmount = - (row.votes + 2 - oldIncrementAmount); - type = incrementAmount; - } else if (row !== undefined && (row.votes > 8 || row.views > 15) && incrementAmount < 0) { - //increase the power of this downvote - incrementAmount = -Math.abs(Math.min(10, row.votes + 2 - oldIncrementAmount)); - type = incrementAmount; - } + if (type == 1 || type == 11) { + //upvote + incrementAmount = 1; + } else if (type == 0 || type == 10) { + //downvote + incrementAmount = -1; + } else { + //unrecongnised type of vote + res.sendStatus(400); + return; + } + if (votesRow != undefined) { + if (votesRow.type == 1 || type == 11) { + //upvote + oldIncrementAmount = 1; + } else if (votesRow.type == 0 || type == 10) { + //downvote + oldIncrementAmount = -1; + } else if (votesRow.type == 2) { + //extra downvote + oldIncrementAmount = -4; + } else if (votesRow.type < 0) { + //vip downvote + oldIncrementAmount = votesRow.type; + } else if (votesRow.type == 12) { + // VIP downvote for completely incorrect + oldIncrementAmount = -500; + } else if (votesRow.type == 13) { + // VIP upvote for completely incorrect + oldIncrementAmount = 500; + } + } - // Send discord message - if (type != 1) { - // Get video ID - let submissionInfoRow = db.prepare("SELECT videoID, userID, startTime, endTime FROM sponsorTimes WHERE UUID = ?").get(UUID); + //check if this user is on the vip list + let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID); - let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID); + //check if the increment amount should be multiplied (downvotes have more power if there have been many views) + let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID); + + if (voteTypeEnum == voteTypes.normal) { + if (vipRow.userCount != 0 && incrementAmount < 0) { + //this user is a vip and a downvote + incrementAmount = - (row.votes + 2 - oldIncrementAmount); + type = incrementAmount; + } else if (row !== undefined && (row.votes > 8 || row.views > 15) && incrementAmount < 0) { + //increase the power of this downvote + incrementAmount = -Math.abs(Math.min(10, row.votes + 2 - oldIncrementAmount)); + type = incrementAmount; + } + } else if (voteTypeEnum == voteTypes.incorrect) { + if (vipRow.userCount != 0) { + //this user is a vip and a downvote + incrementAmount = 500 * incrementAmount; + type = incrementAmount < 0 ? 12 : 13; + } + } - if (config.youtubeAPIKey !== null && config.discordReportChannelWebhookURL !== null) { - YouTubeAPI.videos.list({ - part: "snippet", - id: submissionInfoRow.videoID - }, function (err, data) { - if (err || data.items.length === 0) { - err && console.log(err); - return; - } - - request.post(config.discordReportChannelWebhookURL, { - json: { - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + - "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), - "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + - " Views**\n\nSubmission ID: " + UUID + - "\n\nSubmitted by: " + submissionInfoRow.userID + "\n\nTimestamp: " + - getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), - "color": 10813440, - "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "") - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - } - }] - } + // Send discord message + if (incrementAmount < 0) { + // Get video ID + let submissionInfoRow = db.prepare("SELECT videoID, userID, startTime, endTime FROM sponsorTimes WHERE UUID = ?").get(UUID); + + let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID); + + let webhookURL = null; + if (voteTypeEnum === voteTypes.normal) { + webhookURL = config.discordReportChannelWebhookURL; + } else if (voteTypeEnum === voteTypes.incorrect) { + webhookURL = config.discordCompletelyIncorrectReportWebhookURL; + } + + if (config.youtubeAPIKey !== null && webhookURL !== null) { + YouTubeAPI.videos.list({ + part: "snippet", + id: submissionInfoRow.videoID + }, function (err, data) { + if (err || data.items.length === 0) { + err && console.log(err); + return; + } + + request.post(webhookURL, { + json: { + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), + "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + + " Views**\n\nSubmission ID: " + UUID + + "\n\nSubmitted by: " + submissionInfoRow.userID + "\n\nTimestamp: " + + getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), + "color": 10813440, + "author": { + "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "") + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + } + }] + } }, (err, res) => { if (err) { console.log("Failed to send reported submission Discord hook."); @@ -129,43 +162,50 @@ module.exports = async function voteOnSponsorTime(req, res) { console.log(JSON.stringify(res)); console.log("\n"); } - }); - }); - } - } + }); + }); + } + } - //update the votes table - if (votesRow != undefined) { - privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID); - } else { - privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type); - } + //update the votes table + if (votesRow != undefined) { + privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID); + } else { + privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type); + } - //update the vote count on this sponsorTime - //oldIncrementAmount will be zero is row is null - db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); + let tableName = ""; + if (voteTypeEnum === voteTypes.normal) { + tableName = "votes"; + } else if (voteTypeEnum === voteTypes.incorrect) { + tableName = "incorrectVotes"; + } - //for each positive vote, see if a hidden submission can be shown again - if (incrementAmount > 0) { - //find the UUID that submitted the submission that was voted on - let submissionUserID = db.prepare("SELECT userID FROM sponsorTimes WHERE UUID = ?").get(UUID).userID; + //update the vote count on this sponsorTime + //oldIncrementAmount will be zero is row is null + db.prepare("UPDATE sponsorTimes SET " + tableName + " += ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); - //check if any submissions are hidden - let hiddenSubmissionsRow = db.prepare("SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0").get(submissionUserID); + //for each positive vote, see if a hidden submission can be shown again + if (incrementAmount > 0) { + //find the UUID that submitted the submission that was voted on + let submissionUserID = db.prepare("SELECT userID FROM sponsorTimes WHERE UUID = ?").get(UUID).userID; - if (hiddenSubmissionsRow.hiddenSubmissions > 0) { - //see if some of this users submissions should be visible again - - if (await isUserTrustworthy(submissionUserID)) { - //they are trustworthy again, show 2 of their submissions again, if there are two to show - db.prepare("UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)").run(submissionUserID) - } - } - } + //check if any submissions are hidden + let hiddenSubmissionsRow = db.prepare("SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0").get(submissionUserID); - //added to db - res.sendStatus(200); - } catch (err) { - console.error(err); - } + if (hiddenSubmissionsRow.hiddenSubmissions > 0) { + //see if some of this users submissions should be visible again + + if (await isUserTrustworthy(submissionUserID)) { + //they are trustworthy again, show 2 of their submissions again, if there are two to show + db.prepare("UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)").run(submissionUserID) + } + } + } + + //added to db + res.sendStatus(200); + } catch (err) { + console.error(err); + } } \ No newline at end of file diff --git a/test.json b/test.json index f07fbf8..28373a1 100644 --- a/test.json +++ b/test.json @@ -6,12 +6,14 @@ "youtubeAPIKey": "", "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", + "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", "behindProxy": true, "db": "./test/databases/sponsorTimes.db", "privateDB": "./test/databases/private.db", "createDatabaseIfNotExist": true, - "dbSchema": "./test/databases/_sponsorTimes.db.sql", - "privateDBSchema": "./test/databases/_private.db.sql", + "schemaFolder": "./databases", + "dbSchema": "./databases/_sponsorTimes.db.sql", + "privateDBSchema": "./databases/_private.db.sql", "mode": "test", "readOnly": false } diff --git a/test/databases/_private.db.sql b/test/databases/_private.db.sql deleted file mode 100644 index b93aaee..0000000 --- a/test/databases/_private.db.sql +++ /dev/null @@ -1,18 +0,0 @@ -BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "shadowBannedUsers" ( - "userID" TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS "votes" ( - "UUID" TEXT NOT NULL, - "userID" INTEGER NOT NULL, - "hashedIP" INTEGER NOT NULL, - "type" INTEGER NOT NULL -); -CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "hashedIP" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL -); -CREATE INDEX IF NOT EXISTS sponsorTimes_hashedIP on sponsorTimes(hashedIP); -CREATE INDEX IF NOT EXISTS votes_userID on votes(UUID); -COMMIT; diff --git a/test/databases/_sponsorTimes.db.sql b/test/databases/_sponsorTimes.db.sql deleted file mode 100644 index dfca8bf..0000000 --- a/test/databases/_sponsorTimes.db.sql +++ /dev/null @@ -1,23 +0,0 @@ -BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "vipUsers" ( - "userID" TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "category" TEXT NOT NULL, - "shadowHidden" INTEGER NOT NULL -); -CREATE TABLE IF NOT EXISTS "userNames" ( - "userID" TEXT NOT NULL, - "userName" TEXT NOT NULL -); -CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID); -CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID); -COMMIT; \ No newline at end of file diff --git a/test/mocks.js b/test/mocks.js index 49060af..b58a47f 100644 --- a/test/mocks.js +++ b/test/mocks.js @@ -11,6 +11,10 @@ app.post('/FirstTimeSubmissionsWebhook', (req, res) => { res.sendStatus(200); }); +app.post('/CompletelyIncorrectReportWebhook', (req, res) => { + res.sendStatus(200); +}); + module.exports = function createMockServer(callback) { return app.listen(config.mockPort, callback); } \ No newline at end of file From cde51a30594e639b67734a3e10555b226e4e38cd Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 27 Apr 2020 23:02:29 -0400 Subject: [PATCH 02/13] Only unhide submissions with normal vote --- src/routes/voteOnSponsorTime.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 32ed465..6cffbdf 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -186,7 +186,7 @@ module.exports = async function voteOnSponsorTime(req, res) { db.prepare("UPDATE sponsorTimes SET " + tableName + " += ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); //for each positive vote, see if a hidden submission can be shown again - if (incrementAmount > 0) { + if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { //find the UUID that submitted the submission that was voted on let submissionUserID = db.prepare("SELECT userID FROM sponsorTimes WHERE UUID = ?").get(UUID).userID; From e27ef39dcf11d3cf8cd313b2c5ca2f7a50a93af8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 18:26:05 -0400 Subject: [PATCH 03/13] Remove unused function --- src/routes/voteOnSponsorTime.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 6cffbdf..87fc97a 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -11,11 +11,6 @@ var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); var request = require('request'); -function completelyIncorrectVote(req, res, params) { - - -} - module.exports = async function voteOnSponsorTime(req, res) { let UUID = req.query.UUID; let userID = req.query.userID; From 0eec924c0201a58dd5190e3f1c321d4fcee1335d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 18:31:19 -0400 Subject: [PATCH 04/13] Made all updates required be performed at once. --- src/databases/databases.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/databases/databases.js b/src/databases/databases.js index 8b4b407..ab3d8a3 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -18,8 +18,11 @@ if (config.createDatabaseIfNotExist && !config.readOnly) { // Upgrade database if required let versionCode = db.prepare("SELECT code FROM version").get() || 0; let path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; -if (fs.existsSync(path)) { +while (fs.existsSync(path)) { db.exec(fs.readFileSync(path).toString()); + + versionCode = db.prepare("SELECT code FROM version").get(); + path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; } // Enable WAL mode checkpoint number From b5fcdea62f7bc7b15f059d633428cfc3272fd9f0 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 19:40:17 -0400 Subject: [PATCH 05/13] Don't upgrade if read only --- databases/_upgrade_1.sql | 25 +++++++++++++++++++++++++ src/databases/databases.js | 14 ++++++++------ 2 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 databases/_upgrade_1.sql diff --git a/databases/_upgrade_1.sql b/databases/_upgrade_1.sql new file mode 100644 index 0000000..4f747bc --- /dev/null +++ b/databases/_upgrade_1.sql @@ -0,0 +1,25 @@ +BEGIN TRANSACTION; + +/* Add incorrectVotes field */ +CREATE TABLE "sqlb_temp_table_1" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT "sponsor", + "shadowHidden" INTEGER NOT NULL +); +INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes; + +DROP TABLE sponsorTimes; +ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; + +/* Increase version number */ +INSERT INTO version VALUES(1); + +COMMIT; \ No newline at end of file diff --git a/src/databases/databases.js b/src/databases/databases.js index ab3d8a3..73b0897 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -16,13 +16,15 @@ if (config.createDatabaseIfNotExist && !config.readOnly) { } // Upgrade database if required -let versionCode = db.prepare("SELECT code FROM version").get() || 0; -let path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; -while (fs.existsSync(path)) { - db.exec(fs.readFileSync(path).toString()); +if (!config.readOnly) { + let versionCode = db.prepare("SELECT code FROM version").get() || 0; + let path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; + while (fs.existsSync(path)) { + db.exec(fs.readFileSync(path).toString()); - versionCode = db.prepare("SELECT code FROM version").get(); - path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; + versionCode = db.prepare("SELECT code FROM version").get(); + path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; + } } // Enable WAL mode checkpoint number From 98f4d973e7b2445ca853e054cd32ca67dc14e33d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 20:56:34 -0400 Subject: [PATCH 06/13] Added wrong category vote option. --- databases/_private.db.sql | 16 +++++++- databases/_sponsorTimes.db.sql | 7 ++++ databases/_upgrade_1.sql | 25 ----------- src/routes/voteOnSponsorTime.js | 73 ++++++++++++++++++++++++++++----- 4 files changed, 83 insertions(+), 38 deletions(-) delete mode 100644 databases/_upgrade_1.sql diff --git a/databases/_private.db.sql b/databases/_private.db.sql index b93aaee..6db6afd 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -1,18 +1,30 @@ BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "shadowBannedUsers" ( "userID" TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS "votes" ( "UUID" TEXT NOT NULL, - "userID" INTEGER NOT NULL, - "hashedIP" INTEGER NOT NULL, + "userID" TEXT NOT NULL, + "hashedIP" TEXT NOT NULL, "type" INTEGER NOT NULL ); + +CREATE TABLE IF NOT EXISTS "categoryVotes" ( + "UUID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "hashedIP" TEXT NOT NULL, + "category" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, +); + CREATE TABLE IF NOT EXISTS "sponsorTimes" ( "videoID" TEXT NOT NULL, "hashedIP" TEXT NOT NULL, "timeSubmitted" INTEGER NOT NULL ); + CREATE INDEX IF NOT EXISTS sponsorTimes_hashedIP on sponsorTimes(hashedIP); CREATE INDEX IF NOT EXISTS votes_userID on votes(UUID); + COMMIT; diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 1fd0f29..3924c6e 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -1,4 +1,5 @@ BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "vipUsers" ( "userID" TEXT NOT NULL ); @@ -18,6 +19,11 @@ CREATE TABLE IF NOT EXISTS "userNames" ( "userID" TEXT NOT NULL, "userName" TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS "categoryVotes" ( + "UUID" TEXT NOT NULL UNIQUE, + "category" TEXT NOT NULL, + "votes" INTEGER NOT NULL default '0' +); CREATE TABLE IF NOT EXISTS "version" ( "code" INTEGER NOT NULL default '0' @@ -25,4 +31,5 @@ CREATE TABLE IF NOT EXISTS "version" ( CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID); CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID); + COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_1.sql b/databases/_upgrade_1.sql deleted file mode 100644 index 4f747bc..0000000 --- a/databases/_upgrade_1.sql +++ /dev/null @@ -1,25 +0,0 @@ -BEGIN TRANSACTION; - -/* Add incorrectVotes field */ -CREATE TABLE "sqlb_temp_table_1" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "incorrectVotes" INTEGER NOT NULL default '1', - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "category" TEXT NOT NULL DEFAULT "sponsor", - "shadowHidden" INTEGER NOT NULL -); -INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes; - -DROP TABLE sponsorTimes; -ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; - -/* Increase version number */ -INSERT INTO version VALUES(1); - -COMMIT; \ No newline at end of file diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 5fb2c74..94ac13f 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -11,12 +11,59 @@ var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); var request = require('request'); +function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { + // Check if they've already made a vote + let previousVoteInfo = privateDB.prepare("select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?").get(UUID, userID, category); + + if (previousVoteInfo > 0 && previousVoteInfo.category === category) { + // Double vote, ignore + res.sendStatus(200); + return; + } + + let timeSubmitted = Date.now(); + + // Add the vote + if (db.prepare("select count(*) as count from categoryVotes where UUID = ? and category = ?").get(UUID, category).count > 0) { + // Update the already existing db entry + db.prepare("update categoryVotes set count += 1 where UUID = ? and category = ?").run(UUID, category); + } else { + // Add a db entry + db.prepare("insert into categoryVotes (UUID, category, count) values (?, ?, 1)").run(UUID, category); + } + + // Add the info into the private db + if (previousVoteInfo > 0) { + // Reverse the previous vote + db.prepare("update categoryVotes set votes -= 1 where UUID = ? and category = ?").run(UUID, previousVoteInfo.category); + + privateDB.prepare("update categoryVotes set category = ?, timeSubmitted = ?, hashedIP = ?").run(category, timeSubmitted, hashedIP) + } else { + privateDB.prepare("insert into categoryVotes (UUID, userID, hashedIP, category, timeSubmitted) values (?, ?, ?, ?, ?)").run(UUID, userID, hashedIP, category, timeSubmitted); + } + + // See if the submissions categort is ready to change + let currentCategory = db.prepare("select category from sponsorTimes where UUID = ?").get(UUID); + // Change this value from 1 in the future to make it harder to change categories + let currentCategoryCount = db.prepare("select votes from categoryVotes where UUID = ? and category = ?").get(UUID, currentCategory).votes ?? 1; + let nextCategoryCount = previousVoteInfo.votes ?? 1; + + //TODO: In the future, raise this number from zero to make it harder to change categories + if (nextCategoryCount - currentCategoryCount >= 0) { + // Replace the category + db.prepare("update sponsorTimes set category = ? where UUID = ?").run(category, UUID); + } + + res.sendStatus(200); +} + module.exports = async function voteOnSponsorTime(req, res) { let UUID = req.query.UUID; let userID = req.query.userID; let type = req.query.type; + let category = req.query.category; - if (UUID == undefined || userID == undefined || type == undefined) { + if (UUID === undefined || userID === undefined || (type === undefined && category === undefined)) { //invalid request res.sendStatus(400); return; @@ -32,6 +79,13 @@ module.exports = async function voteOnSponsorTime(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(ip + config.globalSalt); + //check if this user is on the vip list + let isVIP = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID).userCount > 0; + + if (type === undefined && category !== undefined) { + return categoryVote(UUID, userID, isVIP, category, hashedIP, res); + } + let voteTypes = { normal: 0, incorrect: 1 @@ -80,14 +134,11 @@ module.exports = async function voteOnSponsorTime(req, res) { } } - //check if this user is on the vip list - let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID); - //check if the increment amount should be multiplied (downvotes have more power if there have been many views) let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID); if (voteTypeEnum == voteTypes.normal) { - if (vipRow.userCount != 0 && incrementAmount < 0) { + if (isVIP && incrementAmount < 0) { //this user is a vip and a downvote incrementAmount = - (row.votes + 2 - oldIncrementAmount); type = incrementAmount; @@ -97,7 +148,7 @@ module.exports = async function voteOnSponsorTime(req, res) { type = incrementAmount; } } else if (voteTypeEnum == voteTypes.incorrect) { - if (vipRow.userCount != 0) { + if (isVIP) { //this user is a vip and a downvote incrementAmount = 500 * incrementAmount; type = incrementAmount < 0 ? 12 : 13; @@ -147,7 +198,7 @@ module.exports = async function voteOnSponsorTime(req, res) { getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), "color": 10813440, "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "") + "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (isVIP ? "Report by VIP User" : "") }, "thumbnail": { "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", @@ -176,16 +227,16 @@ module.exports = async function voteOnSponsorTime(req, res) { privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type); } - let tableName = ""; + let columnName = ""; if (voteTypeEnum === voteTypes.normal) { - tableName = "votes"; + columnName = "votes"; } else if (voteTypeEnum === voteTypes.incorrect) { - tableName = "incorrectVotes"; + columnName = "incorrectVotes"; } //update the vote count on this sponsorTime //oldIncrementAmount will be zero is row is null - db.prepare("UPDATE sponsorTimes SET " + tableName + " += ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); + db.prepare("UPDATE sponsorTimes SET " + columnName + " += ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); //for each positive vote, see if a hidden submission can be shown again if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { From f23ead56ade096b2b716c0ada896ba4b7b61cb8b Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 21:26:11 -0400 Subject: [PATCH 07/13] Updated inserts to new database schema and removed null coalescing --- databases/_private.db.sql | 2 +- databases/_sponsorTimes.db.sql | 6 +++--- src/routes/postSkipSegments.js | 4 +++- src/routes/voteOnSponsorTime.js | 7 +++++-- test/cases/getSavedTimeForUser.js | 3 ++- test/cases/getSkipSegments.js | 15 ++++++++------- test/cases/oldGetSponsorTime.js | 5 +++-- 7 files changed, 25 insertions(+), 17 deletions(-) diff --git a/databases/_private.db.sql b/databases/_private.db.sql index 6db6afd..0324481 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS "categoryVotes" ( "userID" TEXT NOT NULL, "hashedIP" TEXT NOT NULL, "category" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, + "timeSubmitted" INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS "sponsorTimes" ( diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 3924c6e..a7d5e4d 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -20,9 +20,9 @@ CREATE TABLE IF NOT EXISTS "userNames" ( "userName" TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS "categoryVotes" ( - "UUID" TEXT NOT NULL UNIQUE, - "category" TEXT NOT NULL, - "votes" INTEGER NOT NULL default '0' + "UUID" TEXT NOT NULL UNIQUE, + "category" TEXT NOT NULL, + "votes" INTEGER NOT NULL default '0' ); CREATE TABLE IF NOT EXISTS "version" ( diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index d410568..3c33a83 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -239,7 +239,9 @@ module.exports = async function postSkipSegments(req, res) { segmentInfo.segment[1] + segmentInfo.category + userID, 1); try { - db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], + db.prepare("INSERT INTO sponsorTimes " + + "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); //add to private db as well diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 94ac13f..992512a 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -44,9 +44,12 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // See if the submissions categort is ready to change let currentCategory = db.prepare("select category from sponsorTimes where UUID = ?").get(UUID); + let currentCategoryCount = db.prepare("select votes from categoryVotes where UUID = ? and category = ?").get(UUID, currentCategory).votes; // Change this value from 1 in the future to make it harder to change categories - let currentCategoryCount = db.prepare("select votes from categoryVotes where UUID = ? and category = ?").get(UUID, currentCategory).votes ?? 1; - let nextCategoryCount = previousVoteInfo.votes ?? 1; + // Done this way without ORs incase the value is zero + if (currentCategoryCount === undefined || currentCategoryCount === null) currentCategoryCount = 1; + + let nextCategoryCount = (previousVoteInfo.votes || 0) + 1; //TODO: In the future, raise this number from zero to make it harder to change categories if (nextCategoryCount - currentCategoryCount >= 0) { diff --git a/test/cases/getSavedTimeForUser.js b/test/cases/getSavedTimeForUser.js index e3976c1..5df8a48 100644 --- a/test/cases/getSavedTimeForUser.js +++ b/test/cases/getSavedTimeForUser.js @@ -5,7 +5,8 @@ var getHash = require('../../src/utils/getHash.js'); describe('getSavedTimeForUser', () => { before(() => { - db.exec("INSERT INTO sponsorTimes VALUES ('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; + db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)"); }); it('Should be able to get a 200', (done) => { diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index 21c9f91..5c15c63 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -18,13 +18,14 @@ var utils = require('../utils.js'); describe('getSkipSegments', () => { before(() => { - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; + db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)"); }); diff --git a/test/cases/oldGetSponsorTime.js b/test/cases/oldGetSponsorTime.js index ffb1264..569b109 100644 --- a/test/cases/oldGetSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -19,8 +19,9 @@ var utils = require('../utils.js'); describe('getVideoSponsorTime (Old get method)', () => { before(() => { - db.exec("INSERT INTO sponsorTimes VALUES ('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; + db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); }); it('Should be able to get a time', (done) => { From ad169fd6e7cd4cebff0d0dff7fbd424ba2d4d0ff Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 21:30:07 -0400 Subject: [PATCH 08/13] Added VIP support to category vote --- src/routes/voteOnSponsorTime.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 992512a..c7c1e63 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -23,6 +23,8 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { let timeSubmitted = Date.now(); + let voteAmount = isVIP ? 500 : 1; + // Add the vote if (db.prepare("select count(*) as count from categoryVotes where UUID = ? and category = ?").get(UUID, category).count > 0) { // Update the already existing db entry @@ -52,7 +54,8 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { let nextCategoryCount = (previousVoteInfo.votes || 0) + 1; //TODO: In the future, raise this number from zero to make it harder to change categories - if (nextCategoryCount - currentCategoryCount >= 0) { + // VIPs change it every time + if (nextCategoryCount - currentCategoryCount >= 0 || isVIP) { // Replace the category db.prepare("update sponsorTimes set category = ? where UUID = ?").run(category, UUID); } From 6a17e4d1413d1ce0b7fe53fc424a604be5dae56a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 22:15:16 -0400 Subject: [PATCH 09/13] Added test cases for voting and fixed code to support them. --- databases/_sponsorTimes.db.sql | 2 +- src/routes/voteOnSponsorTime.js | 135 ++++++++++++++++---------------- test/cases/voteOnSponsorTime.js | 130 ++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 67 deletions(-) create mode 100644 test/cases/voteOnSponsorTime.js diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index a7d5e4d..707cb5c 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS "userNames" ( "userName" TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS "categoryVotes" ( - "UUID" TEXT NOT NULL UNIQUE, + "UUID" TEXT NOT NULL, "category" TEXT NOT NULL, "votes" INTEGER NOT NULL default '0' ); diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index c7c1e63..2918d3f 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -13,7 +13,7 @@ var request = require('request'); function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // Check if they've already made a vote - let previousVoteInfo = privateDB.prepare("select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?").get(UUID, userID, category); + let previousVoteInfo = privateDB.prepare("select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?").get(UUID, userID); if (previousVoteInfo > 0 && previousVoteInfo.category === category) { // Double vote, ignore @@ -28,10 +28,10 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // Add the vote if (db.prepare("select count(*) as count from categoryVotes where UUID = ? and category = ?").get(UUID, category).count > 0) { // Update the already existing db entry - db.prepare("update categoryVotes set count += 1 where UUID = ? and category = ?").run(UUID, category); + db.prepare("update categoryVotes set votes = votes + ? where UUID = ? and category = ?").run(voteAmount, UUID, category); } else { // Add a db entry - db.prepare("insert into categoryVotes (UUID, category, count) values (?, ?, 1)").run(UUID, category); + db.prepare("insert into categoryVotes (UUID, category, votes) values (?, ?, ?)").run(UUID, category, voteAmount); } // Add the info into the private db @@ -46,10 +46,11 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // See if the submissions categort is ready to change let currentCategory = db.prepare("select category from sponsorTimes where UUID = ?").get(UUID); - let currentCategoryCount = db.prepare("select votes from categoryVotes where UUID = ? and category = ?").get(UUID, currentCategory).votes; + let currentCategoryInfo = db.prepare("select votes from categoryVotes where UUID = ? and category = ?").get(UUID, currentCategory.category); + // Change this value from 1 in the future to make it harder to change categories // Done this way without ORs incase the value is zero - if (currentCategoryCount === undefined || currentCategoryCount === null) currentCategoryCount = 1; + let currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? 1 : currentCategoryInfo.votes; let nextCategoryCount = (previousVoteInfo.votes || 0) + 1; @@ -119,22 +120,22 @@ module.exports = async function voteOnSponsorTime(req, res) { return; } if (votesRow != undefined) { - if (votesRow.type == 1 || type == 11) { + if (votesRow.type === 1 || type === 11) { //upvote oldIncrementAmount = 1; - } else if (votesRow.type == 0 || type == 10) { + } else if (votesRow.type === 0 || type === 10) { //downvote oldIncrementAmount = -1; - } else if (votesRow.type == 2) { + } else if (votesRow.type === 2) { //extra downvote oldIncrementAmount = -4; } else if (votesRow.type < 0) { //vip downvote oldIncrementAmount = votesRow.type; - } else if (votesRow.type == 12) { + } else if (votesRow.type === 12) { // VIP downvote for completely incorrect oldIncrementAmount = -500; - } else if (votesRow.type == 13) { + } else if (votesRow.type === 13) { // VIP upvote for completely incorrect oldIncrementAmount = 500; } @@ -142,8 +143,8 @@ module.exports = async function voteOnSponsorTime(req, res) { //check if the increment amount should be multiplied (downvotes have more power if there have been many views) let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID); - - if (voteTypeEnum == voteTypes.normal) { + + if (voteTypeEnum === voteTypes.normal) { if (isVIP && incrementAmount < 0) { //this user is a vip and a downvote incrementAmount = - (row.votes + 2 - oldIncrementAmount); @@ -164,65 +165,67 @@ module.exports = async function voteOnSponsorTime(req, res) { // Send discord message if (incrementAmount < 0) { // Get video ID - let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, "+ - "(select count(1) from sponsorTimes where userID = s.userID) count, "+ - "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded "+ - "FROM sponsorTimes s inner join userNames u on s.userID = u.userID where s.UUID=?" + let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, " + + "(select count(1) from sponsorTimes where userID = s.userID) count, " + + "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " + + "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?" ).get(UUID); let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID); - let webhookURL = null; - if (voteTypeEnum === voteTypes.normal) { - webhookURL = config.discordReportChannelWebhookURL; - } else if (voteTypeEnum === voteTypes.incorrect) { - webhookURL = config.discordCompletelyIncorrectReportWebhookURL; - } - - if (config.youtubeAPIKey !== null && webhookURL !== null) { - YouTubeAPI.videos.list({ - part: "snippet", - id: submissionInfoRow.videoID - }, function (err, data) { - if (err || data.items.length === 0) { - err && console.log(err); - return; - } - - request.post(webhookURL, { - json: { - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID - + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), - "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views - + " Views**\n\n**Submission ID:** " + UUID - + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID - + "\n\n**Total User Submissions:** "+submissionInfoRow.count - + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded - +"\n\n**Timestamp:** " + - getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), - "color": 10813440, - "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (isVIP ? "Report by VIP User" : "") - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - } - }] - } - }, (err, res) => { - if (err) { - console.log("Failed to send reported submission Discord hook."); - console.log(JSON.stringify(err)); - console.log("\n"); - } else if (res && res.statusCode >= 400) { - console.log("Error sending reported submission Discord hook"); - console.log(JSON.stringify(res)); - console.log("\n"); + if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) { + let webhookURL = null; + if (voteTypeEnum === voteTypes.normal) { + webhookURL = config.discordReportChannelWebhookURL; + } else if (voteTypeEnum === voteTypes.incorrect) { + webhookURL = config.discordCompletelyIncorrectReportWebhookURL; + } + + if (config.youtubeAPIKey !== null && webhookURL !== null) { + YouTubeAPI.videos.list({ + part: "snippet", + id: submissionInfoRow.videoID + }, function (err, data) { + if (err || data.items.length === 0) { + err && console.log(err); + return; } + + request.post(webhookURL, { + json: { + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), + "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + + " Views**\n\n**Submission ID:** " + UUID + + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID + + "\n\n**Total User Submissions:** "+submissionInfoRow.count + + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded + +"\n\n**Timestamp:** " + + getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), + "color": 10813440, + "author": { + "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (isVIP ? "Report by VIP User" : "") + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + } + }] + } + }, (err, res) => { + if (err) { + console.log("Failed to send reported submission Discord hook."); + console.log(JSON.stringify(err)); + console.log("\n"); + } else if (res && res.statusCode >= 400) { + console.log("Error sending reported submission Discord hook"); + console.log(JSON.stringify(res)); + console.log("\n"); + } + }); }); - }); + } } } @@ -242,7 +245,7 @@ module.exports = async function voteOnSponsorTime(req, res) { //update the vote count on this sponsorTime //oldIncrementAmount will be zero is row is null - db.prepare("UPDATE sponsorTimes SET " + columnName + " += ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); + db.prepare("UPDATE sponsorTimes SET " + columnName + " = " + columnName + " + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); //for each positive vote, see if a hidden submission can be shown again if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js new file mode 100644 index 0000000..167347c --- /dev/null +++ b/test/cases/voteOnSponsorTime.js @@ -0,0 +1,130 @@ +var request = require('request'); +var db = require('../../src/databases/databases.js').db; +var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js') + +describe('voteOnSponsorTime', () => { + before(() => { + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('vote-testtesttest', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0)"); + + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); + }); + + + it('Should be able to upvote a segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID&UUID=vote-uuid-0&type=1", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT votes FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-0"); + if (row.votes === 3) { + done() + } else { + done("Vote did not succeed. Submission went from 2 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to downvote a segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-2&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT votes FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-2"); + if (row.votes < 10) { + done() + } else { + done("Vote did not succeed. Submission went from 10 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('VIP should be able to completely downvote a segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-3&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT votes FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-3"); + if (row.votes <= -2) { + done() + } else { + done("Vote did not succeed. Submission went from 100 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to vote for a category and it should immediately change (for now)', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=intro", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT category FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-4"); + if (row.category === "intro") { + done() + } else { + done("Vote did not succeed. Submission went from sponsor to " + row.category); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to change your vote for a category and it should immediately change (for now)', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT category FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-4"); + if (row.category === "outro") { + done() + } else { + done("Vote did not succeed. Submission went from intro to " + row.category); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('VIP should be able to vote for a category and it should immediately change', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&category=outro", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT category FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-5"); + let row2 = db.prepare("SELECT votes FROM categoryVotes WHERE UUID = ? and category = ?").get("vote-uuid-5", "outro"); + if (row.category === "outro" && row2.votes === 500) { + done() + } else { + done("Vote did not succeed. Submission went from intro to " + row.category + ". Category votes are " + row2.votes + " and should be 500."); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + +}); \ No newline at end of file From bd9a411106c76d860ffb31aa3b445856bc4ea5a3 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 22:15:31 -0400 Subject: [PATCH 10/13] Updated packages --- package-lock.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package-lock.json b/package-lock.json index 2c6dcc0..ecb01c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1398,6 +1398,11 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, + "iso8601-duration": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/iso8601-duration/-/iso8601-duration-1.2.0.tgz", + "integrity": "sha512-ErTBd++b17E8nmWII1K1uZtBgD1E8RjyvwmxlCjPHNqHMD7gmcMHOw0E8Ro/6+QT4PhHRSnnMo7bxa1vFPkwhg==" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", From 5c0062d9dfc3d4e42b75f6e49b8238422f5d4b86 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 29 Apr 2020 22:21:15 -0400 Subject: [PATCH 11/13] Make directories if needed for the databases --- src/databases/databases.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/databases/databases.js b/src/databases/databases.js index 73b0897..7f81c77 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -7,6 +7,9 @@ let options = { fileMustExist: !config.createDatabaseIfNotExist }; +fs.mkdirSync(config.db); +fs.mkdirSync(config.db); + var db = new Sqlite3(config.db, options); var privateDB = new Sqlite3(config.privateDB, options); From e303405ee0bda6003f1b87613680fcf296026281 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 00:31:42 -0400 Subject: [PATCH 12/13] Check before mkdir --- src/databases/databases.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/databases/databases.js b/src/databases/databases.js index 7f81c77..bd3aad1 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -1,14 +1,20 @@ var config = require('../config.js'); var Sqlite3 = require('better-sqlite3'); var fs = require('fs'); +var path = require('path'); let options = { readonly: config.readOnly, fileMustExist: !config.createDatabaseIfNotExist }; -fs.mkdirSync(config.db); -fs.mkdirSync(config.db); +// Make dirs if required +if (!fs.existsSync(path.join(config.db, "../"))) { + fs.mkdirSync(path.join(config.db, "../")); +} +if (!fs.existsSync(path.join(config.privateDB, "../"))) { + fs.mkdirSync(path.join(config.privateDB, "../")); +} var db = new Sqlite3(config.db, options); var privateDB = new Sqlite3(config.privateDB, options); From 9a317a2c23e0d332284835749bfb9521dcf65606 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 30 Apr 2020 00:34:34 -0400 Subject: [PATCH 13/13] Added getIP options --- config.json.example | 2 +- src/utils/getIP.js | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index ccda7e2..c548ffc 100644 --- a/config.json.example +++ b/config.json.example @@ -8,7 +8,7 @@ "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] - "behindProxy": true, + "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For" "db": "./databases/sponsorTimes.db", "privateDB": "./databases/private.db", "createDatabaseIfNotExist": true, //This will run on startup every time (unless readOnly is true) - so ensure "create table if not exists" is used in the schema diff --git a/src/utils/getIP.js b/src/utils/getIP.js index 78244fa..e8d0c26 100644 --- a/src/utils/getIP.js +++ b/src/utils/getIP.js @@ -2,5 +2,16 @@ var fs = require('fs'); var config = require('../config.js'); module.exports = function getIP(req) { - return config.behindProxy ? req.headers['x-forwarded-for'] : req.connection.remoteAddress; + if (config.behindProxy === true) config.behindProxy = "X-Forwarded-For"; + + switch (config.behindProxy) { + case "X-Forwarded-For": + return req.headers['X-Forwarded-For']; + case "Cloudflare": + return req.headers['CF-Connecting-IP']; + case "X-Real-IP": + return req.headers['X-Real-IP']; + default: + return req.connection.remoteAddress; + } } \ No newline at end of file