diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..efca3de --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM node:12 +WORKDIR /usr/src/app +COPY package.json . +RUN npm install +COPY index.js . +COPY src src +COPY entrypoint.sh . +EXPOSE 8080 +CMD ./entrypoint.sh \ No newline at end of file diff --git a/config.json.example b/config.json.example index 104893f..55e5e78 100644 --- a/config.json.example +++ b/config.json.example @@ -9,6 +9,7 @@ "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] "neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app + "proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash) "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For" "db": "./databases/sponsorTimes.db", "privateDB": "./databases/private.db", diff --git a/databases/_private.db.sql b/databases/_private.db.sql index 0324481..5239cab 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -24,6 +24,11 @@ CREATE TABLE IF NOT EXISTS "sponsorTimes" ( "timeSubmitted" INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS "config" ( + "key" TEXT NOT NULL, + "value" TEXT NOT NULL +); + CREATE INDEX IF NOT EXISTS sponsorTimes_hashedIP on sponsorTimes(hashedIP); CREATE INDEX IF NOT EXISTS votes_userID on votes(UUID); diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 707cb5c..03d02f1 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -25,8 +25,9 @@ CREATE TABLE IF NOT EXISTS "categoryVotes" ( "votes" INTEGER NOT NULL default '0' ); -CREATE TABLE IF NOT EXISTS "version" ( - "code" INTEGER NOT NULL default '0' +CREATE TABLE IF NOT EXISTS "config" ( + "key" TEXT NOT NULL UNIQUE, + "value" TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID); diff --git a/databases/_upgrade_0.sql b/databases/_upgrade_sponsorTimes_1.sql similarity index 89% rename from databases/_upgrade_0.sql rename to databases/_upgrade_sponsorTimes_1.sql index 4f747bc..5629cc8 100644 --- a/databases/_upgrade_0.sql +++ b/databases/_upgrade_sponsorTimes_1.sql @@ -19,7 +19,7 @@ INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,us DROP TABLE sponsorTimes; ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; -/* Increase version number */ -INSERT INTO version VALUES(1); +/* Add version to config */ +INSERT INTO config (key, value) VALUES("version", 1); COMMIT; \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..6215525 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e +echo 'Entrypoint script' +cd /usr/src/app +cp /etc/sponsorblock/config.json . || cat < config.json +{ + "port": 8080, + "globalSalt": "[CHANGE THIS]", + "adminUserID": "[CHANGE THIS]", + "youtubeAPIKey": null, + "discordReportChannelWebhookURL": null, + "discordFirstTimeSubmissionsWebhookURL": null, + "discordAutoModWebhookURL": null, + "proxySubmission": null, + "behindProxy": "X-Forwarded-For", + "db": "./databases/sponsorTimes.db", + "privateDB": "./databases/private.db", + "createDatabaseIfNotExist": true, + "schemaFolder": "./databases", + "dbSchema": "./databases/_sponsorTimes.db.sql", + "privateDBSchema": "./databases/_private.db.sql", + "mode": "development", + "readOnly": false +} +EOF +node index.js \ No newline at end of file diff --git a/index.js b/index.js index 5e4b977..a877bab 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,6 @@ var config = require('./src/config.js'); var createServer = require('./src/app.js'); +const logger = require('./src/utils/logger.js'); var server = createServer(() => { - console.log("Server started."); -}); \ No newline at end of file + logger.info("Server started on port " + config.port + "."); +}); diff --git a/package-lock.json b/package-lock.json index ecb01c3..d314594 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1502,9 +1502,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" }, "lodash.noop": { "version": "3.0.1", diff --git a/package.json b/package.json index b8f62a8..ffd1a63 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "express": "^4.17.1", "http": "0.0.0", "iso8601-duration": "^1.2.0", + "sync-mysql": "^3.0.1", "uuid": "^3.3.2", "youtube-api": "^2.0.10", "node-fetch": "^2.6.0" diff --git a/src/app.js b/src/app.js index 9561dc1..1ce1674 100644 --- a/src/app.js +++ b/src/app.js @@ -49,8 +49,8 @@ app.get('/api/skipSegments', getSkipSegments); app.post('/api/skipSegments', postSkipSegments); //voting endpoint -app.get('/api/voteOnSponsorTime', voteOnSponsorTime); -app.post('/api/voteOnSponsorTime', voteOnSponsorTime); +app.get('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint); +app.post('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint); //Endpoint when a sponsorTime is used up app.get('/api/viewedVideoSponsorTime', viewedVideoSponsorTime); diff --git a/src/databases/Mysql.js b/src/databases/Mysql.js new file mode 100644 index 0000000..8f3d8ba --- /dev/null +++ b/src/databases/Mysql.js @@ -0,0 +1,29 @@ +var MysqlInterface = require('sync-mysql'); +var config = require('../config.js'); +const logger = require('../utils/logger.js'); + +class Mysql { + constructor(msConfig) { + this.connection = new MysqlInterface(msConfig); + } + + exec(query) { + this.prepare('run', query, []); + } + + prepare (type, query, params) { + logger.debug("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params); + if (type === 'get') { + return this.connection.query(query, params)[0]; + } else if (type === 'run') { + this.connection.query(query, params); + } else if (type === 'all') { + return this.connection.query(query, params); + } else { + logger.warn('returning undefined...'); + return undefined; + } + } +} + +module.exports = Mysql; \ No newline at end of file diff --git a/src/databases/Sqlite.js b/src/databases/Sqlite.js new file mode 100644 index 0000000..6b71fec --- /dev/null +++ b/src/databases/Sqlite.js @@ -0,0 +1,33 @@ +const { db } = require("./databases"); +var config = require('../config.js'); +const logger = require('../utils/logger.js'); + +class Sqlite { + constructor(connection) { + this.connection = connection; + } + + getConnection() { + return this.connection; + } + + prepare(type, query, params) { + if (type === 'get') { + return this.connection.prepare(query).get(...params); + } else if (type === 'run') { + this.connection.prepare(query).run(...params); + } else if (type === 'all') { + return this.connection.prepare(query).all(...params); + } else { + logger.debug('sqlite query: returning undefined') + logger.debug("prepare: type: " + type + ", query: " + query + ", params: " + params); + return undefined; + } + } + + exec(query) { + return this.connection.exec(query); + } +} + +module.exports = Sqlite; \ No newline at end of file diff --git a/src/databases/databases.js b/src/databases/databases.js index bd3aad1..1a1b448 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -2,51 +2,70 @@ var config = require('../config.js'); var Sqlite3 = require('better-sqlite3'); var fs = require('fs'); var path = require('path'); +var Sqlite = require('./Sqlite.js') +var Mysql = require('./Mysql.js') let options = { readonly: config.readOnly, fileMustExist: !config.createDatabaseIfNotExist }; -// Make dirs if required -if (!fs.existsSync(path.join(config.db, "../"))) { - fs.mkdirSync(path.join(config.db, "../")); -} -if (!fs.existsSync(path.join(config.privateDB, "../"))) { - fs.mkdirSync(path.join(config.privateDB, "../")); -} - -var db = new Sqlite3(config.db, options); -var privateDB = new Sqlite3(config.privateDB, options); - -if (config.createDatabaseIfNotExist && !config.readOnly) { - if (fs.existsSync(config.dbSchema)) db.exec(fs.readFileSync(config.dbSchema).toString()); - if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString()); -} - -// Upgrade database if required -if (!config.readOnly) { - let versionCode = db.prepare("SELECT code FROM version").get() || 0; - let path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; - while (fs.existsSync(path)) { - db.exec(fs.readFileSync(path).toString()); - - versionCode = db.prepare("SELECT code FROM version").get(); - path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql"; +if (config.mysql) { + module.exports = { + db: new Mysql(config.mysql), + privateDB: new Mysql(config.privateMysql) + }; +} else { + // Make dirs if required + if (!fs.existsSync(path.join(config.db, "../"))) { + fs.mkdirSync(path.join(config.db, "../")); + } + if (!fs.existsSync(path.join(config.privateDB, "../"))) { + fs.mkdirSync(path.join(config.privateDB, "../")); } -} -// Enable WAL mode checkpoint number -if (!config.readOnly && config.mode === "production") { - db.exec("PRAGMA journal_mode=WAL;"); - db.exec("PRAGMA wal_autocheckpoint=1;"); -} + var db = new Sqlite3(config.db, options); + var privateDB = new Sqlite3(config.privateDB, options); -// Enable Memory-Mapped IO -db.exec("pragma mmap_size= 500000000;"); -privateDB.exec("pragma mmap_size= 500000000;"); + if (config.createDatabaseIfNotExist && !config.readOnly) { + if (fs.existsSync(config.dbSchema)) db.exec(fs.readFileSync(config.dbSchema).toString()); + if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString()); + } -module.exports = { - db: db, - privateDB: privateDB -}; \ No newline at end of file + if (!config.readOnly) { + // Upgrade database if required + ugradeDB(db, "sponsorTimes"); + ugradeDB(privateDB, "private") + + // Attach private db to main db + db.prepare("ATTACH ? as privateDB").run(config.privateDB); + } + + // Enable WAL mode checkpoint number + if (!config.readOnly && config.mode === "production") { + db.exec("PRAGMA journal_mode=WAL;"); + db.exec("PRAGMA wal_autocheckpoint=1;"); + } + + // Enable Memory-Mapped IO + db.exec("pragma mmap_size= 500000000;"); + privateDB.exec("pragma mmap_size= 500000000;"); + + module.exports = { + db: new Sqlite(db), + privateDB: new Sqlite(privateDB) + }; + + function ugradeDB(db, prefix) { + let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); + let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; + + let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + while (fs.existsSync(path)) { + db.exec(fs.readFileSync(path).toString()); + + versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value; + path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + } + } +} \ No newline at end of file diff --git a/src/middleware/logger.js b/src/middleware/logger.js index 23ffb16..43a0c36 100644 --- a/src/middleware/logger.js +++ b/src/middleware/logger.js @@ -1,7 +1,6 @@ -var fs = require('fs'); -var config = require('../config.js'); +const log = require('../utils/logger.js'); // log not logger to not interfere with function name module.exports = function logger (req, res, next) { - (config.mode === "development") && console.log('Request recieved: ' + req.url); + log.info("Request recieved: " + req.method + " " + req.url); next(); } \ No newline at end of file diff --git a/src/routes/addUserAsVIP.js b/src/routes/addUserAsVIP.js index a5eadd3..03d8502 100644 --- a/src/routes/addUserAsVIP.js +++ b/src/routes/addUserAsVIP.js @@ -4,7 +4,6 @@ var config = require('../config.js'); var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); - module.exports = async function addUserAsVIP (req, res) { let userID = req.query.userID; let adminUserIDInput = req.query.adminUserID; @@ -25,21 +24,21 @@ module.exports = async function addUserAsVIP (req, res) { //hash the userID adminUserIDInput = getHash(adminUserIDInput); - if (adminUserIDInput !== adminUserID) { + if (adminUserIDInput !== config.adminUserID) { //not authorized res.sendStatus(403); return; } //check to see if this user is already a vip - let row = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(userID); + let row = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]); if (enabled && row.userCount == 0) { //add them to the vip list - db.prepare("INSERT INTO vipUsers VALUES(?)").run(userID); + db.prepare('run', "INSERT INTO vipUsers VALUES(?)", [userID]); } else if (!enabled && row.userCount > 0) { //remove them from the shadow ban list - db.prepare("DELETE FROM vipUsers WHERE userID = ?").run(userID); + db.prepare('run', "DELETE FROM vipUsers WHERE userID = ?", [userID]); } res.sendStatus(200); diff --git a/src/routes/getDaysSavedFormatted.js b/src/routes/getDaysSavedFormatted.js index 45f13d8..94793b7 100644 --- a/src/routes/getDaysSavedFormatted.js +++ b/src/routes/getDaysSavedFormatted.js @@ -1,7 +1,7 @@ var db = require('../databases/databases.js').db; module.exports = function getDaysSavedFormatted (req, res) { - let row = db.prepare("SELECT SUM((endTime - startTime) / 60 / 60 / 24 * views) as daysSaved from sponsorTimes where shadowHidden != 1").get(); + let row = db.prepare('get', "SELECT SUM((endTime - startTime) / 60 / 60 / 24 * views) as daysSaved from sponsorTimes where shadowHidden != 1", []); if (row !== undefined) { //send this result diff --git a/src/routes/getSavedTimeForUser.js b/src/routes/getSavedTimeForUser.js index ea23dc6..fd04d28 100644 --- a/src/routes/getSavedTimeForUser.js +++ b/src/routes/getSavedTimeForUser.js @@ -14,7 +14,7 @@ module.exports = function getSavedTimeForUser (req, res) { userID = getHash(userID); try { - let row = db.prepare("SELECT SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE userID = ? AND votes > -1 AND shadowHidden != 1 ").get(userID); + let row = db.prepare("get", "SELECT SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE userID = ? AND votes > -1 AND shadowHidden != 1 ", [userID]); if (row.minutesSaved != null) { res.send({ diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index 1b4af64..a003a6c 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -5,300 +5,182 @@ var databases = require('../databases/databases.js'); var db = databases.db; var privateDB = databases.privateDB; +var logger = require('../utils/logger.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); - -//gets the getWeightedRandomChoice for each group in an array of groups -function getWeightedRandomChoiceForArray(choiceGroups, weights) { - let finalChoices = []; - //the indexes either chosen to be added to final indexes or chosen not to be added - let choicesDealtWith = []; - //for each choice group, what are the sums of the weights - let weightSums = []; - - for (let i = 0; i < choiceGroups.length; i++) { - //find weight sums for this group - weightSums.push(0); - for (let j = 0; j < choiceGroups[i].length; j++) { - //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time - if (weights[choiceGroups[i][j]] > 0) { - weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]]; - } - } - - //create a random choice for this group - let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1) - finalChoices.push(randomChoice.finalChoices); - - for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) { - choicesDealtWith.push(randomChoice.choicesDealtWith[j]) - } - } - - return { - finalChoices: finalChoices, - choicesDealtWith: choicesDealtWith, - weightSums: weightSums - }; -} - -//gets a weighted random choice from the indexes array based on the weights. -//amountOfChoices speicifies the amount of choices to return, 1 or more. +//gets a weighted random choice from the choices array based on their `votes` property. +//amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique -function getWeightedRandomChoice(choices, weights, amountOfChoices) { - if (amountOfChoices > choices.length) { - //not possible, since all choices must be unique - return null; +function getWeightedRandomChoice(choices, amountOfChoices) { + //trivial case: no need to go through the whole process + if (amountOfChoices >= choices.length) { + return choices; } - let finalChoices = []; - let choicesDealtWith = []; + //assign a weight to each choice + let totalWeight = 0; + choices = choices.map(choice => { + //The 3 makes -2 the minimum votes before being ignored completely + //https://www.desmos.com/calculator/c1duhfrmts + //this can be changed if this system increases in popularity. + const weight = Math.exp((choice.votes + 3), 0.85); + totalWeight += weight; - let sqrtWeightsList = []; - //the total of all the weights run through the cutom sqrt function - let totalSqrtWeights = 0; - for (let j = 0; j < choices.length; j++) { - //multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3) - //The 3 makes -2 the minimum votes before being ignored completely - //https://www.desmos.com/calculator/ljftxolg9j - //this can be changed if this system increases in popularity. - let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10); - sqrtWeightsList.push(sqrtVote) - totalSqrtWeights += sqrtVote; - - //this index has now been deat with - choicesDealtWith.push(choices[j]); - } + return { ...choice, weight }; + }); //iterate and find amountOfChoices choices - let randomNumber = Math.random(); - - //this array will keep adding to this variable each time one sqrt vote has been dealt with - //this is the sum of all the sqrtVotes under this index - let currentVoteNumber = 0; - for (let j = 0; j < sqrtWeightsList.length; j++) { - if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) { - //this one was randomly generated - finalChoices.push(choices[j]); - //remove that from original array, for next recursion pass if it happens - choices.splice(j, 1); - break; - } + const chosen = []; + while (amountOfChoices-- > 0) { + //weighted random draw of one element of choices + const randomNumber = Math.random() * totalWeight; + let stackWeight = choices[0].weight; + let i = 0; + while (stackWeight < randomNumber) { + stackWeight += choices[++i].weight; + } - //add on to the count - currentVoteNumber += sqrtWeightsList[j]; - } - - //add on the other choices as well using recursion - if (amountOfChoices > 1) { - let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices; - //add all these choices to the finalChoices array being returned - for (let i = 0; i < otherChoices.length; i++) { - finalChoices.push(otherChoices[i]); - } + //add it to the chosen ones and remove it from the choices before the next iteration + chosen.push(choices[i]); + totalWeight -= choices[i].weight; + choices.splice(i, 1); } - return { - finalChoices: finalChoices, - choicesDealtWith: choicesDealtWith - }; + return chosen; } - -//This function will find sponsor times that are contained inside of eachother, called similar sponsor times +//This function will find segments that are contained inside of eachother, called similar segments //Only one similar time will be returned, randomly generated based on the sqrt of votes. //This allows new less voted items to still sometimes appear to give them a chance at getting votes. -//Sponsor times with less than -1 votes are already ignored before this function is called -function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) { - //list of sponsors that are contained inside eachother - let similarSponsors = []; +//Segments with less than -1 votes are already ignored before this function is called +function chooseSegments(segments) { + //Create groups of segments that are similar to eachother + //Segments must be sorted by their startTime so that we can build groups chronologically: + //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group + //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall + // inside that group (because they're sorted) so we can create a new one + const similarSegmentsGroups = []; + let currentGroup; + let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created + segments.forEach(segment => { + if (segment.startTime > cursor) { + currentGroup = { segments: [], votes: 0 }; + similarSegmentsGroups.push(currentGroup); + } - for (let i = 0; i < sponsorTimes.length; i++) { - //see if the start time is located between the start and end time of the other sponsor time. - for (let j = i + 1; j < sponsorTimes.length; j++) { - if (sponsorTimes[j][0] >= sponsorTimes[i][0] && sponsorTimes[j][0] <= sponsorTimes[i][1]) { - //sponsor j is contained in sponsor i - similarSponsors.push([i, j]); - } - } - } + currentGroup.segments.push(segment); + //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time + if (segment.votes > 0) { + currentGroup.votes += segment.votes; + } - let similarSponsorsGroups = []; - //once they have been added to a group, they don't need to be dealt with anymore - let dealtWithSimilarSponsors = []; + cursor = Math.max(cursor, segment.endTime); + }); - //create lists of all the similar groups (if 1 and 2 are similar, and 2 and 3 are similar, the group is 1, 2, 3) - for (let i = 0; i < similarSponsors.length; i++) { - if (dealtWithSimilarSponsors.includes(i)) { - //dealt with already - continue; - } - - //this is the group of indexes that are similar - let group = similarSponsors[i]; - for (let j = 0; j < similarSponsors.length; j++) { - if (group.includes(similarSponsors[j][0]) || group.includes(similarSponsors[j][1])) { - //this is a similar group - group.push(similarSponsors[j][0]); - group.push(similarSponsors[j][1]); - dealtWithSimilarSponsors.push(j); - } - } - similarSponsorsGroups.push(group); - } - - //remove duplicate indexes in group arrays - for (let i = 0; i < similarSponsorsGroups.length; i++) { - uniqueArray = similarSponsorsGroups[i].filter(function(item, pos, self) { - return self.indexOf(item) == pos; - }); - - similarSponsorsGroups[i] = uniqueArray; - } - - let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes); - - let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices; - //the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added - let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith; - - let voteSums = weightedRandomIndexes.weightSums; - //convert these into the votes - for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { - //it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor. - votes[finalSponsorTimeIndexes[i]] = voteSums[i]; - } - - //find the indexes never dealt with and add them - for (let i = 0; i < sponsorTimes.length; i++) { - if (!finalSponsorTimeIndexesDealtWith.includes(i)) { - finalSponsorTimeIndexes.push(i) - } - } - - //if there are too many indexes, find the best 4 - if (finalSponsorTimeIndexes.length > 8) { - finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices; - } - - //convert this to a final array to return - let finalSponsorTimes = []; - for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { - finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]); - } - - //convert this to a final array of UUIDs as well - let finalUUIDs = []; - for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { - finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]); - } - - return { - sponsorTimes: finalSponsorTimes, - UUIDs: finalUUIDs - }; + //if there are too many groups, find the best 8 + return getWeightedRandomChoice(similarSegmentsGroups, 8).map( + //randomly choose 1 good segment per group and return them + group => getWeightedRandomChoice(group.segments, 1)[0] + ); } /** - * + * * Returns what would be sent to the client. - * Will resond with errors if required. Returns false if it errors. - * - * @param req - * @param res - * + * Will respond with errors if required. Returns false if it errors. + * + * @param req + * @param res + * * @returns */ function handleGetSegments(req, res) { - const videoID = req.query.videoID; - // Default to sponsor - // If using params instead of JSON, only one category can be pulled - const categories = req.query.categories ? JSON.parse(req.query.categories) - : (req.query.category ? [req.query.category] : ["sponsor"]); + const videoID = req.query.videoID; + // Default to sponsor + // If using params instead of JSON, only one category can be pulled + const categories = req.query.categories + ? JSON.parse(req.query.categories) + : req.query.category + ? [req.query.category] + : ['sponsor']; - /** - * @type {Array<{ - * segment: number[], - * category: string, - * UUID: string - * }> - * } - */ - let segments = []; + /** + * @type {Array<{ + * segment: number[], + * category: string, + * UUID: string + * }> + * } + */ + const segments = []; - let hashedIP = getHash(getIP(req) + config.globalSalt); + let userHashedIP, shadowHiddenSegments; - try { - for (const category of categories) { - let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime") - .all(videoID, category); + try { + for (const category of categories) { + const categorySegments = db + .prepare( + 'all', + 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', + [videoID, category] + ) + .filter(segment => { + if (segment.votes < -1) { + return false; //too untrustworthy, just ignore it + } - let sponsorTimes = []; - let votes = [] - let UUIDs = []; - - for (let i = 0; i < rows.length; i++) { - //check if votes are above -1 - if (rows[i].votes < -1) { - //too untrustworthy, just ignore it - continue; - } - - //check if shadowHidden - //this means it is hidden to everyone but the original ip that submitted it - if (rows[i].shadowHidden == 1) { - //get the ip - //await the callback - let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID); - - if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) { - //this isn't their ip, don't send it to them - continue; - } - } - - sponsorTimes.push([rows[i].startTime, rows[i].endTime]); - votes.push(rows[i].votes); - UUIDs.push(rows[i].UUID); + //check if shadowHidden + //this means it is hidden to everyone but the original ip that submitted it + if (segment.shadowHidden != 1) { + return true; + } + + if (shadowHiddenSegments === undefined) { + shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); + } + + //if this isn't their ip, don't send it to them + return shadowHiddenSegments.some(shadowHiddenSegment => { + if (userHashedIP === undefined) { + //hash the IP only if it's strictly necessary + userHashedIP = getHash(getIP(req) + config.globalSalt); } - - organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs); - sponsorTimes = organisedData.sponsorTimes; - UUIDs = organisedData.UUIDs; - - for (let i = 0; i < sponsorTimes.length; i++) { - segments.push({ - segment: sponsorTimes[i], - category: category, - UUID: UUIDs[i] - }); - } - } - } catch(error) { - console.error(error); - res.send(500); + return shadowHiddenSegment.hashedIP === userHashedIP; + }); + }); - return false; + chooseSegments(categorySegments).forEach(chosenSegment => { + segments.push({ + category, + segment: [chosenSegment.startTime, chosenSegment.endTime], + UUID: chosenSegment.UUID, + }); + }); } if (segments.length == 0) { - res.sendStatus(404); - return false; + res.sendStatus(404); + return false; } return segments; + } catch (error) { + logger.error(error); + res.sendStatus(500); + + return false; + } } - module.exports = { - handleGetSegments, - endpoint: function (req, res) { - let segments = handleGetSegments(req, res); + handleGetSegments, + endpoint: function (req, res) { + let segments = handleGetSegments(req, res); - if (segments) { - //send result - res.send(segments) - } + if (segments) { + //send result + res.send(segments); } -} \ No newline at end of file + }, +}; diff --git a/src/routes/getTopUsers.js b/src/routes/getTopUsers.js index 8230b53..3df5c65 100644 --- a/src/routes/getTopUsers.js +++ b/src/routes/getTopUsers.js @@ -2,6 +2,7 @@ var db = require('../databases/databases.js').db; module.exports = function getTopUsers (req, res) { let sortType = req.query.sortType; + let categoryStatsEnabled = req.query.categoryStats; if (sortType == undefined) { //invalid request @@ -27,11 +28,27 @@ module.exports = function getTopUsers (req, res) { let viewCounts = []; let totalSubmissions = []; let minutesSaved = []; + let categoryStats = categoryStatsEnabled ? [] : undefined; + + let additionalFields = ''; + if (categoryStatsEnabled) { + additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " + + "SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " + + "SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " + + "SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " + + "SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " + + "SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, "; + } - let rows = db.prepare("SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + + let rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " + + "SUM(votes) as userVotes, " + + additionalFields + "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " + - "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 GROUP BY IFNULL(userName, sponsorTimes.userID) ORDER BY " + sortBy + " DESC LIMIT 100").all(); + "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " + + "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " + + "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 50 " + + "ORDER BY " + sortBy + " DESC LIMIT 100", []); for (let i = 0; i < rows.length; i++) { userNames[i] = rows[i].userName; @@ -39,13 +56,24 @@ module.exports = function getTopUsers (req, res) { viewCounts[i] = rows[i].viewCount; totalSubmissions[i] = rows[i].totalSubmissions; minutesSaved[i] = rows[i].minutesSaved; + if (categoryStatsEnabled) { + categoryStats[i] = [ + rows[i].categorySponsor, + rows[i].categorySumIntro, + rows[i].categorySumOutro, + rows[i].categorySumInteraction, + rows[i].categorySelfpromo, + rows[i].categoryMusicOfftopic, + ]; + } } //send this result res.send({ - userNames: userNames, - viewCounts: viewCounts, - totalSubmissions: totalSubmissions, - minutesSaved: minutesSaved + userNames, + viewCounts, + totalSubmissions, + minutesSaved, + categoryStats }); } \ No newline at end of file diff --git a/src/routes/getTotalStats.js b/src/routes/getTotalStats.js index 86b1531..e4cfb67 100644 --- a/src/routes/getTotalStats.js +++ b/src/routes/getTotalStats.js @@ -8,8 +8,8 @@ var lastUserCountCheck = 0; module.exports = function getTotalStats (req, res) { - let row = db.prepare("SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, " + - "SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE shadowHidden != 1").get(); + let row = db.prepare('get', "SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, " + + "SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE shadowHidden != 1 AND votes >= 0", []); if (row !== undefined) { //send this result diff --git a/src/routes/getUsername.js b/src/routes/getUsername.js index 290b6cd..d04b8ee 100644 --- a/src/routes/getUsername.js +++ b/src/routes/getUsername.js @@ -1,6 +1,7 @@ var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); +const logger = require('../utils/logger.js'); module.exports = function getUsername (req, res) { let userID = req.query.userID; @@ -15,7 +16,7 @@ module.exports = function getUsername (req, res) { userID = getHash(userID); try { - let row = db.prepare("SELECT userName FROM userNames WHERE userID = ?").get(userID); + let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]); if (row !== undefined) { res.send({ @@ -28,7 +29,7 @@ module.exports = function getUsername (req, res) { }); } } catch (err) { - console.log(err); + logger.error(err); res.sendStatus(500); return; diff --git a/src/routes/getViewsForUser.js b/src/routes/getViewsForUser.js index e3a0ebf..79d8961 100644 --- a/src/routes/getViewsForUser.js +++ b/src/routes/getViewsForUser.js @@ -1,6 +1,6 @@ var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); - +var logger = require('../utils/logger.js'); module.exports = function getViewsForUser(req, res) { let userID = req.query.userID; @@ -14,7 +14,7 @@ module.exports = function getViewsForUser(req, res) { userID = getHash(userID); try { - let row = db.prepare("SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?").get(userID); + let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?", [userID]); //increase the view count by one if (row.viewCount != null) { @@ -25,7 +25,7 @@ module.exports = function getViewsForUser(req, res) { res.sendStatus(404); } } catch (err) { - console.log(err); + logger.error(err); res.sendStatus(500); return; diff --git a/src/routes/oldSubmitSponsorTimes.js b/src/routes/oldSubmitSponsorTimes.js index b818a24..613ec9c 100644 --- a/src/routes/oldSubmitSponsorTimes.js +++ b/src/routes/oldSubmitSponsorTimes.js @@ -1,5 +1,3 @@ -var config = require('../config.js'); - var postSkipSegments = require('./postSkipSegments.js'); module.exports = async function submitSponsorTimes(req, res) { diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index f883c76..61a8ac2 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -4,37 +4,20 @@ var databases = require('../databases/databases.js'); var db = databases.db; var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); +var logger = require('../utils/logger.js'); var request = require('request'); var isoDurations = require('iso8601-duration'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); var getFormattedTime = require('../utils/getFormattedTime.js'); -const fetch = require('node-fetch'); - -// TODO: might need to be a util -//returns true if the user is considered trustworthy -//this happens after a user has made 5 submissions and has less than 60% downvoted submissions -async function isUserTrustworthy(userID) { - //check to see if this user how many submissions this user has submitted - let totalSubmissionsRow = db.prepare("SELECT count(*) as totalSubmissions, sum(votes) as voteSum FROM sponsorTimes WHERE userID = ?").get(userID); - - if (totalSubmissionsRow.totalSubmissions > 5) { - //check if they have a high downvote ratio - let downvotedSubmissionsRow = db.prepare("SELECT count(*) as downvotedSubmissions FROM sponsorTimes WHERE userID = ? AND (votes < 0 OR shadowHidden > 0)").get(userID); - - return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || - (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions); - } - - return true; -} +var isUserTrustworthy = require('../utils/isUserTrustworthy.js') function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { //check if they are a first time user //if so, send a notification to discord if (config.youtubeAPIKey !== null && config.discordFirstTimeSubmissionsWebhookURL !== null) { - let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(userID); + let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]); // If it is a first time submission if (userSubmissionCountRow.submissionCount <= 1) { @@ -43,7 +26,7 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { id: videoID }, function (err, data) { if (err || data.items.length === 0) { - err && console.log(err); + err && logger.error(err); return; } @@ -70,13 +53,13 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { } }, (err, res) => { if (err) { - console.log("Failed to send first time submission Discord hook."); - console.log(JSON.stringify(err)); - console.log("\n"); + logger.error("Failed to send first time submission Discord hook."); + logger.error(JSON.stringify(err)); + logger.error("\n"); } else if (res && res.statusCode >= 400) { - console.log("Error sending first time submission Discord hook"); - console.log(JSON.stringify(res)); - console.log("\n"); + logger.error("Error sending first time submission Discord hook"); + logger.error(JSON.stringify(res)); + logger.error("\n"); } }); }); @@ -86,9 +69,13 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { // callback: function(reject: "String containing reason the submission was rejected") // returns: string when an error, false otherwise -async function autoModerateSubmission(videoID, segments) { + +// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return +// false for a pass - it was confusing and lead to this bug - any use of this function in +// the future could have the same problem. +async function autoModerateSubmission(submission, callback) { // Get the video information from the youtube API - if (config.youtubeAPI !== null) { + if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { YouTubeAPI.videos.list({ part: "contentDetails", @@ -119,7 +106,7 @@ async function autoModerateSubmission(videoID, segments) { let neuralBlockURL = config.neuralBlockURL; if (!neuralBlockURL) return false; - + let overlap = true; let response = await fetch(neuralBlockURL + "/api/getSponsorSegments?vid=" + videoID); @@ -142,7 +129,7 @@ async function autoModerateSubmission(videoID, segments) { if (!thisSegmentOverlaps){ overlap = false; break; - } + } } if (overlap) { @@ -154,15 +141,31 @@ async function autoModerateSubmission(videoID, segments) { } } else { - console.log("Skipped YouTube API"); + logger.debug("Skipped YouTube API"); // Can't moderate the submission without calling the youtube API // so allow by default. - return; + return false; } } +function proxySubmission(req) { + request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => { + if (config.mode === 'development') { + if (!err) { + logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); + } else { + logger.debug("Proxy Submission: Failed to make call"); + } + } + }); +} + module.exports = async function postSkipSegments(req, res) { + if (config.proxySubmission) { + proxySubmission(req); + } + let videoID = req.query.videoID || req.body.videoID; let userID = req.query.userID || req.body.userID; @@ -203,16 +206,22 @@ module.exports = async function postSkipSegments(req, res) { let startTime = parseFloat(segments[i].segment[0]); let endTime = parseFloat(segments[i].segment[1]); - if (Math.abs(startTime - endTime) < 1 || isNaN(startTime) || isNaN(endTime) - || startTime === Infinity || endTime === Infinity || startTime > endTime) { + if (isNaN(startTime) || isNaN(endTime) + || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request res.sendStatus(400); return; } + if (segments[i].category === "sponsor" && Math.abs(startTime - endTime) < 1) { + // Too short + res.status(400).send("Sponsors must be longer than 1 second long"); + return; + } + //check if this info has already been submitted before - let duplicateCheck2Row = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE startTime = ? " + - "and endTime = ? and category = ? and videoID = ?").get(startTime, endTime, segments[i].category, videoID); + let duplicateCheck2Row = db.prepare('get', "SELECT COUNT(*) as count FROM sponsorTimes WHERE startTime = ? " + + "and endTime = ? and category = ? and videoID = ?", [startTime, endTime, segments[i].category, videoID]); if (duplicateCheck2Row.count > 0) { res.sendStatus(409); return; @@ -229,33 +238,42 @@ module.exports = async function postSkipSegments(req, res) { } try { + //check if this user is on the vip list + let vipRow = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]); + //get current time let timeSubmitted = Date.now(); let yesterday = timeSubmitted - 86400000; - //check to see if this ip has submitted too many sponsors today - let rateLimitCheckRow = privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday]); + // Disable IP ratelimiting for now + if (false) { + //check to see if this ip has submitted too many sponsors today + let rateLimitCheckRow = privateDB.prepare('get', "SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?", [hashedIP, videoID, yesterday]); - if (rateLimitCheckRow.count >= 10) { - //too many sponsors for the same video from the same ip address - res.sendStatus(429); + if (rateLimitCheckRow.count >= 10) { + //too many sponsors for the same video from the same ip address + res.sendStatus(429); - return; + return; + } } - //check to see if the user has already submitted sponsors for this video - let duplicateCheckRow = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID]); + // Disable max submissions for now + if (false) { + //check to see if the user has already submitted sponsors for this video + let duplicateCheckRow = db.prepare('get', "SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?", [userID, videoID]); - if (duplicateCheckRow.count >= 8) { - //too many sponsors for the same video from the same user - res.sendStatus(429); + if (duplicateCheckRow.count >= 16) { + //too many sponsors for the same video from the same user + res.sendStatus(429); - return; + return; + } } //check to see if this user is shadowbanned - let shadowBanRow = privateDB.prepare("SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?").get(userID); + let shadowBanRow = privateDB.prepare('get', "SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?", [userID]); let shadowBanned = shadowBanRow.userCount; @@ -267,7 +285,7 @@ module.exports = async function postSkipSegments(req, res) { let startingVotes = 0; if (isVIP) { //this user is a vip, start them at a higher approval rating - startingVotes = 10; + startingVotes = 10000; } for (const segmentInfo of segments) { @@ -278,17 +296,17 @@ module.exports = async function postSkipSegments(req, res) { segmentInfo.segment[1] + segmentInfo.category + userID, 1); try { - db.prepare("INSERT INTO sponsorTimes " + + db.prepare('run', "INSERT INTO sponsorTimes " + "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" + - "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], - segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [videoID, segmentInfo.segment[0], + segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned]); //add to private db as well - privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); + privateDB.prepare('run', "INSERT INTO sponsorTimes VALUES(?, ?, ?)", [videoID, hashedIP, timeSubmitted]); } catch (err) { //a DB change probably occurred res.sendStatus(502); - console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + + logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err); return; @@ -298,7 +316,7 @@ module.exports = async function postSkipSegments(req, res) { sendDiscordNotification(userID, videoID, UUID, segmentInfo); } } catch (err) { - console.error(err); + logger.error(err); res.sendStatus(500); diff --git a/src/routes/setUsername.js b/src/routes/setUsername.js index 666afb1..7165c07 100644 --- a/src/routes/setUsername.js +++ b/src/routes/setUsername.js @@ -3,6 +3,7 @@ var config = require('../config.js'); var db = require('../databases/databases.js').db; var getHash = require('../utils/getHash.js'); +const logger = require('../utils/logger.js'); module.exports = function setUsername(req, res) { @@ -17,6 +18,12 @@ module.exports = function setUsername(req, res) { return; } + if (userName.includes("discord")) { + // Don't allow + res.sendStatus(200); + return; + } + if (adminUserIDInput != undefined) { //this is the admin controlling the other users account, don't hash the controling account's ID adminUserIDInput = getHash(adminUserIDInput); @@ -33,19 +40,19 @@ module.exports = function setUsername(req, res) { try { //check if username is already set - let row = db.prepare("SELECT count(*) as count FROM userNames WHERE userID = ?").get(userID); + let row = db.prepare('get', "SELECT count(*) as count FROM userNames WHERE userID = ?", [userID]); if (row.count > 0) { //already exists, update this row - db.prepare("UPDATE userNames SET userName = ? WHERE userID = ?").run(userName, userID); + db.prepare('run', "UPDATE userNames SET userName = ? WHERE userID = ?", [userName, userID]); } else { //add to the db - db.prepare("INSERT INTO userNames VALUES(?, ?)").run(userID, userName); + db.prepare('run', "INSERT INTO userNames VALUES(?, ?)", [userID, userName]); } res.sendStatus(200); } catch (err) { - console.log(err); + logger.error(err); res.sendStatus(500); return; diff --git a/src/routes/shadowBanUser.js b/src/routes/shadowBanUser.js index 6eb2d21..ba5052e 100644 --- a/src/routes/shadowBanUser.js +++ b/src/routes/shadowBanUser.js @@ -8,6 +8,7 @@ var getHash = require('../utils/getHash.js'); module.exports = async function shadowBanUser(req, res) { let userID = req.query.userID; + let hashedIP = req.query.hashedIP; let adminUserIDInput = req.query.adminUserID; let enabled = req.query.enabled; @@ -18,14 +19,9 @@ module.exports = async function shadowBanUser(req, res) { } //if enabled is false and the old submissions should be made visible again - let unHideOldSubmissions = req.query.unHideOldSubmissions; - if (enabled === undefined){ - unHideOldSubmissions = true; - } else { - unHideOldSubmissions = unHideOldSubmissions === "true"; - } + let unHideOldSubmissions = req.query.unHideOldSubmissions !== "false"; - if (adminUserIDInput == undefined || userID == undefined) { + if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined)) { //invalid request res.sendStatus(400); return; @@ -40,25 +36,57 @@ module.exports = async function shadowBanUser(req, res) { return; } - //check to see if this user is already shadowbanned - let row = privateDB.prepare("SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?").get(userID); + if (userID) { + //check to see if this user is already shadowbanned + let row = privateDB.prepare('get', "SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?", [userID]); - if (enabled && row.userCount == 0) { - //add them to the shadow ban list + if (enabled && row.userCount == 0) { + //add them to the shadow ban list - //add it to the table - privateDB.prepare("INSERT INTO shadowBannedUsers VALUES(?)").run(userID); + //add it to the table + privateDB.prepare('run', "INSERT INTO shadowBannedUsers VALUES(?)", [userID]); - //find all previous submissions and hide them - db.prepare("UPDATE sponsorTimes SET shadowHidden = 1 WHERE userID = ?").run(userID); - } else if (!enabled && row.userCount > 0) { - //remove them from the shadow ban list - privateDB.prepare("DELETE FROM shadowBannedUsers WHERE userID = ?").run(userID); + //find all previous submissions and hide them + if (unHideOldSubmissions) { + db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 1 WHERE userID = ?", [userID]); + } + } else if (!enabled && row.userCount > 0) { + //remove them from the shadow ban list + privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]); - //find all previous submissions and unhide them - if (unHideOldSubmissions) { - db.prepare("UPDATE sponsorTimes SET shadowHidden = 0 WHERE userID = ?").run(userID); - } + //find all previous submissions and unhide them + if (unHideOldSubmissions) { + db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 0 WHERE userID = ?", [userID]); + } + } + } else if (hashedIP) { + //check to see if this user is already shadowbanned + // let row = privateDB.prepare('get', "SELECT count(*) as userCount FROM shadowBannedIPs WHERE hashedIP = ?", [hashedIP]); + + // if (enabled && row.userCount == 0) { + if (enabled) { + //add them to the shadow ban list + + //add it to the table + // privateDB.prepare('run', "INSERT INTO shadowBannedIPs VALUES(?)", [hashedIP]); + + + + //find all previous submissions and hide them + if (unHideOldSubmissions) { + db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 1 WHERE timeSubmitted IN " + + "(SELECT privateDB.timeSubmitted FROM sponsorTimes LEFT JOIN privateDB.sponsorTimes as privateDB ON sponsorTimes.timeSubmitted=privateDB.timeSubmitted " + + "WHERE privateDB.hashedIP = ?)", [hashedIP]); + } + } else if (!enabled && row.userCount > 0) { + // //remove them from the shadow ban list + // privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]); + + // //find all previous submissions and unhide them + // if (unHideOldSubmissions) { + // db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 0 WHERE userID = ?", [userID]); + // } + } } res.sendStatus(200); diff --git a/src/routes/viewedVideoSponsorTime.js b/src/routes/viewedVideoSponsorTime.js index 9af6960..fcc22ea 100644 --- a/src/routes/viewedVideoSponsorTime.js +++ b/src/routes/viewedVideoSponsorTime.js @@ -10,7 +10,7 @@ module.exports = function viewedVideoSponsorTime(req, res) { } //up the view count by one - db.prepare("UPDATE sponsorTimes SET views = views + 1 WHERE UUID = ?").run(UUID); + db.prepare('run', "UPDATE sponsorTimes SET views = views + 1 WHERE UUID = ?", [UUID]); res.sendStatus(200); } diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 2918d3f..abe9262 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -4,16 +4,30 @@ var config = require('../config.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); var getFormattedTime = require('../utils/getFormattedTime.js'); +var isUserTrustworthy = require('../utils/isUserTrustworthy.js') var databases = require('../databases/databases.js'); var db = databases.db; var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); var request = require('request'); +const logger = require('../utils/logger.js'); + +function getVoteAuthor(submissionCount, isVIP, isOwnSubmission) { + if (submissionCount === 0) { + return "Report by New User"; + } else if (isVIP) { + return "Report by VIP User"; + } else if (isOwnSubmission) { + return "Report by Submitter"; + } + + return ""; +} function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // Check if they've already made a vote - let previousVoteInfo = privateDB.prepare("select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?").get(UUID, userID); + let previousVoteInfo = privateDB.prepare('get', "select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?", [UUID, userID]); if (previousVoteInfo > 0 && previousVoteInfo.category === category) { // Double vote, ignore @@ -21,32 +35,38 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { return; } + let currentCategory = db.prepare('get', "select category from sponsorTimes where UUID = ?", [UUID]); + if (!currentCategory) { + // Submission doesn't exist + res.status("400").send("Submission doesn't exist."); + return; + } + let timeSubmitted = Date.now(); let voteAmount = isVIP ? 500 : 1; // Add the vote - if (db.prepare("select count(*) as count from categoryVotes where UUID = ? and category = ?").get(UUID, category).count > 0) { + if (db.prepare('get', "select count(*) as count from categoryVotes where UUID = ? and category = ?", [UUID, category]).count > 0) { // Update the already existing db entry - db.prepare("update categoryVotes set votes = votes + ? where UUID = ? and category = ?").run(voteAmount, UUID, category); + db.prepare('run', "update categoryVotes set votes = votes + ? where UUID = ? and category = ?", [voteAmount, UUID, category]); } else { // Add a db entry - db.prepare("insert into categoryVotes (UUID, category, votes) values (?, ?, ?)").run(UUID, category, voteAmount); + db.prepare('run', "insert into categoryVotes (UUID, category, votes) values (?, ?, ?)", [UUID, category, voteAmount]); } // Add the info into the private db if (previousVoteInfo > 0) { // Reverse the previous vote - db.prepare("update categoryVotes set votes -= 1 where UUID = ? and category = ?").run(UUID, previousVoteInfo.category); + db.prepare('run', "update categoryVotes set votes -= 1 where UUID = ? and category = ?", [UUID, previousVoteInfo.category]); - privateDB.prepare("update categoryVotes set category = ?, timeSubmitted = ?, hashedIP = ?").run(category, timeSubmitted, hashedIP) + privateDB.prepare('run', "update categoryVotes set category = ?, timeSubmitted = ?, hashedIP = ?", [category, timeSubmitted, hashedIP]); } else { - privateDB.prepare("insert into categoryVotes (UUID, userID, hashedIP, category, timeSubmitted) values (?, ?, ?, ?, ?)").run(UUID, userID, hashedIP, category, timeSubmitted); + privateDB.prepare('run', "insert into categoryVotes (UUID, userID, hashedIP, category, timeSubmitted) values (?, ?, ?, ?, ?)", [UUID, userID, hashedIP, category, timeSubmitted]); } - // See if the submissions categort is ready to change - let currentCategory = db.prepare("select category from sponsorTimes where UUID = ?").get(UUID); - let currentCategoryInfo = db.prepare("select votes from categoryVotes where UUID = ? and category = ?").get(UUID, currentCategory.category); + // See if the submissions category is ready to change + let currentCategoryInfo = db.prepare('get', "select votes from categoryVotes where UUID = ? and category = ?", [UUID, currentCategory.category]); // Change this value from 1 in the future to make it harder to change categories // Done this way without ORs incase the value is zero @@ -58,13 +78,13 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) { // VIPs change it every time if (nextCategoryCount - currentCategoryCount >= 0 || isVIP) { // Replace the category - db.prepare("update sponsorTimes set category = ? where UUID = ?").run(category, UUID); + db.prepare('run', "update sponsorTimes set category = ? where UUID = ?", [category, UUID]); } res.sendStatus(200); } -module.exports = async function voteOnSponsorTime(req, res) { +async function voteOnSponsorTime(req, res) { let UUID = req.query.UUID; let userID = req.query.userID; let type = req.query.type; @@ -87,12 +107,25 @@ module.exports = async function voteOnSponsorTime(req, res) { let hashedIP = getHash(ip + config.globalSalt); //check if this user is on the vip list - let isVIP = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID).userCount > 0; + let isVIP = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [nonAnonUserID]).userCount > 0; + //check if user voting on own submission + let isOwnSubmission = db.prepare("get", "SELECT UUID as submissionCount FROM sponsorTimes where userID = ? AND UUID = ?", [nonAnonUserID, UUID]) !== undefined; + if (type === undefined && category !== undefined) { return categoryVote(UUID, userID, isVIP, category, hashedIP, res); } + if (type == 1 && !isVIP && !isOwnSubmission) { + // Check if upvoting hidden segment + let voteInfo = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", [UUID]); + + if (voteInfo && voteInfo.votes <= -2) { + res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.") + return; + } + } + let voteTypes = { normal: 0, incorrect: 1 @@ -102,7 +135,7 @@ module.exports = async function voteOnSponsorTime(req, res) { try { //check if vote has already happened - let votesRow = privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID); + let votesRow = privateDB.prepare('get', "SELECT type FROM votes WHERE userID = ? AND UUID = ?", [userID, UUID]); //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future let incrementAmount = 0; @@ -114,6 +147,9 @@ module.exports = async function voteOnSponsorTime(req, res) { } else if (type == 0 || type == 10) { //downvote incrementAmount = -1; + } else if (type == 20) { + //undo/cancel vote + incrementAmount = 0; } else { //unrecongnised type of vote res.sendStatus(400); @@ -142,20 +178,16 @@ module.exports = async function voteOnSponsorTime(req, res) { } //check if the increment amount should be multiplied (downvotes have more power if there have been many views) - let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID); + let row = db.prepare('get', "SELECT votes, views FROM sponsorTimes WHERE UUID = ?", [UUID]); if (voteTypeEnum === voteTypes.normal) { - if (isVIP && incrementAmount < 0) { + if ((isVIP || isOwnSubmission) && incrementAmount < 0) { //this user is a vip and a downvote incrementAmount = - (row.votes + 2 - oldIncrementAmount); type = incrementAmount; - } else if (row !== undefined && (row.votes > 8 || row.views > 15) && incrementAmount < 0) { - //increase the power of this downvote - incrementAmount = -Math.abs(Math.min(10, row.votes + 2 - oldIncrementAmount)); - type = incrementAmount; } } else if (voteTypeEnum == voteTypes.incorrect) { - if (isVIP) { + if (isVIP || isOwnSubmission) { //this user is a vip and a downvote incrementAmount = 500 * incrementAmount; type = incrementAmount < 0 ? 12 : 13; @@ -165,13 +197,13 @@ module.exports = async function voteOnSponsorTime(req, res) { // Send discord message if (incrementAmount < 0) { // Get video ID - let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, " + + let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " + "(select count(1) from sponsorTimes where userID = s.userID) count, " + "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " + - "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?" - ).get(UUID); + "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?", + [UUID]); - let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID); + let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [nonAnonUserID]); if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) { let webhookURL = null; @@ -187,7 +219,7 @@ module.exports = async function voteOnSponsorTime(req, res) { id: submissionInfoRow.videoID }, function (err, data) { if (err || data.items.length === 0) { - err && console.log(err); + err && logger.error(err); return; } @@ -199,6 +231,7 @@ module.exports = async function voteOnSponsorTime(req, res) { + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + " Views**\n\n**Submission ID:** " + UUID + + "\n**Category:** " + submissionInfoRow.category + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID + "\n\n**Total User Submissions:** "+submissionInfoRow.count + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded @@ -206,7 +239,7 @@ module.exports = async function voteOnSponsorTime(req, res) { getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), "color": 10813440, "author": { - "name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (isVIP ? "Report by VIP User" : "") + "name": getVoteAuthor(userSubmissionCountRow.submissionCount, isVIP, isOwnSubmission) }, "thumbnail": { "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", @@ -215,13 +248,13 @@ module.exports = async function voteOnSponsorTime(req, res) { } }, (err, res) => { if (err) { - console.log("Failed to send reported submission Discord hook."); - console.log(JSON.stringify(err)); - console.log("\n"); + logger.error("Failed to send reported submission Discord hook."); + logger.error(JSON.stringify(err)); + logger.error("\n"); } else if (res && res.statusCode >= 400) { - console.log("Error sending reported submission Discord hook"); - console.log(JSON.stringify(res)); - console.log("\n"); + logger.error("Error sending reported submission Discord hook"); + logger.error(JSON.stringify(res)); + logger.error("\n"); } }); }); @@ -229,47 +262,68 @@ module.exports = async function voteOnSponsorTime(req, res) { } } - //update the votes table - if (votesRow != undefined) { - privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID); - } else { - privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type); - } + // Only change the database if they have made a submission before and haven't voted recently + let ableToVote = isVIP + || (db.prepare("get", "SELECT userID FROM sponsorTimes WHERE userID = ?", [nonAnonUserID]) !== undefined + && privateDB.prepare("get", "SELECT userID FROM shadowBannedUsers WHERE userID = ?", [nonAnonUserID]) === undefined + && privateDB.prepare("get", "SELECT UUID FROM votes WHERE UUID = ? AND hashedIP = ? AND userID != ?", [UUID, hashedIP, userID]) === undefined); - let columnName = ""; - if (voteTypeEnum === voteTypes.normal) { - columnName = "votes"; - } else if (voteTypeEnum === voteTypes.incorrect) { - columnName = "incorrectVotes"; - } + if (ableToVote) { + //update the votes table + if (votesRow != undefined) { + privateDB.prepare('run', "UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?", [type, userID, UUID]); + } else { + privateDB.prepare('run', "INSERT INTO votes VALUES(?, ?, ?, ?)", [UUID, userID, hashedIP, type]); + } - //update the vote count on this sponsorTime - //oldIncrementAmount will be zero is row is null - db.prepare("UPDATE sponsorTimes SET " + columnName + " = " + columnName + " + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); + let columnName = ""; + if (voteTypeEnum === voteTypes.normal) { + columnName = "votes"; + } else if (voteTypeEnum === voteTypes.incorrect) { + columnName = "incorrectVotes"; + } - //for each positive vote, see if a hidden submission can be shown again - if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { - //find the UUID that submitted the submission that was voted on - let submissionUserID = db.prepare("SELECT userID FROM sponsorTimes WHERE UUID = ?").get(UUID).userID; + //update the vote count on this sponsorTime + //oldIncrementAmount will be zero is row is null + db.prepare('run', "UPDATE sponsorTimes SET " + columnName + " = " + columnName + " + ? WHERE UUID = ?", [incrementAmount - oldIncrementAmount, UUID]); - //check if any submissions are hidden - let hiddenSubmissionsRow = db.prepare("SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0").get(submissionUserID); + //for each positive vote, see if a hidden submission can be shown again + if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { + //find the UUID that submitted the submission that was voted on + let submissionUserIDInfo = db.prepare('get', "SELECT userID FROM sponsorTimes WHERE UUID = ?", [UUID]); + if (!submissionUserIDInfo) { + // They are voting on a non-existent submission + res.status(400).send("Voting on a non-existent submission"); + return; + } - if (hiddenSubmissionsRow.hiddenSubmissions > 0) { - //see if some of this users submissions should be visible again - - if (await isUserTrustworthy(submissionUserID)) { - //they are trustworthy again, show 2 of their submissions again, if there are two to show - db.prepare("UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)").run(submissionUserID) + let submissionUserID = submissionUserIDInfo.userID; + + //check if any submissions are hidden + let hiddenSubmissionsRow = db.prepare('get', "SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0", [submissionUserID]); + + if (hiddenSubmissionsRow.hiddenSubmissions > 0) { + //see if some of this users submissions should be visible again + + if (await isUserTrustworthy(submissionUserID)) { + //they are trustworthy again, show 2 of their submissions again, if there are two to show + db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)", [submissionUserID]); + } } } } - //added to db res.sendStatus(200); } catch (err) { - console.error(err); + logger.error(err); res.status(500).json({error: 'Internal error creating segment vote'}); } -} \ No newline at end of file +} + +module.exports = { + voteOnSponsorTime, + endpoint: function (req, res) { + voteOnSponsorTime(req, res); + }, + }; diff --git a/src/utils/getFormattedTime.js b/src/utils/getFormattedTime.js index 6f3ef40..0d86271 100644 --- a/src/utils/getFormattedTime.js +++ b/src/utils/getFormattedTime.js @@ -1,8 +1,9 @@ //converts time in seconds to minutes:seconds -module.exports = function getFormattedTime(seconds) { - let minutes = Math.floor(seconds / 60); - let secondsDisplay = Math.round(seconds - minutes * 60); - if (secondsDisplay < 10) { +module.exports = function getFormattedTime(totalSeconds) { + let minutes = Math.floor(totalSeconds / 60); + let seconds = totalSeconds - minutes * 60; + let secondsDisplay = seconds.toFixed(3); + if (seconds < 10) { //add a zero secondsDisplay = "0" + secondsDisplay; } diff --git a/src/utils/getIP.js b/src/utils/getIP.js index e8d0c26..a26130b 100644 --- a/src/utils/getIP.js +++ b/src/utils/getIP.js @@ -1,17 +1,16 @@ -var fs = require('fs'); var config = require('../config.js'); module.exports = function getIP(req) { - if (config.behindProxy === true) config.behindProxy = "X-Forwarded-For"; + if (config.behindProxy === true || config.behindProxy === "true") config.behindProxy = "X-Forwarded-For"; switch (config.behindProxy) { case "X-Forwarded-For": - return req.headers['X-Forwarded-For']; + return req.headers['x-forwarded-for']; case "Cloudflare": - return req.headers['CF-Connecting-IP']; + return req.headers['cf-connecting-ip']; case "X-Real-IP": - return req.headers['X-Real-IP']; + return req.headers['x-real-ip']; default: return req.connection.remoteAddress; } -} \ No newline at end of file +} diff --git a/src/utils/isUserTrustworthy.js b/src/utils/isUserTrustworthy.js new file mode 100644 index 0000000..4e46562 --- /dev/null +++ b/src/utils/isUserTrustworthy.js @@ -0,0 +1,20 @@ +var databases = require('../databases/databases.js'); +var db = databases.db; + +//returns true if the user is considered trustworthy +//this happens after a user has made 5 submissions and has less than 60% downvoted submissions + +module.exports = async (userID) => { + //check to see if this user how many submissions this user has submitted + let totalSubmissionsRow = db.prepare('get', "SELECT count(*) as totalSubmissions, sum(votes) as voteSum FROM sponsorTimes WHERE userID = ?", [userID]); + + if (totalSubmissionsRow.totalSubmissions > 5) { + //check if they have a high downvote ratio + let downvotedSubmissionsRow = db.prepare('get', "SELECT count(*) as downvotedSubmissions FROM sponsorTimes WHERE userID = ? AND (votes < 0 OR shadowHidden > 0)", [userID]); + + return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || + (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions); + } + + return true; +} \ No newline at end of file diff --git a/src/utils/logger.js b/src/utils/logger.js new file mode 100644 index 0000000..dd3f62c --- /dev/null +++ b/src/utils/logger.js @@ -0,0 +1,68 @@ +const config = require('../config.js'); + +const levels = { + ERROR: "ERROR", + WARN: "WARN", + INFO: "INFO", + DEBUG: "DEBUG" +}; + +const colors = { + Reset: "\x1b[0m", + Bright: "\x1b[1m", + Dim: "\x1b[2m", + Underscore: "\x1b[4m", + Blink: "\x1b[5m", + Reverse: "\x1b[7m", + Hidden: "\x1b[8m", + + FgBlack: "\x1b[30m", + FgRed: "\x1b[31m", + FgGreen: "\x1b[32m", + FgYellow: "\x1b[33m", + FgBlue: "\x1b[34m", + FgMagenta: "\x1b[35m", + FgCyan: "\x1b[36m", + FgWhite: "\x1b[37m", + + BgBlack: "\x1b[40m", + BgRed: "\x1b[41m", + BgGreen: "\x1b[42m", + BgYellow: "\x1b[43m", + BgBlue: "\x1b[44m", + BgMagenta: "\x1b[45m", + BgCyan: "\x1b[46m", + BgWhite: "\x1b[47m", +} + +const settings = { + ERROR: true, + WARN: true, + INFO: false, + DEBUG: false +}; + +if (config.mode === 'development') { + settings.INFO = true; + settings.DEBUG = true; +} + +function log(level, string) { + if (!!settings[level]) { + let color = colors.Bright; + if (level === levels.ERROR) color = colors.FgRed; + if (level === levels.WARN) color = colors.FgYellow; + + if (level.length === 4) {level = level + " "}; // ensure logs are aligned + console.log(colors.Dim, level + " " + new Date().toISOString() + ": ", color, string, colors.Reset); + } +} + +module.exports = { + levels, + log, + error: (string) => {log(levels.ERROR, string)}, + warn: (string) => {log(levels.WARN, string)}, + info: (string) => {log(levels.INFO, string)}, + debug: (string) => {log(levels.DEBUG, string)}, +}; \ No newline at end of file diff --git a/test/cases/oldSubmitSponsorTimes.js b/test/cases/oldSubmitSponsorTimes.js index bf3ac7a..31f90ba 100644 --- a/test/cases/oldSubmitSponsorTimes.js +++ b/test/cases/oldSubmitSponsorTimes.js @@ -13,7 +13,7 @@ describe('postVideoSponsorTime (Old submission method)', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcQ"); + let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcQ"]); if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { done() } else { @@ -31,7 +31,7 @@ describe('postVideoSponsorTime (Old submission method)', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcE"); + let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcE"]); if (row.startTime === 1 && row.endTime === 11 && row.category === "sponsor") { done() } else { diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index ab743c7..b31fedb 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -13,7 +13,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcR"); + let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcR"]); if (row.startTime === 2 && row.endTime === 10 && row.category === "sponsor") { done() } else { @@ -40,7 +40,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcF"); + let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcF"]); if (row.startTime === 0 && row.endTime === 10 && row.category === "sponsor") { done() } else { @@ -70,7 +70,7 @@ describe('postSkipSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let rows = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").all("dQw4w9WgXcR"); + let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcR"]); let success = true; if (rows.length === 2) { for (const row of rows) { @@ -90,6 +90,26 @@ describe('postSkipSegments', () => { }); }).timeout(5000); + it('Should be accepted if a non-sponsor is less than 1 second', (done) => { + request.post(utils.getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=intro", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 200) done(); // pass + else done("non 200 status code: " + res.statusCode + " ("+body+")"); + }); + }); + + it('Should be rejected if a sponsor is less than 1 second', (done) => { + request.post(utils.getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 400) done(); // pass + else done("non 403 status code: " + res.statusCode + " ("+body+")"); + }); + }); + it('Should be rejected if over 80% of the video', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=qqwerty&startTime=30&endTime=1000000&userID=testing", null, diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index 167347c..bcbca78 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -1,22 +1,31 @@ -var request = require('request'); -var db = require('../../src/databases/databases.js').db; -var utils = require('../utils.js'); -var getHash = require('../../src/utils/getHash.js') +const request = require('request'); +const { db, privateDB } = require('../../src/databases/databases.js'); +const utils = require('../utils.js'); +const getHash = require('../../src/utils/getHash.js'); describe('voteOnSponsorTime', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-testtesttest', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0)"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0)"); + db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)"); db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0)"); db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0)"); db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0)"); db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0)"); + db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)"); + db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); + privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); }); - it('Should be able to upvote a segment', (done) => { request.get(utils.getbaseURL() @@ -24,7 +33,7 @@ describe('voteOnSponsorTime', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT votes FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-0"); + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-0"]); if (row.votes === 3) { done() } else { @@ -42,7 +51,7 @@ describe('voteOnSponsorTime', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT votes FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-2"); + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-2"]); if (row.votes < 10) { done() } else { @@ -54,13 +63,85 @@ describe('voteOnSponsorTime', () => { }); }); + it('Should not be able to downvote the same segment when voting from a different user on the same IP', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID3&UUID=vote-uuid-2&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-2"]); + if (row.votes === 9) { + done() + } else { + done("Vote did not fail. Submission went from 9 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it("Should not be able to downvote a segment if the user is shadow banned", (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID4&UUID=vote-uuid-1.6&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-1.6"]); + if (row.votes === 10) { + done() + } else { + done("Vote did not fail. Submission went from 10 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it("Should not be able to upvote a segment if the user hasn't submitted yet", (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=hasNotSubmittedID&UUID=vote-uuid-1&type=1", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-1"]); + if (row.votes === 2) { + done() + } else { + done("Vote did not fail. Submission went from 2 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it("Should not be able to downvote a segment if the user hasn't submitted yet", (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=hasNotSubmittedID&UUID=vote-uuid-1.5&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-1.5"]); + if (row.votes === 10) { + done() + } else { + done("Vote did not fail. Submission went from 10 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + it('VIP should be able to completely downvote a segment', (done) => { request.get(utils.getbaseURL() + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-3&type=0", null, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT votes FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-3"); + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-3"]); if (row.votes <= -2) { done() } else { @@ -72,13 +153,49 @@ describe('voteOnSponsorTime', () => { }); }); + it('should be able to completely downvote your own segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=own-submission-id&UUID=own-submission-uuid&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["own-submission-uuid"]); + if (row.votes <= -2) { + done() + } else { + done("Vote did not succeed. Submission went from 500 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('should not be able to completely downvote somebody elses segment', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=not-own-submission-uuid&type=0", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["not-own-submission-uuid"]); + if (row.votes === 499) { + done() + } else { + done("Vote did not succeed. Submission went from 500 votes to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + it('Should be able to vote for a category and it should immediately change (for now)', (done) => { request.get(utils.getbaseURL() + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=intro", null, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT category FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-4"); + let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-4"]); if (row.category === "intro") { done() } else { @@ -96,7 +213,7 @@ describe('voteOnSponsorTime', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT category FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-4"); + let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-4"]); if (row.category === "outro") { done() } else { @@ -114,8 +231,8 @@ describe('voteOnSponsorTime', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT category FROM sponsorTimes WHERE UUID = ?").get("vote-uuid-5"); - let row2 = db.prepare("SELECT votes FROM categoryVotes WHERE UUID = ? and category = ?").get("vote-uuid-5", "outro"); + let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-5"]); + let row2 = db.prepare('get', "SELECT votes FROM categoryVotes WHERE UUID = ? and category = ?", ["vote-uuid-5", "outro"]); if (row.category === "outro" && row2.votes === 500) { done() } else { @@ -127,4 +244,48 @@ describe('voteOnSponsorTime', () => { }); }); + it('Should not be able to category-vote on an invalid UUID submission', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID3&UUID=invalid-uuid&category=intro", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done(); + } else { + done("Status code was " + res.statusCode + " instead of 400."); + } + }); + }); + + it('Non-VIP should not be able to upvote "dead" submission', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); + } else { + done("Status code was " + res.statusCode + " instead of 403"); + } + }); + }); + + it('VIP should be able to upvote "dead" submission', (done) => { + request.get(utils.getbaseURL() + + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&type=1", null, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-5"]); + if (row.votes > -3) { + done() + } else { + done("Vote did not succeed. Votes raised from -3 to " + row.votes); + } + } else { + done("Status code was " + res.statusCode); + } + }); + }); + }); \ No newline at end of file