diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql new file mode 100644 index 0000000..f6816bc --- /dev/null +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -0,0 +1,13 @@ +BEGIN TRANSACTION; + +/* Add new table: noSegments */ +CREATE TABLE "noSegments" ( + "videoID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "category" TEXT NOT NULL +); + +/* Add version to config */ +UPDATE config SET value = 2 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/app.js b/src/app.js index c70115d..45715b7 100644 --- a/src/app.js +++ b/src/app.js @@ -22,6 +22,8 @@ var getViewsForUser = require('./routes/getViewsForUser.js'); var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); +var postNoSegments = require('./routes/postNoSegments.js'); +var getIsUserVIP = require('./routes/getIsUserVIP.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -89,6 +91,13 @@ app.get('/api/getTotalStats', getTotalStats); //send out a formatted time saved total app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); +//submit video containing no segments +app.post('/api/noSegments', postNoSegments); + +//get if user is a vip +app.get('/api/isUserVIP', getIsUserVIP); + + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); }); diff --git a/src/databases/Mysql.js b/src/databases/Mysql.js index 8f3d8ba..a6b2850 100644 --- a/src/databases/Mysql.js +++ b/src/databases/Mysql.js @@ -1,5 +1,4 @@ var MysqlInterface = require('sync-mysql'); -var config = require('../config.js'); const logger = require('../utils/logger.js'); class Mysql { diff --git a/src/databases/databases.js b/src/databases/databases.js index 1a1b448..a3aeb59 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -3,7 +3,8 @@ var Sqlite3 = require('better-sqlite3'); var fs = require('fs'); var path = require('path'); var Sqlite = require('./Sqlite.js') -var Mysql = require('./Mysql.js') +var Mysql = require('./Mysql.js'); +const logger = require('../utils/logger.js'); let options = { readonly: config.readOnly, @@ -60,12 +61,16 @@ if (config.mysql) { let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; - let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; + logger.debug('db update: trying ' + path); while (fs.existsSync(path)) { + logger.debug('db update: updating ' + path); db.exec(fs.readFileSync(path).toString()); versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value; - path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; + logger.debug('db update: trying ' + path); } + logger.debug('db update: no file ' + path); } } \ No newline at end of file diff --git a/src/routes/getIsUserVIP.js b/src/routes/getIsUserVIP.js new file mode 100644 index 0000000..63efd02 --- /dev/null +++ b/src/routes/getIsUserVIP.js @@ -0,0 +1,31 @@ +var db = require('../databases/databases.js').db; + +var getHash = require('../utils/getHash.js'); +const logger = require('../utils/logger.js'); +const isUserVIP = require('../utils/isUserVIP.js'); + +module.exports = (req, res) => { + let userID = req.query.userID; + + if (userID == undefined) { + //invalid request + res.sendStatus(400); + return; + } + + //hash the userID + userID = getHash(userID); + + try { + let vipState = isUserVIP(userID); + res.status(200).json({ + hashedUserID: userID, + vip: vipState + }); + } catch (err) { + logger.error(err); + res.sendStatus(500); + + return; + } +} diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index a003a6c..81aae1c 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -1,4 +1,3 @@ -var fs = require('fs'); var config = require('../config.js'); var databases = require('../databases/databases.js'); diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js new file mode 100644 index 0000000..cdbfce9 --- /dev/null +++ b/src/routes/postNoSegments.js @@ -0,0 +1,74 @@ +const db = require('../databases/databases.js').db; +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); + +module.exports = (req, res) => { + // Collect user input data + let videoID = req.body.videoID; + let userID = req.body.userID; + let categories = req.body.categories; + + // Check input data is valid + if (!videoID + || !userID + || !categories + || !Array.isArray(categories) + || categories.length === 0 + ) { + res.status(400).json({ + message: 'Bad Format' + }); + return; + } + + // Check if user is VIP + userID = getHash(userID); + let userIsVIP = isUserVIP(userID); + + if (!userIsVIP) { + res.status(403).json({ + message: 'Must be a VIP to mark videos.' + }); + return; + } + + // Get existing no segment markers + let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]); + if (!noSegmentList || noSegmentList.length === 0) { + noSegmentList = []; + } else { + noSegmentList = noSegmentList.map((obj) => { + return obj.category; + }); + } + + // get user categories not already submitted that match accepted format + let categoriesToMark = categories.filter((category) => { + return !!category.match(/^[_a-zA-Z]+$/); + }).filter((category) => { + return noSegmentList.indexOf(category) === -1; + }); + + // remove any duplicates + categoriesToMark = categoriesToMark.filter((category, index) => { + return categoriesToMark.indexOf(category) === index; + }); + + // create database entry + categoriesToMark.forEach((category) => { + try { + db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]); + } catch (err) { + logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'"); + logger.error(err); + res.status(500).json({ + message: "Internal Server Error: Could not write marker to the database." + }); + } + }); + + res.status(200).json({ + submitted: categoriesToMark + }); +}; \ No newline at end of file diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 799711f..69b3e22 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -148,9 +148,9 @@ function proxySubmission(req) { request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => { if (config.mode === 'development') { if (!err) { - logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); + logger.debug('Proxy Submission: ' + result.statusCode + ' ('+result.body+')'); } else { - logger.debug("Proxy Submission: Failed to make call"); + logger.error("Proxy Submission: Failed to make call"); } } }); @@ -187,6 +187,7 @@ module.exports = async function postSkipSegments(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(getIP(req) + config.globalSalt); + let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category }); // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { @@ -195,6 +196,18 @@ module.exports = async function postSkipSegments(req, res) { return; } + // Reject segemnt if it's in the no segments list + if (noSegmentList.indexOf(segments[i].category) !== -1) { + // TODO: Do something about the fradulent submission + logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'"); + res.status(403).send( + "Request rejected by auto moderator: This video has been reported as not containing any segments with the category '" + + segments[i].category + "'. If you believe this is incorrect, contact someone on Discord." + ); + return; + } + + let startTime = parseFloat(segments[i].segment[0]); let endTime = parseFloat(segments[i].segment[1]); diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index f3be68b..8f9b381 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -5,7 +5,7 @@ var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); var getFormattedTime = require('../utils/getFormattedTime.js'); var isUserTrustworthy = require('../utils/isUserTrustworthy.js'); -const {getVoteAuthor, getVoteAuthorRaw, dispatchEvent} = require('../utils/webhookUtils.js'); +const { getVoteAuthor, getVoteAuthorRaw, dispatchEvent } = require('../utils/webhookUtils.js'); var databases = require('../databases/databases.js'); var db = databases.db; diff --git a/src/utils/isUserVIP.js b/src/utils/isUserVIP.js new file mode 100644 index 0000000..eeb25dc --- /dev/null +++ b/src/utils/isUserVIP.js @@ -0,0 +1,8 @@ +const databases = require('../databases/databases.js'); +const db = databases.db; + +module.exports = (userID) => { + return db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0; +} + + diff --git a/src/utils/logger.js b/src/utils/logger.js index 731b5cf..66ebb03 100644 --- a/src/utils/logger.js +++ b/src/utils/logger.js @@ -46,7 +46,7 @@ if (config.mode === 'development') { settings.INFO = true; settings.DEBUG = true; } else if (config.mode === 'test') { - settings.WARN = false; + settings.WARN = false; } function log(level, string) { diff --git a/test.js b/test.js index 45ace90..9810786 100644 --- a/test.js +++ b/test.js @@ -9,6 +9,7 @@ if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB); var createServer = require('./src/app.js'); var createMockServer = require('./test/mocks.js'); +const logger = require('./src/utils/logger.js'); // Instantiate a Mocha instance. var mocha = new Mocha(); @@ -27,9 +28,9 @@ fs.readdirSync(testDir).filter(function(file) { }); var mockServer = createMockServer(() => { - console.log("Started mock HTTP Server"); + logger.info("Started mock HTTP Server"); var server = createServer(() => { - console.log("Started main HTTP server"); + logger.info("Started main HTTP server"); // Run the tests. mocha.run(function(failures) { mockServer.close(); diff --git a/test/cases/getIsUserVIP.js b/test/cases/getIsUserVIP.js new file mode 100644 index 0000000..1bbfbfe --- /dev/null +++ b/test/cases/getIsUserVIP.js @@ -0,0 +1,57 @@ +var request = require('request'); +var utils = require('../utils.js'); +var db = require('../../src/databases/databases.js').db; +var getHash = require('../../src/utils/getHash.js'); + +describe('getIsUserVIP', () => { + before(() => { + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("supertestman") + "')"); + }); + + it('Should be able to get a 200', (done) => { + request.get(utils.getbaseURL() + + "/api/isUserVIP?userID=supertestman", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200: " + res.statusCode); + else done(); // pass + }); + }); + + + it('Should get a 400 if no userID', (done) => { + request.get(utils.getbaseURL() + + "/api/isUserVIP", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 400) done("non 400: " + res.statusCode); + else done(); // pass + }); + }); + + it('Should say a VIP is a VIP', (done) => { + request.get(utils.getbaseURL() + + "/api/isUserVIP?userID=supertestman", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200: " + res.statusCode); + else { + if (JSON.parse(body).vip === true) done(); // pass + else done("Result was non-vip when should have been vip"); + } + }); + }); + + it('Should say a normal user is not a VIP', (done) => { + request.get(utils.getbaseURL() + + "/api/isUserVIP?userID=regulartestman", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200: " + res.statusCode); + else { + if (JSON.parse(body).vip === false) done(); // pass + else done("Result was vip when should have been non-vip"); + } + }); + }); +}); \ No newline at end of file diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js new file mode 100644 index 0000000..91dea2a --- /dev/null +++ b/test/cases/noSegmentRecords.js @@ -0,0 +1,408 @@ +var request = require('request'); + +var utils = require('../utils.js'); +const getHash = require('../../src/utils/getHash.js'); + +var databases = require('../../src/databases/databases.js'); +const logger = require('../../src/utils/logger.js'); +var db = databases.db; + +describe('noSegmentRecords', () => { + before(() => { + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')"); + + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')"); + + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')"); + }); + + it('Should update the database version when starting the application', (done) => { + let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (version > 1) done(); + else done('Version isn\'t greater that 1. Version is ' + version); + }); + + it('Should be able to submit categories not in video (http response)', (done) => { + let json = { + videoID: 'no-segments-video-id', + userID: 'VIPUser-noSegments', + categories: [ + 'outro', + 'shilling', + 'shilling', + 'shil ling', + '', + 'intro' + ] + }; + + let expected = { + submitted: [ + 'outro', + 'shilling' + ] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + if (JSON.stringify(body) === JSON.stringify(expected)) { + done(); + } else { + done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(body)); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to submit categories not in video (sql check)', (done) => { + let json = { + videoID: 'no-segments-video-id-1', + userID: 'VIPUser-noSegments', + categories: [ + 'outro', + 'shilling', + 'shilling', + 'shil ling', + '', + 'intro' + ] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['no-segments-video-id-1']); + if (result.length !== 4) { + console.log(result); + done("Expected 4 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to submit categories with _ in the category', (done) => { + let json = { + videoID: 'underscore', + userID: 'VIPUser-noSegments', + categories: [ + 'word_word', + ] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['underscore']); + if (result.length !== 1) { + console.log(result); + done("Expected 1 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to submit categories with upper and lower case in the category', (done) => { + let json = { + videoID: 'bothCases', + userID: 'VIPUser-noSegments', + categories: [ + 'wordWord', + ] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['bothCases']); + if (result.length !== 1) { + console.log(result); + done("Expected 1 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should not be able to submit categories with $ in the category', (done) => { + let json = { + videoID: 'specialChar', + userID: 'VIPUser-noSegments', + categories: [ + 'word&word', + ] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['specialChar']); + if (result.length !== 0) { + console.log(result); + done("Expected 0 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for missing params', (done) => { + request.post(utils.getbaseURL() + + "/api/noSegments", {json: {}}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for no categories', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categories: [] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for no userID', (done) => { + let json = { + videoID: 'test', + userID: null, + categories: ['sponsor'] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for no videoID', (done) => { + let json = { + videoID: null, + userID: 'test', + categories: ['sponsor'] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 object categories)', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categories: {} + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 bad format categories', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categories: 'sponsor' + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 403 if user is not VIP', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categories: [ + 'sponsor' + ] + }; + + request.post(utils.getbaseURL() + + "/api/noSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done(); + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + /* + * Submission tests in this file do not check database records, only status codes. + * To test the submission code properly see ./test/cases/postSkipSegments.js + */ + + it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "noSubmitVideo", + segments: [{ + segment: [20, 40], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should not be able to submit segments to a video where any of the submissions with a no-segment record', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "noSubmitVideo", + segments: [{ + segment: [20, 40], + category: "sponsor" + },{ + segment: [50, 60], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + + it('Should be able to submit a segment to a video with a different no-segment record', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "noSubmitVideo", + segments: [{ + segment: [20, 40], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to submit a segment to a video with no no-segment records', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "normalVideo", + segments: [{ + segment: [20, 40], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); +}); \ No newline at end of file