From b74ca3962bfe396b03268ad987d801416d0ec5b7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 14:05:42 -0400 Subject: [PATCH 01/20] Updated table to include category --- databases/_sponsorTimes.db.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 675ab5e..1514e0e 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS "sponsorTimes" ( "userID" TEXT NOT NULL, "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, + "category" TEXT NOT NULL; "shadowHidden" INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS "userNames" ( From bd2f00d5f72dae1d487ea6266358b881d3778eb8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 16:43:47 -0400 Subject: [PATCH 02/20] Added new submission method --- src/app.js | 11 +- src/routes/oldSubmitSponsorTimes.js | 9 + src/routes/postSkipSegments.js | 212 ++++++++++++++++++ src/routes/submitSponsorTimes.js | 192 ---------------- ...onsorTimes.js => oldSubmitSponsorTimes.js} | 14 +- test/cases/postSkipSegments.js | 150 +++++++++++++ 6 files changed, 391 insertions(+), 197 deletions(-) create mode 100644 src/routes/oldSubmitSponsorTimes.js create mode 100644 src/routes/postSkipSegments.js delete mode 100644 src/routes/submitSponsorTimes.js rename test/cases/{submitSponsorTimes.js => oldSubmitSponsorTimes.js} (60%) create mode 100644 test/cases/postSkipSegments.js diff --git a/src/app.js b/src/app.js index 5924715..fa29c38 100644 --- a/src/app.js +++ b/src/app.js @@ -9,7 +9,8 @@ var loggerMiddleware = require('./middleware/logger.js'); // Routes var getVideoSponsorTimes = require('./routes/getVideoSponsorTimes.js'); -var submitSponsorTimes = require('./routes/submitSponsorTimes.js'); +var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); +var postSkipSegments = require('./routes/postSkipSegments.js'); var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js'); var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js'); var setUsername = require('./routes/setUsername.js'); @@ -26,13 +27,17 @@ var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); //setup CORS correctly app.use(corsMiddleware); app.use(loggerMiddleware); +app.use(express.json()) //add the get function app.get('/api/getVideoSponsorTimes', getVideoSponsorTimes); +//add the oldpost function +app.get('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); +app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); + //add the post function -app.get('/api/postVideoSponsorTimes', submitSponsorTimes); -app.post('/api/postVideoSponsorTimes', submitSponsorTimes); +app.post('/api/skipSegments', postSkipSegments); //voting endpoint app.get('/api/voteOnSponsorTime', voteOnSponsorTime); diff --git a/src/routes/oldSubmitSponsorTimes.js b/src/routes/oldSubmitSponsorTimes.js new file mode 100644 index 0000000..b818a24 --- /dev/null +++ b/src/routes/oldSubmitSponsorTimes.js @@ -0,0 +1,9 @@ +var config = require('../config.js'); + +var postSkipSegments = require('./postSkipSegments.js'); + +module.exports = async function submitSponsorTimes(req, res) { + req.query.category = "sponsor"; + + return postSkipSegments(req, res); +} diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js new file mode 100644 index 0000000..6562d94 --- /dev/null +++ b/src/routes/postSkipSegments.js @@ -0,0 +1,212 @@ +var config = require('../config.js'); + +var databases = require('../databases/databases.js'); +var db = databases.db; +var privateDB = databases.privateDB; +var YouTubeAPI = require('../utils/youtubeAPI.js'); + +var getHash = require('../utils/getHash.js'); +var getIP = require('../utils/getIP.js'); +var getFormattedTime = require('../utils/getFormattedTime.js'); + +// TODO: might need to be a util +//returns true if the user is considered trustworthy +//this happens after a user has made 5 submissions and has less than 60% downvoted submissions +async function isUserTrustworthy(userID) { + //check to see if this user how many submissions this user has submitted + let totalSubmissionsRow = db.prepare("SELECT count(*) as totalSubmissions, sum(votes) as voteSum FROM sponsorTimes WHERE userID = ?").get(userID); + + if (totalSubmissionsRow.totalSubmissions > 5) { + //check if they have a high downvote ratio + let downvotedSubmissionsRow = db.prepare("SELECT count(*) as downvotedSubmissions FROM sponsorTimes WHERE userID = ? AND (votes < 0 OR shadowHidden > 0)").get(userID); + + return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || + (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions); + } + + return true; +} + +function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { + //check if they are a first time user + //if so, send a notification to discord + if (config.youtubeAPIKey !== null && config.discordFirstTimeSubmissionsWebhookURL !== null) { + let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(userID); + + // If it is a first time submission + if (userSubmissionCountRow.submissionCount === 0) { + YouTubeAPI.videos.list({ + part: "snippet", + id: videoID + }, function (err, data) { + if (err || data.items.length === 0) { + err && console.log(err); + return; + } + + request.post(config.discordFirstTimeSubmissionsWebhookURL, { + json: { + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (segmentInfo.segment.toFixed(0) - 2), + "description": "Submission ID: " + UUID + + "\n\nTimestamp: " + + getFormattedTime(segmentInfo.segment[0]) + " to " + getFormattedTime(segmentInfo.segment[1]) + + "\n\nCategory: " + segmentInfo.category, + "color": 10813440, + "author": { + "name": userID + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + } + }] + } + }, (err, res) => { + if (err) { + console.log("Failed to send first time submission Discord hook."); + console.log(JSON.stringify(err)); + console.log("\n"); + } else if (res && res.statusCode >= 400) { + console.log("Error sending first time submission Discord hook"); + console.log(JSON.stringify(res)); + console.log("\n"); + } + }); + }); + } + } +} + +module.exports = async function postSkipSegments(req, res) { + let videoID = req.query.videoID || req.body.videoID; + let userID = req.query.userID || req.body.userID; + + let segments = req.body.segments; + + if (segments === undefined) { + // Use query instead + segments = [{ + segment: [req.query.startTime, req.query.endTime], + category: req.query.category + }]; + } + + //check if all correct inputs are here and the length is 1 second or more + if (videoID == undefined || userID == undefined || segments == undefined || segments.length < 1) { + //invalid request + res.sendStatus(400); + return; + } + + //hash the userID + userID = getHash(userID); + + //hash the ip 5000 times so no one can get it from the database + let hashedIP = getHash(getIP(req) + config.globalSalt); + + // Check if all submissions are correct + for (let i = 0; i < segments.length; i++) { + if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { + //invalid request + res.sendStatus(400); + return; + } + + let startTime = parseFloat(segments[i].segment[0]); + let endTime = parseFloat(segments[i].segment[1]); + + if (Math.abs(startTime - endTime) < 1 || isNaN(startTime) || isNaN(endTime) + || startTime === Infinity || endTime === Infinity || startTime > endTime) { + //invalid request + res.sendStatus(400); + return; + } + + //check if this info has already been submitted before + let duplicateCheck2Row = + db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID]); + if (duplicateCheck2Row == null) { + res.sendStatus(409); + return; + } + } + + try { + //check if this user is on the vip list + let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(userID); + + //get current time + let timeSubmitted = Date.now(); + + let yesterday = timeSubmitted - 86400000; + + //check to see if this ip has submitted too many sponsors today + let rateLimitCheckRow = privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday]); + + if (rateLimitCheckRow.count >= 10) { + //too many sponsors for the same video from the same ip address + res.sendStatus(429); + + return; + } + + //check to see if the user has already submitted sponsors for this video + let duplicateCheckRow = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID]); + + if (duplicateCheckRow.count >= 8) { + //too many sponsors for the same video from the same user + res.sendStatus(429); + + return; + } + + //check to see if this user is shadowbanned + let shadowBanRow = privateDB.prepare("SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?").get(userID); + + let shadowBanned = shadowBanRow.userCount; + + if (!(await isUserTrustworthy(userID))) { + //hide this submission as this user is untrustworthy + shadowBanned = 1; + } + + let startingVotes = 0; + if (vipRow.userCount > 0) { + //this user is a vip, start them at a higher approval rating + startingVotes = 10; + } + + for (const segmentInfo of segments) { + //this can just be a hash of the data + //it's better than generating an actual UUID like what was used before + //also better for duplication checking + let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] + + segmentInfo.segment[1] + segmentInfo.category + userID, 1); + + try { + db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], + segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); + + //add to private db as well + privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); + + res.sendStatus(200); + } catch (err) { + //a DB change probably occurred + res.sendStatus(502); + console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category); + + return; + } + + // Discord notification + sendDiscordNotification(userID, videoID, UUID, segmentInfo); + } + } catch (err) { + console.error(err); + + res.send(500); + } +} diff --git a/src/routes/submitSponsorTimes.js b/src/routes/submitSponsorTimes.js deleted file mode 100644 index a550f00..0000000 --- a/src/routes/submitSponsorTimes.js +++ /dev/null @@ -1,192 +0,0 @@ -var config = require('../config.js'); - -var databases = require('../databases/databases.js'); -var db = databases.db; -var privateDB = databases.privateDB; -var YouTubeAPI = require('../utils/youtubeAPI.js'); - -var getHash = require('../utils/getHash.js'); -var getIP = require('../utils/getIP.js'); -var getFormattedTime = require('../utils/getFormattedTime.js'); - -// TODO: might need to be a util -//returns true if the user is considered trustworthy -//this happens after a user has made 5 submissions and has less than 60% downvoted submissions -async function isUserTrustworthy(userID) { - //check to see if this user how many submissions this user has submitted - let totalSubmissionsRow = db.prepare("SELECT count(*) as totalSubmissions, sum(votes) as voteSum FROM sponsorTimes WHERE userID = ?").get(userID); - - if (totalSubmissionsRow.totalSubmissions > 5) { - //check if they have a high downvote ratio - let downvotedSubmissionsRow = db.prepare("SELECT count(*) as downvotedSubmissions FROM sponsorTimes WHERE userID = ? AND (votes < 0 OR shadowHidden > 0)").get(userID); - - return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || - (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions); - } - - return true; -} - -module.exports = async function submitSponsorTimes(req, res) { - let videoID = req.query.videoID; - let startTime = req.query.startTime; - let endTime = req.query.endTime; - let userID = req.query.userID; - - //check if all correct inputs are here and the length is 1 second or more - if (videoID == undefined || startTime == undefined || endTime == undefined || userID == undefined - || Math.abs(startTime - endTime) < 1) { - //invalid request - res.sendStatus(400); - return; - } - - //hash the userID - userID = getHash(userID); - - //hash the ip 5000 times so no one can get it from the database - let hashedIP = getHash(getIP(req) + config.globalSalt); - - startTime = parseFloat(startTime); - endTime = parseFloat(endTime); - - if (isNaN(startTime) || isNaN(endTime)) { - //invalid request - res.sendStatus(400); - return; - } - - if (startTime === Infinity || endTime === Infinity) { - //invalid request - res.sendStatus(400); - return; - } - - if (startTime > endTime) { - //time can't go backwards - res.sendStatus(400); - return; - } - - try { - //check if this user is on the vip list - let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(userID); - - //this can just be a hash of the data - //it's better than generating an actual UUID like what was used before - //also better for duplication checking - let UUID = getHash(videoID + startTime + endTime + userID, 1); - - //get current time - let timeSubmitted = Date.now(); - - let yesterday = timeSubmitted - 86400000; - - //check to see if this ip has submitted too many sponsors today - let rateLimitCheckRow = privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday]); - - if (rateLimitCheckRow.count >= 10) { - //too many sponsors for the same video from the same ip address - res.sendStatus(429); - } else { - //check to see if the user has already submitted sponsors for this video - let duplicateCheckRow = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID]); - - if (duplicateCheckRow.count >= 8) { - //too many sponsors for the same video from the same user - res.sendStatus(429); - } else { - //check if this info has already been submitted first - let duplicateCheck2Row = db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID]); - - //check to see if this user is shadowbanned - let shadowBanRow = privateDB.prepare("SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?").get(userID); - - let shadowBanned = shadowBanRow.userCount; - - if (!(await isUserTrustworthy(userID))) { - //hide this submission as this user is untrustworthy - shadowBanned = 1; - } - - let startingVotes = 0; - if (vipRow.userCount > 0) { - //this user is a vip, start them at a higher approval rating - startingVotes = 10; - } - - if (duplicateCheck2Row == null) { - //not a duplicate, execute query - try { - db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, startingVotes, UUID, userID, timeSubmitted, 0, shadowBanned); - - //add to private db as well - privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); - - res.sendStatus(200); - } catch (err) { - //a DB change probably occurred - res.sendStatus(502); - console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + startTime + ", " + "endTime" + ", " + userID); - - return; - } - } else { - res.sendStatus(409); - } - - //check if they are a first time user - //if so, send a notification to discord - if (config.youtubeAPIKey !== null && config.discordFirstTimeSubmissionsWebhookURL !== null && duplicateCheck2Row == null) { - let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(userID); - - // If it is a first time submission - if (userSubmissionCountRow.submissionCount === 0) { - YouTubeAPI.videos.list({ - part: "snippet", - id: videoID - }, function (err, data) { - if (err || data.items.length === 0) { - err && console.log(err); - return; - } - - request.post(config.discordFirstTimeSubmissionsWebhookURL, { - json: { - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), - "description": "Submission ID: " + UUID + - "\n\nTimestamp: " + - getFormattedTime(startTime) + " to " + getFormattedTime(endTime), - "color": 10813440, - "author": { - "name": userID - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - } - }] - } - }, (err, res) => { - if (err) { - console.log("Failed to send first time submission Discord hook."); - console.log(JSON.stringify(err)); - console.log("\n"); - } else if (res && res.statusCode >= 400) { - console.log("Error sending first time submission Discord hook"); - console.log(JSON.stringify(res)); - console.log("\n"); - } - }); - }); - } - } - } - } - } catch (err) { - console.error(err); - - res.send(500); - } -} diff --git a/test/cases/submitSponsorTimes.js b/test/cases/oldSubmitSponsorTimes.js similarity index 60% rename from test/cases/submitSponsorTimes.js rename to test/cases/oldSubmitSponsorTimes.js index 181b550..4c0b8a1 100644 --- a/test/cases/submitSponsorTimes.js +++ b/test/cases/oldSubmitSponsorTimes.js @@ -3,8 +3,8 @@ var request = require('request'); var utils = require('../utils.js'); -describe('postVideoSponsorTime', () => { - it('Should be able to create a time', (done) => { +describe('postVideoSponsorTime (Old submission method)', () => { + it('Should be able to submit a time (GET)', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test", null, (err, res, body) => { @@ -14,6 +14,16 @@ describe('postVideoSponsorTime', () => { }); }); + it('Should be able to submit a time (POST)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test", null, + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + it('Should return 400 for missing params', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?startTime=1&endTime=10&userID=test", null, diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js new file mode 100644 index 0000000..e058466 --- /dev/null +++ b/test/cases/postSkipSegments.js @@ -0,0 +1,150 @@ +var assert = require('assert'); +var request = require('request'); + +var utils = require('../utils.js'); + +describe('postSkipSegments', () => { + it('Should be able to submit a single time (Params method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test&category=sponsor", null, + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + + it('Should be able to submit a single time (JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + videoID: "djgofQKWmXc", + segments: [{ + segment: [0, 10], + category: "sponsor" + }] + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + + it('Should be able to submit multiple times (JSON method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + videoID: "djgofQKWmXc", + segments: [{ + segment: [0, 10], + category: "sponsor" + }, { + segment: [30, 60], + category: "intro" + }] + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + + it('Should return 400 for missing params (Params method)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes?startTime=1&endTime=10&userID=test", null, + (err, res, body) => { + if (err) done(false); + if (res.statusCode === 400) done(); + else done(false); + }); + }); + + it('Should return 400 for missing params (JSON method) 1', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + segments: [{ + segment: [0, 10], + category: "sponsor" + }, { + segment: [30, 60], + category: "intro" + }] + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + it('Should return 400 for missing params (JSON method) 2', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + videoID: "djgofQKWmXc" + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + it('Should return 400 for missing params (JSON method) 3', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + videoID: "djgofQKWmXc", + segments: [{ + segment: [0], + category: "sponsor" + }, { + segment: [30, 60], + category: "intro" + }] + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + it('Should return 400 for missing params (JSON method) 4', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + videoID: "djgofQKWmXc", + segments: [{ + segment: [0, 10] + }, { + segment: [30, 60], + category: "intro" + }] + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); + it('Should return 400 for missing params (JSON method) 5', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", JSON.stringify({ + body: { + videoID: "djgofQKWmXc" + } + }), + (err, res, body) => { + if (err) done(false); + else if (res.statusCode === 200) done(); + else done(false); + }); + }); +}); \ No newline at end of file From 3a203d249e4a6671d37a9eeaaba5d8439076188b Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 16:46:19 -0400 Subject: [PATCH 03/20] Added Github actions --- .github/workflows/ci.yml | 18 ++++++++++++++++++ config.json.example | 2 -- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f1cbd7d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: [push, pull_request] + +jobs: + + build: + name: Run Tests + runs-on: ubuntu-latest + + steps: + # Initialization + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + - run: npm install + + - name: Run Tests + run: npm test diff --git a/config.json.example b/config.json.example index b8a9d79..0d76f8b 100644 --- a/config.json.example +++ b/config.json.example @@ -1,6 +1,4 @@ { - // Remove all comments from the config when using it! - "port": 80, "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]", "adminUserID": "[the hashed id of the user who can perform admin actions]", From 67c608e76fe2df2a4414abdb39f6f631ea4b2b18 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 16:59:57 -0400 Subject: [PATCH 04/20] Added github actions logging --- src/config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config.js b/src/config.js index 1bc3721..86fbefd 100644 --- a/src/config.js +++ b/src/config.js @@ -2,6 +2,8 @@ var fs = require('fs'); var config = undefined; +console.log(process.env.npm_lifecycle_script) + // Check to see if launched in test mode if (process.env.npm_lifecycle_script === 'node test.js') { config = JSON.parse(fs.readFileSync('test.json')); From 8eca458e381e6dc4a7774229ae2f6338f4fe469d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 17:25:20 -0400 Subject: [PATCH 05/20] Added DB checks to tests and fixed getIP using the wrong config --- src/config.js | 2 -- src/utils/getIP.js | 2 +- test/cases/oldSubmitSponsorTimes.js | 25 ++++++++++++++--- test/cases/postSkipSegments.js | 42 ++++++++++++++++++++++++----- 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/config.js b/src/config.js index 86fbefd..1bc3721 100644 --- a/src/config.js +++ b/src/config.js @@ -2,8 +2,6 @@ var fs = require('fs'); var config = undefined; -console.log(process.env.npm_lifecycle_script) - // Check to see if launched in test mode if (process.env.npm_lifecycle_script === 'node test.js') { config = JSON.parse(fs.readFileSync('test.json')); diff --git a/src/utils/getIP.js b/src/utils/getIP.js index 8379ec0..cc11057 100644 --- a/src/utils/getIP.js +++ b/src/utils/getIP.js @@ -1,5 +1,5 @@ var fs = require('fs'); -var config = JSON.parse(fs.readFileSync('config.json')); +var config = require('./config.js'); module.exports = function getIP(req) { return config.behindProxy ? req.headers['x-forwarded-for'] : req.connection.remoteAddress; diff --git a/test/cases/oldSubmitSponsorTimes.js b/test/cases/oldSubmitSponsorTimes.js index 4c0b8a1..d205df5 100644 --- a/test/cases/oldSubmitSponsorTimes.js +++ b/test/cases/oldSubmitSponsorTimes.js @@ -3,14 +3,24 @@ var request = require('request'); var utils = require('../utils.js'); +var databases = require('../../src/databases/databases.js'); +var db = databases.db; + describe('postVideoSponsorTime (Old submission method)', () => { it('Should be able to submit a time (GET)', (done) => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test", null, (err, res, body) => { if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); + if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { + done() + return; + } + } + + done(false); }); }); @@ -19,8 +29,15 @@ describe('postVideoSponsorTime (Old submission method)', () => { + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test", null, (err, res, body) => { if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); + if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { + done() + return; + } + } + + done(false); }); }); diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index e058466..446d802 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -3,14 +3,24 @@ var request = require('request'); var utils = require('../utils.js'); +var databases = require('../../src/databases/databases.js'); +var db = databases.db; + describe('postSkipSegments', () => { it('Should be able to submit a single time (Params method)', (done) => { request.post(utils.getbaseURL() + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test&category=sponsor", null, (err, res, body) => { if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); + if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { + done() + return; + } + } + + done(false); }); }); @@ -27,8 +37,15 @@ describe('postSkipSegments', () => { }), (err, res, body) => { if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + else if (res.statusCode === 200) { + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); + if (row.startTime === 0 && row.endTime === 10 && row.category === "sponsor") { + done() + return; + } + } + + done(false); }); }); @@ -48,8 +65,21 @@ describe('postSkipSegments', () => { }), (err, res, body) => { if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + else if (res.statusCode === 200) { + let rows = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").all(videoID); + + if (rows.length !== 2) done(false); + for (const row of rows) { + if (row.startTime !== 1 || row.endTime !== 10 || row.category !== "sponsor") { + done(false) + return; + } + } + + done() + } + + done(false); }); }); From f1a7524acf3f283bbb4844bf91ca91720e22b1e8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 17:29:52 -0400 Subject: [PATCH 06/20] Fixed getIP module import --- src/utils/getIP.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/getIP.js b/src/utils/getIP.js index cc11057..78244fa 100644 --- a/src/utils/getIP.js +++ b/src/utils/getIP.js @@ -1,5 +1,5 @@ var fs = require('fs'); -var config = require('./config.js'); +var config = require('../config.js'); module.exports = function getIP(req) { return config.behindProxy ? req.headers['x-forwarded-for'] : req.connection.remoteAddress; From 9807d3e9c76a9667e0871ffd66b98a2f8a60e57f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:05:26 -0400 Subject: [PATCH 07/20] Merge --- databases/_sponsorTimes.db.sql | 2 +- package.json | 2 + src/routes/getSkipSegment.js | 273 ++++++++++++++++++++++++++++ src/routes/postSkipSegments.js | 68 ++++++- src/utils/youtubeAPI.js | 20 +- test/cases/getSavedTimeForUser.js | 6 +- test/cases/getSponsorTime.js | 25 +-- test/cases/oldSubmitSponsorTimes.js | 32 ++-- test/cases/postSkipSegments.js | 180 ++++++++++-------- test/databases/_private.db.sql | 4 - test/databases/_sponsorTimes.db.sql | 5 +- test/youtubeMock.js | 38 ++++ 12 files changed, 529 insertions(+), 126 deletions(-) create mode 100644 src/routes/getSkipSegment.js create mode 100644 test/youtubeMock.js diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 1514e0e..dfca8bf 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS "sponsorTimes" ( "userID" TEXT NOT NULL, "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, - "category" TEXT NOT NULL; + "category" TEXT NOT NULL, "shadowHidden" INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS "userNames" ( diff --git a/package.json b/package.json index e7408a0..e162a6b 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "test": "node test.js", "dev": "nodemon -x \"(npm test || echo test failed) && npm start\"", + "dev:bash": "nodemon -x 'npm test ; npm start'", "start": "node index.js" }, "author": "Ajay Ramachandran", @@ -14,6 +15,7 @@ "better-sqlite3": "^5.4.3", "express": "^4.17.1", "http": "0.0.0", + "iso8601-duration": "^1.2.0", "uuid": "^3.3.2", "youtube-api": "^2.0.10" }, diff --git a/src/routes/getSkipSegment.js b/src/routes/getSkipSegment.js new file mode 100644 index 0000000..8918349 --- /dev/null +++ b/src/routes/getSkipSegment.js @@ -0,0 +1,273 @@ +var fs = require('fs'); +var config = require('../config.js'); + +var databases = require('../databases/databases.js'); +var db = databases.db; +var privateDB = databases.privateDB; + +var getHash = require('../utils/getHash.js'); +var getIP = require('../utils/getIP.js'); + + +//gets the getWeightedRandomChoice for each group in an array of groups +function getWeightedRandomChoiceForArray(choiceGroups, weights) { + let finalChoices = []; + //the indexes either chosen to be added to final indexes or chosen not to be added + let choicesDealtWith = []; + //for each choice group, what are the sums of the weights + let weightSums = []; + + for (let i = 0; i < choiceGroups.length; i++) { + //find weight sums for this group + weightSums.push(0); + for (let j = 0; j < choiceGroups[i].length; j++) { + //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time + if (weights[choiceGroups[i][j]] > 0) { + weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]]; + } + } + + //create a random choice for this group + let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1) + finalChoices.push(randomChoice.finalChoices); + + for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) { + choicesDealtWith.push(randomChoice.choicesDealtWith[j]) + } + } + + return { + finalChoices: finalChoices, + choicesDealtWith: choicesDealtWith, + weightSums: weightSums + }; +} + +//gets a weighted random choice from the indexes array based on the weights. +//amountOfChoices speicifies the amount of choices to return, 1 or more. +//choices are unique +function getWeightedRandomChoice(choices, weights, amountOfChoices) { + if (amountOfChoices > choices.length) { + //not possible, since all choices must be unique + return null; + } + + let finalChoices = []; + let choicesDealtWith = []; + + let sqrtWeightsList = []; + //the total of all the weights run through the cutom sqrt function + let totalSqrtWeights = 0; + for (let j = 0; j < choices.length; j++) { + //multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3) + //The 3 makes -2 the minimum votes before being ignored completely + //https://www.desmos.com/calculator/ljftxolg9j + //this can be changed if this system increases in popularity. + let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10); + sqrtWeightsList.push(sqrtVote) + totalSqrtWeights += sqrtVote; + + //this index has now been deat with + choicesDealtWith.push(choices[j]); + } + + //iterate and find amountOfChoices choices + let randomNumber = Math.random(); + + //this array will keep adding to this variable each time one sqrt vote has been dealt with + //this is the sum of all the sqrtVotes under this index + let currentVoteNumber = 0; + for (let j = 0; j < sqrtWeightsList.length; j++) { + if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) { + //this one was randomly generated + finalChoices.push(choices[j]); + //remove that from original array, for next recursion pass if it happens + choices.splice(j, 1); + break; + } + + //add on to the count + currentVoteNumber += sqrtWeightsList[j]; + } + + //add on the other choices as well using recursion + if (amountOfChoices > 1) { + let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices; + //add all these choices to the finalChoices array being returned + for (let i = 0; i < otherChoices.length; i++) { + finalChoices.push(otherChoices[i]); + } + } + + return { + finalChoices: finalChoices, + choicesDealtWith: choicesDealtWith + }; +} + + +//This function will find sponsor times that are contained inside of eachother, called similar sponsor times +//Only one similar time will be returned, randomly generated based on the sqrt of votes. +//This allows new less voted items to still sometimes appear to give them a chance at getting votes. +//Sponsor times with less than -1 votes are already ignored before this function is called +function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) { + //list of sponsors that are contained inside eachother + let similarSponsors = []; + + for (let i = 0; i < sponsorTimes.length; i++) { + //see if the start time is located between the start and end time of the other sponsor time. + for (let j = i + 1; j < sponsorTimes.length; j++) { + if (sponsorTimes[j][0] >= sponsorTimes[i][0] && sponsorTimes[j][0] <= sponsorTimes[i][1]) { + //sponsor j is contained in sponsor i + similarSponsors.push([i, j]); + } + } + } + + let similarSponsorsGroups = []; + //once they have been added to a group, they don't need to be dealt with anymore + let dealtWithSimilarSponsors = []; + + //create lists of all the similar groups (if 1 and 2 are similar, and 2 and 3 are similar, the group is 1, 2, 3) + for (let i = 0; i < similarSponsors.length; i++) { + if (dealtWithSimilarSponsors.includes(i)) { + //dealt with already + continue; + } + + //this is the group of indexes that are similar + let group = similarSponsors[i]; + for (let j = 0; j < similarSponsors.length; j++) { + if (group.includes(similarSponsors[j][0]) || group.includes(similarSponsors[j][1])) { + //this is a similar group + group.push(similarSponsors[j][0]); + group.push(similarSponsors[j][1]); + dealtWithSimilarSponsors.push(j); + } + } + similarSponsorsGroups.push(group); + } + + //remove duplicate indexes in group arrays + for (let i = 0; i < similarSponsorsGroups.length; i++) { + uniqueArray = similarSponsorsGroups[i].filter(function(item, pos, self) { + return self.indexOf(item) == pos; + }); + + similarSponsorsGroups[i] = uniqueArray; + } + + let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes); + + let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices; + //the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added + let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith; + + let voteSums = weightedRandomIndexes.weightSums; + //convert these into the votes + for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { + //it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor. + votes[finalSponsorTimeIndexes[i]] = voteSums[i]; + } + + //find the indexes never dealt with and add them + for (let i = 0; i < sponsorTimes.length; i++) { + if (!finalSponsorTimeIndexesDealtWith.includes(i)) { + finalSponsorTimeIndexes.push(i) + } + } + + //if there are too many indexes, find the best 4 + if (finalSponsorTimeIndexes.length > 8) { + finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices; + } + + //convert this to a final array to return + let finalSponsorTimes = []; + for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { + finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]); + } + + //convert this to a final array of UUIDs as well + let finalUUIDs = []; + for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { + finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]); + } + + return { + sponsorTimes: finalSponsorTimes, + UUIDs: finalUUIDs + }; +} + + + +module.exports = function (req, res) { + const videoID = req.body.videoID || req.query.videoID; + // Default to sponsor + // If using params instead of JSON, only one category can be pulled + const categories = req.body.categories || [req.query.category] ["sponsor"]; + + let sponsorTimes = []; + let votes = [] + let UUIDs = []; + + let hashedIP = getHash(getIP(req) + config.globalSalt); + + try { + let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? ORDER BY startTime").all(videoID); + + for (let i = 0; i < rows.length; i++) { + //check if votes are above -1 + if (rows[i].votes < -1) { + //too untrustworthy, just ignore it + continue; + } + + //check if shadowHidden + //this means it is hidden to everyone but the original ip that submitted it + if (rows[i].shadowHidden == 1) { + //get the ip + //await the callback + let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID); + + if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) { + //this isn't their ip, don't send it to them + continue; + } + } + + sponsorTimes.push([]); + + let index = sponsorTimes.length - 1; + + sponsorTimes[index][0] = rows[i].startTime; + sponsorTimes[index][1] = rows[i].endTime; + + votes[index] = rows[i].votes; + UUIDs[index] = rows[i].UUID; + } + + if (sponsorTimes.length == 0) { + res.sendStatus(404); + return; + } + + organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs); + sponsorTimes = organisedData.sponsorTimes; + UUIDs = organisedData.UUIDs; + + if (sponsorTimes.length == 0) { + res.sendStatus(404); + } else { + //send result + res.send({ + sponsorTimes: sponsorTimes, + UUIDs: UUIDs + }) + } + } catch(error) { + console.error(error); + res.send(500); + } +} \ No newline at end of file diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 6562d94..08a5c7f 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -4,6 +4,8 @@ var databases = require('../databases/databases.js'); var db = databases.db; var privateDB = databases.privateDB; var YouTubeAPI = require('../utils/youtubeAPI.js'); +var request = require('request'); +var isoDurations = require('iso8601-duration'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); @@ -34,7 +36,7 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(userID); // If it is a first time submission - if (userSubmissionCountRow.submissionCount === 0) { + if (userSubmissionCountRow.submissionCount <= 1) { YouTubeAPI.videos.list({ part: "snippet", id: videoID @@ -78,6 +80,47 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { } } +// submission: {videoID, startTime, endTime} +// callback: function(reject: "String containing reason the submission was rejected") +// returns: string when an error, false otherwise +async function autoModerateSubmission(submission, callback) { + // Get the video information from the youtube API + if (config.youtubeAPI !== null) { + let {err, data} = await new Promise((resolve, reject) => { + YouTubeAPI.videos.list({ + part: "contentDetails", + id: submission.videoID + }, (err, data) => resolve({err, data})); + }); + + if (err) { + return "Couldn't get video information."; + } else { + // Check to see if video exists + if (data.pageInfo.totalResults === 0) { + callback("No video exists with id " + submission.videoID); + } else { + let duration = data.items[0].contentDetails.duration; + duration = isoDurations.toSeconds(isoDurations.parse(duration)); + + // Reject submission if over 80% of the video + if ((submission.endTime - submission.startTime) > (duration/100)*80) { + return "Sponsor segment is over 80% of the video."; + } else { + return false; + } + } + } + + } else { + console.log("Skipped YouTube API"); + + // Can't moderate the submission without calling the youtube API + // so allow by default. + return; + } +} + module.exports = async function postSkipSegments(req, res) { let videoID = req.query.videoID || req.body.videoID; let userID = req.query.userID || req.body.userID; @@ -124,10 +167,17 @@ module.exports = async function postSkipSegments(req, res) { } //check if this info has already been submitted before - let duplicateCheck2Row = - db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID]); - if (duplicateCheck2Row == null) { - res.sendStatus(409); + let duplicateCheck2Row = db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get(startTime, endTime, videoID); + if (duplicateCheck2Row !== null) { + // console.log(duplicateCheck2Row) + // console.log(db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").all(1,10,"dQw4w9WgXcQ")) + // res.sendStatus(409); + // return; + } + + let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime}); + if (autoModerateResult) { + res.status(403).send("Request rejected by auto moderator: " + autoModerateResult); return; } } @@ -184,8 +234,10 @@ module.exports = async function postSkipSegments(req, res) { let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] + segmentInfo.segment[1] + segmentInfo.category + userID, 1); + console.log(UUID) + try { - db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], + db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); //add to private db as well @@ -196,7 +248,7 @@ module.exports = async function postSkipSegments(req, res) { //a DB change probably occurred res.sendStatus(502); console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + - segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category); + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err); return; } @@ -207,6 +259,6 @@ module.exports = async function postSkipSegments(req, res) { } catch (err) { console.error(err); - res.send(500); + res.sendStatus(500); } } diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index 7449815..c28f882 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -2,8 +2,18 @@ var config = require('../config.js'); // YouTube API const YouTubeAPI = require("youtube-api"); -YouTubeAPI.authenticate({ - type: "key", - key: config.youtubeAPIKey -}); -module.exports = YouTubeAPI; \ No newline at end of file + +var exportObject; +// If in test mode, return a mocked youtube object +// otherwise return an authenticated youtube api +if (config.mode === "test") { + exportObject = require("../../test/youtubeMock.js"); +} else { + YouTubeAPI.authenticate({ + type: "key", + key: config.youtubeAPIKey + }); + exportObject = YouTubeAPI; +} + +module.exports = exportObject; \ No newline at end of file diff --git a/test/cases/getSavedTimeForUser.js b/test/cases/getSavedTimeForUser.js index 5488c67..e3976c1 100644 --- a/test/cases/getSavedTimeForUser.js +++ b/test/cases/getSavedTimeForUser.js @@ -5,16 +5,16 @@ var getHash = require('../../src/utils/getHash.js'); describe('getSavedTimeForUser', () => { before(() => { - db.exec("INSERT INTO sponsorTimes VALUES ('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '"+getHash("testman")+"', 0, 50, 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)"); }); it('Should be able to get a 200', (done) => { request.get(utils.getbaseURL() + "/api/getSavedTimeForUser?userID=testman", null, (err, res, body) => { - if (err) done(false); + if (err) done("couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200"); - else done(); + else done(); // pass }); }); }); \ No newline at end of file diff --git a/test/cases/getSponsorTime.js b/test/cases/getSponsorTime.js index 0565d28..306f563 100644 --- a/test/cases/getSponsorTime.js +++ b/test/cases/getSponsorTime.js @@ -19,17 +19,17 @@ var utils = require('../utils.js'); describe('getVideoSponsorTime', () => { before(() => { - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); }); it('Should be able to get a time', (done) => { request.get(utils.getbaseURL() + "/api/getVideoSponsorTimes?videoID=testtesttest", null, (err, res, body) => { - if (err) done(false); + if (err) done("Couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200"); - else done(); + else done(); // pass }); }); @@ -37,9 +37,9 @@ describe('getVideoSponsorTime', () => { request.get(utils.getbaseURL() + "/api/getVideoSponsorTimes?videoID=notarealvideo", null, (err, res, body) => { - if (err) done(false); + if (err) done("couldn't call endpoint"); else if (res.statusCode !== 404) done("non 404 respone code: " + res.statusCode); - else done(); + else done(); // pass }); }); @@ -48,9 +48,9 @@ describe('getVideoSponsorTime', () => { request.get(utils.getbaseURL() + "/api/getVideoSponsorTimes?videoID=testtesttest&fakeparam=hello", null, (err, res, body) => { - if (err) done(false); + if (err) done("couldn't callendpoint"); else if (res.statusCode !== 200) done("non 200"); - else done(); + else done(); // pass }); }); @@ -58,9 +58,10 @@ describe('getVideoSponsorTime', () => { request.get(utils.getbaseURL() + "/api/getVideoSponsorTimes?videoID=testtesttest,test", null, (err, res, body) => { - if (err) done(false); + if (err) done("couln't call endpoint"); else if (res.statusCode !== 200) done("non 200 response: " + res.statusCode); - else (JSON.parse(body).UUIDs[0] === 'uuid-1') && done(); + else if (JSON.parse(body).UUIDs[0] === 'uuid-1') done(); // pass + else done("couldn't parse response"); }); }); @@ -68,14 +69,14 @@ describe('getVideoSponsorTime', () => { request.get(utils.getbaseURL() + "/api/getVideoSponsorTimes?videoID=testtesttest", null, (err, res, body) => { - if (err) done(false); + if (err) done("couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200"); else { let parsedBody = JSON.parse(body); if (parsedBody.sponsorTimes[0][0] === 1 && parsedBody.sponsorTimes[0][1] === 11 && parsedBody.UUIDs[0] === 'uuid-0') { - done(); + done(); // pass } else { done("Wrong data was returned + " + parsedBody); } diff --git a/test/cases/oldSubmitSponsorTimes.js b/test/cases/oldSubmitSponsorTimes.js index d205df5..bf3ac7a 100644 --- a/test/cases/oldSubmitSponsorTimes.js +++ b/test/cases/oldSubmitSponsorTimes.js @@ -9,35 +9,37 @@ var db = databases.db; describe('postVideoSponsorTime (Old submission method)', () => { it('Should be able to submit a time (GET)', (done) => { request.get(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test", null, + + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcQ&startTime=1&endTime=10&userID=test", null, (err, res, body) => { - if (err) done(false); + if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcQ"); if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { done() - return; + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); } + } else { + done("Status code was " + res.statusCode); } - - done(false); }); }); it('Should be able to submit a time (POST)', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test", null, + + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcE&startTime=1&endTime=11&userID=test", null, (err, res, body) => { - if (err) done(false); + if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); - if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcE"); + if (row.startTime === 1 && row.endTime === 11 && row.category === "sponsor") { done() - return; + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); } + } else { + done("Status code was " + res.statusCode); } - - done(false); }); }); @@ -45,9 +47,9 @@ describe('postVideoSponsorTime (Old submission method)', () => { request.get(utils.getbaseURL() + "/api/postVideoSponsorTimes?startTime=1&endTime=10&userID=test", null, (err, res, body) => { - if (err) done(false); + if (err) done(err); if (res.statusCode === 400) done(); - else done(false); + else done("Status code was: " + res.statusCode); }); }); }); \ No newline at end of file diff --git a/test/cases/postSkipSegments.js b/test/cases/postSkipSegments.js index 446d802..05c7374 100644 --- a/test/cases/postSkipSegments.js +++ b/test/cases/postSkipSegments.js @@ -9,172 +9,204 @@ var db = databases.db; describe('postSkipSegments', () => { it('Should be able to submit a single time (Params method)', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes?videoID=djgofQKWmXc&startTime=1&endTime=10&userID=test&category=sponsor", null, + + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null, (err, res, body) => { - if (err) done(false); + if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); - if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") { + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcR"); + if (row.startTime === 2 && row.endTime === 10 && row.category === "sponsor") { done() - return; + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); } + } else { + done("Status code was " + res.statusCode); } - - done(false); }); }); it('Should be able to submit a single time (JSON method)', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { - videoID: "djgofQKWmXc", + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "dQw4w9WgXcF", segments: [{ segment: [0, 10], category: "sponsor" }] } - }), + }, (err, res, body) => { - if (err) done(false); + if (err) done(err); else if (res.statusCode === 200) { - let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get(videoID); + let row = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").get("dQw4w9WgXcF"); if (row.startTime === 0 && row.endTime === 10 && row.category === "sponsor") { done() - return; + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); } + } else { + done("Status code was " + res.statusCode); } - - done(false); }); }); it('Should be able to submit multiple times (JSON method)', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { - videoID: "djgofQKWmXc", + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "dQw4w9WgXcQ", segments: [{ - segment: [0, 10], + segment: [3, 10], category: "sponsor" }, { segment: [30, 60], category: "intro" }] } - }), + }, (err, res, body) => { - if (err) done(false); + if (err) done(err); else if (res.statusCode === 200) { - let rows = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").all(videoID); - - if (rows.length !== 2) done(false); - for (const row of rows) { - if (row.startTime !== 1 || row.endTime !== 10 || row.category !== "sponsor") { - done(false) - return; + let rows = db.prepare("SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?").all("dQw4w9WgXcR"); + let success = true; + if (rows.length === 2) { + for (const row of rows) { + if ((row.startTime !== 3 || row.endTime !== 10 || row.category !== "sponsor") && + (row.startTime !== 30 || row.endTime !== 60 || row.category !== "intro")) { + success = false; + break; + } } } - done() + if (success) done(); + else done("Submitted times were not saved. Actual submissions: " + JSON.stringify(row)); + } else { + done("Status code was " + res.statusCode); } - - done(false); + }); + }); + + it('Should be rejected if over 80% of the video', (done) => { + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=qqwerty&startTime=30&endTime=1000000&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 403) done(); // pass + else done("non 403 status code: " + res.statusCode + " ("+body+")"); + }); + }); + + it('Should be rejected if not a valid videoID', (done) => { + request.get(utils.getbaseURL() + + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 403) done(); // pass + else done("non 403 status code: " + res.statusCode + " ("+body+")"); }); }); it('Should return 400 for missing params (Params method)', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes?startTime=1&endTime=10&userID=test", null, + + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", null, (err, res, body) => { - if (err) done(false); + if (err) done(true); if (res.statusCode === 400) done(); - else done(false); + else done(true); }); }); it('Should return 400 for missing params (JSON method) 1', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", segments: [{ - segment: [0, 10], + segment: [9, 10], category: "sponsor" }, { - segment: [30, 60], + segment: [31, 60], category: "intro" }] } - }), + }, (err, res, body) => { - if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + if (err) done(true); + else if (res.statusCode === 400) done(); + else done(true); }); }); it('Should return 400 for missing params (JSON method) 2', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { - videoID: "djgofQKWmXc" + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "dQw4w9WgXcQ" } - }), + }, (err, res, body) => { - if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + if (err) done(true); + else if (res.statusCode === 400) done(); + else done(true); }); }); it('Should return 400 for missing params (JSON method) 3', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { - videoID: "djgofQKWmXc", + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "dQw4w9WgXcQ", segments: [{ segment: [0], category: "sponsor" }, { - segment: [30, 60], + segment: [31, 60], category: "intro" }] } - }), + }, (err, res, body) => { - if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + if (err) done(true); + else if (res.statusCode === 400) done(); + else done(true); }); }); it('Should return 400 for missing params (JSON method) 4', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { - videoID: "djgofQKWmXc", + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "dQw4w9WgXcQ", segments: [{ - segment: [0, 10] + segment: [9, 10] }, { - segment: [30, 60], + segment: [31, 60], category: "intro" }] } - }), + }, (err, res, body) => { - if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + if (err) done(true); + else if (res.statusCode === 400) done(); + else done(true); }); }); it('Should return 400 for missing params (JSON method) 5', (done) => { request.post(utils.getbaseURL() - + "/api/postVideoSponsorTimes", JSON.stringify({ - body: { - videoID: "djgofQKWmXc" + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: "dQw4w9WgXcQ" } - }), + }, (err, res, body) => { - if (err) done(false); - else if (res.statusCode === 200) done(); - else done(false); + if (err) done(true); + else if (res.statusCode === 400) done(); + else done(true); }); }); }); \ No newline at end of file diff --git a/test/databases/_private.db.sql b/test/databases/_private.db.sql index 317603a..b93aaee 100644 --- a/test/databases/_private.db.sql +++ b/test/databases/_private.db.sql @@ -1,8 +1,4 @@ BEGIN TRANSACTION; -DROP TABLE IF EXISTS "shadowBannedUsers"; -DROP TABLE IF EXISTS "votes"; -DROP TABLE IF EXISTS "sponsorTimes"; - CREATE TABLE IF NOT EXISTS "shadowBannedUsers" ( "userID" TEXT NOT NULL ); diff --git a/test/databases/_sponsorTimes.db.sql b/test/databases/_sponsorTimes.db.sql index bbf501e..dfca8bf 100644 --- a/test/databases/_sponsorTimes.db.sql +++ b/test/databases/_sponsorTimes.db.sql @@ -1,8 +1,4 @@ BEGIN TRANSACTION; -DROP TABLE IF EXISTS "vipUsers"; -DROP TABLE IF EXISTS "sponsorTimes"; -DROP TABLE IF EXISTS "userNames"; - CREATE TABLE IF NOT EXISTS "vipUsers" ( "userID" TEXT NOT NULL ); @@ -15,6 +11,7 @@ CREATE TABLE IF NOT EXISTS "sponsorTimes" ( "userID" TEXT NOT NULL, "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, + "category" TEXT NOT NULL, "shadowHidden" INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS "userNames" ( diff --git a/test/youtubeMock.js b/test/youtubeMock.js new file mode 100644 index 0000000..56300fd --- /dev/null +++ b/test/youtubeMock.js @@ -0,0 +1,38 @@ +/* +YouTubeAPI.videos.list({ + part: "snippet", + id: videoID +}, function (err, data) {}); + */ + + // https://developers.google.com/youtube/v3/docs/videos + +const YouTubeAPI = { + videos: { + list: (obj, callback) => { + if (obj.videoID === "knownWrongID") { + callback(undefined, { + pageInfo: { + totalResults: 0 + }, + items: [] + }); + } else { + callback(undefined, { + pageInfo: { + totalResults: 1 + }, + items: [ + { + contentDetails: { + duration: "PT1H23M30S" + } + } + ] + }); + } + } + } +}; + +module.exports = YouTubeAPI; \ No newline at end of file From 557c6ad05a44b710922f9ea87051a4bb5b43d86d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:12:12 -0400 Subject: [PATCH 08/20] Fixed discord tests --- src/routes/postSkipSegments.js | 9 +++++---- test/mocks.js | 2 -- test/youtubeMock.js | 8 ++++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 08a5c7f..afa26f5 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -45,15 +45,18 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) { err && console.log(err); return; } + + let startTime = parseFloat(segmentInfo.segment[0]); + let endTime = parseFloat(segmentInfo.segment[1]); request.post(config.discordFirstTimeSubmissionsWebhookURL, { json: { "embeds": [{ "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (segmentInfo.segment.toFixed(0) - 2), + "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2), "description": "Submission ID: " + UUID + "\n\nTimestamp: " + - getFormattedTime(segmentInfo.segment[0]) + " to " + getFormattedTime(segmentInfo.segment[1]) + + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + "\n\nCategory: " + segmentInfo.category, "color": 10813440, "author": { @@ -234,8 +237,6 @@ module.exports = async function postSkipSegments(req, res) { let UUID = getHash("v2-categories" + videoID + segmentInfo.segment[0] + segmentInfo.segment[1] + segmentInfo.category + userID, 1); - console.log(UUID) - try { db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); diff --git a/test/mocks.js b/test/mocks.js index df19c8a..a124646 100644 --- a/test/mocks.js +++ b/test/mocks.js @@ -4,12 +4,10 @@ var app = express(); var config = require('../src/config.js'); app.post('/ReportChannelWebhook', (req, res) => { - console.log("report mock hit"); res.status(200); }); app.post('/FirstTimeSubmissionsWebhook', (req, res) => { - console.log("first time submisson mock hit"); res.status(200); }); diff --git a/test/youtubeMock.js b/test/youtubeMock.js index 56300fd..0a9998d 100644 --- a/test/youtubeMock.js +++ b/test/youtubeMock.js @@ -26,6 +26,14 @@ const YouTubeAPI = { { contentDetails: { duration: "PT1H23M30S" + }, + snippet: { + title: "Example Title", + thumbnails: { + maxres: { + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png" + } + } } } ] From 06de0d61fa360e686f00b78e25b073f201335ca4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:14:50 -0400 Subject: [PATCH 09/20] Fixed crashes from multiple submissions. --- src/routes/postSkipSegments.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index afa26f5..795796f 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -243,8 +243,6 @@ module.exports = async function postSkipSegments(req, res) { //add to private db as well privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); - - res.sendStatus(200); } catch (err) { //a DB change probably occurred res.sendStatus(502); @@ -261,5 +259,9 @@ module.exports = async function postSkipSegments(req, res) { console.error(err); res.sendStatus(500); + + return; } + + res.sendStatus(200); } From 739b59c65d1dce9efd6b0ec4014f024ddb72741e Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:17:49 -0400 Subject: [PATCH 10/20] Clear database before running tests --- test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test.js b/test.js index 0241386..aedb8c5 100644 --- a/test.js +++ b/test.js @@ -2,6 +2,11 @@ var Mocha = require('mocha'), fs = require('fs'), path = require('path'); +var config = require('./src/config.js'); +// delete old test database +fs.unlinkSync(config.db); +fs.unlinkSync(config.privateDB); + var createServer = require('./src/app.js'); var createMockServer = require('./test/mocks.js'); From 329abfa6171ddd0a2d9f97311176c497b4ed468b Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:26:01 -0400 Subject: [PATCH 11/20] Only delete test db if it exists --- test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.js b/test.js index aedb8c5..3a13418 100644 --- a/test.js +++ b/test.js @@ -4,8 +4,8 @@ var Mocha = require('mocha'), var config = require('./src/config.js'); // delete old test database -fs.unlinkSync(config.db); -fs.unlinkSync(config.privateDB); +if (fs.existsSync(config.db)) fs.unlinkSync(config.db); +if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB); var createServer = require('./src/app.js'); var createMockServer = require('./test/mocks.js'); From 70a39eeea4ad776916a33677e058a1136b7c153f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:29:58 -0400 Subject: [PATCH 12/20] Fixed duplicate check code --- src/routes/postSkipSegments.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 795796f..c8fbbdf 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -170,12 +170,13 @@ module.exports = async function postSkipSegments(req, res) { } //check if this info has already been submitted before - let duplicateCheck2Row = db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get(startTime, endTime, videoID); - if (duplicateCheck2Row !== null) { - // console.log(duplicateCheck2Row) + let duplicateCheck2Row = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE startTime = ? " + + "and endTime = ? and category = ? and videoID = ?").get(startTime, endTime, segments[i].category, videoID); + if (duplicateCheck2Row.count > 0) { + console.log(duplicateCheck2Row.count) // console.log(db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").all(1,10,"dQw4w9WgXcQ")) - // res.sendStatus(409); - // return; + res.sendStatus(409); + return; } let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime}); From ca8f57197899b22bed4443c94bbfba04f447d647 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 6 Apr 2020 20:30:27 -0400 Subject: [PATCH 13/20] Removed extra logging --- src/routes/postSkipSegments.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index c8fbbdf..d8fe2a3 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -173,8 +173,6 @@ module.exports = async function postSkipSegments(req, res) { let duplicateCheck2Row = db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE startTime = ? " + "and endTime = ? and category = ? and videoID = ?").get(startTime, endTime, segments[i].category, videoID); if (duplicateCheck2Row.count > 0) { - console.log(duplicateCheck2Row.count) - // console.log(db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").all(1,10,"dQw4w9WgXcQ")) res.sendStatus(409); return; } From 3c9380c2a26df0b9990a4fbae0ea081ecf1e1e71 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 01:54:19 -0400 Subject: [PATCH 14/20] Added new segment getting endpoint --- src/app.js | 13 +- .../{getSkipSegment.js => getSkipSegments.js} | 136 ++++++++++-------- 2 files changed, 85 insertions(+), 64 deletions(-) rename src/routes/{getSkipSegment.js => getSkipSegments.js} (73%) diff --git a/src/app.js b/src/app.js index fa29c38..ab50509 100644 --- a/src/app.js +++ b/src/app.js @@ -8,8 +8,7 @@ var corsMiddleware = require('./middleware/cors.js'); var loggerMiddleware = require('./middleware/logger.js'); // Routes -var getVideoSponsorTimes = require('./routes/getVideoSponsorTimes.js'); -var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); +var getSkipSegments = require('./routes/getSkipSegments.js'); var postSkipSegments = require('./routes/postSkipSegments.js'); var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js'); var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js'); @@ -23,12 +22,19 @@ var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); +// Old Routes +var getVideoSponsorTimes = require('./routes/getVideoSponsorTimes.js'); +var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); + //setup CORS correctly app.use(corsMiddleware); app.use(loggerMiddleware); app.use(express.json()) +// Setup pretty JSON +if (config.mode === "development") app.set('json spaces', 2); + //add the get function app.get('/api/getVideoSponsorTimes', getVideoSponsorTimes); @@ -36,7 +42,8 @@ app.get('/api/getVideoSponsorTimes', getVideoSponsorTimes); app.get('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); -//add the post function +//add the skip segments functions +app.get('/api/skipSegments', getSkipSegments); app.post('/api/skipSegments', postSkipSegments); //voting endpoint diff --git a/src/routes/getSkipSegment.js b/src/routes/getSkipSegments.js similarity index 73% rename from src/routes/getSkipSegment.js rename to src/routes/getSkipSegments.js index 8918349..95f39c7 100644 --- a/src/routes/getSkipSegment.js +++ b/src/routes/getSkipSegments.js @@ -203,71 +203,85 @@ function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) { module.exports = function (req, res) { - const videoID = req.body.videoID || req.query.videoID; - // Default to sponsor - // If using params instead of JSON, only one category can be pulled - const categories = req.body.categories || [req.query.category] ["sponsor"]; + const videoID = req.body.videoID || req.query.videoID; + // Default to sponsor + // If using params instead of JSON, only one category can be pulled + const categories = req.body.categories || (req.query.category ? [req.query.category] : ["sponsor"]); - let sponsorTimes = []; - let votes = [] - let UUIDs = []; + /** + * @type {Array<{ + * segment: number[], + * category: string, + * UUID: string + * }> + * } + */ + let segments = []; - let hashedIP = getHash(getIP(req) + config.globalSalt); + let hashedIP = getHash(getIP(req) + config.globalSalt); - try { - let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? ORDER BY startTime").all(videoID); - - for (let i = 0; i < rows.length; i++) { - //check if votes are above -1 - if (rows[i].votes < -1) { - //too untrustworthy, just ignore it - continue; - } + try { + for (const category of categories) { + let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime") + .all(videoID, category); - //check if shadowHidden - //this means it is hidden to everyone but the original ip that submitted it - if (rows[i].shadowHidden == 1) { - //get the ip - //await the callback - let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID); + let sponsorTimes = []; + let votes = [] + let UUIDs = []; + + for (let i = 0; i < rows.length; i++) { + //check if votes are above -1 + if (rows[i].votes < -1) { + //too untrustworthy, just ignore it + continue; + } + + //check if shadowHidden + //this means it is hidden to everyone but the original ip that submitted it + if (rows[i].shadowHidden == 1) { + //get the ip + //await the callback + let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID); + + if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) { + //this isn't their ip, don't send it to them + continue; + } + } + + sponsorTimes.push([rows[i].startTime, rows[i].endTime]); + votes.push(rows[i].votes); + UUIDs.push(rows[i].UUID); + } + + if (sponsorTimes.length == 0) { + res.sendStatus(404); + return; + } + + organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs); + sponsorTimes = organisedData.sponsorTimes; + UUIDs = organisedData.UUIDs; + + for (let i = 0; i < sponsorTimes.length; i++) { + segments.push({ + segment: sponsorTimes[i], + category: category, + UUID: UUIDs[i] + }); + } + } + } catch(error) { + console.error(error); + res.send(500); - if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) { - //this isn't their ip, don't send it to them - continue; - } - } + return; + } - sponsorTimes.push([]); - - let index = sponsorTimes.length - 1; - - sponsorTimes[index][0] = rows[i].startTime; - sponsorTimes[index][1] = rows[i].endTime; - - votes[index] = rows[i].votes; - UUIDs[index] = rows[i].UUID; - } - - if (sponsorTimes.length == 0) { - res.sendStatus(404); - return; - } - - organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs); - sponsorTimes = organisedData.sponsorTimes; - UUIDs = organisedData.UUIDs; - - if (sponsorTimes.length == 0) { - res.sendStatus(404); - } else { - //send result - res.send({ - sponsorTimes: sponsorTimes, - UUIDs: UUIDs - }) - } - } catch(error) { - console.error(error); - res.send(500); - } + if (segments.length == 0) { + res.sendStatus(404); + } else { + //send result + res.send(segments) + } } \ No newline at end of file From 4c9aa11b9aa08842b00a0fc0155d70045cc03b0d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 02:03:02 -0400 Subject: [PATCH 15/20] Made old get function use new function --- src/app.js | 2 +- src/routes/getSkipSegments.js | 37 +++- src/routes/getVideoSponsorTimes.js | 278 ++--------------------------- 3 files changed, 47 insertions(+), 270 deletions(-) diff --git a/src/app.js b/src/app.js index ab50509..9dd41a2 100644 --- a/src/app.js +++ b/src/app.js @@ -43,7 +43,7 @@ app.get('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); //add the skip segments functions -app.get('/api/skipSegments', getSkipSegments); +app.get('/api/skipSegments', getSkipSegments.endpoint); app.post('/api/skipSegments', postSkipSegments); //voting endpoint diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index 95f39c7..4925af3 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -200,9 +200,17 @@ function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) { }; } - - -module.exports = function (req, res) { +/** + * + * Returns what would be sent to the client. + * Will resond with errors if required. Returns false if it errors. + * + * @param req + * @param res + * + * @returns + */ +function handleGetSegments(req, res) { const videoID = req.body.videoID || req.query.videoID; // Default to sponsor // If using params instead of JSON, only one category can be pulled @@ -256,7 +264,7 @@ module.exports = function (req, res) { if (sponsorTimes.length == 0) { res.sendStatus(404); - return; + return false; } organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs); @@ -275,13 +283,26 @@ module.exports = function (req, res) { console.error(error); res.send(500); - return; + return false; } if (segments.length == 0) { res.sendStatus(404); - } else { - //send result - res.send(segments) + return false; + } + + return segments; +} + + +module.exports = { + handleGetSegments, + endpoint: function (req, res) { + let segments = handleGetSegments(req, res); + + if (segments) { + //send result + res.send(segments) + } } } \ No newline at end of file diff --git a/src/routes/getVideoSponsorTimes.js b/src/routes/getVideoSponsorTimes.js index 67447bc..d898d38 100644 --- a/src/routes/getVideoSponsorTimes.js +++ b/src/routes/getVideoSponsorTimes.js @@ -1,270 +1,26 @@ -var fs = require('fs'); -var config = require('../config.js'); - -var databases = require('../databases/databases.js'); -var db = databases.db; -var privateDB = databases.privateDB; - -var getHash = require('../utils/getHash.js'); -var getIP = require('../utils/getIP.js'); - - -//gets the getWeightedRandomChoice for each group in an array of groups -function getWeightedRandomChoiceForArray(choiceGroups, weights) { - let finalChoices = []; - //the indexes either chosen to be added to final indexes or chosen not to be added - let choicesDealtWith = []; - //for each choice group, what are the sums of the weights - let weightSums = []; - - for (let i = 0; i < choiceGroups.length; i++) { - //find weight sums for this group - weightSums.push(0); - for (let j = 0; j < choiceGroups[i].length; j++) { - //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time - if (weights[choiceGroups[i][j]] > 0) { - weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]]; - } - } - - //create a random choice for this group - let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1) - finalChoices.push(randomChoice.finalChoices); - - for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) { - choicesDealtWith.push(randomChoice.choicesDealtWith[j]) - } - } - - return { - finalChoices: finalChoices, - choicesDealtWith: choicesDealtWith, - weightSums: weightSums - }; -} - -//gets a weighted random choice from the indexes array based on the weights. -//amountOfChoices speicifies the amount of choices to return, 1 or more. -//choices are unique -function getWeightedRandomChoice(choices, weights, amountOfChoices) { - if (amountOfChoices > choices.length) { - //not possible, since all choices must be unique - return null; - } - - let finalChoices = []; - let choicesDealtWith = []; - - let sqrtWeightsList = []; - //the total of all the weights run through the cutom sqrt function - let totalSqrtWeights = 0; - for (let j = 0; j < choices.length; j++) { - //multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3) - //The 3 makes -2 the minimum votes before being ignored completely - //https://www.desmos.com/calculator/ljftxolg9j - //this can be changed if this system increases in popularity. - let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10); - sqrtWeightsList.push(sqrtVote) - totalSqrtWeights += sqrtVote; - - //this index has now been deat with - choicesDealtWith.push(choices[j]); - } - - //iterate and find amountOfChoices choices - let randomNumber = Math.random(); - - //this array will keep adding to this variable each time one sqrt vote has been dealt with - //this is the sum of all the sqrtVotes under this index - let currentVoteNumber = 0; - for (let j = 0; j < sqrtWeightsList.length; j++) { - if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) { - //this one was randomly generated - finalChoices.push(choices[j]); - //remove that from original array, for next recursion pass if it happens - choices.splice(j, 1); - break; - } - - //add on to the count - currentVoteNumber += sqrtWeightsList[j]; - } - - //add on the other choices as well using recursion - if (amountOfChoices > 1) { - let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices; - //add all these choices to the finalChoices array being returned - for (let i = 0; i < otherChoices.length; i++) { - finalChoices.push(otherChoices[i]); - } - } - - return { - finalChoices: finalChoices, - choicesDealtWith: choicesDealtWith - }; -} - - -//This function will find sponsor times that are contained inside of eachother, called similar sponsor times -//Only one similar time will be returned, randomly generated based on the sqrt of votes. -//This allows new less voted items to still sometimes appear to give them a chance at getting votes. -//Sponsor times with less than -1 votes are already ignored before this function is called -function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) { - //list of sponsors that are contained inside eachother - let similarSponsors = []; - - for (let i = 0; i < sponsorTimes.length; i++) { - //see if the start time is located between the start and end time of the other sponsor time. - for (let j = i + 1; j < sponsorTimes.length; j++) { - if (sponsorTimes[j][0] >= sponsorTimes[i][0] && sponsorTimes[j][0] <= sponsorTimes[i][1]) { - //sponsor j is contained in sponsor i - similarSponsors.push([i, j]); - } - } - } - - let similarSponsorsGroups = []; - //once they have been added to a group, they don't need to be dealt with anymore - let dealtWithSimilarSponsors = []; - - //create lists of all the similar groups (if 1 and 2 are similar, and 2 and 3 are similar, the group is 1, 2, 3) - for (let i = 0; i < similarSponsors.length; i++) { - if (dealtWithSimilarSponsors.includes(i)) { - //dealt with already - continue; - } - - //this is the group of indexes that are similar - let group = similarSponsors[i]; - for (let j = 0; j < similarSponsors.length; j++) { - if (group.includes(similarSponsors[j][0]) || group.includes(similarSponsors[j][1])) { - //this is a similar group - group.push(similarSponsors[j][0]); - group.push(similarSponsors[j][1]); - dealtWithSimilarSponsors.push(j); - } - } - similarSponsorsGroups.push(group); - } - - //remove duplicate indexes in group arrays - for (let i = 0; i < similarSponsorsGroups.length; i++) { - uniqueArray = similarSponsorsGroups[i].filter(function(item, pos, self) { - return self.indexOf(item) == pos; - }); - - similarSponsorsGroups[i] = uniqueArray; - } - - let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes); - - let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices; - //the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added - let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith; - - let voteSums = weightedRandomIndexes.weightSums; - //convert these into the votes - for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { - //it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor. - votes[finalSponsorTimeIndexes[i]] = voteSums[i]; - } - - //find the indexes never dealt with and add them - for (let i = 0; i < sponsorTimes.length; i++) { - if (!finalSponsorTimeIndexesDealtWith.includes(i)) { - finalSponsorTimeIndexes.push(i) - } - } - - //if there are too many indexes, find the best 4 - if (finalSponsorTimeIndexes.length > 8) { - finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices; - } - - //convert this to a final array to return - let finalSponsorTimes = []; - for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { - finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]); - } - - //convert this to a final array of UUIDs as well - let finalUUIDs = []; - for (let i = 0; i < finalSponsorTimeIndexes.length; i++) { - finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]); - } - - return { - sponsorTimes: finalSponsorTimes, - UUIDs: finalUUIDs - }; -} - +var getSkipSegments = require("./getSkipSegments.js") module.exports = function (req, res) { - let videoID = req.query.videoID; + let videoID = req.query.videoID; - let sponsorTimes = []; - let votes = [] - let UUIDs = []; + let segments = getSkipSegments.handleGetSegments(req, res); - let hashedIP = getHash(getIP(req) + config.globalSalt); + if (segments) { + // Convert to old outputs + let sponsorTimes = []; + let UUIDs = []; - try { - let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? ORDER BY startTime").all(videoID); - - for (let i = 0; i < rows.length; i++) { - //check if votes are above -1 - if (rows[i].votes < -1) { - //too untrustworthy, just ignore it - continue; - } + for (const segment of segments) { + sponsorTimes.push(segment.segment); + UUIDs.push(segment.UUID); + } - //check if shadowHidden - //this means it is hidden to everyone but the original ip that submitted it - if (rows[i].shadowHidden == 1) { - //get the ip - //await the callback - let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID); + res.send({ + sponsorTimes, + UUIDs + }) + } - if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) { - //this isn't their ip, don't send it to them - continue; - } - } - - sponsorTimes.push([]); - - let index = sponsorTimes.length - 1; - - sponsorTimes[index][0] = rows[i].startTime; - sponsorTimes[index][1] = rows[i].endTime; - - votes[index] = rows[i].votes; - UUIDs[index] = rows[i].UUID; - } - - if (sponsorTimes.length == 0) { - res.sendStatus(404); - return; - } - - organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs); - sponsorTimes = organisedData.sponsorTimes; - UUIDs = organisedData.UUIDs; - - if (sponsorTimes.length == 0) { - res.sendStatus(404); - } else { - //send result - res.send({ - sponsorTimes: sponsorTimes, - UUIDs: UUIDs - }) - } - } catch(error) { - console.error(error); - res.send(500); - } + // Error has already been handled in the other method } \ No newline at end of file From 0b4416d9eb18858f6dea5fa5eeb77150c9fdd435 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 02:04:37 -0400 Subject: [PATCH 16/20] Renamed old get method --- src/app.js | 8 ++++---- ...getVideoSponsorTimes.js => oldGetVideoSponsorTimes.js} | 0 test/cases/{getSponsorTime.js => oldGetSponsorTime.js} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/routes/{getVideoSponsorTimes.js => oldGetVideoSponsorTimes.js} (100%) rename test/cases/{getSponsorTime.js => oldGetSponsorTime.js} (98%) diff --git a/src/app.js b/src/app.js index 9dd41a2..3c7ec8a 100644 --- a/src/app.js +++ b/src/app.js @@ -8,7 +8,7 @@ var corsMiddleware = require('./middleware/cors.js'); var loggerMiddleware = require('./middleware/logger.js'); // Routes -var getSkipSegments = require('./routes/getSkipSegments.js'); +var getSkipSegments = require('./routes/getSkipSegments.js').endpoint; var postSkipSegments = require('./routes/postSkipSegments.js'); var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js'); var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js'); @@ -23,7 +23,7 @@ var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); // Old Routes -var getVideoSponsorTimes = require('./routes/getVideoSponsorTimes.js'); +var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js'); @@ -36,14 +36,14 @@ app.use(express.json()) if (config.mode === "development") app.set('json spaces', 2); //add the get function -app.get('/api/getVideoSponsorTimes', getVideoSponsorTimes); +app.get('/api/getVideoSponsorTimes', oldGetVideoSponsorTimes); //add the oldpost function app.get('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); //add the skip segments functions -app.get('/api/skipSegments', getSkipSegments.endpoint); +app.get('/api/skipSegments', getSkipSegments); app.post('/api/skipSegments', postSkipSegments); //voting endpoint diff --git a/src/routes/getVideoSponsorTimes.js b/src/routes/oldGetVideoSponsorTimes.js similarity index 100% rename from src/routes/getVideoSponsorTimes.js rename to src/routes/oldGetVideoSponsorTimes.js diff --git a/test/cases/getSponsorTime.js b/test/cases/oldGetSponsorTime.js similarity index 98% rename from test/cases/getSponsorTime.js rename to test/cases/oldGetSponsorTime.js index 306f563..33fed2d 100644 --- a/test/cases/getSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -17,7 +17,7 @@ var utils = require('../utils.js'); ); */ -describe('getVideoSponsorTime', () => { +describe('getVideoSponsorTime (Old get method)', () => { before(() => { db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); From 57cc4f698f32841930a328f5b21840ecd3b8bb12 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 02:24:58 -0400 Subject: [PATCH 17/20] Added get sponsor test cases --- test/cases/getSkipSegments.js | 94 +++++++++++++++++++++++++++++++++ test/cases/oldGetSponsorTime.js | 12 ++--- 2 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 test/cases/getSkipSegments.js diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js new file mode 100644 index 0000000..7e33518 --- /dev/null +++ b/test/cases/getSkipSegments.js @@ -0,0 +1,94 @@ +var request = require('request'); +var db = require('../../src/databases/databases.js').db; +var utils = require('../utils.js'); + +/* + *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "shadowHidden" INTEGER NOT NULL +); + */ + +describe('getSkipSegments', () => { + before(() => { + db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('test3', 1, 11, 2, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); + }); + + + it('Should be able to get a time by category', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest&category=sponsor", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = JSON.parse(res.body); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + done(); + } else { + done("Received incorrect body: " + res.body); + } + } + }); + }); + + it('Should be possible to send unexpected query parameters', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest&fakeparam=hello&category=sponsor", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = JSON.parse(res.body); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + done(); + } else { + done("Received incorrect body: " + res.body); + } + } + }); + }); + + it('Should return 404 if no segment found', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=notarealvideo", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 404) done("non 404 respone code: " + res.statusCode); + else done(); // pass + }); + }); + + + it('Should be able send a comma in a query param', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest,test&category=sponsor", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = JSON.parse(res.body); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-1") { + done(); + } else { + done("Received incorrect body: " + res.body); + } + } + }); + }); + +}); \ No newline at end of file diff --git a/test/cases/oldGetSponsorTime.js b/test/cases/oldGetSponsorTime.js index 33fed2d..ffb1264 100644 --- a/test/cases/oldGetSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -19,13 +19,13 @@ var utils = require('../utils.js'); describe('getVideoSponsorTime (Old get method)', () => { before(() => { - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); }); it('Should be able to get a time', (done) => { request.get(utils.getbaseURL() - + "/api/getVideoSponsorTimes?videoID=testtesttest", null, + + "/api/getVideoSponsorTimes?videoID=old-testtesttest", null, (err, res, body) => { if (err) done("Couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200"); @@ -46,7 +46,7 @@ describe('getVideoSponsorTime (Old get method)', () => { it('Should be possible to send unexpected query parameters', (done) => { request.get(utils.getbaseURL() - + "/api/getVideoSponsorTimes?videoID=testtesttest&fakeparam=hello", null, + + "/api/getVideoSponsorTimes?videoID=old-testtesttest&fakeparam=hello", null, (err, res, body) => { if (err) done("couldn't callendpoint"); else if (res.statusCode !== 200) done("non 200"); @@ -56,7 +56,7 @@ describe('getVideoSponsorTime (Old get method)', () => { it('Should be able send a comma in a query param', (done) => { request.get(utils.getbaseURL() - + "/api/getVideoSponsorTimes?videoID=testtesttest,test", null, + + "/api/getVideoSponsorTimes?videoID=old-testtesttest,test", null, (err, res, body) => { if (err) done("couln't call endpoint"); else if (res.statusCode !== 200) done("non 200 response: " + res.statusCode); @@ -67,7 +67,7 @@ describe('getVideoSponsorTime (Old get method)', () => { it('Should be able to get the correct time', (done) => { request.get(utils.getbaseURL() - + "/api/getVideoSponsorTimes?videoID=testtesttest", null, + + "/api/getVideoSponsorTimes?videoID=old-testtesttest", null, (err, res, body) => { if (err) done("couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200"); From d331ece0be96a17b02ab1933ac4a4182e29fb5b5 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 02:39:29 -0400 Subject: [PATCH 18/20] Added JSON get test cases --- test/cases/getSkipSegments.js | 123 +++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 2 deletions(-) diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index 7e33518..46feae3 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -22,11 +22,13 @@ describe('getSkipSegments', () => { db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)"); db.exec("INSERT INTO sponsorTimes VALUES ('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); db.exec("INSERT INTO sponsorTimes VALUES ('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); - db.exec("INSERT INTO sponsorTimes VALUES ('test3', 1, 11, 2, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)"); + db.exec("INSERT INTO sponsorTimes VALUES ('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)"); }); - it('Should be able to get a time by category', (done) => { + it('Should be able to get a time by category (Query Method) 1', (done) => { request.get(utils.getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor", null, (err, res, body) => { @@ -44,6 +46,105 @@ describe('getSkipSegments', () => { }); }); + it('Should be able to get a time by category (Query Method) 2', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest&category=intro", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = JSON.parse(res.body); + if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 + && data[0].category === "intro" && data[0].UUID === "1-uuid-2") { + done(); + } else { + done("Received incorrect body: " + res.body); + } + } + }); + }); + + it('Should be able to get a time by category (JSON Method) 1', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest&category=sponsor", { + json: { + videoID: "testtesttest", + categories: ["sponsor"] + } + }, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = res.body; + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + done(); + } else { + done("Received incorrect body: " + JSON.stringify(res.body)); + } + } + }); + }); + + it('Should be able to get a time by category (JSON Method) 2', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest&category=sponsor", { + json: { + videoID: "testtesttest", + categories: ["intro"] + } + }, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = res.body; + if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 + && data[0].category === "intro" && data[0].UUID === "1-uuid-2") { + done(); + } else { + done("Received incorrect body: " + JSON.stringify(res.body)); + } + } + }); + }); + + it('Should be able to get multiple times by category (JSON Method) 1', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=testtesttest&category=sponsor", { + json: { + videoID: "multiple", + categories: ["intro"] + } + }, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = res.body; + if (data.length === 2) { + + let success = true; + for (const segment of data) { + if ((segment.segment[0] !== 20 || segment.segment[1] !== 33 + || segment.category !== "intro" || segment.UUID !== "1-uuid-7") && + (segment.segment[0] !== 1 || segment.segment[1] !== 11 + || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) { + success = false; + break; + } + } + + if (success) done(); + else done("Received incorrect body: " + JSON.stringify(res.body)); + } else { + done("Received incorrect body: " + JSON.stringify(res.body)); + } + } + }); + }); + it('Should be possible to send unexpected query parameters', (done) => { request.get(utils.getbaseURL() + "/api/skipSegments?videoID=testtesttest&fakeparam=hello&category=sponsor", null, @@ -62,6 +163,24 @@ describe('getSkipSegments', () => { }); }); + it('Low voted submissions should be hidden', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments?videoID=test3&category=sponsor", null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = JSON.parse(res.body); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-4") { + done(); + } else { + done("Received incorrect body: " + res.body); + } + } + }); + }); + it('Should return 404 if no segment found', (done) => { request.get(utils.getbaseURL() + "/api/skipSegments?videoID=notarealvideo", null, From f9ec4e37e9bd246e850d337da020bb5d5c4091f7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 02:41:20 -0400 Subject: [PATCH 19/20] Added another test case for get --- test/cases/getSkipSegments.js | 41 ++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index 46feae3..5338311 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -66,7 +66,7 @@ describe('getSkipSegments', () => { it('Should be able to get a time by category (JSON Method) 1', (done) => { request.get(utils.getbaseURL() - + "/api/skipSegments?videoID=testtesttest&category=sponsor", { + + "/api/skipSegments", { json: { videoID: "testtesttest", categories: ["sponsor"] @@ -89,7 +89,7 @@ describe('getSkipSegments', () => { it('Should be able to get a time by category (JSON Method) 2', (done) => { request.get(utils.getbaseURL() - + "/api/skipSegments?videoID=testtesttest&category=sponsor", { + + "/api/skipSegments", { json: { videoID: "testtesttest", categories: ["intro"] @@ -112,7 +112,7 @@ describe('getSkipSegments', () => { it('Should be able to get multiple times by category (JSON Method) 1', (done) => { request.get(utils.getbaseURL() - + "/api/skipSegments?videoID=testtesttest&category=sponsor", { + + "/api/skipSegments", { json: { videoID: "multiple", categories: ["intro"] @@ -145,6 +145,41 @@ describe('getSkipSegments', () => { }); }); + it('Should be able to get multiple times by multiple categories (JSON Method)', (done) => { + request.get(utils.getbaseURL() + + "/api/skipSegments", { + json: { + videoID: "testtesttest", + categories: ["sponsor", "intro"] + } + }, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("Status code was: " + res.statusCode); + else { + let data = res.body; + if (data.length === 2) { + + let success = true; + for (const segment of data) { + if ((segment.segment[0] !== 20 || segment.segment[1] !== 33 + || segment.category !== "intro" || segment.UUID !== "1-uuid-2") && + (segment.segment[0] !== 1 || segment.segment[1] !== 11 + || segment.category !== "sponsor" || segment.UUID !== "1-uuid-0")) { + success = false; + break; + } + } + + if (success) done(); + else done("Received incorrect body: " + JSON.stringify(res.body)); + } else { + done("Received incorrect body: " + JSON.stringify(res.body)); + } + } + }); + }); + it('Should be possible to send unexpected query parameters', (done) => { request.get(utils.getbaseURL() + "/api/skipSegments?videoID=testtesttest&fakeparam=hello&category=sponsor", null, From 08a76dc42051afbb5c699632f6fa355323445e2a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 7 Apr 2020 14:45:47 -0400 Subject: [PATCH 20/20] Added back example config comment --- config.json.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.json.example b/config.json.example index 0d76f8b..b8a9d79 100644 --- a/config.json.example +++ b/config.json.example @@ -1,4 +1,6 @@ { + // Remove all comments from the config when using it! + "port": 80, "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]", "adminUserID": "[the hashed id of the user who can perform admin actions]",