From ca46f4c9ac8517c55e1b4456f3666fdf7f3b06ec Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 16:53:45 +0200 Subject: [PATCH 01/14] Enable database upgrades to run --- src/databases/databases.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databases/databases.js b/src/databases/databases.js index 7c3991f..d7008a0 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -49,11 +49,11 @@ function ugradeDB(db, prefix) { let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; - let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; while (fs.existsSync(path)) { db.exec(fs.readFileSync(path).toString()); versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value; - path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; } } \ No newline at end of file From 803b1fa5050233c43b062c934cebd0a1365957ca Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 16:54:21 +0200 Subject: [PATCH 02/14] Add initial version of querying by hash prefix --- databases/_upgrade_sponsorTimes_2.sql | 26 ++++++++++ src/app.js | 4 ++ src/databases/databases.js | 4 ++ src/routes/getSkipSegmentsByHash.js | 69 +++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 databases/_upgrade_sponsorTimes_2.sql create mode 100644 src/routes/getSkipSegmentsByHash.js diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql new file mode 100644 index 0000000..5472499 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -0,0 +1,26 @@ +BEGIN TRANSACTION; + +/* Add hash field */ +CREATE TABLE "sqlb_temp_table_1" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT "sponsor", + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL +); +INSERT INTO sqlb_temp_table_1 SELECT *, sha256(videoID) FROM sponsorTimes; + +DROP TABLE sponsorTimes; +ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; + +/* Bump version in config */ +UPDATE config SET value = 2 WHERE key = "version"; + +COMMIT; \ No newline at end of file diff --git a/src/app.js b/src/app.js index 9561dc1..5595625 100644 --- a/src/app.js +++ b/src/app.js @@ -10,6 +10,7 @@ var loggerMiddleware = require('./middleware/logger.js'); // Routes var getSkipSegments = require('./routes/getSkipSegments.js').endpoint; var postSkipSegments = require('./routes/postSkipSegments.js'); +var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js'); var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js'); var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js'); var setUsername = require('./routes/setUsername.js'); @@ -48,6 +49,9 @@ app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); app.get('/api/skipSegments', getSkipSegments); app.post('/api/skipSegments', postSkipSegments); +// add the privacy protecting skip segments functions +app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash); + //voting endpoint app.get('/api/voteOnSponsorTime', voteOnSponsorTime); app.post('/api/voteOnSponsorTime', voteOnSponsorTime); diff --git a/src/databases/databases.js b/src/databases/databases.js index d7008a0..621ad8b 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -26,6 +26,10 @@ if (config.createDatabaseIfNotExist && !config.readOnly) { // Upgrade database if required if (!config.readOnly) { + // Register hashing function needed for running database upgrade + db.function("sha256", function (string) { + return require('crypto').createHash("sha256").update(string).digest("hex"); + }); ugradeDB(db, "sponsorTimes"); ugradeDB(privateDB, "private") } diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js new file mode 100644 index 0000000..3672eed --- /dev/null +++ b/src/routes/getSkipSegmentsByHash.js @@ -0,0 +1,69 @@ +const config = require('../config.js'); +const { db, privateDB } = require('../databases/databases.js'); + +const getHash = require('../utils/getHash.js'); +const getIP = require('../utils/getIP.js'); + +/** + * @typedef {Object} Segment + * @property {string} videoID YouTube video ID the segment is meant for + * @property {number[]} segment Tuple of start and end times in seconds + * @property {string} category Category of content to skip + * @property {string} UUID Unique identifier for the specific segment + */ + +/** + * @param {string} prefix Lowercased hexadecimal hash prefix + * @param {string} hashedIP Custom hash of the visitor’s IP address + * @returns {Segment[]} + */ +function getSkipSegmentsByHash(prefix, hashedIP) { + /** + * @constant + * @type {Segment[]} + * @default + */ + const segments = []; + + const rows = db.prepare('SELECT videoID, startTime, endTime, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY startTime') + .all(prefix + '%'); + + const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID); + + for (const row of rows) { + /** @TODO check if this logic does what is expected. */ + if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) { + // The current visitor’s IP did not submit for the current video. + // Do not send shadowHidden segments to them. + continue; + } + + segments.push({ + videoID: row.videoID, + segment: [row.startTime, row.endTime], + category: row.category, + UUID: row.UUID + }); + } + + return segments; +} + +const minimumPrefix = config.minimumPrefix || '3'; +const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. +const prefixChecker = new RegExp('^[\\dA-F]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i'); + +module.exports = async function (req, res) { + if (!prefixChecker.test(req.params.prefix)) { + res.sendStatus(400).end(); // Exit early on faulty prefix + } + + const segments = getSkipSegmentsByHash( + req.params.prefix.toLowerCase(), + getHash(getIP(req) + config.globalSalt) + ); + + if (segments) { + res.send(segments) + } +} \ No newline at end of file From d8203dce77a5962076b93ab08fcc9213dc9d7d54 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 17:57:33 +0200 Subject: [PATCH 03/14] Make sure to always insert hashed video ID --- src/routes/postSkipSegments.js | 6 +++--- test/cases/getSavedTimeForUser.js | 4 ++-- test/cases/getSkipSegments.js | 17 +++++++++-------- test/cases/oldGetSponsorTime.js | 7 ++++--- test/cases/voteOnSponsorTime.js | 16 ++++++++-------- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e178ee6..e13280f 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -232,9 +232,9 @@ module.exports = async function postSkipSegments(req, res) { try { db.prepare("INSERT INTO sponsorTimes " + - "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" + - "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], - segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); + "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], + segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 0)); //add to private db as well privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); diff --git a/test/cases/getSavedTimeForUser.js b/test/cases/getSavedTimeForUser.js index 5df8a48..242204d 100644 --- a/test/cases/getSavedTimeForUser.js +++ b/test/cases/getSavedTimeForUser.js @@ -5,8 +5,8 @@ var getHash = require('../../src/utils/getHash.js'); describe('getSavedTimeForUser', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0, '" + getHash('getSavedTimeForUser', 0) + "')"); }); it('Should be able to get a 200', (done) => { diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index 5c15c63..db4698e 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -1,6 +1,7 @@ var request = require('request'); var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js'); /* *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( @@ -18,14 +19,14 @@ var utils = require('../utils.js'); describe('getSkipSegments', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 0) + "')"); + db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 0) + "')"); + db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 0) + "')"); + db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')"); + db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')"); + db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')"); + db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')"); }); diff --git a/test/cases/oldGetSponsorTime.js b/test/cases/oldGetSponsorTime.js index 569b109..8fe19d2 100644 --- a/test/cases/oldGetSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -1,6 +1,7 @@ var request = require('request'); var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js'); /* @@ -19,9 +20,9 @@ var utils = require('../utils.js'); describe('getVideoSponsorTime (Old get method)', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 0) + "')"); + db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 0) + "')"); }); it('Should be able to get a time', (done) => { diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index db618d2..437bbea 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -5,14 +5,14 @@ var getHash = require('../../src/utils/getHash.js') describe('voteOnSponsorTime', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-testtesttest', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 0) + "')"); + db.exec(startOfQuery + "('vote-testtesttest', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-testtesttest', 0) + "')"); + db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest,test', 0) + "')"); + db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-test3', 0) + "')"); + db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-test3', 0) + "')"); + db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-multiple', 0) + "')"); + db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-multiple', 0) + "')"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); }); From c2510d302a3934a8c895c03b943403721b6df763 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 21:17:47 +0200 Subject: [PATCH 04/14] Limit overlapping segments to just one through weighted randomness --- src/routes/getSkipSegmentsByHash.js | 90 +++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 3672eed..8bb5fcc 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -12,24 +12,64 @@ const getIP = require('../utils/getIP.js'); * @property {string} UUID Unique identifier for the specific segment */ +/** + * @typedef {Object} Row + * @property {string} videoID + * @property {number} startTime + * @property {number} endTime + * @property {number} votes + * @property {string} UUID + * @property {string} category + * @property {number} shadowHidden + */ + +/** + * Input an array of database records and get only one back, weighed on votes. + * The logic is taken from getWeightedRandomChoice, just simplified input and output to not work on indices only. + * + * @param {Row[]} rows + * @returns {?Row} + */ +function pickWeightedRandomRow(rows) { + if (rows.length === 0) { + return null; + } else if (rows.length === 1) { + return rows[0]; + } + + const sqrtWeightsList = []; + let totalSqrtWeights = 0; + for (const row of rows) { + let sqrtVote = Math.sqrt((row.votes + 3) * 10); + sqrtWeightsList.push(sqrtVote); + totalSqrtWeights += sqrtVote; + } + + const randomNumber = Math.random(); + let currentVoteNumber = 0; + for (let i = 0; i < sqrtWeightsList.length; i++) { + if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[i]) / totalSqrtWeights) { + return rows[i]; + } + currentVoteNumber += sqrtWeightsList[i]; + } +} /** * @param {string} prefix Lowercased hexadecimal hash prefix * @param {string} hashedIP Custom hash of the visitor’s IP address - * @returns {Segment[]} + * @returns {Object.} */ function getSkipSegmentsByHash(prefix, hashedIP) { - /** - * @constant - * @type {Segment[]} - * @default - */ - const segments = []; - - const rows = db.prepare('SELECT videoID, startTime, endTime, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY startTime') + /** @type Row[] */ + const rows = db.prepare('SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY videoID, startTime') .all(prefix + '%'); - + /** @type {string[]} */ const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID); + /** @type {Object.} */ + const rowGroupsPerVideo = {}; + let previousVideoID = null; + let previousEndTime = null; for (const row of rows) { /** @TODO check if this logic does what is expected. */ if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) { @@ -37,16 +77,30 @@ function getSkipSegmentsByHash(prefix, hashedIP) { // Do not send shadowHidden segments to them. continue; } - - segments.push({ - videoID: row.videoID, - segment: [row.startTime, row.endTime], - category: row.category, - UUID: row.UUID - }); + // Split up the rows per video and group overlapping segments together. + if (!(row.videoID in rowGroupsPerVideo)) { + rowGroupsPerVideo[row.videoID] = []; + } + if (previousVideoID === row.videoID && row.startTime <= previousEndTime) { + rowGroupsPerVideo[row.videoID][rowGroupsPerVideo[row.videoID].length - 1].push(row); + previousEndTime = Math.max(previousEndTime, row.endTime); + } else { + rowGroupsPerVideo[row.videoID].push([row]); + previousVideoID = row.videoID; + previousEndTime = row.endTime; + } } - return segments; + /** @type {Object.} */ + const output = {}; + for (const videoID in rowGroupsPerVideo) { + const pickedVideosForVideoID = []; + for (const group of rowGroupsPerVideo[videoID]) { + pickedVideosForVideoID.push(pickWeightedRandomRow(group)); + } + output[videoID] = pickedVideosForVideoID.map(row => ({ videoID: row.videoID, segment: [row.startTime, row.endTime], category: row.category, UUID: row.UUID })); + } + return output; } const minimumPrefix = config.minimumPrefix || '3'; From c7fd603933e7d33661c7d3ba7fb0486411b619b4 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 21:25:38 +0200 Subject: [PATCH 05/14] Respond Not Found when a prefix is empty --- src/routes/getSkipSegmentsByHash.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 8bb5fcc..1762ef3 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -117,7 +117,9 @@ module.exports = async function (req, res) { getHash(getIP(req) + config.globalSalt) ); - if (segments) { - res.send(segments) + if (Object.keys(segments).length > 0) { + res.send(segments); + } else { + res.sendStatus(404); // No skipable segments within this prefix } } \ No newline at end of file From 3167c24f75bcd9c390baff39c72a47130e2d16f8 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 21:27:21 +0200 Subject: [PATCH 06/14] Put hashedVideoID column before shadowHidden column --- databases/_upgrade_sponsorTimes_2.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql index 5472499..1c7270b 100644 --- a/databases/_upgrade_sponsorTimes_2.sql +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -12,8 +12,8 @@ CREATE TABLE "sqlb_temp_table_1" ( "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, "category" TEXT NOT NULL DEFAULT "sponsor", - "shadowHidden" INTEGER NOT NULL, - "hashedVideoID" TEXT NOT NULL + "hashedVideoID" TEXT NOT NULL, + "shadowHidden" INTEGER NOT NULL ); INSERT INTO sqlb_temp_table_1 SELECT *, sha256(videoID) FROM sponsorTimes; From 1a06502806fe2e09a5c19abd8b20bd7fd90900fc Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 00:45:06 +0100 Subject: [PATCH 07/14] added get segments by hash prefix --- src/routes/getSkipSegments.js | 118 +++++++++++----------- src/routes/getSkipSegmentsByHash.js | 145 ++++++---------------------- src/routes/voteOnSponsorTime.js | 9 +- src/utils/hashPrefixTester.js | 11 +++ test/cases/getSegmentsByHash.js | 85 ++++++++++++++++ test/cases/oldGetSponsorTime.js | 4 +- test/cases/voteOnSponsorTime.js | 32 +++--- 7 files changed, 206 insertions(+), 198 deletions(-) create mode 100644 src/utils/hashPrefixTester.js create mode 100644 test/cases/getSegmentsByHash.js diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index 81aae1c..db6ce76 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -8,6 +8,62 @@ var logger = require('../utils/logger.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); +function cleanGetSegments(videoID, categories) { + let userHashedIP, shadowHiddenSegments; + + let segments = []; + + try { + for (const category of categories) { + const categorySegments = db + .prepare( + 'all', + 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', + [videoID, category] + ) + .filter(segment => { + if (segment.votes < -1) { + return false; //too untrustworthy, just ignore it + } + + //check if shadowHidden + //this means it is hidden to everyone but the original ip that submitted it + if (segment.shadowHidden != 1) { + return true; + } + + if (shadowHiddenSegments === undefined) { + shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); + } + + //if this isn't their ip, don't send it to them + return shadowHiddenSegments.some(shadowHiddenSegment => { + if (userHashedIP === undefined) { + //hash the IP only if it's strictly necessary + userHashedIP = getHash(getIP(req) + config.globalSalt); + } + return shadowHiddenSegment.hashedIP === userHashedIP; + }); + }); + + chooseSegments(categorySegments).forEach(chosenSegment => { + segments.push({ + category, + segment: [chosenSegment.startTime, chosenSegment.endTime], + UUID: chosenSegment.UUID, + }); + }); + } + + return segments; + } catch (err) { + if (err) { + logger.error('j 2 Query failed'); + return undefined; + } + } +} + //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique @@ -104,58 +160,11 @@ function handleGetSegments(req, res) { ? [req.query.category] : ['sponsor']; - /** - * @type {Array<{ - * segment: number[], - * category: string, - * UUID: string - * }> - * } - */ - const segments = []; + let segments = cleanGetSegments(videoID, categories); - let userHashedIP, shadowHiddenSegments; - - try { - for (const category of categories) { - const categorySegments = db - .prepare( - 'all', - 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', - [videoID, category] - ) - .filter(segment => { - if (segment.votes < -1) { - return false; //too untrustworthy, just ignore it - } - - //check if shadowHidden - //this means it is hidden to everyone but the original ip that submitted it - if (segment.shadowHidden != 1) { - return true; - } - - if (shadowHiddenSegments === undefined) { - shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); - } - - //if this isn't their ip, don't send it to them - return shadowHiddenSegments.some(shadowHiddenSegment => { - if (userHashedIP === undefined) { - //hash the IP only if it's strictly necessary - userHashedIP = getHash(getIP(req) + config.globalSalt); - } - return shadowHiddenSegment.hashedIP === userHashedIP; - }); - }); - - chooseSegments(categorySegments).forEach(chosenSegment => { - segments.push({ - category, - segment: [chosenSegment.startTime, chosenSegment.endTime], - UUID: chosenSegment.UUID, - }); - }); + if (segments === undefined) { + res.sendStatus(500); + return false; } if (segments.length == 0) { @@ -164,16 +173,11 @@ function handleGetSegments(req, res) { } return segments; - } catch (error) { - logger.error(error); - res.sendStatus(500); - - return false; - } } module.exports = { handleGetSegments, + cleanGetSegments, endpoint: function (req, res) { let segments = handleGetSegments(req, res); diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 1762ef3..eec0e0e 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -1,125 +1,36 @@ -const config = require('../config.js'); -const { db, privateDB } = require('../databases/databases.js'); +const hashPrefixTester = require('../utils/hashPrefixTester.js'); +const getSegments = require('./getSkipSegments.js').cleanGetSegments; -const getHash = require('../utils/getHash.js'); -const getIP = require('../utils/getIP.js'); - -/** - * @typedef {Object} Segment - * @property {string} videoID YouTube video ID the segment is meant for - * @property {number[]} segment Tuple of start and end times in seconds - * @property {string} category Category of content to skip - * @property {string} UUID Unique identifier for the specific segment - */ - -/** - * @typedef {Object} Row - * @property {string} videoID - * @property {number} startTime - * @property {number} endTime - * @property {number} votes - * @property {string} UUID - * @property {string} category - * @property {number} shadowHidden - */ - -/** - * Input an array of database records and get only one back, weighed on votes. - * The logic is taken from getWeightedRandomChoice, just simplified input and output to not work on indices only. - * - * @param {Row[]} rows - * @returns {?Row} - */ -function pickWeightedRandomRow(rows) { - if (rows.length === 0) { - return null; - } else if (rows.length === 1) { - return rows[0]; - } - - const sqrtWeightsList = []; - let totalSqrtWeights = 0; - for (const row of rows) { - let sqrtVote = Math.sqrt((row.votes + 3) * 10); - sqrtWeightsList.push(sqrtVote); - totalSqrtWeights += sqrtVote; - } - - const randomNumber = Math.random(); - let currentVoteNumber = 0; - for (let i = 0; i < sqrtWeightsList.length; i++) { - if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[i]) / totalSqrtWeights) { - return rows[i]; - } - currentVoteNumber += sqrtWeightsList[i]; - } -} -/** - * @param {string} prefix Lowercased hexadecimal hash prefix - * @param {string} hashedIP Custom hash of the visitor’s IP address - * @returns {Object.} - */ -function getSkipSegmentsByHash(prefix, hashedIP) { - /** @type Row[] */ - const rows = db.prepare('SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY videoID, startTime') - .all(prefix + '%'); - /** @type {string[]} */ - const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID); - /** @type {Object.} */ - const rowGroupsPerVideo = {}; - - let previousVideoID = null; - let previousEndTime = null; - for (const row of rows) { - /** @TODO check if this logic does what is expected. */ - if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) { - // The current visitor’s IP did not submit for the current video. - // Do not send shadowHidden segments to them. - continue; - } - // Split up the rows per video and group overlapping segments together. - if (!(row.videoID in rowGroupsPerVideo)) { - rowGroupsPerVideo[row.videoID] = []; - } - if (previousVideoID === row.videoID && row.startTime <= previousEndTime) { - rowGroupsPerVideo[row.videoID][rowGroupsPerVideo[row.videoID].length - 1].push(row); - previousEndTime = Math.max(previousEndTime, row.endTime); - } else { - rowGroupsPerVideo[row.videoID].push([row]); - previousVideoID = row.videoID; - previousEndTime = row.endTime; - } - } - - /** @type {Object.} */ - const output = {}; - for (const videoID in rowGroupsPerVideo) { - const pickedVideosForVideoID = []; - for (const group of rowGroupsPerVideo[videoID]) { - pickedVideosForVideoID.push(pickWeightedRandomRow(group)); - } - output[videoID] = pickedVideosForVideoID.map(row => ({ videoID: row.videoID, segment: [row.startTime, row.endTime], category: row.category, UUID: row.UUID })); - } - return output; -} - -const minimumPrefix = config.minimumPrefix || '3'; -const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. -const prefixChecker = new RegExp('^[\\dA-F]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i'); +const databases = require('../databases/databases.js'); +const db = databases.db; module.exports = async function (req, res) { - if (!prefixChecker.test(req.params.prefix)) { - res.sendStatus(400).end(); // Exit early on faulty prefix + let hashPrefix = req.params.prefix; + if (!hashPrefixTester(req.params.prefix)) { + res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix + return; } - const segments = getSkipSegmentsByHash( - req.params.prefix.toLowerCase(), - getHash(getIP(req) + config.globalSalt) - ); + const categories = req.query.categories + ? JSON.parse(req.query.categories) + : req.query.category + ? [req.query.category] + : ['sponsor']; - if (Object.keys(segments).length > 0) { - res.send(segments); - } else { - res.sendStatus(404); // No skipable segments within this prefix + // Get all video id's that match hash prefix + const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']); + if (videoIds.length === 0) { + res.sendStatus(404); + return; } + + let segments = videoIds.map((video) => { + return { + videoID: video.videoID, + hash: video.hashedVideoID, + segments: getSegments(video.videoID, categories) + }; + }); + + res.status(200).json(segments); } \ No newline at end of file diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 8f9b381..5ebdc6d 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -1,4 +1,3 @@ -var fs = require('fs'); var config = require('../config.js'); var getHash = require('../utils/getHash.js'); @@ -370,8 +369,6 @@ async function voteOnSponsorTime(req, res) { } module.exports = { - voteOnSponsorTime, - endpoint: function (req, res) { - voteOnSponsorTime(req, res); - }, - }; + voteOnSponsorTime, + endpoint: voteOnSponsorTime +}; diff --git a/src/utils/hashPrefixTester.js b/src/utils/hashPrefixTester.js new file mode 100644 index 0000000..0a6639e --- /dev/null +++ b/src/utils/hashPrefixTester.js @@ -0,0 +1,11 @@ +const config = require('../config.js'); +const logger = require('./logger.js'); + +const minimumPrefix = config.minimumPrefix || '3'; +const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. + +const prefixChecker = new RegExp('^[\\da-f]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i'); + +module.exports = (prefix) => { + return prefixChecker.test(prefix); +}; \ No newline at end of file diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js new file mode 100644 index 0000000..6b6de50 --- /dev/null +++ b/test/cases/getSegmentsByHash.js @@ -0,0 +1,85 @@ +var request = require('request'); +var db = require('../../src/databases/databases.js').db; +var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js'); + + +/* + *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "shadowHidden" INTEGER NOT NULL +); + */ + +describe('getSegmentsByHash', () => { + before(() => { + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + db.exec(startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + db.exec(startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b + }); + + it('Should be able to get a 200', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + done(); + } // pass + }); + }); + + it('Should be able to get a 200 with empty segments for video but no matching categories', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/3272f?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + if (JSON.parse(body) && JSON.parse(body).length > 0 && JSON.parse(body)[0].segments.length === 0) { + done(); // pass + } else { + done("response had segments"); + } + } + }); + }); + + it('Should be able to get a 404 if no videos', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11111?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should be able to get multiple videos', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + body = JSON.parse(body); + if (body.length !== 2) done("expected 2 video, got " + body.length); + else if (body[0].segments.length !== 2) done("expected 2 segments for first video, got " + body[0].segments.length); + else if (body[1].segments.length !== 1) done("expected 1 segment for second video, got " + body[1].segments.length); + else done(); + } + }); + }); +}); \ No newline at end of file diff --git a/test/cases/oldGetSponsorTime.js b/test/cases/oldGetSponsorTime.js index 8fe19d2..d4b9a88 100644 --- a/test/cases/oldGetSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -21,8 +21,8 @@ var getHash = require('../../src/utils/getHash.js'); describe('getVideoSponsorTime (Old get method)', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; - db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 0) + "')"); - db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 0) + "')"); + db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 1) + "')"); + db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 1) + "')"); }); it('Should be able to get a time', (done) => { diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index 8ecb3b3..e99c2fe 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -6,22 +6,22 @@ const getHash = require('../../src/utils/getHash.js'); describe('voteOnSponsorTime', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; - db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest2', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest2', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest2', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest3', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest,test', 1)+"')"); + db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-test3', 1)+"')"); + db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-test3', 1)+"')"); + db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')"); + db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')"); + db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('own-submission-video', 1)+"')"); + db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('not-own-submission-video', 1)+"')"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); From 84c8eeccc6172bffe5444d1b890b273b89a2b57c Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:06:50 +0100 Subject: [PATCH 08/14] added hash prefix test for missing categories --- test/cases/getSegmentsByHash.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 6b6de50..fb83526 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -82,4 +82,22 @@ describe('getSegmentsByHash', () => { } }); }); + + it('Should be able to get 200 for no categories (default sponsor)', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/fdaf', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + console.log(body); + body = JSON.parse(body); + if (body.length !== 2) done("expected 2 videos, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); + else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length); + else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else done(); + } + }); + }); }); \ No newline at end of file From 28bd24022b3792d9c96cecbafe6f60b4dc85c4e1 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:08:18 +0100 Subject: [PATCH 09/14] removed schema comments and removed test log --- test/cases/getSegmentsByHash.js | 16 ---------------- test/cases/getSkipSegments.js | 13 ------------- 2 files changed, 29 deletions(-) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index fb83526..1c60484 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -3,21 +3,6 @@ var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); var getHash = require('../../src/utils/getHash.js'); - -/* - *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "shadowHidden" INTEGER NOT NULL -); - */ - describe('getSegmentsByHash', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; @@ -90,7 +75,6 @@ describe('getSegmentsByHash', () => { if (err) done("Couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); else { - console.log(body); body = JSON.parse(body); if (body.length !== 2) done("expected 2 videos, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index db4698e..0314028 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -3,19 +3,6 @@ var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); var getHash = require('../../src/utils/getHash.js'); -/* - *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "shadowHidden" INTEGER NOT NULL -); - */ describe('getSkipSegments', () => { before(() => { From 82d59e159fc5136021fbe5a62bcbd99ae42351a6 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 02:42:25 +0100 Subject: [PATCH 10/14] added post and get test to hash prefix --- databases/_upgrade_sponsorTimes_3.sql | 2 ++ src/databases/databases.js | 5 ++-- test/cases/getHash.js | 4 +++ test/cases/getSegmentsByHash.js | 36 +++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/databases/_upgrade_sponsorTimes_3.sql b/databases/_upgrade_sponsorTimes_3.sql index ee2e041..1d2edbd 100644 --- a/databases/_upgrade_sponsorTimes_3.sql +++ b/databases/_upgrade_sponsorTimes_3.sql @@ -15,6 +15,8 @@ CREATE TABLE "sqlb_temp_table_3" ( "shadowHidden" INTEGER NOT NULL, "hashedVideoID" TEXT NOT NULL ); + +/* hash upgade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */ INSERT INTO sqlb_temp_table_3 SELECT *, sha256(videoID) FROM sponsorTimes; DROP TABLE sponsorTimes; diff --git a/src/databases/databases.js b/src/databases/databases.js index 5625d6c..a6f38ab 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -5,6 +5,7 @@ var path = require('path'); var Sqlite = require('./Sqlite.js') var Mysql = require('./Mysql.js'); const logger = require('../utils/logger.js'); +const getHash = require('../utils/getHash.js'); let options = { readonly: config.readOnly, @@ -34,8 +35,8 @@ if (config.mysql) { } if (!config.readOnly) { - db.function("sha256", function (string) { - return require('crypto').createHash("sha256").update(string).digest("hex"); + db.function("sha256", (string) => { + return getHash(string, 1); }); // Upgrade database if required diff --git a/test/cases/getHash.js b/test/cases/getHash.js index 15d7cc5..896076d 100644 --- a/test/cases/getHash.js +++ b/test/cases/getHash.js @@ -13,6 +13,10 @@ describe('getHash', () => { assert.equal(getHash("test"), "2f327ef967ade1ebf4319163f7debbda9cc17bb0c8c834b00b30ca1cf1c256ee"); }); + it ('Should be able to output the same has the DB upgrade script will output', () => { + assert.equal(getHash("vid", 1), "1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663"); + }); + it ('Should take a variable number of passes', () => { assert.equal(getHash("test", 1), "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); assert.equal(getHash("test", 2), "7b3d979ca8330a94fa7e9e1b466d8b99e0bcdea1ec90596c0dcc8d7ef6b4300c"); diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 1c60484..1717e98 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -84,4 +84,40 @@ describe('getSegmentsByHash', () => { } }); }); + + it('Should be able to post a segment and get it using endpoint', (done) => { + let testID = 'abc123goodVideo'; + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: testID, + segments: [{ + segment: [13, 17], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done('(post) ' + err); + else if (res.statusCode === 200) { + request.get(utils.getbaseURL() + + '/api/skipSegments/'+getHash(testID, 1).substring(0,3), null, + (err, res, body) => { + if (err) done("(get) Couldn't call endpoint"); + else if (res.statusCode !== 200) done("(get) non 200 status code, was " + res.statusCode); + else { + body = JSON.parse(body); + if (body.length !== 1) done("(get) expected 1 video, got " + body.length); + else if (body[0].segments.length !== 1) done("(get) expected 1 segments for first video, got " + body[0].segments.length); + else if (body[0].segments[0].category !== 'sponsor') done("(get) segment should be sponsor, was "+body[0].segments[0].category); + else done(); + } + }); + } else { + done("(post) non 200 status code, was " + res.statusCode); + } + } + ); + }); }); \ No newline at end of file From 88e6c6f93c05769a7847464e294029526eaa7c5d Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 02:52:12 +0100 Subject: [PATCH 11/14] added db upgrade tests --- databases/_upgrade_private_1.sql | 6 ++++++ test/cases/dbUpgrade.js | 12 ++++++++++++ test/cases/getSegmentsByHash.js | 6 ++++++ test/cases/noSegmentRecords.js | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 databases/_upgrade_private_1.sql create mode 100644 test/cases/dbUpgrade.js diff --git a/databases/_upgrade_private_1.sql b/databases/_upgrade_private_1.sql new file mode 100644 index 0000000..8c7ab33 --- /dev/null +++ b/databases/_upgrade_private_1.sql @@ -0,0 +1,6 @@ +BEGIN TRANSACTION; + +/* Add version to config */ +INSERT INTO config (key, value) VALUES("version", 1); + +COMMIT; \ No newline at end of file diff --git a/test/cases/dbUpgrade.js b/test/cases/dbUpgrade.js new file mode 100644 index 0000000..8228d27 --- /dev/null +++ b/test/cases/dbUpgrade.js @@ -0,0 +1,12 @@ +const databases = require('../../src/databases/databases.js'); +const db = databases.db; +const privateDB = databases.privateDB; + +describe('dbUpgrade', () => { + it('Should update the database version when starting the application', (done) => { + let dbVersion = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + let privateVersion = privateDB.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (dbVersion >= 1 && privateVersion >= 1) done(); + else done('Versions are not at least 1. db is ' + dbVersion + ', private is ' + privateVersion); + }); +}); \ No newline at end of file diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 1717e98..191b58a 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -12,6 +12,12 @@ describe('getSegmentsByHash', () => { db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b }); + it('Should update the database version when starting the application', (done) => { + let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (version > 2) done(); + else done('Version isn\'t greater than 2. Version is ' + version); + }); + it('Should be able to get a 200', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index d902478..8c2c393 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -22,7 +22,7 @@ describe('noSegmentRecords', () => { it('Should update the database version when starting the application', (done) => { let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; if (version > 1) done(); - else done('Version isn\'t greater that 1. Version is ' + version); + else done('Version isn\'t greater than 1. Version is ' + version); }); it('Should be able to submit categorys not in video (http response)', (done) => { From 20675998437c5f5a405eff00464ddd448b44f539 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 03:27:12 +0100 Subject: [PATCH 12/14] added test for hash prefix length --- src/utils/hashPrefixTester.js | 1 - test/cases/getSegmentsByHash.js | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/utils/hashPrefixTester.js b/src/utils/hashPrefixTester.js index 0a6639e..f5aec61 100644 --- a/src/utils/hashPrefixTester.js +++ b/src/utils/hashPrefixTester.js @@ -1,5 +1,4 @@ const config = require('../config.js'); -const logger = require('./logger.js'); const minimumPrefix = config.minimumPrefix || '3'; const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 191b58a..1084ee3 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -58,6 +58,48 @@ describe('getSegmentsByHash', () => { }); }); + it('Should return 400 prefix too short', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should return 400 prefix too long', (done) => { + let prefix = new Array(50).join('1'); + if (prefix.length <= 32) { // default value, config can change this + done('failed to generate a long enough string for the test ' + prefix.length); + return; + } + + request.get(utils.getbaseURL() + + '/api/skipSegments/'+prefix+'?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should not return 400 prefix in range', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11111?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 400) done("prefix length 5 gave 400 " + res.statusCode); + else { + done(); // pass + } + }); + }); + it('Should be able to get multiple videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, From 9c9c2a23cc7efcbbeecf6fac93b3128d7aa3dc03 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 03:31:53 +0100 Subject: [PATCH 13/14] added comment to db upgrade sql file --- databases/_upgrade_private_1.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/databases/_upgrade_private_1.sql b/databases/_upgrade_private_1.sql index 8c7ab33..c4de7e9 100644 --- a/databases/_upgrade_private_1.sql +++ b/databases/_upgrade_private_1.sql @@ -1,5 +1,7 @@ BEGIN TRANSACTION; +/* for testing the db upgrade, don't remove because it looks empty */ + /* Add version to config */ INSERT INTO config (key, value) VALUES("version", 1); From 2c32460a6e9c698c70a3ca224c5b994e8ef08356 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 04:17:50 +0100 Subject: [PATCH 14/14] more tests --- src/routes/getSkipSegmentsByHash.js | 1 + test/cases/getSegmentsByHash.js | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index eec0e0e..702a28d 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -2,6 +2,7 @@ const hashPrefixTester = require('../utils/hashPrefixTester.js'); const getSegments = require('./getSkipSegments.js').cleanGetSegments; const databases = require('../databases/databases.js'); +const logger = require('../utils/logger.js'); const db = databases.db; module.exports = async function (req, res) { diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 1084ee3..6571fe4 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -100,6 +100,30 @@ describe('getSegmentsByHash', () => { }); }); + it('Should return 404 for no hash', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 404) done("expected 404, got " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should return 500 for bad format categories', (done) => { // should probably be 400 + request.get(utils.getbaseURL() + + '/api/skipSegments/?categories=shilling', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 500) done("expected 500 got " + res.statusCode); + else { + done(); // pass + } + }); + }); + it('Should be able to get multiple videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null,