added get segments by hash prefix

This commit is contained in:
Joe Dowd
2020-08-31 00:45:06 +01:00
parent 26c72b006c
commit 1a06502806
7 changed files with 206 additions and 198 deletions

View File

@@ -8,6 +8,62 @@ var logger = require('../utils/logger.js');
var getHash = require('../utils/getHash.js'); var getHash = require('../utils/getHash.js');
var getIP = require('../utils/getIP.js'); var getIP = require('../utils/getIP.js');
function cleanGetSegments(videoID, categories) {
let userHashedIP, shadowHiddenSegments;
let segments = [];
try {
for (const category of categories) {
const categorySegments = db
.prepare(
'all',
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
[videoID, category]
)
.filter(segment => {
if (segment.votes < -1) {
return false; //too untrustworthy, just ignore it
}
//check if shadowHidden
//this means it is hidden to everyone but the original ip that submitted it
if (segment.shadowHidden != 1) {
return true;
}
if (shadowHiddenSegments === undefined) {
shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
}
//if this isn't their ip, don't send it to them
return shadowHiddenSegments.some(shadowHiddenSegment => {
if (userHashedIP === undefined) {
//hash the IP only if it's strictly necessary
userHashedIP = getHash(getIP(req) + config.globalSalt);
}
return shadowHiddenSegment.hashedIP === userHashedIP;
});
});
chooseSegments(categorySegments).forEach(chosenSegment => {
segments.push({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
});
});
}
return segments;
} catch (err) {
if (err) {
logger.error('j 2 Query failed');
return undefined;
}
}
}
//gets a weighted random choice from the choices array based on their `votes` property. //gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more. //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique //choices are unique
@@ -104,58 +160,11 @@ function handleGetSegments(req, res) {
? [req.query.category] ? [req.query.category]
: ['sponsor']; : ['sponsor'];
/** let segments = cleanGetSegments(videoID, categories);
* @type {Array<{
* segment: number[],
* category: string,
* UUID: string
* }>
* }
*/
const segments = [];
let userHashedIP, shadowHiddenSegments; if (segments === undefined) {
res.sendStatus(500);
try { return false;
for (const category of categories) {
const categorySegments = db
.prepare(
'all',
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
[videoID, category]
)
.filter(segment => {
if (segment.votes < -1) {
return false; //too untrustworthy, just ignore it
}
//check if shadowHidden
//this means it is hidden to everyone but the original ip that submitted it
if (segment.shadowHidden != 1) {
return true;
}
if (shadowHiddenSegments === undefined) {
shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
}
//if this isn't their ip, don't send it to them
return shadowHiddenSegments.some(shadowHiddenSegment => {
if (userHashedIP === undefined) {
//hash the IP only if it's strictly necessary
userHashedIP = getHash(getIP(req) + config.globalSalt);
}
return shadowHiddenSegment.hashedIP === userHashedIP;
});
});
chooseSegments(categorySegments).forEach(chosenSegment => {
segments.push({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
});
});
} }
if (segments.length == 0) { if (segments.length == 0) {
@@ -164,16 +173,11 @@ function handleGetSegments(req, res) {
} }
return segments; return segments;
} catch (error) {
logger.error(error);
res.sendStatus(500);
return false;
}
} }
module.exports = { module.exports = {
handleGetSegments, handleGetSegments,
cleanGetSegments,
endpoint: function (req, res) { endpoint: function (req, res) {
let segments = handleGetSegments(req, res); let segments = handleGetSegments(req, res);

View File

@@ -1,125 +1,36 @@
const config = require('../config.js'); const hashPrefixTester = require('../utils/hashPrefixTester.js');
const { db, privateDB } = require('../databases/databases.js'); const getSegments = require('./getSkipSegments.js').cleanGetSegments;
const getHash = require('../utils/getHash.js'); const databases = require('../databases/databases.js');
const getIP = require('../utils/getIP.js'); const db = databases.db;
/**
* @typedef {Object} Segment
* @property {string} videoID YouTube video ID the segment is meant for
* @property {number[]} segment Tuple of start and end times in seconds
* @property {string} category Category of content to skip
* @property {string} UUID Unique identifier for the specific segment
*/
/**
* @typedef {Object} Row
* @property {string} videoID
* @property {number} startTime
* @property {number} endTime
* @property {number} votes
* @property {string} UUID
* @property {string} category
* @property {number} shadowHidden
*/
/**
* Input an array of database records and get only one back, weighed on votes.
* The logic is taken from getWeightedRandomChoice, just simplified input and output to not work on indices only.
*
* @param {Row[]} rows
* @returns {?Row}
*/
function pickWeightedRandomRow(rows) {
if (rows.length === 0) {
return null;
} else if (rows.length === 1) {
return rows[0];
}
const sqrtWeightsList = [];
let totalSqrtWeights = 0;
for (const row of rows) {
let sqrtVote = Math.sqrt((row.votes + 3) * 10);
sqrtWeightsList.push(sqrtVote);
totalSqrtWeights += sqrtVote;
}
const randomNumber = Math.random();
let currentVoteNumber = 0;
for (let i = 0; i < sqrtWeightsList.length; i++) {
if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[i]) / totalSqrtWeights) {
return rows[i];
}
currentVoteNumber += sqrtWeightsList[i];
}
}
/**
* @param {string} prefix Lowercased hexadecimal hash prefix
* @param {string} hashedIP Custom hash of the visitors IP address
* @returns {Object.<string, Segment[]>}
*/
function getSkipSegmentsByHash(prefix, hashedIP) {
/** @type Row[] */
const rows = db.prepare('SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY videoID, startTime')
.all(prefix + '%');
/** @type {string[]} */
const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID);
/** @type {Object.<string, Segment[][]>} */
const rowGroupsPerVideo = {};
let previousVideoID = null;
let previousEndTime = null;
for (const row of rows) {
/** @TODO check if this logic does what is expected. */
if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) {
// The current visitors IP did not submit for the current video.
// Do not send shadowHidden segments to them.
continue;
}
// Split up the rows per video and group overlapping segments together.
if (!(row.videoID in rowGroupsPerVideo)) {
rowGroupsPerVideo[row.videoID] = [];
}
if (previousVideoID === row.videoID && row.startTime <= previousEndTime) {
rowGroupsPerVideo[row.videoID][rowGroupsPerVideo[row.videoID].length - 1].push(row);
previousEndTime = Math.max(previousEndTime, row.endTime);
} else {
rowGroupsPerVideo[row.videoID].push([row]);
previousVideoID = row.videoID;
previousEndTime = row.endTime;
}
}
/** @type {Object.<string, Segment[]>} */
const output = {};
for (const videoID in rowGroupsPerVideo) {
const pickedVideosForVideoID = [];
for (const group of rowGroupsPerVideo[videoID]) {
pickedVideosForVideoID.push(pickWeightedRandomRow(group));
}
output[videoID] = pickedVideosForVideoID.map(row => ({ videoID: row.videoID, segment: [row.startTime, row.endTime], category: row.category, UUID: row.UUID }));
}
return output;
}
const minimumPrefix = config.minimumPrefix || '3';
const maximumPrefix = config.maximumPrefix || '32'; // Half the hash.
const prefixChecker = new RegExp('^[\\dA-F]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i');
module.exports = async function (req, res) { module.exports = async function (req, res) {
if (!prefixChecker.test(req.params.prefix)) { let hashPrefix = req.params.prefix;
res.sendStatus(400).end(); // Exit early on faulty prefix if (!hashPrefixTester(req.params.prefix)) {
res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
return;
} }
const segments = getSkipSegmentsByHash( const categories = req.query.categories
req.params.prefix.toLowerCase(), ? JSON.parse(req.query.categories)
getHash(getIP(req) + config.globalSalt) : req.query.category
); ? [req.query.category]
: ['sponsor'];
if (Object.keys(segments).length > 0) { // Get all video id's that match hash prefix
res.send(segments); const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
} else { if (videoIds.length === 0) {
res.sendStatus(404); // No skipable segments within this prefix res.sendStatus(404);
return;
} }
let segments = videoIds.map((video) => {
return {
videoID: video.videoID,
hash: video.hashedVideoID,
segments: getSegments(video.videoID, categories)
};
});
res.status(200).json(segments);
} }

View File

@@ -1,4 +1,3 @@
var fs = require('fs');
var config = require('../config.js'); var config = require('../config.js');
var getHash = require('../utils/getHash.js'); var getHash = require('../utils/getHash.js');
@@ -370,8 +369,6 @@ async function voteOnSponsorTime(req, res) {
} }
module.exports = { module.exports = {
voteOnSponsorTime, voteOnSponsorTime,
endpoint: function (req, res) { endpoint: voteOnSponsorTime
voteOnSponsorTime(req, res); };
},
};

View File

@@ -0,0 +1,11 @@
const config = require('../config.js');
const logger = require('./logger.js');
const minimumPrefix = config.minimumPrefix || '3';
const maximumPrefix = config.maximumPrefix || '32'; // Half the hash.
const prefixChecker = new RegExp('^[\\da-f]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i');
module.exports = (prefix) => {
return prefixChecker.test(prefix);
};

View File

@@ -0,0 +1,85 @@
var request = require('request');
var db = require('../../src/databases/databases.js').db;
var utils = require('../utils.js');
var getHash = require('../../src/utils/getHash.js');
/*
*CREATE TABLE IF NOT EXISTS "sponsorTimes" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"shadowHidden" INTEGER NOT NULL
);
*/
describe('getSegmentsByHash', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
db.exec(startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
db.exec(startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
});
it('Should be able to get a 200', (done) => {
request.get(utils.getbaseURL()
+ '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null,
(err, res, body) => {
if (err) done("Couldn't call endpoint");
else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
else {
done();
} // pass
});
});
it('Should be able to get a 200 with empty segments for video but no matching categories', (done) => {
request.get(utils.getbaseURL()
+ '/api/skipSegments/3272f?categories=["shilling"]', null,
(err, res, body) => {
if (err) done("Couldn't call endpoint");
else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
else {
if (JSON.parse(body) && JSON.parse(body).length > 0 && JSON.parse(body)[0].segments.length === 0) {
done(); // pass
} else {
done("response had segments");
}
}
});
});
it('Should be able to get a 404 if no videos', (done) => {
request.get(utils.getbaseURL()
+ '/api/skipSegments/11111?categories=["shilling"]', null,
(err, res, body) => {
if (err) done("Couldn't call endpoint");
else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode);
else {
done(); // pass
}
});
});
it('Should be able to get multiple videos', (done) => {
request.get(utils.getbaseURL()
+ '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null,
(err, res, body) => {
if (err) done("Couldn't call endpoint");
else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
else {
body = JSON.parse(body);
if (body.length !== 2) done("expected 2 video, got " + body.length);
else if (body[0].segments.length !== 2) done("expected 2 segments for first video, got " + body[0].segments.length);
else if (body[1].segments.length !== 1) done("expected 1 segment for second video, got " + body[1].segments.length);
else done();
}
});
});
});

View File

@@ -21,8 +21,8 @@ var getHash = require('../../src/utils/getHash.js');
describe('getVideoSponsorTime (Old get method)', () => { describe('getVideoSponsorTime (Old get method)', () => {
before(() => { before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 0) + "')"); db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 1) + "')");
db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 0) + "')"); db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 1) + "')");
}); });
it('Should be able to get a time', (done) => { it('Should be able to get a time', (done) => {

View File

@@ -6,22 +6,22 @@ const getHash = require('../../src/utils/getHash.js');
describe('voteOnSponsorTime', () => { describe('voteOnSponsorTime', () => {
before(() => { before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest', 1)+"')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest2', 1)+"')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest2', 1)+"')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest2', 1)+"')");
db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest3', 1)+"')");
db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest,test', 1)+"')");
db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-test3', 1)+"')");
db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-test3', 1)+"')");
db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')");
db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')");
db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter', 1)+"')");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')");
db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('own-submission-video', 1)+"')");
db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('not-own-submission-video', 1)+"')");
db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");