Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 11:36:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

origin testing merge

Browse Source
This commit is contained in:
Joe Dowd
2020-08-30 22:24:37 +01:00
parent 48bb93e8d4 3167c24f75
commit b8d7eef536
9 changed files with 193 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
databases/_upgrade_sponsorTimes_3.sql Normal file
View File

@@ -0,0 +1,26 @@
BEGIN TRANSACTION;
/* Add hash field */
CREATE TABLE "sqlb_temp_table_1" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT "sponsor",
"hashedVideoID" TEXT NOT NULL,
"shadowHidden" INTEGER NOT NULL
);
INSERT INTO sqlb_temp_table_1 SELECT *, sha256(videoID) FROM sponsorTimes;
DROP TABLE sponsorTimes;
ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
/* Bump version in config */
UPDATE config SET value = 2 WHERE key = "version";
COMMIT;

4
src/app.js
View File

@@ -11,6 +11,7 @@ const userCounter = require('./middleware/userCounter.js');
// Routes
var getSkipSegments = require('./routes/getSkipSegments.js').endpoint;
var postSkipSegments = require('./routes/postSkipSegments.js');
var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js');
var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js');
var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js');
var setUsername = require('./routes/setUsername.js');
@@ -51,6 +52,9 @@ app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
app.get('/api/skipSegments', getSkipSegments);
app.post('/api/skipSegments', postSkipSegments);
// add the privacy protecting skip segments functions
app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash);
//voting endpoint
app.get('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);
app.post('/api/voteOnSponsorTime', voteOnSponsorTime.endpoint);

4
src/databases/databases.js
View File

@@ -60,12 +60,12 @@ if (config.mysql) {
let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql";
let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
while (fs.existsSync(path)) {
db.exec(fs.readFileSync(path).toString());
versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql";
path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
}
}
}

125
src/routes/getSkipSegmentsByHash.js Normal file
View File

@@ -0,0 +1,125 @@
const config = require('../config.js');
const { db, privateDB } = require('../databases/databases.js');
const getHash = require('../utils/getHash.js');
const getIP = require('../utils/getIP.js');
/**
* @typedef {Object} Segment
* @property {string} videoID YouTube video ID the segment is meant for
* @property {number[]} segment Tuple of start and end times in seconds
* @property {string} category Category of content to skip
* @property {string} UUID Unique identifier for the specific segment
*/
/**
* @typedef {Object} Row
* @property {string} videoID
* @property {number} startTime
* @property {number} endTime
* @property {number} votes
* @property {string} UUID
* @property {string} category
* @property {number} shadowHidden
*/
/**
* Input an array of database records and get only one back, weighed on votes.
* The logic is taken from getWeightedRandomChoice, just simplified input and output to not work on indices only.
*
* @param {Row[]} rows
* @returns {?Row}
*/
function pickWeightedRandomRow(rows) {
if (rows.length === 0) {
return null;
} else if (rows.length === 1) {
return rows[0];
}
const sqrtWeightsList = [];
let totalSqrtWeights = 0;
for (const row of rows) {
let sqrtVote = Math.sqrt((row.votes + 3) * 10);
sqrtWeightsList.push(sqrtVote);
totalSqrtWeights += sqrtVote;
}
const randomNumber = Math.random();
let currentVoteNumber = 0;
for (let i = 0; i < sqrtWeightsList.length; i++) {
if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[i]) / totalSqrtWeights) {
return rows[i];
}
currentVoteNumber += sqrtWeightsList[i];
}
}
/**
* @param {string} prefix Lowercased hexadecimal hash prefix
* @param {string} hashedIP Custom hash of the visitor’s IP address
* @returns {Object.<string, Segment[]>}
*/
function getSkipSegmentsByHash(prefix, hashedIP) {
/** @type Row[] */
const rows = db.prepare('SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY videoID, startTime')
.all(prefix + '%');
/** @type {string[]} */
const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID);
/** @type {Object.<string, Segment[][]>} */
const rowGroupsPerVideo = {};
let previousVideoID = null;
let previousEndTime = null;
for (const row of rows) {
/** @TODO check if this logic does what is expected. */
if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) {
// The current visitor’s IP did not submit for the current video.
// Do not send shadowHidden segments to them.
continue;
}
// Split up the rows per video and group overlapping segments together.
if (!(row.videoID in rowGroupsPerVideo)) {
rowGroupsPerVideo[row.videoID] = [];
}
if (previousVideoID === row.videoID && row.startTime <= previousEndTime) {
rowGroupsPerVideo[row.videoID][rowGroupsPerVideo[row.videoID].length - 1].push(row);
previousEndTime = Math.max(previousEndTime, row.endTime);
} else {
rowGroupsPerVideo[row.videoID].push([row]);
previousVideoID = row.videoID;
previousEndTime = row.endTime;
}
}
/** @type {Object.<string, Segment[]>} */
const output = {};
for (const videoID in rowGroupsPerVideo) {
const pickedVideosForVideoID = [];
for (const group of rowGroupsPerVideo[videoID]) {
pickedVideosForVideoID.push(pickWeightedRandomRow(group));
}
output[videoID] = pickedVideosForVideoID.map(row => ({ videoID: row.videoID, segment: [row.startTime, row.endTime], category: row.category, UUID: row.UUID }));
}
return output;
}
const minimumPrefix = config.minimumPrefix || '3';
const maximumPrefix = config.maximumPrefix || '32'; // Half the hash.
const prefixChecker = new RegExp('^[\\dA-F]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i');
module.exports = async function (req, res) {
if (!prefixChecker.test(req.params.prefix)) {
res.sendStatus(400).end(); // Exit early on faulty prefix
}
const segments = getSkipSegmentsByHash(
req.params.prefix.toLowerCase(),
getHash(getIP(req) + config.globalSalt)
);
if (Object.keys(segments).length > 0) {
res.send(segments);
} else {
res.sendStatus(404); // No skipable segments within this prefix
}
}

8
src/routes/postSkipSegments.js
View File

@@ -289,9 +289,11 @@ module.exports = async function postSkipSegments(req, res) {
try {
db.prepare('run', "INSERT INTO sponsorTimes " +
"(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [videoID, segmentInfo.segment[0],
segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned]);
"(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1)
]
);
//add to private db as well
privateDB.prepare('run', "INSERT INTO sponsorTimes VALUES(?, ?, ?)", [videoID, hashedIP, timeSubmitted]);

4
test/cases/getSavedTimeForUser.js
View File

@@ -5,8 +5,8 @@ var getHash = require('../../src/utils/getHash.js');
describe('getSavedTimeForUser', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)");
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0, '" + getHash('getSavedTimeForUser', 0) + "')");
});
it('Should be able to get a 200', (done) => {

17
test/cases/getSkipSegments.js
View File

@@ -1,6 +1,7 @@
var request = require('request');
var db = require('../../src/databases/databases.js').db;
var utils = require('../utils.js');
var getHash = require('../../src/utils/getHash.js');
/*
*CREATE TABLE IF NOT EXISTS "sponsorTimes" (
@@ -18,14 +19,14 @@ var utils = require('../utils.js');
describe('getSkipSegments', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)");
db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)");
db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)");
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 0) + "')");
db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 0) + "')");
db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 0) + "')");
db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')");
db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')");
db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')");
db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')");
});

7
test/cases/oldGetSponsorTime.js
View File

@@ -1,6 +1,7 @@
var request = require('request');
var db = require('../../src/databases/databases.js').db;
var utils = require('../utils.js');
var getHash = require('../../src/utils/getHash.js');
/*
@@ -19,9 +20,9 @@ var utils = require('../utils.js');
describe('getVideoSponsorTime (Old get method)', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)");
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 0) + "')");
db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 0) + "')");
});
it('Should be able to get a time', (done) => {

32
test/cases/voteOnSponsorTime.js
View File

@@ -6,22 +6,22 @@ const getHash = require('../../src/utils/getHash.js');
describe('voteOnSponsorTime', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0)");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0)");
db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)");
db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0)");
db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0)");
db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0)");
db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')");
db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");