Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-10 05:27:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added schema upgrade system and started on new vote type

Browse Source
This commit is contained in:
Ajay Ramachandran
2020-04-27 23:01:51 -04:00
parent 74169e6caf
commit d767f2ff6b
9 changed files with 216 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config.json.example
View File

@@ -7,10 +7,12 @@
"youtubeAPIKey": null, //get this from Google Cloud Platform [optional] "youtubeAPIKey": null, //get this from Google Cloud Platform [optional]
"discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
"discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
"discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
"behindProxy": true, "behindProxy": true,
"db": "./databases/sponsorTimes.db", "db": "./databases/sponsorTimes.db",
"privateDB": "./databases/private.db", "privateDB": "./databases/private.db",
"createDatabaseIfNotExist": true, //This will run on startup every time (unless readOnly is true) - so ensure "create table if not exists" is used in the schema "createDatabaseIfNotExist": true, //This will run on startup every time (unless readOnly is true) - so ensure "create table if not exists" is used in the schema
"schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql", "dbSchema": "./databases/_sponsorTimes.db.sql",
"privateDBSchema": "./databases/_private.db.sql", "privateDBSchema": "./databases/_private.db.sql",
"mode": "development", "mode": "development",

5
databases/_sponsorTimes.db.sql
View File

@@ -18,6 +18,11 @@ CREATE TABLE IF NOT EXISTS "userNames" (
"userID" TEXT NOT NULL, "userID" TEXT NOT NULL,
"userName" TEXT NOT NULL "userName" TEXT NOT NULL
); );
CREATE TABLE IF NOT EXISTS "version" (
"code" INTEGER NOT NULL default '0'
);
CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID); CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID);
CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID); CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID);
COMMIT; COMMIT;

25
databases/_upgrade_0.sql Normal file
View File

@@ -0,0 +1,25 @@
BEGIN TRANSACTION;
/* Add incorrectVotes field */
CREATE TABLE "sqlb_temp_table_1" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT "sponsor",
"shadowHidden" INTEGER NOT NULL
);
INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
DROP TABLE sponsorTimes;
ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
/* Increase version number */
INSERT INTO version VALUES(1);
COMMIT;

7
src/databases/databases.js
View File

@@ -15,6 +15,13 @@ if (config.createDatabaseIfNotExist && !config.readOnly) {
if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString()); if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString());
} }
// Upgrade database if required
let versionCode = db.prepare("SELECT code FROM version").get() || 0;
let path = config.schemaFolder + "/_upgrade_" + versionCode + ".sql";
if (fs.existsSync(path)) {
db.exec(fs.readFileSync(path).toString());
}
// Enable WAL mode checkpoint number // Enable WAL mode checkpoint number
if (!config.readOnly && config.mode === "production") { if (!config.readOnly && config.mode === "production") {
db.exec("PRAGMA journal_mode=WAL;"); db.exec("PRAGMA journal_mode=WAL;");

290
src/routes/voteOnSponsorTime.js
View File

@@ -11,114 +11,147 @@ var privateDB = databases.privateDB;
var YouTubeAPI = require('../utils/youtubeAPI.js'); var YouTubeAPI = require('../utils/youtubeAPI.js');
var request = require('request'); var request = require('request');
function completelyIncorrectVote(req, res, params) {
}
module.exports = async function voteOnSponsorTime(req, res) { module.exports = async function voteOnSponsorTime(req, res) {
let UUID = req.query.UUID; let UUID = req.query.UUID;
let userID = req.query.userID; let userID = req.query.userID;
let type = req.query.type; let type = req.query.type;
if (UUID == undefined || userID == undefined || type == undefined) { if (UUID == undefined || userID == undefined || type == undefined) {
//invalid request //invalid request
res.sendStatus(400); res.sendStatus(400);
return; return;
} }
//hash the userID //hash the userID
let nonAnonUserID = getHash(userID); let nonAnonUserID = getHash(userID);
userID = getHash(userID + UUID); userID = getHash(userID + UUID);
//x-forwarded-for if this server is behind a proxy //x-forwarded-for if this server is behind a proxy
let ip = getIP(req); let ip = getIP(req);
//hash the ip 5000 times so no one can get it from the database //hash the ip 5000 times so no one can get it from the database
let hashedIP = getHash(ip + config.globalSalt); let hashedIP = getHash(ip + config.globalSalt);
try { let voteTypes = {
//check if vote has already happened normal: 0,
let votesRow = privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID); incorrect: 1
}
//-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;
let incrementAmount = 0;
let oldIncrementAmount = 0;
if (type == 1) { try {
//upvote //check if vote has already happened
incrementAmount = 1; let votesRow = privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID);
} else if (type == 0) {
//downvote
incrementAmount = -1;
} else {
//unrecongnised type of vote
res.sendStatus(400);
return;
}
if (votesRow != undefined) {
if (votesRow.type == 1) {
//upvote
oldIncrementAmount = 1;
} else if (votesRow.type == 0) {
//downvote
oldIncrementAmount = -1;
} else if (votesRow.type == 2) {
//extra downvote
oldIncrementAmount = -4;
} else if (votesRow.type < 0) {
//vip downvote
oldIncrementAmount = votesRow.type;
}
}
//check if this user is on the vip list //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID); let incrementAmount = 0;
let oldIncrementAmount = 0;
//check if the increment amount should be multiplied (downvotes have more power if there have been many views) if (type == 1 || type == 11) {
let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID); //upvote
incrementAmount = 1;
} else if (type == 0 || type == 10) {
//downvote
incrementAmount = -1;
} else {
//unrecongnised type of vote
res.sendStatus(400);
return;
}
if (votesRow != undefined) {
if (votesRow.type == 1 || type == 11) {
//upvote
oldIncrementAmount = 1;
} else if (votesRow.type == 0 || type == 10) {
//downvote
oldIncrementAmount = -1;
} else if (votesRow.type == 2) {
//extra downvote
oldIncrementAmount = -4;
} else if (votesRow.type < 0) {
//vip downvote
oldIncrementAmount = votesRow.type;
} else if (votesRow.type == 12) {
// VIP downvote for completely incorrect
oldIncrementAmount = -500;
} else if (votesRow.type == 13) {
// VIP upvote for completely incorrect
oldIncrementAmount = 500;
}
}
if (vipRow.userCount != 0 && incrementAmount < 0) { //check if this user is on the vip list
//this user is a vip and a downvote let vipRow = db.prepare("SELECT count(*) as userCount FROM vipUsers WHERE userID = ?").get(nonAnonUserID);
incrementAmount = - (row.votes + 2 - oldIncrementAmount);
type = incrementAmount;
} else if (row !== undefined && (row.votes > 8 || row.views > 15) && incrementAmount < 0) {
//increase the power of this downvote
incrementAmount = -Math.abs(Math.min(10, row.votes + 2 - oldIncrementAmount));
type = incrementAmount;
}
// Send discord message //check if the increment amount should be multiplied (downvotes have more power if there have been many views)
if (type != 1) { let row = db.prepare("SELECT votes, views FROM sponsorTimes WHERE UUID = ?").get(UUID);
// Get video ID
let submissionInfoRow = db.prepare("SELECT videoID, userID, startTime, endTime FROM sponsorTimes WHERE UUID = ?").get(UUID);
let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID); if (voteTypeEnum == voteTypes.normal) {
if (vipRow.userCount != 0 && incrementAmount < 0) {
//this user is a vip and a downvote
incrementAmount = - (row.votes + 2 - oldIncrementAmount);
type = incrementAmount;
} else if (row !== undefined && (row.votes > 8 || row.views > 15) && incrementAmount < 0) {
//increase the power of this downvote
incrementAmount = -Math.abs(Math.min(10, row.votes + 2 - oldIncrementAmount));
type = incrementAmount;
}
} else if (voteTypeEnum == voteTypes.incorrect) {
if (vipRow.userCount != 0) {
//this user is a vip and a downvote
incrementAmount = 500 * incrementAmount;
type = incrementAmount < 0 ? 12 : 13;
}
}
if (config.youtubeAPIKey !== null && config.discordReportChannelWebhookURL !== null) { // Send discord message
YouTubeAPI.videos.list({ if (incrementAmount < 0) {
part: "snippet", // Get video ID
id: submissionInfoRow.videoID let submissionInfoRow = db.prepare("SELECT videoID, userID, startTime, endTime FROM sponsorTimes WHERE UUID = ?").get(UUID);
}, function (err, data) {
if (err || data.items.length === 0) {
err && console.log(err);
return;
}
request.post(config.discordReportChannelWebhookURL, { let userSubmissionCountRow = db.prepare("SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?").get(nonAnonUserID);
json: {
"embeds": [{ let webhookURL = null;
"title": data.items[0].snippet.title, if (voteTypeEnum === voteTypes.normal) {
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + webhookURL = config.discordReportChannelWebhookURL;
"&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), } else if (voteTypeEnum === voteTypes.incorrect) {
"description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views + webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
" Views**\n\nSubmission ID: " + UUID + }
"\n\nSubmitted by: " + submissionInfoRow.userID + "\n\nTimestamp: " +
getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), if (config.youtubeAPIKey !== null && webhookURL !== null) {
"color": 10813440, YouTubeAPI.videos.list({
"author": { part: "snippet",
"name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "") id: submissionInfoRow.videoID
}, }, function (err, data) {
"thumbnail": { if (err || data.items.length === 0) {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", err && console.log(err);
} return;
}] }
}
request.post(webhookURL, {
json: {
"embeds": [{
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID +
"&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
"description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views +
" Views**\n\nSubmission ID: " + UUID +
"\n\nSubmitted by: " + submissionInfoRow.userID + "\n\nTimestamp: " +
getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
"color": 10813440,
"author": {
"name": userSubmissionCountRow.submissionCount === 0 ? "Report by New User" : (vipRow.userCount !== 0 ? "Report by VIP User" : "")
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
}
}]
}
}, (err, res) => { }, (err, res) => {
if (err) { if (err) {
console.log("Failed to send reported submission Discord hook."); console.log("Failed to send reported submission Discord hook.");
@@ -129,43 +162,50 @@ module.exports = async function voteOnSponsorTime(req, res) {
console.log(JSON.stringify(res)); console.log(JSON.stringify(res));
console.log("\n"); console.log("\n");
} }
}); });
}); });
} }
} }
//update the votes table //update the votes table
if (votesRow != undefined) { if (votesRow != undefined) {
privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID); privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID);
} else { } else {
privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type); privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type);
} }
//update the vote count on this sponsorTime let tableName = "";
//oldIncrementAmount will be zero is row is null if (voteTypeEnum === voteTypes.normal) {
db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID); tableName = "votes";
} else if (voteTypeEnum === voteTypes.incorrect) {
tableName = "incorrectVotes";
}
//for each positive vote, see if a hidden submission can be shown again //update the vote count on this sponsorTime
if (incrementAmount > 0) { //oldIncrementAmount will be zero is row is null
//find the UUID that submitted the submission that was voted on db.prepare("UPDATE sponsorTimes SET " + tableName + " += ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID);
let submissionUserID = db.prepare("SELECT userID FROM sponsorTimes WHERE UUID = ?").get(UUID).userID;
//check if any submissions are hidden //for each positive vote, see if a hidden submission can be shown again
let hiddenSubmissionsRow = db.prepare("SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0").get(submissionUserID); if (incrementAmount > 0) {
//find the UUID that submitted the submission that was voted on
let submissionUserID = db.prepare("SELECT userID FROM sponsorTimes WHERE UUID = ?").get(UUID).userID;
if (hiddenSubmissionsRow.hiddenSubmissions > 0) { //check if any submissions are hidden
//see if some of this users submissions should be visible again let hiddenSubmissionsRow = db.prepare("SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0").get(submissionUserID);
if (await isUserTrustworthy(submissionUserID)) { if (hiddenSubmissionsRow.hiddenSubmissions > 0) {
//they are trustworthy again, show 2 of their submissions again, if there are two to show //see if some of this users submissions should be visible again
db.prepare("UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)").run(submissionUserID)
}
}
}
//added to db if (await isUserTrustworthy(submissionUserID)) {
res.sendStatus(200); //they are trustworthy again, show 2 of their submissions again, if there are two to show
} catch (err) { db.prepare("UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)").run(submissionUserID)
console.error(err); }
} }
}
//added to db
res.sendStatus(200);
} catch (err) {
console.error(err);
}
} }

6
test.json
View File

@@ -6,12 +6,14 @@
"youtubeAPIKey": "", "youtubeAPIKey": "",
"discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
"discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
"discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",
"behindProxy": true, "behindProxy": true,
"db": "./test/databases/sponsorTimes.db", "db": "./test/databases/sponsorTimes.db",
"privateDB": "./test/databases/private.db", "privateDB": "./test/databases/private.db",
"createDatabaseIfNotExist": true, "createDatabaseIfNotExist": true,
"dbSchema": "./test/databases/_sponsorTimes.db.sql", "schemaFolder": "./databases",
"privateDBSchema": "./test/databases/_private.db.sql", "dbSchema": "./databases/_sponsorTimes.db.sql",
"privateDBSchema": "./databases/_private.db.sql",
"mode": "test", "mode": "test",
"readOnly": false "readOnly": false
} }

18
test/databases/_private.db.sql
View File

@@ -1,18 +0,0 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
"userID" TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "votes" (
"UUID" TEXT NOT NULL,
"userID" INTEGER NOT NULL,
"hashedIP" INTEGER NOT NULL,
"type" INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS "sponsorTimes" (
"videoID" TEXT NOT NULL,
"hashedIP" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS sponsorTimes_hashedIP on sponsorTimes(hashedIP);
CREATE INDEX IF NOT EXISTS votes_userID on votes(UUID);
COMMIT;

23
test/databases/_sponsorTimes.db.sql
View File

@@ -1,23 +0,0 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "vipUsers" (
"userID" TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "sponsorTimes" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL,
"shadowHidden" INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS "userNames" (
"userID" TEXT NOT NULL,
"userName" TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID);
CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID);
COMMIT;

4
test/mocks.js
View File

@@ -11,6 +11,10 @@ app.post('/FirstTimeSubmissionsWebhook', (req, res) => {
res.sendStatus(200); res.sendStatus(200);
}); });
app.post('/CompletelyIncorrectReportWebhook', (req, res) => {
res.sendStatus(200);
});
module.exports = function createMockServer(callback) { module.exports = function createMockServer(callback) {
return app.listen(config.mockPort, callback); return app.listen(config.mockPort, callback);
} }