Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-14 07:27:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into testing

Browse Source 
# Conflicts:
#	src/routes/postSkipSegments.js
#	test/cases/voteOnSponsorTime.js
This commit is contained in:
Ajay Ramachandran
2020-09-15 11:56:14 -04:00
parent 7a83e7901a 4a865c4fb8
commit d3463bbef2
9 changed files with 307 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -100,3 +100,6 @@ test/databases/private.db
# Config files
config.json
# Mac files
.DS_Store

5
config.json.example
View File

@@ -8,6 +8,8 @@
"discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
"discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
"discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
"neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app
"discordNeuralBlockRejectWebhookURL": null, //URL from discord if you would like notifications when NeuralBlock rejects a submission [optional]
"userCounterURL": null, // For user counting. URL to instance of https://github.com/ajayyy/PrivacyUserCount
"proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash)
"behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For"
@@ -19,5 +21,6 @@
"privateDBSchema": "./databases/_private.db.sql",
"mode": "development",
"readOnly": false,
"webhooks": []
"webhooks": [],
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected
}

3
package.json
View File

@@ -19,7 +19,8 @@
"redis": "^3.0.2",
"sync-mysql": "^3.0.1",
"uuid": "^3.3.2",
"youtube-api": "^2.0.10"
"youtube-api": "^2.0.10",
"node-fetch": "^2.6.0"
},
"devDependencies": {
"mocha": "^7.1.1",

167
src/routes/postSkipSegments.js
View File

@@ -1,17 +1,18 @@
var config = require('../config.js');
const config = require('../config.js');
var databases = require('../databases/databases.js');
var db = databases.db;
var privateDB = databases.privateDB;
var YouTubeAPI = require('../utils/youtubeAPI.js');
var logger = require('../utils/logger.js');
var request = require('request');
var isoDurations = require('iso8601-duration');
const databases = require('../databases/databases.js');
const db = databases.db;
const privateDB = databases.privateDB;
const YouTubeAPI = require('../utils/youtubeAPI.js');
const logger = require('../utils/logger.js');
const request = require('request');
const isoDurations = require('iso8601-duration');
const fetch = require('node-fetch');
var getHash = require('../utils/getHash.js');
var getIP = require('../utils/getIP.js');
var getFormattedTime = require('../utils/getFormattedTime.js');
var isUserTrustworthy = require('../utils/isUserTrustworthy.js');
const getHash = require('../utils/getHash.js');
const getIP = require('../utils/getIP.js');
const getFormattedTime = require('../utils/getFormattedTime.js');
const isUserTrustworthy = require('../utils/isUserTrustworthy.js')
const { dispatchEvent } = require('../utils/webhookUtils.js');
function sendWebhookNotification(userID, videoID, UUID, submissionCount, youtubeData, {submissionStart, submissionEnd}, segmentInfo) {
@@ -94,35 +95,124 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) {
}
}
// submission: {videoID, startTime, endTime}
function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, probability, ytData) {
let submissionInfoRow = db.prepare('get', "SELECT " +
"(select count(1) from sponsorTimes where userID = ?) count, " +
"(select count(1) from sponsorTimes where userID = ? and votes <= -2) disregarded, " +
"coalesce((select userName FROM userNames WHERE userID = ?), ?) userName",
[userID, userID, userID, userID]);
let submittedBy = "";
// If a userName was created then show both
if (submissionInfoRow.userName !== userID){
submittedBy = submissionInfoRow.userName + "\n " + userID;
} else {
submittedBy = userID;
}
// Send discord message
if (config.discordNeuralBlockRejectWebhookURL === null) return;
request.post(config.discordNeuralBlockRejectWebhookURL, {
json: {
"embeds": [{
"title": ytData.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2),
"description": "**Submission ID:** " + UUID +
"\n**Timestamp:** " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
"\n**Predicted Probability:** " + probability +
"\n**Category:** " + category +
"\n**Submitted by:** "+ submittedBy +
"\n**Total User Submissions:** "+submissionInfoRow.count +
"\n**Ignored User Submissions:** "+submissionInfoRow.disregarded,
"color": 10813440,
"thumbnail": {
"url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
}
}]
}
}, (err, res) => {
if (err) {
logger.error("Failed to send NeuralBlock Discord hook.");
logger.error(JSON.stringify(err));
logger.error("\n");
} else if (res && res.statusCode >= 400) {
logger.error("Error sending NeuralBlock Discord hook");
logger.error(JSON.stringify(res));
logger.error("\n");
}
});
}
// callback: function(reject: "String containing reason the submission was rejected")
// returns: string when an error, false otherwise
// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
async function autoModerateSubmission(submission, callback) {
async function autoModerateSubmission(submission) {
// Get the video information from the youtube API
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve, reject) => {
YouTubeAPI.listVideos(submission.videoID, "contentDetails", (err, data) => resolve({err, data}));
YouTubeAPI.listVideos(submission.videoID, "contentDetails,snippet", (err, data) => resolve({err, data}));
});
if (err) {
return "Couldn't get video information.";
return false;
} else {
// Check to see if video exists
if (data.pageInfo.totalResults === 0) {
return "No video exists with id " + submission.videoID;
} else {
let segments = submission.segments;
let nbString = "";
for (let i = 0; i < segments.length; i++) {
let startTime = parseFloat(segments[i].segment[0]);
let endTime = parseFloat(segments[i].segment[1]);
let duration = data.items[0].contentDetails.duration;
duration = isoDurations.toSeconds(isoDurations.parse(duration));
if (duration == 0) {
// Allow submission if the duration is 0 (bug in youtube api)
return false;
} else if ((submission.endTime - submission.startTime) > (duration/100)*80) {
} else if ((endTime - startTime) > (duration/100)*80) {
// Reject submission if over 80% of the video
return "Sponsor segment is over 80% of the video.";
return "One of your submitted segments is over 80% of the video.";
} else {
if (segments[i].category === "sponsor") {
//Prepare timestamps to send to NB all at once
nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
}
}
}
// Check NeuralBlock
let neuralBlockURL = config.neuralBlockURL;
if (!neuralBlockURL) return false;
let response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
"&segments=" + nbString.substring(0,nbString.length-1));
if (!response.ok) return false;
let nbPredictions = await response.json();
nbDecision = false;
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
for (let i = 0; i < segments.length; i++){
if (segments[i].category === "sponsor"){
if (nbPredictions.probabilities[predictionIdx] < 0.70){
nbDecision = true; // At least one bad entry
startTime = parseFloat(segments[i].segment[0]);
endTime = parseFloat(segments[i].segment[1]);
let UUID = getHash("v2-categories" + submission.videoID + startTime +
endTime + segments[i].category + submission.userID, 1);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
}
predictionIdx++;
}
}
if (nbDecision){
return "Rejected based on NeuralBlock predictions.";
} else {
return false;
}
@@ -170,7 +260,7 @@ module.exports = async function postSkipSegments(req, res) {
//check if all correct inputs are here and the length is 1 second or more
if (videoID == undefined || userID == undefined || segments == undefined || segments.length < 1) {
//invalid request
res.sendStatus(400);
res.status(400).send("Parameters are not valid");
return;
}
@@ -181,11 +271,22 @@ module.exports = async function postSkipSegments(req, res) {
let hashedIP = getHash(getIP(req) + config.globalSalt);
let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
//check if this user is on the vip list
let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0;
let decreaseVotes = 0;
// Check if all submissions are correct
for (let i = 0; i < segments.length; i++) {
if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
//invalid request
res.sendStatus(400);
res.status(400).send("One of your segments are invalid");
return;
}
if (!config.categoryList.includes(segments[i].category)) {
res.status("400").send("Category doesn't exist.");
return;
}
@@ -207,7 +308,7 @@ module.exports = async function postSkipSegments(req, res) {
if (isNaN(startTime) || isNaN(endTime)
|| startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) {
//invalid request
res.sendStatus(400);
res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)");
return;
}
@@ -224,21 +325,27 @@ module.exports = async function postSkipSegments(req, res) {
res.sendStatus(409);
return;
}
}
let autoModerateResult = await autoModerateSubmission({videoID, startTime, endTime});
if (autoModerateResult) {
res.status(403).send("Request rejected by auto moderator: " + autoModerateResult);
// Auto moderator check
if (!isVIP) {
let autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions."){
// If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected.
// However, this behavior is consistent with other automod functions
// already in place.
//decreaseVotes = -2; //Disable for now
} else if (autoModerateResult) {
//Normal automod behavior
res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord.");
return;
}
}
// Will be filled when submitting
let UUIDs = [];
try {
//check if this user is on the vip list
let vipRow = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]);
//get current time
let timeSubmitted = Date.now();
@@ -280,8 +387,8 @@ module.exports = async function postSkipSegments(req, res) {
shadowBanned = 1;
}
let startingVotes = 0;
if (vipRow.userCount > 0) {
let startingVotes = 0 + decreaseVotes;
if (isVIP) {
//this user is a vip, start them at a higher approval rating
startingVotes = 10000;
}

5
src/routes/voteOnSponsorTime.js
View File

@@ -147,6 +147,11 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) {
return;
}
if (!config.categoryList.includes(category)) {
res.status("400").send("Category doesn't exist.");
return;
}
let timeSubmitted = Date.now();
let voteAmount = isVIP ? 500 : 1;

5
test.json
View File

@@ -7,6 +7,8 @@
"discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
"discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
"discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",
"discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
"neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
"behindProxy": true,
"db": "./test/databases/sponsorTimes.db",
"privateDB": "./test/databases/private.db",
@@ -46,5 +48,6 @@
"vote.down"
]
}
]
],
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"]
}

12
test/cases/postSkipSegments.js
View File

@@ -88,7 +88,7 @@ describe('postSkipSegments', () => {
done("Status code was " + res.statusCode);
}
});
});
}).timeout(5000);
it('Should be accepted if a non-sponsor is less than 1 second', (done) => {
request.post(utils.getbaseURL()
@@ -120,6 +120,16 @@ describe('postSkipSegments', () => {
});
});
it("Should be rejected if NB's predicted probability is <70%.", (done) => {
request.get(utils.getbaseURL()
+ "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null,
(err, res, body) => {
if (err) done("Couldn't call endpoint");
else if (res.statusCode === 200) done(); // pass
else done("non 200 status code: " + res.statusCode + " ("+body+")");
});
});
it('Should be allowed if youtube thinks duration is 0', (done) => {
request.get(utils.getbaseURL()
+ "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null,

44
test/cases/voteOnSponsorTime.js
View File

@@ -6,6 +6,7 @@ const getHash = require('../../src/utils/getHash.js');
describe('voteOnSponsorTime', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest2', 1) + "')");
db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '" + getHash('vote-testtesttest2', 1) + "')");
@@ -22,6 +23,8 @@ describe('voteOnSponsorTime', () => {
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '" + getHash('voter-submitter2', 1) + "')");
db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '" + getHash('own-submission-video', 1) + "')");
db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('not-own-submission-video', 1) + "')");
db.exec(startOfQuery + "('incorrect-category', 1, 11, 500, 'incorrect-category', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category', 1) + "')");
db.exec(startOfQuery + "('incorrect-category-change', 1, 11, 500, 'incorrect-category-change', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category-change', 1) + "')");
db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");
@@ -207,6 +210,24 @@ describe('voteOnSponsorTime', () => {
});
});
it('Should not able to change to an invalid category', (done) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=fakecategory", null,
(err, res, body) => {
if (err) done(err);
else if (res.statusCode === 400) {
let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category"]);
if (row.category === "sponsor") {
done()
} else {
done("Vote did not succeed. Submission went from sponsor to " + row.category);
}
} else {
done("Status code was " + res.statusCode);
}
});
});
it('Should be able to change your vote for a category and it should immediately change (for now)', (done) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro", null,
@@ -225,6 +246,29 @@ describe('voteOnSponsorTime', () => {
});
});
it('Should not be able to change your vote to an invalid category', (done) => {
const vote = (inputCat, assertCat, callback) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category-change&category="+inputCat, null,
(err) => {
if (err) done(err);
else{
let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category-change"]);
if (row.category === assertCat) {
callback();
} else {
done("Vote did not succeed. Submission went from sponsor to " + row.category);
}
}
});
};
vote("sponsor", "sponsor", () => {
vote("fakeCategory", "sponsor", done);
});
});
it('VIP should be able to vote for a category and it should immediately change', (done) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&category=outro", null,

27
test/mocks.js
View File

@@ -15,12 +15,37 @@ app.post('/CompletelyIncorrectReportWebhook', (req, res) => {
res.sendStatus(200);
});
// Testing NeuralBlock
app.post('/NeuralBlockRejectWebhook', (req, res) => {
res.sendStatus(200);
});
app.get('/NeuralBlock/api/checkSponsorSegments', (req, res) => {
if (req.query.vid === "LevkAjUE6d4") {
res.json({
probabilities: [0.69]
});
return;
}
res.sendStatus(500);
});
//getSponsorSegments is no longer being used for automod
app.get('/NeuralBlock/api/getSponsorSegments', (req, res) => {
if (req.query.vid === "LevkAjUE6d4") {
res.json({
sponsorSegments: [[0.47,7.549],[264.023,317.293]]
});
return;
}
res.sendStatus(500);
});
// Testing webhooks
app.post('/CustomWebhook', (req, res) => {
res.sendStatus(200);
});
module.exports = function createMockServer(callback) {
return app.listen(config.mockPort, callback);
}