Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2026-03-26 07:47:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/getuserinfo

Browse Source
This commit is contained in:
Nanobyte
2020-09-15 19:08:56 +02:00
committed by GitHub
parent 43db13ab3f 1fe68c66c5
commit b52a862f4f
32 changed files with 1427 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/routes/getIsUserVIP.js Normal file
View File

@@ -0,0 +1,31 @@
var db = require('../databases/databases.js').db;
var getHash = require('../utils/getHash.js');
const logger = require('../utils/logger.js');
const isUserVIP = require('../utils/isUserVIP.js');
module.exports = (req, res) => {
let userID = req.query.userID;
if (userID == undefined) {
//invalid request
res.sendStatus(400);
return;
}
//hash the userID
userID = getHash(userID);
try {
let vipState = isUserVIP(userID);
res.status(200).json({
hashedUserID: userID,
vip: vipState
});
} catch (err) {
logger.error(err);
res.sendStatus(500);
return;
}
}

269
src/routes/getSkipSegments.js
View File

@@ -1,4 +1,3 @@
var fs = require('fs');
var config = require('../config.js');
var databases = require('../databases/databases.js');
@@ -9,45 +8,101 @@ var logger = require('../utils/logger.js');
var getHash = require('../utils/getHash.js');
var getIP = require('../utils/getIP.js');
function cleanGetSegments(req, videoID, categories) {
let userHashedIP, shadowHiddenSegments;
let segments = [];
try {
for (const category of categories) {
const categorySegments = db
.prepare(
'all',
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
[videoID, category]
)
.filter(segment => {
if (segment.votes < -1) {
return false; //too untrustworthy, just ignore it
}
//check if shadowHidden
//this means it is hidden to everyone but the original ip that submitted it
if (segment.shadowHidden != 1) {
return true;
}
if (shadowHiddenSegments === undefined) {
shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
}
//if this isn't their ip, don't send it to them
return shadowHiddenSegments.some(shadowHiddenSegment => {
if (userHashedIP === undefined) {
//hash the IP only if it's strictly necessary
userHashedIP = getHash(getIP(req) + config.globalSalt);
}
return shadowHiddenSegment.hashedIP === userHashedIP;
});
});
chooseSegments(categorySegments).forEach(chosenSegment => {
segments.push({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
});
});
}
return segments;
} catch (err) {
if (err) {
logger.error(err);
return null;
}
}
}
//gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique
function getWeightedRandomChoice(choices, amountOfChoices) {
//trivial case: no need to go through the whole process
if (amountOfChoices >= choices.length) {
return choices;
}
//assign a weight to each choice
let totalWeight = 0;
choices = choices.map(choice => {
//The 3 makes -2 the minimum votes before being ignored completely
//https://www.desmos.com/calculator/c1duhfrmts
//this can be changed if this system increases in popularity.
const weight = Math.exp((choice.votes + 3), 0.85);
totalWeight += weight;
return { ...choice, weight };
});
//iterate and find amountOfChoices choices
const chosen = [];
while (amountOfChoices-- > 0) {
//weighted random draw of one element of choices
const randomNumber = Math.random() * totalWeight;
let stackWeight = choices[0].weight;
let i = 0;
while (stackWeight < randomNumber) {
stackWeight += choices[++i].weight;
//trivial case: no need to go through the whole process
if (amountOfChoices >= choices.length) {
return choices;
}
//add it to the chosen ones and remove it from the choices before the next iteration
chosen.push(choices[i]);
totalWeight -= choices[i].weight;
choices.splice(i, 1);
}
//assign a weight to each choice
let totalWeight = 0;
choices = choices.map(choice => {
//The 3 makes -2 the minimum votes before being ignored completely
//https://www.desmos.com/calculator/c1duhfrmts
//this can be changed if this system increases in popularity.
const weight = Math.exp((choice.votes + 3), 0.85);
totalWeight += weight;
return chosen;
return { ...choice, weight };
});
//iterate and find amountOfChoices choices
const chosen = [];
while (amountOfChoices-- > 0) {
//weighted random draw of one element of choices
const randomNumber = Math.random() * totalWeight;
let stackWeight = choices[0].weight;
let i = 0;
while (stackWeight < randomNumber) {
stackWeight += choices[++i].weight;
}
//add it to the chosen ones and remove it from the choices before the next iteration
chosen.push(choices[i]);
totalWeight -= choices[i].weight;
choices.splice(i, 1);
}
return chosen;
}
//This function will find segments that are contained inside of eachother, called similar segments
@@ -55,34 +110,34 @@ function getWeightedRandomChoice(choices, amountOfChoices) {
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
//Segments with less than -1 votes are already ignored before this function is called
function chooseSegments(segments) {
//Create groups of segments that are similar to eachother
//Segments must be sorted by their startTime so that we can build groups chronologically:
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
//2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
// inside that group (because they're sorted) so we can create a new one
const similarSegmentsGroups = [];
let currentGroup;
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
if (segment.startTime > cursor) {
currentGroup = { segments: [], votes: 0 };
similarSegmentsGroups.push(currentGroup);
}
//Create groups of segments that are similar to eachother
//Segments must be sorted by their startTime so that we can build groups chronologically:
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
//2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
// inside that group (because they're sorted) so we can create a new one
const similarSegmentsGroups = [];
let currentGroup;
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
if (segment.startTime > cursor) {
currentGroup = { segments: [], votes: 0 };
similarSegmentsGroups.push(currentGroup);
}
currentGroup.segments.push(segment);
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
if (segment.votes > 0) {
currentGroup.votes += segment.votes;
}
currentGroup.segments.push(segment);
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
if (segment.votes > 0) {
currentGroup.votes += segment.votes;
}
cursor = Math.max(cursor, segment.endTime);
});
cursor = Math.max(cursor, segment.endTime);
});
//if there are too many groups, find the best 8
return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
//randomly choose 1 good segment per group and return them
group => getWeightedRandomChoice(group.segments, 1)[0]
);
//if there are too many groups, find the best 8
return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
//randomly choose 1 good segment per group and return them
group => getWeightedRandomChoice(group.segments, 1)[0]
);
}
/**
@@ -96,91 +151,39 @@ function chooseSegments(segments) {
* @returns
*/
function handleGetSegments(req, res) {
const videoID = req.query.videoID;
// Default to sponsor
// If using params instead of JSON, only one category can be pulled
const categories = req.query.categories
? JSON.parse(req.query.categories)
: req.query.category
? [req.query.category]
: ['sponsor'];
const videoID = req.query.videoID;
// Default to sponsor
// If using params instead of JSON, only one category can be pulled
const categories = req.query.categories
? JSON.parse(req.query.categories)
: req.query.category
? [req.query.category]
: ['sponsor'];
/**
* @type {Array<{
* segment: number[],
* category: string,
* UUID: string
* }>
* }
*/
const segments = [];
let segments = cleanGetSegments(req, videoID, categories);
let userHashedIP, shadowHiddenSegments;
try {
for (const category of categories) {
const categorySegments = db
.prepare(
'all',
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
[videoID, category]
)
.filter(segment => {
if (segment.votes < -1) {
return false; //too untrustworthy, just ignore it
}
//check if shadowHidden
//this means it is hidden to everyone but the original ip that submitted it
if (segment.shadowHidden != 1) {
return true;
}
if (shadowHiddenSegments === undefined) {
shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
}
//if this isn't their ip, don't send it to them
return shadowHiddenSegments.some(shadowHiddenSegment => {
if (userHashedIP === undefined) {
//hash the IP only if it's strictly necessary
userHashedIP = getHash(getIP(req) + config.globalSalt);
}
return shadowHiddenSegment.hashedIP === userHashedIP;
});
});
chooseSegments(categorySegments).forEach(chosenSegment => {
segments.push({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
});
});
if (segments === null || segments === undefined) {
res.sendStatus(500);
return false;
}
if (segments.length == 0) {
res.sendStatus(404);
return false;
res.sendStatus(404);
return false;
}
return segments;
} catch (error) {
logger.error(error);
res.sendStatus(500);
return false;
}
}
module.exports = {
handleGetSegments,
endpoint: function (req, res) {
let segments = handleGetSegments(req, res);
handleGetSegments,
cleanGetSegments,
endpoint: function (req, res) {
let segments = handleGetSegments(req, res);
if (segments) {
//send result
res.send(segments);
}
},
if (segments) {
//send result
res.send(segments);
}
},
};

37
src/routes/getSkipSegmentsByHash.js Normal file
View File

@@ -0,0 +1,37 @@
const hashPrefixTester = require('../utils/hashPrefixTester.js');
const getSegments = require('./getSkipSegments.js').cleanGetSegments;
const databases = require('../databases/databases.js');
const logger = require('../utils/logger.js');
const db = databases.db;
module.exports = async function (req, res) {
let hashPrefix = req.params.prefix;
if (!hashPrefixTester(req.params.prefix)) {
res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
return;
}
const categories = req.query.categories
? JSON.parse(req.query.categories)
: req.query.category
? [req.query.category]
: ['sponsor'];
// Get all video id's that match hash prefix
const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
if (videoIds.length === 0) {
res.sendStatus(404);
return;
}
let segments = videoIds.map((video) => {
return {
videoID: video.videoID,
hash: video.hashedVideoID,
segments: getSegments(req, video.videoID, categories)
};
});
res.status(200).json(segments);
}

74
src/routes/postNoSegments.js Normal file
View File

@@ -0,0 +1,74 @@
const db = require('../databases/databases.js').db;
const getHash = require('../utils/getHash.js');
const isUserVIP = require('../utils/isUserVIP.js');
const logger = require('../utils/logger.js');
module.exports = (req, res) => {
// Collect user input data
let videoID = req.body.videoID;
let userID = req.body.userID;
let categories = req.body.categories;
// Check input data is valid
if (!videoID
|| !userID
|| !categories
|| !Array.isArray(categories)
|| categories.length === 0
) {
res.status(400).json({
message: 'Bad Format'
});
return;
}
// Check if user is VIP
userID = getHash(userID);
let userIsVIP = isUserVIP(userID);
if (!userIsVIP) {
res.status(403).json({
message: 'Must be a VIP to mark videos.'
});
return;
}
// Get existing no segment markers
let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]);
if (!noSegmentList || noSegmentList.length === 0) {
noSegmentList = [];
} else {
noSegmentList = noSegmentList.map((obj) => {
return obj.category;
});
}
// get user categories not already submitted that match accepted format
let categoriesToMark = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/);
}).filter((category) => {
return noSegmentList.indexOf(category) === -1;
});
// remove any duplicates
categoriesToMark = categoriesToMark.filter((category, index) => {
return categoriesToMark.indexOf(category) === index;
});
// create database entry
categoriesToMark.forEach((category) => {
try {
db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]);
} catch (err) {
logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'");
logger.error(err);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database."
});
}
});
res.status(200).json({
submitted: categoriesToMark
});
};

41
src/routes/postSkipSegments.js
View File

@@ -49,10 +49,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) {
if (config.youtubeAPIKey !== null) {
let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]);
YouTubeAPI.videos.list({
part: "snippet",
id: videoID
}, function (err, data) {
YouTubeAPI.listVideos(videoID, "snippet", (err, data) => {
if (err || data.items.length === 0) {
err && logger.error(err);
return;
@@ -156,10 +153,7 @@ async function autoModerateSubmission(submission) {
// Get the video information from the youtube API
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve, reject) => {
YouTubeAPI.videos.list({
part: "contentDetails,snippet",
id: submission.videoID
}, (err, data) => resolve({err, data}));
YouTubeAPI.listVideos(submission.videoID, "contentDetails,snippet", (err, data) => resolve({err, data}));
});
if (err) {
@@ -224,7 +218,6 @@ async function autoModerateSubmission(submission) {
}
}
}
} else {
logger.debug("Skipped YouTube API");
@@ -238,9 +231,9 @@ function proxySubmission(req) {
request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => {
if (config.mode === 'development') {
if (!err) {
logger.error('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
logger.debug('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
} else {
logger.debug("Proxy Submission: Failed to make call");
logger.error("Proxy Submission: Failed to make call");
}
}
});
@@ -277,6 +270,8 @@ module.exports = async function postSkipSegments(req, res) {
//hash the ip 5000 times so no one can get it from the database
let hashedIP = getHash(getIP(req) + config.globalSalt);
let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
//check if this user is on the vip list
let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0;
@@ -295,6 +290,18 @@ module.exports = async function postSkipSegments(req, res) {
return;
}
// Reject segemnt if it's in the no segments list
if (noSegmentList.indexOf(segments[i].category) !== -1) {
// TODO: Do something about the fradulent submission
logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
res.status(403).send(
"Request rejected by auto moderator: This video has been reported as not containing any segments with the category '"
+ segments[i].category + "'. If you believe this is incorrect, contact someone on Discord."
);
return;
}
let startTime = parseFloat(segments[i].segment[0]);
let endTime = parseFloat(segments[i].segment[1]);
@@ -394,11 +401,13 @@ module.exports = async function postSkipSegments(req, res) {
segmentInfo.segment[1] + segmentInfo.category + userID, 1);
try {
db.prepare('run', "INSERT INTO sponsorTimes " +
"(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [videoID, segmentInfo.segment[0],
segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned]);
db.prepare('run', "INSERT INTO sponsorTimes " +
"(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1)
]
);
//add to private db as well
privateDB.prepare('run', "INSERT INTO sponsorTimes VALUES(?, ?, ?)", [videoID, hashedIP, timeSubmitted]);
} catch (err) {

16
src/routes/voteOnSponsorTime.js
View File

@@ -1,11 +1,10 @@
var fs = require('fs');
var config = require('../config.js');
var getHash = require('../utils/getHash.js');
var getIP = require('../utils/getIP.js');
var getFormattedTime = require('../utils/getFormattedTime.js');
var isUserTrustworthy = require('../utils/isUserTrustworthy.js');
const {getVoteAuthor, getVoteAuthorRaw, dispatchEvent} = require('../utils/webhookUtils.js');
const { getVoteAuthor, getVoteAuthorRaw, dispatchEvent } = require('../utils/webhookUtils.js');
var databases = require('../databases/databases.js');
var db = databases.db;
@@ -49,10 +48,7 @@ function sendWebhooks(voteData) {
}
if (config.youtubeAPIKey !== null) {
YouTubeAPI.videos.list({
part: "snippet",
id: submissionInfoRow.videoID
}, function (err, data) {
YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => {
if (err || data.items.length === 0) {
err && logger.error(err);
return;
@@ -375,8 +371,6 @@ async function voteOnSponsorTime(req, res) {
}
module.exports = {
voteOnSponsorTime,
endpoint: function (req, res) {
voteOnSponsorTime(req, res);
},
};
voteOnSponsorTime,
endpoint: voteOnSponsorTime
};