mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2025-12-06 11:36:58 +03:00
Rework of getSkipSegments
This commit is contained in:
Pierre Donias
@@ -8,271 +8,179 @@ var privateDB = databases.privateDB;
|
||||
var getHash = require('../utils/getHash.js');
|
||||
var getIP = require('../utils/getIP.js');
|
||||
|
||||
|
||||
//gets the getWeightedRandomChoice for each group in an array of groups
|
||||
function getWeightedRandomChoiceForArray(choiceGroups, weights) {
|
||||
let finalChoices = [];
|
||||
//the indexes either chosen to be added to final indexes or chosen not to be added
|
||||
let choicesDealtWith = [];
|
||||
//for each choice group, what are the sums of the weights
|
||||
let weightSums = [];
|
||||
|
||||
for (let i = 0; i < choiceGroups.length; i++) {
|
||||
//find weight sums for this group
|
||||
weightSums.push(0);
|
||||
for (let j = 0; j < choiceGroups[i].length; j++) {
|
||||
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
|
||||
if (weights[choiceGroups[i][j]] > 0) {
|
||||
weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]];
|
||||
}
|
||||
}
|
||||
|
||||
//create a random choice for this group
|
||||
let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1)
|
||||
finalChoices.push(randomChoice.finalChoices);
|
||||
|
||||
for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) {
|
||||
choicesDealtWith.push(randomChoice.choicesDealtWith[j])
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
finalChoices: finalChoices,
|
||||
choicesDealtWith: choicesDealtWith,
|
||||
weightSums: weightSums
|
||||
};
|
||||
}
|
||||
|
||||
//gets a weighted random choice from the indexes array based on the weights.
|
||||
//amountOfChoices speicifies the amount of choices to return, 1 or more.
|
||||
//gets a weighted random choice from the choices array based on their `votes` property.
|
||||
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
|
||||
//choices are unique
|
||||
function getWeightedRandomChoice(choices, weights, amountOfChoices) {
|
||||
if (amountOfChoices > choices.length) {
|
||||
//not possible, since all choices must be unique
|
||||
return null;
|
||||
function getWeightedRandomChoice(choices, amountOfChoices) {
|
||||
//trivial case: no need to go through the whole process
|
||||
if (amountOfChoices >= choices.length) {
|
||||
return choices;
|
||||
}
|
||||
|
||||
let finalChoices = [];
|
||||
let choicesDealtWith = [];
|
||||
//assign a weight to each choice
|
||||
let totalWeight = 0;
|
||||
choices = choices.map(choice => {
|
||||
//multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
|
||||
//The 3 makes -2 the minimum votes before being ignored completely
|
||||
//https://www.desmos.com/calculator/ljftxolg9j
|
||||
//this can be changed if this system increases in popularity.
|
||||
const weight = Math.sqrt((choice.votes + 3) * 10);
|
||||
totalWeight += weight;
|
||||
|
||||
let sqrtWeightsList = [];
|
||||
//the total of all the weights run through the cutom sqrt function
|
||||
let totalSqrtWeights = 0;
|
||||
for (let j = 0; j < choices.length; j++) {
|
||||
//multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
|
||||
//The 3 makes -2 the minimum votes before being ignored completely
|
||||
//https://www.desmos.com/calculator/ljftxolg9j
|
||||
//this can be changed if this system increases in popularity.
|
||||
let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10);
|
||||
sqrtWeightsList.push(sqrtVote)
|
||||
totalSqrtWeights += sqrtVote;
|
||||
|
||||
//this index has now been deat with
|
||||
choicesDealtWith.push(choices[j]);
|
||||
}
|
||||
return { ...choice, weight };
|
||||
});
|
||||
|
||||
//iterate and find amountOfChoices choices
|
||||
let randomNumber = Math.random();
|
||||
|
||||
//this array will keep adding to this variable each time one sqrt vote has been dealt with
|
||||
//this is the sum of all the sqrtVotes under this index
|
||||
let currentVoteNumber = 0;
|
||||
for (let j = 0; j < sqrtWeightsList.length; j++) {
|
||||
if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) {
|
||||
//this one was randomly generated
|
||||
finalChoices.push(choices[j]);
|
||||
//remove that from original array, for next recursion pass if it happens
|
||||
choices.splice(j, 1);
|
||||
break;
|
||||
}
|
||||
const chosen = [];
|
||||
while (amountOfChoices-- > 0) {
|
||||
//weighted random draw of one element of choices
|
||||
const randomNumber = Math.random() * totalWeight;
|
||||
let stackWeight = choices[0].weight;
|
||||
let i = 0;
|
||||
while (stackWeight < randomNumber) {
|
||||
stackWeight += choices[++i].weight;
|
||||
}
|
||||
|
||||
//add on to the count
|
||||
currentVoteNumber += sqrtWeightsList[j];
|
||||
}
|
||||
|
||||
//add on the other choices as well using recursion
|
||||
if (amountOfChoices > 1) {
|
||||
let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices;
|
||||
//add all these choices to the finalChoices array being returned
|
||||
for (let i = 0; i < otherChoices.length; i++) {
|
||||
finalChoices.push(otherChoices[i]);
|
||||
}
|
||||
//add it to the chosen ones and remove it from the choices before the next iteration
|
||||
chosen.push(choices[i]);
|
||||
choices.splice(i, 1);
|
||||
}
|
||||
|
||||
return {
|
||||
finalChoices: finalChoices,
|
||||
choicesDealtWith: choicesDealtWith
|
||||
};
|
||||
return chosen;
|
||||
}
|
||||
|
||||
|
||||
//This function will find sponsor times that are contained inside of eachother, called similar sponsor times
|
||||
//This function will find segments that are contained inside of eachother, called similar segments
|
||||
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
|
||||
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
|
||||
//Sponsor times with less than -1 votes are already ignored before this function is called
|
||||
function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
|
||||
//create groups of sponsor times that are similar to eachother
|
||||
const groups = [];
|
||||
sponsorTimes.forEach(([startTime, endTime], i) => {
|
||||
//find a group that overlaps with the current segment
|
||||
//sponsorTimes are sorted by their startTime so there should never be more than 1 similar group
|
||||
const similarGroup = groups.find(group => group.start < endTime && startTime < group.end);
|
||||
|
||||
//add the sponsor to that group or create a new group if there aren't any
|
||||
if (similarGroup === undefined) {
|
||||
groups.push({ start: startTime, end: endTime, sponsors: [i] });
|
||||
} else {
|
||||
similarGroup.sponsors.push(i)
|
||||
similarGroup.start = Math.min(similarGroup.start, startTime);
|
||||
similarGroup.end = Math.max(similarGroup.end, endTime);
|
||||
}
|
||||
})
|
||||
//Segments with less than -1 votes are already ignored before this function is called
|
||||
function chooseSegments(segments) {
|
||||
//Create groups of segments that are similar to eachother
|
||||
//Segments must be sorted by their startTime so that we can build groups chronologically:
|
||||
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
|
||||
//2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
|
||||
// inside that group (because they're sorted) so we can create a new one
|
||||
const similarSegmentsGroups = [];
|
||||
let currentGroup;
|
||||
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
|
||||
segments.forEach(segment => {
|
||||
if (segment.startTime > cursor) {
|
||||
currentGroup = { segments: [], votes: 0 };
|
||||
similarSegmentsGroups.push(currentGroup);
|
||||
}
|
||||
|
||||
//once all the groups have been created, get rid of the metadata and remove single-sponsor groups
|
||||
const similarSponsorsGroups = groups.map(group => group.sponsors).filter(group => group.length > 1);
|
||||
currentGroup.segments.push(segment);
|
||||
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
|
||||
if (segment.votes > 0) {
|
||||
currentGroup.votes += segment.votes;
|
||||
}
|
||||
|
||||
let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes);
|
||||
cursor = Math.max(cursor, segment.endTime);
|
||||
});
|
||||
|
||||
let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices;
|
||||
//the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added
|
||||
let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith;
|
||||
|
||||
let voteSums = weightedRandomIndexes.weightSums;
|
||||
//convert these into the votes
|
||||
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
|
||||
//it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor.
|
||||
votes[finalSponsorTimeIndexes[i]] = voteSums[i];
|
||||
}
|
||||
|
||||
//find the indexes never dealt with and add them
|
||||
for (let i = 0; i < sponsorTimes.length; i++) {
|
||||
if (!finalSponsorTimeIndexesDealtWith.includes(i)) {
|
||||
finalSponsorTimeIndexes.push(i)
|
||||
}
|
||||
}
|
||||
|
||||
//if there are too many indexes, find the best 4
|
||||
if (finalSponsorTimeIndexes.length > 8) {
|
||||
finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices;
|
||||
}
|
||||
|
||||
//convert this to a final array to return
|
||||
let finalSponsorTimes = [];
|
||||
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
|
||||
finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]);
|
||||
}
|
||||
|
||||
//convert this to a final array of UUIDs as well
|
||||
let finalUUIDs = [];
|
||||
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
|
||||
finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]);
|
||||
}
|
||||
|
||||
return {
|
||||
sponsorTimes: finalSponsorTimes,
|
||||
UUIDs: finalUUIDs
|
||||
};
|
||||
//if there are too many groups, find the best 8
|
||||
return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
|
||||
//randomly choose 1 good segment per group and return them
|
||||
group => getWeightedRandomChoice(group.segments, 1)[0]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
* Returns what would be sent to the client.
|
||||
* Will resond with errors if required. Returns false if it errors.
|
||||
*
|
||||
* @param req
|
||||
* @param res
|
||||
*
|
||||
* Will respond with errors if required. Returns false if it errors.
|
||||
*
|
||||
* @param req
|
||||
* @param res
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
function handleGetSegments(req, res) {
|
||||
const videoID = req.query.videoID;
|
||||
// Default to sponsor
|
||||
// If using params instead of JSON, only one category can be pulled
|
||||
const categories = req.query.categories ? JSON.parse(req.query.categories)
|
||||
: (req.query.category ? [req.query.category] : ["sponsor"]);
|
||||
const videoID = req.query.videoID;
|
||||
// Default to sponsor
|
||||
// If using params instead of JSON, only one category can be pulled
|
||||
const categories = req.query.categories
|
||||
? JSON.parse(req.query.categories)
|
||||
: req.query.category
|
||||
? [req.query.category]
|
||||
: ['sponsor'];
|
||||
|
||||
/**
|
||||
* @type {Array<{
|
||||
* segment: number[],
|
||||
* category: string,
|
||||
* UUID: string
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
let segments = [];
|
||||
/**
|
||||
* @type {Array<{
|
||||
* segment: number[],
|
||||
* category: string,
|
||||
* UUID: string
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
const segments = [];
|
||||
|
||||
let hashedIP = getHash(getIP(req) + config.globalSalt);
|
||||
let userHashedIP, shadowHiddenSegments;
|
||||
|
||||
try {
|
||||
for (const category of categories) {
|
||||
let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime")
|
||||
.all(videoID, category);
|
||||
try {
|
||||
for (const category of categories) {
|
||||
const categorySegments = db
|
||||
.prepare(
|
||||
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime'
|
||||
)
|
||||
.all(videoID, category)
|
||||
.filter(segment => {
|
||||
if (segment.votes < -1) {
|
||||
return false; //too untrustworthy, just ignore it
|
||||
}
|
||||
|
||||
let sponsorTimes = [];
|
||||
let votes = []
|
||||
let UUIDs = [];
|
||||
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
//check if votes are above -1
|
||||
if (rows[i].votes < -1) {
|
||||
//too untrustworthy, just ignore it
|
||||
continue;
|
||||
}
|
||||
|
||||
//check if shadowHidden
|
||||
//this means it is hidden to everyone but the original ip that submitted it
|
||||
if (rows[i].shadowHidden == 1) {
|
||||
//get the ip
|
||||
//await the callback
|
||||
let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID);
|
||||
|
||||
if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) {
|
||||
//this isn't their ip, don't send it to them
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
sponsorTimes.push([rows[i].startTime, rows[i].endTime]);
|
||||
votes.push(rows[i].votes);
|
||||
UUIDs.push(rows[i].UUID);
|
||||
//check if shadowHidden
|
||||
//this means it is hidden to everyone but the original ip that submitted it
|
||||
if (segment.shadowHidden != 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (shadowHiddenSegments === undefined) {
|
||||
shadowHiddenSegments = privateDB
|
||||
.prepare('SELECT hashedIP FROM sponsorTimes WHERE videoID = ?')
|
||||
.all(videoID);
|
||||
}
|
||||
|
||||
//if this isn't their ip, don't send it to them
|
||||
return shadowHiddenSegments.some(shadowHiddenSegment => {
|
||||
if (userHashedIP === undefined) {
|
||||
//hash the IP only if it's strictly necessary
|
||||
userHashedIP = getHash(getIP(req) + config.globalSalt);
|
||||
}
|
||||
|
||||
organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs);
|
||||
sponsorTimes = organisedData.sponsorTimes;
|
||||
UUIDs = organisedData.UUIDs;
|
||||
|
||||
for (let i = 0; i < sponsorTimes.length; i++) {
|
||||
segments.push({
|
||||
segment: sponsorTimes[i],
|
||||
category: category,
|
||||
UUID: UUIDs[i]
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch(error) {
|
||||
console.error(error);
|
||||
res.send(500);
|
||||
return shadowHiddenSegment.hashedIP === userHashedIP;
|
||||
});
|
||||
});
|
||||
|
||||
return false;
|
||||
chooseSegments(categorySegments).forEach(chosenSegment => {
|
||||
segments.push({
|
||||
category,
|
||||
segment: [chosenSegment.startTime, chosenSegment.endTime],
|
||||
UUID: chosenSegment.UUID,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (segments.length == 0) {
|
||||
res.sendStatus(404);
|
||||
return false;
|
||||
res.sendStatus(404);
|
||||
return false;
|
||||
}
|
||||
|
||||
return segments;
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
res.sendStatus(500);
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
module.exports = {
|
||||
handleGetSegments,
|
||||
endpoint: function (req, res) {
|
||||
let segments = handleGetSegments(req, res);
|
||||
handleGetSegments,
|
||||
endpoint: function (req, res) {
|
||||
let segments = handleGetSegments(req, res);
|
||||
|
||||
if (segments) {
|
||||
//send result
|
||||
res.send(segments)
|
||||
}
|
||||
if (segments) {
|
||||
//send result
|
||||
res.send(segments);
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user