mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2025-12-07 03:57:06 +03:00
Merge pull request #91 from pdonias/rework-getSkipSegments
`getSkipSegments` rework
This commit is contained in:
Ajay Ramachandran
@@ -8,175 +8,86 @@ var privateDB = databases.privateDB;
|
|||||||
var getHash = require('../utils/getHash.js');
|
var getHash = require('../utils/getHash.js');
|
||||||
var getIP = require('../utils/getIP.js');
|
var getIP = require('../utils/getIP.js');
|
||||||
|
|
||||||
|
//gets a weighted random choice from the choices array based on their `votes` property.
|
||||||
//gets the getWeightedRandomChoice for each group in an array of groups
|
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
|
||||||
function getWeightedRandomChoiceForArray(choiceGroups, weights) {
|
|
||||||
let finalChoices = [];
|
|
||||||
//the indexes either chosen to be added to final indexes or chosen not to be added
|
|
||||||
let choicesDealtWith = [];
|
|
||||||
//for each choice group, what are the sums of the weights
|
|
||||||
let weightSums = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < choiceGroups.length; i++) {
|
|
||||||
//find weight sums for this group
|
|
||||||
weightSums.push(0);
|
|
||||||
for (let j = 0; j < choiceGroups[i].length; j++) {
|
|
||||||
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
|
|
||||||
if (weights[choiceGroups[i][j]] > 0) {
|
|
||||||
weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//create a random choice for this group
|
|
||||||
let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1)
|
|
||||||
finalChoices.push(randomChoice.finalChoices);
|
|
||||||
|
|
||||||
for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) {
|
|
||||||
choicesDealtWith.push(randomChoice.choicesDealtWith[j])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
finalChoices: finalChoices,
|
|
||||||
choicesDealtWith: choicesDealtWith,
|
|
||||||
weightSums: weightSums
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
//gets a weighted random choice from the indexes array based on the weights.
|
|
||||||
//amountOfChoices speicifies the amount of choices to return, 1 or more.
|
|
||||||
//choices are unique
|
//choices are unique
|
||||||
function getWeightedRandomChoice(choices, weights, amountOfChoices) {
|
function getWeightedRandomChoice(choices, amountOfChoices) {
|
||||||
if (amountOfChoices > choices.length) {
|
//trivial case: no need to go through the whole process
|
||||||
//not possible, since all choices must be unique
|
if (amountOfChoices >= choices.length) {
|
||||||
return null;
|
return choices;
|
||||||
}
|
}
|
||||||
|
|
||||||
let finalChoices = [];
|
//assign a weight to each choice
|
||||||
let choicesDealtWith = [];
|
let totalWeight = 0;
|
||||||
|
choices = choices.map(choice => {
|
||||||
|
//multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
|
||||||
|
//The 3 makes -2 the minimum votes before being ignored completely
|
||||||
|
//https://www.desmos.com/calculator/ljftxolg9j
|
||||||
|
//this can be changed if this system increases in popularity.
|
||||||
|
const weight = Math.sqrt((choice.votes + 3) * 10);
|
||||||
|
totalWeight += weight;
|
||||||
|
|
||||||
let sqrtWeightsList = [];
|
return { ...choice, weight };
|
||||||
//the total of all the weights run through the cutom sqrt function
|
});
|
||||||
let totalSqrtWeights = 0;
|
|
||||||
for (let j = 0; j < choices.length; j++) {
|
|
||||||
//multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
|
|
||||||
//The 3 makes -2 the minimum votes before being ignored completely
|
|
||||||
//https://www.desmos.com/calculator/ljftxolg9j
|
|
||||||
//this can be changed if this system increases in popularity.
|
|
||||||
let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10);
|
|
||||||
sqrtWeightsList.push(sqrtVote)
|
|
||||||
totalSqrtWeights += sqrtVote;
|
|
||||||
|
|
||||||
//this index has now been deat with
|
|
||||||
choicesDealtWith.push(choices[j]);
|
|
||||||
}
|
|
||||||
|
|
||||||
//iterate and find amountOfChoices choices
|
//iterate and find amountOfChoices choices
|
||||||
let randomNumber = Math.random();
|
const chosen = [];
|
||||||
|
while (amountOfChoices-- > 0) {
|
||||||
|
//weighted random draw of one element of choices
|
||||||
|
const randomNumber = Math.random() * totalWeight;
|
||||||
|
let stackWeight = choices[0].weight;
|
||||||
|
let i = 0;
|
||||||
|
while (stackWeight < randomNumber) {
|
||||||
|
stackWeight += choices[++i].weight;
|
||||||
|
}
|
||||||
|
|
||||||
//this array will keep adding to this variable each time one sqrt vote has been dealt with
|
//add it to the chosen ones and remove it from the choices before the next iteration
|
||||||
//this is the sum of all the sqrtVotes under this index
|
chosen.push(choices[i]);
|
||||||
let currentVoteNumber = 0;
|
choices.splice(i, 1);
|
||||||
for (let j = 0; j < sqrtWeightsList.length; j++) {
|
|
||||||
if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) {
|
|
||||||
//this one was randomly generated
|
|
||||||
finalChoices.push(choices[j]);
|
|
||||||
//remove that from original array, for next recursion pass if it happens
|
|
||||||
choices.splice(j, 1);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
//add on to the count
|
|
||||||
currentVoteNumber += sqrtWeightsList[j];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//add on the other choices as well using recursion
|
return chosen;
|
||||||
if (amountOfChoices > 1) {
|
|
||||||
let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices;
|
|
||||||
//add all these choices to the finalChoices array being returned
|
|
||||||
for (let i = 0; i < otherChoices.length; i++) {
|
|
||||||
finalChoices.push(otherChoices[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
finalChoices: finalChoices,
|
|
||||||
choicesDealtWith: choicesDealtWith
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//This function will find segments that are contained inside of eachother, called similar segments
|
||||||
//This function will find sponsor times that are contained inside of eachother, called similar sponsor times
|
|
||||||
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
|
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
|
||||||
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
|
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
|
||||||
//Sponsor times with less than -1 votes are already ignored before this function is called
|
//Segments with less than -1 votes are already ignored before this function is called
|
||||||
function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
|
function chooseSegments(segments) {
|
||||||
//create groups of sponsor times that are similar to eachother
|
//Create groups of segments that are similar to eachother
|
||||||
const groups = []
|
//Segments must be sorted by their startTime so that we can build groups chronologically:
|
||||||
sponsorTimes.forEach(([sponsorStart, sponsorEnd], i) => {
|
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
|
||||||
//find a group that overlaps with the current segment
|
//2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
|
||||||
//sponsorTimes are sorted by their startTime so there should never be more than 1 similar group
|
// inside that group (because they're sorted) so we can create a new one
|
||||||
const similarGroup = groups.find(group => group.start < sponsorEnd && sponsorStart < group.end)
|
const similarSegmentsGroups = [];
|
||||||
//add the sponsor to that group or create a new group if there aren't any
|
let currentGroup;
|
||||||
if (similarGroup === undefined) {
|
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
|
||||||
groups.push({ start: sponsorStart, end: sponsorEnd, sponsors: [i] })
|
segments.forEach(segment => {
|
||||||
} else {
|
if (segment.startTime > cursor) {
|
||||||
similarGroup.sponsors.push(i)
|
currentGroup = { segments: [], votes: 0 };
|
||||||
similarGroup.start = Math.min(similarGroup.start, sponsorStart)
|
similarSegmentsGroups.push(currentGroup);
|
||||||
similarGroup.end = Math.max(similarGroup.end, sponsorEnd)
|
}
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
//once all the groups have been created, get rid of the metadata and remove single-sponsor groups
|
currentGroup.segments.push(segment);
|
||||||
const similarSponsorsGroups = groups.map(group => group.sponsors).filter(group => group.length > 1)
|
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
|
||||||
|
if (segment.votes > 0) {
|
||||||
|
currentGroup.votes += segment.votes;
|
||||||
|
}
|
||||||
|
|
||||||
let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes);
|
cursor = Math.max(cursor, segment.endTime);
|
||||||
|
});
|
||||||
|
|
||||||
let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices;
|
//if there are too many groups, find the best 8
|
||||||
//the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added
|
return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
|
||||||
let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith;
|
//randomly choose 1 good segment per group and return them
|
||||||
|
group => getWeightedRandomChoice(group.segments, 1)[0]
|
||||||
let voteSums = weightedRandomIndexes.weightSums;
|
);
|
||||||
//convert these into the votes
|
|
||||||
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
|
|
||||||
//it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor.
|
|
||||||
votes[finalSponsorTimeIndexes[i]] = voteSums[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
//find the indexes never dealt with and add them
|
|
||||||
for (let i = 0; i < sponsorTimes.length; i++) {
|
|
||||||
if (!finalSponsorTimeIndexesDealtWith.includes(i)) {
|
|
||||||
finalSponsorTimeIndexes.push(i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//if there are too many indexes, find the best 4
|
|
||||||
if (finalSponsorTimeIndexes.length > 8) {
|
|
||||||
finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices;
|
|
||||||
}
|
|
||||||
|
|
||||||
//convert this to a final array to return
|
|
||||||
let finalSponsorTimes = [];
|
|
||||||
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
|
|
||||||
finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]);
|
|
||||||
}
|
|
||||||
|
|
||||||
//convert this to a final array of UUIDs as well
|
|
||||||
let finalUUIDs = [];
|
|
||||||
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
|
|
||||||
finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
sponsorTimes: finalSponsorTimes,
|
|
||||||
UUIDs: finalUUIDs
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* Returns what would be sent to the client.
|
* Returns what would be sent to the client.
|
||||||
* Will resond with errors if required. Returns false if it errors.
|
* Will respond with errors if required. Returns false if it errors.
|
||||||
*
|
*
|
||||||
* @param req
|
* @param req
|
||||||
* @param res
|
* @param res
|
||||||
@@ -184,94 +95,92 @@ function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
|
|||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
function handleGetSegments(req, res) {
|
function handleGetSegments(req, res) {
|
||||||
const videoID = req.query.videoID;
|
const videoID = req.query.videoID;
|
||||||
// Default to sponsor
|
// Default to sponsor
|
||||||
// If using params instead of JSON, only one category can be pulled
|
// If using params instead of JSON, only one category can be pulled
|
||||||
const categories = req.query.categories ? JSON.parse(req.query.categories)
|
const categories = req.query.categories
|
||||||
: (req.query.category ? [req.query.category] : ["sponsor"]);
|
? JSON.parse(req.query.categories)
|
||||||
|
: req.query.category
|
||||||
|
? [req.query.category]
|
||||||
|
: ['sponsor'];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @type {Array<{
|
* @type {Array<{
|
||||||
* segment: number[],
|
* segment: number[],
|
||||||
* category: string,
|
* category: string,
|
||||||
* UUID: string
|
* UUID: string
|
||||||
* }>
|
* }>
|
||||||
* }
|
* }
|
||||||
*/
|
*/
|
||||||
let segments = [];
|
const segments = [];
|
||||||
|
|
||||||
let hashedIP = getHash(getIP(req) + config.globalSalt);
|
let userHashedIP, shadowHiddenSegments;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (const category of categories) {
|
for (const category of categories) {
|
||||||
let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime")
|
const categorySegments = db
|
||||||
.all(videoID, category);
|
.prepare(
|
||||||
|
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime'
|
||||||
|
)
|
||||||
|
.all(videoID, category)
|
||||||
|
.filter(segment => {
|
||||||
|
if (segment.votes < -1) {
|
||||||
|
return false; //too untrustworthy, just ignore it
|
||||||
|
}
|
||||||
|
|
||||||
let sponsorTimes = [];
|
//check if shadowHidden
|
||||||
let votes = []
|
//this means it is hidden to everyone but the original ip that submitted it
|
||||||
let UUIDs = [];
|
if (segment.shadowHidden != 1) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
for (let i = 0; i < rows.length; i++) {
|
if (shadowHiddenSegments === undefined) {
|
||||||
//check if votes are above -1
|
shadowHiddenSegments = privateDB
|
||||||
if (rows[i].votes < -1) {
|
.prepare('SELECT hashedIP FROM sponsorTimes WHERE videoID = ?')
|
||||||
//too untrustworthy, just ignore it
|
.all(videoID);
|
||||||
continue;
|
}
|
||||||
}
|
|
||||||
|
|
||||||
//check if shadowHidden
|
//if this isn't their ip, don't send it to them
|
||||||
//this means it is hidden to everyone but the original ip that submitted it
|
return shadowHiddenSegments.some(shadowHiddenSegment => {
|
||||||
if (rows[i].shadowHidden == 1) {
|
if (userHashedIP === undefined) {
|
||||||
//get the ip
|
//hash the IP only if it's strictly necessary
|
||||||
//await the callback
|
userHashedIP = getHash(getIP(req) + config.globalSalt);
|
||||||
let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID);
|
|
||||||
|
|
||||||
if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) {
|
|
||||||
//this isn't their ip, don't send it to them
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sponsorTimes.push([rows[i].startTime, rows[i].endTime]);
|
|
||||||
votes.push(rows[i].votes);
|
|
||||||
UUIDs.push(rows[i].UUID);
|
|
||||||
}
|
}
|
||||||
|
return shadowHiddenSegment.hashedIP === userHashedIP;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs);
|
chooseSegments(categorySegments).forEach(chosenSegment => {
|
||||||
sponsorTimes = organisedData.sponsorTimes;
|
segments.push({
|
||||||
UUIDs = organisedData.UUIDs;
|
category,
|
||||||
|
segment: [chosenSegment.startTime, chosenSegment.endTime],
|
||||||
for (let i = 0; i < sponsorTimes.length; i++) {
|
UUID: chosenSegment.UUID,
|
||||||
segments.push({
|
});
|
||||||
segment: sponsorTimes[i],
|
});
|
||||||
category: category,
|
|
||||||
UUID: UUIDs[i]
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch(error) {
|
|
||||||
console.error(error);
|
|
||||||
res.send(500);
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (segments.length == 0) {
|
if (segments.length == 0) {
|
||||||
res.sendStatus(404);
|
res.sendStatus(404);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return segments;
|
return segments;
|
||||||
}
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
res.sendStatus(500);
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
handleGetSegments,
|
handleGetSegments,
|
||||||
endpoint: function (req, res) {
|
endpoint: function (req, res) {
|
||||||
let segments = handleGetSegments(req, res);
|
let segments = handleGetSegments(req, res);
|
||||||
|
|
||||||
if (segments) {
|
if (segments) {
|
||||||
//send result
|
//send result
|
||||||
res.send(segments)
|
res.send(segments);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
|
};
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ module.exports = async function voteOnSponsorTime(req, res) {
|
|||||||
// Send discord message
|
// Send discord message
|
||||||
if (incrementAmount < 0) {
|
if (incrementAmount < 0) {
|
||||||
// Get video ID
|
// Get video ID
|
||||||
let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, " +
|
let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " +
|
||||||
"(select count(1) from sponsorTimes where userID = s.userID) count, " +
|
"(select count(1) from sponsorTimes where userID = s.userID) count, " +
|
||||||
"(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
|
"(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
|
||||||
"FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?"
|
"FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?"
|
||||||
@@ -216,6 +216,7 @@ module.exports = async function voteOnSponsorTime(req, res) {
|
|||||||
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
|
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
|
||||||
"description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views
|
"description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views
|
||||||
+ " Views**\n\n**Submission ID:** " + UUID
|
+ " Views**\n\n**Submission ID:** " + UUID
|
||||||
|
+ "\n**Category:** " + submissionInfoRow.category
|
||||||
+ "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID
|
+ "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID
|
||||||
+ "\n\n**Total User Submissions:** "+submissionInfoRow.count
|
+ "\n\n**Total User Submissions:** "+submissionInfoRow.count
|
||||||
+ "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded
|
+ "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded
|
||||||
|
|||||||
Reference in New Issue
Block a user