Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 19:47:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #91 from pdonias/rework-getSkipSegments

Browse Source 
`getSkipSegments` rework
This commit is contained in:
Ajay Ramachandran
2020-06-08 22:58:01 -04:00
committed by GitHub
parent 45d10f2667 85dcb39f5e
commit 325981b9c9
2 changed files with 138 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

285
src/routes/getSkipSegments.js
View File

@@ -8,175 +8,86 @@ var privateDB = databases.privateDB;
var getHash = require('../utils/getHash.js');
var getIP = require('../utils/getIP.js');
//gets the getWeightedRandomChoice for each group in an array of groups
function getWeightedRandomChoiceForArray(choiceGroups, weights) {
let finalChoices = [];
//the indexes either chosen to be added to final indexes or chosen not to be added
let choicesDealtWith = [];
//for each choice group, what are the sums of the weights
let weightSums = [];
for (let i = 0; i < choiceGroups.length; i++) {
//find weight sums for this group
weightSums.push(0);
for (let j = 0; j < choiceGroups[i].length; j++) {
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
if (weights[choiceGroups[i][j]] > 0) {
weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]];
}
}
//create a random choice for this group
let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1)
finalChoices.push(randomChoice.finalChoices);
for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) {
choicesDealtWith.push(randomChoice.choicesDealtWith[j])
}
}
return {
finalChoices: finalChoices,
choicesDealtWith: choicesDealtWith,
weightSums: weightSums
};
}
//gets a weighted random choice from the indexes array based on the weights.
//amountOfChoices speicifies the amount of choices to return, 1 or more.
//gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique
function getWeightedRandomChoice(choices, weights, amountOfChoices) {
if (amountOfChoices > choices.length) {
//not possible, since all choices must be unique
return null;
function getWeightedRandomChoice(choices, amountOfChoices) {
//trivial case: no need to go through the whole process
if (amountOfChoices >= choices.length) {
return choices;
}
let finalChoices = [];
let choicesDealtWith = [];
let sqrtWeightsList = [];
//the total of all the weights run through the cutom sqrt function
let totalSqrtWeights = 0;
for (let j = 0; j < choices.length; j++) {
//assign a weight to each choice
let totalWeight = 0;
choices = choices.map(choice => {
//multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
//The 3 makes -2 the minimum votes before being ignored completely
//https://www.desmos.com/calculator/ljftxolg9j
//this can be changed if this system increases in popularity.
let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10);
sqrtWeightsList.push(sqrtVote)
totalSqrtWeights += sqrtVote;
const weight = Math.sqrt((choice.votes + 3) * 10);
totalWeight += weight;
//this index has now been deat with
choicesDealtWith.push(choices[j]);
}
return { ...choice, weight };
});
//iterate and find amountOfChoices choices
let randomNumber = Math.random();
//this array will keep adding to this variable each time one sqrt vote has been dealt with
//this is the sum of all the sqrtVotes under this index
let currentVoteNumber = 0;
for (let j = 0; j < sqrtWeightsList.length; j++) {
if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) {
//this one was randomly generated
finalChoices.push(choices[j]);
//remove that from original array, for next recursion pass if it happens
choices.splice(j, 1);
break;
const chosen = [];
while (amountOfChoices-- > 0) {
//weighted random draw of one element of choices
const randomNumber = Math.random() * totalWeight;
let stackWeight = choices[0].weight;
let i = 0;
while (stackWeight < randomNumber) {
stackWeight += choices[++i].weight;
}
//add on to the count
currentVoteNumber += sqrtWeightsList[j];
//add it to the chosen ones and remove it from the choices before the next iteration
chosen.push(choices[i]);
choices.splice(i, 1);
}
//add on the other choices as well using recursion
if (amountOfChoices > 1) {
let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices;
//add all these choices to the finalChoices array being returned
for (let i = 0; i < otherChoices.length; i++) {
finalChoices.push(otherChoices[i]);
}
}
return {
finalChoices: finalChoices,
choicesDealtWith: choicesDealtWith
};
return chosen;
}
//This function will find sponsor times that are contained inside of eachother, called similar sponsor times
//This function will find segments that are contained inside of eachother, called similar segments
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
//Sponsor times with less than -1 votes are already ignored before this function is called
function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
//create groups of sponsor times that are similar to eachother
const groups = []
sponsorTimes.forEach(([sponsorStart, sponsorEnd], i) => {
//find a group that overlaps with the current segment
//sponsorTimes are sorted by their startTime so there should never be more than 1 similar group
const similarGroup = groups.find(group => group.start < sponsorEnd && sponsorStart < group.end)
//add the sponsor to that group or create a new group if there aren't any
if (similarGroup === undefined) {
groups.push({ start: sponsorStart, end: sponsorEnd, sponsors: [i] })
} else {
similarGroup.sponsors.push(i)
similarGroup.start = Math.min(similarGroup.start, sponsorStart)
similarGroup.end = Math.max(similarGroup.end, sponsorEnd)
}
})
//once all the groups have been created, get rid of the metadata and remove single-sponsor groups
const similarSponsorsGroups = groups.map(group => group.sponsors).filter(group => group.length > 1)
let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes);
let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices;
//the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added
let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith;
let voteSums = weightedRandomIndexes.weightSums;
//convert these into the votes
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
//it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor.
votes[finalSponsorTimeIndexes[i]] = voteSums[i];
//Segments with less than -1 votes are already ignored before this function is called
function chooseSegments(segments) {
//Create groups of segments that are similar to eachother
//Segments must be sorted by their startTime so that we can build groups chronologically:
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
//2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
// inside that group (because they're sorted) so we can create a new one
const similarSegmentsGroups = [];
let currentGroup;
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
if (segment.startTime > cursor) {
currentGroup = { segments: [], votes: 0 };
similarSegmentsGroups.push(currentGroup);
}
//find the indexes never dealt with and add them
for (let i = 0; i < sponsorTimes.length; i++) {
if (!finalSponsorTimeIndexesDealtWith.includes(i)) {
finalSponsorTimeIndexes.push(i)
}
currentGroup.segments.push(segment);
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
if (segment.votes > 0) {
currentGroup.votes += segment.votes;
}
//if there are too many indexes, find the best 4
if (finalSponsorTimeIndexes.length > 8) {
finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices;
}
cursor = Math.max(cursor, segment.endTime);
});
//convert this to a final array to return
let finalSponsorTimes = [];
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]);
}
//convert this to a final array of UUIDs as well
let finalUUIDs = [];
for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]);
}
return {
sponsorTimes: finalSponsorTimes,
UUIDs: finalUUIDs
};
//if there are too many groups, find the best 8
return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
//randomly choose 1 good segment per group and return them
group => getWeightedRandomChoice(group.segments, 1)[0]
);
}
/**
*
* Returns what would be sent to the client.
* Will resond with errors if required. Returns false if it errors.
* Will respond with errors if required. Returns false if it errors.
*
* @param req
* @param res
@@ -187,8 +98,11 @@ function handleGetSegments(req, res) {
const videoID = req.query.videoID;
// Default to sponsor
// If using params instead of JSON, only one category can be pulled
const categories = req.query.categories ? JSON.parse(req.query.categories)
: (req.query.category ? [req.query.category] : ["sponsor"]);
const categories = req.query.categories
? JSON.parse(req.query.categories)
: req.query.category
? [req.query.category]
: ['sponsor'];
/**
* @type {Array<{
@@ -198,61 +112,51 @@ function handleGetSegments(req, res) {
* }>
* }
*/
let segments = [];
const segments = [];
let hashedIP = getHash(getIP(req) + config.globalSalt);
let userHashedIP, shadowHiddenSegments;
try {
for (const category of categories) {
let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime")
.all(videoID, category);
let sponsorTimes = [];
let votes = []
let UUIDs = [];
for (let i = 0; i < rows.length; i++) {
//check if votes are above -1
if (rows[i].votes < -1) {
//too untrustworthy, just ignore it
continue;
const categorySegments = db
.prepare(
'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime'
)
.all(videoID, category)
.filter(segment => {
if (segment.votes < -1) {
return false; //too untrustworthy, just ignore it
}
//check if shadowHidden
//this means it is hidden to everyone but the original ip that submitted it
if (rows[i].shadowHidden == 1) {
//get the ip
//await the callback
let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID);
if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) {
//this isn't their ip, don't send it to them
continue;
}
if (segment.shadowHidden != 1) {
return true;
}
sponsorTimes.push([rows[i].startTime, rows[i].endTime]);
votes.push(rows[i].votes);
UUIDs.push(rows[i].UUID);
if (shadowHiddenSegments === undefined) {
shadowHiddenSegments = privateDB
.prepare('SELECT hashedIP FROM sponsorTimes WHERE videoID = ?')
.all(videoID);
}
organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs);
sponsorTimes = organisedData.sponsorTimes;
UUIDs = organisedData.UUIDs;
for (let i = 0; i < sponsorTimes.length; i++) {
segments.push({
segment: sponsorTimes[i],
category: category,
UUID: UUIDs[i]
//if this isn't their ip, don't send it to them
return shadowHiddenSegments.some(shadowHiddenSegment => {
if (userHashedIP === undefined) {
//hash the IP only if it's strictly necessary
userHashedIP = getHash(getIP(req) + config.globalSalt);
}
return shadowHiddenSegment.hashedIP === userHashedIP;
});
});
}
}
} catch(error) {
console.error(error);
res.send(500);
return false;
chooseSegments(categorySegments).forEach(chosenSegment => {
segments.push({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
});
});
}
if (segments.length == 0) {
@@ -261,8 +165,13 @@ function handleGetSegments(req, res) {
}
return segments;
}
} catch (error) {
console.error(error);
res.sendStatus(500);
return false;
}
}
module.exports = {
handleGetSegments,
@@ -271,7 +180,7 @@ module.exports = {
if (segments) {
//send result
res.send(segments)
res.send(segments);
}
}
}
},
};

3
src/routes/voteOnSponsorTime.js
View File

@@ -182,7 +182,7 @@ module.exports = async function voteOnSponsorTime(req, res) {
// Send discord message
if (incrementAmount < 0) {
// Get video ID
let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, " +
let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " +
"(select count(1) from sponsorTimes where userID = s.userID) count, " +
"(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
"FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?"
@@ -216,6 +216,7 @@ module.exports = async function voteOnSponsorTime(req, res) {
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
"description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views
+ " Views**\n\n**Submission ID:** " + UUID
+ "\n**Category:** " + submissionInfoRow.category
+ "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID
+ "\n\n**Total User Submissions:** "+submissionInfoRow.count
+ "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded