mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2025-12-09 21:17:15 +03:00
431 lines
18 KiB
TypeScript
431 lines
18 KiB
TypeScript
import { Request, Response } from "express";
|
|
import { partition } from "lodash";
|
|
import { config } from "../config";
|
|
import { db, privateDB } from "../databases/databases";
|
|
import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey } from "../utils/redisKeys";
|
|
import { SBRecord } from "../types/lib.model";
|
|
import { ActionType, Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, SegmentUUID, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
|
|
import { getCategoryActionType } from "../utils/categoryInfo";
|
|
import { getHashCache } from "../utils/getHashCache";
|
|
import { getIP } from "../utils/getIP";
|
|
import { Logger } from "../utils/logger";
|
|
import { QueryCacher } from "../utils/queryCacher";
|
|
import { getReputation } from "../utils/reputation";
|
|
import { getService } from "../utils/getService";
|
|
|
|
|
|
async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean): Promise<Segment[]> {
|
|
const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
|
|
if (segment.votes < -1 && !segment.required) {
|
|
return false; //too untrustworthy, just ignore it
|
|
}
|
|
|
|
//check if shadowHidden
|
|
//this means it is hidden to everyone but the original ip that submitted it
|
|
if (segment.shadowHidden != Visibility.HIDDEN) {
|
|
return true;
|
|
}
|
|
|
|
if (cache.shadowHiddenSegmentIPs[videoID] === undefined) cache.shadowHiddenSegmentIPs[videoID] = {};
|
|
if (cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] === undefined) {
|
|
const service = getService(req?.query?.service as string);
|
|
cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ? AND "service" = ?',
|
|
[videoID, segment.timeSubmitted, service]) as { hashedIP: HashedIP }[];
|
|
}
|
|
|
|
const ipList = cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted];
|
|
|
|
if (ipList?.length > 0 && cache.userHashedIP === undefined) {
|
|
//hash the IP only if it's strictly necessary
|
|
cache.userHashedIP = await getHashCache((getIP(req) + config.globalSalt) as IPAddress);
|
|
}
|
|
//if this isn't their ip, don't send it to them
|
|
const shouldShadowHide = cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some(
|
|
(shadowHiddenSegment) => shadowHiddenSegment.hashedIP === cache.userHashedIP) ?? false;
|
|
|
|
if (shouldShadowHide) useCache = false;
|
|
return shouldShadowHide;
|
|
}));
|
|
|
|
const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
|
|
|
|
return (await chooseSegments(videoID, service, filteredSegments, useCache)).map((chosenSegment) => ({
|
|
category: chosenSegment.category,
|
|
actionType: chosenSegment.actionType,
|
|
segment: [chosenSegment.startTime, chosenSegment.endTime],
|
|
UUID: chosenSegment.UUID,
|
|
locked: chosenSegment.locked,
|
|
votes: chosenSegment.votes,
|
|
videoDuration: chosenSegment.videoDuration,
|
|
userID: chosenSegment.userID,
|
|
description: chosenSegment.description
|
|
}));
|
|
}
|
|
|
|
async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories: Category[],
|
|
actionTypes: ActionType[], requiredSegments: SegmentUUID[], service: Service): Promise<Segment[]> {
|
|
const cache: SegmentCache = { shadowHiddenSegmentIPs: {} };
|
|
|
|
try {
|
|
categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
|
|
if (categories.length === 0) return null;
|
|
|
|
const segments: DBSegment[] = (await getSegmentsFromDBByVideoID(videoID, service))
|
|
.map((segment: DBSegment) => {
|
|
if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
|
|
return segment;
|
|
}, {});
|
|
|
|
const canUseCache = requiredSegments.length === 0;
|
|
const processedSegments: Segment[] = await prepareCategorySegments(req, videoID, service, segments, cache, canUseCache);
|
|
|
|
return processedSegments.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType));
|
|
} catch (err) {
|
|
if (err) {
|
|
Logger.error(err as string);
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[],
|
|
actionTypes: ActionType[], requiredSegments: SegmentUUID[], service: Service): Promise<SBRecord<VideoID, VideoData>> {
|
|
const cache: SegmentCache = { shadowHiddenSegmentIPs: {} };
|
|
const segments: SBRecord<VideoID, VideoData> = {};
|
|
|
|
try {
|
|
type SegmentWithHashPerVideoID = SBRecord<VideoID, { hash: VideoIDHash, segments: DBSegment[] }>;
|
|
|
|
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
|
|
if (categories.length === 0) return null;
|
|
|
|
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
|
|
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
|
|
acc[segment.videoID] = acc[segment.videoID] || {
|
|
hash: segment.hashedVideoID,
|
|
segments: []
|
|
};
|
|
if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
|
|
|
|
acc[segment.videoID].segments ??= [];
|
|
acc[segment.videoID].segments.push(segment);
|
|
|
|
return acc;
|
|
}, {});
|
|
|
|
for (const [videoID, videoData] of Object.entries(segmentPerVideoID)) {
|
|
const data: VideoData = {
|
|
hash: videoData.hash,
|
|
segments: [],
|
|
};
|
|
|
|
const canUseCache = requiredSegments.length === 0;
|
|
data.segments = (await prepareCategorySegments(req, videoID as VideoID, service, videoData.segments, cache, canUseCache))
|
|
.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType));
|
|
|
|
if (data.segments.length > 0) {
|
|
segments[videoID] = data;
|
|
}
|
|
}
|
|
|
|
return segments;
|
|
} catch (err) {
|
|
Logger.error(err as string);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
|
|
const fetchFromDB = () => db
|
|
.prepare(
|
|
"all",
|
|
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description" FROM "sponsorTimes"
|
|
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
|
|
[`${hashedVideoIDPrefix}%`, service]
|
|
) as Promise<DBSegment[]>;
|
|
|
|
if (hashedVideoIDPrefix.length === 4) {
|
|
return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service));
|
|
}
|
|
|
|
return await fetchFromDB();
|
|
}
|
|
|
|
async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
|
|
const fetchFromDB = () => db
|
|
.prepare(
|
|
"all",
|
|
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "timeSubmitted", "description" FROM "sponsorTimes"
|
|
WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
|
|
[videoID, service]
|
|
) as Promise<DBSegment[]>;
|
|
|
|
return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service));
|
|
}
|
|
|
|
// Gets a weighted random choice from the choices array based on their `votes` property.
|
|
// amountOfChoices specifies the maximum amount of choices to return, 1 or more.
|
|
// Choices are unique
|
|
// If a predicate is given, it will only filter choices following it, and will leave the rest in the list
|
|
function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOfChoices: number, predicate?: (choice: T) => void): T[] {
|
|
//trivial case: no need to go through the whole process
|
|
if (amountOfChoices >= choices.length) {
|
|
return choices;
|
|
}
|
|
|
|
type TWithWeight = T & {
|
|
weight: number
|
|
}
|
|
|
|
let forceIncludedChoices: T[] = [];
|
|
let filteredChoices = choices;
|
|
if (predicate) {
|
|
const splitArray = partition(choices, predicate);
|
|
filteredChoices = splitArray[0];
|
|
forceIncludedChoices = splitArray[1];
|
|
}
|
|
|
|
//assign a weight to each choice
|
|
let totalWeight = 0;
|
|
const choicesWithWeights: TWithWeight[] = filteredChoices.map(choice => {
|
|
const boost = Math.min(choice.reputation, 4);
|
|
|
|
//The 3 makes -2 the minimum votes before being ignored completely
|
|
//this can be changed if this system increases in popularity.
|
|
const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost);
|
|
totalWeight += Math.max(weight, 0);
|
|
|
|
return { ...choice, weight };
|
|
});
|
|
|
|
// Nothing to filter for
|
|
if (amountOfChoices >= choicesWithWeights.length) {
|
|
return choices;
|
|
}
|
|
|
|
//iterate and find amountOfChoices choices
|
|
const chosen = [...forceIncludedChoices];
|
|
while (amountOfChoices-- > 0) {
|
|
//weighted random draw of one element of choices
|
|
const randomNumber = Math.random() * totalWeight;
|
|
let stackWeight = choicesWithWeights[0].weight;
|
|
let i = 0;
|
|
while (stackWeight < randomNumber) {
|
|
stackWeight += choicesWithWeights[++i].weight;
|
|
}
|
|
|
|
//add it to the chosen ones and remove it from the choices before the next iteration
|
|
chosen.push(choicesWithWeights[i]);
|
|
totalWeight -= choicesWithWeights[i].weight;
|
|
choicesWithWeights.splice(i, 1);
|
|
}
|
|
|
|
return chosen;
|
|
}
|
|
|
|
async function chooseSegments(videoID: VideoID, service: Service, segments: DBSegment[], useCache: boolean): Promise<DBSegment[]> {
|
|
const fetchData = async () => await buildSegmentGroups(segments);
|
|
|
|
const groups = useCache
|
|
? await QueryCacher.get(fetchData, skipSegmentGroupsKey(videoID, service))
|
|
: await fetchData();
|
|
|
|
// Filter for only 1 item for POI categories
|
|
return getWeightedRandomChoice(groups, 1, (choice) => getCategoryActionType(choice.segments[0].category) === CategoryActionType.POI)
|
|
.map(//randomly choose 1 good segment per group and return them
|
|
group => getWeightedRandomChoice(group.segments, 1)[0]
|
|
);
|
|
}
|
|
|
|
//This function will find segments that are contained inside of eachother, called similar segments
|
|
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
|
|
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
|
|
//Segments with less than -1 votes are already ignored before this function is called
|
|
async function buildSegmentGroups(segments: DBSegment[]): Promise<OverlappingSegmentGroup[]> {
|
|
//Create groups of segments that are similar to eachother
|
|
//Segments must be sorted by their startTime so that we can build groups chronologically:
|
|
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
|
|
//2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
|
|
// inside that group (because they're sorted) so we can create a new one
|
|
let overlappingSegmentsGroups: OverlappingSegmentGroup[] = [];
|
|
let currentGroup: OverlappingSegmentGroup;
|
|
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
|
|
for (const segment of segments) {
|
|
if (segment.startTime >= cursor) {
|
|
currentGroup = { segments: [], votes: 0, reputation: 0, locked: false, required: false };
|
|
overlappingSegmentsGroups.push(currentGroup);
|
|
}
|
|
|
|
currentGroup.segments.push(segment);
|
|
//only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
|
|
if (segment.votes > 0) {
|
|
currentGroup.votes += segment.votes;
|
|
}
|
|
|
|
if (segment.userID) segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID));
|
|
if (segment.reputation > 0) {
|
|
currentGroup.reputation += segment.reputation;
|
|
}
|
|
|
|
if (segment.locked) {
|
|
currentGroup.locked = true;
|
|
}
|
|
|
|
if (segment.required) {
|
|
currentGroup.required = true;
|
|
}
|
|
|
|
cursor = Math.max(cursor, segment.endTime);
|
|
}
|
|
|
|
overlappingSegmentsGroups = splitPercentOverlap(overlappingSegmentsGroups);
|
|
overlappingSegmentsGroups.forEach((group) => {
|
|
if (group.required) {
|
|
// Required beats locked
|
|
group.segments = group.segments.filter((segment) => segment.required);
|
|
} else if (group.locked) {
|
|
group.segments = group.segments.filter((segment) => segment.locked);
|
|
}
|
|
|
|
group.reputation = group.reputation / group.segments.length;
|
|
});
|
|
|
|
//if there are too many groups, find the best ones
|
|
return overlappingSegmentsGroups;
|
|
}
|
|
|
|
function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegmentGroup[] {
|
|
return groups.flatMap((group) => {
|
|
const result: OverlappingSegmentGroup[] = [];
|
|
group.segments.forEach((segment) => {
|
|
const bestGroup = result.find((group) => {
|
|
// At least one segment in the group must have high % overlap or the same action type
|
|
// Since POI segments will always have 0 overlap, they will always be in their own groups
|
|
return group.segments.some((compareSegment) => {
|
|
const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime);
|
|
const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime);
|
|
const overlapPercent = overlap / overallDuration;
|
|
return (overlapPercent > 0 && segment.actionType === compareSegment.actionType && segment.category === compareSegment.category && segment.actionType !== ActionType.Chapter)
|
|
|| (overlapPercent >= 0.6 && segment.actionType !== compareSegment.actionType && segment.category === compareSegment.category)
|
|
|| (overlapPercent >= 0.95 && segment.actionType === compareSegment.actionType && segment.category !== compareSegment.category
|
|
&& segment.category !== "music_offtopic" && compareSegment.category !== "music_offtopic")
|
|
|| (overlapPercent >= 0.8 && segment.actionType === ActionType.Chapter && compareSegment.actionType === ActionType.Chapter);
|
|
});
|
|
});
|
|
|
|
if (bestGroup) {
|
|
bestGroup.segments.push(segment);
|
|
bestGroup.votes += segment.votes;
|
|
bestGroup.reputation += segment.reputation;
|
|
bestGroup.locked ||= segment.locked;
|
|
bestGroup.required ||= segment.required;
|
|
} else {
|
|
result.push({ segments: [segment], votes: segment.votes, reputation: segment.reputation, locked: segment.locked, required: segment.required });
|
|
}
|
|
});
|
|
|
|
return result;
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* Returns what would be sent to the client.
|
|
* Will respond with errors if required. Returns false if it errors.
|
|
*
|
|
* @param req
|
|
* @param res
|
|
*
|
|
* @returns
|
|
*/
|
|
async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
|
|
const videoID = req.query.videoID as VideoID;
|
|
if (!videoID) {
|
|
res.status(400).send("videoID not specified");
|
|
return false;
|
|
}
|
|
// Default to sponsor
|
|
// If using params instead of JSON, only one category can be pulled
|
|
const categories: Category[] = req.query.categories
|
|
? JSON.parse(req.query.categories as string)
|
|
: req.query.category
|
|
? Array.isArray(req.query.category)
|
|
? req.query.category
|
|
: [req.query.category]
|
|
: ["sponsor"];
|
|
if (!Array.isArray(categories)) {
|
|
res.status(400).send("Categories parameter does not match format requirements.");
|
|
return false;
|
|
}
|
|
|
|
const actionTypes: ActionType[] = req.query.actionTypes
|
|
? JSON.parse(req.query.actionTypes as string)
|
|
: req.query.actionType
|
|
? Array.isArray(req.query.actionType)
|
|
? req.query.actionType
|
|
: [req.query.actionType]
|
|
: [ActionType.Skip];
|
|
if (!Array.isArray(actionTypes)) {
|
|
res.status(400).send("actionTypes parameter does not match format requirements.");
|
|
return false;
|
|
}
|
|
|
|
const requiredSegments: SegmentUUID[] = req.query.requiredSegments
|
|
? JSON.parse(req.query.requiredSegments as string)
|
|
: req.query.requiredSegment
|
|
? Array.isArray(req.query.requiredSegment)
|
|
? req.query.requiredSegment
|
|
: [req.query.requiredSegment]
|
|
: [];
|
|
if (!Array.isArray(requiredSegments)) {
|
|
res.status(400).send("requiredSegments parameter does not match format requirements.");
|
|
return false;
|
|
}
|
|
|
|
const service = getService(req.query.service, req.body.service);
|
|
|
|
const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
|
|
|
|
if (segments === null || segments === undefined) {
|
|
res.sendStatus(500);
|
|
return false;
|
|
}
|
|
|
|
if (segments.length === 0) {
|
|
res.sendStatus(404);
|
|
return false;
|
|
}
|
|
|
|
return segments;
|
|
}
|
|
|
|
const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID[]): boolean => {
|
|
for (const search of requiredSegments) {
|
|
if (search === UUID || UUID.indexOf(search) == 0) return true;
|
|
}
|
|
return false;
|
|
};
|
|
|
|
async function endpoint(req: Request, res: Response): Promise<Response> {
|
|
try {
|
|
const segments = await handleGetSegments(req, res);
|
|
|
|
// If false, res.send has already been called
|
|
if (segments) {
|
|
//send result
|
|
return res.send(segments);
|
|
}
|
|
} catch (err) {
|
|
if (err instanceof SyntaxError) {
|
|
return res.status(400).send("Categories parameter does not match format requirements.");
|
|
} else return res.sendStatus(500);
|
|
}
|
|
}
|
|
|
|
export {
|
|
getSegmentsByVideoID,
|
|
getSegmentsByHash,
|
|
endpoint,
|
|
handleGetSegments
|
|
};
|