Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-30 03:18:23 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into clickbait

Browse Source
This commit is contained in:
Ajay Ramachandran
2023-01-28 02:13:42 -05:00
committed by GitHub
parent b91114165e 894cd48182
commit d07aac3a33
39 changed files with 1207 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
src/routes/getSkipSegments.ts
View File

@@ -12,7 +12,8 @@ import { QueryCacher } from "../utils/queryCacher";
import { getReputation } from "../utils/reputation";
import { getService } from "../utils/getService";
import { promiseOrTimeout } from "../utils/promise";
import { parseSkipSegments } from "../utils/parseSkipSegments";
import { getEtag } from "../middleware/etag";
async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean): Promise<Segment[]> {
const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
@@ -86,9 +87,6 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
}
try {
categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
if (categories.length === 0) return null;
const segments: DBSegment[] = (await getSegmentsFromDBByVideoID(videoID, service))
.map((segment: DBSegment) => {
if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
@@ -97,7 +95,17 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
const canUseCache = requiredSegments.length === 0;
let processedSegments: Segment[] = (await prepareCategorySegments(req, videoID, service, segments, cache, canUseCache))
.filter((segment: Segment) => categories.includes(segment?.category) && (actionTypes.includes(segment?.actionType)));
.filter((segment: Segment) => categories.includes(segment?.category) && (actionTypes.includes(segment?.actionType)))
.map((segment: Segment) => ({
category: segment.category,
actionType: segment.actionType,
segment: segment.segment,
UUID: segment.UUID,
videoDuration: segment.videoDuration,
locked: segment.locked,
votes: segment.votes,
description: segment.description
}));
if (forcePoiAsSkip) {
processedSegments = processedSegments.map((segment) => ({
@@ -127,15 +135,11 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}
try {
type SegmentWithHashPerVideoID = SBRecord<VideoID, { hash: VideoIDHash, segments: DBSegment[] }>;
type SegmentPerVideoID = SBRecord<VideoID, { segments: DBSegment[] }>;
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null;
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
const segmentPerVideoID: SegmentPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.reduce((acc: SegmentPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
hash: segment.hashedVideoID,
segments: []
};
if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
@@ -148,13 +152,22 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
await Promise.all(Object.entries(segmentPerVideoID).map(async ([videoID, videoData]) => {
const data: VideoData = {
hash: videoData.hash,
segments: [],
};
const canUseCache = requiredSegments.length === 0;
data.segments = (await prepareCategorySegments(req, videoID as VideoID, service, videoData.segments, cache, canUseCache))
.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType));
.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType))
.map((segment) => ({
category: segment.category,
actionType: segment.actionType,
segment: segment.segment,
UUID: segment.UUID,
videoDuration: segment.videoDuration,
locked: segment.locked,
votes: segment.votes,
description: segment.description
}));
if (forcePoiAsSkip) {
data.segments = data.segments.map((segment) => ({
@@ -378,75 +391,59 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm
});
}
/**
*
* Returns what would be sent to the client.
* Will respond with errors if required. Returns false if it errors.
*
* @param req
* @param res
*
* @returns
*/
async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
async function getSkipSegments(req: Request, res: Response): Promise<Response> {
const videoID = req.query.videoID as VideoID;
if (!videoID) {
res.status(400).send("videoID not specified");
return false;
}
// Default to sponsor
// If using params instead of JSON, only one category can be pulled
const categories: Category[] = req.query.categories
? JSON.parse(req.query.categories as string)
: req.query.category
? Array.isArray(req.query.category)
? req.query.category
: [req.query.category]
: ["sponsor"];
if (!Array.isArray(categories)) {
res.status(400).send("Categories parameter does not match format requirements.");
return false;
return res.status(400).send("videoID not specified");
}
const actionTypes: ActionType[] = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip];
if (!Array.isArray(actionTypes)) {
res.status(400).send("actionTypes parameter does not match format requirements.");
return false;
const parseResult = parseSkipSegments(req);
if (parseResult.errors.length > 0) {
return res.status(400).send(parseResult.errors);
}
const requiredSegments: SegmentUUID[] = req.query.requiredSegments
? JSON.parse(req.query.requiredSegments as string)
: req.query.requiredSegment
? Array.isArray(req.query.requiredSegment)
? req.query.requiredSegment
: [req.query.requiredSegment]
: [];
if (!Array.isArray(requiredSegments)) {
res.status(400).send("requiredSegments parameter does not match format requirements.");
return false;
}
const service = getService(req.query.service, req.body.service);
const { categories, actionTypes, requiredSegments, service } = parseResult;
const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
if (segments === null || segments === undefined) {
res.sendStatus(500);
return false;
return res.sendStatus(500);
} else if (segments.length === 0) {
return res.sendStatus(404);
}
if (segments.length === 0) {
res.sendStatus(404);
return false;
await getEtag("skipSegments", (videoID as string), service)
.then(etag => res.set("ETag", etag))
.catch(() => null);
return res.send(segments);
}
async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
const videoID = req.query.videoID as VideoID;
if (!videoID) {
return res.status(400).send("videoID not specified");
}
return segments;
const segments = await getSegmentsByVideoID(req, videoID, ["sponsor"] as Category[], [ActionType.Skip], [], Service.YouTube);
if (segments === null || segments === undefined) {
return res.sendStatus(500);
} else if (segments.length === 0) {
return res.sendStatus(404);
}
// Convert to old outputs
const sponsorTimes = [];
const UUIDs = [];
for (const segment of segments) {
sponsorTimes.push(segment.segment);
UUIDs.push(segment.UUID);
}
return res.send({
sponsorTimes,
UUIDs,
});
}
const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID[]): boolean => {
@@ -456,25 +453,9 @@ const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID
return false;
};
async function endpoint(req: Request, res: Response): Promise<Response> {
try {
const segments = await handleGetSegments(req, res);
// If false, res.send has already been called
if (segments) {
//send result
return res.send(segments);
}
} catch (err) /* istanbul ignore next */ {
if (err instanceof SyntaxError) {
return res.status(400).send("Categories parameter does not match format requirements.");
} else return res.sendStatus(500);
}
}
export {
getSegmentsByVideoID,
getSegmentsByHash,
endpoint,
handleGetSegments
getSkipSegments,
oldGetVideoSponsorTimes
};

64
src/routes/getSkipSegmentsByHash.ts
View File

@@ -1,9 +1,10 @@
import { hashPrefixTester } from "../utils/hashPrefixTester";
import { getSegmentsByHash } from "./getSkipSegments";
import { Request, Response } from "express";
import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model";
import { getService } from "../utils/getService";
import { VideoIDHash } from "../types/segments.model";
import { Logger } from "../utils/logger";
import { parseSkipSegments } from "../utils/parseSkipSegments";
import { getEtag } from "../middleware/etag";
export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
let hashPrefix = req.params.prefix as VideoIDHash;
@@ -12,66 +13,21 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
}
hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
let categories: Category[] = [];
try {
categories = req.query.categories
? JSON.parse(req.query.categories as string)
: req.query.category
? Array.isArray(req.query.category)
? req.query.category
: [req.query.category]
: ["sponsor"];
if (!Array.isArray(categories)) {
return res.status(400).send("Categories parameter does not match format requirements.");
}
} catch(error) {
return res.status(400).send("Bad parameter: categories (invalid JSON)");
const parseResult = parseSkipSegments(req);
if (parseResult.errors.length > 0) {
return res.status(400).send(parseResult.errors);
}
let actionTypes: ActionType[] = [];
try {
actionTypes = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip];
if (!Array.isArray(actionTypes)) {
return res.status(400).send("actionTypes parameter does not match format requirements.");
}
} catch(error) {
return res.status(400).send("Bad parameter: actionTypes (invalid JSON)");
}
let requiredSegments: SegmentUUID[] = [];
try {
requiredSegments = req.query.requiredSegments
? JSON.parse(req.query.requiredSegments as string)
: req.query.requiredSegment
? Array.isArray(req.query.requiredSegment)
? req.query.requiredSegment
: [req.query.requiredSegment]
: [];
if (!Array.isArray(requiredSegments)) {
return res.status(400).send("requiredSegments parameter does not match format requirements.");
}
} catch(error) {
return res.status(400).send("Bad parameter: requiredSegments (invalid JSON)");
}
const service: Service = getService(req.query.service, req.body.service);
// filter out none string elements, only flat array with strings is valid
categories = categories.filter((item: any) => typeof item === "string");
const { categories, actionTypes, requiredSegments, service } = parseResult;
// Get all video id's that match hash prefix
const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service);
try {
await getEtag("skipSegmentsHash", hashPrefix, service)
.then(etag => res.set("ETag", etag))
.catch(() => null);
const output = Object.entries(segments).map(([videoID, data]) => ({
videoID,
hash: data.hash,
segments: data.segments,
}));
return res.status(output.length === 0 ? 404 : 200).json(output);

1
src/routes/getVideoLabel.ts
View File

@@ -54,7 +54,6 @@ async function getLabelsByHash(hashedVideoIDPrefix: VideoIDHash, service: Servic
for (const [videoID, videoData] of Object.entries(segmentPerVideoID)) {
const data: VideoData = {
hash: videoData.hash,
segments: chooseSegment(videoData.segments),
};

1
src/routes/getVideoLabelByHash.ts
View File

@@ -20,7 +20,6 @@ export async function getVideoLabelsByHash(req: Request, res: Response): Promise
const output = Object.entries(segments).map(([videoID, data]) => ({
videoID,
hash: data.hash,
segments: data.segments,
}));
return res.status(output.length === 0 ? 404 : 200).json(output);

24
src/routes/oldGetVideoSponsorTimes.ts
View File

@@ -1,24 +0,0 @@
import { handleGetSegments } from "./getSkipSegments";
import { Request, Response } from "express";
export async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
const segments = await handleGetSegments(req, res);
if (segments) {
// Convert to old outputs
const sponsorTimes = [];
const UUIDs = [];
for (const segment of segments) {
sponsorTimes.push(segment.segment);
UUIDs.push(segment.UUID);
}
return res.send({
sponsorTimes,
UUIDs,
});
}
// Error has already been handled in the other method
}

22
src/routes/postSkipSegments.ts
View File

@@ -22,6 +22,7 @@ import axios from "axios";
import { vote } from "./voteOnSponsorTime";
import { canSubmit } from "../utils/permissions";
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
import * as youtubeID from "../utils/youtubeID";
type CheckResult = {
pass: boolean,
@@ -185,15 +186,23 @@ async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
}
async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID: HashedUserID
, segments: IncomingSegment[], videoDurationParam: number, userAgent: string): Promise<CheckResult> {
, segments: IncomingSegment[], videoDurationParam: number, userAgent: string, service: Service): Promise<CheckResult> {
const invalidFields = [];
const errors = [];
if (typeof videoID !== "string" || videoID?.length == 0) {
invalidFields.push("videoID");
}
if (typeof userID !== "string" || userID?.length < 30) {
if (service === Service.YouTube && config.mode !== "test") {
const sanitizedVideoID = youtubeID.validate(videoID) ? videoID : youtubeID.sanitize(videoID);
if (!youtubeID.validate(sanitizedVideoID)) {
invalidFields.push("videoID");
errors.push("YouTube videoID could not be extracted");
}
}
const minLength = config.minUserIDLength;
if (typeof userID !== "string" || userID?.length < minLength) {
invalidFields.push("userID");
if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`);
if (userID?.length < minLength) errors.push(`userID must be at least ${minLength} characters long`);
}
if (!Array.isArray(segments) || segments.length == 0) {
invalidFields.push("segments");
@@ -335,7 +344,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
return {
pass: false,
errorCode: 403,
errorMessage: `Hi, currently there are server issues and you might have not recieved segments even though they exist. Sorry about this, I'm working on it. Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
errorMessage: `Submissions rejected: ${autoModerateResult} If this is an issue, send a message on Discord.`
};
}
}
@@ -484,7 +493,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//hash the userID
const userID = await getHashCache(paramUserID || "");
const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent);
const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent, service);
if (!invalidCheckResult.pass) {
return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage);
}
@@ -546,7 +555,8 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
//also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.category, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const UUID = getSubmissionUUID(videoID, segmentInfo.category, segmentInfo.actionType,
segmentInfo.description, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0;

15
src/routes/setUsername.ts
View File

@@ -32,6 +32,11 @@ export async function setUsername(req: Request, res: Response): Promise<Response
// eslint-disable-next-line no-control-regex
userName = userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, "");
// check privateID against publicID
if (!await checkPrivateUsername(userName, userID)) {
return res.sendStatus(400);
}
if (adminUserIDInput != undefined) {
//this is the admin controlling the other users account, don't hash the controling account's ID
adminUserIDInput = await getHashCache(adminUserIDInput);
@@ -88,3 +93,13 @@ export async function setUsername(req: Request, res: Response): Promise<Response
return res.sendStatus(500);
}
}
async function checkPrivateUsername(username: string, userID: string): Promise<boolean> {
const userIDHash = await getHashCache(userID);
const userNameHash = await getHashCache(username);
if (userIDHash == userNameHash) return false;
const sponsorTimeRow = await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ? LIMIT 1`, [userNameHash]);
const userNameRow = await db.prepare("get", `SELECT "userID" FROM "userNames" WHERE "userID" = ? LIMIT 1`, [userNameHash]);
if ((sponsorTimeRow || userNameRow)?.userID) return false;
return true;
}

4
src/routes/viewedVideoSponsorTime.ts
View File

@@ -2,9 +2,9 @@ import { db } from "../databases/databases";
import { Request, Response } from "express";
export async function viewedVideoSponsorTime(req: Request, res: Response): Promise<Response> {
const UUID = req.query.UUID;
const UUID = req.query?.UUID;
if (UUID == undefined) {
if (!UUID) {
//invalid request
return res.sendStatus(400);
}

2
src/routes/voteOnSponsorTime.ts
View File

@@ -325,7 +325,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
return { status: 400 };
}
// Ignore this vote, invalid
if (paramUserID.length < 30 && config.mode !== "test") {
if (paramUserID.length < config.minUserIDLength) {
return { status: 200 };
}