Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-14 07:27:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into clickbait

Browse Source
This commit is contained in:
Ajay Ramachandran
2023-01-28 02:13:42 -05:00
committed by GitHub
parent b91114165e 894cd48182
commit d07aac3a33
39 changed files with 1207 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/app.ts
View File

@@ -1,7 +1,6 @@
import express, { Request, RequestHandler, Response, Router } from "express";
import { config } from "./config";
import { oldSubmitSponsorTimes } from "./routes/oldSubmitSponsorTimes";
import { oldGetVideoSponsorTimes } from "./routes/oldGetVideoSponsorTimes";
import { postSegmentShift } from "./routes/postSegmentShift";
import { postWarning } from "./routes/postWarning";
import { getIsUserVIP } from "./routes/getIsUserVIP";
@@ -21,13 +20,13 @@ import { viewedVideoSponsorTime } from "./routes/viewedVideoSponsorTime";
import { voteOnSponsorTime, getUserID as voteGetUserID } from "./routes/voteOnSponsorTime";
import { getSkipSegmentsByHash } from "./routes/getSkipSegmentsByHash";
import { postSkipSegments } from "./routes/postSkipSegments";
import { endpoint as getSkipSegments } from "./routes/getSkipSegments";
import { getSkipSegments, oldGetVideoSponsorTimes } from "./routes/getSkipSegments";
import { userCounter } from "./middleware/userCounter";
import { loggerMiddleware } from "./middleware/logger";
import { corsMiddleware } from "./middleware/cors";
import { apiCspMiddleware } from "./middleware/apiCsp";
import { rateLimitMiddleware } from "./middleware/requestRateLimit";
import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
import dumpDatabase from "./routes/dumpDatabase";
import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
import { postClearCache } from "./routes/postClearCache";
import { addUnlistedVideo } from "./routes/addUnlistedVideo";
@@ -52,6 +51,7 @@ import { generateTokenRequest } from "./routes/generateToken";
import { verifyTokenRequest } from "./routes/verifyToken";
import { getBranding, getBrandingByHashEndpoint } from "./routes/getBranding";
import { postBranding } from "./routes/postBranding";
import { cacheMiddlware } from "./middleware/etag";
export function createServer(callback: () => void): Server {
// Create a service (the app object is just a callback).
@@ -59,11 +59,13 @@ export function createServer(callback: () => void): Server {
const router = ExpressPromiseRouter();
app.use(router);
app.set("etag", false); // disable built in etag
//setup CORS correctly
router.use(corsMiddleware);
router.use(loggerMiddleware);
router.use("/api/", apiCspMiddleware);
router.use(cacheMiddlware);
router.use(express.json());
if (config.userCounterURL) router.use(userCounter);

3
src/config.ts
View File

@@ -166,7 +166,8 @@ addDefaults(config, {
},
gumroad: {
productPermalinks: ["sponsorblock"]
}
},
minUserIDLength: 30
});
loadFromEnv(config);
migrate(config);

2
src/middleware/cors.ts
View File

@@ -3,6 +3,6 @@ import { NextFunction, Request, Response } from "express";
export function corsMiddleware(req: Request, res: Response, next: NextFunction): void {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE");
res.header("Access-Control-Allow-Headers", "Content-Type");
res.header("Access-Control-Allow-Headers", "Content-Type, If-None-Match");
next();
}

49
src/middleware/etag.ts Normal file
View File

@@ -0,0 +1,49 @@
import { NextFunction, Request, Response } from "express";
import { VideoID, VideoIDHash, Service } from "../types/segments.model";
import { QueryCacher } from "../utils/queryCacher";
import { skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKey } from "../utils/redisKeys";
type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash";
type ETag = `${hashType};${VideoIDHash};${Service};${number}`;
type hashKey = string | VideoID | VideoIDHash;
export function cacheMiddlware(req: Request, res: Response, next: NextFunction): void {
const reqEtag = req.get("If-None-Match") as string;
// if weak etag, do not handle
if (!reqEtag || reqEtag.startsWith("W/")) return next();
// split into components
const [hashType, hashKey, service, lastModified] = reqEtag.split(";");
// fetch last-modified
getLastModified(hashType as hashType, hashKey as VideoIDHash, service as Service)
.then(redisLastModified => {
if (redisLastModified <= new Date(Number(lastModified) + 1000)) {
// match cache, generate etag
const etag = `${hashType};${hashKey};${service};${redisLastModified.getTime()}` as ETag;
res.status(304).set("etag", etag).send();
}
else next();
})
.catch(next);
}
function getLastModified(hashType: hashType, hashKey: hashKey, service: Service): Promise<Date | null> {
let redisKey: string | null;
if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service);
else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service);
else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service);
else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service);
else return Promise.reject();
return QueryCacher.getKeyLastModified(redisKey);
}
export async function getEtag(hashType: hashType, hashKey: hashKey, service: Service): Promise<ETag> {
const lastModified = await getLastModified(hashType, hashKey, service);
return `${hashType};${hashKey};${service};${lastModified.getTime()}` as ETag;
}
/* example usage
import { getEtag } from "../middleware/etag";
await getEtag(hashType, hashPrefix, service)
.then(etag => res.set("ETag", etag))
.catch(() => null);
*/

157
src/routes/getSkipSegments.ts
View File

@@ -12,7 +12,8 @@ import { QueryCacher } from "../utils/queryCacher";
import { getReputation } from "../utils/reputation";
import { getService } from "../utils/getService";
import { promiseOrTimeout } from "../utils/promise";
import { parseSkipSegments } from "../utils/parseSkipSegments";
import { getEtag } from "../middleware/etag";
async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean): Promise<Segment[]> {
const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
@@ -86,9 +87,6 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
}
try {
categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
if (categories.length === 0) return null;
const segments: DBSegment[] = (await getSegmentsFromDBByVideoID(videoID, service))
.map((segment: DBSegment) => {
if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
@@ -97,7 +95,17 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
const canUseCache = requiredSegments.length === 0;
let processedSegments: Segment[] = (await prepareCategorySegments(req, videoID, service, segments, cache, canUseCache))
.filter((segment: Segment) => categories.includes(segment?.category) && (actionTypes.includes(segment?.actionType)));
.filter((segment: Segment) => categories.includes(segment?.category) && (actionTypes.includes(segment?.actionType)))
.map((segment: Segment) => ({
category: segment.category,
actionType: segment.actionType,
segment: segment.segment,
UUID: segment.UUID,
videoDuration: segment.videoDuration,
locked: segment.locked,
votes: segment.votes,
description: segment.description
}));
if (forcePoiAsSkip) {
processedSegments = processedSegments.map((segment) => ({
@@ -127,15 +135,11 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}
try {
type SegmentWithHashPerVideoID = SBRecord<VideoID, { hash: VideoIDHash, segments: DBSegment[] }>;
type SegmentPerVideoID = SBRecord<VideoID, { segments: DBSegment[] }>;
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null;
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
const segmentPerVideoID: SegmentPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.reduce((acc: SegmentPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
hash: segment.hashedVideoID,
segments: []
};
if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
@@ -148,13 +152,22 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
await Promise.all(Object.entries(segmentPerVideoID).map(async ([videoID, videoData]) => {
const data: VideoData = {
hash: videoData.hash,
segments: [],
};
const canUseCache = requiredSegments.length === 0;
data.segments = (await prepareCategorySegments(req, videoID as VideoID, service, videoData.segments, cache, canUseCache))
.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType));
.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType))
.map((segment) => ({
category: segment.category,
actionType: segment.actionType,
segment: segment.segment,
UUID: segment.UUID,
videoDuration: segment.videoDuration,
locked: segment.locked,
votes: segment.votes,
description: segment.description
}));
if (forcePoiAsSkip) {
data.segments = data.segments.map((segment) => ({
@@ -378,75 +391,59 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm
});
}
/**
*
* Returns what would be sent to the client.
* Will respond with errors if required. Returns false if it errors.
*
* @param req
* @param res
*
* @returns
*/
async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
async function getSkipSegments(req: Request, res: Response): Promise<Response> {
const videoID = req.query.videoID as VideoID;
if (!videoID) {
res.status(400).send("videoID not specified");
return false;
}
// Default to sponsor
// If using params instead of JSON, only one category can be pulled
const categories: Category[] = req.query.categories
? JSON.parse(req.query.categories as string)
: req.query.category
? Array.isArray(req.query.category)
? req.query.category
: [req.query.category]
: ["sponsor"];
if (!Array.isArray(categories)) {
res.status(400).send("Categories parameter does not match format requirements.");
return false;
return res.status(400).send("videoID not specified");
}
const actionTypes: ActionType[] = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip];
if (!Array.isArray(actionTypes)) {
res.status(400).send("actionTypes parameter does not match format requirements.");
return false;
const parseResult = parseSkipSegments(req);
if (parseResult.errors.length > 0) {
return res.status(400).send(parseResult.errors);
}
const requiredSegments: SegmentUUID[] = req.query.requiredSegments
? JSON.parse(req.query.requiredSegments as string)
: req.query.requiredSegment
? Array.isArray(req.query.requiredSegment)
? req.query.requiredSegment
: [req.query.requiredSegment]
: [];
if (!Array.isArray(requiredSegments)) {
res.status(400).send("requiredSegments parameter does not match format requirements.");
return false;
}
const service = getService(req.query.service, req.body.service);
const { categories, actionTypes, requiredSegments, service } = parseResult;
const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
if (segments === null || segments === undefined) {
res.sendStatus(500);
return false;
return res.sendStatus(500);
} else if (segments.length === 0) {
return res.sendStatus(404);
}
if (segments.length === 0) {
res.sendStatus(404);
return false;
await getEtag("skipSegments", (videoID as string), service)
.then(etag => res.set("ETag", etag))
.catch(() => null);
return res.send(segments);
}
async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
const videoID = req.query.videoID as VideoID;
if (!videoID) {
return res.status(400).send("videoID not specified");
}
return segments;
const segments = await getSegmentsByVideoID(req, videoID, ["sponsor"] as Category[], [ActionType.Skip], [], Service.YouTube);
if (segments === null || segments === undefined) {
return res.sendStatus(500);
} else if (segments.length === 0) {
return res.sendStatus(404);
}
// Convert to old outputs
const sponsorTimes = [];
const UUIDs = [];
for (const segment of segments) {
sponsorTimes.push(segment.segment);
UUIDs.push(segment.UUID);
}
return res.send({
sponsorTimes,
UUIDs,
});
}
const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID[]): boolean => {
@@ -456,25 +453,9 @@ const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID
return false;
};
async function endpoint(req: Request, res: Response): Promise<Response> {
try {
const segments = await handleGetSegments(req, res);
// If false, res.send has already been called
if (segments) {
//send result
return res.send(segments);
}
} catch (err) /* istanbul ignore next */ {
if (err instanceof SyntaxError) {
return res.status(400).send("Categories parameter does not match format requirements.");
} else return res.sendStatus(500);
}
}
export {
getSegmentsByVideoID,
getSegmentsByHash,
endpoint,
handleGetSegments
getSkipSegments,
oldGetVideoSponsorTimes
};

64
src/routes/getSkipSegmentsByHash.ts
View File

@@ -1,9 +1,10 @@
import { hashPrefixTester } from "../utils/hashPrefixTester";
import { getSegmentsByHash } from "./getSkipSegments";
import { Request, Response } from "express";
import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model";
import { getService } from "../utils/getService";
import { VideoIDHash } from "../types/segments.model";
import { Logger } from "../utils/logger";
import { parseSkipSegments } from "../utils/parseSkipSegments";
import { getEtag } from "../middleware/etag";
export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
let hashPrefix = req.params.prefix as VideoIDHash;
@@ -12,66 +13,21 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
}
hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
let categories: Category[] = [];
try {
categories = req.query.categories
? JSON.parse(req.query.categories as string)
: req.query.category
? Array.isArray(req.query.category)
? req.query.category
: [req.query.category]
: ["sponsor"];
if (!Array.isArray(categories)) {
return res.status(400).send("Categories parameter does not match format requirements.");
}
} catch(error) {
return res.status(400).send("Bad parameter: categories (invalid JSON)");
const parseResult = parseSkipSegments(req);
if (parseResult.errors.length > 0) {
return res.status(400).send(parseResult.errors);
}
let actionTypes: ActionType[] = [];
try {
actionTypes = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip];
if (!Array.isArray(actionTypes)) {
return res.status(400).send("actionTypes parameter does not match format requirements.");
}
} catch(error) {
return res.status(400).send("Bad parameter: actionTypes (invalid JSON)");
}
let requiredSegments: SegmentUUID[] = [];
try {
requiredSegments = req.query.requiredSegments
? JSON.parse(req.query.requiredSegments as string)
: req.query.requiredSegment
? Array.isArray(req.query.requiredSegment)
? req.query.requiredSegment
: [req.query.requiredSegment]
: [];
if (!Array.isArray(requiredSegments)) {
return res.status(400).send("requiredSegments parameter does not match format requirements.");
}
} catch(error) {
return res.status(400).send("Bad parameter: requiredSegments (invalid JSON)");
}
const service: Service = getService(req.query.service, req.body.service);
// filter out none string elements, only flat array with strings is valid
categories = categories.filter((item: any) => typeof item === "string");
const { categories, actionTypes, requiredSegments, service } = parseResult;
// Get all video id's that match hash prefix
const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service);
try {
await getEtag("skipSegmentsHash", hashPrefix, service)
.then(etag => res.set("ETag", etag))
.catch(() => null);
const output = Object.entries(segments).map(([videoID, data]) => ({
videoID,
hash: data.hash,
segments: data.segments,
}));
return res.status(output.length === 0 ? 404 : 200).json(output);

1
src/routes/getVideoLabel.ts
View File

@@ -54,7 +54,6 @@ async function getLabelsByHash(hashedVideoIDPrefix: VideoIDHash, service: Servic
for (const [videoID, videoData] of Object.entries(segmentPerVideoID)) {
const data: VideoData = {
hash: videoData.hash,
segments: chooseSegment(videoData.segments),
};

1
src/routes/getVideoLabelByHash.ts
View File

@@ -20,7 +20,6 @@ export async function getVideoLabelsByHash(req: Request, res: Response): Promise
const output = Object.entries(segments).map(([videoID, data]) => ({
videoID,
hash: data.hash,
segments: data.segments,
}));
return res.status(output.length === 0 ? 404 : 200).json(output);

24
src/routes/oldGetVideoSponsorTimes.ts
View File

@@ -1,24 +0,0 @@
import { handleGetSegments } from "./getSkipSegments";
import { Request, Response } from "express";
export async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
const segments = await handleGetSegments(req, res);
if (segments) {
// Convert to old outputs
const sponsorTimes = [];
const UUIDs = [];
for (const segment of segments) {
sponsorTimes.push(segment.segment);
UUIDs.push(segment.UUID);
}
return res.send({
sponsorTimes,
UUIDs,
});
}
// Error has already been handled in the other method
}

22
src/routes/postSkipSegments.ts
View File

@@ -22,6 +22,7 @@ import axios from "axios";
import { vote } from "./voteOnSponsorTime";
import { canSubmit } from "../utils/permissions";
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
import * as youtubeID from "../utils/youtubeID";
type CheckResult = {
pass: boolean,
@@ -185,15 +186,23 @@ async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
}
async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID: HashedUserID
, segments: IncomingSegment[], videoDurationParam: number, userAgent: string): Promise<CheckResult> {
, segments: IncomingSegment[], videoDurationParam: number, userAgent: string, service: Service): Promise<CheckResult> {
const invalidFields = [];
const errors = [];
if (typeof videoID !== "string" || videoID?.length == 0) {
invalidFields.push("videoID");
}
if (typeof userID !== "string" || userID?.length < 30) {
if (service === Service.YouTube && config.mode !== "test") {
const sanitizedVideoID = youtubeID.validate(videoID) ? videoID : youtubeID.sanitize(videoID);
if (!youtubeID.validate(sanitizedVideoID)) {
invalidFields.push("videoID");
errors.push("YouTube videoID could not be extracted");
}
}
const minLength = config.minUserIDLength;
if (typeof userID !== "string" || userID?.length < minLength) {
invalidFields.push("userID");
if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`);
if (userID?.length < minLength) errors.push(`userID must be at least ${minLength} characters long`);
}
if (!Array.isArray(segments) || segments.length == 0) {
invalidFields.push("segments");
@@ -335,7 +344,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
return {
pass: false,
errorCode: 403,
errorMessage: `Hi, currently there are server issues and you might have not recieved segments even though they exist. Sorry about this, I'm working on it. Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
errorMessage: `Submissions rejected: ${autoModerateResult} If this is an issue, send a message on Discord.`
};
}
}
@@ -484,7 +493,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//hash the userID
const userID = await getHashCache(paramUserID || "");
const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent);
const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent, service);
if (!invalidCheckResult.pass) {
return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage);
}
@@ -546,7 +555,8 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
//also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.category, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const UUID = getSubmissionUUID(videoID, segmentInfo.category, segmentInfo.actionType,
segmentInfo.description, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0;

15
src/routes/setUsername.ts
View File

@@ -32,6 +32,11 @@ export async function setUsername(req: Request, res: Response): Promise<Response
// eslint-disable-next-line no-control-regex
userName = userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, "");
// check privateID against publicID
if (!await checkPrivateUsername(userName, userID)) {
return res.sendStatus(400);
}
if (adminUserIDInput != undefined) {
//this is the admin controlling the other users account, don't hash the controling account's ID
adminUserIDInput = await getHashCache(adminUserIDInput);
@@ -88,3 +93,13 @@ export async function setUsername(req: Request, res: Response): Promise<Response
return res.sendStatus(500);
}
}
async function checkPrivateUsername(username: string, userID: string): Promise<boolean> {
const userIDHash = await getHashCache(userID);
const userNameHash = await getHashCache(username);
if (userIDHash == userNameHash) return false;
const sponsorTimeRow = await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ? LIMIT 1`, [userNameHash]);
const userNameRow = await db.prepare("get", `SELECT "userID" FROM "userNames" WHERE "userID" = ? LIMIT 1`, [userNameHash]);
if ((sponsorTimeRow || userNameRow)?.userID) return false;
return true;
}

4
src/routes/viewedVideoSponsorTime.ts
View File

@@ -2,9 +2,9 @@ import { db } from "../databases/databases";
import { Request, Response } from "express";
export async function viewedVideoSponsorTime(req: Request, res: Response): Promise<Response> {
const UUID = req.query.UUID;
const UUID = req.query?.UUID;
if (UUID == undefined) {
if (!UUID) {
//invalid request
return res.sendStatus(400);
}

2
src/routes/voteOnSponsorTime.ts
View File

@@ -325,7 +325,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
return { status: 400 };
}
// Ignore this vote, invalid
if (paramUserID.length < 30 && config.mode !== "test") {
if (paramUserID.length < config.minUserIDLength) {
return { status: 200 };
}

3
src/types/config.model.ts
View File

@@ -94,7 +94,8 @@ export interface SBSConfig {
}
gumroad: {
productPermalinks: string[],
}
},
minUserIDLength: number
}
export interface WebhookConfig {

4
src/types/segments.model.ts
View File

@@ -45,6 +45,9 @@ export interface Segment {
segment: number[];
UUID: SegmentUUID;
videoDuration: VideoDuration;
locked: boolean;
votes: number;
description: string;
}
export enum Visibility {
@@ -94,7 +97,6 @@ export interface VotableObjectWithWeight extends VotableObject {
}
export interface VideoData {
hash: VideoIDHash;
segments: Segment[];
}

3
src/utils/getSubmissionUUID.ts
View File

@@ -7,10 +7,11 @@ export function getSubmissionUUID(
videoID: VideoID,
category: Category,
actionType: ActionType,
description: string,
userID: UserID,
startTime: number,
endTime: number,
service: Service
) : HashedValue {
return `${getHash(`${videoID}${startTime}${endTime}${userID}${category}${actionType}${service}`, 1)}6` as HashedValue;
return `${getHash(`${videoID}${startTime}${endTime}${userID}${description}${category}${actionType}${service}`, 1)}7` as HashedValue;
}

2
src/utils/innerTubeAPI.ts
View File

@@ -34,7 +34,7 @@ async function getFromITube (videoID: string): Promise<innerTubeVideoDetails> {
context: {
client: {
clientName: "WEB",
clientVersion: "2.20211129.09.00"
clientVersion: "2.20221215.04.01"
}
},
videoId: videoID

75
src/utils/parseSkipSegments.ts Normal file
View File

@@ -0,0 +1,75 @@
import { Request } from "express";
import { ActionType, SegmentUUID, Category, Service } from "../types/segments.model";
import { getService } from "./getService";
type fn = (req: Request) => any[];
const syntaxErrorWrapper = (fn: fn, req: Request) => {
try { return fn(req); }
catch (e) { return undefined; }
};
// Default to sponsor
const getCategories = (req: Request): Category[] =>
req.query.categories
? JSON.parse(req.query.categories as string)
: req.query.category
? Array.isArray(req.query.category)
? req.query.category
: [req.query.category]
: ["sponsor"];
// Default to skip
const getActionTypes = (req: Request): ActionType[] =>
req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip];
// Default to empty array
const getRequiredSegments = (req: Request): SegmentUUID[] =>
req.query.requiredSegments
? JSON.parse(req.query.requiredSegments as string)
: req.query.requiredSegment
? Array.isArray(req.query.requiredSegment)
? req.query.requiredSegment
: [req.query.requiredSegment]
: [];
const errorMessage = (parameter: string) => `${parameter} parameter does not match format requirements.`;
export function parseSkipSegments(req: Request): {
categories: Category[];
actionTypes: ActionType[];
requiredSegments: SegmentUUID[];
service: Service;
errors: string[];
} {
let categories: Category[] = syntaxErrorWrapper(getCategories, req);
const actionTypes: ActionType[] = syntaxErrorWrapper(getActionTypes, req);
const requiredSegments: SegmentUUID[] = syntaxErrorWrapper(getRequiredSegments, req);
const service: Service = getService(req.query.service, req.body.services);
const errors: string[] = [];
if (!Array.isArray(categories)) errors.push(errorMessage("categories"));
else {
// check category names for invalid characters
// and none string elements
categories = categories
.filter((item: any) => typeof item === "string")
.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) errors.push("No valid categories provided.");
}
if (!Array.isArray(actionTypes)) errors.push(errorMessage("actionTypes"));
if (!Array.isArray(requiredSegments)) errors.push(errorMessage("requiredSegments"));
// finished parsing
return {
categories,
actionTypes,
requiredSegments,
service,
errors
};
}

14
src/utils/queryCacher.ts
View File

@@ -87,6 +87,17 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID
}
}
async function getKeyLastModified(key: string): Promise<Date> {
if (!config.redis?.enabled) return Promise.reject("ETag - Redis not enabled");
return await redis.ttl(key)
.then(ttl => {
const sinceLive = config.redis?.expiryTime - ttl;
const now = Math.floor(Date.now() / 1000);
return new Date((now-sinceLive) * 1000);
})
.catch(() => Promise.reject("ETag - Redis error"));
}
function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
if (videoInfo) {
redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
@@ -101,6 +112,7 @@ export const QueryCacher = {
get,
getAndSplit,
clearSegmentCache,
getKeyLastModified,
clearRatingCache,
clearFeatureCache
clearFeatureCache,
};

2
src/utils/redis.ts
View File

@@ -19,6 +19,7 @@ interface RedisSB {
del(...keys: [RedisCommandArgument]): Promise<number>;
increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
ttl(key: RedisCommandArgument): Promise<number>;
quit(): Promise<void>;
}
@@ -30,6 +31,7 @@ let exportClient: RedisSB = {
increment: () => new Promise((resolve) => resolve(null)),
sendCommand: () => new Promise((resolve) => resolve(null)),
quit: () => new Promise((resolve) => resolve(null)),
ttl: () => new Promise((resolve) => resolve(null)),
};
let lastClientFail = 0;

26
src/utils/youtubeID.ts Normal file
View File

@@ -0,0 +1,26 @@
import { VideoID } from "../types/segments.model";
const idRegex = new RegExp(/([0-9A-Za-z_-]{11})/); // group to always be index 1
const exclusiveIdegex = new RegExp(`^${idRegex.source}$`);
// match /c/, /channel/, /@channel, full UUIDs
const negativeRegex = new RegExp(/(\/(channel|c)\/.+)|(\/@.+)|([a-f0-9]{64,65})|(youtube\.com\/clip\/)/);
const urlRegex = new RegExp(`(?:v=|/|youtu.be/)${idRegex.source}(?:|/|[?&]t=\\d+s?)>?(?:\\s|$)`);
const negateIdRegex = new RegExp(/(?:[^0-9A-Za-z_-]*?)/);
const looseEndsRegex = new RegExp(`${negateIdRegex.source}${idRegex.source}${negateIdRegex.source}`);
export const validate = (id: string): boolean => exclusiveIdegex.test(id);
export const sanitize = (id: string): VideoID | null => {
// first decode URI
id = decodeURIComponent(id);
// strict matching
const strictMatch = id.match(exclusiveIdegex)?.[1];
const urlMatch = id.match(urlRegex)?.[1];
// return match, if not negative, return looseMatch
const looseMatch = id.match(looseEndsRegex)?.[1];
return strictMatch ? (strictMatch as VideoID)
: negativeRegex.test(id) ? null
: urlMatch ? (urlMatch as VideoID)
: looseMatch ? (looseMatch as VideoID)
: null;
};