Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-25 00:48:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into fullVideoLabels

Browse Source
This commit is contained in:
Michael C
2022-10-28 16:49:50 -04:00
parent cc6c5980a1 9d7236282c
commit 80c018c9d5
78 changed files with 2035 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/app.ts
View File

@@ -82,7 +82,7 @@ function setupRoutes(router: Router) {
// Rate limit endpoint lists
const voteEndpoints: RequestHandler[] = [voteOnSponsorTime];
const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime];
if (config.rateLimit) {
if (config.rateLimit && config.redisRateLimit) {
if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID));
if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
}
@@ -206,6 +206,7 @@ function setupRoutes(router: Router) {
router.get("/api/videoLabels", getVideoLabels);
router.get("/api/videoLabels/:prefix", getVideoLabelsByHash);
/* istanbul ignore next */
if (config.postgres?.enabled) {
router.get("/database", (req, res) => dumpDatabase(req, res, true));
router.get("/database.json", (req, res) => dumpDatabase(req, res, false));

15
src/config.ts
View File

@@ -76,8 +76,7 @@ addDefaults(config, {
port: 5432,
max: 10,
idleTimeoutMillis: 10000,
maxTries: 3,
maxConcurrentRequests: 3500
maxTries: 3
},
postgresReadOnly: {
enabled: false,
@@ -91,7 +90,7 @@ addDefaults(config, {
idleTimeoutMillis: 10000,
maxTries: 3,
fallbackOnFail: true,
maxConcurrentRequests: 3500
stopRetryThreshold: 800
},
dumpDatabase: {
enabled: false,
@@ -139,6 +138,16 @@ addDefaults(config, {
expiryTime: 24 * 60 * 60,
getTimeout: 40
},
redisRead: {
enabled: false,
socket: {
host: "",
port: 0
},
disableOfflineQueue: true,
weight: 1
},
redisRateLimit: true,
patreon: {
clientId: "",
clientSecret: "",

77
src/databases/Postgres.ts
View File

@@ -33,8 +33,7 @@ export class Postgres implements IDatabase {
private poolRead: Pool;
private lastPoolReadFail = 0;
private concurrentRequests = 0;
private concurrentReadRequests = 0;
activePostgresRequests = 0;
constructor(private config: DatabaseConfig) {}
@@ -54,19 +53,23 @@ export class Postgres implements IDatabase {
});
if (this.config.postgresReadOnly && this.config.postgresReadOnly.enabled) {
this.poolRead = new Pool({
...this.config.postgresReadOnly
});
this.poolRead.on("error", (err, client) => {
Logger.error(err.stack);
this.lastPoolReadFail = Date.now();
try {
this.poolRead = new Pool({
...this.config.postgresReadOnly
});
this.poolRead.on("error", (err, client) => {
Logger.error(err.stack);
this.lastPoolReadFail = Date.now();
try {
client.release(true);
} catch (err) {
Logger.error(`poolRead (postgres): ${err}`);
}
});
try {
client.release(true);
} catch (err) {
Logger.error(`poolRead (postgres): ${err}`);
}
});
} catch (e) {
Logger.error(`poolRead (postgres): ${e}`);
}
}
if (!this.config.readOnly) {
@@ -102,22 +105,6 @@ export class Postgres implements IDatabase {
Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
if (this.config.readOnly) {
if (this.concurrentReadRequests > this.config.postgresReadOnly?.maxConcurrentRequests) {
Logger.error(`prepare (postgres): cancelling read query because too many concurrent requests, query: ${query}`);
throw new Error("Too many concurrent requests");
}
this.concurrentReadRequests++;
} else {
if (this.concurrentRequests > this.config.postgres.maxConcurrentRequests) {
Logger.error(`prepare (postgres): cancelling query because too many concurrent requests, query: ${query}`);
throw new Error("Too many concurrent requests");
}
this.concurrentRequests++;
}
const pendingQueries: PromiseWithState<QueryResult<any>>[] = [];
let tries = 0;
let lastPool: Pool = null;
@@ -127,6 +114,7 @@ export class Postgres implements IDatabase {
tries++;
try {
this.activePostgresRequests++;
lastPool = this.getPool(type, options);
pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params })));
@@ -134,12 +122,7 @@ export class Postgres implements IDatabase {
if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout)));
const queryResult = await nextFulfilment(currentPromises);
if (this.config.readOnly) {
this.concurrentReadRequests--;
} else {
this.concurrentRequests--;
}
this.activePostgresRequests--;
switch (type) {
case "get": {
const value = queryResult.rows[0];
@@ -159,30 +142,30 @@ export class Postgres implements IDatabase {
if (lastPool === this.pool) {
// Only applies if it is get or all request
options.forceReplica = true;
} else if (lastPool === this.poolRead && maxTries() - tries <= 1) {
options.useReplica = false;
} else if (lastPool === this.poolRead) {
this.lastPoolReadFail = Date.now();
if (maxTries() - tries <= 1) {
options.useReplica = false;
}
}
Logger.error(`prepare (postgres) try ${tries}: ${err}`);
}
} while (this.isReadQuery(type) && tries < maxTries());
if (this.config.readOnly) {
this.concurrentReadRequests--;
} else {
this.concurrentRequests--;
}
} while (this.isReadQuery(type) && tries < maxTries()
&& this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold);
this.activePostgresRequests--;
throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`);
}
private getPool(type: string, options: QueryOption): Pool {
const readAvailable = this.poolRead && options.useReplica && this.isReadQuery(type);
const ignroreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
const ignoreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
&& this.lastPoolReadFail > Date.now() - 1000 * 30;
const readDueToFailure = this.config.postgresReadOnly.fallbackOnFail
&& this.lastPoolFail > Date.now() - 1000 * 30;
if (readAvailable && !ignroreReadDueToFailure && (options.forceReplica || readDueToFailure ||
if (readAvailable && !ignoreReadDueToFailure && (options.forceReplica || readDueToFailure ||
Math.random() > 1 / (this.config.postgresReadOnly.weight + 1))) {
return this.poolRead;
} else {

10
src/index.ts
View File

@@ -12,7 +12,13 @@ async function init() {
process.exit(1);
});
await initDb();
try {
await initDb();
} catch (e) {
Logger.error(`Init Db: ${e}`);
process.exit(1);
}
// edge case clause for creating compatible .db files, do not enable
if (config.mode === "init-db-and-exit") process.exit(0);
// do not enable init-db-only mode for usage.
@@ -27,4 +33,4 @@ async function init() {
}).setTimeout(15000);
}
init().catch((err) => Logger.error(err));
init().catch((err) => Logger.error(`Index.js: ${err}`));

3
src/routes/deleteLockCategories.ts
View File

@@ -35,6 +35,7 @@ export async function deleteLockCategoriesEndpoint(req: DeleteLockCategoriesRequ
|| !categories
|| !Array.isArray(categories)
|| categories.length === 0
|| actionTypes && !Array.isArray(actionTypes)
|| actionTypes.length === 0
) {
return res.status(400).json({
@@ -48,7 +49,7 @@ export async function deleteLockCategoriesEndpoint(req: DeleteLockCategoriesRequ
if (!userIsVIP) {
return res.status(403).json({
message: "Must be a VIP to mark videos.",
message: "Must be a VIP to lock videos.",
});
}

8
src/routes/generateToken.ts
View File

@@ -1,7 +1,7 @@
import { Request, Response } from "express";
import { config } from "../config";
import { createAndSaveToken, TokenType } from "../utils/tokenUtils";
import { getHashCache } from "../utils/getHashCache";
interface GenerateTokenRequest extends Request {
query: {
@@ -15,14 +15,16 @@ interface GenerateTokenRequest extends Request {
export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise<Response> {
const { query: { code, adminUserID }, params: { type } } = req;
const adminUserIDHash = adminUserID ? (await getHashCache(adminUserID)) : null;
if (!code || !type) {
return res.status(400).send("Invalid request");
}
if (type === TokenType.patreon || (type === TokenType.local && adminUserID === config.adminUserID)) {
if (type === TokenType.patreon || (type === TokenType.local && adminUserIDHash === config.adminUserID)) {
const licenseKey = await createAndSaveToken(type, code);
/* istanbul ignore else */
if (licenseKey) {
return res.status(200).send(`
<h1>
@@ -44,5 +46,7 @@ export async function generateTokenRequest(req: GenerateTokenRequest, res: Respo
</h1>
`);
}
} else {
return res.sendStatus(403);
}
}

6
src/routes/getDaysSavedFormatted.ts
View File

@@ -7,7 +7,11 @@ export async function getDaysSavedFormatted(req: Request, res: Response): Promis
if (row !== undefined) {
//send this result
return res.send({
daysSaved: row.daysSaved.toFixed(2),
daysSaved: row.daysSaved?.toFixed(2) ?? "0",
});
} else {
return res.send({
daysSaved: 0
});
}
}

2
src/routes/getIsUserVIP.ts
View File

@@ -21,7 +21,7 @@ export async function getIsUserVIP(req: Request, res: Response): Promise<Respons
hashedUserID: hashedUserID,
vip: vipState,
});
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/getLockCategories.ts
View File

@@ -33,7 +33,7 @@ export async function getLockCategories(req: Request, res: Response): Promise<Re
categories,
actionTypes
});
} catch (err) {
} catch (err) /* istanbul ignore next */{
Logger.error(err as string);
return res.sendStatus(500);
}

27
src/routes/getLockCategoriesByHash.ts
View File

@@ -44,14 +44,25 @@ const mergeLocks = (source: DBLock[], actionTypes: ActionType[]): LockResultByHa
export async function getLockCategoriesByHash(req: Request, res: Response): Promise<Response> {
let hashPrefix = req.params.prefix as VideoIDHash;
const actionTypes: ActionType[] = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip, ActionType.Mute];
let actionTypes: ActionType[] = [];
try {
actionTypes = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip, ActionType.Mute];
if (!Array.isArray(actionTypes)) {
//invalid request
return res.sendStatus(400);
}
} catch (err) {
//invalid request
return res.status(400).send("Invalid request: JSON parse error (actionTypes)");
}
if (!hashPrefixTester(req.params.prefix)) {
return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
}
hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
@@ -62,7 +73,7 @@ export async function getLockCategoriesByHash(req: Request, res: Response): Prom
if (lockedRows.length === 0 || !lockedRows[0]) return res.sendStatus(404);
// merge all locks
return res.send(mergeLocks(lockedRows, actionTypes));
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

35
src/routes/getLockReason.ts
View File

@@ -32,18 +32,24 @@ export async function getLockReason(req: Request, res: Response): Promise<Respon
return res.status(400).send("No videoID provided");
}
let categories: Category[] = [];
const actionTypes: ActionType[] = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip, ActionType.Mute];
const possibleCategories = filterActionType(actionTypes);
if (!Array.isArray(actionTypes)) {
//invalid request
return res.status(400).send("actionTypes parameter does not match format requirements");
let actionTypes: ActionType[] = [];
try {
actionTypes = req.query.actionTypes
? JSON.parse(req.query.actionTypes as string)
: req.query.actionType
? Array.isArray(req.query.actionType)
? req.query.actionType
: [req.query.actionType]
: [ActionType.Skip, ActionType.Mute];
if (!Array.isArray(actionTypes)) {
//invalid request
return res.status(400).send("actionTypes parameter does not match format requirements");
}
} catch (error) {
return res.status(400).send("Bad parameter: actionTypes (invalid JSON)");
}
const possibleCategories = filterActionType(actionTypes);
try {
categories = req.query.categories
? JSON.parse(req.query.categories as string)
@@ -64,11 +70,6 @@ export async function getLockReason(req: Request, res: Response): Promise<Respon
: categories.filter(x =>
possibleCategories.includes(x));
if (!videoID || !Array.isArray(actionTypes)) {
//invalid request
return res.sendStatus(400);
}
try {
// Get existing lock categories markers
const row = await db.prepare("all", 'SELECT "category", "reason", "actionType", "userID" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string, actionType: ActionType, userID: string }[];
@@ -115,7 +116,7 @@ export async function getLockReason(req: Request, res: Response): Promise<Respon
}
return res.send(results);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/getSavedTimeForUser.ts
View File

@@ -27,7 +27,7 @@ export async function getSavedTimeForUser(req: Request, res: Response): Promise<
} else {
return res.sendStatus(404);
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(`getSavedTimeForUser ${err}`);
return res.sendStatus(500);
}

11
src/routes/getSearchSegments.ts
View File

@@ -14,7 +14,7 @@ type searchSegmentResponse = {
function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
return db.prepare(
"all",
`SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID" FROM "sponsorTimes"
`SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes"
WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`,
[videoID, service]
) as Promise<DBSegment[]>;
@@ -128,12 +128,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<searchSeg
const segments = await getSegmentsFromDBByVideoID(videoID, service);
if (segments === null || segments === undefined) {
res.sendStatus(500);
return false;
}
if (segments.length === 0) {
if (!segments?.length) {
res.sendStatus(404);
return false;
}
@@ -155,6 +150,7 @@ function filterSegments(segments: DBSegment[], filters: Record<string, any>, pag
);
if (sortBy !== SortableFields.timeSubmitted) {
/* istanbul ignore next */
filteredSegments.sort((a,b) => {
const key = sortDir === "desc" ? 1 : -1;
if (a[sortBy] < b[sortBy]) {
@@ -187,6 +183,7 @@ async function endpoint(req: Request, res: Response): Promise<Response> {
return res.send(segmentResponse);
}
} catch (err) {
/* istanbul ignore next */
if (err instanceof SyntaxError) {
return res.status(400).send("Invalid array in parameters");
} else return res.sendStatus(500);

4
src/routes/getSegmentInfo.ts
View File

@@ -7,7 +7,7 @@ const isValidSegmentUUID = (str: string): boolean => /^([a-f0-9]{64}|[a-f0-9]{8}
async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise<DBSegment> {
try {
return await db.prepare("get", `SELECT * FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return null;
}
}
@@ -62,7 +62,7 @@ async function endpoint(req: Request, res: Response): Promise<Response> {
//send result
return res.send(DBSegments);
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
if (err instanceof SyntaxError) { // catch JSON.parse error
return res.status(400).send("UUIDs parameter does not match format requirements.");
} else return res.sendStatus(500);

6
src/routes/getSkipSegments.ts
View File

@@ -107,7 +107,7 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
}
return processedSegments;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
if (err) {
Logger.error(err as string);
return null;
@@ -169,7 +169,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}));
return segments;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return null;
}
@@ -465,7 +465,7 @@ async function endpoint(req: Request, res: Response): Promise<Response> {
//send result
return res.send(segments);
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
if (err instanceof SyntaxError) {
return res.status(400).send("Categories parameter does not match format requirements.");
} else return res.sendStatus(500);

2
src/routes/getSkipSegmentsByHash.ts
View File

@@ -67,8 +67,6 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
// Get all video id's that match hash prefix
const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service);
if (!segments) return res.status(404).json([]);
const output = Object.entries(segments).map(([videoID, data]) => ({
videoID,
hash: data.hash,

21
src/routes/getStatus.ts
View File

@@ -2,8 +2,9 @@ import { db } from "../databases/databases";
import { Logger } from "../utils/logger";
import { Request, Response } from "express";
import os from "os";
import redis from "../utils/redis";
import redis, { getRedisActiveRequests } from "../utils/redis";
import { promiseOrTimeout } from "../utils/promise";
import { Postgres } from "../databases/Postgres";
export async function getStatus(req: Request, res: Response): Promise<Response> {
const startTime = Date.now();
@@ -11,21 +12,23 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
value = Array.isArray(value) ? value[0] : value;
let processTime, redisProcessTime = -1;
try {
const dbStartTime = Date.now();
const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000)
.then(e => {
processTime = Date.now() - startTime;
processTime = Date.now() - dbStartTime;
return e.value;
})
.catch(e => {
.catch(e => /* istanbul ignore next */ {
Logger.error(`status: SQL query timed out: ${e}`);
return -1;
});
let statusRequests: unknown = 0;
const redisStartTime = Date.now();
const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000)
.then(e => {
redisProcessTime = Date.now() - startTime;
redisProcessTime = Date.now() - redisStartTime;
return e;
}).catch(e => {
}).catch(e => /* istanbul ignore next */ {
Logger.error(`status: redis increment timed out ${e}`);
return [-1];
});
@@ -33,17 +36,19 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
const statusValues: Record<string, any> = {
uptime: process.uptime(),
commit: (global as any).HEADCOMMIT || "unknown",
commit: (global as any)?.HEADCOMMIT ?? "unknown",
db: Number(dbVersion),
startTime,
processTime,
redisProcessTime,
loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average
statusRequests,
hostname: os.hostname()
hostname: os.hostname(),
activePostgresRequests: (db as Postgres)?.activePostgresRequests,
activeRedisRequests: getRedisActiveRequests(),
};
return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/getTopCategoryUsers.ts
View File

@@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 2
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
if (rows) {

13
src/routes/getTopUsers.ts
View File

@@ -28,14 +28,15 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
SUM(CASE WHEN category = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight",
SUM(CASE WHEN category = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller",
SUM(CASE WHEN category = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess",
SUM(CASE WHEN category = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter",
`;
}
const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views" END) as "minutesSaved",
SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "sponsorTimes"."actionType" != 'chapter' AND "shadowBannedUsers"."userID" IS NULL
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]);
@@ -55,7 +56,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
row.categorySumPreview,
row.categorySumHighlight,
row.categorySumFiller,
row.categorySumExclusiveAccess
row.categorySumExclusiveAccess,
row.categorySumChapter
]);
}
}
@@ -73,11 +75,6 @@ export async function getTopUsers(req: Request, res: Response): Promise<Response
const sortType = parseInt(req.query.sortType as string);
const categoryStatsEnabled = req.query.categoryStats;
if (sortType == undefined) {
//invalid request
return res.sendStatus(400);
}
//setup which sort type to use
let sortBy = "";
if (sortType == 0) {

5
src/routes/getUserID.ts
View File

@@ -12,7 +12,7 @@ function getFuzzyUserID(userName: string): Promise<{userName: string, userID: Us
try {
return db.prepare("all", `SELECT "userName", "userID" FROM "userNames" WHERE "userName"
LIKE ? ESCAPE '\\' LIMIT 10`, [userName]);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return null;
}
}
@@ -20,7 +20,7 @@ function getFuzzyUserID(userName: string): Promise<{userName: string, userID: Us
function getExactUserID(userName: string): Promise<{userName: string, userID: UserID }[]> {
try {
return db.prepare("all", `SELECT "userName", "userID" from "userNames" WHERE "userName" = ? LIMIT 10`, [userName]);
} catch (err) {
} catch (err) /* istanbul ignore next */{
return null;
}
}
@@ -42,6 +42,7 @@ export async function getUserID(req: Request, res: Response): Promise<Response>
: await getFuzzyUserID(userName);
if (results === undefined || results === null) {
/* istanbul ignore next */
return res.sendStatus(500);
} else if (results.length === 0) {
return res.sendStatus(404);

23
src/routes/getUserInfo.ts
View File

@@ -28,7 +28,7 @@ async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ min
segmentCount: 0,
};
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return null;
}
}
@@ -37,7 +37,7 @@ async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<number> {
try {
const row = await db.prepare("get", `SELECT COUNT(*) as "ignoredSegmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID], { useReplica: true });
return row?.ignoredSegmentCount ?? 0;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return null;
}
}
@@ -46,7 +46,7 @@ async function dbGetUsername(userID: HashedUserID) {
try {
const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
return row?.userName ?? userID;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return false;
}
}
@@ -55,7 +55,7 @@ async function dbGetViewsForUser(userID: HashedUserID) {
try {
const row = await db.prepare("get", `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID], { useReplica: true });
return row?.viewCount ?? 0;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return false;
}
}
@@ -64,7 +64,7 @@ async function dbGetIgnoredViewsForUser(userID: HashedUserID) {
try {
const row = await db.prepare("get", `SELECT SUM("views") as "ignoredViewCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID], { useReplica: true });
return row?.ignoredViewCount ?? 0;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return false;
}
}
@@ -73,7 +73,7 @@ async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
try {
const row = await db.prepare("get", `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID], { useReplica: true });
return row?.total ?? 0;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(`Couldn't get warnings for user ${userID}. returning 0`);
return 0;
}
@@ -83,7 +83,7 @@ async function dbGetLastSegmentForUser(userID: HashedUserID): Promise<SegmentUUI
try {
const row = await db.prepare("get", `SELECT "UUID" FROM "sponsorTimes" WHERE "userID" = ? ORDER BY "timeSubmitted" DESC LIMIT 1`, [userID], { useReplica: true });
return row?.UUID ?? null;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return null;
}
}
@@ -92,7 +92,7 @@ async function dbGetActiveWarningReasonForUser(userID: HashedUserID): Promise<st
try {
const row = await db.prepare("get", `SELECT reason FROM "warnings" WHERE "userID" = ? AND "enabled" = 1 ORDER BY "issueTime" DESC LIMIT 1`, [userID], { useReplica: true });
return row?.reason ?? "";
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(`Couldn't get reason for user ${userID}. returning blank`);
return "";
}
@@ -102,7 +102,7 @@ async function dbGetBanned(userID: HashedUserID): Promise<boolean> {
try {
const row = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID], { useReplica: true });
return row?.userCount > 0 ?? false;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return false;
}
}
@@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise<Record<string, bool
async function getFreeChaptersAccess(userID: HashedUserID): Promise<boolean> {
return await oneOf([isUserVIP(userID),
(async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(),
(async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
(async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
]);
}
@@ -195,7 +194,7 @@ async function getUserInfo(req: Request, res: Response): Promise<Response> {
export async function endpoint(req: Request, res: Response): Promise<Response> {
try {
return await getUserInfo(req, res);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
if (err instanceof SyntaxError) { // catch JSON.parse error
return res.status(400).send("Invalid values JSON");
} else return res.sendStatus(500);

4
src/routes/getUserStats.ts
View File

@@ -75,7 +75,7 @@ async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolea
};
}
return result;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return null;
}
@@ -85,7 +85,7 @@ async function dbGetUsername(userID: HashedUserID) {
try {
const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
return row?.userName ?? userID;
} catch (err) {
} catch (err) /* istanbul ignore next */ {
return false;
}
}

2
src/routes/getUsername.ts
View File

@@ -27,7 +27,7 @@ export async function getUsername(req: Request, res: Response): Promise<Response
userName: userID,
});
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/getViewsForUser.ts
View File

@@ -25,7 +25,7 @@ export async function getViewsForUser(req: Request, res: Response): Promise<Resp
} else {
return res.sendStatus(404);
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/postClearCache.ts
View File

@@ -47,7 +47,7 @@ export async function postClearCache(req: Request, res: Response): Promise<Respo
return res.status(200).json({
message: `Cache cleared on video ${videoID}`
});
} catch(err) {
} catch(err) /* istanbul ignore next */ {
return res.sendStatus(500);
}
}

6
src/routes/postLockCategories.ts
View File

@@ -37,7 +37,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
if (!userIsVIP) {
res.status(403).json({
message: "Must be a VIP to mark videos.",
message: "Must be a VIP to lock videos.",
});
return;
}
@@ -66,7 +66,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
for (const lock of locksToApply) {
try {
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "actionType", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?, ?)`, [videoID, userID, lock.actionType, lock.category, hashedVideoID, reason, service]);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`);
Logger.error(err as string);
res.status(500).json({
@@ -82,7 +82,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
await db.prepare("run",
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "actionType" = ? AND "category" = ? AND "service" = ?',
[reason, userID, videoID, lock.actionType, lock.category, service]);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`);
Logger.error(err as string);
res.status(500).json({

2
src/routes/postPurgeAllSegments.ts
View File

@@ -37,7 +37,7 @@ export async function postPurgeAllSegments(req: Request, res: Response): Promise
service
});
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/postSegmentShift.ts
View File

@@ -91,7 +91,7 @@ export async function postSegmentShift(req: Request, res: Response): Promise<Res
break;
}
}
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/postSkipSegments.ts
View File

@@ -334,7 +334,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
return {
pass: false,
errorCode: 403,
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
errorMessage: `Hi, currently there are server issues and you might have not recieved segments even though they exist. Sorry about this, I'm working on it. Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
};
}
}

4
src/routes/setUsername.ts
View File

@@ -56,7 +56,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
return res.sendStatus(200);
}
}
catch (error) {
catch (error) /* istanbul ignore next */ {
Logger.error(error as string);
return res.sendStatus(500);
}
@@ -83,7 +83,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
await logUserNameChange(userID, userName, oldUserName, adminUserIDInput !== undefined);
return res.sendStatus(200);
} catch (err) {
} catch (err) /* istanbul ignore next */ {
Logger.error(err as string);
return res.sendStatus(500);
}

23
src/routes/verifyToken.ts
View File

@@ -4,7 +4,6 @@ import { config } from "../config";
import { privateDB } from "../databases/databases";
import { Logger } from "../utils/logger";
import { getPatreonIdentity, PatronStatus, refreshToken, TokenType } from "../utils/tokenUtils";
import FormData from "form-data";
interface VerifyTokenRequest extends Request {
query: {
@@ -12,14 +11,16 @@ interface VerifyTokenRequest extends Request {
}
}
export const validatelicenseKeyRegex = (token: string) =>
new RegExp(/[A-Za-z0-9]{40}|[A-Za-z0-9-]{35}/).test(token);
export async function verifyTokenRequest(req: VerifyTokenRequest, res: Response): Promise<Response> {
const { query: { licenseKey } } = req;
if (!licenseKey) {
return res.status(400).send("Invalid request");
}
const licenseRegex = new RegExp(/[a-zA-Z0-9]{40}|[A-Z0-9-]{35}/);
if (!licenseRegex.test(licenseKey)) {
} else if (!validatelicenseKeyRegex(licenseKey)) {
// fast check for invalid licence key
return res.status(200).send({
allowed: false
});
@@ -34,6 +35,7 @@ export async function verifyTokenRequest(req: VerifyTokenRequest, res: Response)
refreshToken(TokenType.patreon, licenseKey, tokens.refreshToken).catch(Logger.error);
}
/* istanbul ignore else */
if (identity) {
const membership = identity.included?.[0]?.attributes;
const allowed = !!membership && ((membership.patron_status === PatronStatus.active && membership.currently_entitled_amount_cents > 0)
@@ -65,20 +67,13 @@ export async function verifyTokenRequest(req: VerifyTokenRequest, res: Response)
async function checkAllGumroadProducts(licenseKey: string): Promise<boolean> {
for (const link of config.gumroad.productPermalinks) {
try {
const formData = new FormData();
formData.append("product_permalink", link);
formData.append("license_key", licenseKey);
const result = await axios.request({
url: "https://api.gumroad.com/v2/licenses/verify",
data: formData,
method: "POST",
headers: formData.getHeaders()
const result = await axios.post("https://api.gumroad.com/v2/licenses/verify", {
params: { product_permalink: link, license_key: licenseKey }
});
const allowed = result.status === 200 && result.data?.success;
if (allowed) return allowed;
} catch (e) {
} catch (e) /* istanbul ignore next */ {
Logger.error(`Gumroad fetch for ${link} failed: ${e}`);
}
}

6
src/routes/voteOnSponsorTime.ts
View File

@@ -14,6 +14,7 @@ import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID
import { QueryCacher } from "../utils/queryCacher";
import axios from "axios";
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
import { deleteLockCategories } from "./deleteLockCategories";
const voteTypes = {
normal: 0,
@@ -59,7 +60,7 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) {
let apiVideoDetails: videoDetails = null;
if (service == Service.YouTube) {
// don't use cache since we have no information about the video length
apiVideoDetails = await getVideoDetails(videoID);
apiVideoDetails = await getVideoDetails(videoID, true);
}
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
if (videoDurationChanged(videoDuration, apiVideoDuration)) {
@@ -95,6 +96,7 @@ async function checkVideoDuration(UUID: SegmentUUID) {
AND "hidden" = 0 AND "shadowHidden" = 0 AND
"actionType" != 'full' AND "votes" > -2`,
[videoID, service, latestSubmission.timeSubmitted]);
deleteLockCategories(videoID, null, null, service).catch(Logger.error);
}
}
@@ -219,7 +221,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
[UUID], { useReplica: true })) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number};
if (!config.categorySupport[category]?.includes(segmentInfo.actionType) || segmentInfo.actionType === ActionType.Full) {
return { status: 400, message: `Not allowed to change to ${category} when for segment of type ${segmentInfo.actionType}`};
return { status: 400, message: `Not allowed to change to ${category} when for segment of type ${segmentInfo.actionType}` };
}
if (!config.categoryList.includes(category)) {
return { status: 400, message: "Category doesn't exist." };

9
src/types/config.model.ts
View File

@@ -7,16 +7,21 @@ interface RedisConfig extends redis.RedisClientOptions {
getTimeout: number;
}
interface RedisReadOnlyConfig extends redis.RedisClientOptions {
enabled: boolean;
weight: number;
}
export interface CustomPostgresConfig extends PoolConfig {
enabled: boolean;
maxTries: number;
maxConcurrentRequests: number;
}
export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig {
weight: number;
readTimeout: number;
fallbackOnFail: boolean;
stopRetryThreshold: number;
}
export interface SBSConfig {
@@ -61,6 +66,8 @@ export interface SBSConfig {
minimumPrefix?: string;
maximumPrefix?: string;
redis?: RedisConfig;
redisRead?: RedisReadOnlyConfig;
redisRateLimit: boolean;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: CustomPostgresConfig;
postgresReadOnly?: CustomPostgresReadOnlyConfig;

5
src/utils/getIP.ts
View File

@@ -3,6 +3,9 @@ import { Request } from "express";
import { IPAddress } from "../types/segments.model";
export function getIP(req: Request): IPAddress {
// if in testing mode, return immediately
if (config.mode === "test") return "127.0.0.1" as IPAddress;
if (config.behindProxy === true || config.behindProxy === "true") {
config.behindProxy = "X-Forwarded-For";
}
@@ -15,6 +18,6 @@ export function getIP(req: Request): IPAddress {
case "X-Real-IP":
return req.headers["x-real-ip"] as IPAddress;
default:
return (req.connection?.remoteAddress || req.socket?.remoteAddress) as IPAddress;
return req.socket?.remoteAddress as IPAddress;
}
}

2
src/utils/innerTubeAPI.ts
View File

@@ -18,6 +18,7 @@ async function getFromITube (videoID: string): Promise<innerTubeVideoDetails> {
const result = await axios.post(url, data, {
timeout: 3500
});
/* istanbul ignore else */
if (result.status === 200) {
return result.data.videoDetails;
} else {
@@ -39,6 +40,7 @@ export async function getPlayerData (videoID: string, ignoreCache = false): Prom
return data as innerTubeVideoDetails;
}
} catch (err) {
/* istanbul ignore next */
return Promise.reject(err);
}
}

8
src/utils/permissions.ts
View File

@@ -9,7 +9,7 @@ import { getReputation } from "./reputation";
interface CanSubmitResult {
canSubmit: boolean;
reason?: string;
reason: string;
}
async function lowDownvotes(userID: HashedUserID): Promise<boolean> {
@@ -27,11 +27,13 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi
lowDownvotes(userID),
(async () => (await getReputation(userID)) > config.minReputationToSubmitChapter)(),
hasFeature(userID, Feature.ChapterSubmitter)
])
]),
reason: "Submitting chapters requires a minimum reputation. You can ask on Discord/Matrix to get permission with less reputation."
};
default:
return {
canSubmit: true
canSubmit: true,
reason: ""
};
}
}

49
src/utils/redis.ts
View File

@@ -25,19 +25,40 @@ let exportClient: RedisSB = {
quit: () => new Promise((resolve) => resolve(null)),
};
let lastClientFail = 0;
let lastReadFail = 0;
let activeRequests = 0;
if (config.redis?.enabled) {
Logger.info("Connected to redis");
const client = createClient(config.redis);
const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null;
void client.connect(); // void as we don't care about the promise
void readClient?.connect();
exportClient = client as RedisSB;
const get = client.get.bind(client);
const getRead = readClient?.get?.bind(readClient);
exportClient.get = (key) => new Promise((resolve, reject) => {
activeRequests++;
const timeout = config.redis.getTimeout ? setTimeout(() => reject(), config.redis.getTimeout) : null;
get(key).then((reply) => {
const chosenGet = pickChoice(get, getRead);
chosenGet(key).then((reply) => {
if (timeout !== null) clearTimeout(timeout);
activeRequests--;
resolve(reply);
}).catch((err) => reject(err));
}).catch((err) => {
if (chosenGet === get) {
lastClientFail = Date.now();
} else {
lastReadFail = Date.now();
}
activeRequests--;
reject(err);
});
});
exportClient.increment = (key) => new Promise((resolve, reject) =>
void client.multi()
@@ -48,11 +69,35 @@ if (config.redis?.enabled) {
.catch((err) => reject(err))
);
client.on("error", function(error) {
lastClientFail = Date.now();
Logger.error(`Redis Error: ${error}`);
});
client.on("reconnect", () => {
Logger.info("Redis: trying to reconnect");
});
readClient?.on("error", function(error) {
lastReadFail = Date.now();
Logger.error(`Redis Read-Only Error: ${error}`);
});
readClient?.on("reconnect", () => {
Logger.info("Redis Read-Only: trying to reconnect");
});
}
function pickChoice<T>(client: T, readClient: T): T {
const readAvailable = !!readClient;
const ignoreReadDueToFailure = lastReadFail > Date.now() - 1000 * 30;
const readDueToFailure = lastClientFail > Date.now() - 1000 * 30;
if (readAvailable && !ignoreReadDueToFailure && (readDueToFailure ||
Math.random() > 1 / (config.redisRead?.weight + 1))) {
return readClient;
} else {
return client;
}
}
export function getRedisActiveRequests(): number {
return activeRequests;
}
export default exportClient;

14
src/utils/tokenUtils.ts
View File

@@ -58,12 +58,11 @@ export async function createAndSaveToken(type: TokenType, code?: string): Promis
return licenseKey;
}
} catch (e) {
break;
} catch (e) /* istanbul ignore next */ {
Logger.error(`token creation: ${e}`);
return null;
}
break;
}
case TokenType.local: {
const licenseKey = generateToken();
@@ -74,7 +73,6 @@ export async function createAndSaveToken(type: TokenType, code?: string): Promis
return licenseKey;
}
}
return null;
}
@@ -102,15 +100,12 @@ export async function refreshToken(type: TokenType, licenseKey: string, refreshT
return true;
}
} catch (e) {
} catch (e) /* istanbul ignore next */ {
Logger.error(`token refresh: ${e}`);
return false;
}
break;
}
}
return false;
}
@@ -136,9 +131,8 @@ export async function getPatreonIdentity(accessToken: string): Promise<PatreonId
if (identityRequest.status === 200) {
return identityRequest.data;
}
} catch (e) {
} catch (e) /* istanbul ignore next */ {
Logger.error(`identity request: ${e}`);
}
return null;
}