Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 11:36:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into more-coverage

Browse Source
This commit is contained in:
Michael C
2022-10-27 01:38:08 -04:00
parent 9286f16e7b c2acc6227b
commit 622511c8fd
16 changed files with 136 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@@ -6,11 +6,11 @@ RUN npm ci && npm run tsc
FROM node:16-alpine as app
WORKDIR /usr/src/app
RUN apk add git postgresql-client
RUN apk add --no-cache git postgresql-client
COPY --from=builder ./node_modules ./node_modules
COPY --from=builder ./dist ./dist
COPY ./.git ./.git
COPY entrypoint.sh .
COPY databases/*.sql databases/
EXPOSE 8080
CMD ./entrypoint.sh
CMD ./entrypoint.sh

14
src/config.ts
View File

@@ -76,8 +76,7 @@ addDefaults(config, {
port: 5432,
max: 10,
idleTimeoutMillis: 10000,
maxTries: 3,
maxConcurrentRequests: 3500
maxTries: 3
},
postgresReadOnly: {
enabled: false,
@@ -91,7 +90,7 @@ addDefaults(config, {
idleTimeoutMillis: 10000,
maxTries: 3,
fallbackOnFail: true,
maxConcurrentRequests: 3500
stopRetryThreshold: 800
},
dumpDatabase: {
enabled: false,
@@ -139,6 +138,15 @@ addDefaults(config, {
expiryTime: 24 * 60 * 60,
getTimeout: 40
},
redisRead: {
enabled: false,
socket: {
host: "",
port: 0
},
disableOfflineQueue: true,
weight: 1
},
patreon: {
clientId: "",
clientSecret: "",

77
src/databases/Postgres.ts
View File

@@ -33,8 +33,7 @@ export class Postgres implements IDatabase {
private poolRead: Pool;
private lastPoolReadFail = 0;
private concurrentRequests = 0;
private concurrentReadRequests = 0;
activePostgresRequests = 0;
constructor(private config: DatabaseConfig) {}
@@ -54,19 +53,23 @@ export class Postgres implements IDatabase {
});
if (this.config.postgresReadOnly && this.config.postgresReadOnly.enabled) {
this.poolRead = new Pool({
...this.config.postgresReadOnly
});
this.poolRead.on("error", (err, client) => {
Logger.error(err.stack);
this.lastPoolReadFail = Date.now();
try {
this.poolRead = new Pool({
...this.config.postgresReadOnly
});
this.poolRead.on("error", (err, client) => {
Logger.error(err.stack);
this.lastPoolReadFail = Date.now();
try {
client.release(true);
} catch (err) {
Logger.error(`poolRead (postgres): ${err}`);
}
});
try {
client.release(true);
} catch (err) {
Logger.error(`poolRead (postgres): ${err}`);
}
});
} catch (e) {
Logger.error(`poolRead (postgres): ${e}`);
}
}
if (!this.config.readOnly) {
@@ -102,22 +105,6 @@ export class Postgres implements IDatabase {
Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
if (this.config.readOnly) {
if (this.concurrentReadRequests > this.config.postgresReadOnly?.maxConcurrentRequests) {
Logger.error(`prepare (postgres): cancelling read query because too many concurrent requests, query: ${query}`);
throw new Error("Too many concurrent requests");
}
this.concurrentReadRequests++;
} else {
if (this.concurrentRequests > this.config.postgres.maxConcurrentRequests) {
Logger.error(`prepare (postgres): cancelling query because too many concurrent requests, query: ${query}`);
throw new Error("Too many concurrent requests");
}
this.concurrentRequests++;
}
const pendingQueries: PromiseWithState<QueryResult<any>>[] = [];
let tries = 0;
let lastPool: Pool = null;
@@ -127,6 +114,7 @@ export class Postgres implements IDatabase {
tries++;
try {
this.activePostgresRequests++;
lastPool = this.getPool(type, options);
pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params })));
@@ -134,12 +122,7 @@ export class Postgres implements IDatabase {
if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout)));
const queryResult = await nextFulfilment(currentPromises);
if (this.config.readOnly) {
this.concurrentReadRequests--;
} else {
this.concurrentRequests--;
}
this.activePostgresRequests--;
switch (type) {
case "get": {
const value = queryResult.rows[0];
@@ -159,30 +142,30 @@ export class Postgres implements IDatabase {
if (lastPool === this.pool) {
// Only applies if it is get or all request
options.forceReplica = true;
} else if (lastPool === this.poolRead && maxTries() - tries <= 1) {
options.useReplica = false;
} else if (lastPool === this.poolRead) {
this.lastPoolReadFail = Date.now();
if (maxTries() - tries <= 1) {
options.useReplica = false;
}
}
Logger.error(`prepare (postgres) try ${tries}: ${err}`);
}
} while (this.isReadQuery(type) && tries < maxTries());
if (this.config.readOnly) {
this.concurrentReadRequests--;
} else {
this.concurrentRequests--;
}
} while (this.isReadQuery(type) && tries < maxTries()
&& this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold);
this.activePostgresRequests--;
throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`);
}
private getPool(type: string, options: QueryOption): Pool {
const readAvailable = this.poolRead && options.useReplica && this.isReadQuery(type);
const ignroreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
const ignoreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
&& this.lastPoolReadFail > Date.now() - 1000 * 30;
const readDueToFailure = this.config.postgresReadOnly.fallbackOnFail
&& this.lastPoolFail > Date.now() - 1000 * 30;
if (readAvailable && !ignroreReadDueToFailure && (options.forceReplica || readDueToFailure ||
if (readAvailable && !ignoreReadDueToFailure && (options.forceReplica || readDueToFailure ||
Math.random() > 1 / (this.config.postgresReadOnly.weight + 1))) {
return this.poolRead;
} else {

10
src/index.ts
View File

@@ -12,7 +12,13 @@ async function init() {
process.exit(1);
});
await initDb();
try {
await initDb();
} catch (e) {
Logger.error(`Init Db: ${e}`);
process.exit(1);
}
// edge case clause for creating compatible .db files, do not enable
if (config.mode === "init-db-and-exit") process.exit(0);
// do not enable init-db-only mode for usage.
@@ -27,4 +33,4 @@ async function init() {
}).setTimeout(15000);
}
init().catch((err) => Logger.error(err));
init().catch((err) => Logger.error(`Index.js: ${err}`));

5
src/routes/generateToken.ts
View File

@@ -1,7 +1,7 @@
import { Request, Response } from "express";
import { config } from "../config";
import { createAndSaveToken, TokenType } from "../utils/tokenUtils";
import { getHashCache } from "../utils/getHashCache";
interface GenerateTokenRequest extends Request {
query: {
@@ -15,12 +15,13 @@ interface GenerateTokenRequest extends Request {
export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise<Response> {
const { query: { code, adminUserID }, params: { type } } = req;
const adminUserIDHash = adminUserID ? (await getHashCache(adminUserID)) : null;
if (!code || !type) {
return res.status(400).send("Invalid request");
}
if (type === TokenType.patreon || (type === TokenType.local && adminUserID === config.adminUserID)) {
if (type === TokenType.patreon || (type === TokenType.local && adminUserIDHash === config.adminUserID)) {
const licenseKey = await createAndSaveToken(type, code);
/* istanbul ignore else */

2
src/routes/getSearchSegments.ts
View File

@@ -14,7 +14,7 @@ type searchSegmentResponse = {
function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
return db.prepare(
"all",
`SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID" FROM "sponsorTimes"
`SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes"
WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`,
[videoID, service]
) as Promise<DBSegment[]>;

13
src/routes/getStatus.ts
View File

@@ -2,8 +2,9 @@ import { db } from "../databases/databases";
import { Logger } from "../utils/logger";
import { Request, Response } from "express";
import os from "os";
import redis from "../utils/redis";
import redis, { getRedisActiveRequests } from "../utils/redis";
import { promiseOrTimeout } from "../utils/promise";
import { Postgres } from "../databases/Postgres";
export async function getStatus(req: Request, res: Response): Promise<Response> {
const startTime = Date.now();
@@ -11,9 +12,10 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
value = Array.isArray(value) ? value[0] : value;
let processTime, redisProcessTime = -1;
try {
const dbStartTime = Date.now();
const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000)
.then(e => {
processTime = Date.now() - startTime;
processTime = Date.now() - dbStartTime;
return e.value;
})
.catch(e => /* istanbul ignore next */ {
@@ -21,9 +23,10 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
return -1;
});
let statusRequests: unknown = 0;
const redisStartTime = Date.now();
const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000)
.then(e => {
redisProcessTime = Date.now() - startTime;
redisProcessTime = Date.now() - redisStartTime;
return e;
}).catch(e => /* istanbul ignore next */ {
Logger.error(`status: redis increment timed out ${e}`);
@@ -40,7 +43,9 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
redisProcessTime,
loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average
statusRequests,
hostname: os.hostname()
hostname: os.hostname(),
activePostgresRequests: (db as Postgres)?.activePostgresRequests,
activeRedisRequests: getRedisActiveRequests(),
};
return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues);
} catch (err) /* istanbul ignore next */ {

2
src/routes/getTopCategoryUsers.ts
View File

@@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 2
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
if (rows) {

8
src/routes/getTopUsers.ts
View File

@@ -28,14 +28,15 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
SUM(CASE WHEN category = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight",
SUM(CASE WHEN category = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller",
SUM(CASE WHEN category = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess",
SUM(CASE WHEN category = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter",
`;
}
const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views" END) as "minutesSaved",
SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "sponsorTimes"."actionType" != 'chapter' AND "shadowBannedUsers"."userID" IS NULL
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]);
@@ -55,7 +56,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
row.categorySumPreview,
row.categorySumHighlight,
row.categorySumFiller,
row.categorySumExclusiveAccess
row.categorySumExclusiveAccess,
row.categorySumChapter
]);
}
}

3
src/routes/getUserInfo.ts
View File

@@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise<Record<string, bool
async function getFreeChaptersAccess(userID: HashedUserID): Promise<boolean> {
return await oneOf([isUserVIP(userID),
(async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(),
(async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
(async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
]);
}

2
src/routes/postSkipSegments.ts
View File

@@ -334,7 +334,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
return {
pass: false,
errorCode: 403,
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
errorMessage: `Hi, currently there are server issues and you might have not recieved segments even though they exist. Sorry about this, I'm working on it. Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
};
}
}

4
src/routes/voteOnSponsorTime.ts
View File

@@ -14,6 +14,7 @@ import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID
import { QueryCacher } from "../utils/queryCacher";
import axios from "axios";
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
import { deleteLockCategories } from "./deleteLockCategories";
const voteTypes = {
normal: 0,
@@ -59,7 +60,7 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) {
let apiVideoDetails: videoDetails = null;
if (service == Service.YouTube) {
// don't use cache since we have no information about the video length
apiVideoDetails = await getVideoDetails(videoID);
apiVideoDetails = await getVideoDetails(videoID, true);
}
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
if (videoDurationChanged(videoDuration, apiVideoDuration)) {
@@ -95,6 +96,7 @@ async function checkVideoDuration(UUID: SegmentUUID) {
AND "hidden" = 0 AND "shadowHidden" = 0 AND
"actionType" != 'full' AND "votes" > -2`,
[videoID, service, latestSubmission.timeSubmitted]);
deleteLockCategories(videoID, null, null, service).catch(Logger.error);
}
}

8
src/types/config.model.ts
View File

@@ -7,16 +7,21 @@ interface RedisConfig extends redis.RedisClientOptions {
getTimeout: number;
}
interface RedisReadOnlyConfig extends redis.RedisClientOptions {
enabled: boolean;
weight: number;
}
export interface CustomPostgresConfig extends PoolConfig {
enabled: boolean;
maxTries: number;
maxConcurrentRequests: number;
}
export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig {
weight: number;
readTimeout: number;
fallbackOnFail: boolean;
stopRetryThreshold: number;
}
export interface SBSConfig {
@@ -61,6 +66,7 @@ export interface SBSConfig {
minimumPrefix?: string;
maximumPrefix?: string;
redis?: RedisConfig;
redisRead?: RedisReadOnlyConfig;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: CustomPostgresConfig;
postgresReadOnly?: CustomPostgresReadOnlyConfig;

8
src/utils/permissions.ts
View File

@@ -9,7 +9,7 @@ import { getReputation } from "./reputation";
interface CanSubmitResult {
canSubmit: boolean;
reason?: string;
reason: string;
}
async function lowDownvotes(userID: HashedUserID): Promise<boolean> {
@@ -27,11 +27,13 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi
lowDownvotes(userID),
(async () => (await getReputation(userID)) > config.minReputationToSubmitChapter)(),
hasFeature(userID, Feature.ChapterSubmitter)
])
]),
reason: "Submitting chapters requires a minimum reputation. You can ask on Discord/Matrix to get permission with less reputation."
};
default:
return {
canSubmit: true
canSubmit: true,
reason: ""
};
}
}

49
src/utils/redis.ts
View File

@@ -25,19 +25,40 @@ let exportClient: RedisSB = {
quit: () => new Promise((resolve) => resolve(null)),
};
let lastClientFail = 0;
let lastReadFail = 0;
let activeRequests = 0;
if (config.redis?.enabled) {
Logger.info("Connected to redis");
const client = createClient(config.redis);
const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null;
void client.connect(); // void as we don't care about the promise
void readClient?.connect();
exportClient = client as RedisSB;
const get = client.get.bind(client);
const getRead = readClient?.get?.bind(readClient);
exportClient.get = (key) => new Promise((resolve, reject) => {
activeRequests++;
const timeout = config.redis.getTimeout ? setTimeout(() => reject(), config.redis.getTimeout) : null;
get(key).then((reply) => {
const chosenGet = pickChoice(get, getRead);
chosenGet(key).then((reply) => {
if (timeout !== null) clearTimeout(timeout);
activeRequests--;
resolve(reply);
}).catch((err) => reject(err));
}).catch((err) => {
if (chosenGet === get) {
lastClientFail = Date.now();
} else {
lastReadFail = Date.now();
}
activeRequests--;
reject(err);
});
});
exportClient.increment = (key) => new Promise((resolve, reject) =>
void client.multi()
@@ -48,11 +69,35 @@ if (config.redis?.enabled) {
.catch((err) => reject(err))
);
client.on("error", function(error) {
lastClientFail = Date.now();
Logger.error(`Redis Error: ${error}`);
});
client.on("reconnect", () => {
Logger.info("Redis: trying to reconnect");
});
readClient?.on("error", function(error) {
lastReadFail = Date.now();
Logger.error(`Redis Read-Only Error: ${error}`);
});
readClient?.on("reconnect", () => {
Logger.info("Redis Read-Only: trying to reconnect");
});
}
function pickChoice<T>(client: T, readClient: T): T {
const readAvailable = !!readClient;
const ignoreReadDueToFailure = lastReadFail > Date.now() - 1000 * 30;
const readDueToFailure = lastClientFail > Date.now() - 1000 * 30;
if (readAvailable && !ignoreReadDueToFailure && (readDueToFailure ||
Math.random() > 1 / (config.redisRead?.weight + 1))) {
return readClient;
} else {
return client;
}
}
export function getRedisActiveRequests(): number {
return activeRequests;
}
export default exportClient;

3
test/cases/getSearchSegments.ts
View File

@@ -335,7 +335,8 @@ describe("getSearchSegments", () => {
locked: 1,
hidden: 0,
shadowHidden: 0,
userID: "searchTestUser"
userID: "searchTestUser",
description: ""
};
assert.deepStrictEqual(segment0, expected);
done();