Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into more-coverage

Dockerfile

@@ -6,11 +6,11 @@ RUN npm ci && npm run tsc
 
 FROM node:16-alpine as app
 WORKDIR /usr/src/app
-RUN apk add git postgresql-client
+RUN apk add --no-cache git postgresql-client
 COPY --from=builder ./node_modules ./node_modules
 COPY --from=builder ./dist ./dist
 COPY ./.git ./.git
 COPY entrypoint.sh .
 COPY databases/*.sql databases/
 EXPOSE 8080
-CMD ./entrypoint.sh
+CMD ./entrypoint.sh

src/config.ts

@@ -76,8 +76,7 @@ addDefaults(config, {
         port: 5432,
         max: 10,
         idleTimeoutMillis: 10000,
-        maxTries: 3,
+        maxTries: 3
-        maxConcurrentRequests: 3500
     },
     postgresReadOnly: {
         enabled: false,
@@ -91,7 +90,7 @@ addDefaults(config, {
         idleTimeoutMillis: 10000,
         maxTries: 3,
         fallbackOnFail: true,
-        maxConcurrentRequests: 3500
+        stopRetryThreshold: 800
     },
     dumpDatabase: {
         enabled: false,
@@ -139,6 +138,15 @@ addDefaults(config, {
         expiryTime: 24 * 60 * 60,
         getTimeout: 40
     },
+    redisRead: {
+        enabled: false,
+        socket: {
+            host: "",
+            port: 0
+        },
+        disableOfflineQueue: true,
+        weight: 1
+    },
     patreon: {
         clientId: "",
         clientSecret: "",

src/databases/Postgres.ts

@@ -33,8 +33,7 @@ export class Postgres implements IDatabase {
     private poolRead: Pool;
     private lastPoolReadFail = 0;
 
-    private concurrentRequests = 0;
+    activePostgresRequests = 0;
-    private concurrentReadRequests = 0;
 
     constructor(private config: DatabaseConfig) {}
 
@@ -54,19 +53,23 @@ export class Postgres implements IDatabase {
         });
 
         if (this.config.postgresReadOnly && this.config.postgresReadOnly.enabled) {
-            this.poolRead = new Pool({
+            try {
-                ...this.config.postgresReadOnly
+                this.poolRead = new Pool({
-            });
+                    ...this.config.postgresReadOnly
-            this.poolRead.on("error", (err, client) => {
+                });
-                Logger.error(err.stack);
+                this.poolRead.on("error", (err, client) => {
-                this.lastPoolReadFail = Date.now();
+                    Logger.error(err.stack);
+                    this.lastPoolReadFail = Date.now();
 
-                try {
+                    try {
-                    client.release(true);
+                        client.release(true);
-                } catch (err) {
+                    } catch (err) {
-                    Logger.error(`poolRead (postgres): ${err}`);
+                        Logger.error(`poolRead (postgres): ${err}`);
-                }
+                    }
-            });
+                });
+            } catch (e) {
+                Logger.error(`poolRead (postgres): ${e}`);
+            }
         }
 
         if (!this.config.readOnly) {
@@ -102,22 +105,6 @@ export class Postgres implements IDatabase {
 
         Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
 
-        if (this.config.readOnly) {
-            if (this.concurrentReadRequests > this.config.postgresReadOnly?.maxConcurrentRequests) {
-                Logger.error(`prepare (postgres): cancelling read query because too many concurrent requests, query: ${query}`);
-                throw new Error("Too many concurrent requests");
-            }
-
-            this.concurrentReadRequests++;
-        } else {
-            if (this.concurrentRequests > this.config.postgres.maxConcurrentRequests) {
-                Logger.error(`prepare (postgres): cancelling query because too many concurrent requests, query: ${query}`);
-                throw new Error("Too many concurrent requests");
-            }
-
-            this.concurrentRequests++;
-        }
-
         const pendingQueries: PromiseWithState<QueryResult<any>>[] = [];
         let tries = 0;
         let lastPool: Pool = null;
@@ -127,6 +114,7 @@ export class Postgres implements IDatabase {
             tries++;
 
             try {
+                this.activePostgresRequests++;
                 lastPool = this.getPool(type, options);
 
                 pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params })));
@@ -134,12 +122,7 @@ export class Postgres implements IDatabase {
                 if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout)));
                 const queryResult = await nextFulfilment(currentPromises);
 
-                if (this.config.readOnly) {
+                this.activePostgresRequests--;
-                    this.concurrentReadRequests--;
-                } else {
-                    this.concurrentRequests--;
-                }
-
                 switch (type) {
                     case "get": {
                         const value = queryResult.rows[0];
@@ -159,30 +142,30 @@ export class Postgres implements IDatabase {
                 if (lastPool === this.pool) {
                     // Only applies if it is get or all request
                     options.forceReplica = true;
-                } else if (lastPool === this.poolRead && maxTries() - tries <= 1) {
+                } else if (lastPool === this.poolRead) {
-                    options.useReplica = false;
+                    this.lastPoolReadFail = Date.now();
+
+                    if (maxTries() - tries <= 1) {
+                        options.useReplica = false;
+                    }
                 }
 
                 Logger.error(`prepare (postgres) try ${tries}: ${err}`);
             }
-        } while (this.isReadQuery(type) && tries < maxTries());
+        } while (this.isReadQuery(type) && tries < maxTries()
-
+            && this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold);
-        if (this.config.readOnly) {
-            this.concurrentReadRequests--;
-        } else {
-            this.concurrentRequests--;
-        }
 
+        this.activePostgresRequests--;
         throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`);
     }
 
     private getPool(type: string, options: QueryOption): Pool {
         const readAvailable = this.poolRead && options.useReplica && this.isReadQuery(type);
-        const ignroreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
+        const ignoreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
             && this.lastPoolReadFail > Date.now() - 1000 * 30;
         const readDueToFailure = this.config.postgresReadOnly.fallbackOnFail
             && this.lastPoolFail > Date.now() - 1000 * 30;
-        if (readAvailable && !ignroreReadDueToFailure && (options.forceReplica || readDueToFailure ||
+        if (readAvailable && !ignoreReadDueToFailure && (options.forceReplica || readDueToFailure ||
                 Math.random() > 1 / (this.config.postgresReadOnly.weight + 1))) {
             return this.poolRead;
         } else {

src/index.ts

@@ -12,7 +12,13 @@ async function init() {
         process.exit(1);
     });
 
-    await initDb();
+    try {
+        await initDb();
+    } catch (e) {
+        Logger.error(`Init Db: ${e}`);
+        process.exit(1);
+    }
+
     // edge case clause for creating compatible .db files, do not enable
     if (config.mode === "init-db-and-exit") process.exit(0);
     // do not enable init-db-only mode for usage.
@@ -27,4 +33,4 @@ async function init() {
     }).setTimeout(15000);
 }
 
-init().catch((err) => Logger.error(err));
+init().catch((err) => Logger.error(`Index.js: ${err}`));

src/routes/generateToken.ts

@@ -1,7 +1,7 @@
 import { Request, Response } from "express";
 import { config } from "../config";
 import { createAndSaveToken, TokenType } from "../utils/tokenUtils";
-
+import { getHashCache } from "../utils/getHashCache";
 
 interface GenerateTokenRequest extends Request {
     query: {
@@ -15,12 +15,13 @@ interface GenerateTokenRequest extends Request {
 
 export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise<Response> {
     const { query: { code, adminUserID }, params: { type } } = req;
+    const adminUserIDHash = adminUserID ? (await getHashCache(adminUserID)) : null;
 
     if (!code || !type) {
         return res.status(400).send("Invalid request");
     }
 
-    if (type === TokenType.patreon || (type === TokenType.local && adminUserID === config.adminUserID)) {
+    if (type === TokenType.patreon || (type === TokenType.local && adminUserIDHash === config.adminUserID)) {
         const licenseKey = await createAndSaveToken(type, code);
 
         /* istanbul ignore else */

src/routes/getSearchSegments.ts

@@ -14,7 +14,7 @@ type searchSegmentResponse = {
 function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
     return db.prepare(
         "all",
-        `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID" FROM "sponsorTimes" 
+        `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes" 
         WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`,
         [videoID, service]
     ) as Promise<DBSegment[]>;

src/routes/getStatus.ts

@@ -2,8 +2,9 @@ import { db } from "../databases/databases";
 import { Logger } from "../utils/logger";
 import { Request, Response } from "express";
 import os from "os";
-import redis from "../utils/redis";
+import redis, { getRedisActiveRequests } from "../utils/redis";
 import { promiseOrTimeout } from "../utils/promise";
+import { Postgres } from "../databases/Postgres";
 
 export async function getStatus(req: Request, res: Response): Promise<Response> {
     const startTime = Date.now();
@@ -11,9 +12,10 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
     value = Array.isArray(value) ? value[0] : value;
     let processTime, redisProcessTime = -1;
     try {
+        const dbStartTime = Date.now();
         const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000)
             .then(e => {
-                processTime = Date.now() - startTime;
+                processTime = Date.now() - dbStartTime;
                 return e.value;
             })
             .catch(e => /* istanbul ignore next */ {
@@ -21,9 +23,10 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
                 return -1;
             });
         let statusRequests: unknown = 0;
+        const redisStartTime = Date.now();
         const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000)
             .then(e => {
-                redisProcessTime = Date.now() - startTime;
+                redisProcessTime = Date.now() - redisStartTime;
                 return e;
             }).catch(e => /* istanbul ignore next */ {
                 Logger.error(`status: redis increment timed out ${e}`);
@@ -40,7 +43,9 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
             redisProcessTime,
             loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average
             statusRequests,
-            hostname: os.hostname()
+            hostname: os.hostname(),
+            activePostgresRequests: (db as Postgres)?.activePostgresRequests,
+            activeRedisRequests: getRedisActiveRequests(),
         };
         return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues);
     } catch (err) /* istanbul ignore next */ {

src/routes/getTopCategoryUsers.ts

@@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
         SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
         LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
         WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
-        GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
+        GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 2
         ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
 
     if (rows) {

src/routes/getTopUsers.ts

@@ -28,14 +28,15 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
             SUM(CASE WHEN category = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight",
             SUM(CASE WHEN category = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller",
             SUM(CASE WHEN category = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess",
+            SUM(CASE WHEN category = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter",
         `;
     }
 
     const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
-        SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
+        SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views" END) as "minutesSaved",
         SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
         LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
-        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "sponsorTimes"."actionType" != 'chapter' AND "shadowBannedUsers"."userID" IS NULL
+        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
         GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
         ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]);
 
@@ -55,7 +56,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
                 row.categorySumPreview,
                 row.categorySumHighlight,
                 row.categorySumFiller,
-                row.categorySumExclusiveAccess
+                row.categorySumExclusiveAccess,
+                row.categorySumChapter
             ]);
         }
     }

src/routes/getUserInfo.ts

@@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise<Record<string, bool
 
 async function getFreeChaptersAccess(userID: HashedUserID): Promise<boolean> {
     return await oneOf([isUserVIP(userID),
-        (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(),
+        (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
-        (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
     ]);
 }
 

src/routes/postSkipSegments.ts

@@ -334,7 +334,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
             return {
                 pass: false,
                 errorCode: 403,
-                errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
+                errorMessage: `Hi, currently there are server issues and you might have not recieved segments even though they exist. Sorry about this, I'm working on it. Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
             };
         }
     }

src/routes/voteOnSponsorTime.ts

@@ -14,6 +14,7 @@ import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID
 import { QueryCacher } from "../utils/queryCacher";
 import axios from "axios";
 import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
+import { deleteLockCategories } from "./deleteLockCategories";
 
 const voteTypes = {
     normal: 0,
@@ -59,7 +60,7 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) {
     let apiVideoDetails: videoDetails = null;
     if (service == Service.YouTube) {
         // don't use cache since we have no information about the video length
-        apiVideoDetails = await getVideoDetails(videoID);
+        apiVideoDetails = await getVideoDetails(videoID, true);
     }
     const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
     if (videoDurationChanged(videoDuration, apiVideoDuration)) {
@@ -95,6 +96,7 @@ async function checkVideoDuration(UUID: SegmentUUID) {
             AND "hidden" = 0 AND "shadowHidden" = 0 AND 
             "actionType" != 'full' AND "votes" > -2`,
         [videoID, service, latestSubmission.timeSubmitted]);
+        deleteLockCategories(videoID, null, null, service).catch(Logger.error);
     }
 }
 

src/types/config.model.ts

@@ -7,16 +7,21 @@ interface RedisConfig extends redis.RedisClientOptions {
     getTimeout: number;
 }
 
+interface RedisReadOnlyConfig extends redis.RedisClientOptions {
+    enabled: boolean;
+    weight: number;
+}
+
 export interface CustomPostgresConfig extends PoolConfig {
     enabled: boolean;
     maxTries: number;
-    maxConcurrentRequests: number;
 }
 
 export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig {
     weight: number;
     readTimeout: number;
     fallbackOnFail: boolean;
+    stopRetryThreshold: number;
 }
 
 export interface SBSConfig {
@@ -61,6 +66,7 @@ export interface SBSConfig {
     minimumPrefix?: string;
     maximumPrefix?: string;
     redis?: RedisConfig;
+    redisRead?: RedisReadOnlyConfig;
     maxRewardTimePerSegmentInSeconds?: number;
     postgres?: CustomPostgresConfig;
     postgresReadOnly?: CustomPostgresReadOnlyConfig;

src/utils/permissions.ts

@@ -9,7 +9,7 @@ import { getReputation } from "./reputation";
 
 interface CanSubmitResult {
     canSubmit: boolean;
-    reason?: string;
+    reason: string;
 }
 
 async function lowDownvotes(userID: HashedUserID): Promise<boolean> {
@@ -27,11 +27,13 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi
                     lowDownvotes(userID),
                     (async () => (await getReputation(userID)) > config.minReputationToSubmitChapter)(),
                     hasFeature(userID, Feature.ChapterSubmitter)
-                ])
+                ]),
+                reason: "Submitting chapters requires a minimum reputation. You can ask on Discord/Matrix to get permission with less reputation."
             };
         default:
             return {
-                canSubmit: true
+                canSubmit: true,
+                reason: ""
             };
     }
 }

src/utils/redis.ts

@@ -25,19 +25,40 @@ let exportClient: RedisSB = {
     quit: () => new Promise((resolve) => resolve(null)),
 };
 
+let lastClientFail = 0;
+let lastReadFail = 0;
+let activeRequests = 0;
+
 if (config.redis?.enabled) {
     Logger.info("Connected to redis");
     const client = createClient(config.redis);
+    const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null;
     void client.connect(); // void as we don't care about the promise
+    void readClient?.connect();
     exportClient = client as RedisSB;
 
+
     const get = client.get.bind(client);
+    const getRead = readClient?.get?.bind(readClient);
     exportClient.get = (key) => new Promise((resolve, reject) => {
+        activeRequests++;
         const timeout = config.redis.getTimeout ? setTimeout(() => reject(), config.redis.getTimeout) : null;
-        get(key).then((reply) => {
+        const chosenGet = pickChoice(get, getRead);
+        chosenGet(key).then((reply) => {
             if (timeout !== null) clearTimeout(timeout);
+
+            activeRequests--;
             resolve(reply);
-        }).catch((err) => reject(err));
+        }).catch((err) => {
+            if (chosenGet === get) {
+                lastClientFail = Date.now();
+            } else {
+                lastReadFail = Date.now();
+            }
+
+            activeRequests--;
+            reject(err);
+        });
     });
     exportClient.increment = (key) => new Promise((resolve, reject) =>
         void client.multi()
@@ -48,11 +69,35 @@ if (config.redis?.enabled) {
             .catch((err) => reject(err))
     );
     client.on("error", function(error) {
+        lastClientFail = Date.now();
         Logger.error(`Redis Error: ${error}`);
     });
     client.on("reconnect", () => {
         Logger.info("Redis: trying to reconnect");
     });
+    readClient?.on("error", function(error) {
+        lastReadFail = Date.now();
+        Logger.error(`Redis Read-Only Error: ${error}`);
+    });
+    readClient?.on("reconnect", () => {
+        Logger.info("Redis Read-Only: trying to reconnect");
+    });
+}
+
+function pickChoice<T>(client: T, readClient: T): T {
+    const readAvailable = !!readClient;
+    const ignoreReadDueToFailure = lastReadFail > Date.now() - 1000 * 30;
+    const readDueToFailure = lastClientFail > Date.now() - 1000 * 30;
+    if (readAvailable && !ignoreReadDueToFailure && (readDueToFailure ||
+            Math.random() > 1 / (config.redisRead?.weight + 1))) {
+        return readClient;
+    } else {
+        return client;
+    }
+}
+
+export function getRedisActiveRequests(): number {
+    return activeRequests;
 }
 
 export default exportClient;

test/cases/getSearchSegments.ts

@@ -335,7 +335,8 @@ describe("getSearchSegments", () => {
                     locked: 1,
                     hidden: 0,
                     shadowHidden: 0,
-                    userID: "searchTestUser"
+                    userID: "searchTestUser",
+                    description: ""
                 };
                 assert.deepStrictEqual(segment0, expected);
                 done();