diff --git a/Dockerfile b/Dockerfile index 26b77cb..ee5805b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,11 @@ RUN npm ci && npm run tsc FROM node:16-alpine as app WORKDIR /usr/src/app -RUN apk add git postgresql-client +RUN apk add --no-cache git postgresql-client COPY --from=builder ./node_modules ./node_modules COPY --from=builder ./dist ./dist COPY ./.git ./.git COPY entrypoint.sh . COPY databases/*.sql databases/ EXPOSE 8080 -CMD ./entrypoint.sh \ No newline at end of file +CMD ./entrypoint.sh diff --git a/src/config.ts b/src/config.ts index 70e1fc3..bbccfae 100644 --- a/src/config.ts +++ b/src/config.ts @@ -76,8 +76,7 @@ addDefaults(config, { port: 5432, max: 10, idleTimeoutMillis: 10000, - maxTries: 3, - maxConcurrentRequests: 3500 + maxTries: 3 }, postgresReadOnly: { enabled: false, @@ -91,7 +90,7 @@ addDefaults(config, { idleTimeoutMillis: 10000, maxTries: 3, fallbackOnFail: true, - maxConcurrentRequests: 3500 + stopRetryThreshold: 800 }, dumpDatabase: { enabled: false, @@ -139,6 +138,15 @@ addDefaults(config, { expiryTime: 24 * 60 * 60, getTimeout: 40 }, + redisRead: { + enabled: false, + socket: { + host: "", + port: 0 + }, + disableOfflineQueue: true, + weight: 1 + }, patreon: { clientId: "", clientSecret: "", diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 01f59b1..db72021 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -33,8 +33,7 @@ export class Postgres implements IDatabase { private poolRead: Pool; private lastPoolReadFail = 0; - private concurrentRequests = 0; - private concurrentReadRequests = 0; + activePostgresRequests = 0; constructor(private config: DatabaseConfig) {} @@ -54,19 +53,23 @@ export class Postgres implements IDatabase { }); if (this.config.postgresReadOnly && this.config.postgresReadOnly.enabled) { - this.poolRead = new Pool({ - ...this.config.postgresReadOnly - }); - this.poolRead.on("error", (err, client) => { - Logger.error(err.stack); - this.lastPoolReadFail = Date.now(); + try { + this.poolRead = new Pool({ + ...this.config.postgresReadOnly + }); + this.poolRead.on("error", (err, client) => { + Logger.error(err.stack); + this.lastPoolReadFail = Date.now(); - try { - client.release(true); - } catch (err) { - Logger.error(`poolRead (postgres): ${err}`); - } - }); + try { + client.release(true); + } catch (err) { + Logger.error(`poolRead (postgres): ${err}`); + } + }); + } catch (e) { + Logger.error(`poolRead (postgres): ${e}`); + } } if (!this.config.readOnly) { @@ -102,22 +105,6 @@ export class Postgres implements IDatabase { Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); - if (this.config.readOnly) { - if (this.concurrentReadRequests > this.config.postgresReadOnly?.maxConcurrentRequests) { - Logger.error(`prepare (postgres): cancelling read query because too many concurrent requests, query: ${query}`); - throw new Error("Too many concurrent requests"); - } - - this.concurrentReadRequests++; - } else { - if (this.concurrentRequests > this.config.postgres.maxConcurrentRequests) { - Logger.error(`prepare (postgres): cancelling query because too many concurrent requests, query: ${query}`); - throw new Error("Too many concurrent requests"); - } - - this.concurrentRequests++; - } - const pendingQueries: PromiseWithState>[] = []; let tries = 0; let lastPool: Pool = null; @@ -127,6 +114,7 @@ export class Postgres implements IDatabase { tries++; try { + this.activePostgresRequests++; lastPool = this.getPool(type, options); pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params }))); @@ -134,12 +122,7 @@ export class Postgres implements IDatabase { if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout))); const queryResult = await nextFulfilment(currentPromises); - if (this.config.readOnly) { - this.concurrentReadRequests--; - } else { - this.concurrentRequests--; - } - + this.activePostgresRequests--; switch (type) { case "get": { const value = queryResult.rows[0]; @@ -159,30 +142,30 @@ export class Postgres implements IDatabase { if (lastPool === this.pool) { // Only applies if it is get or all request options.forceReplica = true; - } else if (lastPool === this.poolRead && maxTries() - tries <= 1) { - options.useReplica = false; + } else if (lastPool === this.poolRead) { + this.lastPoolReadFail = Date.now(); + + if (maxTries() - tries <= 1) { + options.useReplica = false; + } } Logger.error(`prepare (postgres) try ${tries}: ${err}`); } - } while (this.isReadQuery(type) && tries < maxTries()); - - if (this.config.readOnly) { - this.concurrentReadRequests--; - } else { - this.concurrentRequests--; - } + } while (this.isReadQuery(type) && tries < maxTries() + && this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold); + this.activePostgresRequests--; throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`); } private getPool(type: string, options: QueryOption): Pool { const readAvailable = this.poolRead && options.useReplica && this.isReadQuery(type); - const ignroreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail + const ignoreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail && this.lastPoolReadFail > Date.now() - 1000 * 30; const readDueToFailure = this.config.postgresReadOnly.fallbackOnFail && this.lastPoolFail > Date.now() - 1000 * 30; - if (readAvailable && !ignroreReadDueToFailure && (options.forceReplica || readDueToFailure || + if (readAvailable && !ignoreReadDueToFailure && (options.forceReplica || readDueToFailure || Math.random() > 1 / (this.config.postgresReadOnly.weight + 1))) { return this.poolRead; } else { diff --git a/src/index.ts b/src/index.ts index e34882a..88c6562 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,13 @@ async function init() { process.exit(1); }); - await initDb(); + try { + await initDb(); + } catch (e) { + Logger.error(`Init Db: ${e}`); + process.exit(1); + } + // edge case clause for creating compatible .db files, do not enable if (config.mode === "init-db-and-exit") process.exit(0); // do not enable init-db-only mode for usage. @@ -27,4 +33,4 @@ async function init() { }).setTimeout(15000); } -init().catch((err) => Logger.error(err)); \ No newline at end of file +init().catch((err) => Logger.error(`Index.js: ${err}`)); \ No newline at end of file diff --git a/src/routes/generateToken.ts b/src/routes/generateToken.ts index cc88992..d617ecc 100644 --- a/src/routes/generateToken.ts +++ b/src/routes/generateToken.ts @@ -1,7 +1,7 @@ import { Request, Response } from "express"; import { config } from "../config"; import { createAndSaveToken, TokenType } from "../utils/tokenUtils"; - +import { getHashCache } from "../utils/getHashCache"; interface GenerateTokenRequest extends Request { query: { @@ -15,12 +15,13 @@ interface GenerateTokenRequest extends Request { export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise { const { query: { code, adminUserID }, params: { type } } = req; + const adminUserIDHash = adminUserID ? (await getHashCache(adminUserID)) : null; if (!code || !type) { return res.status(400).send("Invalid request"); } - if (type === TokenType.patreon || (type === TokenType.local && adminUserID === config.adminUserID)) { + if (type === TokenType.patreon || (type === TokenType.local && adminUserIDHash === config.adminUserID)) { const licenseKey = await createAndSaveToken(type, code); /* istanbul ignore else */ diff --git a/src/routes/getSearchSegments.ts b/src/routes/getSearchSegments.ts index 9dec9df..31b2c93 100644 --- a/src/routes/getSearchSegments.ts +++ b/src/routes/getSearchSegments.ts @@ -14,7 +14,7 @@ type searchSegmentResponse = { function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise { return db.prepare( "all", - `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID" FROM "sponsorTimes" + `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`, [videoID, service] ) as Promise; diff --git a/src/routes/getStatus.ts b/src/routes/getStatus.ts index c7413ff..6315fdf 100644 --- a/src/routes/getStatus.ts +++ b/src/routes/getStatus.ts @@ -2,8 +2,9 @@ import { db } from "../databases/databases"; import { Logger } from "../utils/logger"; import { Request, Response } from "express"; import os from "os"; -import redis from "../utils/redis"; +import redis, { getRedisActiveRequests } from "../utils/redis"; import { promiseOrTimeout } from "../utils/promise"; +import { Postgres } from "../databases/Postgres"; export async function getStatus(req: Request, res: Response): Promise { const startTime = Date.now(); @@ -11,9 +12,10 @@ export async function getStatus(req: Request, res: Response): Promise value = Array.isArray(value) ? value[0] : value; let processTime, redisProcessTime = -1; try { + const dbStartTime = Date.now(); const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000) .then(e => { - processTime = Date.now() - startTime; + processTime = Date.now() - dbStartTime; return e.value; }) .catch(e => /* istanbul ignore next */ { @@ -21,9 +23,10 @@ export async function getStatus(req: Request, res: Response): Promise return -1; }); let statusRequests: unknown = 0; + const redisStartTime = Date.now(); const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000) .then(e => { - redisProcessTime = Date.now() - startTime; + redisProcessTime = Date.now() - redisStartTime; return e; }).catch(e => /* istanbul ignore next */ { Logger.error(`status: redis increment timed out ${e}`); @@ -40,7 +43,9 @@ export async function getStatus(req: Request, res: Response): Promise redisProcessTime, loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average statusRequests, - hostname: os.hostname() + hostname: os.hostname(), + activePostgresRequests: (db as Postgres)?.activePostgresRequests, + activeRedisRequests: getRedisActiveRequests(), }; return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues); } catch (err) /* istanbul ignore next */ { diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index 1197e6a..21bc9eb 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL - GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 + GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 2 ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]); if (rows) { diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index b4b83a3..552aaf9 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -28,14 +28,15 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals SUM(CASE WHEN category = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight", SUM(CASE WHEN category = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller", SUM(CASE WHEN category = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess", + SUM(CASE WHEN category = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter", `; } const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", - SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved", + SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views" END) as "minutesSaved", SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" - WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "sponsorTimes"."actionType" != 'chapter' AND "shadowBannedUsers"."userID" IS NULL + WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]); @@ -55,7 +56,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals row.categorySumPreview, row.categorySumHighlight, row.categorySumFiller, - row.categorySumExclusiveAccess + row.categorySumExclusiveAccess, + row.categorySumChapter ]); } } diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index b8913ef..70f3464 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise { return await oneOf([isUserVIP(userID), - (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(), - (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))() + (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))() ]); } diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 97cd3c7..f3efaac 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -334,7 +334,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array -2`, [videoID, service, latestSubmission.timeSubmitted]); + deleteLockCategories(videoID, null, null, service).catch(Logger.error); } } diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 406643c..c9ae96e 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -7,16 +7,21 @@ interface RedisConfig extends redis.RedisClientOptions { getTimeout: number; } +interface RedisReadOnlyConfig extends redis.RedisClientOptions { + enabled: boolean; + weight: number; +} + export interface CustomPostgresConfig extends PoolConfig { enabled: boolean; maxTries: number; - maxConcurrentRequests: number; } export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig { weight: number; readTimeout: number; fallbackOnFail: boolean; + stopRetryThreshold: number; } export interface SBSConfig { @@ -61,6 +66,7 @@ export interface SBSConfig { minimumPrefix?: string; maximumPrefix?: string; redis?: RedisConfig; + redisRead?: RedisReadOnlyConfig; maxRewardTimePerSegmentInSeconds?: number; postgres?: CustomPostgresConfig; postgresReadOnly?: CustomPostgresReadOnlyConfig; diff --git a/src/utils/permissions.ts b/src/utils/permissions.ts index f64000d..b981883 100644 --- a/src/utils/permissions.ts +++ b/src/utils/permissions.ts @@ -9,7 +9,7 @@ import { getReputation } from "./reputation"; interface CanSubmitResult { canSubmit: boolean; - reason?: string; + reason: string; } async function lowDownvotes(userID: HashedUserID): Promise { @@ -27,11 +27,13 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi lowDownvotes(userID), (async () => (await getReputation(userID)) > config.minReputationToSubmitChapter)(), hasFeature(userID, Feature.ChapterSubmitter) - ]) + ]), + reason: "Submitting chapters requires a minimum reputation. You can ask on Discord/Matrix to get permission with less reputation." }; default: return { - canSubmit: true + canSubmit: true, + reason: "" }; } } \ No newline at end of file diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 8e8c6d0..27b552c 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -25,19 +25,40 @@ let exportClient: RedisSB = { quit: () => new Promise((resolve) => resolve(null)), }; +let lastClientFail = 0; +let lastReadFail = 0; +let activeRequests = 0; + if (config.redis?.enabled) { Logger.info("Connected to redis"); const client = createClient(config.redis); + const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null; void client.connect(); // void as we don't care about the promise + void readClient?.connect(); exportClient = client as RedisSB; + const get = client.get.bind(client); + const getRead = readClient?.get?.bind(readClient); exportClient.get = (key) => new Promise((resolve, reject) => { + activeRequests++; const timeout = config.redis.getTimeout ? setTimeout(() => reject(), config.redis.getTimeout) : null; - get(key).then((reply) => { + const chosenGet = pickChoice(get, getRead); + chosenGet(key).then((reply) => { if (timeout !== null) clearTimeout(timeout); + + activeRequests--; resolve(reply); - }).catch((err) => reject(err)); + }).catch((err) => { + if (chosenGet === get) { + lastClientFail = Date.now(); + } else { + lastReadFail = Date.now(); + } + + activeRequests--; + reject(err); + }); }); exportClient.increment = (key) => new Promise((resolve, reject) => void client.multi() @@ -48,11 +69,35 @@ if (config.redis?.enabled) { .catch((err) => reject(err)) ); client.on("error", function(error) { + lastClientFail = Date.now(); Logger.error(`Redis Error: ${error}`); }); client.on("reconnect", () => { Logger.info("Redis: trying to reconnect"); }); + readClient?.on("error", function(error) { + lastReadFail = Date.now(); + Logger.error(`Redis Read-Only Error: ${error}`); + }); + readClient?.on("reconnect", () => { + Logger.info("Redis Read-Only: trying to reconnect"); + }); +} + +function pickChoice(client: T, readClient: T): T { + const readAvailable = !!readClient; + const ignoreReadDueToFailure = lastReadFail > Date.now() - 1000 * 30; + const readDueToFailure = lastClientFail > Date.now() - 1000 * 30; + if (readAvailable && !ignoreReadDueToFailure && (readDueToFailure || + Math.random() > 1 / (config.redisRead?.weight + 1))) { + return readClient; + } else { + return client; + } +} + +export function getRedisActiveRequests(): number { + return activeRequests; } export default exportClient; diff --git a/test/cases/getSearchSegments.ts b/test/cases/getSearchSegments.ts index c18bef2..6dbef4f 100644 --- a/test/cases/getSearchSegments.ts +++ b/test/cases/getSearchSegments.ts @@ -335,7 +335,8 @@ describe("getSearchSegments", () => { locked: 1, hidden: 0, shadowHidden: 0, - userID: "searchTestUser" + userID: "searchTestUser", + description: "" }; assert.deepStrictEqual(segment0, expected); done();