From eb5458427d1d984688248d7579bb97d005684785 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 1 Oct 2022 13:46:27 +0200 Subject: [PATCH 01/23] Make /searchSegments return the segment description --- src/routes/getSearchSegments.ts | 2 +- test/cases/getSearchSegments.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/routes/getSearchSegments.ts b/src/routes/getSearchSegments.ts index d8fc860..a7b9110 100644 --- a/src/routes/getSearchSegments.ts +++ b/src/routes/getSearchSegments.ts @@ -14,7 +14,7 @@ type searchSegmentResponse = { function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise { return db.prepare( "all", - `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID" FROM "sponsorTimes" + `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`, [videoID, service] ) as Promise; diff --git a/test/cases/getSearchSegments.ts b/test/cases/getSearchSegments.ts index 0084204..4e78604 100644 --- a/test/cases/getSearchSegments.ts +++ b/test/cases/getSearchSegments.ts @@ -274,7 +274,8 @@ describe("getSearchSegments", () => { locked: 1, hidden: 0, shadowHidden: 0, - userID: "searchTestUser" + userID: "searchTestUser", + description: "" }; assert.deepStrictEqual(segment0, expected); done(); From c4af2449c34215e87e43d3d25e33024de2fcb513 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 1 Oct 2022 15:50:47 +0200 Subject: [PATCH 02/23] Use different startTime variables for each processTime check This should make `processTime` and `redisProcessTime` values from /api/status more accurate --- src/routes/getStatus.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/routes/getStatus.ts b/src/routes/getStatus.ts index b817450..1f6dfcf 100644 --- a/src/routes/getStatus.ts +++ b/src/routes/getStatus.ts @@ -11,9 +11,10 @@ export async function getStatus(req: Request, res: Response): Promise value = Array.isArray(value) ? value[0] : value; let processTime, redisProcessTime = -1; try { + const dbStartTime = Date.now(); const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000) .then(e => { - processTime = Date.now() - startTime; + processTime = Date.now() - dbStartTime; return e.value; }) .catch(e => { @@ -21,9 +22,10 @@ export async function getStatus(req: Request, res: Response): Promise return -1; }); let statusRequests: unknown = 0; + const redisStartTime = Date.now(); const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000) .then(e => { - redisProcessTime = Date.now() - startTime; + redisProcessTime = Date.now() - redisStartTime; return e; }).catch(e => { Logger.error(`status: redis increment timed out ${e}`); From b63572ec431225037c4952262ca64ec041c0f72c Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 5 Oct 2022 22:00:12 -0400 Subject: [PATCH 03/23] Add chapter to top users --- src/routes/getTopUsers.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index 5a2b86c..d47d94c 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -28,14 +28,15 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals SUM(CASE WHEN category = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight", SUM(CASE WHEN category = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller", SUM(CASE WHEN category = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess", + SUM(CASE WHEN category = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter", `; } const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", - SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved", + SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 6) * "sponsorTimes"."views" END) as "minutesSaved", SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" - WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "sponsorTimes"."actionType" != 'chapter' AND "shadowBannedUsers"."userID" IS NULL + WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]); @@ -55,7 +56,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals row.categorySumPreview, row.categorySumHighlight, row.categorySumFiller, - row.categorySumExclusiveAccess + row.categorySumExclusiveAccess, + row.categorySumChapter ]); } } From 52a7d7e79167f72c3c88fc3f03703246855426c6 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 5 Oct 2022 23:08:16 -0400 Subject: [PATCH 04/23] allow locked to be in top list and chapter --- src/routes/getTopCategoryUsers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index 1197e6a..c368519 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL - GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 + GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 OR SUM("locked") > 0 OR "category" = 'chapter' ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]); if (rows) { From b6da103c3d5af915c5d9087a26fec44c170c87d8 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 5 Oct 2022 23:15:01 -0400 Subject: [PATCH 05/23] switch to just lower vote min --- src/routes/getTopCategoryUsers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index c368519..21bc9eb 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL - GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 OR SUM("locked") > 0 OR "category" = 'chapter' + GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 2 ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]); if (rows) { From ee3d94e7b777b949500ab7c51003e8139d00a42b Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 7 Oct 2022 12:04:44 -0400 Subject: [PATCH 06/23] Add clearer chapter permission error --- src/utils/permissions.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/permissions.ts b/src/utils/permissions.ts index f64000d..716a258 100644 --- a/src/utils/permissions.ts +++ b/src/utils/permissions.ts @@ -27,7 +27,8 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi lowDownvotes(userID), (async () => (await getReputation(userID)) > config.minReputationToSubmitChapter)(), hasFeature(userID, Feature.ChapterSubmitter) - ]) + ]), + reason: "Submitting chapters requires a minimum reputation. You can ask on Discord/Matrix to get permission with less reputation." }; default: return { From b417241ca033a1c2a0b2c03ef726e78b37087194 Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 7 Oct 2022 12:05:11 -0400 Subject: [PATCH 07/23] make permission reason not optional --- src/utils/permissions.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/permissions.ts b/src/utils/permissions.ts index 716a258..b981883 100644 --- a/src/utils/permissions.ts +++ b/src/utils/permissions.ts @@ -9,7 +9,7 @@ import { getReputation } from "./reputation"; interface CanSubmitResult { canSubmit: boolean; - reason?: string; + reason: string; } async function lowDownvotes(userID: HashedUserID): Promise { @@ -32,7 +32,8 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi }; default: return { - canSubmit: true + canSubmit: true, + reason: "" }; } } \ No newline at end of file From 1becebdcd5565db9e3ba8fff1b5d444ab26c7ec6 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 8 Oct 2022 09:04:37 +0200 Subject: [PATCH 08/23] Actually pass the ignoreCache param, as the comment suggests --- src/routes/voteOnSponsorTime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 97b7c5c..bba06fb 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -59,7 +59,7 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) { let apiVideoDetails: videoDetails = null; if (service == Service.YouTube) { // don't use cache since we have no information about the video length - apiVideoDetails = await getVideoDetails(videoID); + apiVideoDetails = await getVideoDetails(videoID, true); } const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; if (videoDurationChanged(videoDuration, apiVideoDuration)) { From d229003f6e61d5ef1cb882ed73483bc1c8fbff04 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 8 Oct 2022 09:08:40 +0200 Subject: [PATCH 09/23] Unlock the video on duration change detected when voting --- src/routes/voteOnSponsorTime.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index bba06fb..1f8c619 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -14,6 +14,7 @@ import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID import { QueryCacher } from "../utils/queryCacher"; import axios from "axios"; import { getVideoDetails, videoDetails } from "../utils/getVideoDetails"; +import { deleteLockCategories } from "./deleteLockCategories"; const voteTypes = { normal: 0, @@ -95,6 +96,7 @@ async function checkVideoDuration(UUID: SegmentUUID) { AND "hidden" = 0 AND "shadowHidden" = 0 AND "actionType" != 'full' AND "votes" > -2`, [videoID, service, latestSubmission.timeSubmitted]); + deleteLockCategories(videoID, null, null, service).catch(Logger.error); } } From ceaf9ec6f629127b19613eb3a1b926a187da8eb7 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 8 Oct 2022 09:09:19 +0200 Subject: [PATCH 10/23] Fix linter warning --- src/routes/voteOnSponsorTime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 1f8c619..bbda1e2 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -221,7 +221,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i [UUID], { useReplica: true })) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; if (!config.categorySupport[category]?.includes(segmentInfo.actionType) || segmentInfo.actionType === ActionType.Full) { - return { status: 400, message: `Not allowed to change to ${category} when for segment of type ${segmentInfo.actionType}`}; + return { status: 400, message: `Not allowed to change to ${category} when for segment of type ${segmentInfo.actionType}` }; } if (!config.categoryList.includes(category)) { return { status: 400, message: "Category doesn't exist." }; From 9386f25f9fbf5778fd4cb2e7877dfff5bf1cb01d Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 9 Oct 2022 16:10:46 -0400 Subject: [PATCH 11/23] Add read only redis ability --- src/config.ts | 9 +++++++++ src/databases/Postgres.ts | 4 ++-- src/types/config.model.ts | 6 ++++++ src/utils/redis.ts | 40 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 70e1fc3..1d5ed94 100644 --- a/src/config.ts +++ b/src/config.ts @@ -139,6 +139,15 @@ addDefaults(config, { expiryTime: 24 * 60 * 60, getTimeout: 40 }, + redisRead: { + enabled: false, + socket: { + host: "", + port: 0 + }, + disableOfflineQueue: true, + weight: 1 + }, patreon: { clientId: "", clientSecret: "", diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 01f59b1..c10f094 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -178,11 +178,11 @@ export class Postgres implements IDatabase { private getPool(type: string, options: QueryOption): Pool { const readAvailable = this.poolRead && options.useReplica && this.isReadQuery(type); - const ignroreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail + const ignoreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail && this.lastPoolReadFail > Date.now() - 1000 * 30; const readDueToFailure = this.config.postgresReadOnly.fallbackOnFail && this.lastPoolFail > Date.now() - 1000 * 30; - if (readAvailable && !ignroreReadDueToFailure && (options.forceReplica || readDueToFailure || + if (readAvailable && !ignoreReadDueToFailure && (options.forceReplica || readDueToFailure || Math.random() > 1 / (this.config.postgresReadOnly.weight + 1))) { return this.poolRead; } else { diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 406643c..e0498fb 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -7,6 +7,11 @@ interface RedisConfig extends redis.RedisClientOptions { getTimeout: number; } +interface RedisReadOnlyConfig extends redis.RedisClientOptions { + enabled: boolean; + weight: number; +} + export interface CustomPostgresConfig extends PoolConfig { enabled: boolean; maxTries: number; @@ -61,6 +66,7 @@ export interface SBSConfig { minimumPrefix?: string; maximumPrefix?: string; redis?: RedisConfig; + redisRead?: RedisReadOnlyConfig; maxRewardTimePerSegmentInSeconds?: number; postgres?: CustomPostgresConfig; postgresReadOnly?: CustomPostgresReadOnlyConfig; diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 8e8c6d0..6c13133 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -25,19 +25,35 @@ let exportClient: RedisSB = { quit: () => new Promise((resolve) => resolve(null)), }; +let lastClientFail = 0; +let lastReadFail = 0; + if (config.redis?.enabled) { Logger.info("Connected to redis"); const client = createClient(config.redis); + const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null; void client.connect(); // void as we don't care about the promise + void readClient?.connect(); exportClient = client as RedisSB; + const get = client.get.bind(client); + const getRead = readClient?.get?.bind(readClient); exportClient.get = (key) => new Promise((resolve, reject) => { const timeout = config.redis.getTimeout ? setTimeout(() => reject(), config.redis.getTimeout) : null; - get(key).then((reply) => { + const chosenGet = pickChoice(get, getRead); + chosenGet(key).then((reply) => { if (timeout !== null) clearTimeout(timeout); resolve(reply); - }).catch((err) => reject(err)); + }).catch((err) => { + if (chosenGet === get) { + lastClientFail = Date.now(); + } else { + lastReadFail = Date.now(); + } + + reject(err); + }); }); exportClient.increment = (key) => new Promise((resolve, reject) => void client.multi() @@ -48,11 +64,31 @@ if (config.redis?.enabled) { .catch((err) => reject(err)) ); client.on("error", function(error) { + lastClientFail = Date.now(); Logger.error(`Redis Error: ${error}`); }); client.on("reconnect", () => { Logger.info("Redis: trying to reconnect"); }); + readClient?.on("error", function(error) { + lastReadFail = Date.now(); + Logger.error(`Redis Read-Only Error: ${error}`); + }); + readClient?.on("reconnect", () => { + Logger.info("Redis Read-Only: trying to reconnect"); + }); +} + +function pickChoice(client: T, readClient: T): T { + const readAvailable = !!readClient; + const ignoreReadDueToFailure = lastReadFail > Date.now() - 1000 * 30; + const readDueToFailure = lastClientFail > Date.now() - 1000 * 30; + if (readAvailable && !ignoreReadDueToFailure && (readDueToFailure || + Math.random() > 1 / (config.redisRead?.weight + 1))) { + return readClient; + } else { + return client; + } } export default exportClient; From 415bb31e366822a91c7d929ff2750f44fe2b3f40 Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 11 Oct 2022 11:59:53 -0400 Subject: [PATCH 12/23] Revert "Add concurrent request limit" --- src/config.ts | 6 ++---- src/databases/Postgres.ts | 31 ------------------------------- src/types/config.model.ts | 1 - 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/src/config.ts b/src/config.ts index 1d5ed94..96d84d4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -76,8 +76,7 @@ addDefaults(config, { port: 5432, max: 10, idleTimeoutMillis: 10000, - maxTries: 3, - maxConcurrentRequests: 3500 + maxTries: 3 }, postgresReadOnly: { enabled: false, @@ -90,8 +89,7 @@ addDefaults(config, { max: 10, idleTimeoutMillis: 10000, maxTries: 3, - fallbackOnFail: true, - maxConcurrentRequests: 3500 + fallbackOnFail: true }, dumpDatabase: { enabled: false, diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index c10f094..6711897 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -33,9 +33,6 @@ export class Postgres implements IDatabase { private poolRead: Pool; private lastPoolReadFail = 0; - private concurrentRequests = 0; - private concurrentReadRequests = 0; - constructor(private config: DatabaseConfig) {} async init(): Promise { @@ -102,22 +99,6 @@ export class Postgres implements IDatabase { Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); - if (this.config.readOnly) { - if (this.concurrentReadRequests > this.config.postgresReadOnly?.maxConcurrentRequests) { - Logger.error(`prepare (postgres): cancelling read query because too many concurrent requests, query: ${query}`); - throw new Error("Too many concurrent requests"); - } - - this.concurrentReadRequests++; - } else { - if (this.concurrentRequests > this.config.postgres.maxConcurrentRequests) { - Logger.error(`prepare (postgres): cancelling query because too many concurrent requests, query: ${query}`); - throw new Error("Too many concurrent requests"); - } - - this.concurrentRequests++; - } - const pendingQueries: PromiseWithState>[] = []; let tries = 0; let lastPool: Pool = null; @@ -134,12 +115,6 @@ export class Postgres implements IDatabase { if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout))); const queryResult = await nextFulfilment(currentPromises); - if (this.config.readOnly) { - this.concurrentReadRequests--; - } else { - this.concurrentRequests--; - } - switch (type) { case "get": { const value = queryResult.rows[0]; @@ -167,12 +142,6 @@ export class Postgres implements IDatabase { } } while (this.isReadQuery(type) && tries < maxTries()); - if (this.config.readOnly) { - this.concurrentReadRequests--; - } else { - this.concurrentRequests--; - } - throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`); } diff --git a/src/types/config.model.ts b/src/types/config.model.ts index e0498fb..e6b221d 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -15,7 +15,6 @@ interface RedisReadOnlyConfig extends redis.RedisClientOptions { export interface CustomPostgresConfig extends PoolConfig { enabled: boolean; maxTries: number; - maxConcurrentRequests: number; } export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig { From e799821ad9c4da603a64ea95055fd35110eadf4d Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 13 Oct 2022 00:59:34 -0400 Subject: [PATCH 13/23] add additional token validation --- src/routes/generateToken.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/routes/generateToken.ts b/src/routes/generateToken.ts index 4c0771e..46069e0 100644 --- a/src/routes/generateToken.ts +++ b/src/routes/generateToken.ts @@ -1,7 +1,7 @@ import { Request, Response } from "express"; import { config } from "../config"; import { createAndSaveToken, TokenType } from "../utils/tokenUtils"; - +import { getHashCache } from "../utils/getHashCache"; interface GenerateTokenRequest extends Request { query: { @@ -15,12 +15,13 @@ interface GenerateTokenRequest extends Request { export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise { const { query: { code, adminUserID }, params: { type } } = req; + const adminUserIDHash = await getHashCache(adminUserID); if (!code || !type) { return res.status(400).send("Invalid request"); } - if (type === TokenType.patreon || (type === TokenType.local && adminUserID === config.adminUserID)) { + if (type === TokenType.patreon || (type === TokenType.local && adminUserIDHash === config.adminUserID)) { const licenseKey = await createAndSaveToken(type, code); if (licenseKey) { From b5434ae234a118e8adb1cf3044b7828da43ed0fc Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 18 Oct 2022 16:50:21 -0400 Subject: [PATCH 14/23] expand chapter free access --- src/routes/getUserInfo.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index a5e8ea3..62ac131 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise { return await oneOf([isUserVIP(userID), - (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(), - (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))() + (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))() ]); } From 10397fbde2d6f811d8ac5222fa078895fdd9a465 Mon Sep 17 00:00:00 2001 From: Brian Choromanski Date: Fri, 21 Oct 2022 17:27:34 -0400 Subject: [PATCH 15/23] Fixed minutesSaved typo --- src/routes/getTopUsers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index d47d94c..a40981e 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -33,7 +33,7 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals } const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", - SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 6) * "sponsorTimes"."views" END) as "minutesSaved", + SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views" END) as "minutesSaved", SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL From a6275b3607fc49044117a0e0a6286c614851244a Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 22 Oct 2022 13:32:48 -0400 Subject: [PATCH 16/23] Add message about server outage --- src/routes/postSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 97cd3c7..f3efaac 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -334,7 +334,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array Date: Sun, 23 Oct 2022 20:19:06 +0800 Subject: [PATCH 17/23] Leave no apk cache in the Docker image This will make the Docker image tidier --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 26b77cb..ee5805b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,11 @@ RUN npm ci && npm run tsc FROM node:16-alpine as app WORKDIR /usr/src/app -RUN apk add git postgresql-client +RUN apk add --no-cache git postgresql-client COPY --from=builder ./node_modules ./node_modules COPY --from=builder ./dist ./dist COPY ./.git ./.git COPY entrypoint.sh . COPY databases/*.sql databases/ EXPOSE 8080 -CMD ./entrypoint.sh \ No newline at end of file +CMD ./entrypoint.sh From 94eb37cb1ca69a67c1bdf7ee2ce937d1f7aff228 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 26 Oct 2022 01:31:29 -0400 Subject: [PATCH 18/23] Count active postgres and redis requests --- src/databases/Postgres.ts | 5 +++++ src/routes/getStatus.ts | 7 +++++-- src/utils/redis.ts | 9 +++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 6711897..1fa910d 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -33,6 +33,8 @@ export class Postgres implements IDatabase { private poolRead: Pool; private lastPoolReadFail = 0; + activePostgresRequests = 0; + constructor(private config: DatabaseConfig) {} async init(): Promise { @@ -108,6 +110,7 @@ export class Postgres implements IDatabase { tries++; try { + this.activePostgresRequests++; lastPool = this.getPool(type, options); pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params }))); @@ -115,6 +118,7 @@ export class Postgres implements IDatabase { if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout))); const queryResult = await nextFulfilment(currentPromises); + this.activePostgresRequests--; switch (type) { case "get": { const value = queryResult.rows[0]; @@ -142,6 +146,7 @@ export class Postgres implements IDatabase { } } while (this.isReadQuery(type) && tries < maxTries()); + this.activePostgresRequests--; throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`); } diff --git a/src/routes/getStatus.ts b/src/routes/getStatus.ts index 1f6dfcf..3f50fa1 100644 --- a/src/routes/getStatus.ts +++ b/src/routes/getStatus.ts @@ -2,8 +2,9 @@ import { db } from "../databases/databases"; import { Logger } from "../utils/logger"; import { Request, Response } from "express"; import os from "os"; -import redis from "../utils/redis"; +import redis, { getRedisActiveRequests } from "../utils/redis"; import { promiseOrTimeout } from "../utils/promise"; +import { Postgres } from "../databases/Postgres"; export async function getStatus(req: Request, res: Response): Promise { const startTime = Date.now(); @@ -42,7 +43,9 @@ export async function getStatus(req: Request, res: Response): Promise redisProcessTime, loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average statusRequests, - hostname: os.hostname() + hostname: os.hostname(), + activePostgresRequests: (db as Postgres)?.activePostgresRequests, + activeRedisRequests: getRedisActiveRequests(), }; return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues); } catch (err) { diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 6c13133..27b552c 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -27,6 +27,7 @@ let exportClient: RedisSB = { let lastClientFail = 0; let lastReadFail = 0; +let activeRequests = 0; if (config.redis?.enabled) { Logger.info("Connected to redis"); @@ -40,10 +41,13 @@ if (config.redis?.enabled) { const get = client.get.bind(client); const getRead = readClient?.get?.bind(readClient); exportClient.get = (key) => new Promise((resolve, reject) => { + activeRequests++; const timeout = config.redis.getTimeout ? setTimeout(() => reject(), config.redis.getTimeout) : null; const chosenGet = pickChoice(get, getRead); chosenGet(key).then((reply) => { if (timeout !== null) clearTimeout(timeout); + + activeRequests--; resolve(reply); }).catch((err) => { if (chosenGet === get) { @@ -52,6 +56,7 @@ if (config.redis?.enabled) { lastReadFail = Date.now(); } + activeRequests--; reject(err); }); }); @@ -91,4 +96,8 @@ function pickChoice(client: T, readClient: T): T { } } +export function getRedisActiveRequests(): number { + return activeRequests; +} + export default exportClient; From ee56a8dea4de5b77a6e7e251dfce0286e093ebb6 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 26 Oct 2022 12:49:39 -0400 Subject: [PATCH 19/23] Last pool fail when timeout --- src/databases/Postgres.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 1fa910d..fab0fb1 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -138,8 +138,12 @@ export class Postgres implements IDatabase { if (lastPool === this.pool) { // Only applies if it is get or all request options.forceReplica = true; - } else if (lastPool === this.poolRead && maxTries() - tries <= 1) { - options.useReplica = false; + } else if (lastPool === this.poolRead) { + this.lastPoolReadFail = Date.now(); + + if (maxTries() - tries <= 1) { + options.useReplica = false; + } } Logger.error(`prepare (postgres) try ${tries}: ${err}`); From 13ae4681cbdf74a153df2e6daf5d7924afc26985 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 26 Oct 2022 12:56:40 -0400 Subject: [PATCH 20/23] Fix hashing empty value for patreon sign in --- src/routes/generateToken.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/generateToken.ts b/src/routes/generateToken.ts index 46069e0..294a948 100644 --- a/src/routes/generateToken.ts +++ b/src/routes/generateToken.ts @@ -15,7 +15,7 @@ interface GenerateTokenRequest extends Request { export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise { const { query: { code, adminUserID }, params: { type } } = req; - const adminUserIDHash = await getHashCache(adminUserID); + const adminUserIDHash = adminUserID ? (await getHashCache(adminUserID)) : null; if (!code || !type) { return res.status(400).send("Invalid request"); From 9e3d059d109128fb19299cf7d95bfe4574aebb9c Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 26 Oct 2022 22:58:35 -0400 Subject: [PATCH 21/23] Stop postgres retries when over a threshold --- src/config.ts | 3 ++- src/databases/Postgres.ts | 3 ++- src/types/config.model.ts | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 96d84d4..bbccfae 100644 --- a/src/config.ts +++ b/src/config.ts @@ -89,7 +89,8 @@ addDefaults(config, { max: 10, idleTimeoutMillis: 10000, maxTries: 3, - fallbackOnFail: true + fallbackOnFail: true, + stopRetryThreshold: 800 }, dumpDatabase: { enabled: false, diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index fab0fb1..161a9c9 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -148,7 +148,8 @@ export class Postgres implements IDatabase { Logger.error(`prepare (postgres) try ${tries}: ${err}`); } - } while (this.isReadQuery(type) && tries < maxTries()); + } while (this.isReadQuery(type) && tries < maxTries() + && this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold); this.activePostgresRequests--; throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`); diff --git a/src/types/config.model.ts b/src/types/config.model.ts index e6b221d..c9ae96e 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -21,6 +21,7 @@ export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig { weight: number; readTimeout: number; fallbackOnFail: boolean; + stopRetryThreshold: number; } export interface SBSConfig { From 176f2ce7b9ff57671433e3b0dfdf88a162f088b2 Mon Sep 17 00:00:00 2001 From: Ajay Date: Thu, 27 Oct 2022 01:30:59 -0400 Subject: [PATCH 22/23] Kill if db can't connect --- src/index.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index e34882a..88c6562 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,13 @@ async function init() { process.exit(1); }); - await initDb(); + try { + await initDb(); + } catch (e) { + Logger.error(`Init Db: ${e}`); + process.exit(1); + } + // edge case clause for creating compatible .db files, do not enable if (config.mode === "init-db-and-exit") process.exit(0); // do not enable init-db-only mode for usage. @@ -27,4 +33,4 @@ async function init() { }).setTimeout(15000); } -init().catch((err) => Logger.error(err)); \ No newline at end of file +init().catch((err) => Logger.error(`Index.js: ${err}`)); \ No newline at end of file From c2acc6227b5820f0daffcf25b395dd99c8c2c788 Mon Sep 17 00:00:00 2001 From: Ajay Date: Thu, 27 Oct 2022 01:31:41 -0400 Subject: [PATCH 23/23] Don't kill program if can't connect to read server --- src/databases/Postgres.ts | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 161a9c9..db72021 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -53,19 +53,23 @@ export class Postgres implements IDatabase { }); if (this.config.postgresReadOnly && this.config.postgresReadOnly.enabled) { - this.poolRead = new Pool({ - ...this.config.postgresReadOnly - }); - this.poolRead.on("error", (err, client) => { - Logger.error(err.stack); - this.lastPoolReadFail = Date.now(); + try { + this.poolRead = new Pool({ + ...this.config.postgresReadOnly + }); + this.poolRead.on("error", (err, client) => { + Logger.error(err.stack); + this.lastPoolReadFail = Date.now(); - try { - client.release(true); - } catch (err) { - Logger.error(`poolRead (postgres): ${err}`); - } - }); + try { + client.release(true); + } catch (err) { + Logger.error(`poolRead (postgres): ${err}`); + } + }); + } catch (e) { + Logger.error(`poolRead (postgres): ${e}`); + } } if (!this.config.readOnly) {