From 50743070de16bfd3aa1dd2224c38477c9c6ac579 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 12:41:00 -0500 Subject: [PATCH 01/21] Clarify locked category error --- src/routes/postSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index f3efaac..eb976e8 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -266,7 +266,7 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user pass: false, errorCode: 403, errorMessage: - `Users have voted that new segments aren't needed for the following category: ` + + `Users have voted that new segments aren't needed on this video for the following category: ` + `'${segments[i].category}'\n` + `${lockedCategoryList[lockIndex].reason?.length !== 0 ? `\nReason: '${lockedCategoryList[lockIndex].reason}'` : ""}\n` + `${(segments[i].category === "sponsor" ? "\nMaybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + From e6e8c8e5a1ce28af7ac260b19d8e7179377a91d3 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 12:43:16 -0500 Subject: [PATCH 02/21] Added suggestion to refresh for locks --- src/routes/postSkipSegments.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index eb976e8..558946d 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -266,9 +266,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user pass: false, errorCode: 403, errorMessage: - `Users have voted that new segments aren't needed on this video for the following category: ` + + `Users have voted that all the necessary segments have been submitted for the following category: ` + `'${segments[i].category}'\n` + - `${lockedCategoryList[lockIndex].reason?.length !== 0 ? `\nReason: '${lockedCategoryList[lockIndex].reason}'` : ""}\n` + + `${lockedCategoryList[lockIndex].reason?.length !== 0 ? `\nReason: '${lockedCategoryList[lockIndex].reason}\n'` : ""}` + + `You may need to refresh if you don't see the segments.\n` + `${(segments[i].category === "sponsor" ? "\nMaybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n" : "")}` + `\nIf you believe this is incorrect, please contact someone on chat.sponsor.ajay.app, discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app` From 6c18579a78616cbcc1c55acc0c7eb0751c252a1b Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 12:45:35 -0500 Subject: [PATCH 03/21] Added back mention of video to lock message --- src/routes/postSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 558946d..aaf8eda 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -266,7 +266,7 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user pass: false, errorCode: 403, errorMessage: - `Users have voted that all the necessary segments have been submitted for the following category: ` + + `Users have voted that all the segments required for this video have already been submitted for the following category: ` + `'${segments[i].category}'\n` + `${lockedCategoryList[lockIndex].reason?.length !== 0 ? `\nReason: '${lockedCategoryList[lockIndex].reason}\n'` : ""}` + `You may need to refresh if you don't see the segments.\n` + From ed5e3373a56a5156d4388041207b66feb10ae9fc Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 15:29:55 -0500 Subject: [PATCH 04/21] Add max active postgres requests --- src/config.ts | 3 ++- src/databases/Postgres.ts | 8 ++++++-- src/types/config.model.ts | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 568aa0c..91020b6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -76,7 +76,8 @@ addDefaults(config, { port: 5432, max: 10, idleTimeoutMillis: 10000, - maxTries: 3 + maxTries: 3, + maxActiveRequests: -1 }, postgresReadOnly: { enabled: false, diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index db72021..a47da54 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -3,7 +3,7 @@ import { IDatabase, QueryOption, QueryType } from "./IDatabase"; import { Client, Pool, QueryResult, types } from "pg"; import fs from "fs"; -import { CustomPostgresConfig, CustomPostgresReadOnlyConfig } from "../types/config.model"; +import { CustomPostgresReadOnlyConfig, CustomWritePostgresConfig } from "../types/config.model"; import { timeoutPomise, PromiseWithState, savePromiseState, nextFulfilment } from "../utils/promise"; // return numeric (pg_type oid=1700) as float @@ -22,7 +22,7 @@ export interface DatabaseConfig { fileNamePrefix: string, readOnly: boolean, createDbIfNotExists: boolean, - postgres: CustomPostgresConfig, + postgres: CustomWritePostgresConfig, postgresReadOnly: CustomPostgresReadOnlyConfig } @@ -105,6 +105,10 @@ export class Postgres implements IDatabase { Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); + if (this.activePostgresRequests > this.config.postgres.maxActiveRequests) { + throw new Error("Too many active postgres requests"); + } + const pendingQueries: PromiseWithState>[] = []; let tries = 0; let lastPool: Pool = null; diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 3450b72..8b06c56 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -19,6 +19,10 @@ export interface CustomPostgresConfig extends PoolConfig { maxTries: number; } +export interface CustomWritePostgresConfig extends CustomPostgresConfig { + maxActiveRequests: number; +} + export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig { weight: number; readTimeout: number; @@ -71,7 +75,7 @@ export interface SBSConfig { redisRead?: RedisReadOnlyConfig; redisRateLimit: boolean; maxRewardTimePerSegmentInSeconds?: number; - postgres?: CustomPostgresConfig; + postgres?: CustomWritePostgresConfig; postgresReadOnly?: CustomPostgresReadOnlyConfig; dumpDatabase?: DumpDatabase; diskCacheURL: string; From 517c9512e4ca18af5f790871cb91558a31945f2b Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 15:31:21 -0500 Subject: [PATCH 05/21] Make default have no max requests --- src/config.ts | 2 +- src/databases/Postgres.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 91020b6..3c28819 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,7 +77,7 @@ addDefaults(config, { max: 10, idleTimeoutMillis: 10000, maxTries: 3, - maxActiveRequests: -1 + maxActiveRequests: 0 }, postgresReadOnly: { enabled: false, diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index a47da54..5253ad1 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -105,7 +105,8 @@ export class Postgres implements IDatabase { Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); - if (this.activePostgresRequests > this.config.postgres.maxActiveRequests) { + if (this.config.postgres.maxActiveRequests + && this.activePostgresRequests > this.config.postgres.maxActiveRequests) { throw new Error("Too many active postgres requests"); } From d3d53d0758a82a351442f46ca778478032a1fd16 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 15:35:07 -0500 Subject: [PATCH 06/21] Allow no max redis connections --- src/utils/redis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 68240ea..3997465 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -53,7 +53,7 @@ if (config.redis?.enabled) { const get = client.get.bind(client); const getRead = readClient?.get?.bind(readClient); exportClient.get = (key) => new Promise((resolve, reject) => { - if (activeRequests > config.redis.maxConnections) { + if (config.redis.maxConnections && activeRequests > config.redis.maxConnections) { reject("Too many active requests"); return; } @@ -85,7 +85,7 @@ if (config.redis?.enabled) { const setFun = >(func: (...args: T) => Promise , params: T): Promise => new Promise((resolve, reject) => { - if (activeRequests > config.redis.maxWriteConnections) { + if (config.redis.maxWriteConnections && activeRequests > config.redis.maxWriteConnections) { reject("Too many active requests"); return; } From a417299d3eec092518f712caad53f0c2935b12bd Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 12 Nov 2022 15:44:09 -0500 Subject: [PATCH 07/21] Only limit connections for reads --- src/databases/Postgres.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 5253ad1..a0b0128 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -105,7 +105,7 @@ export class Postgres implements IDatabase { Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); - if (this.config.postgres.maxActiveRequests + if (this.config.postgres.maxActiveRequests && this.isReadQuery(type) && this.activePostgresRequests > this.config.postgres.maxActiveRequests) { throw new Error("Too many active postgres requests"); } From 1fffd2e6ac8d88e3364f1d86f1d5cb5eb918cf0c Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 13 Nov 2022 17:01:32 -0500 Subject: [PATCH 08/21] Remove set limitation code --- src/utils/redis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 3997465..4b62bc3 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -108,8 +108,8 @@ if (config.redis?.enabled) { }); }); - exportClient.set = (key, value) => setFun(client.set.bind(client), [key, value]); - exportClient.setEx = (key, seconds, value) => setFun(client.setEx.bind(client), [key, seconds, value]); + // exportClient.set = (key, value) => setFun(client.set.bind(client), [key, value]); + // exportClient.setEx = (key, seconds, value) => setFun(client.setEx.bind(client), [key, seconds, value]); exportClient.increment = (key) => new Promise((resolve, reject) => void client.multi() .incr(key) From 4139bf8f8ce9a8d2922b61aa09003d9754517b93 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 13 Nov 2022 23:21:46 -0500 Subject: [PATCH 09/21] lower active connections on error --- src/databases/Postgres.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index a0b0128..05ebb4c 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -155,12 +155,12 @@ export class Postgres implements IDatabase { } } + this.activePostgresRequests--; Logger.error(`prepare (postgres) try ${tries}: ${err}`); } } while (this.isReadQuery(type) && tries < maxTries() && this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold); - this.activePostgresRequests--; throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`); } From 9e2e1343da552c8dcb33416b9930ad1b6f4450f3 Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 15 Nov 2022 14:19:15 -0500 Subject: [PATCH 10/21] Add timeout for all postgres requests --- src/config.ts | 3 ++- src/databases/Postgres.ts | 1 + src/types/config.model.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 3c28819..5ea715e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,7 +77,8 @@ addDefaults(config, { max: 10, idleTimeoutMillis: 10000, maxTries: 3, - maxActiveRequests: 0 + maxActiveRequests: 0, + timeout: 60000 }, postgresReadOnly: { enabled: false, diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 05ebb4c..a6dcceb 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -125,6 +125,7 @@ export class Postgres implements IDatabase { pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params }))); const currentPromises = [...pendingQueries]; if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout))); + else if (this.config.postgres.timeout) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgres.timeout))); const queryResult = await nextFulfilment(currentPromises); this.activePostgresRequests--; diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 8b06c56..459d231 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -21,6 +21,7 @@ export interface CustomPostgresConfig extends PoolConfig { export interface CustomWritePostgresConfig extends CustomPostgresConfig { maxActiveRequests: number; + timeout: number; } export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig { From 849ca52ef8c5a19414452e7fdde474b24d8a1aa7 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 19 Nov 2022 20:05:59 -0500 Subject: [PATCH 11/21] Allow disabling hash cache --- src/config.ts | 3 ++- src/types/config.model.ts | 1 + src/utils/getHashCache.ts | 22 ++++++++++++++-------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/config.ts b/src/config.ts index 5ea715e..6a939f1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -140,7 +140,8 @@ addDefaults(config, { expiryTime: 24 * 60 * 60, getTimeout: 40, maxConnections: 15000, - maxWriteConnections: 1000 + maxWriteConnections: 1000, + disableHashCache: false }, redisRead: { enabled: false, diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 459d231..08729d8 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -7,6 +7,7 @@ interface RedisConfig extends redis.RedisClientOptions { getTimeout: number; maxConnections: number; maxWriteConnections: number; + disableHashCache: boolean; } interface RedisReadOnlyConfig extends redis.RedisClientOptions { diff --git a/src/utils/getHashCache.ts b/src/utils/getHashCache.ts index 7ff0ef0..0c15cc0 100644 --- a/src/utils/getHashCache.ts +++ b/src/utils/getHashCache.ts @@ -3,6 +3,7 @@ import { shaHashKey } from "../utils/redisKeys"; import { HashedValue } from "../types/hash.model"; import { Logger } from "../utils/logger"; import { getHash } from "../utils/getHash"; +import { config } from "../config"; const defaultedHashTimes = 5000; const cachedHashTimes = defaultedHashTimes - 1; @@ -19,20 +20,25 @@ export async function getHashCache(value: T, times = defaulted async function getFromRedis(key: HashedValue): Promise { const redisKey = shaHashKey(key); - try { - const reply = await redis.get(redisKey); + if (!config.redis?.disableHashCache) { + try { + const reply = await redis.get(redisKey); - if (reply) { - Logger.debug(`Got data from redis: ${reply}`); - return reply as T & HashedValue; + if (reply) { + Logger.debug(`Got data from redis: ${reply}`); + return reply as T & HashedValue; + } + } catch (err) { + Logger.error(err as string); } - } catch (err) { - Logger.error(err as string); } // Otherwise, calculate it const data = getHash(key, cachedHashTimes); - redis.set(redisKey, data).catch((err) => Logger.error(err)); + + if (!config.redis?.disableHashCache) { + redis.set(redisKey, data).catch((err) => Logger.error(err)); + } return data as T & HashedValue; } \ No newline at end of file From 7bdcb10994b945a181a9ccd95a3d260982b20bfe Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 19 Nov 2022 23:32:09 -0500 Subject: [PATCH 12/21] Fix redis set limiting --- src/utils/redis.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 4b62bc3..330f863 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -83,10 +83,10 @@ if (config.redis?.enabled) { }); }); - const setFun = >(func: (...args: T) => Promise , params: T): Promise => + const setFun = >(func: (...args: T) => Promise, params: T): Promise => new Promise((resolve, reject) => { if (config.redis.maxWriteConnections && activeRequests > config.redis.maxWriteConnections) { - reject("Too many active requests"); + reject("Too many active requests to write"); return; } @@ -108,8 +108,10 @@ if (config.redis?.enabled) { }); }); - // exportClient.set = (key, value) => setFun(client.set.bind(client), [key, value]); - // exportClient.setEx = (key, seconds, value) => setFun(client.setEx.bind(client), [key, seconds, value]); + const set = client.set.bind(client); + const setEx = client.setEx.bind(client); + exportClient.set = (key, value) => setFun(set, [key, value]); + exportClient.setEx = (key, seconds, value) => setFun(setEx, [key, seconds, value]); exportClient.increment = (key) => new Promise((resolve, reject) => void client.multi() .incr(key) From 9bd6e9df4ff62ee6f326778edf21162cf8226360 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 19 Nov 2022 23:35:46 -0500 Subject: [PATCH 13/21] Add error catching in segment by hash --- src/routes/getSkipSegmentsByHash.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.ts b/src/routes/getSkipSegmentsByHash.ts index 454f232..d6ea9de 100644 --- a/src/routes/getSkipSegmentsByHash.ts +++ b/src/routes/getSkipSegmentsByHash.ts @@ -3,6 +3,7 @@ import { getSegmentsByHash } from "./getSkipSegments"; import { Request, Response } from "express"; import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model"; import { getService } from "../utils/getService"; +import { Logger } from "../utils/logger"; export async function getSkipSegmentsByHash(req: Request, res: Response): Promise { let hashPrefix = req.params.prefix as VideoIDHash; @@ -67,10 +68,16 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis // Get all video id's that match hash prefix const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service); - const output = Object.entries(segments).map(([videoID, data]) => ({ - videoID, - hash: data.hash, - segments: data.segments, - })); - return res.status(output.length === 0 ? 404 : 200).json(output); + try { + const output = Object.entries(segments).map(([videoID, data]) => ({ + videoID, + hash: data.hash, + segments: data.segments, + })); + return res.status(output.length === 0 ? 404 : 200).json(output); + } catch(e) { + Logger.error(`skip segments by hash error: ${e}`); + + return res.status(500).send("Internal server error"); + } } From 238ccc83d9c1b674207c3c315b192bce414362fb Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 19 Nov 2022 23:58:55 -0500 Subject: [PATCH 14/21] Stop writing if response time exceeds value --- src/config.ts | 2 ++ src/types/config.model.ts | 2 ++ src/utils/redis.ts | 12 ++++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 6a939f1..dc86962 100644 --- a/src/config.ts +++ b/src/config.ts @@ -141,6 +141,8 @@ addDefaults(config, { getTimeout: 40, maxConnections: 15000, maxWriteConnections: 1000, + stopWritingAfterResponseTime: 20, + responseTimePause: 1000, disableHashCache: false }, redisRead: { diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 08729d8..2fa6106 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -7,6 +7,8 @@ interface RedisConfig extends redis.RedisClientOptions { getTimeout: number; maxConnections: number; maxWriteConnections: number; + stopWritingAfterResponseTime: number; + responseTimePause: number; disableHashCache: boolean; } diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 330f863..13002c6 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -39,6 +39,7 @@ let writeRequests = 0; const readResponseTime: number[] = []; const writeResponseTime: number[] = []; +let lastResponseTimeLimit = 0; const maxStoredTimes = 200; if (config.redis?.enabled) { @@ -69,8 +70,13 @@ if (config.redis?.enabled) { activeRequests--; resolve(reply); - readResponseTime.push(Date.now() - start); + const responseTime = Date.now() - start; + readResponseTime.push(responseTime); if (readResponseTime.length > maxStoredTimes) readResponseTime.shift(); + if (config.redis.stopWritingAfterResponseTime + && responseTime > config.redis.stopWritingAfterResponseTime) { + lastResponseTimeLimit = Date.now(); + } }).catch((err) => { if (chosenGet === get) { lastClientFail = Date.now(); @@ -85,7 +91,9 @@ if (config.redis?.enabled) { const setFun = >(func: (...args: T) => Promise, params: T): Promise => new Promise((resolve, reject) => { - if (config.redis.maxWriteConnections && activeRequests > config.redis.maxWriteConnections) { + if ((config.redis.maxWriteConnections && activeRequests > config.redis.maxWriteConnections) + || (config.redis.responseTimePause + && Date.now() - lastResponseTimeLimit < config.redis.responseTimePause)) { reject("Too many active requests to write"); return; } From 71d30c0b51bf8e9ca728885f9cab999745143c34 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 19 Nov 2022 23:59:50 -0500 Subject: [PATCH 15/21] change redis max response time default --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index dc86962..06f2a43 100644 --- a/src/config.ts +++ b/src/config.ts @@ -141,7 +141,7 @@ addDefaults(config, { getTimeout: 40, maxConnections: 15000, maxWriteConnections: 1000, - stopWritingAfterResponseTime: 20, + stopWritingAfterResponseTime: 50, responseTimePause: 1000, disableHashCache: false }, From df76b5f053c2d4a47ac77bd0be5daa3800fd0807 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 20 Nov 2022 00:23:22 -0500 Subject: [PATCH 16/21] Add max queue to redis --- src/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.ts b/src/config.ts index 06f2a43..b6935d9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -141,6 +141,7 @@ addDefaults(config, { getTimeout: 40, maxConnections: 15000, maxWriteConnections: 1000, + commandsQueueMaxLength: 3000, stopWritingAfterResponseTime: 50, responseTimePause: 1000, disableHashCache: false From 87e38c8bc4462d8cb60264ac65e1a8445fe138cb Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 20 Nov 2022 00:47:41 -0500 Subject: [PATCH 17/21] Show total stats if not under high load --- src/config.ts | 3 ++- src/databases/IDatabase.ts | 2 ++ src/databases/Mysql.ts | 4 ++++ src/databases/Postgres.ts | 4 ++++ src/databases/Sqlite.ts | 4 ++++ src/routes/getTotalStats.ts | 33 +++++++++++++++++++++++++++++---- src/types/config.model.ts | 1 + 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index b6935d9..cb0bf34 100644 --- a/src/config.ts +++ b/src/config.ts @@ -78,7 +78,8 @@ addDefaults(config, { idleTimeoutMillis: 10000, maxTries: 3, maxActiveRequests: 0, - timeout: 60000 + timeout: 60000, + highLoadThreshold: 10 }, postgresReadOnly: { enabled: false, diff --git a/src/databases/IDatabase.ts b/src/databases/IDatabase.ts index 9cc82d4..717c700 100644 --- a/src/databases/IDatabase.ts +++ b/src/databases/IDatabase.ts @@ -7,6 +7,8 @@ export interface IDatabase { init(): Promise; prepare(type: QueryType, query: string, params?: any[], options?: QueryOption): Promise; + + highLoad(): boolean; } export type QueryType = "get" | "all" | "run"; \ No newline at end of file diff --git a/src/databases/Mysql.ts b/src/databases/Mysql.ts index d15b692..a68a12c 100644 --- a/src/databases/Mysql.ts +++ b/src/databases/Mysql.ts @@ -32,4 +32,8 @@ export class Mysql implements IDatabase { } } + highLoad() { + return false; + } + } diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index a6dcceb..7f65c9f 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -235,4 +235,8 @@ export class Postgres implements IDatabase { return result; } + + highLoad() { + return this.activePostgresRequests > this.config.postgres.highLoadThreshold; + } } diff --git a/src/databases/Sqlite.ts b/src/databases/Sqlite.ts index f5dd371..ac03753 100644 --- a/src/databases/Sqlite.ts +++ b/src/databases/Sqlite.ts @@ -95,6 +95,10 @@ export class Sqlite implements IDatabase { private static processUpgradeQuery(query: string): string { return query.replace(/^.*--!sqlite-ignore/gm, ""); } + + highLoad() { + return false; + } } export interface SqliteConfig { diff --git a/src/routes/getTotalStats.ts b/src/routes/getTotalStats.ts index e001d5d..187a59d 100644 --- a/src/routes/getTotalStats.ts +++ b/src/routes/getTotalStats.ts @@ -12,13 +12,26 @@ let firefoxUsersCache = 0; let apiUsersCache = 0; let lastUserCountCheck = 0; +interface DBStatsData { + userCount: number, + viewCount: number, + totalSubmissions: number, + minutesSaved: number +} + +let lastFetch: DBStatsData = { + userCount: 0, + viewCount: 0, + totalSubmissions: 0, + minutesSaved: 0 +}; + updateExtensionUsers(); export async function getTotalStats(req: Request, res: Response): Promise { - const userCountQuery = `(SELECT COUNT(*) FROM (SELECT DISTINCT "userID" from "sponsorTimes") t) "userCount",`; - const row = await db.prepare("get", `SELECT ${req.query.countContributingUsers ? userCountQuery : ""} COUNT(*) as "totalSubmissions", - SUM("views") as "viewCount", SUM(("endTime" - "startTime") / 60 * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "shadowHidden" != 1 AND "votes" >= 0 AND "actionType" != 'chapter'`, []); + const row = await getStats(!!req.query.countContributingUsers); + lastFetch = row; if (row !== undefined) { const extensionUsers = chromeUsersCache + firefoxUsersCache; @@ -43,6 +56,18 @@ export async function getTotalStats(req: Request, res: Response): Promise } } +function getStats(countContributingUsers: boolean): Promise { + if (db.highLoad()) { + return Promise.resolve(lastFetch); + } else { + const userCountQuery = `(SELECT COUNT(*) FROM (SELECT DISTINCT "userID" from "sponsorTimes") t) "userCount",`; + + return db.prepare("get", `SELECT ${countContributingUsers ? userCountQuery : ""} COUNT(*) as "totalSubmissions", + SUM("views") as "viewCount", SUM(("endTime" - "startTime") / 60 * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "shadowHidden" != 1 AND "votes" >= 0 AND "actionType" != 'chapter'`, []); + } +} + + function updateExtensionUsers() { if (config.userCounterURL) { axios.get(`${config.userCounterURL}/api/v1/userCount`) @@ -68,7 +93,7 @@ function updateExtensionUsers() { const userDownloadsStartIndex = body.indexOf(matchingString); if (userDownloadsStartIndex >= 0) { const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen); - const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",","").replace(".",""); + const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", ""); chromeUsersCache = parseInt(userDownloadsStr); } else { diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 2fa6106..4adaa51 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -25,6 +25,7 @@ export interface CustomPostgresConfig extends PoolConfig { export interface CustomWritePostgresConfig extends CustomPostgresConfig { maxActiveRequests: number; timeout: number; + highLoadThreshold: number; } export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig { From 0cd808a2d9fb0a3b157743b4086767d37cd1ec9a Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 20 Nov 2022 00:50:32 -0500 Subject: [PATCH 18/21] Add high load checks to leaderboard pages --- src/routes/getTopCategoryUsers.ts | 4 ++++ src/routes/getTopUsers.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index 21bc9eb..877ed2e 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -56,6 +56,10 @@ export async function getTopCategoryUsers(req: Request, res: Response): Promise< return res.sendStatus(400); } + if (db.highLoad()) { + return res.status(503).send("Disabled for load reasons"); + } + //setup which sort type to use let sortBy = ""; if (sortType == 0) { diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index 552aaf9..f77bc8b 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -88,6 +88,10 @@ export async function getTopUsers(req: Request, res: Response): Promise Date: Sun, 20 Nov 2022 01:20:05 -0500 Subject: [PATCH 19/21] Wait for redis to connect before starting server --- src/index.ts | 2 ++ src/utils/redis.ts | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 88c6562..ec44960 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { createServer } from "./app"; import { Logger } from "./utils/logger"; import { startAllCrons } from "./cronjob"; import { getCommit } from "./utils/getCommit"; +import { connectionPromise } from "./utils/redis"; async function init() { process.on("unhandledRejection", (error: any) => { @@ -14,6 +15,7 @@ async function init() { try { await initDb(); + await connectionPromise; } catch (e) { Logger.error(`Init Db: ${e}`); process.exit(1); diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 13002c6..01761ae 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -42,12 +42,14 @@ const writeResponseTime: number[] = []; let lastResponseTimeLimit = 0; const maxStoredTimes = 200; +export let connectionPromise = Promise.resolve(); + if (config.redis?.enabled) { Logger.info("Connected to redis"); const client = createClient(config.redis); const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null; - void client.connect(); // void as we don't care about the promise - void readClient?.connect(); + connectionPromise = client.connect(); + void readClient?.connect(); // void as we don't care about the promise exportClient = client as RedisSB; From e252411fe1e13ef4d950256b56082e0309d5462c Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 20 Nov 2022 17:30:13 -0500 Subject: [PATCH 20/21] No time saved for chapter --- src/routes/getTopCategoryUsers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index 877ed2e..74853b3 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -34,7 +34,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { userNames.push(row.userName); viewCounts.push(row.viewCount); totalSubmissions.push(row.totalSubmissions); - minutesSaved.push(row.minutesSaved); + minutesSaved.push(category === "chapter" ? 0 : row.minutesSaved); } } From 30ac91c537029d393e7cdd4f11b51d5084fce2b2 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 20 Nov 2022 19:17:47 -0500 Subject: [PATCH 21/21] Don't refetch reputation if under high db load --- src/routes/getSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 6c93822..4dfe589 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -291,7 +291,7 @@ async function chooseSegments(videoID: VideoID, service: Service, segments: DBSe //Segments with less than -1 votes are already ignored before this function is called async function buildSegmentGroups(segments: DBSegment[]): Promise { const reputationPromises = segments.map(segment => - segment.userID ? getReputation(segment.userID).catch((e) => Logger.error(e)) : null); + segment.userID && !db.highLoad() ? getReputation(segment.userID).catch((e) => Logger.error(e)) : null); //Create groups of segments that are similar to eachother //Segments must be sorted by their startTime so that we can build groups chronologically: