From 14b6f84f947fb4acf41a5101bb50be6f0f407491 Mon Sep 17 00:00:00 2001 From: SashaXser <24498484+SashaXser@users.noreply.github.com> Date: Fri, 19 Jan 2024 08:50:45 +0400 Subject: [PATCH 1/3] 2 things Consider using "forEach" instead of "map" as its return value is not being used here. Replace this trivial promise with "Promise.resolve". --- src/routes/getBranding.ts | 12 ++++++------ src/utils/createMemoryCache.ts | 4 +--- src/utils/redis.ts | 17 +++++++++-------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index 82b67aa..66cc395 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -131,21 +131,21 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi }; }; - (await branding.titles).map((title) => { + (await branding.titles).forEach((title) => { title.title = title.title.replace("<", "‹"); - + initResult(title); dbResult[title.videoID].titles.push(title); }); - (await branding.thumbnails).map((thumbnail) => { + (await branding.thumbnails).forEach((thumbnail) => { initResult(thumbnail); dbResult[thumbnail.videoID].thumbnails.push(thumbnail); }); - - (await branding.segments).map((segment) => { + + (await branding.segments).forEach((segment) => { initResult(segment); dbResult[segment.videoID].segments.push(segment); - }); + }); return dbResult; }, brandingHashKey(videoHashPrefix, service)); diff --git a/src/utils/createMemoryCache.ts b/src/utils/createMemoryCache.ts index ded7c0b..0215657 100644 --- a/src/utils/createMemoryCache.ts +++ b/src/utils/createMemoryCache.ts @@ -23,9 +23,7 @@ export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeM } } // create new promise - const promise = new Promise((resolve) => { - resolve(memoryFn(...args)); - }); + const promise = Promise.resolve(memoryFn(...args)); // store promise reference until fulfilled promiseMemory.set(cacheKey, promise); return promise.then(result => { diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 3a495e7..9b93a18 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -26,16 +26,17 @@ interface RedisSB { } let exportClient: RedisSB = { - get: () => new Promise((resolve) => resolve(null)), - set: () => new Promise((resolve) => resolve(null)), - setEx: () => new Promise((resolve) => resolve(null)), - del: () => new Promise((resolve) => resolve(null)), - increment: () => new Promise((resolve) => resolve(null)), - sendCommand: () => new Promise((resolve) => resolve(null)), - quit: () => new Promise((resolve) => resolve(null)), - ttl: () => new Promise((resolve) => resolve(null)), + get: () => Promise.resolve(null), + set: () => Promise.resolve(null), + setEx: () => Promise.resolve(null), + del: () => Promise.resolve(null), + increment: () => Promise.resolve(null), + sendCommand: () => Promise.resolve(null), + quit: () => Promise.resolve(null), + ttl: () => Promise.resolve(null), }; + let lastClientFail = 0; let lastReadFail = 0; let activeRequests = 0; From ea609470926e4137e8a77b5d8cbfb0633aa0f5b5 Mon Sep 17 00:00:00 2001 From: SashaXser <24498484+SashaXser@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:31:03 +0400 Subject: [PATCH 2/3] format fix --- src/routes/getBranding.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index 66cc395..7be8e43 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -133,7 +133,7 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi (await branding.titles).forEach((title) => { title.title = title.title.replace("<", "‹"); - + initResult(title); dbResult[title.videoID].titles.push(title); }); @@ -141,11 +141,11 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi initResult(thumbnail); dbResult[thumbnail.videoID].thumbnails.push(thumbnail); }); - + (await branding.segments).forEach((segment) => { initResult(segment); dbResult[segment.videoID].segments.push(segment); - }); + }); return dbResult; }, brandingHashKey(videoHashPrefix, service)); @@ -317,4 +317,4 @@ export async function getBrandingByHashEndpoint(req: Request, res: Response) { Logger.error(e as string); return res.status(500).send([]); } -} \ No newline at end of file +} From 5b95aa8aba14465d97adaec9b82475a3e2154095 Mon Sep 17 00:00:00 2001 From: SashaXser <24498484+SashaXser@users.noreply.github.com> Date: Sat, 20 Jan 2024 06:59:12 +0400 Subject: [PATCH 3/3] Resolve conflicts --- src/utils/redis.ts | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 9b93a18..c366e99 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -6,6 +6,7 @@ import { RedisClientOptions } from "@redis/client/dist/lib/client"; import { RedisReply } from "rate-limit-redis"; import { db } from "../databases/databases"; import { Postgres } from "../databases/Postgres"; +import { compress, uncompress } from "lz4-napi"; export interface RedisStats { activeRequests: number; @@ -16,8 +17,11 @@ export interface RedisStats { interface RedisSB { get(key: RedisCommandArgument): Promise; + getCompressed(key: RedisCommandArgument): Promise; set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; + setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; + setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; del(...keys: [RedisCommandArgument]): Promise; increment?(key: RedisCommandArgument): Promise; sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise; @@ -27,8 +31,11 @@ interface RedisSB { let exportClient: RedisSB = { get: () => Promise.resolve(null), + getCompressed: () => Promise.resolve(null), set: () => Promise.resolve(null), + setCompressed: () => Promise.resolve(null), setEx: () => Promise.resolve(null), + setExCompressed: () => Promise.resolve(null), del: () => Promise.resolve(null), increment: () => Promise.resolve(null), sendCommand: () => Promise.resolve(null), @@ -36,7 +43,6 @@ let exportClient: RedisSB = { ttl: () => Promise.resolve(null), }; - let lastClientFail = 0; let lastReadFail = 0; let activeRequests = 0; @@ -47,7 +53,7 @@ const writeResponseTime: number[] = []; let lastResponseTimeLimit = 0; const maxStoredTimes = 200; -export class TooManyActiveConnectionsError extends Error {} +export class TooManyActiveConnectionsError extends Error { } export let connectionPromise = Promise.resolve(); @@ -57,8 +63,24 @@ if (config.redis?.enabled) { const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null; connectionPromise = client.connect(); void readClient?.connect(); // void as we don't care about the promise - exportClient = client as RedisSB; + exportClient = client as unknown as RedisSB; + exportClient.getCompressed = (key) => { + return exportClient.get(key).then((reply) => { + if (reply === null) return null; + return uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8")); + }); + }; + exportClient.setCompressed = (key, value, options) => { + return compress(Buffer.from(value as string, "utf-8")).then((compressed) => + exportClient.set(key, compressed.toString("base64"), options) + ); + }; + exportClient.setExCompressed = (key, seconds, value) => { + return compress(Buffer.from(value as string, "utf-8")).then((compressed) => + exportClient.setEx(key, seconds, compressed.toString("base64")) + ); + }; const get = client.get.bind(client); const getRead = readClient?.get?.bind(readClient); @@ -84,7 +106,7 @@ if (config.redis?.enabled) { readResponseTime.push(responseTime); if (readResponseTime.length > maxStoredTimes) readResponseTime.shift(); if (config.redis.stopWritingAfterResponseTime - && responseTime > config.redis.stopWritingAfterResponseTime) { + && responseTime > config.redis.stopWritingAfterResponseTime) { Logger.error(`Hit response time limit at ${responseTime}ms`); lastResponseTimeLimit = Date.now(); } @@ -104,7 +126,7 @@ if (config.redis?.enabled) { new Promise((resolve, reject) => { if ((config.redis.maxWriteConnections && activeRequests > config.redis.maxWriteConnections) || (config.redis.responseTimePause - && Date.now() - lastResponseTimeLimit < config.redis.responseTimePause)) { + && Date.now() - lastResponseTimeLimit < config.redis.responseTimePause)) { reject(`Too many active requests to write due to ${activeRequests} requests and ${Date.now() - lastResponseTimeLimit}ms since last limit. ${(db as Postgres)?.getStats?.()?.activeRequests} active db requests with ${(db as Postgres)?.getStats?.()?.avgReadTime}ms`); return; } @@ -140,7 +162,7 @@ if (config.redis?.enabled) { .catch((err) => reject(err)) ); /* istanbul ignore next */ - client.on("error", function(error) { + client.on("error", function (error) { lastClientFail = Date.now(); Logger.error(`Redis Error: ${error}`); }); @@ -149,7 +171,7 @@ if (config.redis?.enabled) { Logger.info("Redis: trying to reconnect"); }); /* istanbul ignore next */ - readClient?.on("error", function(error) { + readClient?.on("error", function (error) { lastReadFail = Date.now(); Logger.error(`Redis Read-Only Error: ${error}`); }); @@ -164,7 +186,7 @@ function pickChoice(client: T, readClient: T): T { const ignoreReadDueToFailure = lastReadFail > Date.now() - 1000 * 30; const readDueToFailure = lastClientFail > Date.now() - 1000 * 30; if (readAvailable && !ignoreReadDueToFailure && (readDueToFailure || - Math.random() > 1 / (config.redisRead?.weight + 1))) { + Math.random() > 1 / (config.redisRead?.weight + 1))) { return readClient; } else { return client;