From 2ad51842ccd1fac5cf4404f912b0c205da84c26f Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 19 Jan 2024 14:34:18 -0500 Subject: [PATCH] Compress redis values --- package-lock.json | 179 +++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/utils/queryCacher.ts | 8 +- src/utils/redis.ts | 25 +++++- src/utils/redisKeys.ts | 24 +++--- 5 files changed, 220 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index 06ddadc..216c12a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "AGPL-3.0-only", "dependencies": { + "@mongodb-js/zstd": "^1.2.0", "axios": "^1.1.3", "better-sqlite3": "^8.0.1", "cron": "^2.1.0", @@ -792,6 +793,128 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mongodb-js/zstd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz", + "integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mongodb-js/zstd-darwin-arm64": "1.2.0", + "@mongodb-js/zstd-darwin-x64": "1.2.0", + "@mongodb-js/zstd-linux-arm64-gnu": "1.2.0", + "@mongodb-js/zstd-linux-arm64-musl": "1.2.0", + "@mongodb-js/zstd-linux-x64-gnu": "1.2.0", + "@mongodb-js/zstd-linux-x64-musl": "1.2.0", + "@mongodb-js/zstd-win32-x64-msvc": "1.2.0" + } + }, + "node_modules/@mongodb-js/zstd-darwin-arm64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz", + "integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-darwin-x64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz", + "integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-arm64-gnu": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz", + "integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-arm64-musl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz", + "integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-x64-gnu": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz", + "integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-x64-musl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz", + "integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-win32-x64-msvc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz", + "integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -6296,6 +6419,62 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "@mongodb-js/zstd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz", + "integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==", + "requires": { + "@mongodb-js/zstd-darwin-arm64": "1.2.0", + "@mongodb-js/zstd-darwin-x64": "1.2.0", + "@mongodb-js/zstd-linux-arm64-gnu": "1.2.0", + "@mongodb-js/zstd-linux-arm64-musl": "1.2.0", + "@mongodb-js/zstd-linux-x64-gnu": "1.2.0", + "@mongodb-js/zstd-linux-x64-musl": "1.2.0", + "@mongodb-js/zstd-win32-x64-msvc": "1.2.0" + } + }, + "@mongodb-js/zstd-darwin-arm64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz", + "integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==", + "optional": true + }, + "@mongodb-js/zstd-darwin-x64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz", + "integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==", + "optional": true + }, + "@mongodb-js/zstd-linux-arm64-gnu": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz", + "integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==", + "optional": true + }, + "@mongodb-js/zstd-linux-arm64-musl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz", + "integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==", + "optional": true + }, + "@mongodb-js/zstd-linux-x64-gnu": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz", + "integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==", + "optional": true + }, + "@mongodb-js/zstd-linux-x64-musl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz", + "integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==", + "optional": true + }, + "@mongodb-js/zstd-win32-x64-msvc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz", + "integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==", + "optional": true + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/package.json b/package.json index ac5864d..3db577d 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "author": "Ajay Ramachandran", "license": "AGPL-3.0-only", "dependencies": { + "@mongodb-js/zstd": "^1.2.0", "axios": "^1.1.3", "better-sqlite3": "^8.0.1", "cron": "^2.1.0", diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index fe89c65..85ec76f 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -7,7 +7,7 @@ import { config } from "../config"; async function get(fetchFromDB: () => Promise, key: string): Promise { try { - const reply = await redis.get(key); + const reply = await redis.getCompressed(key); if (reply) { Logger.debug(`Got data from redis: ${reply}`); @@ -21,7 +21,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { const data = await fetchFromDB(); - redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); + redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); return data; } @@ -36,7 +36,7 @@ async function getTraced(fetchFromDB: () => Promise, key: string): Promise const startTime = Date.now(); try { - const reply = await redis.get(key); + const reply = await redis.getCompressed(key); if (reply) { Logger.debug(`Got data from redis: ${reply}`); @@ -55,7 +55,7 @@ async function getTraced(fetchFromDB: () => Promise, key: string): Promise const dbStartTime = Date.now(); const data = await fetchFromDB(); - redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); + redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); return { data, diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 3a495e7..c27d3ce 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -6,6 +6,7 @@ import { RedisClientOptions } from "@redis/client/dist/lib/client"; import { RedisReply } from "rate-limit-redis"; import { db } from "../databases/databases"; import { Postgres } from "../databases/Postgres"; +import { compress, decompress } from "@mongodb-js/zstd"; export interface RedisStats { activeRequests: number; @@ -16,8 +17,11 @@ export interface RedisStats { interface RedisSB { get(key: RedisCommandArgument): Promise; + getCompressed(key: RedisCommandArgument): Promise; set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; + setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; + setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; del(...keys: [RedisCommandArgument]): Promise; increment?(key: RedisCommandArgument): Promise; sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise; @@ -27,8 +31,11 @@ interface RedisSB { let exportClient: RedisSB = { get: () => new Promise((resolve) => resolve(null)), + getCompressed: () => new Promise((resolve) => resolve(null)), set: () => new Promise((resolve) => resolve(null)), + setCompressed: () => new Promise((resolve) => resolve(null)), setEx: () => new Promise((resolve) => resolve(null)), + setExCompressed: () => new Promise((resolve) => resolve(null)), del: () => new Promise((resolve) => resolve(null)), increment: () => new Promise((resolve) => resolve(null)), sendCommand: () => new Promise((resolve) => resolve(null)), @@ -56,8 +63,24 @@ if (config.redis?.enabled) { const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null; connectionPromise = client.connect(); void readClient?.connect(); // void as we don't care about the promise - exportClient = client as RedisSB; + exportClient = client as unknown as RedisSB; + exportClient.getCompressed = (key) => { + return exportClient.get(key).then((reply) => { + if (reply === null) return null; + return decompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8")); + }); + }; + exportClient.setCompressed = (key, value, options) => { + return compress(Buffer.from(value as string, "utf-8")).then((compressed) => + exportClient.set(key, compressed.toString("base64"), options) + ); + }; + exportClient.setExCompressed = (key, seconds, value) => { + return compress(Buffer.from(value as string, "utf-8")).then((compressed) => + exportClient.setEx(key, seconds, compressed.toString("base64")) + ); + }; const get = client.get.bind(client); const getRead = readClient?.get?.bind(readClient); diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index ae6927e..760220d 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -5,43 +5,43 @@ import { Logger } from "./logger"; import { BrandingUUID } from "../types/branding.model"; export const skipSegmentsKey = (videoID: VideoID, service: Service): string => - `segments.v4.${service}.videoID.${videoID}`; + `segments.v5.${service}.videoID.${videoID}`; export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string => - `segments.groups.v3.${service}.videoID.${videoID}`; + `segments.groups.v4.${service}.videoID.${videoID}`; export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `segments.v4.${service}.${hashedVideoIDPrefix}`; + return `segments.v5.${service}.${hashedVideoIDPrefix}`; } export const brandingKey = (videoID: VideoID, service: Service): string => - `branding.v2.${service}.videoID.${videoID}`; + `branding.v3.${service}.videoID.${videoID}`; export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `branding.v2.${service}.${hashedVideoIDPrefix}`; + return `branding.v3.${service}.${hashedVideoIDPrefix}`; } export const brandingIPKey = (uuid: BrandingUUID): string => - `branding.shadow.${uuid}`; + `branding.v1.shadow.${uuid}`; export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => - `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; + `segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; export const reputationKey = (userID: UserID): string => - `reputation.user.${userID}`; + `reputation.v1.user.${userID}`; export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string { hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash; if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`); - return `rating.${service}.${hashPrefix}`; + return `rating.v1.${service}.${hashPrefix}`; } export function shaHashKey(singleIter: HashedValue): string { @@ -54,15 +54,15 @@ export const tempVIPKey = (userID: HashedUserID): string => `vip.temp.${userID}`; export const videoLabelsKey = (videoID: VideoID, service: Service): string => - `labels.v1.${service}.videoID.${videoID}`; + `labels.v2.${service}.videoID.${videoID}`; export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash; if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`); - return `labels.v1.${service}.${hashedVideoIDPrefix}`; + return `labels.v2.${service}.${hashedVideoIDPrefix}`; } export function userFeatureKey (userID: HashedUserID, feature: Feature): string { - return `user.${userID}.feature.${feature}`; + return `user.v1.${userID}.feature.${feature}`; } \ No newline at end of file