Compress redis values

package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.1.0",
       "license": "AGPL-3.0-only",
       "dependencies": {
+        "@mongodb-js/zstd": "^1.2.0",
         "axios": "^1.1.3",
         "better-sqlite3": "^8.0.1",
         "cron": "^2.1.0",
@@ -792,6 +793,128 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@mongodb-js/zstd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz",
+      "integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==",
+      "engines": {
+        "node": ">= 10"
+      },
+      "optionalDependencies": {
+        "@mongodb-js/zstd-darwin-arm64": "1.2.0",
+        "@mongodb-js/zstd-darwin-x64": "1.2.0",
+        "@mongodb-js/zstd-linux-arm64-gnu": "1.2.0",
+        "@mongodb-js/zstd-linux-arm64-musl": "1.2.0",
+        "@mongodb-js/zstd-linux-x64-gnu": "1.2.0",
+        "@mongodb-js/zstd-linux-x64-musl": "1.2.0",
+        "@mongodb-js/zstd-win32-x64-msvc": "1.2.0"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-darwin-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz",
+      "integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-darwin-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz",
+      "integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-linux-arm64-gnu": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz",
+      "integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-linux-arm64-musl": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz",
+      "integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-linux-x64-gnu": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz",
+      "integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-linux-x64-musl": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz",
+      "integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@mongodb-js/zstd-win32-x64-msvc": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz",
+      "integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -6296,6 +6419,62 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "@mongodb-js/zstd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz",
+      "integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==",
+      "requires": {
+        "@mongodb-js/zstd-darwin-arm64": "1.2.0",
+        "@mongodb-js/zstd-darwin-x64": "1.2.0",
+        "@mongodb-js/zstd-linux-arm64-gnu": "1.2.0",
+        "@mongodb-js/zstd-linux-arm64-musl": "1.2.0",
+        "@mongodb-js/zstd-linux-x64-gnu": "1.2.0",
+        "@mongodb-js/zstd-linux-x64-musl": "1.2.0",
+        "@mongodb-js/zstd-win32-x64-msvc": "1.2.0"
+      }
+    },
+    "@mongodb-js/zstd-darwin-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz",
+      "integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==",
+      "optional": true
+    },
+    "@mongodb-js/zstd-darwin-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz",
+      "integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==",
+      "optional": true
+    },
+    "@mongodb-js/zstd-linux-arm64-gnu": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz",
+      "integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==",
+      "optional": true
+    },
+    "@mongodb-js/zstd-linux-arm64-musl": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz",
+      "integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==",
+      "optional": true
+    },
+    "@mongodb-js/zstd-linux-x64-gnu": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz",
+      "integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==",
+      "optional": true
+    },
+    "@mongodb-js/zstd-linux-x64-musl": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz",
+      "integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==",
+      "optional": true
+    },
+    "@mongodb-js/zstd-win32-x64-msvc": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz",
+      "integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==",
+      "optional": true
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

package.json

@@ -19,6 +19,7 @@
   "author": "Ajay Ramachandran",
   "license": "AGPL-3.0-only",
   "dependencies": {
+    "@mongodb-js/zstd": "^1.2.0",
     "axios": "^1.1.3",
     "better-sqlite3": "^8.0.1",
     "cron": "^2.1.0",

src/utils/queryCacher.ts

@@ -7,7 +7,7 @@ import { config } from "../config";
 
 async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
     try {
-        const reply = await redis.get(key);
+        const reply = await redis.getCompressed(key);
         if (reply) {
             Logger.debug(`Got data from redis: ${reply}`);
 
@@ -21,7 +21,7 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
 
     const data = await fetchFromDB();
 
-    redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
+    redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
 
     return data;
 }
@@ -36,7 +36,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
     const startTime = Date.now();
 
     try {
-        const reply = await redis.get(key);
+        const reply = await redis.getCompressed(key);
         if (reply) {
             Logger.debug(`Got data from redis: ${reply}`);
 
@@ -55,7 +55,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
     const dbStartTime = Date.now();
     const data = await fetchFromDB();
 
-    redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
+    redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
 
     return {
         data,

src/utils/redis.ts

@@ -6,6 +6,7 @@ import { RedisClientOptions } from "@redis/client/dist/lib/client";
 import { RedisReply } from "rate-limit-redis";
 import { db } from "../databases/databases";
 import { Postgres } from "../databases/Postgres";
+import { compress, decompress } from "@mongodb-js/zstd";
 
 export interface RedisStats {
     activeRequests: number;
@@ -16,8 +17,11 @@ export interface RedisStats {
 
 interface RedisSB {
     get(key: RedisCommandArgument): Promise<string>;
+    getCompressed(key: RedisCommandArgument): Promise<string>;
     set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
+    setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
     setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
+    setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
     del(...keys: [RedisCommandArgument]): Promise<number>;
     increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
     sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
@@ -27,8 +31,11 @@ interface RedisSB {
 
 let exportClient: RedisSB = {
     get: () => new Promise((resolve) => resolve(null)),
+    getCompressed: () => new Promise((resolve) => resolve(null)),
     set: () => new Promise((resolve) => resolve(null)),
+    setCompressed: () => new Promise((resolve) => resolve(null)),
     setEx: () => new Promise((resolve) => resolve(null)),
+    setExCompressed: () => new Promise((resolve) => resolve(null)),
     del: () => new Promise((resolve) => resolve(null)),
     increment: () => new Promise((resolve) => resolve(null)),
     sendCommand: () => new Promise((resolve) => resolve(null)),
@@ -56,8 +63,24 @@ if (config.redis?.enabled) {
     const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null;
     connectionPromise = client.connect();
     void readClient?.connect(); // void as we don't care about the promise
-    exportClient = client as RedisSB;
+    exportClient = client as unknown as RedisSB;
 
+    exportClient.getCompressed = (key) => {
+        return exportClient.get(key).then((reply) => {
+            if (reply === null) return null;
+            return decompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
+        });
+    };
+    exportClient.setCompressed = (key, value, options) => {
+        return compress(Buffer.from(value as string, "utf-8")).then((compressed) => 
+            exportClient.set(key, compressed.toString("base64"), options)
+        );
+    };
+    exportClient.setExCompressed = (key, seconds, value) => {
+        return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
+            exportClient.setEx(key, seconds, compressed.toString("base64"))
+        );
+    };
 
     const get = client.get.bind(client);
     const getRead = readClient?.get?.bind(readClient);

src/utils/redisKeys.ts

@@ -5,43 +5,43 @@ import { Logger } from "./logger";
 import { BrandingUUID } from "../types/branding.model";
 
 export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
-    `segments.v4.${service}.videoID.${videoID}`;
+    `segments.v5.${service}.videoID.${videoID}`;
 
 export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string =>
-    `segments.groups.v3.${service}.videoID.${videoID}`;
+    `segments.groups.v4.${service}.videoID.${videoID}`;
 
 export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
     hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
     if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
 
-    return `segments.v4.${service}.${hashedVideoIDPrefix}`;
+    return `segments.v5.${service}.${hashedVideoIDPrefix}`;
 }
 
 export const brandingKey = (videoID: VideoID, service: Service): string =>
-    `branding.v2.${service}.videoID.${videoID}`;
+    `branding.v3.${service}.videoID.${videoID}`;
 
 export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
     hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
     if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
 
-    return `branding.v2.${service}.${hashedVideoIDPrefix}`;
+    return `branding.v3.${service}.${hashedVideoIDPrefix}`;
 }
 
 export const brandingIPKey = (uuid: BrandingUUID): string =>
-    `branding.shadow.${uuid}`;
+    `branding.v1.shadow.${uuid}`;
 
 
 export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
-    `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
+    `segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
 
 export const reputationKey = (userID: UserID): string =>
-    `reputation.user.${userID}`;
+    `reputation.v1.user.${userID}`;
 
 export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
     hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
     if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
 
-    return `rating.${service}.${hashPrefix}`;
+    return `rating.v1.${service}.${hashPrefix}`;
 }
 
 export function shaHashKey(singleIter: HashedValue): string {
@@ -54,15 +54,15 @@ export const tempVIPKey = (userID: HashedUserID): string =>
     `vip.temp.${userID}`;
 
 export const videoLabelsKey = (videoID: VideoID, service: Service): string =>
-    `labels.v1.${service}.videoID.${videoID}`;
+    `labels.v2.${service}.videoID.${videoID}`;
 
 export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
     hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash;
     if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`);
 
-    return `labels.v1.${service}.${hashedVideoIDPrefix}`;
+    return `labels.v2.${service}.${hashedVideoIDPrefix}`;
 }
 
 export function userFeatureKey (userID: HashedUserID, feature: Feature): string {
-    return `user.${userID}.feature.${feature}`;
+    return `user.v1.${userID}.feature.${feature}`;
 }