Allow toggling redis compression and disable by default

src/config.ts

@@ -167,7 +167,8 @@ addDefaults(config, {
         stopWritingAfterResponseTime: 50,
         responseTimePause: 1000,
         disableHashCache: false,
-        clientCacheSize: 2000
+        clientCacheSize: 2000,
+        useCompression: false
     },
     redisRead: {
         enabled: false,

src/types/config.model.ts

@@ -12,6 +12,7 @@ interface RedisConfig extends redis.RedisClientOptions {
     responseTimePause: number;
     disableHashCache: boolean;
     clientCacheSize: number;
+    useCompression: boolean;
 }
 
 interface RedisReadOnlyConfig extends redis.RedisClientOptions {

src/utils/queryCacher.ts

@@ -7,7 +7,7 @@ import { config } from "../config";
 
 async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
     try {
-        const reply = await redis.getCompressed(key);
+        const reply = await redis.getWithCache(key);
         if (reply) {
             Logger.debug(`Got data from redis: ${reply}`);
 
@@ -21,7 +21,7 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
 
     const data = await fetchFromDB();
 
-    redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
+    redis.setExWithCache(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
 
     return data;
 }
@@ -36,7 +36,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
     const startTime = Date.now();
 
     try {
-        const reply = await redis.getCompressed(key);
+        const reply = await redis.getWithCache(key);
         if (reply) {
             Logger.debug(`Got data from redis: ${reply}`);
 
@@ -55,7 +55,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
     const dbStartTime = Date.now();
     const data = await fetchFromDB();
 
-    redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
+    redis.setExWithCache(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
 
     return {
         data,

src/utils/redis.ts

@@ -25,11 +25,11 @@ export interface RedisStats {
 
 interface RedisSB {
     get(key: RedisCommandArgument, useClientCache?: boolean): Promise<string>;
-    getCompressed(key: RedisCommandArgument): Promise<string>;
+    getWithCache(key: RedisCommandArgument): Promise<string>;
     set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
-    setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
+    setWithCache(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
     setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
-    setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
+    setExWithCache(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
     del(...keys: [RedisCommandArgument]): Promise<number>;
     increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
     sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
@@ -39,11 +39,11 @@ interface RedisSB {
 
 let exportClient: RedisSB = {
     get: () => Promise.resolve(null),
-    getCompressed: () => Promise.resolve(null),
+    getWithCache: () => Promise.resolve(null),
     set: () => Promise.resolve(null),
-    setCompressed: () => Promise.resolve(null),
+    setWithCache: () => Promise.resolve(null),
     setEx: () => Promise.resolve(null),
-    setExCompressed: () => Promise.resolve(null),
+    setExWithCache: () => Promise.resolve(null),
     del: () => Promise.resolve(null),
     increment: () => Promise.resolve(null),
     sendCommand: () => Promise.resolve(null),
@@ -95,7 +95,9 @@ if (config.redis?.enabled) {
 
     let cacheClient = null as RedisClientType | null;
 
-    exportClient.getCompressed = (key) => {
+    const createKeyName = (key: RedisCommandArgument) => (key + (config.redis.useCompression ? ".c" : "")) as RedisCommandArgument;
+
+    exportClient.getWithCache = (key) => {
         if (cache && cacheClient && cache.has(key)) {
             memoryCacheHits++;
             return Promise.resolve(cache.get(key));
@@ -115,23 +117,34 @@ if (config.redis?.enabled) {
             return activeRequestPromises[key as string];
         }
 
-        const request = exportClient.get(key).then((reply) => {
+        const request = exportClient.get(createKeyName(key)).then((reply) => {
             if (reply === null) return null;
 
-            const decompressed = uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
+            if (config.redis.useCompression) {
-            if (cache && shouldClientCacheKey(key)) {
+                const decompressed = uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
-                decompressed.then((d) => {
+                if (cache && shouldClientCacheKey(key)) {
-                    if (!resetKeys.has(key)) {
+                    decompressed.then((d) => {
-                        cache.set(key, d);
+                        if (!resetKeys.has(key)) {
-                    }
+                            cache.set(key, d);
+                        }
 
+                        resetKeys.delete(key);
+                    }).catch(Logger.error);
+                } else {
                     resetKeys.delete(key);
-                }).catch(Logger.error);
+                }
-            } else {
-                resetKeys.delete(key);
-            }
 
-            return decompressed;
+                return decompressed;
+            } else {
+                if (cache && shouldClientCacheKey(key)) {
+                    if (!resetKeys.has(key)) {
+                        cache.set(key, reply);
+                    }
+                }
+
+                resetKeys.delete(key);
+                return reply;
+            }
         });
 
         activeRequestPromises[key as string] = request;
@@ -140,15 +153,32 @@ if (config.redis?.enabled) {
 
         return request;
     };
-    exportClient.setCompressed = (key, value, options) => {
+    exportClient.setWithCache = (key, value, options) => {
-        return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
+        if (config.redis.useCompression) {
-            exportClient.set(key, compressed.toString("base64"), options)
+            return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
-        );
+                exportClient.set(createKeyName(key), compressed.toString("base64"), options)
+            );
+        } else {
+            return exportClient.set(createKeyName(key), value, options);
+        }
     };
-    exportClient.setExCompressed = (key, seconds, value) => {
+    exportClient.setExWithCache = (key, seconds, value) => {
-        return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
+        if (config.redis.useCompression) {
-            exportClient.setEx(key, seconds, compressed.toString("base64"))
+            return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
-        );
+                exportClient.setEx(createKeyName(key), seconds, compressed.toString("base64"))
+            );
+        } else {
+            return exportClient.setEx(createKeyName(key), seconds, value);
+        }
+    };
+
+    const del = client.del.bind(client);
+    exportClient.del = (...keys) => {
+        if (config.redis.useCompression) {
+            return del(...keys.map((key) => createKeyName(key)) as [RedisCommandArgument]);
+        } else {
+            return del(...keys);
+        }
     };
 
     const get = client.get.bind(client);

src/utils/redisKeys.ts

@@ -6,43 +6,43 @@ import { BrandingUUID } from "../types/branding.model";
 import { RedisCommandArgument } from "@redis/client/dist/lib/commands";
 
 export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
-    `segments.v6.${service}.videoID.${videoID}`;
+    `segments.v4.${service}.videoID.${videoID}`;
 
 export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string =>
-    `segments.groups.v5.${service}.videoID.${videoID}`;
+    `segments.groups.v3.${service}.videoID.${videoID}`;
 
 export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
     hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
     if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
 
-    return `segments.v6.${service}.${hashedVideoIDPrefix}`;
+    return `segments.v4.${service}.${hashedVideoIDPrefix}`;
 }
 
 export const brandingKey = (videoID: VideoID, service: Service): string =>
-    `branding.v4.${service}.videoID.${videoID}`;
+    `branding.v2.${service}.videoID.${videoID}`;
 
 export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
     hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
     if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
 
-    return `branding.v4.${service}.${hashedVideoIDPrefix}`;
+    return `branding.v2.${service}.${hashedVideoIDPrefix}`;
 }
 
 export const brandingIPKey = (uuid: BrandingUUID): string =>
-    `branding.v2.shadow.${uuid}`;
+    `branding.v1.shadow.${uuid}`;
 
 
 export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
-    `segments.v2.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
+    `segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
 
 export const reputationKey = (userID: UserID): string =>
-    `reputation.v2.user.${userID}`;
+    `reputation.v1.user.${userID}`;
 
 export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
     hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
     if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
 
-    return `rating.v2.${service}.${hashPrefix}`;
+    return `rating.v1.${service}.${hashPrefix}`;
 }
 
 export function shaHashKey(singleIter: HashedValue): string {
@@ -55,17 +55,17 @@ export const tempVIPKey = (userID: HashedUserID): string =>
     `vip.temp.${userID}`;
 
 export const videoLabelsKey = (videoID: VideoID, service: Service): string =>
-    `labels.v3.${service}.videoID.${videoID}`;
+    `labels.v1.${service}.videoID.${videoID}`;
 
 export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
     hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash;
     if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`);
 
-    return `labels.v3.${service}.${hashedVideoIDPrefix}`;
+    return `labels.v1.${service}.${hashedVideoIDPrefix}`;
 }
 
 export function userFeatureKey (userID: HashedUserID, feature: Feature): string {
-    return `user.v2.${userID}.feature.${feature}`;
+    return `user.v1.${userID}.feature.${feature}`;
 }
 
 export function shouldClientCacheKey(key: RedisCommandArgument): boolean {