diff --git a/src/config.ts b/src/config.ts index a9fbbf7..1a74d5c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -167,7 +167,8 @@ addDefaults(config, { stopWritingAfterResponseTime: 50, responseTimePause: 1000, disableHashCache: false, - clientCacheSize: 2000 + clientCacheSize: 2000, + useCompression: false }, redisRead: { enabled: false, diff --git a/src/types/config.model.ts b/src/types/config.model.ts index aa354d4..ce56622 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -12,6 +12,7 @@ interface RedisConfig extends redis.RedisClientOptions { responseTimePause: number; disableHashCache: boolean; clientCacheSize: number; + useCompression: boolean; } interface RedisReadOnlyConfig extends redis.RedisClientOptions { diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index 85ec76f..317bb58 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -7,7 +7,7 @@ import { config } from "../config"; async function get(fetchFromDB: () => Promise, key: string): Promise { try { - const reply = await redis.getCompressed(key); + const reply = await redis.getWithCache(key); if (reply) { Logger.debug(`Got data from redis: ${reply}`); @@ -21,7 +21,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { const data = await fetchFromDB(); - redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); + redis.setExWithCache(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); return data; } @@ -36,7 +36,7 @@ async function getTraced(fetchFromDB: () => Promise, key: string): Promise const startTime = Date.now(); try { - const reply = await redis.getCompressed(key); + const reply = await redis.getWithCache(key); if (reply) { Logger.debug(`Got data from redis: ${reply}`); @@ -55,7 +55,7 @@ async function getTraced(fetchFromDB: () => Promise, key: string): Promise const dbStartTime = Date.now(); const data = await fetchFromDB(); - redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); + redis.setExWithCache(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); return { data, diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 6e012d1..3eef756 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -25,11 +25,11 @@ export interface RedisStats { interface RedisSB { get(key: RedisCommandArgument, useClientCache?: boolean): Promise; - getCompressed(key: RedisCommandArgument): Promise; + getWithCache(key: RedisCommandArgument): Promise; set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; - setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; + setWithCache(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; - setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; + setExWithCache(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; del(...keys: [RedisCommandArgument]): Promise; increment?(key: RedisCommandArgument): Promise; sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise; @@ -39,11 +39,11 @@ interface RedisSB { let exportClient: RedisSB = { get: () => Promise.resolve(null), - getCompressed: () => Promise.resolve(null), + getWithCache: () => Promise.resolve(null), set: () => Promise.resolve(null), - setCompressed: () => Promise.resolve(null), + setWithCache: () => Promise.resolve(null), setEx: () => Promise.resolve(null), - setExCompressed: () => Promise.resolve(null), + setExWithCache: () => Promise.resolve(null), del: () => Promise.resolve(null), increment: () => Promise.resolve(null), sendCommand: () => Promise.resolve(null), @@ -95,7 +95,9 @@ if (config.redis?.enabled) { let cacheClient = null as RedisClientType | null; - exportClient.getCompressed = (key) => { + const createKeyName = (key: RedisCommandArgument) => (key + (config.redis.useCompression ? ".c" : "")) as RedisCommandArgument; + + exportClient.getWithCache = (key) => { if (cache && cacheClient && cache.has(key)) { memoryCacheHits++; return Promise.resolve(cache.get(key)); @@ -115,23 +117,34 @@ if (config.redis?.enabled) { return activeRequestPromises[key as string]; } - const request = exportClient.get(key).then((reply) => { + const request = exportClient.get(createKeyName(key)).then((reply) => { if (reply === null) return null; - const decompressed = uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8")); - if (cache && shouldClientCacheKey(key)) { - decompressed.then((d) => { - if (!resetKeys.has(key)) { - cache.set(key, d); - } + if (config.redis.useCompression) { + const decompressed = uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8")); + if (cache && shouldClientCacheKey(key)) { + decompressed.then((d) => { + if (!resetKeys.has(key)) { + cache.set(key, d); + } + resetKeys.delete(key); + }).catch(Logger.error); + } else { resetKeys.delete(key); - }).catch(Logger.error); - } else { - resetKeys.delete(key); - } + } - return decompressed; + return decompressed; + } else { + if (cache && shouldClientCacheKey(key)) { + if (!resetKeys.has(key)) { + cache.set(key, reply); + } + } + + resetKeys.delete(key); + return reply; + } }); activeRequestPromises[key as string] = request; @@ -140,15 +153,32 @@ if (config.redis?.enabled) { return request; }; - exportClient.setCompressed = (key, value, options) => { - return compress(Buffer.from(value as string, "utf-8")).then((compressed) => - exportClient.set(key, compressed.toString("base64"), options) - ); + exportClient.setWithCache = (key, value, options) => { + if (config.redis.useCompression) { + return compress(Buffer.from(value as string, "utf-8")).then((compressed) => + exportClient.set(createKeyName(key), compressed.toString("base64"), options) + ); + } else { + return exportClient.set(createKeyName(key), value, options); + } }; - exportClient.setExCompressed = (key, seconds, value) => { - return compress(Buffer.from(value as string, "utf-8")).then((compressed) => - exportClient.setEx(key, seconds, compressed.toString("base64")) - ); + exportClient.setExWithCache = (key, seconds, value) => { + if (config.redis.useCompression) { + return compress(Buffer.from(value as string, "utf-8")).then((compressed) => + exportClient.setEx(createKeyName(key), seconds, compressed.toString("base64")) + ); + } else { + return exportClient.setEx(createKeyName(key), seconds, value); + } + }; + + const del = client.del.bind(client); + exportClient.del = (...keys) => { + if (config.redis.useCompression) { + return del(...keys.map((key) => createKeyName(key)) as [RedisCommandArgument]); + } else { + return del(...keys); + } }; const get = client.get.bind(client); diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index 0b96d8c..c5ced20 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -6,43 +6,43 @@ import { BrandingUUID } from "../types/branding.model"; import { RedisCommandArgument } from "@redis/client/dist/lib/commands"; export const skipSegmentsKey = (videoID: VideoID, service: Service): string => - `segments.v6.${service}.videoID.${videoID}`; + `segments.v4.${service}.videoID.${videoID}`; export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string => - `segments.groups.v5.${service}.videoID.${videoID}`; + `segments.groups.v3.${service}.videoID.${videoID}`; export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `segments.v6.${service}.${hashedVideoIDPrefix}`; + return `segments.v4.${service}.${hashedVideoIDPrefix}`; } export const brandingKey = (videoID: VideoID, service: Service): string => - `branding.v4.${service}.videoID.${videoID}`; + `branding.v2.${service}.videoID.${videoID}`; export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `branding.v4.${service}.${hashedVideoIDPrefix}`; + return `branding.v2.${service}.${hashedVideoIDPrefix}`; } export const brandingIPKey = (uuid: BrandingUUID): string => - `branding.v2.shadow.${uuid}`; + `branding.v1.shadow.${uuid}`; export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => - `segments.v2.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; + `segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; export const reputationKey = (userID: UserID): string => - `reputation.v2.user.${userID}`; + `reputation.v1.user.${userID}`; export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string { hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash; if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`); - return `rating.v2.${service}.${hashPrefix}`; + return `rating.v1.${service}.${hashPrefix}`; } export function shaHashKey(singleIter: HashedValue): string { @@ -55,17 +55,17 @@ export const tempVIPKey = (userID: HashedUserID): string => `vip.temp.${userID}`; export const videoLabelsKey = (videoID: VideoID, service: Service): string => - `labels.v3.${service}.videoID.${videoID}`; + `labels.v1.${service}.videoID.${videoID}`; export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash; if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`); - return `labels.v3.${service}.${hashedVideoIDPrefix}`; + return `labels.v1.${service}.${hashedVideoIDPrefix}`; } export function userFeatureKey (userID: HashedUserID, feature: Feature): string { - return `user.v2.${userID}.feature.${feature}`; + return `user.v1.${userID}.feature.${feature}`; } export function shouldClientCacheKey(key: RedisCommandArgument): boolean {