Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-12 06:27:10 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow toggling redis compression and disable by default

Browse Source
This commit is contained in:
Ajay
2024-02-08 21:58:10 -05:00
parent db225f8a84
commit bf1fe1ff61
5 changed files with 76 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/config.ts
View File

@@ -167,7 +167,8 @@ addDefaults(config, {
stopWritingAfterResponseTime: 50,
responseTimePause: 1000,
disableHashCache: false,
clientCacheSize: 2000
clientCacheSize: 2000,
useCompression: false
},
redisRead: {
enabled: false,

1
src/types/config.model.ts
View File

@@ -12,6 +12,7 @@ interface RedisConfig extends redis.RedisClientOptions {
responseTimePause: number;
disableHashCache: boolean;
clientCacheSize: number;
useCompression: boolean;
}
interface RedisReadOnlyConfig extends redis.RedisClientOptions {

8
src/utils/queryCacher.ts
View File

@@ -7,7 +7,7 @@ import { config } from "../config";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
try {
const reply = await redis.getCompressed(key);
const reply = await redis.getWithCache(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
@@ -21,7 +21,7 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const data = await fetchFromDB();
redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
redis.setExWithCache(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
return data;
}
@@ -36,7 +36,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
const startTime = Date.now();
try {
const reply = await redis.getCompressed(key);
const reply = await redis.getWithCache(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
@@ -55,7 +55,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
const dbStartTime = Date.now();
const data = await fetchFromDB();
redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
redis.setExWithCache(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
return {
data,

84
src/utils/redis.ts
View File

@@ -25,11 +25,11 @@ export interface RedisStats {
interface RedisSB {
get(key: RedisCommandArgument, useClientCache?: boolean): Promise<string>;
getCompressed(key: RedisCommandArgument): Promise<string>;
getWithCache(key: RedisCommandArgument): Promise<string>;
set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
setWithCache(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
setExWithCache(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
del(...keys: [RedisCommandArgument]): Promise<number>;
increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
@@ -39,11 +39,11 @@ interface RedisSB {
let exportClient: RedisSB = {
get: () => Promise.resolve(null),
getCompressed: () => Promise.resolve(null),
getWithCache: () => Promise.resolve(null),
set: () => Promise.resolve(null),
setCompressed: () => Promise.resolve(null),
setWithCache: () => Promise.resolve(null),
setEx: () => Promise.resolve(null),
setExCompressed: () => Promise.resolve(null),
setExWithCache: () => Promise.resolve(null),
del: () => Promise.resolve(null),
increment: () => Promise.resolve(null),
sendCommand: () => Promise.resolve(null),
@@ -95,7 +95,9 @@ if (config.redis?.enabled) {
let cacheClient = null as RedisClientType | null;
exportClient.getCompressed = (key) => {
const createKeyName = (key: RedisCommandArgument) => (key + (config.redis.useCompression ? ".c" : "")) as RedisCommandArgument;
exportClient.getWithCache = (key) => {
if (cache && cacheClient && cache.has(key)) {
memoryCacheHits++;
return Promise.resolve(cache.get(key));
@@ -115,23 +117,34 @@ if (config.redis?.enabled) {
return activeRequestPromises[key as string];
}
const request = exportClient.get(key).then((reply) => {
const request = exportClient.get(createKeyName(key)).then((reply) => {
if (reply === null) return null;
const decompressed = uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
if (cache && shouldClientCacheKey(key)) {
decompressed.then((d) => {
if (!resetKeys.has(key)) {
cache.set(key, d);
}
if (config.redis.useCompression) {
const decompressed = uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
if (cache && shouldClientCacheKey(key)) {
decompressed.then((d) => {
if (!resetKeys.has(key)) {
cache.set(key, d);
}
resetKeys.delete(key);
}).catch(Logger.error);
} else {
resetKeys.delete(key);
}).catch(Logger.error);
} else {
resetKeys.delete(key);
}
}
return decompressed;
return decompressed;
} else {
if (cache && shouldClientCacheKey(key)) {
if (!resetKeys.has(key)) {
cache.set(key, reply);
}
}
resetKeys.delete(key);
return reply;
}
});
activeRequestPromises[key as string] = request;
@@ -140,15 +153,32 @@ if (config.redis?.enabled) {
return request;
};
exportClient.setCompressed = (key, value, options) => {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.set(key, compressed.toString("base64"), options)
);
exportClient.setWithCache = (key, value, options) => {
if (config.redis.useCompression) {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.set(createKeyName(key), compressed.toString("base64"), options)
);
} else {
return exportClient.set(createKeyName(key), value, options);
}
};
exportClient.setExCompressed = (key, seconds, value) => {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.setEx(key, seconds, compressed.toString("base64"))
);
exportClient.setExWithCache = (key, seconds, value) => {
if (config.redis.useCompression) {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.setEx(createKeyName(key), seconds, compressed.toString("base64"))
);
} else {
return exportClient.setEx(createKeyName(key), seconds, value);
}
};
const del = client.del.bind(client);
exportClient.del = (...keys) => {
if (config.redis.useCompression) {
return del(...keys.map((key) => createKeyName(key)) as [RedisCommandArgument]);
} else {
return del(...keys);
}
};
const get = client.get.bind(client);

24
src/utils/redisKeys.ts
View File

@@ -6,43 +6,43 @@ import { BrandingUUID } from "../types/branding.model";
import { RedisCommandArgument } from "@redis/client/dist/lib/commands";
export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
`segments.v6.${service}.videoID.${videoID}`;
`segments.v4.${service}.videoID.${videoID}`;
export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string =>
`segments.groups.v5.${service}.videoID.${videoID}`;
`segments.groups.v3.${service}.videoID.${videoID}`;
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `segments.v6.${service}.${hashedVideoIDPrefix}`;
return `segments.v4.${service}.${hashedVideoIDPrefix}`;
}
export const brandingKey = (videoID: VideoID, service: Service): string =>
`branding.v4.${service}.videoID.${videoID}`;
`branding.v2.${service}.videoID.${videoID}`;
export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `branding.v4.${service}.${hashedVideoIDPrefix}`;
return `branding.v2.${service}.${hashedVideoIDPrefix}`;
}
export const brandingIPKey = (uuid: BrandingUUID): string =>
`branding.v2.shadow.${uuid}`;
`branding.v1.shadow.${uuid}`;
export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
`segments.v2.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
`segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
export const reputationKey = (userID: UserID): string =>
`reputation.v2.user.${userID}`;
`reputation.v1.user.${userID}`;
export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
return `rating.v2.${service}.${hashPrefix}`;
return `rating.v1.${service}.${hashPrefix}`;
}
export function shaHashKey(singleIter: HashedValue): string {
@@ -55,17 +55,17 @@ export const tempVIPKey = (userID: HashedUserID): string =>
`vip.temp.${userID}`;
export const videoLabelsKey = (videoID: VideoID, service: Service): string =>
`labels.v3.${service}.videoID.${videoID}`;
`labels.v1.${service}.videoID.${videoID}`;
export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`);
return `labels.v3.${service}.${hashedVideoIDPrefix}`;
return `labels.v1.${service}.${hashedVideoIDPrefix}`;
}
export function userFeatureKey (userID: HashedUserID, feature: Feature): string {
return `user.v2.${userID}.feature.${feature}`;
return `user.v1.${userID}.feature.${feature}`;
}
export function shouldClientCacheKey(key: RedisCommandArgument): boolean {