Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-25 08:58:23 +03:00
Code Issues Packages Projects Releases Wiki Activity

Switch to lz4 compression

Browse Source
This commit is contained in:
Ajay
2024-01-19 15:16:50 -05:00
parent dcb479f3d2
commit fce311377f
4 changed files with 355 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/utils/redis.ts
View File

@@ -6,7 +6,7 @@ import { RedisClientOptions } from "@redis/client/dist/lib/client";
import { RedisReply } from "rate-limit-redis";
import { db } from "../databases/databases";
import { Postgres } from "../databases/Postgres";
import { compress, decompress } from "@mongodb-js/zstd";
import { compress, uncompress } from "lz4-napi";
export interface RedisStats {
activeRequests: number;
@@ -68,11 +68,11 @@ if (config.redis?.enabled) {
exportClient.getCompressed = (key) => {
return exportClient.get(key).then((reply) => {
if (reply === null) return null;
return decompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
return uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
});
};
exportClient.setCompressed = (key, value, options) => {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.set(key, compressed.toString("base64"), options)
);
};

24
src/utils/redisKeys.ts
View File

@@ -5,43 +5,43 @@ import { Logger } from "./logger";
import { BrandingUUID } from "../types/branding.model";
export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
`segments.v5.${service}.videoID.${videoID}`;
`segments.v6.${service}.videoID.${videoID}`;
export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string =>
`segments.groups.v4.${service}.videoID.${videoID}`;
`segments.groups.v5.${service}.videoID.${videoID}`;
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `segments.v5.${service}.${hashedVideoIDPrefix}`;
return `segments.v6.${service}.${hashedVideoIDPrefix}`;
}
export const brandingKey = (videoID: VideoID, service: Service): string =>
`branding.v3.${service}.videoID.${videoID}`;
`branding.v4.${service}.videoID.${videoID}`;
export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `branding.v3.${service}.${hashedVideoIDPrefix}`;
return `branding.v4.${service}.${hashedVideoIDPrefix}`;
}
export const brandingIPKey = (uuid: BrandingUUID): string =>
`branding.v1.shadow.${uuid}`;
`branding.v2.shadow.${uuid}`;
export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
`segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
`segments.v2.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
export const reputationKey = (userID: UserID): string =>
`reputation.v1.user.${userID}`;
`reputation.v2.user.${userID}`;
export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
return `rating.v1.${service}.${hashPrefix}`;
return `rating.v2.${service}.${hashPrefix}`;
}
export function shaHashKey(singleIter: HashedValue): string {
@@ -54,15 +54,15 @@ export const tempVIPKey = (userID: HashedUserID): string =>
`vip.temp.${userID}`;
export const videoLabelsKey = (videoID: VideoID, service: Service): string =>
`labels.v2.${service}.videoID.${videoID}`;
`labels.v3.${service}.videoID.${videoID}`;
export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`);
return `labels.v2.${service}.${hashedVideoIDPrefix}`;
return `labels.v3.${service}.${hashedVideoIDPrefix}`;
}
export function userFeatureKey (userID: HashedUserID, feature: Feature): string {
return `user.v1.${userID}.feature.${feature}`;
return `user.v2.${userID}.feature.${feature}`;
}