Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 11:36:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Compress redis values

Browse Source
This commit is contained in:
Ajay
2024-01-19 14:34:18 -05:00
parent 8e13ec60d6
commit 2ad51842cc
5 changed files with 220 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
package-lock.json generated
View File

@@ -9,6 +9,7 @@
"version": "0.1.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@mongodb-js/zstd": "^1.2.0",
"axios": "^1.1.3",
"better-sqlite3": "^8.0.1",
"cron": "^2.1.0",
@@ -792,6 +793,128 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@mongodb-js/zstd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz",
"integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==",
"engines": {
"node": ">= 10"
},
"optionalDependencies": {
"@mongodb-js/zstd-darwin-arm64": "1.2.0",
"@mongodb-js/zstd-darwin-x64": "1.2.0",
"@mongodb-js/zstd-linux-arm64-gnu": "1.2.0",
"@mongodb-js/zstd-linux-arm64-musl": "1.2.0",
"@mongodb-js/zstd-linux-x64-gnu": "1.2.0",
"@mongodb-js/zstd-linux-x64-musl": "1.2.0",
"@mongodb-js/zstd-win32-x64-msvc": "1.2.0"
}
},
"node_modules/@mongodb-js/zstd-darwin-arm64": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz",
"integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@mongodb-js/zstd-darwin-x64": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz",
"integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@mongodb-js/zstd-linux-arm64-gnu": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz",
"integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@mongodb-js/zstd-linux-arm64-musl": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz",
"integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@mongodb-js/zstd-linux-x64-gnu": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz",
"integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@mongodb-js/zstd-linux-x64-musl": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz",
"integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@mongodb-js/zstd-win32-x64-msvc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz",
"integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -6296,6 +6419,62 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"@mongodb-js/zstd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz",
"integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==",
"requires": {
"@mongodb-js/zstd-darwin-arm64": "1.2.0",
"@mongodb-js/zstd-darwin-x64": "1.2.0",
"@mongodb-js/zstd-linux-arm64-gnu": "1.2.0",
"@mongodb-js/zstd-linux-arm64-musl": "1.2.0",
"@mongodb-js/zstd-linux-x64-gnu": "1.2.0",
"@mongodb-js/zstd-linux-x64-musl": "1.2.0",
"@mongodb-js/zstd-win32-x64-msvc": "1.2.0"
}
},
"@mongodb-js/zstd-darwin-arm64": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz",
"integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==",
"optional": true
},
"@mongodb-js/zstd-darwin-x64": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz",
"integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==",
"optional": true
},
"@mongodb-js/zstd-linux-arm64-gnu": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz",
"integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==",
"optional": true
},
"@mongodb-js/zstd-linux-arm64-musl": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz",
"integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==",
"optional": true
},
"@mongodb-js/zstd-linux-x64-gnu": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz",
"integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==",
"optional": true
},
"@mongodb-js/zstd-linux-x64-musl": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz",
"integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==",
"optional": true
},
"@mongodb-js/zstd-win32-x64-msvc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz",
"integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==",
"optional": true
},
"@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

1
package.json
View File

@@ -19,6 +19,7 @@
"author": "Ajay Ramachandran",
"license": "AGPL-3.0-only",
"dependencies": {
"@mongodb-js/zstd": "^1.2.0",
"axios": "^1.1.3",
"better-sqlite3": "^8.0.1",
"cron": "^2.1.0",

8
src/utils/queryCacher.ts
View File

@@ -7,7 +7,7 @@ import { config } from "../config";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
try {
const reply = await redis.get(key);
const reply = await redis.getCompressed(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
@@ -21,7 +21,7 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const data = await fetchFromDB();
redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
return data;
}
@@ -36,7 +36,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
const startTime = Date.now();
try {
const reply = await redis.get(key);
const reply = await redis.getCompressed(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
@@ -55,7 +55,7 @@ async function getTraced<T>(fetchFromDB: () => Promise<T>, key: string): Promise
const dbStartTime = Date.now();
const data = await fetchFromDB();
redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err));
return {
data,

25
src/utils/redis.ts
View File

@@ -6,6 +6,7 @@ import { RedisClientOptions } from "@redis/client/dist/lib/client";
import { RedisReply } from "rate-limit-redis";
import { db } from "../databases/databases";
import { Postgres } from "../databases/Postgres";
import { compress, decompress } from "@mongodb-js/zstd";
export interface RedisStats {
activeRequests: number;
@@ -16,8 +17,11 @@ export interface RedisStats {
interface RedisSB {
get(key: RedisCommandArgument): Promise<string>;
getCompressed(key: RedisCommandArgument): Promise<string>;
set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise<string>;
setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
del(...keys: [RedisCommandArgument]): Promise<number>;
increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
@@ -27,8 +31,11 @@ interface RedisSB {
let exportClient: RedisSB = {
get: () => new Promise((resolve) => resolve(null)),
getCompressed: () => new Promise((resolve) => resolve(null)),
set: () => new Promise((resolve) => resolve(null)),
setCompressed: () => new Promise((resolve) => resolve(null)),
setEx: () => new Promise((resolve) => resolve(null)),
setExCompressed: () => new Promise((resolve) => resolve(null)),
del: () => new Promise((resolve) => resolve(null)),
increment: () => new Promise((resolve) => resolve(null)),
sendCommand: () => new Promise((resolve) => resolve(null)),
@@ -56,8 +63,24 @@ if (config.redis?.enabled) {
const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null;
connectionPromise = client.connect();
void readClient?.connect(); // void as we don't care about the promise
exportClient = client as RedisSB;
exportClient = client as unknown as RedisSB;
exportClient.getCompressed = (key) => {
return exportClient.get(key).then((reply) => {
if (reply === null) return null;
return decompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8"));
});
};
exportClient.setCompressed = (key, value, options) => {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.set(key, compressed.toString("base64"), options)
);
};
exportClient.setExCompressed = (key, seconds, value) => {
return compress(Buffer.from(value as string, "utf-8")).then((compressed) =>
exportClient.setEx(key, seconds, compressed.toString("base64"))
);
};
const get = client.get.bind(client);
const getRead = readClient?.get?.bind(readClient);

24
src/utils/redisKeys.ts
View File

@@ -5,43 +5,43 @@ import { Logger } from "./logger";
import { BrandingUUID } from "../types/branding.model";
export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
`segments.v4.${service}.videoID.${videoID}`;
`segments.v5.${service}.videoID.${videoID}`;
export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string =>
`segments.groups.v3.${service}.videoID.${videoID}`;
`segments.groups.v4.${service}.videoID.${videoID}`;
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `segments.v4.${service}.${hashedVideoIDPrefix}`;
return `segments.v5.${service}.${hashedVideoIDPrefix}`;
}
export const brandingKey = (videoID: VideoID, service: Service): string =>
`branding.v2.${service}.videoID.${videoID}`;
`branding.v3.${service}.videoID.${videoID}`;
export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `branding.v2.${service}.${hashedVideoIDPrefix}`;
return `branding.v3.${service}.${hashedVideoIDPrefix}`;
}
export const brandingIPKey = (uuid: BrandingUUID): string =>
`branding.shadow.${uuid}`;
`branding.v1.shadow.${uuid}`;
export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
`segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
`segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;
export const reputationKey = (userID: UserID): string =>
`reputation.user.${userID}`;
`reputation.v1.user.${userID}`;
export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
return `rating.${service}.${hashPrefix}`;
return `rating.v1.${service}.${hashPrefix}`;
}
export function shaHashKey(singleIter: HashedValue): string {
@@ -54,15 +54,15 @@ export const tempVIPKey = (userID: HashedUserID): string =>
`vip.temp.${userID}`;
export const videoLabelsKey = (videoID: VideoID, service: Service): string =>
`labels.v1.${service}.videoID.${videoID}`;
`labels.v2.${service}.videoID.${videoID}`;
export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`);
return `labels.v1.${service}.${hashedVideoIDPrefix}`;
return `labels.v2.${service}.${hashedVideoIDPrefix}`;
}
export function userFeatureKey (userID: HashedUserID, feature: Feature): string {
return `user.${userID}.feature.${feature}`;
return `user.v1.${userID}.feature.${feature}`;
}