mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2025-12-06 11:36:58 +03:00
Handle reconnects with client-side caching
Also upgrades redis to fix a library bug
This commit is contained in:
Ajay
126
package-lock.json
generated
126
package-lock.json
generated
@@ -21,7 +21,7 @@
|
||||
"lz4-napi": "^2.2.0",
|
||||
"pg": "^8.8.0",
|
||||
"rate-limit-redis": "^3.0.1",
|
||||
"redis": "^4.5.0",
|
||||
"redis": "^4.6.13",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -1038,19 +1038,19 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/bloom": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.1.0.tgz",
|
||||
"integrity": "sha512-9QovlxmpRtvxVbN0UBcv8WfdSMudNZZTFqCsnBszcQXqaZb/TVe30ScgGEO7u1EAIacTPAo7/oCYjYAxiHLanQ==",
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
|
||||
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/client": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.4.0.tgz",
|
||||
"integrity": "sha512-1gEj1AkyXPlkcC/9/T5xpDcQF8ntERURjLBgEWMTdUZqe181zfI9BY3jc2OzjTLkvZh5GV7VT4ktoJG2fV2ufw==",
|
||||
"version": "1.5.14",
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.14.tgz",
|
||||
"integrity": "sha512-YGn0GqsRBFUQxklhY7v562VMOP0DcmlrHHs3IV1mFE3cbxe31IITUkqhBcIhVSI/2JqtWAJXg5mjV4aU+zD0HA==",
|
||||
"dependencies": {
|
||||
"cluster-key-slot": "1.1.1",
|
||||
"cluster-key-slot": "1.1.2",
|
||||
"generic-pool": "3.9.0",
|
||||
"yallist": "4.0.0"
|
||||
},
|
||||
@@ -1059,33 +1059,33 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/graph": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
|
||||
"integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
|
||||
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/json": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
|
||||
"integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
|
||||
"integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/search": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.0.tgz",
|
||||
"integrity": "sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==",
|
||||
"version": "1.1.6",
|
||||
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz",
|
||||
"integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/time-series": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
|
||||
"integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
|
||||
"integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
@@ -2058,9 +2058,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/cluster-key-slot": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz",
|
||||
"integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==",
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
@@ -4972,16 +4972,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/redis": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.5.0.tgz",
|
||||
"integrity": "sha512-oZGAmOKG+RPnHo0UxM5GGjJ0dBd/Vi4fs3MYwM1p2baDoXC0wpm0yOdpxVS9K+0hM84ycdysp2eHg2xGoQ4FEw==",
|
||||
"version": "4.6.13",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.13.tgz",
|
||||
"integrity": "sha512-MHgkS4B+sPjCXpf+HfdetBwbRz6vCtsceTmw1pHNYJAsYxrfpOP6dz+piJWGos8wqG7qb3vj/Rrc5qOlmInUuA==",
|
||||
"dependencies": {
|
||||
"@redis/bloom": "1.1.0",
|
||||
"@redis/client": "1.4.0",
|
||||
"@redis/graph": "1.1.0",
|
||||
"@redis/json": "1.0.4",
|
||||
"@redis/search": "1.1.0",
|
||||
"@redis/time-series": "1.0.4"
|
||||
"@redis/bloom": "1.2.0",
|
||||
"@redis/client": "1.5.14",
|
||||
"@redis/graph": "1.1.1",
|
||||
"@redis/json": "1.0.6",
|
||||
"@redis/search": "1.1.6",
|
||||
"@redis/time-series": "1.0.5"
|
||||
}
|
||||
},
|
||||
"node_modules/regexpp": {
|
||||
@@ -6658,43 +6658,43 @@
|
||||
}
|
||||
},
|
||||
"@redis/bloom": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.1.0.tgz",
|
||||
"integrity": "sha512-9QovlxmpRtvxVbN0UBcv8WfdSMudNZZTFqCsnBszcQXqaZb/TVe30ScgGEO7u1EAIacTPAo7/oCYjYAxiHLanQ==",
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
|
||||
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/client": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.4.0.tgz",
|
||||
"integrity": "sha512-1gEj1AkyXPlkcC/9/T5xpDcQF8ntERURjLBgEWMTdUZqe181zfI9BY3jc2OzjTLkvZh5GV7VT4ktoJG2fV2ufw==",
|
||||
"version": "1.5.14",
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.14.tgz",
|
||||
"integrity": "sha512-YGn0GqsRBFUQxklhY7v562VMOP0DcmlrHHs3IV1mFE3cbxe31IITUkqhBcIhVSI/2JqtWAJXg5mjV4aU+zD0HA==",
|
||||
"requires": {
|
||||
"cluster-key-slot": "1.1.1",
|
||||
"cluster-key-slot": "1.1.2",
|
||||
"generic-pool": "3.9.0",
|
||||
"yallist": "4.0.0"
|
||||
}
|
||||
},
|
||||
"@redis/graph": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
|
||||
"integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
|
||||
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/json": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
|
||||
"integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
|
||||
"integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/search": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.0.tgz",
|
||||
"integrity": "sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==",
|
||||
"version": "1.1.6",
|
||||
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz",
|
||||
"integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/time-series": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
|
||||
"integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
|
||||
"integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
|
||||
"requires": {}
|
||||
},
|
||||
"@sinonjs/commons": {
|
||||
@@ -7424,9 +7424,9 @@
|
||||
}
|
||||
},
|
||||
"cluster-key-slot": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz",
|
||||
"integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw=="
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "2.0.1",
|
||||
@@ -9564,16 +9564,16 @@
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.5.0.tgz",
|
||||
"integrity": "sha512-oZGAmOKG+RPnHo0UxM5GGjJ0dBd/Vi4fs3MYwM1p2baDoXC0wpm0yOdpxVS9K+0hM84ycdysp2eHg2xGoQ4FEw==",
|
||||
"version": "4.6.13",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.13.tgz",
|
||||
"integrity": "sha512-MHgkS4B+sPjCXpf+HfdetBwbRz6vCtsceTmw1pHNYJAsYxrfpOP6dz+piJWGos8wqG7qb3vj/Rrc5qOlmInUuA==",
|
||||
"requires": {
|
||||
"@redis/bloom": "1.1.0",
|
||||
"@redis/client": "1.4.0",
|
||||
"@redis/graph": "1.1.0",
|
||||
"@redis/json": "1.0.4",
|
||||
"@redis/search": "1.1.0",
|
||||
"@redis/time-series": "1.0.4"
|
||||
"@redis/bloom": "1.2.0",
|
||||
"@redis/client": "1.5.14",
|
||||
"@redis/graph": "1.1.1",
|
||||
"@redis/json": "1.0.6",
|
||||
"@redis/search": "1.1.6",
|
||||
"@redis/time-series": "1.0.5"
|
||||
}
|
||||
},
|
||||
"regexpp": {
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
"lz4-napi": "^2.2.0",
|
||||
"pg": "^8.8.0",
|
||||
"rate-limit-redis": "^3.0.1",
|
||||
"redis": "^4.5.0",
|
||||
"redis": "^4.6.13",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -55,9 +55,12 @@ const writeResponseTime: number[] = [];
|
||||
let lastResponseTimeLimit = 0;
|
||||
const maxStoredTimes = 200;
|
||||
|
||||
// For redis
|
||||
let cacheConnectionClientId = "";
|
||||
|
||||
export class TooManyActiveConnectionsError extends Error {}
|
||||
|
||||
export let connectionPromise = Promise.resolve();
|
||||
export let connectionPromise: Promise<unknown> = Promise.resolve();
|
||||
|
||||
if (config.redis?.enabled) {
|
||||
Logger.info("Connected to redis");
|
||||
@@ -67,7 +70,7 @@ if (config.redis?.enabled) {
|
||||
void readClient?.connect(); // void as we don't care about the promise
|
||||
exportClient = client as unknown as RedisSB;
|
||||
|
||||
const cacheClient = config.redis.clientCacheLength ? createClient(config.redis) : null;
|
||||
let cacheClient = null as RedisClientType | null;
|
||||
const cache = config.redis.clientCacheLength ? new LRUCache<RedisCommandArgument, string>({
|
||||
max: config.redis.clientCacheLength
|
||||
}) : null;
|
||||
@@ -184,7 +187,7 @@ if (config.redis?.enabled) {
|
||||
Logger.error(`Redis Error: ${error}`);
|
||||
});
|
||||
/* istanbul ignore next */
|
||||
client.on("reconnect", () => {
|
||||
client.on("reconnecting", () => {
|
||||
Logger.info("Redis: trying to reconnect");
|
||||
});
|
||||
/* istanbul ignore next */
|
||||
@@ -193,26 +196,54 @@ if (config.redis?.enabled) {
|
||||
Logger.error(`Redis Read-Only Error: ${error}`);
|
||||
});
|
||||
/* istanbul ignore next */
|
||||
readClient?.on("reconnect", () => {
|
||||
readClient?.on("reconnecting", () => {
|
||||
Logger.info("Redis Read-Only: trying to reconnect");
|
||||
});
|
||||
|
||||
if (cacheClient) {
|
||||
// It needs to recreate itself when the connection fails as the queue connection doesn't properly restart
|
||||
const createCacheClient = () => {
|
||||
cacheClient = createClient(config.redis) as RedisClientType;
|
||||
|
||||
/* istanbul ignore next */
|
||||
cacheClient.on("error", function(error) {
|
||||
cacheClient.on("error", function (error) {
|
||||
lastClientFail = Date.now();
|
||||
Logger.error(`Redis Cache Client Error: ${error}`);
|
||||
});
|
||||
/* istanbul ignore next */
|
||||
cacheClient.on("reconnect", () => {
|
||||
cacheClient.on("reconnecting", () => {
|
||||
Logger.info("Redis cache client: trying to reconnect");
|
||||
cache?.clear();
|
||||
|
||||
void cacheClient.disconnect();
|
||||
setTimeout(() => createCacheClient(), 1000);
|
||||
});
|
||||
|
||||
setupClientCache(client as RedisClientType,
|
||||
readClient as RedisClientType,
|
||||
cacheClient as RedisClientType,
|
||||
cache).catch(Logger.error);
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
cacheClient.on("ready", async () => {
|
||||
cache?.clear();
|
||||
|
||||
await setupCacheClientListener(cacheClient as RedisClientType, cache);
|
||||
void setupCacheClientTracking(client as RedisClientType, cacheClient as RedisClientType);
|
||||
void setupCacheClientTracking(readClient as RedisClientType, cacheClient as RedisClientType);
|
||||
});
|
||||
|
||||
void cacheClient.connect();
|
||||
};
|
||||
|
||||
if (config.redis.clientCacheLength) {
|
||||
createCacheClient();
|
||||
|
||||
client.on("ready", () => {
|
||||
if (cacheClient.isReady) {
|
||||
void setupCacheClientTracking(client as RedisClientType, cacheClient as RedisClientType);
|
||||
}
|
||||
});
|
||||
|
||||
readClient?.on("ready", () => {
|
||||
if (cacheClient.isReady) {
|
||||
void setupCacheClientTracking(readClient as RedisClientType, cacheClient as RedisClientType);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,23 +268,22 @@ export function getRedisStats(): RedisStats {
|
||||
};
|
||||
}
|
||||
|
||||
async function setupClientCache(client: RedisClientType,
|
||||
readClient: RedisClientType,
|
||||
cacheClient: RedisClientType,
|
||||
async function setupCacheClientListener(cacheClient: RedisClientType,
|
||||
cache: LRUCache<RedisCommandArgument, string>) {
|
||||
|
||||
await cacheClient.connect();
|
||||
cacheConnectionClientId = String(await cacheClient.clientId());
|
||||
|
||||
const clientId = await cacheClient.sendCommand(["CLIENT", "ID"]);
|
||||
cacheClient.subscribe("__redis__:invalidate", (messages) => {
|
||||
cache.delete(messages[0]);
|
||||
}).catch(Logger.error);
|
||||
}
|
||||
|
||||
await client.sendCommand(["CLIENT", "TRACKING", "ON", "REDIRECT", String(clientId)]);
|
||||
async function setupCacheClientTracking(client: RedisClientType,
|
||||
cacheClient: RedisClientType) {
|
||||
|
||||
if (readClient) {
|
||||
await readClient.sendCommand(["CLIENT", "TRACKING", "ON", "REDIRECT", String(clientId)]);
|
||||
}
|
||||
if (!client || !cacheClient.isReady) return;
|
||||
|
||||
await client.sendCommand(["CLIENT", "TRACKING", "ON", "REDIRECT", cacheConnectionClientId]);
|
||||
}
|
||||
|
||||
export default exportClient;
|
||||
|
||||
Reference in New Issue
Block a user