Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2026-01-01 06:09:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer

Browse Source
This commit is contained in:
Ajay
2022-04-14 01:54:28 -04:00
parent c7905d4062 8a9c7c869b
commit fb414ed6db
16 changed files with 312 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/config.ts
View File

@@ -8,6 +8,7 @@ const configFile = process.env.TEST_POSTGRES ? "ci.json"
: "config.json";
export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8"));
migrate(config);
addDefaults(config, {
port: 80,
behindProxy: "X-Forwarded-For",
@@ -109,3 +110,20 @@ function addDefaults(config: SBSConfig, defaults: SBSConfig) {
}
}
}
function migrate(config: SBSConfig) {
// Redis change
if (config.redis) {
const redisConfig = config.redis as any;
if (redisConfig.host || redisConfig.port) {
config.redis.socket = {
host: redisConfig.host,
port: redisConfig.port
};
}
if (redisConfig.enable_offline_queue !== undefined) {
config.disableOfflineQueue = !redisConfig.enable_offline_queue;
}
}
}

8
src/middleware/requestRateLimit.ts
View File

@@ -6,6 +6,9 @@ import { RateLimitConfig } from "../types/config.model";
import { Request } from "express";
import { isUserVIP } from "../utils/isUserVIP";
import { UserID } from "../types/user.model";
import RedisStore from "rate-limit-redis";
import redis from "../utils/redis";
import { config } from "../config";
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): RateLimitRequestHandler {
return rateLimit({
@@ -24,6 +27,9 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r
} else {
return next();
}
}
},
store: config.redis ? new RedisStore({
sendCommand: (...args: string[]) => redis.sendCommand(args),
}) : null,
});
}

25
src/routes/addUserAsTempVIP.ts
View File

@@ -9,6 +9,7 @@ import { isUserVIP } from "../utils/isUserVIP";
import { HashedUserID } from "../types/user.model";
import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys";
import { Logger } from "../utils/logger";
interface AddUserAsTempVIPRequest extends Request {
query: {
@@ -65,12 +66,22 @@ export async function addUserAsTempVIP(req: AddUserAsTempVIPRequest, res: Respon
if (!channelInfo?.id) {
return res.status(404).send(`No channel found for videoID ${channelVideoID}`);
}
await redis.setAsyncEx(tempVIPKey(userID), channelInfo?.id, dayInSeconds);
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
}
await redis.delAsync(tempVIPKey(userID));
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP removed`);
try {
await redis.setEx(tempVIPKey(userID), dayInSeconds, channelInfo?.id);
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
} catch (e) {
Logger.error(e as string);
return res.status(500).send();
}
}
try {
await redis.del(tempVIPKey(userID));
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP removed`);
} catch (e) {
Logger.error(e as string);
return res.status(500).send();
}
}

8
src/routes/getStatus.ts
View File

@@ -10,8 +10,12 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
value = Array.isArray(value) ? value[0] : value;
try {
const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value;
const numberRequests = await redis.increment("statusRequest");
const statusRequests = numberRequests?.replies?.[0];
let statusRequests: unknown = 0;
try {
const numberRequests = await redis.increment("statusRequest");
statusRequests = numberRequests?.[0];
} catch (error) { } // eslint-disable-line no-empty
const statusValues: Record<string, any> = {
uptime: process.uptime(),
commit: (global as any).HEADCOMMIT || "unknown",

2
src/types/config.model.ts
View File

@@ -41,7 +41,7 @@ export interface SBSConfig {
privateMysql?: any;
minimumPrefix?: string;
maximumPrefix?: string;
redis?: redis.ClientOpts;
redis?: redis.RedisClientOptions;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig;
dumpDatabase?: DumpDatabase;

17
src/utils/getHashCache.ts
View File

@@ -18,18 +18,19 @@ export async function getHashCache<T extends string>(value: T, times = defaulted
async function getFromRedis<T extends string>(key: HashedValue): Promise<T & HashedValue> {
const redisKey = shaHashKey(key);
const { err, reply } = await redis.getAsync(redisKey);
if (!err && reply) {
try {
try {
const reply = await redis.get(redisKey);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
return reply as T & HashedValue;
} catch (e) {
// If all else, continue on hashing
}
}
const data = getHash(key, cachedHashTimes);
} catch (e) {} // eslint-disable-line no-empty
// Otherwise, calculate it
const data = getHash(key, cachedHashTimes);
redis.set(key, data);
redis.setAsync(key, data);
return data as T & HashedValue;
}

11
src/utils/isUserTempVIP.ts
View File

@@ -5,6 +5,7 @@ import { YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model";
import { VideoID } from "../types/segments.model";
import { config } from "../config";
import { Logger } from "./logger";
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
@@ -13,7 +14,11 @@ function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<API
export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
const channelID = apiVideoInfo?.data?.authorId;
const { err, reply } = await redis.getAsync(tempVIPKey(hashedUserID));
return err || !reply ? false : (reply == channelID);
try {
const reply = await redis.get(tempVIPKey(hashedUserID));
return reply && reply == channelID;
} catch (e) {
Logger.error(e as string);
return false;
}
};

36
src/utils/queryCacher.ts
View File

@@ -5,21 +5,18 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { UserID } from "../types/user.model";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const { err, reply } = await redis.getAsync(key);
if (!err && reply) {
try {
try {
const reply = await redis.get(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
} catch (e) { } //eslint-disable-line no-empty
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
redis.set(key, JSON.stringify(data));
return data;
}
@@ -30,18 +27,17 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Promise<Array<T>>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise<Array<T>> {
const cachedValues = await Promise.all(values.map(async (value) => {
const key = keyGenerator(value);
const { err, reply } = await redis.getAsync(key);
if (!err && reply) {
try {
try {
const reply = await redis.get(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
return {
value,
result: JSON.parse(reply)
};
} catch (e) { } //eslint-disable-line no-empty
}
}
} catch (e) { } //eslint-disable-line no-empty
return {
value,
@@ -71,7 +67,7 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
}
for (const key in newResults) {
redis.setAsync(key, JSON.stringify(newResults[key]));
redis.set(key, JSON.stringify(newResults[key]));
}
});
}
@@ -81,16 +77,16 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID));
redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
if (videoInfo.userID) redis.del(reputationKey(videoInfo.userID));
}
}
function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
if (videoInfo) {
redis.delAsync(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
}
}

69
src/utils/redis.ts
View File

@@ -1,59 +1,56 @@
import { config } from "../config";
import { Logger } from "./logger";
import redis, { Callback } from "redis";
import { createClient } from "redis";
import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@node-redis/client/dist/lib/commands";
import { ClientCommandOptions } from "@node-redis/client/dist/lib/client";
import { RedisReply } from "rate-limit-redis";
interface RedisSB {
get(key: string, callback?: Callback<string | null>): void;
getAsync?(key: string): Promise<{err: Error | null, reply: string | null}>;
set(key: string, value: string, callback?: Callback<string | null>): void;
setAsync?(key: string, value: string): Promise<{err: Error | null, reply: string | null}>;
setAsyncEx?(key: string, value: string, seconds: number): Promise<{err: Error | null, reply: string | null}>;
delAsync?(...keys: [string]): Promise<Error | null>;
close?(flush?: boolean): void;
increment?(key: string): Promise<{err: Error| null, replies: any[] | null}>;
get(key: RedisCommandArgument): Promise<string>;
set(key: RedisCommandArgument, value: RedisCommandArgument): Promise<string>;
setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
del(...keys: [RedisCommandArgument]): Promise<number>;
increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
sendCommand(args: RedisCommandArguments, options?: ClientCommandOptions): Promise<RedisReply>;
quit(): Promise<void>;
}
let exportObject: RedisSB = {
get: (key, callback?) => callback(null, undefined),
getAsync: () =>
new Promise((resolve) => resolve({ err: null, reply: undefined })),
set: (key, value, callback) => callback(null, undefined),
setAsync: () =>
new Promise((resolve) => resolve({ err: null, reply: undefined })),
setAsyncEx: () =>
new Promise((resolve) => resolve({ err: null, reply: undefined })),
delAsync: () =>
new Promise((resolve) => resolve(null)),
increment: () =>
new Promise((resolve) => resolve({ err: null, replies: undefined })),
let exportClient: RedisSB = {
get: () => new Promise((resolve, reject) => reject()),
set: () => new Promise((resolve, reject) => reject()),
setEx: () => new Promise((resolve, reject) => reject()),
del: () => new Promise((resolve, reject) => reject()),
increment: () => new Promise((resolve, reject) => reject()),
sendCommand: () => new Promise((resolve, reject) => reject()),
quit: () => new Promise((resolve, reject) => reject()),
};
if (config.redis) {
Logger.info("Connected to redis");
const client = redis.createClient(config.redis);
exportObject = client;
const client = createClient(config.redis);
client.connect();
exportClient = client;
const timeoutDuration = 200;
exportObject.getAsync = (key) => new Promise((resolve) => {
const timeout = setTimeout(() => resolve({ err: null, reply: undefined }), timeoutDuration);
client.get(key, (err, reply) => {
const get = client.get.bind(client);
exportClient.get = (key) => new Promise((resolve, reject) => {
const timeout = setTimeout(() => reject(), timeoutDuration);
get(key).then((reply) => {
clearTimeout(timeout);
resolve({ err, reply });
});
resolve(reply);
}).catch((err) => reject(err));
});
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({ err, reply })));
exportObject.setAsyncEx = (key, value, seconds) => new Promise((resolve) => client.setex(key, seconds, value, (err, reply) => resolve({ err, reply })));
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
exportObject.close = (flush) => client.end(flush);
exportObject.increment = (key) => new Promise((resolve) =>
exportClient.increment = (key) => new Promise((resolve, reject) =>
client.multi()
.incr(key)
.expire(key, 60)
.exec((err, replies) => resolve({ err, replies }))
.exec()
.then((reply) => resolve(reply))
.catch((err) => reject(err))
);
client.on("error", function(error) {
Logger.error(error);
});
}
export default exportObject;
export default exportClient;