Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-07 20:17:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add caching for 5 length skip skip segment query

Browse Source
This commit is contained in:
Ajay
2025-03-12 02:45:59 -04:00
parent 31e678fdc2
commit 4fe7cebcb3
5 changed files with 17 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/middleware/etag.ts
View File

@@ -1,9 +1,9 @@
import { NextFunction, Request, Response } from "express"; import { NextFunction, Request, Response } from "express";
import { VideoID, VideoIDHash, Service } from "../types/segments.model"; import { VideoID, VideoIDHash, Service } from "../types/segments.model";
import { QueryCacher } from "../utils/queryCacher"; import { QueryCacher } from "../utils/queryCacher";
import { brandingHashKey, brandingKey, skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKey, videoLabelsLargerHashKey } from "../utils/redisKeys"; import { brandingHashKey, brandingKey, skipSegmentsHashKey, skipSegmentsKey, skipSegmentsLargerHashKey, videoLabelsHashKey, videoLabelsKey, videoLabelsLargerHashKey } from "../utils/redisKeys";
type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash" | "videoLabelsLargerHash" | "branding" | "brandingHash"; type hashType = "skipSegments" | "skipSegmentsHash" | "skipSegmentsLargerHash" | "videoLabel" | "videoLabelHash" | "videoLabelsLargerHash" | "branding" | "brandingHash";
type ETag = `"${hashType};${VideoIDHash};${Service};${number}"`; type ETag = `"${hashType};${VideoIDHash};${Service};${number}"`;
type hashKey = string | VideoID | VideoIDHash; type hashKey = string | VideoID | VideoIDHash;
@@ -30,6 +30,7 @@ function getLastModified(hashType: hashType, hashKey: hashKey, service: Service)
let redisKey: string | null; let redisKey: string | null;
if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service); if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service);
else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service); else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service);
else if (hashType === "skipSegmentsLargerHash") redisKey = skipSegmentsLargerHashKey(hashKey as VideoIDHash, service);
else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service); else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service);
else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service); else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service);
else if (hashType === "videoLabelsLargerHash") redisKey = videoLabelsLargerHashKey(hashKey as VideoIDHash, service); else if (hashType === "videoLabelsLargerHash") redisKey = videoLabelsLargerHashKey(hashKey as VideoIDHash, service);

4
src/routes/getSkipSegments.ts
View File

@@ -2,7 +2,7 @@ import { Request, Response } from "express";
import { partition } from "lodash"; import { partition } from "lodash";
import { config } from "../config"; import { config } from "../config";
import { db, privateDB } from "../databases/databases"; import { db, privateDB } from "../databases/databases";
import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey, shadowHiddenIPKey } from "../utils/redisKeys"; import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey, shadowHiddenIPKey, skipSegmentsLargerHashKey } from "../utils/redisKeys";
import { SBRecord } from "../types/lib.model"; import { SBRecord } from "../types/lib.model";
import { ActionType, Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, SegmentUUID, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { ActionType, Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, SegmentUUID, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { getHashCache } from "../utils/getHashCache"; import { getHashCache } from "../utils/getHashCache";
@@ -226,6 +226,8 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service
if (hashedVideoIDPrefix.length === 4) { if (hashedVideoIDPrefix.length === 4) {
return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)); return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service));
} else if (hashedVideoIDPrefix.length === 5) {
return await QueryCacher.get(fetchFromDB, skipSegmentsLargerHashKey(hashedVideoIDPrefix, service));
} }
return await fetchFromDB(); return await fetchFromDB();

3
src/routes/getSkipSegmentsByHash.ts
View File

@@ -23,7 +23,8 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, trimUUIDs, requiredSegments, service); const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, trimUUIDs, requiredSegments, service);
try { try {
await getEtag("skipSegmentsHash", hashPrefix, service) const hashKey = hashPrefix.length === 4 ? "skipSegmentsHash" : "skipSegmentsLargerHash";
await getEtag(hashKey, hashPrefix, service)
.then(etag => res.set("ETag", etag)) .then(etag => res.set("ETag", etag))
.catch(/* istanbul ignore next */ () => null); .catch(/* istanbul ignore next */ () => null);
const output = Object.entries(segments).map(([videoID, data]) => ({ const output = Object.entries(segments).map(([videoID, data]) => ({

3
src/utils/queryCacher.ts
View File

@@ -1,6 +1,6 @@
import redis, { TooManyActiveConnectionsError } from "../utils/redis"; import redis, { TooManyActiveConnectionsError } from "../utils/redis";
import { Logger } from "../utils/logger"; import { Logger } from "../utils/logger";
import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey, skipSegmentGroupsKey, userFeatureKey, videoLabelsKey, videoLabelsHashKey, brandingHashKey, brandingKey, videoLabelsLargerHashKey } from "./redisKeys"; import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey, skipSegmentGroupsKey, userFeatureKey, videoLabelsKey, videoLabelsHashKey, brandingHashKey, brandingKey, videoLabelsLargerHashKey, skipSegmentsLargerHashKey } from "./redisKeys";
import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { Feature, HashedUserID, UserID } from "../types/user.model"; import { Feature, HashedUserID, UserID } from "../types/user.model";
import { config } from "../config"; import { config } from "../config";
@@ -125,6 +125,7 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID
redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err));
redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err));
redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
redis.del(skipSegmentsLargerHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
redis.del(videoLabelsKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
redis.del(videoLabelsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
redis.del(videoLabelsLargerHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsLargerHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));

7
src/utils/redisKeys.ts
View File

@@ -18,6 +18,13 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
return `segments.v4.${service}.${hashedVideoIDPrefix}`; return `segments.v4.${service}.${hashedVideoIDPrefix}`;
} }
export function skipSegmentsLargerHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 5) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 5) Logger.warn(`Redis skip segment hash-prefix key is not length 5! ${hashedVideoIDPrefix}`);
return `segments.v4.${service}.${hashedVideoIDPrefix}`;
}
export const brandingKey = (videoID: VideoID, service: Service): string => export const brandingKey = (videoID: VideoID, service: Service): string =>
`branding.v3.${service}.videoID.${videoID}`; `branding.v3.${service}.videoID.${videoID}`;