From 4fe7cebcb3ea80f55c83bc419edee28185c242d4 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 12 Mar 2025 02:45:59 -0400 Subject: [PATCH] Add caching for 5 length skip skip segment query --- src/middleware/etag.ts | 5 +++-- src/routes/getSkipSegments.ts | 4 +++- src/routes/getSkipSegmentsByHash.ts | 3 ++- src/utils/queryCacher.ts | 3 ++- src/utils/redisKeys.ts | 7 +++++++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/middleware/etag.ts b/src/middleware/etag.ts index 5298238..d92b98d 100644 --- a/src/middleware/etag.ts +++ b/src/middleware/etag.ts @@ -1,9 +1,9 @@ import { NextFunction, Request, Response } from "express"; import { VideoID, VideoIDHash, Service } from "../types/segments.model"; import { QueryCacher } from "../utils/queryCacher"; -import { brandingHashKey, brandingKey, skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKey, videoLabelsLargerHashKey } from "../utils/redisKeys"; +import { brandingHashKey, brandingKey, skipSegmentsHashKey, skipSegmentsKey, skipSegmentsLargerHashKey, videoLabelsHashKey, videoLabelsKey, videoLabelsLargerHashKey } from "../utils/redisKeys"; -type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash" | "videoLabelsLargerHash" | "branding" | "brandingHash"; +type hashType = "skipSegments" | "skipSegmentsHash" | "skipSegmentsLargerHash" | "videoLabel" | "videoLabelHash" | "videoLabelsLargerHash" | "branding" | "brandingHash"; type ETag = `"${hashType};${VideoIDHash};${Service};${number}"`; type hashKey = string | VideoID | VideoIDHash; @@ -30,6 +30,7 @@ function getLastModified(hashType: hashType, hashKey: hashKey, service: Service) let redisKey: string | null; if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service); else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service); + else if (hashType === "skipSegmentsLargerHash") redisKey = skipSegmentsLargerHashKey(hashKey as VideoIDHash, service); else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service); else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service); else if (hashType === "videoLabelsLargerHash") redisKey = videoLabelsLargerHashKey(hashKey as VideoIDHash, service); diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index c140828..e21305c 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import { partition } from "lodash"; import { config } from "../config"; import { db, privateDB } from "../databases/databases"; -import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey, shadowHiddenIPKey } from "../utils/redisKeys"; +import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey, shadowHiddenIPKey, skipSegmentsLargerHashKey } from "../utils/redisKeys"; import { SBRecord } from "../types/lib.model"; import { ActionType, Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, SegmentUUID, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getHashCache } from "../utils/getHashCache"; @@ -226,6 +226,8 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service if (hashedVideoIDPrefix.length === 4) { return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)); + } else if (hashedVideoIDPrefix.length === 5) { + return await QueryCacher.get(fetchFromDB, skipSegmentsLargerHashKey(hashedVideoIDPrefix, service)); } return await fetchFromDB(); diff --git a/src/routes/getSkipSegmentsByHash.ts b/src/routes/getSkipSegmentsByHash.ts index 2110c4d..5418257 100644 --- a/src/routes/getSkipSegmentsByHash.ts +++ b/src/routes/getSkipSegmentsByHash.ts @@ -23,7 +23,8 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, trimUUIDs, requiredSegments, service); try { - await getEtag("skipSegmentsHash", hashPrefix, service) + const hashKey = hashPrefix.length === 4 ? "skipSegmentsHash" : "skipSegmentsLargerHash"; + await getEtag(hashKey, hashPrefix, service) .then(etag => res.set("ETag", etag)) .catch(/* istanbul ignore next */ () => null); const output = Object.entries(segments).map(([videoID, data]) => ({ diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index cd34310..81d7d87 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -1,6 +1,6 @@ import redis, { TooManyActiveConnectionsError } from "../utils/redis"; import { Logger } from "../utils/logger"; -import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey, skipSegmentGroupsKey, userFeatureKey, videoLabelsKey, videoLabelsHashKey, brandingHashKey, brandingKey, videoLabelsLargerHashKey } from "./redisKeys"; +import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey, skipSegmentGroupsKey, userFeatureKey, videoLabelsKey, videoLabelsHashKey, brandingHashKey, brandingKey, videoLabelsLargerHashKey, skipSegmentsLargerHashKey } from "./redisKeys"; import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { Feature, HashedUserID, UserID } from "../types/user.model"; import { config } from "../config"; @@ -125,6 +125,7 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); + redis.del(skipSegmentsLargerHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsLargerHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index 0435461..b7594a0 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -18,6 +18,13 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S return `segments.v4.${service}.${hashedVideoIDPrefix}`; } +export function skipSegmentsLargerHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { + hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 5) as VideoIDHash; + if (hashedVideoIDPrefix.length !== 5) Logger.warn(`Redis skip segment hash-prefix key is not length 5! ${hashedVideoIDPrefix}`); + + return `segments.v4.${service}.${hashedVideoIDPrefix}`; +} + export const brandingKey = (videoID: VideoID, service: Service): string => `branding.v3.${service}.videoID.${videoID}`;