diff --git a/src/app.ts b/src/app.ts index 61b2347..d680073 100644 --- a/src/app.ts +++ b/src/app.ts @@ -50,6 +50,7 @@ import { getVideoLabelsByHash } from "./routes/getVideoLabelByHash"; import { addFeature } from "./routes/addFeature"; import { generateTokenRequest } from "./routes/generateToken"; import { verifyTokenRequest } from "./routes/verifyToken"; +import { cacheMiddlware } from "./middleware/etag"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -57,11 +58,13 @@ export function createServer(callback: () => void): Server { const router = ExpressPromiseRouter(); app.use(router); + app.set("etag", false); // disable built in etag //setup CORS correctly router.use(corsMiddleware); router.use(loggerMiddleware); router.use("/api/", apiCspMiddleware); + router.use(cacheMiddlware); router.use(express.json()); if (config.userCounterURL) router.use(userCounter); diff --git a/src/middleware/cors.ts b/src/middleware/cors.ts index e3b71ab..6e5f2c2 100644 --- a/src/middleware/cors.ts +++ b/src/middleware/cors.ts @@ -3,6 +3,6 @@ import { NextFunction, Request, Response } from "express"; export function corsMiddleware(req: Request, res: Response, next: NextFunction): void { res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE"); - res.header("Access-Control-Allow-Headers", "Content-Type"); + res.header("Access-Control-Allow-Headers", "Content-Type, If-None-Match"); next(); } diff --git a/src/middleware/etag.ts b/src/middleware/etag.ts new file mode 100644 index 0000000..4a7cb58 --- /dev/null +++ b/src/middleware/etag.ts @@ -0,0 +1,48 @@ +import { NextFunction, Request, Response } from "express"; +import { VideoID, VideoIDHash, Service } from "../types/segments.model"; +import { QueryCacher } from "../utils/queryCacher"; +import { skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKey } from "../utils/redisKeys"; + +type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash"; +type ETag = `${hashType};${VideoIDHash};${Service};${number}`; + +export function cacheMiddlware(req: Request, res: Response, next: NextFunction): void { + const reqEtag = req.get("If-None-Match") as string; + // if weak etag, do not handle + if (!reqEtag || reqEtag.startsWith("W/")) return next(); + // split into components + const [hashType, hashKey, service, lastModified] = reqEtag.split(";"); + // fetch last-modified + getLastModified(hashType as hashType, hashKey as VideoIDHash, service as Service) + .then(redisLastModified => { + if (redisLastModified <= new Date(Number(lastModified) + 1000)) { + // match cache, generate etag + const etag = `${hashType};${hashKey};${service};${redisLastModified.getTime()}` as ETag; + res.status(304).set("etag", etag).send(); + } + else next(); + }) + .catch(next); +} + +function getLastModified(hashType: hashType, hashKey: (VideoID | VideoIDHash), service: Service): Promise { + let redisKey: string | null; + if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service); + else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service); + else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service); + else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service); + else return Promise.reject(); + return QueryCacher.getKeyLastModified(redisKey); +} + +export async function getEtag(hashType: hashType, hashKey: VideoIDHash, service: Service): Promise { + const lastModified = await getLastModified(hashType, hashKey, service); + return `${hashType};${hashKey};${service};${lastModified.getTime()}` as ETag; +} + +/* example usage +import { getEtag } from "../middleware/etag"; +await getEtag(hashType, hashPrefix, service) + .then(etag => res.set("ETag", etag)) + .catch(() => null); +*/ \ No newline at end of file diff --git a/src/routes/getSkipSegmentsByHash.ts b/src/routes/getSkipSegmentsByHash.ts index 7847879..270ade2 100644 --- a/src/routes/getSkipSegmentsByHash.ts +++ b/src/routes/getSkipSegmentsByHash.ts @@ -4,6 +4,7 @@ import { Request, Response } from "express"; import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model"; import { getService } from "../utils/getService"; import { Logger } from "../utils/logger"; +import { getEtag } from "../middleware/etag"; export async function getSkipSegmentsByHash(req: Request, res: Response): Promise { let hashPrefix = req.params.prefix as VideoIDHash; @@ -69,6 +70,9 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service); try { + await getEtag("skipSegmentsHash", hashPrefix, service) + .then(etag => res.set("ETag", etag)) + .catch(() => null); const output = Object.entries(segments).map(([videoID, data]) => ({ videoID, segments: data.segments, diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index 1b3f6a1..6b02268 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -87,6 +87,17 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID } } +async function getKeyLastModified(key: string): Promise { + if (!config.redis?.enabled) return Promise.reject("ETag - Redis not enabled"); + return await redis.ttl(key) + .then(ttl => { + const sinceLive = config.redis?.expiryTime - ttl; + const now = Math.floor(Date.now() / 1000); + return new Date((now-sinceLive) * 1000); + }) + .catch(() => Promise.reject("ETag - Redis error")); +} + function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void { if (videoInfo) { redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); @@ -101,6 +112,7 @@ export const QueryCacher = { get, getAndSplit, clearSegmentCache, + getKeyLastModified, clearRatingCache, - clearFeatureCache + clearFeatureCache, }; \ No newline at end of file diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 01761ae..ef55906 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -19,6 +19,7 @@ interface RedisSB { del(...keys: [RedisCommandArgument]): Promise; increment?(key: RedisCommandArgument): Promise; sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise; + ttl(key: RedisCommandArgument): Promise; quit(): Promise; } @@ -30,6 +31,7 @@ let exportClient: RedisSB = { increment: () => new Promise((resolve) => resolve(null)), sendCommand: () => new Promise((resolve) => resolve(null)), quit: () => new Promise((resolve) => resolve(null)), + ttl: () => new Promise((resolve) => resolve(null)), }; let lastClientFail = 0;