Cache data for getting hash-prefix segments

src/middleware/redisKeys.ts

@@ -1,5 +1,13 @@
-import { Category, VideoID } from "../types/segments.model";
+import { Service, VideoID, VideoIDHash } from "../types/segments.model";
+import { Logger } from "../utils/logger";
 
 export function skipSegmentsKey(videoID: VideoID): string {
     return "segments-" + videoID;
 } 
+
+export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
+    hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
+    if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
+    
+    return "segments." + service + "." + hashedVideoIDPrefix;
+} 

src/routes/getSkipSegments.ts

@@ -2,7 +2,7 @@ import { Request, Response } from 'express';
 import { RedisClient } from 'redis';
 import { config } from '../config';
 import { db, privateDB } from '../databases/databases';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
 import { SBRecord } from '../types/lib.model';
 import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
 import { getHash } from '../utils/getHash';
@@ -92,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
         categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
         if (categories.length === 0) return null;
 
-        const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
+        const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
-            .prepare(
+            .filter((segment: DBSegment) => categories.includes(segment?.category))
-                'all',
+            .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
-                `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
-                WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
-                [hashedVideoIDPrefix + '%', service]
-            )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
                     hash: segment.hashedVideoID,
                     segmentPerCategory: {},
@@ -131,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
     }
 }
 
+async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
+    const fetchFromDB = () => db
+        .prepare(
+            'all',
+            `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
+            WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`,
+            [hashedVideoIDPrefix + '%', service]
+        );
+
+    if (hashedVideoIDPrefix.length === 4) {
+        const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
+        const {err, reply} = await redis.getAsync(key);
+    
+        if (!err && reply) {
+            try {
+                Logger.debug("Got data from redis: " + reply);
+                return JSON.parse(reply);
+            } catch (e) {
+                // If all else, continue on to fetching from the database
+            }
+        }
+
+        const data = await fetchFromDB();
+
+        redis.setAsync(key, JSON.stringify(data));
+        return data;
+    }
+
+    return await fetchFromDB();
+}
+
 //gets a weighted random choice from the choices array based on their `votes` property.
 //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
 //choices are unique

src/routes/postSkipSegments.ts

@@ -11,7 +11,7 @@ import {getFormattedTime} from '../utils/getFormattedTime';
 import {isUserTrustworthy} from '../utils/isUserTrustworthy';
 import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
 import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
 
@@ -497,13 +497,14 @@ export async function postSkipSegments(req: Request, res: Response) {
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
             const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
+            const hashedVideoID = getHash(videoID, 1);
 
             const startingLocked = isVIP ? 1 : 0;
             try {
                 await db.prepare('run', `INSERT INTO "sponsorTimes" 
                     ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
                     VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1),
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
                     ],
                 );
 
@@ -512,6 +513,7 @@ export async function postSkipSegments(req: Request, res: Response) {
             
                 // Clear redis cache for this video
                 redis.delAsync(skipSegmentsKey(videoID));
+                redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
             } catch (err) {
                 //a DB change probably occurred
                 res.sendStatus(500);

src/routes/voteOnSponsorTime.ts

@@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash';
 import {config} from '../config';
 import { UserID } from '../types/user.model';
 import redis from '../utils/redis';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
-import { VideoID } from '../types/segments.model';
+import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
 
 const voteTypes = {
     normal: 0,
@@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) {
     }
 }
 
-async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string
+async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category
-            , hashedIP: string, finalResponse: FinalResponse, res: Response) {
+            , hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) {
     // Check if they've already made a vote
     const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]);
 
@@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
         return;
     }
 
-    const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]);
+    const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
-    if (!currentCategory) {
+                             [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
+    if (!videoInfo) {
         // Submission doesn't exist
         res.status(400).send("Submission doesn't exist.");
         return;
@@ -196,7 +197,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
     }
 
     // See if the submissions category is ready to change
-    const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
+    const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
 
     const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
     const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
@@ -208,9 +209,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
 
     // Add submission as vote
     if (!currentCategoryInfo && submissionInfo) {
-        await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
+        await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
 
-        await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
+        await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
     }
 
     const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
@@ -222,6 +223,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
         await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
     }
 
+    clearRedisCache(videoInfo);
+
     res.sendStatus(finalResponse.finalStatus);
 }
 
@@ -230,10 +233,10 @@ export function getUserID(req: Request): UserID {
 }
 
 export async function voteOnSponsorTime(req: Request, res: Response) {
-    const UUID = req.query.UUID as string;
+    const UUID = req.query.UUID as SegmentUUID;
     const paramUserID = getUserID(req);
     let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
-    const category = req.query.category as string;
+    const category = req.query.category as Category;
 
     if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
         //invalid request
@@ -255,7 +258,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
     const ip = getIP(req);
 
     //hash the ip 5000 times so no one can get it from the database
-    const hashedIP = getHash(ip + config.globalSalt);
+    const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress);
 
     //check if this user is on the vip list
     const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0;
@@ -350,13 +353,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         }
 
         //check if the increment amount should be multiplied (downvotes have more power if there have been many views)
-        const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as 
+        const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as 
-                        {videoID: VideoID, votes: number, views: number};
+                        {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
 
         if (voteTypeEnum === voteTypes.normal) {
             if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
                 //this user is a vip and a downvote
-                incrementAmount = -(row.votes + 2 - oldIncrementAmount);
+                incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount);
                 type = incrementAmount;
             }
         } else if (voteTypeEnum == voteTypes.incorrect) {
@@ -399,8 +402,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
                  await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
             }
 
-            // Clear redis cache for this video
+            clearRedisCache(videoInfo);
-            redis.delAsync(skipSegmentsKey(row?.videoID));
 
             //for each positive vote, see if a hidden submission can be shown again
             if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
@@ -437,7 +439,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
                 voteTypeEnum,
                 isVIP,
                 isOwnSubmission,
-                row,
+                row: videoInfo,
                 category,
                 incrementAmount,
                 oldIncrementAmount,
@@ -450,3 +452,10 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         res.status(500).json({error: 'Internal error creating segment vote'});
     }
 }
+
+function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
+    if (videoInfo) {
+        redis.delAsync(skipSegmentsKey(videoInfo.videoID));
+        redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
+    }
+}