From 37a07ace72d13f18c281b25f29c2c2c3fd27cd86 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 26 Mar 2021 19:03:30 -0400 Subject: [PATCH] Cache data for getting hash-prefix segments --- src/middleware/redisKeys.ts | 10 ++++++- src/routes/getSkipSegments.ts | 43 ++++++++++++++++++++++++------ src/routes/postSkipSegments.ts | 6 +++-- src/routes/voteOnSponsorTime.ts | 47 ++++++++++++++++++++------------- 4 files changed, 76 insertions(+), 30 deletions(-) diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts index 1335e15..56b1aab 100644 --- a/src/middleware/redisKeys.ts +++ b/src/middleware/redisKeys.ts @@ -1,5 +1,13 @@ -import { Category, VideoID } from "../types/segments.model"; +import { Service, VideoID, VideoIDHash } from "../types/segments.model"; +import { Logger } from "../utils/logger"; export function skipSegmentsKey(videoID: VideoID): string { return "segments-" + videoID; } + +export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { + hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; + if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix); + + return "segments." + service + "." + hashedVideoIDPrefix; +} \ No newline at end of file diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index b6ae72f..c2a9155 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -2,7 +2,7 @@ import { Request, Response } from 'express'; import { RedisClient } from 'redis'; import { config } from '../config'; import { db, privateDB } from '../databases/databases'; -import { skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getHash } from '../utils/getHash'; @@ -92,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category))); if (categories.length === 0) return null; - const segmentPerVideoID: SegmentWithHashPerVideoID = (await db - .prepare( - 'all', - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, - [hashedVideoIDPrefix + '%', service] - )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { + const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service)) + .filter((segment: DBSegment) => categories.includes(segment?.category)) + .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { hash: segment.hashedVideoID, segmentPerCategory: {}, @@ -131,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, } } +async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { + const fetchFromDB = () => db + .prepare( + 'all', + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" + WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`, + [hashedVideoIDPrefix + '%', service] + ); + + if (hashedVideoIDPrefix.length === 4) { + const key = skipSegmentsHashKey(hashedVideoIDPrefix, service); + const {err, reply} = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug("Got data from redis: " + reply); + return JSON.parse(reply); + } catch (e) { + // If all else, continue on to fetching from the database + } + } + + const data = await fetchFromDB(); + + redis.setAsync(key, JSON.stringify(data)); + return data; + } + + return await fetchFromDB(); +} + //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 2f847bf..75b572d 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -11,7 +11,7 @@ import {getFormattedTime} from '../utils/getFormattedTime'; import {isUserTrustworthy} from '../utils/isUserTrustworthy'; import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; -import { skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model'; @@ -497,13 +497,14 @@ export async function postSkipSegments(req: Request, res: Response) { //it's better than generating an actual UUID like what was used before //also better for duplication checking const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1])); + const hashedVideoID = getHash(videoID, 1); const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1), + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID, ], ); @@ -512,6 +513,7 @@ export async function postSkipSegments(req: Request, res: Response) { // Clear redis cache for this video redis.delAsync(skipSegmentsKey(videoID)); + redis.delAsync(skipSegmentsHashKey(hashedVideoID, service)); } catch (err) { //a DB change probably occurred res.sendStatus(500); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 29f0e66..c0aaac0 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash'; import {config} from '../config'; import { UserID } from '../types/user.model'; import redis from '../utils/redis'; -import { skipSegmentsKey } from '../middleware/redisKeys'; -import { VideoID } from '../types/segments.model'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; +import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; const voteTypes = { normal: 0, @@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) { } } -async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string - , hashedIP: string, finalResponse: FinalResponse, res: Response) { +async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category + , hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) { // Check if they've already made a vote const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); @@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS return; } - const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]); - if (!currentCategory) { + const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`, + [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service}; + if (!videoInfo) { // Submission doesn't exist res.status(400).send("Submission doesn't exist."); return; @@ -196,7 +197,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS } // See if the submissions category is ready to change - const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]); + const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]); const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); @@ -208,9 +209,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS // Add submission as vote if (!currentCategoryInfo && submissionInfo) { - await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]); + await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]); - await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]); + await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]); } const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; @@ -222,6 +223,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); } + clearRedisCache(videoInfo); + res.sendStatus(finalResponse.finalStatus); } @@ -230,10 +233,10 @@ export function getUserID(req: Request): UserID { } export async function voteOnSponsorTime(req: Request, res: Response) { - const UUID = req.query.UUID as string; + const UUID = req.query.UUID as SegmentUUID; const paramUserID = getUserID(req); let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; - const category = req.query.category as string; + const category = req.query.category as Category; if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) { //invalid request @@ -255,7 +258,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { const ip = getIP(req); //hash the ip 5000 times so no one can get it from the database - const hashedIP = getHash(ip + config.globalSalt); + const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress); //check if this user is on the vip list const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0; @@ -350,13 +353,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) { } //check if the increment amount should be multiplied (downvotes have more power if there have been many views) - const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as - {videoID: VideoID, votes: number, views: number}; + const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as + {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number}; if (voteTypeEnum === voteTypes.normal) { if ((isVIP || isOwnSubmission) && incrementAmount < 0) { //this user is a vip and a downvote - incrementAmount = -(row.votes + 2 - oldIncrementAmount); + incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount); type = incrementAmount; } } else if (voteTypeEnum == voteTypes.incorrect) { @@ -399,8 +402,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]); } - // Clear redis cache for this video - redis.delAsync(skipSegmentsKey(row?.videoID)); + clearRedisCache(videoInfo); //for each positive vote, see if a hidden submission can be shown again if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { @@ -437,7 +439,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { voteTypeEnum, isVIP, isOwnSubmission, - row, + row: videoInfo, category, incrementAmount, oldIncrementAmount, @@ -449,4 +451,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) { res.status(500).json({error: 'Internal error creating segment vote'}); } -} \ No newline at end of file +} + +function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) { + if (videoInfo) { + redis.delAsync(skipSegmentsKey(videoInfo.videoID)); + redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); + } +}