From a5a90e3c84a060515573d5d4e40ddc1cde5f16ec Mon Sep 17 00:00:00 2001 From: Michael C Date: Sat, 27 Nov 2021 02:41:34 -0500 Subject: [PATCH] add getHashCache where applicable= --- src/middleware/requestRateLimit.ts | 3 ++- src/routes/addUserAsVIP.ts | 4 ++-- src/routes/deleteLockCategories.ts | 4 ++-- src/routes/getIsUserVIP.ts | 4 ++-- src/routes/getSavedTimeForUser.ts | 4 ++-- src/routes/getSkipSegments.ts | 6 +++--- src/routes/getUserInfo.ts | 4 ++-- src/routes/getUserStats.ts | 4 ++-- src/routes/getUsername.ts | 4 ++-- src/routes/getViewsForUser.ts | 4 ++-- src/routes/postClearCache.ts | 6 +++--- src/routes/postLockCategories.ts | 6 +++--- src/routes/postPurgeAllSegments.ts | 6 +++--- src/routes/postSegmentShift.ts | 4 ++-- src/routes/postSkipSegments.ts | 5 +++-- src/routes/postWarning.ts | 4 ++-- src/routes/ratings/postClearCache.ts | 3 ++- src/routes/ratings/postRating.ts | 3 ++- src/routes/setUsername.ts | 6 +++--- src/routes/shadowBanUser.ts | 4 ++-- src/routes/voteOnSponsorTime.ts | 8 +++---- src/utils/getHash.ts | 2 +- src/utils/getHashCache.ts | 32 ++++++++++++++++++++++++++++ src/utils/redisKeys.ts | 7 ++++++ 24 files changed, 90 insertions(+), 47 deletions(-) create mode 100644 src/utils/getHashCache.ts diff --git a/src/middleware/requestRateLimit.ts b/src/middleware/requestRateLimit.ts index e5cb8b9..71f8bd9 100644 --- a/src/middleware/requestRateLimit.ts +++ b/src/middleware/requestRateLimit.ts @@ -1,5 +1,6 @@ import { getIP } from "../utils/getIP"; import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import rateLimit from "express-rate-limit"; import { RateLimitConfig } from "../types/config.model"; import { Request } from "express"; @@ -17,7 +18,7 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r return getHash(getIP(req), 1); }, handler: async (req, res, next) => { - if (getUserID === undefined || !await isUserVIP(getHash(getUserID(req)))) { + if (getUserID === undefined || !await isUserVIP(await getHashCache(getUserID(req)))) { return res.status(limitConfig.statusCode).send(limitConfig.message); } else { return next(); diff --git a/src/routes/addUserAsVIP.ts b/src/routes/addUserAsVIP.ts index 3d94490..3e7f0c7 100644 --- a/src/routes/addUserAsVIP.ts +++ b/src/routes/addUserAsVIP.ts @@ -1,4 +1,4 @@ -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { db } from "../databases/databases"; import { config } from "../config"; import { Request, Response } from "express"; @@ -24,7 +24,7 @@ export async function addUserAsVIP(req: AddUserAsVIPRequest, res: Response): Pro } // hash the userID - const adminUserIDInput = getHash(adminUserID); + const adminUserIDInput = await getHashCache(adminUserID); if (adminUserIDInput !== config.adminUserID) { // not authorized diff --git a/src/routes/deleteLockCategories.ts b/src/routes/deleteLockCategories.ts index e98ed23..369ea52 100644 --- a/src/routes/deleteLockCategories.ts +++ b/src/routes/deleteLockCategories.ts @@ -1,6 +1,6 @@ import { Request, Response } from "express"; import { isUserVIP } from "../utils/isUserVIP"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { db } from "../databases/databases"; import { Category, Service, VideoID } from "../types/segments.model"; import { UserID } from "../types/user.model"; @@ -39,7 +39,7 @@ export async function deleteLockCategoriesEndpoint(req: DeleteLockCategoriesRequ } // Check if user is VIP - const hashedUserID = getHash(userID); + const hashedUserID = await getHashCache(userID); const userIsVIP = await isUserVIP(hashedUserID); if (!userIsVIP) { diff --git a/src/routes/getIsUserVIP.ts b/src/routes/getIsUserVIP.ts index 7b2d523..09f4347 100644 --- a/src/routes/getIsUserVIP.ts +++ b/src/routes/getIsUserVIP.ts @@ -1,5 +1,5 @@ import { Logger } from "../utils/logger"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { isUserVIP } from "../utils/isUserVIP"; import { Request, Response } from "express"; import { HashedUserID, UserID } from "../types/user.model"; @@ -13,7 +13,7 @@ export async function getIsUserVIP(req: Request, res: Response): Promise ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, userID]); diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index b8b0e71..f9b6130 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -5,7 +5,7 @@ import { skipSegmentsHashKey, skipSegmentsKey } from "../utils/redisKeys"; import { SBRecord } from "../types/lib.model"; import { ActionType, Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, SegmentUUID, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getCategoryActionType } from "../utils/categoryInfo"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { getIP } from "../utils/getIP"; import { Logger } from "../utils/logger"; import { QueryCacher } from "../utils/queryCacher"; @@ -33,10 +33,10 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: } //if this isn't their ip, don't send it to them - return cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some((shadowHiddenSegment) => { + return cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some(async (shadowHiddenSegment) => { if (cache.userHashedIP === undefined) { //hash the IP only if it's strictly necessary - cache.userHashedIP = getHash((getIP(req) + config.globalSalt) as IPAddress); + cache.userHashedIP = await getHashCache((getIP(req) + config.globalSalt) as IPAddress); } return shadowHiddenSegment.hashedIP === cache.userHashedIP; diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index ebca98c..37aa678 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -1,5 +1,5 @@ import { db } from "../databases/databases"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { isUserVIP } from "../utils/isUserVIP"; import { Request, Response } from "express"; import { Logger } from "../utils/logger"; @@ -134,7 +134,7 @@ const dbGetValue = (userID: HashedUserID, property: string): Promise { const userID = req.query.userID as UserID; - const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID; + const hashedUserID: HashedUserID = userID ? await getHashCache(userID) : req.query.publicUserID as HashedUserID; const defaultProperties: string[] = ["userID", "userName", "minutesSaved", "segmentCount", "ignoredSegmentCount", "viewCount", "ignoredViewCount", "warnings", "warningReason", "reputation", "vip", "lastSegmentID"]; diff --git a/src/routes/getUserStats.ts b/src/routes/getUserStats.ts index 7708255..b58c611 100644 --- a/src/routes/getUserStats.ts +++ b/src/routes/getUserStats.ts @@ -1,5 +1,5 @@ import { db } from "../databases/databases"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { Request, Response } from "express"; import { HashedUserID, UserID } from "../types/user.model"; import { config } from "../config"; @@ -78,7 +78,7 @@ async function dbGetUsername(userID: HashedUserID) { export async function getUserStats(req: Request, res: Response): Promise { const userID = req.query.userID as UserID; - const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID; + const hashedUserID: HashedUserID = userID ? await getHashCache(userID) : req.query.publicUserID as HashedUserID; const fetchCategoryStats = req.query.fetchCategoryStats == "true"; const fetchActionTypeStats = req.query.fetchActionTypeStats == "true"; diff --git a/src/routes/getUsername.ts b/src/routes/getUsername.ts index 9fbafdf..6411434 100644 --- a/src/routes/getUsername.ts +++ b/src/routes/getUsername.ts @@ -1,5 +1,5 @@ import { db } from "../databases/databases"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { Logger } from "../utils/logger"; import { Request, Response } from "express"; @@ -12,7 +12,7 @@ export async function getUsername(req: Request, res: Response): Promise { @@ -12,7 +12,7 @@ export async function getViewsForUser(req: Request, res: Response): Promise req.body.issuerUserID); + const issuerUserID: HashedUserID = await getHashCache( req.body.issuerUserID); const userID: UserID = req.body.userID; const issueTime = new Date().getTime(); const enabled: boolean = req.body.enabled ?? true; diff --git a/src/routes/ratings/postClearCache.ts b/src/routes/ratings/postClearCache.ts index ed3d5d5..38a8707 100644 --- a/src/routes/ratings/postClearCache.ts +++ b/src/routes/ratings/postClearCache.ts @@ -1,6 +1,7 @@ import { Logger } from "../../utils/logger"; import { HashedUserID, UserID } from "../../types/user.model"; import { getHash } from "../../utils/getHash"; +import { getHashCache } from "../../utils/getHashCache"; import { Request, Response } from "express"; import { Service, VideoID } from "../../types/segments.model"; import { QueryCacher } from "../../utils/queryCacher"; @@ -28,7 +29,7 @@ export async function postClearCache(req: Request, res: Response): Promise } const hashedIP: HashedIP = getHash(getIP(req) + config.globalSalt as IPAddress, 1); - const hashedUserID: HashedUserID = getHash(privateUserID); + const hashedUserID: HashedUserID = await getHashCache(privateUserID); const hashedVideoID = getHash(videoID, 1); try { diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index dee0e99..e31328c 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -1,7 +1,7 @@ import { config } from "../config"; import { Logger } from "../utils/logger"; import { db, privateDB } from "../databases/databases"; -import { getHash } from "../utils/getHash"; +import { getHashCache } from "../utils/getHashCache"; import { Request, Response } from "express"; function logUserNameChange(userID: string, newUserName: string, oldUserName: string, updatedByAdmin: boolean): Promise { @@ -34,7 +34,7 @@ export async function setUsername(req: Request, res: Response): Promise(value: T, times = 5000): T & HashedVal } return value as T & HashedValue; -} +} \ No newline at end of file diff --git a/src/utils/getHashCache.ts b/src/utils/getHashCache.ts new file mode 100644 index 0000000..fac5924 --- /dev/null +++ b/src/utils/getHashCache.ts @@ -0,0 +1,32 @@ +import redis from "../utils/redis"; +import { userHashKey } from "../utils/redisKeys"; +import { HashedValue } from "../types/hash.model"; +import { Logger } from "../utils/logger"; +import { getHash } from "../utils/getHash"; + +export async function getHashCache(value: T, times = 5000): Promise { + if (times === 5000) { + const hashKey = getHash(value, 1); + const result: HashedValue = await getFromRedis(hashKey); + return result as T & HashedValue; + } + return getHash(value, times); +} + +async function getFromRedis(key: HashedValue): Promise { + const redisKey = userHashKey(key); + const { err, reply } = await redis.getAsync(redisKey); + + if (!err && reply) { + try { + Logger.debug(`Got data from redis: ${reply}`); + return reply as T & HashedValue; + } catch (e) { + // If all else, continue on hashing + } + } + const data = getHash(key, 5000-1); + + redis.setAsync(key, data); + return data as T & HashedValue; +} \ No newline at end of file diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index 4d9cd9e..8c5d432 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -1,5 +1,6 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { UserID } from "../types/user.model"; +import { HashedValue } from "../types/hash.model"; import { Logger } from "./logger"; export function skipSegmentsKey(videoID: VideoID, service: Service): string { @@ -22,4 +23,10 @@ export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`); return `rating.${service}.${hashPrefix}`; +} + +export function userHashKey(userID: HashedValue): string { + if (userID.length !== 64) Logger.warn(`Redis userHash key is not length 64! ${userID}`); + + return `user.${userID}`; } \ No newline at end of file