From cbf043ac7e01a1566c8c0bf44aa1cc3bde356b5f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 20 Mar 2021 11:54:50 -0400 Subject: [PATCH 1/7] Add twitch --- src/types/segments.model.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 6c9cd4a..a95bac3 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -12,8 +12,9 @@ export type HashedIP = IPAddress & HashedValue; // Uncomment as needed export enum Service { YouTube = 'YouTube', - // Nebula = 'Nebula', PeerTube = 'PeerTube', + // Twitch = 'Twitch', + // Nebula = 'Nebula', // RSS = 'RSS', // Corridor = 'Corridor', // Lbry = 'Lbry' From 11b4f642a60c973a4060e0416487e4a60a55014c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 21 Mar 2021 19:16:56 -0400 Subject: [PATCH 2/7] Apply indexes after upgrades --- databases/_private_indexes.sql | 32 ++++++++++++++ databases/_sponsorTimes_indexes.sql | 66 +++++++++++++++++++++++++++++ src/databases/Postgres.ts | 11 +++++ 3 files changed, 109 insertions(+) create mode 100644 databases/_private_indexes.sql create mode 100644 databases/_sponsorTimes_indexes.sql diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql new file mode 100644 index 0000000..d7fb746 --- /dev/null +++ b/databases/_private_indexes.sql @@ -0,0 +1,32 @@ +-- sponsorTimes + +CREATE INDEX IF NOT EXISTS "idx_16928_sponsorTimes_hashedIP" + ON public."sponsorTimes" USING btree + ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP" + ON public."sponsorTimes" USING btree + ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- votes + +CREATE INDEX IF NOT EXISTS "votes_userID" + ON public.votes USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- shadowBannedUsers + +CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index" + ON public."shadowBannedUsers" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- categoryVotes + +CREATE INDEX IF NOT EXISTS "categoryVotes_UUID" + ON public."categoryVotes" USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql new file mode 100644 index 0000000..33f1913 --- /dev/null +++ b/databases/_sponsorTimes_indexes.sql @@ -0,0 +1,66 @@ +-- sponsorTimes + +CREATE INDEX IF NOT EXISTS "sponsorTiems_timeSubmitted" + ON public."sponsorTimes" USING btree + ("timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTime_userID" + ON public."sponsorTimes" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID" + ON public."sponsorTimes" USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID" + ON public."sponsorTimes" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "startTime" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID" + ON public."sponsorTimes" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +-- userNames + +CREATE INDEX IF NOT EXISTS "userNames_userID" + ON public."userNames" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- vipUsers + +CREATE INDEX IF NOT EXISTS "vipUsers_index" + ON public."vipUsers" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- warnings + +CREATE INDEX IF NOT EXISTS warnings_index + ON public.warnings USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "warnings_issueTime" + ON public.warnings USING btree + ("issueTime" ASC NULLS LAST) + TABLESPACE pg_default; + +-- noSegments + +CREATE INDEX IF NOT EXISTS "noSegments_videoID" + ON public."noSegments" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- categoryVotes + +CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public" + ON public."categoryVotes" USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; \ No newline at end of file diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index bae2da6..1f87e8f 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -23,6 +23,8 @@ export class Postgres implements IDatabase { // Upgrade database if required await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder); + + await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder); } } @@ -118,6 +120,15 @@ export class Postgres implements IDatabase { Logger.debug('db update: no file ' + path); } + private async applyIndexes(fileNamePrefix: string, schemaFolder: string) { + const path = schemaFolder + "/_" + fileNamePrefix + "_indexes.sql"; + if (fs.existsSync(path)) { + await this.pool.query(fs.readFileSync(path).toString()); + } else { + Logger.debug('failed to apply indexes to ' + fileNamePrefix); + } + } + private processUpgradeQuery(query: string): string { let result = query; result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')"); From 27c2562a7f7319c122bbbf68e6196f361635a037 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 23 Mar 2021 23:46:46 -0400 Subject: [PATCH 3/7] Add more indexes --- databases/_private_indexes.sql | 10 +++++----- databases/_sponsorTimes_indexes.sql | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql index d7fb746..2b4781e 100644 --- a/databases/_private_indexes.sql +++ b/databases/_private_indexes.sql @@ -1,15 +1,15 @@ -- sponsorTimes -CREATE INDEX IF NOT EXISTS "idx_16928_sponsorTimes_hashedIP" - ON public."sponsorTimes" USING btree - ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) - TABLESPACE pg_default; - CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP" ON public."sponsorTimes" USING btree ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; +CREATE INDEX "privateDB_sponsorTimes_videoID" + ON public."sponsorTimes" USING btree + ("videoID" ASC NULLS LAST) +; + -- votes CREATE INDEX IF NOT EXISTS "votes_userID" diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 33f1913..6d4d91d 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -14,10 +14,10 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID" ON public."sponsorTimes" USING btree ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; - -CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID" - ON public."sponsorTimes" USING btree - ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "startTime" ASC NULLS LAST) + +CREATE INDEX "sponsorTimes_hashedVideoID_gin" + ON public."sponsorTimes" USING gin + ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops) TABLESPACE pg_default; CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID" From c7eb5fed35fa520ae35d8fdef56a4e76ffa9ed2a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 26 Mar 2021 18:35:25 -0400 Subject: [PATCH 4/7] Fix video duration precision and use submitted one when possible --- databases/_upgrade_sponsorTimes_9.sql | 29 +++++++++++++++++++++++++++ src/routes/postSkipSegments.ts | 6 +++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 databases/_upgrade_sponsorTimes_9.sql diff --git a/databases/_upgrade_sponsorTimes_9.sql b/databases/_upgrade_sponsorTimes_9.sql new file mode 100644 index 0000000..5015a4a --- /dev/null +++ b/databases/_upgrade_sponsorTimes_9.sql @@ -0,0 +1,29 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_9" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" REAL NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 9 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 3b6f82d..2f847bf 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -423,7 +423,11 @@ export async function postSkipSegments(req: Request, res: Response) { if (service == Service.YouTube) { apiVideoInfo = await getYouTubeVideoInfo(videoID); } - videoDuration = getYouTubeVideoDuration(apiVideoInfo) || videoDuration; + const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo); + if (!apiVideoDuration || Math.abs(videoDuration - apiVideoDuration) > 2) { + // If api duration is far off, take that one instead (it is only precise to seconds, not millis) + videoDuration = apiVideoDuration; + } // Auto moderator check if (!isVIP && service == Service.YouTube) { From 46524e4298e3536a534c90173043290cad213993 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 26 Mar 2021 19:02:32 -0400 Subject: [PATCH 5/7] Fix indexes --- databases/_private_indexes.sql | 2 +- databases/_sponsorTimes_indexes.sql | 8 ++++---- src/databases/Postgres.ts | 7 ++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql index 2b4781e..2b70fa7 100644 --- a/databases/_private_indexes.sql +++ b/databases/_private_indexes.sql @@ -5,7 +5,7 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP" ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; -CREATE INDEX "privateDB_sponsorTimes_videoID" +CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID" ON public."sponsorTimes" USING btree ("videoID" ASC NULLS LAST) ; diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 6d4d91d..c90ef51 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -1,6 +1,6 @@ -- sponsorTimes -CREATE INDEX IF NOT EXISTS "sponsorTiems_timeSubmitted" +CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted" ON public."sponsorTimes" USING btree ("timeSubmitted" ASC NULLS LAST) TABLESPACE pg_default; @@ -14,8 +14,8 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID" ON public."sponsorTimes" USING btree ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; - -CREATE INDEX "sponsorTimes_hashedVideoID_gin" + +CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID_gin" ON public."sponsorTimes" USING gin ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops) TABLESPACE pg_default; @@ -41,7 +41,7 @@ CREATE INDEX IF NOT EXISTS "vipUsers_index" -- warnings -CREATE INDEX IF NOT EXISTS warnings_index +CREATE INDEX IF NOT EXISTS "warnings_index" ON public.warnings USING btree ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST) TABLESPACE pg_default; diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 1f87e8f..e7e16f4 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -24,7 +24,12 @@ export class Postgres implements IDatabase { // Upgrade database if required await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder); - await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder); + try { + await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder); + } catch (e) { + Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information."); + Logger.warn(e); + } } } From 37a07ace72d13f18c281b25f29c2c2c3fd27cd86 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 26 Mar 2021 19:03:30 -0400 Subject: [PATCH 6/7] Cache data for getting hash-prefix segments --- src/middleware/redisKeys.ts | 10 ++++++- src/routes/getSkipSegments.ts | 43 ++++++++++++++++++++++++------ src/routes/postSkipSegments.ts | 6 +++-- src/routes/voteOnSponsorTime.ts | 47 ++++++++++++++++++++------------- 4 files changed, 76 insertions(+), 30 deletions(-) diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts index 1335e15..56b1aab 100644 --- a/src/middleware/redisKeys.ts +++ b/src/middleware/redisKeys.ts @@ -1,5 +1,13 @@ -import { Category, VideoID } from "../types/segments.model"; +import { Service, VideoID, VideoIDHash } from "../types/segments.model"; +import { Logger } from "../utils/logger"; export function skipSegmentsKey(videoID: VideoID): string { return "segments-" + videoID; } + +export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { + hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; + if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix); + + return "segments." + service + "." + hashedVideoIDPrefix; +} \ No newline at end of file diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index b6ae72f..c2a9155 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -2,7 +2,7 @@ import { Request, Response } from 'express'; import { RedisClient } from 'redis'; import { config } from '../config'; import { db, privateDB } from '../databases/databases'; -import { skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getHash } from '../utils/getHash'; @@ -92,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category))); if (categories.length === 0) return null; - const segmentPerVideoID: SegmentWithHashPerVideoID = (await db - .prepare( - 'all', - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, - [hashedVideoIDPrefix + '%', service] - )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { + const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service)) + .filter((segment: DBSegment) => categories.includes(segment?.category)) + .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { hash: segment.hashedVideoID, segmentPerCategory: {}, @@ -131,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, } } +async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { + const fetchFromDB = () => db + .prepare( + 'all', + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" + WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`, + [hashedVideoIDPrefix + '%', service] + ); + + if (hashedVideoIDPrefix.length === 4) { + const key = skipSegmentsHashKey(hashedVideoIDPrefix, service); + const {err, reply} = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug("Got data from redis: " + reply); + return JSON.parse(reply); + } catch (e) { + // If all else, continue on to fetching from the database + } + } + + const data = await fetchFromDB(); + + redis.setAsync(key, JSON.stringify(data)); + return data; + } + + return await fetchFromDB(); +} + //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 2f847bf..75b572d 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -11,7 +11,7 @@ import {getFormattedTime} from '../utils/getFormattedTime'; import {isUserTrustworthy} from '../utils/isUserTrustworthy'; import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; -import { skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model'; @@ -497,13 +497,14 @@ export async function postSkipSegments(req: Request, res: Response) { //it's better than generating an actual UUID like what was used before //also better for duplication checking const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1])); + const hashedVideoID = getHash(videoID, 1); const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1), + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID, ], ); @@ -512,6 +513,7 @@ export async function postSkipSegments(req: Request, res: Response) { // Clear redis cache for this video redis.delAsync(skipSegmentsKey(videoID)); + redis.delAsync(skipSegmentsHashKey(hashedVideoID, service)); } catch (err) { //a DB change probably occurred res.sendStatus(500); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 29f0e66..c0aaac0 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash'; import {config} from '../config'; import { UserID } from '../types/user.model'; import redis from '../utils/redis'; -import { skipSegmentsKey } from '../middleware/redisKeys'; -import { VideoID } from '../types/segments.model'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; +import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; const voteTypes = { normal: 0, @@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) { } } -async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string - , hashedIP: string, finalResponse: FinalResponse, res: Response) { +async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category + , hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) { // Check if they've already made a vote const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); @@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS return; } - const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]); - if (!currentCategory) { + const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`, + [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service}; + if (!videoInfo) { // Submission doesn't exist res.status(400).send("Submission doesn't exist."); return; @@ -196,7 +197,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS } // See if the submissions category is ready to change - const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]); + const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]); const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); @@ -208,9 +209,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS // Add submission as vote if (!currentCategoryInfo && submissionInfo) { - await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]); + await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]); - await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]); + await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]); } const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; @@ -222,6 +223,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); } + clearRedisCache(videoInfo); + res.sendStatus(finalResponse.finalStatus); } @@ -230,10 +233,10 @@ export function getUserID(req: Request): UserID { } export async function voteOnSponsorTime(req: Request, res: Response) { - const UUID = req.query.UUID as string; + const UUID = req.query.UUID as SegmentUUID; const paramUserID = getUserID(req); let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; - const category = req.query.category as string; + const category = req.query.category as Category; if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) { //invalid request @@ -255,7 +258,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { const ip = getIP(req); //hash the ip 5000 times so no one can get it from the database - const hashedIP = getHash(ip + config.globalSalt); + const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress); //check if this user is on the vip list const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0; @@ -350,13 +353,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) { } //check if the increment amount should be multiplied (downvotes have more power if there have been many views) - const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as - {videoID: VideoID, votes: number, views: number}; + const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as + {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number}; if (voteTypeEnum === voteTypes.normal) { if ((isVIP || isOwnSubmission) && incrementAmount < 0) { //this user is a vip and a downvote - incrementAmount = -(row.votes + 2 - oldIncrementAmount); + incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount); type = incrementAmount; } } else if (voteTypeEnum == voteTypes.incorrect) { @@ -399,8 +402,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]); } - // Clear redis cache for this video - redis.delAsync(skipSegmentsKey(row?.videoID)); + clearRedisCache(videoInfo); //for each positive vote, see if a hidden submission can be shown again if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { @@ -437,7 +439,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { voteTypeEnum, isVIP, isOwnSubmission, - row, + row: videoInfo, category, incrementAmount, oldIncrementAmount, @@ -449,4 +451,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) { res.status(500).json({error: 'Internal error creating segment vote'}); } -} \ No newline at end of file +} + +function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) { + if (videoInfo) { + redis.delAsync(skipSegmentsKey(videoInfo.videoID)); + redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); + } +} From 5152d7e6495b40472bb21c9ba893fc5ec3eb2c08 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 26 Mar 2021 19:13:52 -0400 Subject: [PATCH 7/7] Fixed tests --- src/routes/postSkipSegments.ts | 4 ++-- test/cases/postSkipSegments.ts | 36 ++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 75b572d..7738653 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -424,9 +424,9 @@ export async function postSkipSegments(req: Request, res: Response) { apiVideoInfo = await getYouTubeVideoInfo(videoID); } const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo); - if (!apiVideoDuration || Math.abs(videoDuration - apiVideoDuration) > 2) { + if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) { // If api duration is far off, take that one instead (it is only precise to seconds, not millis) - videoDuration = apiVideoDuration; + videoDuration = apiVideoDuration || 0 as VideoDuration; } // Auto moderator check diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 9ca8333..399e922 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -97,7 +97,7 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); - it('Should be able to submit a single time with a duration (JSON method)', (done: Done) => { + it('Should be able to submit a single time with a duration from the YouTube API (JSON method)', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { method: 'POST', @@ -129,7 +129,39 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); - it('Should be able to submit a single time with a duration from the API (JSON method)', (done: Done) => { + it('Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXZH", + videoDuration: 5010.20, + segments: [{ + segment: [1, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]); + if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit a single time with a duration in the body (JSON method)', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { method: 'POST',