diff --git a/databases/_private.db.sql b/databases/_private.db.sql index 0da7016..eb73bb2 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -26,4 +26,30 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS "titles" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "hashedIP" TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS "titleVotes" ( + "id" SERIAL PRIMARY KEY, + "UUID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "hashedIP" TEXT NOT NULL, + "type" INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS "thumbnails" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "hashedIP" TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( + "id" SERIAL PRIMARY KEY, + "UUID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "hashedIP" TEXT NOT NULL, + "type" INTEGER NOT NULL +); + COMMIT; diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 1ab5210..a706283 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -37,6 +37,49 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS "titles" ( + "videoID" TEXT NOT NULL, + "title" TEXT NOT NULL, + "original" BOOLEAN NOT NULL, + "userID" TEXT NOT NULL, + "service" TEXT NOT NULL, + "hashedVideoID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "UUID" TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS "titleVotes" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "votes" INTEGER NOT NULL default 0, + "locked" INTEGER NOT NULL default 0, + "shadowHidden" INTEGER NOT NULL default 0, + FOREIGN KEY("UUID") REFERENCES "titles"("UUID") +); + +CREATE TABLE IF NOT EXISTS "thumbnails" ( + "videoID" TEXT NOT NULL, + "original" INTEGER default 0, + "userID" TEXT NOT NULL, + "service" TEXT NOT NULL, + "hashedVideoID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "UUID" TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "timestamp" INTEGER NOT NULL default 0 + FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") +); + +CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "votes" INTEGER NOT NULL default 0, + "locked" INTEGER NOT NULL default 0, + "shadowHidden" INTEGER NOT NULL default 0, + FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") +); + CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 4f5c52c..afcd16f 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -115,4 +115,38 @@ CREATE INDEX IF NOT EXISTS "ratings_videoID" CREATE INDEX IF NOT EXISTS "userFeatures_userID" ON public."userFeatures" USING btree ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "feature" ASC NULLS LAST) + TABLESPACE pg_default; + +-- titles + +CREATE INDEX IF NOT EXISTS "titles_timeSubmitted" + ON public."titles" USING btree + ("timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "titles_videoID" + ON public."titles" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "titles_hashedVideoID" + ON public."titles" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- thumbnails + +CREATE INDEX IF NOT EXISTS "thumbnails_timeSubmitted" + ON public."thumbnails" USING btree + ("timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "thumbnails_videoID" + ON public."thumbnails" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "thumbnails_hashedVideoID" + ON public."thumbnails" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts new file mode 100644 index 0000000..eae6962 --- /dev/null +++ b/src/routes/getBranding.ts @@ -0,0 +1,209 @@ +import { Request, Response } from "express"; +import { isEmpty } from "lodash"; +import { config } from "../config"; +import { db, privateDB } from "../databases/databases"; +import { BrandingDBSubmission, BrandingHashDBResult, BrandingHashResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model"; +import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model"; +import { shuffleArray } from "../utils/array"; +import { getHashCache } from "../utils/getHashCache"; +import { getIP } from "../utils/getIP"; +import { getService } from "../utils/getService"; +import { hashPrefixTester } from "../utils/hashPrefixTester"; +import { promiseOrTimeout } from "../utils/promise"; +import { QueryCacher } from "../utils/queryCacher"; +import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys"; + +enum BrandingSubmissionType { + Title = "title", + Thumbnail = "thumbnail" +} + +export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise { + const getTitles = () => db.prepare( + "all", + `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID + FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID" + WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`, + [videoID, service], + { useReplica: true } + ) as Promise; + + const getThumbnails = () => db.prepare( + "all", + `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID" + FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID" + WHERE "thumbnails"."videoID" = ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`, + [videoID, service], + { useReplica: true } + ) as Promise; + + // eslint-disable-next-line require-await + const getBranding = async () => ({ + titles: getTitles(), + thumbnails: getThumbnails() + }); + + const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service)); + + const cache = { + currentIP: null as Promise | null + }; + + return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache); +} + +export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise> { + const getTitles = () => db.prepare( + "all", + `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID + FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID" + WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`, + [`${videoHashPrefix}%`, service], + { useReplica: true } + ) as Promise; + + const getThumbnails = () => db.prepare( + "all", + `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID" + FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID" + WHERE "thumbnails"."hashedVideoID" LIKE ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`, + [`${videoHashPrefix}%`, service], + { useReplica: true } + ) as Promise; + + const branding = await QueryCacher.get(async () => { + // Make sure they are both called in parallel + const branding = { + titles: getTitles(), + thumbnails: getThumbnails() + }; + + const dbResult: Record = {}; + const initResult = (submission: BrandingDBSubmission) => { + dbResult[submission.videoID] = dbResult[submission.videoID] || { + hash: submission.hashedVideoID, + branding: { + titles: [], + thumbnails: [] + } + }; + }; + + (await branding.titles).map((title) => { + initResult(title); + dbResult[title.videoID].branding.titles.push(title); + }); + (await branding.thumbnails).map((thumbnail) => { + initResult(thumbnail); + dbResult[thumbnail.videoID].branding.thumbnails.push(thumbnail); + }); + + return dbResult; + }, brandingHashKey(videoHashPrefix, service)); + + + const cache = { + currentIP: null as Promise | null + }; + + const processedResult: Record = {}; + await Promise.all(Object.keys(branding).map(async (key) => { + const castedKey = key as VideoID; + processedResult[castedKey] = { + hash: branding[castedKey].hash, + branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache) + }; + })); + + return processedResult; +} + +async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise | null}): Promise { + const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache); + const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache); + + const titles = shuffleArray(dbTitles.filter(await shouldKeepTitles)) + .sort((a, b) => b.votes - a.votes) + .sort((a, b) => b.locked - a.locked) + .map((r) => ({ + title: r.title, + original: r.original === 1, + votes: r.votes, + locked: r.locked === 1, + UUID: r.UUID, + })) as TitleResult[]; + + const thumbnails = shuffleArray(dbThumbnails.filter(await shouldKeepThumbnails)) + .sort((a, b) => b.votes - a.votes) + .sort((a, b) => b.locked - a.locked) + .map((r) => ({ + timestamp: r.timestamp, + original: r.original === 1, + votes: r.votes, + locked: r.locked === 1, + UUID: r.UUID + })) as ThumbnailResult[]; + + return { + titles, + thumbnails + }; +} + +async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: BrandingSubmissionType, ip: IPAddress, + cache: { currentIP: Promise | null }): Promise<(_: unknown, index: number) => boolean> { + + const shouldKeep = await Promise.all(submissions.map(async (s) => { + if (s.shadowHidden != Visibility.HIDDEN) return true; + const table = type === BrandingSubmissionType.Title ? "titles" : "thumbnails"; + const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`, + [s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>; + try { + const submitterIP = await promiseOrTimeout(QueryCacher.get(fetchData, brandingIPKey(s.UUID)), 150); + if (cache.currentIP === null) cache.currentIP = getHashCache((ip + config.globalSalt) as IPAddress); + const hashedIP = await cache.currentIP; + + return submitterIP.hashedIP !== hashedIP; + } catch (e) { + // give up on shadow hide for now + return true; + } + })); + + return (_, index) => shouldKeep[index]; +} + +export async function getBranding(req: Request, res: Response) { + const videoID: VideoID = req.query.videoID as VideoID; + const service: Service = getService(req.query.service, req.body.service); + + if (!videoID) { + return res.status(400).send("Missing parameter: videoID"); + } + + const ip = getIP(req); + const result = await getVideoBranding(videoID, service, ip); + + const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404; + return res.status(status).json(result); +} + +export async function getBrandingByHash(req: Request, res: Response) { + let hashPrefix = req.params.prefix as VideoIDHash; + if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) { + return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix + } + hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; + + const service: Service = getService(req.query.service, req.body.service); + + if (!hashPrefix || hashPrefix.length !== 4) { + return res.status(400).send("Hash prefix does not match format requirements."); + } + + const ip = getIP(req); + const result = await getVideoBrandingByHash(hashPrefix, service, ip); + + const status = !isEmpty(result) ? 200 : 404; + return res.status(status).json(result); +} \ No newline at end of file diff --git a/src/types/branding.model.ts b/src/types/branding.model.ts new file mode 100644 index 0000000..69bf35d --- /dev/null +++ b/src/types/branding.model.ts @@ -0,0 +1,58 @@ +import { VideoID, VideoIDHash } from "./segments.model"; + +export type BrandingUUID = string & { readonly __brandingUUID: unique symbol }; + +export interface BrandingDBSubmission { + shadowHidden: number, + UUID: BrandingUUID, + videoID: VideoID, + hashedVideoID: VideoIDHash +} + +export interface TitleDBResult extends BrandingDBSubmission { + title: string, + original: number, + votes: number, + locked: number +} + +export interface TitleResult { + title: string, + original: boolean, + votes: number, + locked: boolean, + UUID: BrandingUUID +} + +export interface ThumbnailDBResult extends BrandingDBSubmission { + timestamp?: number, + original: number, + votes: number, + locked: number +} + +export interface ThumbnailResult { + timestamp?: number, + original: boolean, + votes: number, + locked: boolean, + UUID: BrandingUUID +} + +export interface BrandingResult { + titles: TitleResult[], + thumbnails: ThumbnailResult[] +} + +export interface BrandingHashDBResult { + hash: VideoIDHash; + branding: { + titles: TitleDBResult[], + thumbnails: ThumbnailDBResult[] + }; +} + +export interface BrandingHashResult { + hash: VideoIDHash; + branding: BrandingResult; +} \ No newline at end of file diff --git a/src/utils/array.ts b/src/utils/array.ts new file mode 100644 index 0000000..d7cfef6 --- /dev/null +++ b/src/utils/array.ts @@ -0,0 +1,8 @@ +export function shuffleArray(array: T[]): T[] { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } + + return array; +} \ No newline at end of file diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index e482e10..59ecf76 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -2,6 +2,7 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { Feature, HashedUserID, UserID } from "../types/user.model"; import { HashedValue } from "../types/hash.model"; import { Logger } from "./logger"; +import { BrandingUUID } from "../types/branding.model"; export const skipSegmentsKey = (videoID: VideoID, service: Service): string => `segments.v4.${service}.videoID.${videoID}`; @@ -16,6 +17,20 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S return `segments.v4.${service}.${hashedVideoIDPrefix}`; } +export const brandingKey = (videoID: VideoID, service: Service): string => + `branding.${service}.videoID.${videoID}`; + +export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { + hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; + if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); + + return `branding.${service}.${hashedVideoIDPrefix}`; +} + +export const brandingIPKey = (uuid: BrandingUUID): string => + `branding.shadow.${uuid}`; + + export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;