Initial get branding

2025-12-10 13:37:01 +03:00 · 2022-12-23 16:56:27 -05:00
parent cff2325aef
commit cc24a4902f
7 changed files with 393 additions and 0 deletions
--- a/databases/_private.db.sql
+++ b/databases/_private.db.sql
@@ -26,4 +26,30 @@ CREATE TABLE IF NOT EXISTS "config" (
 	"value" TEXT NOT NULL
 );

+CREATE TABLE IF NOT EXISTS "titles" (
+	"UUID"	TEXT NOT NULL PRIMARY KEY,
+	"hashedIP"	TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "titleVotes" (
+	"id" SERIAL PRIMARY KEY,
+	"UUID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"type"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "thumbnails" (
+	"UUID"	TEXT NOT NULL PRIMARY KEY,
+	"hashedIP"	TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "thumbnailVotes" (
+	"id" SERIAL PRIMARY KEY,
+	"UUID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"type"	INTEGER NOT NULL
+);
+
 COMMIT;
--- a/databases/_sponsorTimes.db.sql
+++ b/databases/_sponsorTimes.db.sql
@@ -37,6 +37,49 @@ CREATE TABLE IF NOT EXISTS "config" (
 	"value" TEXT NOT NULL
 );

+CREATE TABLE IF NOT EXISTS "titles" (
+	"videoID"	TEXT NOT NULL,
+	"title"	TEXT NOT NULL,
+	"original" BOOLEAN NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"service"	TEXT NOT NULL,
+	"hashedVideoID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"UUID"	TEXT NOT NULL PRIMARY KEY
+);
+
+CREATE TABLE IF NOT EXISTS "titleVotes" (
+	"UUID"	TEXT NOT NULL PRIMARY KEY,
+	"votes"	INTEGER NOT NULL default 0,
+	"locked"	INTEGER NOT NULL default 0,
+	"shadowHidden"	INTEGER NOT NULL default 0,
+	FOREIGN KEY("UUID") REFERENCES "titles"("UUID")
+);
+
+CREATE TABLE IF NOT EXISTS "thumbnails" (
+	"videoID"	TEXT NOT NULL,
+	"original" INTEGER default 0,
+	"userID"	TEXT NOT NULL,
+	"service"	TEXT NOT NULL,
+	"hashedVideoID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"UUID"	TEXT NOT NULL PRIMARY KEY
+);
+
+CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" (
+	"UUID"	TEXT NOT NULL PRIMARY KEY,
+	"timestamp"	INTEGER NOT NULL default 0
+	FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID")
+);
+
+CREATE TABLE IF NOT EXISTS "thumbnailVotes" (
+	"UUID"	TEXT NOT NULL PRIMARY KEY,
+	"votes"	INTEGER NOT NULL default 0,
+	"locked"	INTEGER NOT NULL default 0,
+		"shadowHidden"	INTEGER NOT NULL default 0,
+	FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID")
+);
+
 CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
 CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore

--- a/databases/_sponsorTimes_indexes.sql
+++ b/databases/_sponsorTimes_indexes.sql
@@ -115,4 +115,38 @@ CREATE INDEX IF NOT EXISTS "ratings_videoID"
 CREATE INDEX IF NOT EXISTS "userFeatures_userID"
    ON public."userFeatures" USING btree
    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "feature" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- titles
+
+CREATE INDEX IF NOT EXISTS "titles_timeSubmitted"
+    ON public."titles" USING btree
+    ("timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "titles_videoID"
+    ON public."titles" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "titles_hashedVideoID"
+    ON public."titles" USING btree
+    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- thumbnails
+
+CREATE INDEX IF NOT EXISTS "thumbnails_timeSubmitted"
+    ON public."thumbnails" USING btree
+    ("timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "thumbnails_videoID"
+    ON public."thumbnails" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "thumbnails_hashedVideoID"
+    ON public."thumbnails" USING btree
+    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
    TABLESPACE pg_default;
--- a/src/routes/getBranding.ts
+++ b/src/routes/getBranding.ts
@@ -0,0 +1,209 @@
+import { Request, Response } from "express";
+import { isEmpty } from "lodash";
+import { config } from "../config";
+import { db, privateDB } from "../databases/databases";
+import { BrandingDBSubmission, BrandingHashDBResult, BrandingHashResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
+import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
+import { shuffleArray } from "../utils/array";
+import { getHashCache } from "../utils/getHashCache";
+import { getIP } from "../utils/getIP";
+import { getService } from "../utils/getService";
+import { hashPrefixTester } from "../utils/hashPrefixTester";
+import { promiseOrTimeout } from "../utils/promise";
+import { QueryCacher } from "../utils/queryCacher";
+import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys";
+
+enum BrandingSubmissionType {
+    Title = "title",
+    Thumbnail = "thumbnail"
+}
+
+export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise<BrandingResult> {
+    const getTitles = () => db.prepare(
+        "all",
+        `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID
+        FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
+        WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
+        [videoID, service],
+        { useReplica: true }
+    ) as Promise<TitleDBResult[]>;
+
+    const getThumbnails = () => db.prepare(
+        "all",
+        `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
+        FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+        WHERE "thumbnails"."videoID" = ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
+        [videoID, service],
+        { useReplica: true }
+    ) as Promise<ThumbnailDBResult[]>;
+
+    // eslint-disable-next-line require-await
+    const getBranding = async () => ({
+        titles: getTitles(),
+        thumbnails: getThumbnails()
+    });
+
+    const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service));
+
+    const cache = {
+        currentIP: null as Promise<HashedIP> | null
+    };
+
+    return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache);
+}
+
+export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingHashResult>> {
+    const getTitles = () => db.prepare(
+        "all",
+        `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID
+        FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
+        WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
+        [`${videoHashPrefix}%`, service],
+        { useReplica: true }
+    ) as Promise<TitleDBResult[]>;
+
+    const getThumbnails = () => db.prepare(
+        "all",
+        `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
+        FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+        WHERE "thumbnails"."hashedVideoID" LIKE ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
+        [`${videoHashPrefix}%`, service],
+        { useReplica: true }
+    ) as Promise<ThumbnailDBResult[]>;
+
+    const branding = await QueryCacher.get(async () => {
+        // Make sure they are both called in parallel
+        const branding = {
+            titles: getTitles(),
+            thumbnails: getThumbnails()
+        };
+
+        const dbResult: Record<VideoID, BrandingHashDBResult> = {};
+        const initResult = (submission: BrandingDBSubmission) => {
+            dbResult[submission.videoID] = dbResult[submission.videoID] || {
+                hash: submission.hashedVideoID,
+                branding: {
+                    titles: [],
+                    thumbnails: []
+                }
+            };
+        };
+
+        (await branding.titles).map((title) => {
+            initResult(title);
+            dbResult[title.videoID].branding.titles.push(title);
+        });
+        (await branding.thumbnails).map((thumbnail) => {
+            initResult(thumbnail);
+            dbResult[thumbnail.videoID].branding.thumbnails.push(thumbnail);
+        });
+
+        return dbResult;
+    }, brandingHashKey(videoHashPrefix, service));
+
+
+    const cache = {
+        currentIP: null as Promise<HashedIP> | null
+    };
+
+    const processedResult: Record<VideoID, BrandingHashResult> = {};
+    await Promise.all(Object.keys(branding).map(async (key) => {
+        const castedKey = key as VideoID;
+        processedResult[castedKey] = {
+            hash: branding[castedKey].hash,
+            branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache)
+        };
+    }));
+
+    return processedResult;
+}
+
+async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise<HashedIP> | null}): Promise<BrandingResult> {
+    const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache);
+    const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache);
+
+    const titles = shuffleArray(dbTitles.filter(await shouldKeepTitles))
+        .sort((a, b) => b.votes - a.votes)
+        .sort((a, b) => b.locked - a.locked)
+        .map((r) => ({
+            title: r.title,
+            original: r.original === 1,
+            votes: r.votes,
+            locked: r.locked === 1,
+            UUID: r.UUID,
+        })) as TitleResult[];
+
+    const thumbnails = shuffleArray(dbThumbnails.filter(await shouldKeepThumbnails))
+        .sort((a, b) => b.votes - a.votes)
+        .sort((a, b) => b.locked - a.locked)
+        .map((r) => ({
+            timestamp: r.timestamp,
+            original: r.original === 1,
+            votes: r.votes,
+            locked: r.locked === 1,
+            UUID: r.UUID
+        })) as ThumbnailResult[];
+
+    return {
+        titles,
+        thumbnails
+    };
+}
+
+async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: BrandingSubmissionType, ip: IPAddress,
+    cache: { currentIP: Promise<HashedIP> | null }): Promise<(_: unknown, index: number) => boolean> {
+
+    const shouldKeep = await Promise.all(submissions.map(async (s) => {
+        if (s.shadowHidden != Visibility.HIDDEN) return true;
+        const table = type === BrandingSubmissionType.Title ? "titles" : "thumbnails";
+        const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`,
+            [s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>;
+        try {
+            const submitterIP = await promiseOrTimeout(QueryCacher.get(fetchData, brandingIPKey(s.UUID)), 150);
+            if (cache.currentIP === null) cache.currentIP = getHashCache((ip + config.globalSalt) as IPAddress);
+            const hashedIP = await cache.currentIP;
+
+            return submitterIP.hashedIP !== hashedIP;
+        } catch (e) {
+            // give up on shadow hide for now
+            return true;
+        }
+    }));
+
+    return (_, index) => shouldKeep[index];
+}
+
+export async function getBranding(req: Request, res: Response) {
+    const videoID: VideoID = req.query.videoID as VideoID;
+    const service: Service = getService(req.query.service, req.body.service);
+
+    if (!videoID) {
+        return res.status(400).send("Missing parameter: videoID");
+    }
+
+    const ip = getIP(req);
+    const result = await getVideoBranding(videoID, service, ip);
+
+    const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404;
+    return res.status(status).json(result);
+}
+
+export async function getBrandingByHash(req: Request, res: Response) {
+    let hashPrefix = req.params.prefix as VideoIDHash;
+    if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) {
+        return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+    }
+    hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
+
+    const service: Service = getService(req.query.service, req.body.service);
+
+    if (!hashPrefix || hashPrefix.length !== 4) {
+        return res.status(400).send("Hash prefix does not match format requirements.");
+    }
+
+    const ip = getIP(req);
+    const result = await getVideoBrandingByHash(hashPrefix, service, ip);
+
+    const status = !isEmpty(result) ? 200 : 404;
+    return res.status(status).json(result);
+}
--- a/src/types/branding.model.ts
+++ b/src/types/branding.model.ts
@@ -0,0 +1,58 @@
+import { VideoID, VideoIDHash } from "./segments.model";
+
+export type BrandingUUID = string & { readonly __brandingUUID: unique symbol };
+
+export interface BrandingDBSubmission {
+    shadowHidden: number,
+    UUID: BrandingUUID,
+    videoID: VideoID,
+    hashedVideoID: VideoIDHash
+}
+
+export interface TitleDBResult extends BrandingDBSubmission {
+    title: string,
+    original: number,
+    votes: number,
+    locked: number
+}
+
+export interface TitleResult {
+    title: string,
+    original: boolean,
+    votes: number,
+    locked: boolean,
+    UUID: BrandingUUID
+}
+
+export interface ThumbnailDBResult extends BrandingDBSubmission {
+    timestamp?: number,
+    original: number,
+    votes: number,
+    locked: number
+}
+
+export interface ThumbnailResult {
+    timestamp?: number,
+    original: boolean,
+    votes: number,
+    locked: boolean,
+    UUID: BrandingUUID
+}
+
+export interface BrandingResult {
+    titles: TitleResult[],
+    thumbnails: ThumbnailResult[]
+}
+
+export interface BrandingHashDBResult {
+    hash: VideoIDHash;
+    branding: {
+        titles: TitleDBResult[],
+        thumbnails: ThumbnailDBResult[]
+    };
+}
+
+export interface BrandingHashResult {
+    hash: VideoIDHash;
+    branding: BrandingResult;
+}
--- a/src/utils/array.ts
+++ b/src/utils/array.ts
@@ -0,0 +1,8 @@
+export function shuffleArray<T>(array: T[]): T[] {
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+
+    return array;
+}
--- a/src/utils/redisKeys.ts
+++ b/src/utils/redisKeys.ts
@@ -2,6 +2,7 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
 import { Feature, HashedUserID, UserID } from "../types/user.model";
 import { HashedValue } from "../types/hash.model";
 import { Logger } from "./logger";
+import { BrandingUUID } from "../types/branding.model";

 export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
    `segments.v4.${service}.videoID.${videoID}`;
@@ -16,6 +17,20 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
    return `segments.v4.${service}.${hashedVideoIDPrefix}`;
 }

+export const brandingKey = (videoID: VideoID, service: Service): string =>
+    `branding.${service}.videoID.${videoID}`;
+
+export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
+    hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
+    if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
+
+    return `branding.${service}.${hashedVideoIDPrefix}`;
+}
+
+export const brandingIPKey = (uuid: BrandingUUID): string =>
+    `branding.shadow.${uuid}`;
+
+
 export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
    `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;