Merge pull request #534 from ajayyy/clickbait

Clickbait

src/app.ts

@@ -49,6 +49,8 @@ import { getVideoLabelsByHash } from "./routes/getVideoLabelByHash";
 import { addFeature } from "./routes/addFeature";
 import { generateTokenRequest } from "./routes/generateToken";
 import { verifyTokenRequest } from "./routes/verifyToken";
+import { getBranding, getBrandingByHashEndpoint } from "./routes/getBranding";
+import { postBranding } from "./routes/postBranding";
 import { cacheMiddlware } from "./middleware/etag";
 
 export function createServer(callback: () => void): Server {
@@ -208,6 +210,10 @@ function setupRoutes(router: Router) {
     router.get("/api/videoLabels", getVideoLabels);
     router.get("/api/videoLabels/:prefix", getVideoLabelsByHash);
 
+    router.get("/api/branding", getBranding);
+    router.get("/api/branding/:prefix", getBrandingByHashEndpoint);
+    router.post("/api/branding", postBranding);
+
     /* istanbul ignore next */
     if (config.postgres?.enabled) {
         router.get("/database", (req, res) => dumpDatabase(req, res, true));

src/routes/getBranding.ts

@@ -0,0 +1,209 @@
+import { Request, Response } from "express";
+import { isEmpty } from "lodash";
+import { config } from "../config";
+import { db, privateDB } from "../databases/databases";
+import { BrandingDBSubmission, BrandingHashDBResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
+import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
+import { shuffleArray } from "../utils/array";
+import { getHashCache } from "../utils/getHashCache";
+import { getIP } from "../utils/getIP";
+import { getService } from "../utils/getService";
+import { hashPrefixTester } from "../utils/hashPrefixTester";
+import { Logger } from "../utils/logger";
+import { promiseOrTimeout } from "../utils/promise";
+import { QueryCacher } from "../utils/queryCacher";
+import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys";
+
+enum BrandingSubmissionType {
+    Title = "title",
+    Thumbnail = "thumbnail"
+}
+
+export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise<BrandingResult> {
+    const getTitles = () => db.prepare(
+        "all",
+        `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID"
+        FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
+        WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
+        [videoID, service],
+        { useReplica: true }
+    ) as Promise<TitleDBResult[]>;
+
+    const getThumbnails = () => db.prepare(
+        "all",
+        `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
+        FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+        WHERE "thumbnails"."videoID" = ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
+        [videoID, service],
+        { useReplica: true }
+    ) as Promise<ThumbnailDBResult[]>;
+
+    const getBranding = async () => ({
+        titles: await getTitles(),
+        thumbnails: await getThumbnails()
+    });
+
+    const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service));
+
+    const cache = {
+        currentIP: null as Promise<HashedIP> | null
+    };
+
+    return filterAndSortBranding(branding.titles, branding.thumbnails, ip, cache);
+}
+
+export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingResult>> {
+    const getTitles = () => db.prepare(
+        "all",
+        `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID"
+        FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
+        WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
+        [`${videoHashPrefix}%`, service],
+        { useReplica: true }
+    ) as Promise<TitleDBResult[]>;
+
+    const getThumbnails = () => db.prepare(
+        "all",
+        `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
+        FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+        WHERE "thumbnails"."hashedVideoID" LIKE ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
+        [`${videoHashPrefix}%`, service],
+        { useReplica: true }
+    ) as Promise<ThumbnailDBResult[]>;
+
+    const branding = await QueryCacher.get(async () => {
+        // Make sure they are both called in parallel
+        const branding = {
+            titles: getTitles(),
+            thumbnails: getThumbnails()
+        };
+
+        const dbResult: Record<VideoID, BrandingHashDBResult> = {};
+        const initResult = (submission: BrandingDBSubmission) => {
+            dbResult[submission.videoID] = dbResult[submission.videoID] || {
+                titles: [],
+                thumbnails: []
+            };
+        };
+
+        (await branding.titles).map((title) => {
+            initResult(title);
+            dbResult[title.videoID].titles.push(title);
+        });
+        (await branding.thumbnails).map((thumbnail) => {
+            initResult(thumbnail);
+            dbResult[thumbnail.videoID].thumbnails.push(thumbnail);
+        });
+
+        return dbResult;
+    }, brandingHashKey(videoHashPrefix, service));
+
+
+    const cache = {
+        currentIP: null as Promise<HashedIP> | null
+    };
+
+    const processedResult: Record<VideoID, BrandingResult> = {};
+    await Promise.all(Object.keys(branding).map(async (key) => {
+        const castedKey = key as VideoID;
+        processedResult[castedKey] = await filterAndSortBranding(branding[castedKey].titles, branding[castedKey].thumbnails, ip, cache);
+    }));
+
+    return processedResult;
+}
+
+async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise<HashedIP> | null }): Promise<BrandingResult> {
+    const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache);
+    const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache);
+
+    const titles = shuffleArray(dbTitles.filter(await shouldKeepTitles))
+        .sort((a, b) => b.votes - a.votes)
+        .sort((a, b) => b.locked - a.locked)
+        .map((r) => ({
+            title: r.title,
+            original: r.original === 1,
+            votes: r.votes,
+            locked: r.locked === 1,
+            UUID: r.UUID,
+        })) as TitleResult[];
+
+    const thumbnails = shuffleArray(dbThumbnails.filter(await shouldKeepThumbnails))
+        .sort((a, b) => b.votes - a.votes)
+        .sort((a, b) => b.locked - a.locked)
+        .map((r) => ({
+            timestamp: r.timestamp,
+            original: r.original === 1,
+            votes: r.votes,
+            locked: r.locked === 1,
+            UUID: r.UUID
+        })) as ThumbnailResult[];
+
+    return {
+        titles,
+        thumbnails
+    };
+}
+
+async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: BrandingSubmissionType, ip: IPAddress,
+    cache: { currentIP: Promise<HashedIP> | null }): Promise<(_: unknown, index: number) => boolean> {
+
+    const shouldKeep = await Promise.all(submissions.map(async (s) => {
+        if (s.shadowHidden != Visibility.HIDDEN) return true;
+        const table = type === BrandingSubmissionType.Title ? "titleVotes" : "thumbnailVotes";
+        const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`,
+            [s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>;
+        try {
+            const submitterIP = await promiseOrTimeout(QueryCacher.get(fetchData, brandingIPKey(s.UUID)), 150);
+            if (cache.currentIP === null) cache.currentIP = getHashCache((ip + config.globalSalt) as IPAddress);
+            const hashedIP = await cache.currentIP;
+
+            return submitterIP.hashedIP !== hashedIP;
+        } catch (e) {
+            // give up on shadow hide for now
+            return false;
+        }
+    }));
+
+    return (_, index) => shouldKeep[index];
+}
+
+export async function getBranding(req: Request, res: Response) {
+    const videoID: VideoID = req.query.videoID as VideoID;
+    const service: Service = getService(req.query.service as string);
+
+    if (!videoID) {
+        return res.status(400).send("Missing parameter: videoID");
+    }
+
+    const ip = getIP(req);
+    try {
+        const result = await getVideoBranding(videoID, service, ip);
+
+        const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404;
+        return res.status(status).json(result);
+    } catch (e) {
+        Logger.error(e as string);
+        return res.status(500).send("Internal server error");
+    }
+}
+
+export async function getBrandingByHashEndpoint(req: Request, res: Response) {
+    let hashPrefix = req.params.prefix as VideoIDHash;
+    if (!hashPrefix || !hashPrefixTester(hashPrefix) || hashPrefix.length !== 4) {
+        return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+    }
+    hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
+
+    const service: Service = getService(req.query.service as string);
+    const ip = getIP(req);
+
+    try {
+        const result = await getVideoBrandingByHash(hashPrefix, service, ip);
+
+        const status = !isEmpty(result) ? 200 : 404;
+        return res.status(status).json(result);
+    } catch (e) {
+        Logger.error(e as string);
+        return res.status(500).send([]);
+    }
+}

src/routes/postBranding.ts

@@ -0,0 +1,146 @@
+import { Request, Response } from "express";
+import { config } from "../config";
+import { db, privateDB } from "../databases/databases";
+
+import { BrandingSubmission, BrandingUUID, TimeThumbnailSubmission } from "../types/branding.model";
+import { HashedIP, IPAddress, VideoID } from "../types/segments.model";
+import { HashedUserID } from "../types/user.model";
+import { getHashCache } from "../utils/getHashCache";
+import { getIP } from "../utils/getIP";
+import { getService } from "../utils/getService";
+import { isUserVIP } from "../utils/isUserVIP";
+import { Logger } from "../utils/logger";
+import crypto from "crypto";
+
+enum BrandingType {
+    Title,
+    Thumbnail
+}
+
+interface ExistingVote {
+    UUID: BrandingUUID;
+    type: number;
+    id: number;
+}
+
+export async function postBranding(req: Request, res: Response) {
+    const { videoID, userID, title, thumbnail } = req.body as BrandingSubmission;
+    const service = getService(req.body.service);
+
+    if (!videoID || !userID || userID.length < 30 || !service
+        || ((!title || !title.title)
+            && (!thumbnail || thumbnail.original == null
+                || (!thumbnail.original && !(thumbnail as TimeThumbnailSubmission).timestamp)))) {
+        res.status(400).send("Bad Request");
+        return;
+    }
+
+    try {
+        const hashedUserID = await getHashCache(userID);
+        const isVip = await isUserVIP(hashedUserID);
+        const hashedVideoID = await getHashCache(videoID, 1);
+        const hashedIP = await getHashCache(getIP(req) + config.globalSalt as IPAddress);
+
+        const now = Date.now();
+        const voteType = 1;
+
+        await Promise.all([(async () => {
+            if (title) {
+                const existingUUID = (await db.prepare("get", `SELECT "UUID" from "titles" where "videoID" = ? AND "title" = ?`, [videoID, title.title]))?.UUID;
+                const UUID = existingUUID || crypto.randomUUID();
+
+                const existingVote = await handleExistingVotes(BrandingType.Title, videoID, hashedUserID, UUID, hashedIP, voteType);
+                if (existingUUID) {
+                    await updateVoteTotals(BrandingType.Title, existingVote, UUID, isVip);
+                } else {
+                    await db.prepare("run", `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+                        [videoID, title.title, title.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID]);
+
+                    await db.prepare("run", `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0);`,
+                        [UUID, isVip ? 1 : 0]);
+                }
+
+                if (isVip) {
+                    // unlock all other titles
+                    await db.prepare("run", `UPDATE "titleVotes" SET "locked" = 0 WHERE "UUID" != ?`, [UUID]);
+                }
+            }
+        })(), (async () => {
+            if (thumbnail) {
+                const existingUUID = thumbnail.original
+                    ? (await db.prepare("get", `SELECT "UUID" from "thumbnails" where "videoID" = ? AND "original" = 1`, [videoID]))?.UUID
+                    : (await db.prepare("get", `SELECT "thumbnails"."UUID" from "thumbnailTimestamps" JOIN "thumbnails" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+                        WHERE "thumbnailTimestamps"."timestamp" = ? AND "thumbnails"."videoID" = ?`, [(thumbnail as TimeThumbnailSubmission).timestamp, videoID]))?.UUID;
+                const UUID = existingUUID || crypto.randomUUID();
+
+                const existingVote = await handleExistingVotes(BrandingType.Thumbnail, videoID, hashedUserID, UUID, hashedIP, voteType);
+                if (existingUUID) {
+                    await updateVoteTotals(BrandingType.Thumbnail, existingVote, UUID, isVip);
+                } else {
+                    await db.prepare("run", `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`,
+                        [videoID, thumbnail.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID]);
+
+                    await db.prepare("run", `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0)`,
+                        [UUID, isVip ? 1 : 0]);
+
+                    if (!thumbnail.original) {
+                        await db.prepare("run", `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`,
+                            [UUID, (thumbnail as TimeThumbnailSubmission).timestamp]);
+                    }
+
+                    if (isVip) {
+                        // unlock all other titles
+                        await db.prepare("run", `UPDATE "thumbnailVotes" SET "locked" = 0 WHERE "UUID" != ?`, [UUID]);
+                    }
+                }
+            }
+        })()]);
+
+        res.status(200).send("OK");
+    } catch (e) {
+        Logger.error(e as string);
+        res.status(500).send("Internal Server Error");
+    }
+}
+
+/**
+ * Finds an existing vote, if found, and it's for a different submission, it undoes it, and points to the new submission.
+ * If no existing vote, it adds one.
+ */
+async function handleExistingVotes(type: BrandingType, videoID: VideoID,
+    hashedUserID: HashedUserID, UUID: BrandingUUID, hashedIP: HashedIP, voteType: number): Promise<ExistingVote> {
+    const table = type === BrandingType.Title ? `"titleVotes"` : `"thumbnailVotes"`;
+
+    const existingVote = await privateDB.prepare("get", `SELECT "id", "UUID", "type" from ${table} where "videoID" = ? AND "userID" = ?`, [videoID, hashedUserID]);
+    if (existingVote && existingVote.UUID !== UUID) {
+        if (existingVote.type === 1) {
+            await db.prepare("run", `UPDATE ${table} SET "votes" = "votes" - 1 WHERE "UUID" = ?`, [existingVote.UUID]);
+        }
+
+        await privateDB.prepare("run", `UPDATE ${table} SET "type" = ?, "UUID" = ? WHERE "id" = ?`, [voteType, UUID, existingVote.id]);
+    } else if (!existingVote) {
+        await privateDB.prepare("run", `INSERT INTO ${table} ("videoID", "UUID", "userID", "hashedIP", "type") VALUES (?, ?, ?, ?, ?)`,
+            [videoID, UUID, hashedUserID, hashedIP, voteType]);
+    }
+
+    return existingVote;
+}
+
+/**
+ * Only called if an existing vote exists.
+ * Will update public vote totals and locked status.
+ */
+async function updateVoteTotals(type: BrandingType, existingVote: ExistingVote, UUID: BrandingUUID, isVip: boolean): Promise<void> {
+    if (!existingVote) return;
+
+    const table = type === BrandingType.Title ? `"titleVotes"` : `"thumbnailVotes"`;
+
+    // Don't upvote if we vote on the same submission
+    if (!existingVote || existingVote.UUID !== UUID) {
+        await db.prepare("run", `UPDATE ${table} SET "votes" = "votes" + 1 WHERE "UUID" = ?`, [UUID]);
+    }
+
+    if (isVip) {
+        await db.prepare("run", `UPDATE ${table} SET "locked" = 1 WHERE "UUID" = ?`, [UUID]);
+    }
+}

src/types/branding.model.ts

@@ -0,0 +1,75 @@
+import { Service, VideoID, VideoIDHash } from "./segments.model";
+import { UserID } from "./user.model";
+
+export type BrandingUUID = string & { readonly __brandingUUID: unique symbol };
+
+export interface BrandingDBSubmission {
+    shadowHidden: number,
+    UUID: BrandingUUID,
+    videoID: VideoID,
+    hashedVideoID: VideoIDHash
+}
+
+export interface TitleDBResult extends BrandingDBSubmission {
+    title: string,
+    original: number,
+    votes: number,
+    locked: number
+}
+
+export interface TitleResult {
+    title: string,
+    original: boolean,
+    votes: number,
+    locked: boolean,
+    UUID: BrandingUUID
+}
+
+export interface ThumbnailDBResult extends BrandingDBSubmission {
+    timestamp?: number,
+    original: number,
+    votes: number,
+    locked: number
+}
+
+export interface ThumbnailResult {
+    timestamp?: number,
+    original: boolean,
+    votes: number,
+    locked: boolean,
+    UUID: BrandingUUID
+}
+
+export interface BrandingResult {
+    titles: TitleResult[],
+    thumbnails: ThumbnailResult[]
+}
+
+export interface BrandingHashDBResult {
+    titles: TitleDBResult[],
+    thumbnails: ThumbnailDBResult[]
+}
+
+export interface OriginalThumbnailSubmission {
+    original: true;
+}
+
+export interface TimeThumbnailSubmission {
+    timestamp: number;
+    original: false;
+}
+
+export type ThumbnailSubmission = OriginalThumbnailSubmission | TimeThumbnailSubmission;
+
+export interface TitleSubmission {
+    title: string;
+    original: boolean;
+}
+
+export interface BrandingSubmission {
+    title: TitleSubmission;
+    thumbnail: ThumbnailSubmission;
+    videoID: VideoID;
+    userID: UserID;
+    service: Service;
+}

src/utils/array.ts

@@ -0,0 +1,8 @@
+export function shuffleArray<T>(array: T[]): T[] {
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+
+    return array;
+}

src/utils/redisKeys.ts

@@ -2,6 +2,7 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
 import { Feature, HashedUserID, UserID } from "../types/user.model";
 import { HashedValue } from "../types/hash.model";
 import { Logger } from "./logger";
+import { BrandingUUID } from "../types/branding.model";
 
 export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
     `segments.v4.${service}.videoID.${videoID}`;
@@ -16,6 +17,20 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
     return `segments.v4.${service}.${hashedVideoIDPrefix}`;
 }
 
+export const brandingKey = (videoID: VideoID, service: Service): string =>
+    `branding.${service}.videoID.${videoID}`;
+
+export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
+    hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
+    if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
+
+    return `branding.${service}.${hashedVideoIDPrefix}`;
+}
+
+export const brandingIPKey = (uuid: BrandingUUID): string =>
+    `branding.shadow.${uuid}`;
+
+
 export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
     `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;