Merge pull request #534 from ajayyy/clickbait

Clickbait
2025-12-26 17:38:28 +03:00 · 2023-03-27 00:54:52 -04:00
parent dde12bcc43 19ebca86c9
commit bffc10e38f
12 changed files with 1218 additions and 10 deletions
--- a/src/routes/getBranding.ts
+++ b/src/routes/getBranding.ts
@@ -0,0 +1,209 @@
+import { Request, Response } from "express";
+import { isEmpty } from "lodash";
+import { config } from "../config";
+import { db, privateDB } from "../databases/databases";
+import { BrandingDBSubmission, BrandingHashDBResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
+import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
+import { shuffleArray } from "../utils/array";
+import { getHashCache } from "../utils/getHashCache";
+import { getIP } from "../utils/getIP";
+import { getService } from "../utils/getService";
+import { hashPrefixTester } from "../utils/hashPrefixTester";
+import { Logger } from "../utils/logger";
+import { promiseOrTimeout } from "../utils/promise";
+import { QueryCacher } from "../utils/queryCacher";
+import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys";
+
+enum BrandingSubmissionType {
+    Title = "title",
+    Thumbnail = "thumbnail"
+}
+
+export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise<BrandingResult> {
+    const getTitles = () => db.prepare(
+        "all",
+        `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID"
+        FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
+        WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
+        [videoID, service],
+        { useReplica: true }
+    ) as Promise<TitleDBResult[]>;
+
+    const getThumbnails = () => db.prepare(
+        "all",
+        `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
+        FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+        WHERE "thumbnails"."videoID" = ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
+        [videoID, service],
+        { useReplica: true }
+    ) as Promise<ThumbnailDBResult[]>;
+
+    const getBranding = async () => ({
+        titles: await getTitles(),
+        thumbnails: await getThumbnails()
+    });
+
+    const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service));
+
+    const cache = {
+        currentIP: null as Promise<HashedIP> | null
+    };
+
+    return filterAndSortBranding(branding.titles, branding.thumbnails, ip, cache);
+}
+
+export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingResult>> {
+    const getTitles = () => db.prepare(
+        "all",
+        `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID"
+        FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
+        WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
+        [`${videoHashPrefix}%`, service],
+        { useReplica: true }
+    ) as Promise<TitleDBResult[]>;
+
+    const getThumbnails = () => db.prepare(
+        "all",
+        `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
+        FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+        WHERE "thumbnails"."hashedVideoID" LIKE ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
+        [`${videoHashPrefix}%`, service],
+        { useReplica: true }
+    ) as Promise<ThumbnailDBResult[]>;
+
+    const branding = await QueryCacher.get(async () => {
+        // Make sure they are both called in parallel
+        const branding = {
+            titles: getTitles(),
+            thumbnails: getThumbnails()
+        };
+
+        const dbResult: Record<VideoID, BrandingHashDBResult> = {};
+        const initResult = (submission: BrandingDBSubmission) => {
+            dbResult[submission.videoID] = dbResult[submission.videoID] || {
+                titles: [],
+                thumbnails: []
+            };
+        };
+
+        (await branding.titles).map((title) => {
+            initResult(title);
+            dbResult[title.videoID].titles.push(title);
+        });
+        (await branding.thumbnails).map((thumbnail) => {
+            initResult(thumbnail);
+            dbResult[thumbnail.videoID].thumbnails.push(thumbnail);
+        });
+
+        return dbResult;
+    }, brandingHashKey(videoHashPrefix, service));
+
+
+    const cache = {
+        currentIP: null as Promise<HashedIP> | null
+    };
+
+    const processedResult: Record<VideoID, BrandingResult> = {};
+    await Promise.all(Object.keys(branding).map(async (key) => {
+        const castedKey = key as VideoID;
+        processedResult[castedKey] = await filterAndSortBranding(branding[castedKey].titles, branding[castedKey].thumbnails, ip, cache);
+    }));
+
+    return processedResult;
+}
+
+async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise<HashedIP> | null }): Promise<BrandingResult> {
+    const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache);
+    const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache);
+
+    const titles = shuffleArray(dbTitles.filter(await shouldKeepTitles))
+        .sort((a, b) => b.votes - a.votes)
+        .sort((a, b) => b.locked - a.locked)
+        .map((r) => ({
+            title: r.title,
+            original: r.original === 1,
+            votes: r.votes,
+            locked: r.locked === 1,
+            UUID: r.UUID,
+        })) as TitleResult[];
+
+    const thumbnails = shuffleArray(dbThumbnails.filter(await shouldKeepThumbnails))
+        .sort((a, b) => b.votes - a.votes)
+        .sort((a, b) => b.locked - a.locked)
+        .map((r) => ({
+            timestamp: r.timestamp,
+            original: r.original === 1,
+            votes: r.votes,
+            locked: r.locked === 1,
+            UUID: r.UUID
+        })) as ThumbnailResult[];
+
+    return {
+        titles,
+        thumbnails
+    };
+}
+
+async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: BrandingSubmissionType, ip: IPAddress,
+    cache: { currentIP: Promise<HashedIP> | null }): Promise<(_: unknown, index: number) => boolean> {
+
+    const shouldKeep = await Promise.all(submissions.map(async (s) => {
+        if (s.shadowHidden != Visibility.HIDDEN) return true;
+        const table = type === BrandingSubmissionType.Title ? "titleVotes" : "thumbnailVotes";
+        const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`,
+            [s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>;
+        try {
+            const submitterIP = await promiseOrTimeout(QueryCacher.get(fetchData, brandingIPKey(s.UUID)), 150);
+            if (cache.currentIP === null) cache.currentIP = getHashCache((ip + config.globalSalt) as IPAddress);
+            const hashedIP = await cache.currentIP;
+
+            return submitterIP.hashedIP !== hashedIP;
+        } catch (e) {
+            // give up on shadow hide for now
+            return false;
+        }
+    }));
+
+    return (_, index) => shouldKeep[index];
+}
+
+export async function getBranding(req: Request, res: Response) {
+    const videoID: VideoID = req.query.videoID as VideoID;
+    const service: Service = getService(req.query.service as string);
+
+    if (!videoID) {
+        return res.status(400).send("Missing parameter: videoID");
+    }
+
+    const ip = getIP(req);
+    try {
+        const result = await getVideoBranding(videoID, service, ip);
+
+        const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404;
+        return res.status(status).json(result);
+    } catch (e) {
+        Logger.error(e as string);
+        return res.status(500).send("Internal server error");
+    }
+}
+
+export async function getBrandingByHashEndpoint(req: Request, res: Response) {
+    let hashPrefix = req.params.prefix as VideoIDHash;
+    if (!hashPrefix || !hashPrefixTester(hashPrefix) || hashPrefix.length !== 4) {
+        return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+    }
+    hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
+
+    const service: Service = getService(req.query.service as string);
+    const ip = getIP(req);
+
+    try {
+        const result = await getVideoBrandingByHash(hashPrefix, service, ip);
+
+        const status = !isEmpty(result) ? 200 : 404;
+        return res.status(status).json(result);
+    } catch (e) {
+        Logger.error(e as string);
+        return res.status(500).send([]);
+    }
+}
--- a/src/routes/postBranding.ts
+++ b/src/routes/postBranding.ts
@@ -0,0 +1,146 @@
+import { Request, Response } from "express";
+import { config } from "../config";
+import { db, privateDB } from "../databases/databases";
+
+import { BrandingSubmission, BrandingUUID, TimeThumbnailSubmission } from "../types/branding.model";
+import { HashedIP, IPAddress, VideoID } from "../types/segments.model";
+import { HashedUserID } from "../types/user.model";
+import { getHashCache } from "../utils/getHashCache";
+import { getIP } from "../utils/getIP";
+import { getService } from "../utils/getService";
+import { isUserVIP } from "../utils/isUserVIP";
+import { Logger } from "../utils/logger";
+import crypto from "crypto";
+
+enum BrandingType {
+    Title,
+    Thumbnail
+}
+
+interface ExistingVote {
+    UUID: BrandingUUID;
+    type: number;
+    id: number;
+}
+
+export async function postBranding(req: Request, res: Response) {
+    const { videoID, userID, title, thumbnail } = req.body as BrandingSubmission;
+    const service = getService(req.body.service);
+
+    if (!videoID || !userID || userID.length < 30 || !service
+        || ((!title || !title.title)
+            && (!thumbnail || thumbnail.original == null
+                || (!thumbnail.original && !(thumbnail as TimeThumbnailSubmission).timestamp)))) {
+        res.status(400).send("Bad Request");
+        return;
+    }
+
+    try {
+        const hashedUserID = await getHashCache(userID);
+        const isVip = await isUserVIP(hashedUserID);
+        const hashedVideoID = await getHashCache(videoID, 1);
+        const hashedIP = await getHashCache(getIP(req) + config.globalSalt as IPAddress);
+
+        const now = Date.now();
+        const voteType = 1;
+
+        await Promise.all([(async () => {
+            if (title) {
+                const existingUUID = (await db.prepare("get", `SELECT "UUID" from "titles" where "videoID" = ? AND "title" = ?`, [videoID, title.title]))?.UUID;
+                const UUID = existingUUID || crypto.randomUUID();
+
+                const existingVote = await handleExistingVotes(BrandingType.Title, videoID, hashedUserID, UUID, hashedIP, voteType);
+                if (existingUUID) {
+                    await updateVoteTotals(BrandingType.Title, existingVote, UUID, isVip);
+                } else {
+                    await db.prepare("run", `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+                        [videoID, title.title, title.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID]);
+
+                    await db.prepare("run", `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0);`,
+                        [UUID, isVip ? 1 : 0]);
+                }
+
+                if (isVip) {
+                    // unlock all other titles
+                    await db.prepare("run", `UPDATE "titleVotes" SET "locked" = 0 WHERE "UUID" != ?`, [UUID]);
+                }
+            }
+        })(), (async () => {
+            if (thumbnail) {
+                const existingUUID = thumbnail.original
+                    ? (await db.prepare("get", `SELECT "UUID" from "thumbnails" where "videoID" = ? AND "original" = 1`, [videoID]))?.UUID
+                    : (await db.prepare("get", `SELECT "thumbnails"."UUID" from "thumbnailTimestamps" JOIN "thumbnails" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
+                        WHERE "thumbnailTimestamps"."timestamp" = ? AND "thumbnails"."videoID" = ?`, [(thumbnail as TimeThumbnailSubmission).timestamp, videoID]))?.UUID;
+                const UUID = existingUUID || crypto.randomUUID();
+
+                const existingVote = await handleExistingVotes(BrandingType.Thumbnail, videoID, hashedUserID, UUID, hashedIP, voteType);
+                if (existingUUID) {
+                    await updateVoteTotals(BrandingType.Thumbnail, existingVote, UUID, isVip);
+                } else {
+                    await db.prepare("run", `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`,
+                        [videoID, thumbnail.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID]);
+
+                    await db.prepare("run", `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0)`,
+                        [UUID, isVip ? 1 : 0]);
+
+                    if (!thumbnail.original) {
+                        await db.prepare("run", `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`,
+                            [UUID, (thumbnail as TimeThumbnailSubmission).timestamp]);
+                    }
+
+                    if (isVip) {
+                        // unlock all other titles
+                        await db.prepare("run", `UPDATE "thumbnailVotes" SET "locked" = 0 WHERE "UUID" != ?`, [UUID]);
+                    }
+                }
+            }
+        })()]);
+
+        res.status(200).send("OK");
+    } catch (e) {
+        Logger.error(e as string);
+        res.status(500).send("Internal Server Error");
+    }
+}
+
+/**
+ * Finds an existing vote, if found, and it's for a different submission, it undoes it, and points to the new submission.
+ * If no existing vote, it adds one.
+ */
+async function handleExistingVotes(type: BrandingType, videoID: VideoID,
+    hashedUserID: HashedUserID, UUID: BrandingUUID, hashedIP: HashedIP, voteType: number): Promise<ExistingVote> {
+    const table = type === BrandingType.Title ? `"titleVotes"` : `"thumbnailVotes"`;
+
+    const existingVote = await privateDB.prepare("get", `SELECT "id", "UUID", "type" from ${table} where "videoID" = ? AND "userID" = ?`, [videoID, hashedUserID]);
+    if (existingVote && existingVote.UUID !== UUID) {
+        if (existingVote.type === 1) {
+            await db.prepare("run", `UPDATE ${table} SET "votes" = "votes" - 1 WHERE "UUID" = ?`, [existingVote.UUID]);
+        }
+
+        await privateDB.prepare("run", `UPDATE ${table} SET "type" = ?, "UUID" = ? WHERE "id" = ?`, [voteType, UUID, existingVote.id]);
+    } else if (!existingVote) {
+        await privateDB.prepare("run", `INSERT INTO ${table} ("videoID", "UUID", "userID", "hashedIP", "type") VALUES (?, ?, ?, ?, ?)`,
+            [videoID, UUID, hashedUserID, hashedIP, voteType]);
+    }
+
+    return existingVote;
+}
+
+/**
+ * Only called if an existing vote exists.
+ * Will update public vote totals and locked status.
+ */
+async function updateVoteTotals(type: BrandingType, existingVote: ExistingVote, UUID: BrandingUUID, isVip: boolean): Promise<void> {
+    if (!existingVote) return;
+
+    const table = type === BrandingType.Title ? `"titleVotes"` : `"thumbnailVotes"`;
+
+    // Don't upvote if we vote on the same submission
+    if (!existingVote || existingVote.UUID !== UUID) {
+        await db.prepare("run", `UPDATE ${table} SET "votes" = "votes" + 1 WHERE "UUID" = ?`, [UUID]);
+    }
+
+    if (isVip) {
+        await db.prepare("run", `UPDATE ${table} SET "locked" = 1 WHERE "UUID" = ?`, [UUID]);
+    }
+}