Add hiding dearrow submissions in ban code

src/config.ts

@@ -34,6 +34,7 @@ addDefaults(config, {
         poi_highlight: ["poi"],
         chapter: ["chapter"]
     },
+    deArrowTypes: ["title", "thumbnail"],
     maxNumberOfActiveWarnings: 1,
     hoursAfterWarningExpires: 16300000,
     adminUserID: "",

src/routes/postSkipSegments.ts

@@ -554,7 +554,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
 
         if (!userBanCount && ipBanCount) {
             // Make sure the whole user is banned
-            banUser(userID, true, true, 1, config.categoryList as Category[])
+            banUser(userID, true, true, 1, config.categoryList as Category[], config.deArrowTypes)
                 .catch((e) => Logger.error(`Error banning user after submitting from a banned IP: ${e}`));
         }
 

src/routes/shadowBanUser.ts

@@ -2,11 +2,11 @@ import { db, privateDB } from "../databases/databases";
 import { getHashCache } from "../utils/getHashCache";
 import { Request, Response } from "express";
 import { config } from "../config";
-import { Category, HashedIP, Service, VideoID, VideoIDHash } from "../types/segments.model";
+import { Category, DeArrowType, HashedIP, Service, VideoID, VideoIDHash } from "../types/segments.model";
 import { UserID } from "../types/user.model";
 import { QueryCacher } from "../utils/queryCacher";
 import { isUserVIP } from "../utils/isUserVIP";
-import { parseCategories } from "../utils/parseParams";
+import { parseCategories, parseDeArrowTypes } from "../utils/parseParams";
 
 export async function shadowBanUser(req: Request, res: Response): Promise<Response> {
     const userID = req.query.userID as UserID;
@@ -29,6 +29,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
     const unHideOldSubmissions = req.query.unHideOldSubmissions !== "false";
 
     const categories: Category[] = parseCategories(req, config.categoryList as Category[]);
+    const deArrowTypes: DeArrowType[] = parseDeArrowTypes(req, config.deArrowTypes);
 
     if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined || type <= 0)) {
         //invalid request
@@ -45,7 +46,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
     }
 
     if (userID) {
-        const result = await banUser(userID, enabled, unHideOldSubmissions, type, categories);
+        const result = await banUser(userID, enabled, unHideOldSubmissions, type, categories, deArrowTypes);
 
         if (enabled && lookForIPs) {
             const ipLoggingFixedTime = 1675295716000;
@@ -55,7 +56,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
             }))).flat();
 
             await Promise.all([...new Set(ips.map((ip) => ip.hashedIP))].map((ip) => {
-                return banIP(ip, enabled, unHideOldSubmissions, type, categories, true);
+                return banIP(ip, enabled, unHideOldSubmissions, type, categories, deArrowTypes, true);
             }));
         }
 
@@ -64,7 +65,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
             return;
         }
     } else if (hashedIP) {
-        const result = await banIP(hashedIP, enabled, unHideOldSubmissions, type, categories, banUsers);
+        const result = await banIP(hashedIP, enabled, unHideOldSubmissions, type, categories, deArrowTypes, banUsers);
         if (result) {
             res.sendStatus(result);
             return;
@@ -73,7 +74,8 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
     return res.sendStatus(200);
 }
 
-export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmissions: boolean, type: number, categories: Category[]): Promise<number> {
+export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmissions: boolean,
+    type: number, categories: Category[], deArrowTypes: DeArrowType[]): Promise<number> {
     //check to see if this user is already shadowbanned
     const row = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
 
@@ -85,12 +87,12 @@ export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmiss
 
         //find all previous submissions and hide them
         if (unHideOldSubmissions) {
-            await unHideSubmissionsByUser(categories, userID, type);
+            await unHideSubmissionsByUser(categories, deArrowTypes, userID, type);
         }
     } else if (enabled && row.userCount > 0) {
         // apply unHideOldSubmissions if applicable
         if (unHideOldSubmissions) {
-            await unHideSubmissionsByUser(categories, userID, type);
+            await unHideSubmissionsByUser(categories, deArrowTypes, userID, type);
         } else {
             // otherwise ban already exists, send 409
             return 409;
@@ -98,7 +100,7 @@ export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmiss
     } else if (!enabled && row.userCount > 0) {
         //find all previous submissions and unhide them
         if (unHideOldSubmissions) {
-            await unHideSubmissionsByUser(categories, userID, 0);
+            await unHideSubmissionsByUser(categories, deArrowTypes, userID, 0);
         }
 
         //remove them from the shadow ban list
@@ -111,7 +113,9 @@ export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmiss
     return 200;
 }
 
-export async function banIP(hashedIP: HashedIP, enabled: boolean, unHideOldSubmissions: boolean, type: number, categories: Category[], banUsers: boolean): Promise<number> {
+export async function banIP(hashedIP: HashedIP, enabled: boolean, unHideOldSubmissions: boolean, type: number,
+    categories: Category[], deArrowTypes: DeArrowType[], banUsers: boolean): Promise<number> {
+
     //check to see if this user is already shadowbanned
     const row = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]);
 
@@ -126,7 +130,7 @@ export async function banIP(hashedIP: HashedIP, enabled: boolean, unHideOldSubmi
 
             if (banUsers) {
                 await Promise.all([...users].map((user) => {
-                    return banUser(user, enabled, unHideOldSubmissions, type, categories);
+                    return banUser(user, enabled, unHideOldSubmissions, type, categories, deArrowTypes);
                 }));
             }
         } else if (row.userCount > 0) {
@@ -148,7 +152,9 @@ export async function banIP(hashedIP: HashedIP, enabled: boolean, unHideOldSubmi
     return 200;
 }
 
-async function unHideSubmissionsByUser(categories: string[], userID: UserID, type = 1) {
+async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArrowType[],
+    userID: UserID, type = 1) {
+
     await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = '${type}' WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
                     AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
                     "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
@@ -158,6 +164,26 @@ async function unHideSubmissionsByUser(categories: string[], userID: UserID, typ
         .forEach((videoInfo: { category: Category; videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) => {
             QueryCacher.clearSegmentCache(videoInfo);
         });
+
+    if (deArrowTypes.includes("title")) {
+        await db.prepare("run", `UPDATE "titleVotes" as tv SET "shadowHidden" = ${type} FROM "titles" t WHERE tv."UUID" = t."UUID" AND t."userID" = ?`,
+            [userID]);
+    }
+
+    if (deArrowTypes.includes("thumbnail")) {
+        await db.prepare("run", `UPDATE "thumbnailVotes" as tv SET "shadowHidden" = ${type} FROM "thumbnails" t WHERE tv."UUID" = t."UUID" AND t."userID" = ?`,
+            [userID]);
+    }
+
+
+    (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service" FROM "titles" WHERE "userID" = ?`, [userID]))
+        .forEach((videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) => {
+            QueryCacher.clearBrandingCache(videoInfo);
+        });
+    (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service" FROM "thumbnails" WHERE "userID" = ?`, [userID]))
+        .forEach((videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) => {
+            QueryCacher.clearBrandingCache(videoInfo);
+        });
 }
 
 async function unHideSubmissionsByIP(categories: string[], hashedIP: HashedIP, type = 1): Promise<Set<UserID>> {

src/types/config.model.ts

@@ -1,5 +1,6 @@
 import { PoolConfig } from "pg";
 import * as redis from "redis";
+import { DeArrowType } from "./segments.model";
 
 interface RedisConfig extends redis.RedisClientOptions {
     enabled: boolean;
@@ -65,6 +66,7 @@ export interface SBSConfig {
     readOnly: boolean;
     webhooks: WebhookConfig[];
     categoryList: string[];
+    deArrowTypes: DeArrowType[];
     categorySupport: Record<string, string[]>;
     getTopUsersCacheTimeMinutes: number;
     maxNumberOfActiveWarnings: number;

src/types/segments.model.ts

@@ -6,6 +6,7 @@ export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
 export type VideoID = string & { __videoIDBrand: unknown };
 export type VideoDuration = number & { __videoDurationBrand: unknown };
 export type Category = ("sponsor" | "selfpromo" | "interaction" | "intro" | "outro" | "preview" | "music_offtopic" | "poi_highlight" | "chapter" | "filler" | "exclusive_access") & { __categoryBrand: unknown };
+export type DeArrowType = "title" | "thumbnail";
 export type VideoIDHash = VideoID & HashedValue;
 export type IPAddress = string & { __ipAddressBrand: unknown };
 export type HashedIP = IPAddress & HashedValue;

src/utils/parseParams.ts

@@ -1,5 +1,5 @@
 import { Request } from "express";
-import { ActionType, SegmentUUID, Category } from "../types/segments.model";
+import { ActionType, SegmentUUID, Category, DeArrowType } from "../types/segments.model";
 import { config } from "../config";
 
 type fn = (req: Request, fallback: any) => any[];
@@ -11,15 +11,21 @@ const syntaxErrorWrapper = (fn: fn, req: Request, fallback: any) => {
     }
 };
 
-const getCategories = (req: Request, fallback: Category[] ): string[] | Category[] =>
+const getQueryList = <T>(req: Request, fallback: T[], param: string, paramPlural: string): string[] | T[] =>
-    req.query.categories
+    req.query[paramPlural]
-        ? JSON.parse(req.query.categories as string)
+        ? JSON.parse(req.query[paramPlural] as string)
-        : req.query.category
+        : req.query[param]
-            ? Array.isArray(req.query.category)
+            ? Array.isArray(req.query[param])
-                ? req.query.category
+                ? req.query[param]
-                : [req.query.category]
+                : [req.query[param]]
             : fallback;
 
+const getCategories = (req: Request, fallback: Category[] ): string[] | Category[] =>
+    getQueryList(req, fallback, "category", "categories");
+
+const getDeArrowTypes = (req: Request, fallback: DeArrowType[] ): string[] | DeArrowType[] =>
+    getQueryList(req, fallback, "deArrowType", "deArrowTypes");
+
 const validateString = (array: any[]): any[] => {
     if (!Array.isArray(array)) return undefined;
     return array
@@ -71,6 +77,11 @@ export const parseActionTypes = (req: Request, fallback: ActionType[]): ActionTy
     return actionTypes ? validateString(actionTypes) : undefined;
 };
 
+export const parseDeArrowTypes = (req: Request, fallback: DeArrowType[]): DeArrowType[] => {
+    const deArrowTypes = syntaxErrorWrapper(getDeArrowTypes, req, fallback);
+    return deArrowTypes ? validateString(deArrowTypes) : undefined;
+};
+
 export const parseRequiredSegments = (req: Request): SegmentUUID[] | undefined =>
     syntaxErrorWrapper(getRequiredSegments, req, []); // never fall back
 

test/cases/shadowBanUser.ts

@@ -1,13 +1,15 @@
 import { db, privateDB } from "../../src/databases/databases";
 import { getHash } from "../../src/utils/getHash";
 import assert from "assert";
-import { Category } from "../../src/types/segments.model";
+import { Category, Service } from "../../src/types/segments.model";
 import { client } from "../utils/httpClient";
 
 describe("shadowBanUser", () => {
     const getShadowBan = (userID: string) => db.prepare("get", `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
     const getShadowBanSegments = (userID: string, status: number) => db.prepare("all", `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, [userID, status]);
     const getShadowBanSegmentCategory = (userID: string, status: number): Promise<{shadowHidden: number, category: Category}[]> => db.prepare("all", `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, [userID, status]);
+    const getShadowBanTitles = (userID: string, status: number) => db.prepare("all", `SELECT tv."shadowHidden" FROM "titles" t JOIN "titleVotes" tv ON t."UUID" = tv."UUID" WHERE t."userID" = ? AND tv."shadowHidden" = ?`, [userID, status]);
+    const getShadowBanThumbnails = (userID: string, status: number) => db.prepare("all", `SELECT tv."shadowHidden" FROM "thumbnails" t JOIN "thumbnailVotes" tv ON t."UUID" = tv."UUID" WHERE t."userID" = ? AND tv."shadowHidden" = ?`, [userID, status]);
 
     const getIPShadowBan = (hashedIP: string) => db.prepare("get", `SELECT * FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]);
 
@@ -67,6 +69,52 @@ describe("shadowBanUser", () => {
         await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062443, "YouTube"]);
         await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062342, "YouTube"]);
         await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916069491, "YouTube"]);
+
+        const titleQuery = `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`;
+        const titleVotesQuery = `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden", "verification") VALUES (?, ?, ?, ?, ?)`;
+        const thumbnailQuery = `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`;
+        const thumbnailTimestampsQuery = `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`;
+        const thumbnailVotesQuery = `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`;
+
+        await Promise.all([
+            db.prepare("run", titleQuery, [video, "title1", 0, "userID1-ban", Service.YouTube, videohash, 1, "UUID1-ban"]),
+            db.prepare("run", titleQuery, [video, "title2", 0, "userID1-ban", Service.YouTube, videohash, 1, "UUID2-ban"]),
+            db.prepare("run", titleQuery, [video, "title3", 1, "userID1-ban", Service.YouTube, videohash, 1, "UUID3-ban"]),
+            db.prepare("run", thumbnailQuery, [video, 0, "userID1-ban", Service.YouTube, videohash, 1, "UUID1T-ban"]),
+            db.prepare("run", thumbnailQuery, [video, 1, "userID1-ban", Service.YouTube, videohash, 1, "UUID2T-ban"]),
+            db.prepare("run", thumbnailQuery, [video, 0, "userID1-ban", Service.YouTube, videohash, 1, "UUID3T-ban"]),
+        ]);
+
+        await Promise.all([
+            db.prepare("run", titleVotesQuery, ["UUID1-ban", 3, 0, 0, 0]),
+            db.prepare("run", titleVotesQuery, ["UUID2-ban", 2, 0, 0, 0]),
+            db.prepare("run", titleVotesQuery, ["UUID3-ban", 1, 0, 0, 0]),
+            db.prepare("run", thumbnailTimestampsQuery, ["UUID1T-ban", 1]),
+            db.prepare("run", thumbnailTimestampsQuery, ["UUID3T-ban", 3]),
+            db.prepare("run", thumbnailVotesQuery, ["UUID1T-ban", 3, 0, 0]),
+            db.prepare("run", thumbnailVotesQuery, ["UUID2T-ban", 2, 0, 0]),
+            db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban", 1, 0, 0])
+        ]);
+
+        await Promise.all([
+            db.prepare("run", titleQuery, [video, "title1", 0, "userID2-ban", Service.YouTube, videohash, 1, "UUID1-ban2"]),
+            db.prepare("run", titleQuery, [video, "title2", 0, "userID2-ban", Service.YouTube, videohash, 1, "UUID2-ban2"]),
+            db.prepare("run", titleQuery, [video, "title3", 1, "userID2-ban", Service.YouTube, videohash, 1, "UUID3-ban2"]),
+            db.prepare("run", thumbnailQuery, [video, 0, "userID2-ban", Service.YouTube, videohash, 1, "UUID1T-ban2"]),
+            db.prepare("run", thumbnailQuery, [video, 1, "userID2-ban", Service.YouTube, videohash, 1, "UUID2T-ban2"]),
+            db.prepare("run", thumbnailQuery, [video, 0, "userID2-ban", Service.YouTube, videohash, 1, "UUID3T-ban2"]),
+        ]);
+
+        await Promise.all([
+            db.prepare("run", titleVotesQuery, ["UUID1-ban2", 3, 0, 0, 0]),
+            db.prepare("run", titleVotesQuery, ["UUID2-ban2", 2, 0, 0, 0]),
+            db.prepare("run", titleVotesQuery, ["UUID3-ban2", 1, 0, 0, 0]),
+            db.prepare("run", thumbnailTimestampsQuery, ["UUID1T-ban2", 1]),
+            db.prepare("run", thumbnailTimestampsQuery, ["UUID3T-ban2", 3]),
+            db.prepare("run", thumbnailVotesQuery, ["UUID1T-ban2", 3, 0, 0]),
+            db.prepare("run", thumbnailVotesQuery, ["UUID2T-ban2", 2, 0, 0]),
+            db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban2", 1, 0, 0])
+        ]);
     });
 
     it("Should be able to ban user and hide submissions", (done) => {
@@ -463,4 +511,51 @@ describe("shadowBanUser", () => {
             })
             .catch(err => done(err));
     });
+
+    it("Should be able to ban user and hide dearrow submissions", (done) => {
+        const userID = "userID1-ban";
+        client({
+            method: "POST",
+            url: endpoint,
+            params: {
+                userID,
+                adminUserID: VIPuserID,
+            }
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const titles = await getShadowBanTitles(userID, 1);
+                const thumbnails = await getShadowBanThumbnails(userID, 1);
+                const shadowRow = await getShadowBan(userID);
+                assert.ok(shadowRow);
+                assert.strictEqual(titles.length, 3);
+                assert.strictEqual(thumbnails.length, 3);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to ban user and hide just dearrow titles", (done) => {
+        const userID = "userID2-ban";
+        client({
+            method: "POST",
+            url: endpoint,
+            params: {
+                userID,
+                adminUserID: VIPuserID,
+                deArrowTypes: `["title"]`
+            }
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const titles = await getShadowBanTitles(userID, 1);
+                const thumbnails = await getShadowBanThumbnails(userID, 1);
+                const shadowRow = await getShadowBan(userID);
+                assert.ok(shadowRow);
+                assert.strictEqual(titles.length, 3);
+                assert.strictEqual(thumbnails.length, 0);
+                done();
+            })
+            .catch(err => done(err));
+    });
 });