create a request validator engine

src/config.ts

@@ -69,10 +69,7 @@ addDefaults(config, {
             message: "OK",
         }
     },
-    validityCheck: {
+    requestValidatorRules: [],
-        userAgent: null,
-        userAgentR: null,
-    },
     userCounterURL: null,
     userCounterRatio: 10,
     newLeafURLs: null,
@@ -276,4 +273,4 @@ function loadFromEnv(config: SBSConfig, prefix = "") {
             }
         }
     }
-}
+}

src/routes/postBranding.ts

@@ -18,8 +18,9 @@ import { checkBanStatus } from "../utils/checkBan";
 import axios from "axios";
 import { getMaxResThumbnail } from "../utils/youtubeApi";
 import { getVideoDetails } from "../utils/getVideoDetails";
-import { canSubmitDeArrow, validSubmittedData } from "../utils/permissions";
+import { canSubmitDeArrow } from "../utils/permissions";
 import { parseUserAgent } from "../utils/userAgent";
+import { isRequestInvalid } from "../utils/requestValidator";
 
 enum BrandingType {
     Title,
@@ -58,8 +59,19 @@ export async function postBranding(req: Request, res: Response) {
         const hashedIP = await getHashCache(getIP(req) + config.globalSalt as IPAddress);
         const isBanned = await checkBanStatus(hashedUserID, hashedIP);
 
-        if (!validSubmittedData(userAgent, req.headers["user-agent"])) {
+        if (isRequestInvalid({
-            Logger.warn(`Rejecting submission based on invalid data: ${hashedUserID} ${videoID} ${videoDuration} ${userAgent} ${req.headers["user-agent"]}`);
+            userAgent,
+            userAgentHeader: req.headers["user-agent"],
+            videoDuration,
+            userID,
+            service,
+            dearrow: {
+                title,
+                thumbnail,
+                downvote,
+            }
+        })) {
+            Logger.warn(`Rejecting submission based on invalid data: ${hashedUserID} ${videoID} ${videoDuration} ${userAgent} ${req.headers["user-agent"]} ${title.title} ${thumbnail.timestamp}`);
             res.status(200).send("OK");
             return;
         }

src/routes/postSkipSegments.ts

@@ -20,11 +20,12 @@ import { parseUserAgent } from "../utils/userAgent";
 import { getService } from "../utils/getService";
 import axios from "axios";
 import { vote } from "./voteOnSponsorTime";
-import { canSubmit, canSubmitGlobal, validSubmittedData } from "../utils/permissions";
+import { canSubmit, canSubmitGlobal } from "../utils/permissions";
 import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
 import * as youtubeID from "../utils/youtubeID";
 import { acquireLock } from "../utils/redisLock";
 import { checkBanStatus } from "../utils/checkBan";
+import { isRequestInvalid } from "../utils/requestValidator";
 
 type CheckResult = {
     pass: boolean,
@@ -509,7 +510,14 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
     }
     const userID: HashedUserID = await getHashCache(paramUserID);
 
-    if (!validSubmittedData(userAgent, req.headers["user-agent"])) {
+    if (isRequestInvalid({
+        userAgent,
+        userAgentHeader: req.headers["user-agent"],
+        videoDuration,
+        userID: paramUserID,
+        service,
+        segments,
+    })) {
         Logger.warn(`Rejecting submission based on invalid data: ${userID} ${videoID} ${videoDurationParam} ${userAgent} ${req.headers["user-agent"]}`);
         return res.status(200).send("OK");
     }

src/types/branding.model.ts

@@ -74,6 +74,7 @@ export interface BrandingHashDBResult {
 }
 
 export interface OriginalThumbnailSubmission {
+    timestamp?: undefined | null;
     original: true;
 }
 
@@ -136,4 +137,4 @@ export interface CasualVoteHashDBResult extends BrandingDBSubmissionData {
     category: CasualCategory;
     upvotes: number;
     downvotes: number;
-}
+}

src/types/config.model.ts

@@ -41,6 +41,28 @@ export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig {
     stopRetryThreshold: number;
 }
 
+export type ValidatorPattern = string | [string, string];
+export interface RequestValidatorRule {
+    // universal
+    userAgent?: ValidatorPattern;
+    userAgentHeader?: ValidatorPattern;
+    videoDuration?: ValidatorPattern;
+    userID?: ValidatorPattern;
+    service?: ValidatorPattern;
+    // sb
+    startTime?: ValidatorPattern;
+    endTime?: ValidatorPattern;
+    category?: ValidatorPattern;
+    actionType?: ValidatorPattern;
+    description?: ValidatorPattern;
+    // dearrow
+    title?: ValidatorPattern;
+    titleOriginal?: boolean;
+    thumbnailTimestamp?: ValidatorPattern;
+    thumbnailOriginal?: boolean;
+    dearrowDownvote?: boolean;
+}
+
 export interface SBSConfig {
     [index: string]: any
     port: number;
@@ -85,10 +107,7 @@ export interface SBSConfig {
         vote: RateLimitConfig;
         view: RateLimitConfig;
     };
-    validityCheck: {
+    requestValidatorRules: RequestValidatorRule[];
-        userAgent: string | null;
-        userAgentR: string | null;
-    }
     minimumPrefix?: string;
     maximumPrefix?: string;
     redis?: RedisConfig;
@@ -173,4 +192,4 @@ export interface CronJobOptions {
 export interface DownvoteSegmentArchiveCron {
     voteThreshold: number;
     timeThresholdInDays: number;
-}
+}

src/utils/permissions.ts

@@ -108,34 +108,6 @@ export async function canSubmit(userID: HashedUserID, category: Category): Promi
     }
 }
 
-export function validSubmittedData(userAgent: string, userAgentR: string): boolean {
-    if (!config.validityCheck.userAgent) {
-        return true;
-    }
-
-    for (const key of Object.keys(config.validityCheck)) {
-        const check = (config.validityCheck as Record<string, string | null>)[key];
-        if (check === null) {
-            continue;
-        } else {
-            switch (key) {
-                case "userAgent":
-                    if (!userAgent.match(check)) {
-                        return false;
-                    }
-                    break;
-                case "userAgentR":
-                    if (!userAgentR.match(new RegExp(check))) {
-                        return false;
-                    }
-                    break;
-            }
-        }
-    }
-
-    return true;
-}
-
 export async function canSubmitGlobal(userID: HashedUserID): Promise<CanSubmitGlobalResult> {
     const oldSubmitterOrAllowedPromise = oldSubmitterOrAllowed(userID);
 
@@ -158,4 +130,4 @@ export async function canSubmitDeArrow(userID: HashedUserID): Promise<CanSubmitG
         newUser: (await oldSubmitterOrAllowedPromise).newUser,
         reason: "We are currently experiencing a mass spam attack, we are restricting submissions for now"
     };
-}
+}

src/utils/requestValidator.ts

@@ -0,0 +1,215 @@
+import { config } from "../config";
+import { ThumbnailSubmission, TitleSubmission } from "../types/branding.model";
+import { ValidatorPattern, RequestValidatorRule } from "../types/config.model";
+import { IncomingSegment } from "../types/segments.model";
+
+export interface RequestValidatorInput {
+    userAgent?: string;
+    userAgentHeader?: string;
+    videoDuration?: string | number;
+    userID?: string;
+    service?: string;
+    segments?: IncomingSegment[];
+    dearrow?: {
+        title?: TitleSubmission;
+        thumbnail?: ThumbnailSubmission;
+        downvote: boolean;
+    };
+}
+export type CompiledValidityCheck = (input: RequestValidatorInput) => boolean;
+type CompiledSegmentCheck = (input: IncomingSegment) => boolean;
+type InputExtractor = (input: RequestValidatorInput) => string | number | undefined | null;
+type SegmentExtractor = (input: IncomingSegment) => string | undefined | null;
+type BooleanRules = "titleOriginal" | "thumbnailOriginal" | "dearrowDownvote";
+type RuleEntry =
+    | [Exclude<keyof RequestValidatorRule, BooleanRules>, ValidatorPattern]
+    | [BooleanRules, boolean];
+
+let compiledRules: CompiledValidityCheck;
+
+function compilePattern(
+    pattern: ValidatorPattern,
+    extractor: InputExtractor,
+): CompiledValidityCheck {
+    const regex =
+        typeof pattern === "string"
+            ? new RegExp(pattern, "i")
+            : new RegExp(...pattern);
+
+    return (input: RequestValidatorInput) => {
+        const field = extractor(input);
+        if (field == undefined) return false;
+        return regex.test(String(field));
+    };
+}
+
+function compileSegmentPattern(
+    pattern: ValidatorPattern,
+    extractor: SegmentExtractor,
+): CompiledSegmentCheck {
+    const regex =
+        typeof pattern === "string"
+            ? new RegExp(pattern, "i")
+            : new RegExp(...pattern);
+
+    return (input: IncomingSegment) => {
+        const field = extractor(input);
+        if (field == undefined) return false;
+        return regex.test(field);
+    };
+}
+
+export function compileRules(
+    ruleDefinitions: RequestValidatorRule[],
+): CompiledValidityCheck {
+    if (ruleDefinitions.length === 0) return () => false;
+
+    const rules: CompiledValidityCheck[] = [];
+    for (const ruleDefinition of ruleDefinitions) {
+        const ruleComponents: CompiledValidityCheck[] = [];
+        const segmentRuleComponents: CompiledSegmentCheck[] = [];
+        for (const [ruleKey, rulePattern] of Object.entries(
+            ruleDefinition,
+        ) as RuleEntry[]) {
+            switch (ruleKey) {
+                case "userAgent":
+                    ruleComponents.push(
+                        compilePattern(rulePattern, (input) => input.userAgent),
+                    );
+                    break;
+                case "userAgentHeader":
+                    ruleComponents.push(
+                        compilePattern(
+                            rulePattern,
+                            (input) => input.userAgentHeader,
+                        ),
+                    );
+                    break;
+                case "videoDuration":
+                    ruleComponents.push(
+                        compilePattern(
+                            rulePattern,
+                            (input) => input.videoDuration,
+                        ),
+                    );
+                    break;
+                case "userID":
+                    ruleComponents.push(
+                        compilePattern(rulePattern, (input) => input.userID),
+                    );
+                    break;
+                case "service":
+                    ruleComponents.push(
+                        compilePattern(rulePattern, (input) => input.service),
+                    );
+                    break;
+                case "startTime":
+                    segmentRuleComponents.push(
+                        compileSegmentPattern(
+                            rulePattern,
+                            (input) => input.segment[0],
+                        ),
+                    );
+                    break;
+                case "endTime":
+                    segmentRuleComponents.push(
+                        compileSegmentPattern(
+                            rulePattern,
+                            (input) => input.segment[1],
+                        ),
+                    );
+                    break;
+                case "category":
+                    segmentRuleComponents.push(
+                        compileSegmentPattern(
+                            rulePattern,
+                            (input) => input.category,
+                        ),
+                    );
+                    break;
+                case "actionType":
+                    segmentRuleComponents.push(
+                        compileSegmentPattern(
+                            rulePattern,
+                            (input) => input.actionType,
+                        ),
+                    );
+                    break;
+                case "description":
+                    segmentRuleComponents.push(
+                        compileSegmentPattern(
+                            rulePattern,
+                            (input) => input.description,
+                        ),
+                    );
+                    break;
+                case "title":
+                    ruleComponents.push(
+                        compilePattern(
+                            rulePattern,
+                            (input) => input.dearrow?.title?.title,
+                        ),
+                    );
+                    break;
+                case "titleOriginal":
+                    ruleComponents.push(
+                        (input) => input.dearrow?.title?.original === rulePattern,
+                    );
+                    break;
+                case "thumbnailTimestamp":
+                    ruleComponents.push(
+                        compilePattern(
+                            rulePattern,
+                            (input) => input.dearrow?.thumbnail?.timestamp,
+                        ),
+                    );
+                    break;
+                case "thumbnailOriginal":
+                    ruleComponents.push(
+                        (input) =>
+                            input.dearrow?.thumbnail?.original === rulePattern,
+                    );
+                    break;
+                case "dearrowDownvote":
+                    ruleComponents.push(
+                        (input) =>
+                            input.dearrow?.downvote === rulePattern,
+                    );
+                    break;
+            }
+        }
+        if (segmentRuleComponents.length > 0) {
+            ruleComponents.push((input) => {
+                if (input.segments === undefined) return false;
+                for (const segment of input.segments) {
+                    let result = true;
+                    for (const rule of segmentRuleComponents) {
+                        if (!rule(segment)) {
+                            result = false;
+                            break;
+                        }
+                    }
+                    if (result) return true;
+                }
+                return false;
+            });
+        }
+        rules.push((input) => {
+            for (const rule of ruleComponents) {
+                if (!rule(input)) return false;
+            }
+            return true;
+        });
+    }
+    return (input) => {
+        for (const rule of rules) {
+            if (rule(input)) return true;
+        }
+        return false;
+    };
+}
+
+export function isRequestInvalid(input: RequestValidatorInput) {
+    compiledRules ??= compileRules(config.requestValidatorRules);
+    return compiledRules(input);
+}