Merge branch 'master' into clickbait

src/utils/getSubmissionUUID.ts

@@ -7,10 +7,11 @@ export function getSubmissionUUID(
     videoID: VideoID,
     category: Category,
     actionType: ActionType,
+    description: string,
     userID: UserID,
     startTime: number,
     endTime: number,
     service: Service
 ) : HashedValue {
-    return `${getHash(`${videoID}${startTime}${endTime}${userID}${category}${actionType}${service}`, 1)}6` as HashedValue;
+    return `${getHash(`${videoID}${startTime}${endTime}${userID}${description}${category}${actionType}${service}`, 1)}7` as HashedValue;
 }

src/utils/innerTubeAPI.ts

@@ -34,7 +34,7 @@ async function getFromITube (videoID: string): Promise<innerTubeVideoDetails> {
         context: {
             client: {
                 clientName: "WEB",
-                clientVersion: "2.20211129.09.00"
+                clientVersion: "2.20221215.04.01"
             }
         },
         videoId: videoID

src/utils/parseSkipSegments.ts

@@ -0,0 +1,75 @@
+import { Request } from "express";
+import { ActionType, SegmentUUID, Category, Service } from "../types/segments.model";
+import { getService } from "./getService";
+
+type fn = (req: Request) => any[];
+
+const syntaxErrorWrapper = (fn: fn, req: Request) => {
+    try { return fn(req); }
+    catch (e) { return undefined; }
+};
+
+// Default to sponsor
+const getCategories = (req: Request): Category[] =>
+    req.query.categories
+        ? JSON.parse(req.query.categories as string)
+        : req.query.category
+            ? Array.isArray(req.query.category)
+                ? req.query.category
+                : [req.query.category]
+            : ["sponsor"];
+
+// Default to skip
+const getActionTypes = (req: Request): ActionType[] =>
+    req.query.actionTypes
+        ? JSON.parse(req.query.actionTypes as string)
+        : req.query.actionType
+            ? Array.isArray(req.query.actionType)
+                ? req.query.actionType
+                : [req.query.actionType]
+            : [ActionType.Skip];
+
+// Default to empty array
+const getRequiredSegments = (req: Request): SegmentUUID[] =>
+    req.query.requiredSegments
+        ? JSON.parse(req.query.requiredSegments as string)
+        : req.query.requiredSegment
+            ? Array.isArray(req.query.requiredSegment)
+                ? req.query.requiredSegment
+                : [req.query.requiredSegment]
+            : [];
+
+const errorMessage = (parameter: string) => `${parameter} parameter does not match format requirements.`;
+
+export function parseSkipSegments(req: Request): {
+    categories: Category[];
+    actionTypes: ActionType[];
+    requiredSegments: SegmentUUID[];
+    service: Service;
+    errors: string[];
+} {
+    let categories: Category[] = syntaxErrorWrapper(getCategories, req);
+    const actionTypes: ActionType[] = syntaxErrorWrapper(getActionTypes, req);
+    const requiredSegments: SegmentUUID[] = syntaxErrorWrapper(getRequiredSegments, req);
+    const service: Service = getService(req.query.service, req.body.services);
+    const errors: string[] = [];
+    if (!Array.isArray(categories)) errors.push(errorMessage("categories"));
+    else {
+        // check category names for invalid characters
+        // and none string elements
+        categories = categories
+            .filter((item: any) => typeof item === "string")
+            .filter((category) => !(/[^a-z|_|-]/.test(category)));
+        if (categories.length === 0) errors.push("No valid categories provided.");
+    }
+    if (!Array.isArray(actionTypes)) errors.push(errorMessage("actionTypes"));
+    if (!Array.isArray(requiredSegments)) errors.push(errorMessage("requiredSegments"));
+    // finished parsing
+    return {
+        categories,
+        actionTypes,
+        requiredSegments,
+        service,
+        errors
+    };
+}

src/utils/queryCacher.ts

@@ -87,6 +87,17 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID
     }
 }
 
+async function getKeyLastModified(key: string): Promise<Date> {
+    if (!config.redis?.enabled) return Promise.reject("ETag - Redis not enabled");
+    return await redis.ttl(key)
+        .then(ttl => {
+            const sinceLive = config.redis?.expiryTime - ttl;
+            const now = Math.floor(Date.now() / 1000);
+            return new Date((now-sinceLive) * 1000);
+        })
+        .catch(() => Promise.reject("ETag - Redis error"));
+}
+
 function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
     if (videoInfo) {
         redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
@@ -101,6 +112,7 @@ export const QueryCacher = {
     get,
     getAndSplit,
     clearSegmentCache,
+    getKeyLastModified,
     clearRatingCache,
-    clearFeatureCache
+    clearFeatureCache,
 };

src/utils/redis.ts

@@ -19,6 +19,7 @@ interface RedisSB {
     del(...keys: [RedisCommandArgument]): Promise<number>;
     increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
     sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
+    ttl(key: RedisCommandArgument): Promise<number>;
     quit(): Promise<void>;
 }
 
@@ -30,6 +31,7 @@ let exportClient: RedisSB = {
     increment: () => new Promise((resolve) => resolve(null)),
     sendCommand: () => new Promise((resolve) => resolve(null)),
     quit: () => new Promise((resolve) => resolve(null)),
+    ttl: () => new Promise((resolve) => resolve(null)),
 };
 
 let lastClientFail = 0;

src/utils/youtubeID.ts

@@ -0,0 +1,26 @@
+import { VideoID } from "../types/segments.model";
+
+const idRegex = new RegExp(/([0-9A-Za-z_-]{11})/); // group to always be index 1
+const exclusiveIdegex = new RegExp(`^${idRegex.source}$`);
+// match /c/, /channel/, /@channel, full UUIDs
+const negativeRegex = new RegExp(/(\/(channel|c)\/.+)|(\/@.+)|([a-f0-9]{64,65})|(youtube\.com\/clip\/)/);
+const urlRegex = new RegExp(`(?:v=|/|youtu.be/)${idRegex.source}(?:|/|[?&]t=\\d+s?)>?(?:\\s|$)`);
+const negateIdRegex = new RegExp(/(?:[^0-9A-Za-z_-]*?)/);
+const looseEndsRegex = new RegExp(`${negateIdRegex.source}${idRegex.source}${negateIdRegex.source}`);
+
+export const validate = (id: string): boolean => exclusiveIdegex.test(id);
+
+export const sanitize = (id: string): VideoID | null => {
+    // first decode URI
+    id = decodeURIComponent(id);
+    // strict matching
+    const strictMatch = id.match(exclusiveIdegex)?.[1];
+    const urlMatch = id.match(urlRegex)?.[1];
+    // return match, if not negative, return looseMatch
+    const looseMatch = id.match(looseEndsRegex)?.[1];
+    return strictMatch ? (strictMatch as VideoID)
+        : negativeRegex.test(id) ? null
+            : urlMatch ? (urlMatch as VideoID)
+                : looseMatch ? (looseMatch as VideoID)
+                    : null;
+};