Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into reWriteTests

src/app.ts

@@ -37,6 +37,7 @@ import {getLockCategories} from "./routes/getLockCategories";
 import {getLockCategoriesByHash} from "./routes/getLockCategoriesByHash";
 import {endpoint as getSearchSegments } from "./routes/getSearchSegments";
 import {getStatus } from "./routes/getStatus";
+import {getUserStats} from "./routes/getUserStats";
 import ExpressPromiseRouter from "express-promise-router";
 import { Server } from "http";
 import { youtubeApiProxy } from "./routes/youtubeApiProxy";
@@ -175,6 +176,8 @@ function setupRoutes(router: Router) {
     router.get("/api/status", getStatus);
 
     router.get("/api/youtubeApiProxy", youtubeApiProxy);
+    // get user category stats
+    router.get("/api/userStats", getUserStats);
 
     if (config.postgres) {
         router.get("/database", (req, res) => dumpDatabase(req, res, true));

src/routes/getSearchSegments.ts

@@ -1,6 +1,7 @@
 import { Request, Response } from "express";
 import { db } from "../databases/databases";
 import { ActionType, Category, DBSegment, Service, VideoID } from "../types/segments.model";
+import { getService } from "../utils/getService";
 const segmentsPerPage = 10;
 
 type searchSegmentResponse = {
@@ -59,10 +60,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<searchSeg
         return false;
     }
 
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val == service)) {
-        service = Service.YouTube;
-    }
 
     let page: number = req.query.page ?? req.body.page ?? 0;
     page = Number(page);

src/routes/getSkipSegments.ts

@@ -10,6 +10,7 @@ import { getIP } from "../utils/getIP";
 import { Logger } from "../utils/logger";
 import { QueryCacher } from "../utils/queryCacher";
 import { getReputation } from "../utils/reputation";
+import { getService } from "../utils/getService";
 
 
 async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
@@ -317,10 +318,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
         return false;
     }
 
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val == service)) {
-        service = Service.YouTube;
-    }
 
     const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
 

src/routes/getSkipSegmentsByHash.ts

@@ -2,6 +2,7 @@ import {hashPrefixTester} from "../utils/hashPrefixTester";
 import {getSegmentsByHash} from "./getSkipSegments";
 import {Request, Response} from "express";
 import { ActionType, Category, SegmentUUID, Service, VideoIDHash } from "../types/segments.model";
+import { getService } from "../utils/getService";
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
     let hashPrefix = req.params.prefix as VideoIDHash;
@@ -58,10 +59,7 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
         return res.status(400).send("Bad parameter: requiredSegments (invalid JSON)");
     }
 
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val == service)) {
-        service = Service.YouTube;
-    }
 
     // filter out none string elements, only flat array with strings is valid
     categories = categories.filter((item: any) => typeof item === "string");

src/routes/getUserInfo.ts

@@ -43,12 +43,7 @@ async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<number> {
 async function dbGetUsername(userID: HashedUserID) {
     try {
         const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
-        if (row !== undefined) {
+        return row?.userName ?? userID;
-            return row.userName;
-        } else {
-            //no username yet, just send back the userID
-            return userID;
-        }
     } catch (err) {
         return false;
     }
@@ -172,7 +167,7 @@ async function getUserInfo(req: Request, res: Response): Promise<Response> {
         responseObj[property] = await dbGetValue(hashedUserID, property);
     }
     // add minutesSaved and segmentCount after to avoid getting overwritten
-    if (paramValues.includes("minutesSaved")) { responseObj["minutesSaved"] = segmentsSummary.minutesSaved; }
+    if (paramValues.includes("minutesSaved")) responseObj["minutesSaved"] = segmentsSummary.minutesSaved;
     if (paramValues.includes("segmentCount")) responseObj["segmentCount"] = segmentsSummary.segmentCount;
     return res.send(responseObj);
 }

src/routes/getUserStats.ts

@@ -0,0 +1,96 @@
+import {db} from "../databases/databases";
+import {getHash} from "../utils/getHash";
+import {Request, Response} from "express";
+import {HashedUserID, UserID} from "../types/user.model";
+import {config} from "../config";
+import { Logger } from "../utils/logger";
+type nestedObj = Record<string, Record<string, number>>;
+const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
+
+async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolean, fetchActionTypeStats: boolean) {
+    let additionalQuery = "";
+    if (fetchCategoryStats) {
+        additionalQuery += `
+            SUM(CASE WHEN "category" = 'sponsor' THEN 1 ELSE 0 END) as "categorySumSponsor",
+            SUM(CASE WHEN "category" = 'intro' THEN 1 ELSE 0 END) as "categorySumIntro",
+            SUM(CASE WHEN "category" = 'outro' THEN 1 ELSE 0 END) as "categorySumOutro",
+            SUM(CASE WHEN "category" = 'interaction' THEN 1 ELSE 0 END) as "categorySumInteraction",
+            SUM(CASE WHEN "category" = 'selfpromo' THEN 1 ELSE 0 END) as "categorySelfpromo",
+            SUM(CASE WHEN "category" = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic",
+            SUM(CASE WHEN "category" = 'preview' THEN 1 ELSE 0 END) as "categorySumPreview",
+            SUM(CASE WHEN "category" = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight",`;
+    }
+    if (fetchActionTypeStats) {
+        additionalQuery += `
+            SUM(CASE WHEN "actionType" = 'skip' THEN 1 ELSE 0 END) as "typeSumSkip",
+            SUM(CASE WHEN "actionType" = 'mute' THEN 1 ELSE 0 END) as "typeSumMute",`;
+    }
+    try {
+        const row = await db.prepare("get", `
+            SELECT SUM(((CASE WHEN "endTime" - "startTime" > ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved",
+            ${additionalQuery}
+            count(*) as "segmentCount"
+            FROM "sponsorTimes"
+            WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" !=1`,
+        [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, userID]);
+        const source = (row.minutesSaved != null) ? row : {};
+        const handler = { get: (target: Record<string, any>, name: string) => target?.[name] || 0 };
+        const proxy = new Proxy(source, handler);
+        const result = {} as nestedObj;
+
+        result.overallStats = {
+            minutesSaved: proxy.minutesSaved,
+            segmentCount: proxy.segmentCount,
+        };
+        if (fetchCategoryStats) {
+            result.categoryCount = {
+                sponsor: proxy.categorySumSponsor,
+                intro: proxy.categorySumIntro,
+                outro: proxy.categorySumOutro,
+                interaction: proxy.categorySumInteraction,
+                selfpromo: proxy.categorySelfpromo,
+                music_offtopic: proxy.categoryMusicOfftopic,
+                preview: proxy.categorySumPreview,
+                poi_highlight: proxy.categorySumHighlight,
+            };
+        }
+        if (fetchActionTypeStats) {
+            result.actionTypeCount = {
+                skip: proxy.typeSumSkip,
+                mute: proxy.typeSumMute,
+            };
+        }
+        return result;
+    } catch (err) {
+        Logger.error(err as string);
+        return null;
+    }
+}
+
+async function dbGetUsername(userID: HashedUserID) {
+    try {
+        const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
+        return row?.userName ?? userID;
+    } catch (err) {
+        return false;
+    }
+}
+
+export async function getUserStats(req: Request, res: Response): Promise<Response> {
+    const userID = req.query.userID as UserID;
+    const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID;
+    const fetchCategoryStats = req.query.fetchCategoryStats == "true";
+    const fetchActionTypeStats = req.query.fetchActionTypeStats == "true";
+
+    if (hashedUserID == undefined) {
+        //invalid request
+        return res.status(400).send("Invalid userID or publicUserID parameter");
+    }
+    const segmentSummary = await dbGetUserSummary(hashedUserID, fetchCategoryStats, fetchActionTypeStats);
+    const responseObj = {
+        userID: hashedUserID,
+        userName: await dbGetUsername(hashedUserID),
+        ...segmentSummary,
+    } as Record<string, nestedObj | string>;
+    return res.send(responseObj);
+}

src/routes/postClearCache.ts

@@ -6,11 +6,12 @@ import { Service, VideoID } from "../types/segments.model";
 import { QueryCacher } from "../utils/queryCacher";
 import { isUserVIP } from "../utils/isUserVIP";
 import { VideoIDHash } from "../types/segments.model";
+import { getService } from "../utils/getService";
 
 export async function postClearCache(req: Request, res: Response): Promise<Response> {
     const videoID = req.query.videoID as VideoID;
     const userID = req.query.userID as UserID;
-    const service = req.query.service as Service ?? Service.YouTube;
+    const service = getService(req.query.service as Service);
 
     const invalidFields = [];
     if (typeof videoID !== "string") {

src/routes/postSkipSegments.ts

@@ -19,6 +19,7 @@ import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
 import { UserID } from "../types/user.model";
 import { isUserVIP } from "../utils/isUserVIP";
 import { parseUserAgent } from "../utils/userAgent";
+import { getService } from "../utils/getService";
 
 type CheckResult = {
     pass: boolean,
@@ -349,7 +350,7 @@ function checkInvalidFields(videoID: any, userID: any, segments: Array<any>): Ch
 }
 
 async function checkEachSegmentValid(userID: string, videoID: VideoID,
-    segments: Array<any>, service: string, isVIP: boolean, lockedCategoryList: Array<any>): Promise<CheckResult> {
+    segments: IncomingSegment[], service: string, isVIP: boolean, lockedCategoryList: Array<any>): Promise<CheckResult> {
 
     for (let i = 0; i < segments.length; i++) {
         if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@@ -406,7 +407,7 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID,
 
         //check if this info has already been submitted before
         const duplicateCheck2Row = await db.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ?
-            and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]);
+            and "endTime" = ? and "category" = ? and "actionType" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, segments[i].actionType, videoID, service]);
         if (duplicateCheck2Row.count > 0) {
             return { pass: false, errorMessage: "Sponsors has already been submitted before.", errorCode: 409};
         }
@@ -545,10 +546,7 @@ function proxySubmission(req: Request) {
 function preprocessInput(req: Request) {
     const videoID = req.query.videoID || req.body.videoID;
     const userID = req.query.userID || req.body.userID;
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val === service)) {
-        service = Service.YouTube;
-    }
     const videoDurationParam: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration;
     const videoDuration = videoDurationParam;
 

src/utils/getService.ts

@@ -0,0 +1,16 @@
+import { Service } from "../types/segments.model";
+
+export function getService<T extends string>(...value: T[]): Service {
+    for (const name of value) {
+        if (name) {
+            const service = Object.values(Service).find(
+                (val) => val.toLowerCase() === name.trim().toLowerCase()
+            );
+            if (service) {
+                return service;
+            }
+        }
+    }
+
+    return Service.YouTube;
+}

src/utils/getSubmissionUUID.ts

@@ -4,5 +4,5 @@ import { ActionType, VideoID } from "../types/segments.model";
 import { UserID } from "../types/user.model";
 
 export function getSubmissionUUID(videoID: VideoID, actionType: ActionType, userID: UserID, startTime: number, endTime: number): HashedValue{
-    return `3${getHash(`v3${videoID}${startTime}${endTime}${userID}`, 1)}` as HashedValue;
+    return `4${getHash(`${videoID}${startTime}${endTime}${userID}${actionType}`, 1)}` as HashedValue;
 }

test/cases/getService.ts

@@ -0,0 +1,29 @@
+import { getService } from "../../src/utils/getService";
+import { Service } from "../../src/types/segments.model";
+
+import assert from "assert";
+
+describe("getService", () => {
+    it("Should return youtube if not match", () => {
+        assert.strictEqual(getService(), Service.YouTube);
+        assert.strictEqual(getService(""), Service.YouTube);
+        assert.strictEqual(getService("test", "not exist"), Service.YouTube);
+        assert.strictEqual(getService(null, null), Service.YouTube);
+        assert.strictEqual(getService(undefined, undefined), Service.YouTube);
+        assert.strictEqual(getService(undefined), Service.YouTube);
+    });
+
+    it("Should return Youtube", () => {
+        assert.strictEqual(getService("youtube"), Service.YouTube);
+        assert.strictEqual(getService("   Youtube   "), Service.YouTube);
+        assert.strictEqual(getService("   YouTube   "), Service.YouTube);
+        assert.strictEqual(getService(undefined, "   YouTube   "), Service.YouTube);
+    });
+
+    it("Should return PeerTube", () => {
+        assert.strictEqual(getService("PeerTube"), Service.PeerTube);
+        assert.strictEqual(getService("   PeerTube   "), Service.PeerTube);
+        assert.strictEqual(getService("   peertube   "), Service.PeerTube);
+        assert.strictEqual(getService(undefined, "   PeerTube   "), Service.PeerTube);
+    });
+});

test/cases/getSubmissionUUID.ts

@@ -5,6 +5,6 @@ import { UserID } from "../../src/types/user.model";
 
 describe("getSubmissionUUID", () => {
     it("Should return the hashed value", () => {
-        assert.strictEqual(getSubmissionUUID("video001" as VideoID, "skip" as ActionType, "testuser001" as UserID, 13.33337, 42.000001), "3572aa64e0a2d6352c3de14ca45f8a83d193c32635669a7ae0b40c9eb36395872");
+        assert.strictEqual(getSubmissionUUID("video001" as VideoID, "skip" as ActionType, "testuser001" as UserID, 13.33337, 42.000001), "48ad47e445e67a7b963d9200037b36ec706eddcb752fdadc7bb2f061b56be6a23");
     });
 });