Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into getLockReason

src/app.ts

@@ -38,6 +38,7 @@ import {getLockCategoriesByHash} from "./routes/getLockCategoriesByHash";
 import {endpoint as getSearchSegments } from "./routes/getSearchSegments";
 import {getStatus } from "./routes/getStatus";
 import { getLockReason } from "./routes/getLockReason";
+import {getUserStats} from "./routes/getUserStats";
 import ExpressPromiseRouter from "express-promise-router";
 import { Server } from "http";
 import { youtubeApiProxy } from "./routes/youtubeApiProxy";
@@ -176,6 +177,8 @@ function setupRoutes(router: Router) {
     router.get("/api/status", getStatus);
 
     router.get("/api/youtubeApiProxy", youtubeApiProxy);
+    // get user category stats
+    router.get("/api/userStats", getUserStats);
 
     router.get("/api/lockReason", getLockReason);
 

src/routes/getSearchSegments.ts

@@ -1,6 +1,7 @@
 import { Request, Response } from "express";
 import { db } from "../databases/databases";
 import { ActionType, Category, DBSegment, Service, VideoID } from "../types/segments.model";
+import { getService } from "../utils/getService";
 const segmentsPerPage = 10;
 
 type searchSegmentResponse = {
@@ -59,10 +60,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<searchSeg
         return false;
     }
 
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val == service)) {
-        service = Service.YouTube;
-    }
 
     let page: number = req.query.page ?? req.body.page ?? 0;
     page = Number(page);

src/routes/getSkipSegments.ts

@@ -10,6 +10,7 @@ import { getIP } from "../utils/getIP";
 import { Logger } from "../utils/logger";
 import { QueryCacher } from "../utils/queryCacher";
 import { getReputation } from "../utils/reputation";
+import { getService } from "../utils/getService";
 
 
 async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
@@ -317,10 +318,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
         return false;
     }
 
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val == service)) {
-        service = Service.YouTube;
-    }
 
     const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
 

src/routes/getSkipSegmentsByHash.ts

@@ -2,6 +2,7 @@ import {hashPrefixTester} from "../utils/hashPrefixTester";
 import {getSegmentsByHash} from "./getSkipSegments";
 import {Request, Response} from "express";
 import { ActionType, Category, SegmentUUID, Service, VideoIDHash } from "../types/segments.model";
+import { getService } from "../utils/getService";
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
     let hashPrefix = req.params.prefix as VideoIDHash;
@@ -58,10 +59,7 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
         return res.status(400).send("Bad parameter: requiredSegments (invalid JSON)");
     }
 
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val == service)) {
-        service = Service.YouTube;
-    }
 
     // filter out none string elements, only flat array with strings is valid
     categories = categories.filter((item: any) => typeof item === "string");

src/routes/getUserInfo.ts

@@ -43,12 +43,7 @@ async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<number> {
 async function dbGetUsername(userID: HashedUserID) {
     try {
         const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
-        if (row !== undefined) {
+        return row?.userName ?? userID;
-            return row.userName;
-        } else {
-            //no username yet, just send back the userID
-            return userID;
-        }
     } catch (err) {
         return false;
     }
@@ -172,7 +167,7 @@ async function getUserInfo(req: Request, res: Response): Promise<Response> {
         responseObj[property] = await dbGetValue(hashedUserID, property);
     }
     // add minutesSaved and segmentCount after to avoid getting overwritten
-    if (paramValues.includes("minutesSaved")) { responseObj["minutesSaved"] = segmentsSummary.minutesSaved; }
+    if (paramValues.includes("minutesSaved")) responseObj["minutesSaved"] = segmentsSummary.minutesSaved;
     if (paramValues.includes("segmentCount")) responseObj["segmentCount"] = segmentsSummary.segmentCount;
     return res.send(responseObj);
 }

src/routes/getUserStats.ts

@@ -0,0 +1,96 @@
+import {db} from "../databases/databases";
+import {getHash} from "../utils/getHash";
+import {Request, Response} from "express";
+import {HashedUserID, UserID} from "../types/user.model";
+import {config} from "../config";
+import { Logger } from "../utils/logger";
+type nestedObj = Record<string, Record<string, number>>;
+const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
+
+async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolean, fetchActionTypeStats: boolean) {
+    let additionalQuery = "";
+    if (fetchCategoryStats) {
+        additionalQuery += `
+            SUM(CASE WHEN "category" = 'sponsor' THEN 1 ELSE 0 END) as "categorySumSponsor",
+            SUM(CASE WHEN "category" = 'intro' THEN 1 ELSE 0 END) as "categorySumIntro",
+            SUM(CASE WHEN "category" = 'outro' THEN 1 ELSE 0 END) as "categorySumOutro",
+            SUM(CASE WHEN "category" = 'interaction' THEN 1 ELSE 0 END) as "categorySumInteraction",
+            SUM(CASE WHEN "category" = 'selfpromo' THEN 1 ELSE 0 END) as "categorySelfpromo",
+            SUM(CASE WHEN "category" = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic",
+            SUM(CASE WHEN "category" = 'preview' THEN 1 ELSE 0 END) as "categorySumPreview",
+            SUM(CASE WHEN "category" = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight",`;
+    }
+    if (fetchActionTypeStats) {
+        additionalQuery += `
+            SUM(CASE WHEN "actionType" = 'skip' THEN 1 ELSE 0 END) as "typeSumSkip",
+            SUM(CASE WHEN "actionType" = 'mute' THEN 1 ELSE 0 END) as "typeSumMute",`;
+    }
+    try {
+        const row = await db.prepare("get", `
+            SELECT SUM(((CASE WHEN "endTime" - "startTime" > ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved",
+            ${additionalQuery}
+            count(*) as "segmentCount"
+            FROM "sponsorTimes"
+            WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" !=1`,
+        [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, userID]);
+        const source = (row.minutesSaved != null) ? row : {};
+        const handler = { get: (target: Record<string, any>, name: string) => target?.[name] || 0 };
+        const proxy = new Proxy(source, handler);
+        const result = {} as nestedObj;
+
+        result.overallStats = {
+            minutesSaved: proxy.minutesSaved,
+            segmentCount: proxy.segmentCount,
+        };
+        if (fetchCategoryStats) {
+            result.categoryCount = {
+                sponsor: proxy.categorySumSponsor,
+                intro: proxy.categorySumIntro,
+                outro: proxy.categorySumOutro,
+                interaction: proxy.categorySumInteraction,
+                selfpromo: proxy.categorySelfpromo,
+                music_offtopic: proxy.categoryMusicOfftopic,
+                preview: proxy.categorySumPreview,
+                poi_highlight: proxy.categorySumHighlight,
+            };
+        }
+        if (fetchActionTypeStats) {
+            result.actionTypeCount = {
+                skip: proxy.typeSumSkip,
+                mute: proxy.typeSumMute,
+            };
+        }
+        return result;
+    } catch (err) {
+        Logger.error(err as string);
+        return null;
+    }
+}
+
+async function dbGetUsername(userID: HashedUserID) {
+    try {
+        const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
+        return row?.userName ?? userID;
+    } catch (err) {
+        return false;
+    }
+}
+
+export async function getUserStats(req: Request, res: Response): Promise<Response> {
+    const userID = req.query.userID as UserID;
+    const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID;
+    const fetchCategoryStats = req.query.fetchCategoryStats == "true";
+    const fetchActionTypeStats = req.query.fetchActionTypeStats == "true";
+
+    if (hashedUserID == undefined) {
+        //invalid request
+        return res.status(400).send("Invalid userID or publicUserID parameter");
+    }
+    const segmentSummary = await dbGetUserSummary(hashedUserID, fetchCategoryStats, fetchActionTypeStats);
+    const responseObj = {
+        userID: hashedUserID,
+        userName: await dbGetUsername(hashedUserID),
+        ...segmentSummary,
+    } as Record<string, nestedObj | string>;
+    return res.send(responseObj);
+}

src/routes/postClearCache.ts

@@ -6,11 +6,12 @@ import { Service, VideoID } from "../types/segments.model";
 import { QueryCacher } from "../utils/queryCacher";
 import { isUserVIP } from "../utils/isUserVIP";
 import { VideoIDHash } from "../types/segments.model";
+import { getService } from "../utils/getService";
 
 export async function postClearCache(req: Request, res: Response): Promise<Response> {
     const videoID = req.query.videoID as VideoID;
     const userID = req.query.userID as UserID;
-    const service = req.query.service as Service ?? Service.YouTube;
+    const service = getService(req.query.service as Service);
 
     const invalidFields = [];
     if (typeof videoID !== "string") {

src/routes/postSkipSegments.ts

@@ -19,6 +19,7 @@ import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
 import { UserID } from "../types/user.model";
 import { isUserVIP } from "../utils/isUserVIP";
 import { parseUserAgent } from "../utils/userAgent";
+import { getService } from "../utils/getService";
 
 type CheckResult = {
     pass: boolean,
@@ -545,10 +546,7 @@ function proxySubmission(req: Request) {
 function preprocessInput(req: Request) {
     const videoID = req.query.videoID || req.body.videoID;
     const userID = req.query.userID || req.body.userID;
-    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    const service = getService(req.query.service, req.body.service);
-    if (!Object.values(Service).some((val) => val === service)) {
-        service = Service.YouTube;
-    }
     const videoDurationParam: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration;
     const videoDuration = videoDurationParam;
 

src/utils/getService.ts

@@ -0,0 +1,16 @@
+import { Service } from "../types/segments.model";
+
+export function getService<T extends string>(...value: T[]): Service {
+    for (const name of value) {
+        if (name) {
+            const service = Object.values(Service).find(
+                (val) => val.toLowerCase() === name.trim().toLowerCase()
+            );
+            if (service) {
+                return service;
+            }
+        }
+    }
+
+    return Service.YouTube;
+}

test/cases/getService.ts

@@ -0,0 +1,29 @@
+import { getService } from "../../src/utils/getService";
+import { Service } from "../../src/types/segments.model";
+
+import assert from "assert";
+
+describe("getService", () => {
+    it("Should return youtube if not match", () => {
+        assert.strictEqual(getService(), Service.YouTube);
+        assert.strictEqual(getService(""), Service.YouTube);
+        assert.strictEqual(getService("test", "not exist"), Service.YouTube);
+        assert.strictEqual(getService(null, null), Service.YouTube);
+        assert.strictEqual(getService(undefined, undefined), Service.YouTube);
+        assert.strictEqual(getService(undefined), Service.YouTube);
+    });
+
+    it("Should return Youtube", () => {
+        assert.strictEqual(getService("youtube"), Service.YouTube);
+        assert.strictEqual(getService("   Youtube   "), Service.YouTube);
+        assert.strictEqual(getService("   YouTube   "), Service.YouTube);
+        assert.strictEqual(getService(undefined, "   YouTube   "), Service.YouTube);
+    });
+
+    it("Should return PeerTube", () => {
+        assert.strictEqual(getService("PeerTube"), Service.PeerTube);
+        assert.strictEqual(getService("   PeerTube   "), Service.PeerTube);
+        assert.strictEqual(getService("   peertube   "), Service.PeerTube);
+        assert.strictEqual(getService(undefined, "   PeerTube   "), Service.PeerTube);
+    });
+});

test/cases/getUserStats.ts

@@ -0,0 +1,124 @@
+import fetch from "node-fetch";
+import {Done, getbaseURL, partialDeepEquals} from "../utils";
+import {db} from "../../src/databases/databases";
+import {getHash} from "../../src/utils/getHash";
+import assert from "assert";
+
+describe("getUserStats", () => {
+    before(async () => {
+        const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)';
+        await db.prepare("run", insertUserNameQuery, [getHash("getuserstats_user_01"), "Username user 01"]);
+
+        const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid1", getHash("getuserstats_user_01"), 1, 1, "sponsor", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid2", getHash("getuserstats_user_01"), 2, 2, "selfpromo", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid3", getHash("getuserstats_user_01"), 3, 3, "interaction", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid4", getHash("getuserstats_user_01"), 4, 4, "intro", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid5", getHash("getuserstats_user_01"), 5, 5, "outro", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid6", getHash("getuserstats_user_01"), 6, 6, "preview", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "getuserstatsuuid7", getHash("getuserstats_user_01"), 7, 7, "music_offtopic", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 11, 11, 0, "getuserstatsuuid8", getHash("getuserstats_user_01"), 8, 8, "poi_highlight", 0]);
+        await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, -2, "getuserstatsuuid9", getHash("getuserstats_user_02"), 8, 2, "sponsor", 0]);
+
+    });
+
+    it("Should be able to get a 400 (No userID parameter)", (done: Done) => {
+        fetch(`${getbaseURL()}/api/userStats`)
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to get all user info", (done: Done) => {
+        fetch(`${getbaseURL()}/api/userStats?userID=getuserstats_user_01&fetchCategoryStats=true&fetchActionTypeStats=true`)
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const expected = {
+                    userName: "Username user 01",
+                    userID: getHash("getuserstats_user_01"),
+                    categoryCount: {
+                        sponsor: 1,
+                        selfpromo: 1,
+                        interaction: 1,
+                        intro: 1,
+                        outro: 1,
+                        preview: 1,
+                        music_offtopic: 1,
+                        poi_highlight: 1,
+                    },
+                    actionTypeCount: {
+                        mute: 0,
+                        skip: 8
+                    },
+                    overallStats: {
+                        minutesSaved: 28,
+                        segmentCount: 8
+                    }
+                };
+                const data = await res.json();
+                assert.ok(partialDeepEquals(data, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to get all zeroes for invalid userid", (done: Done) => {
+        fetch(`${getbaseURL()}/api/userStats?userID=getuserstats_user_invalid`)
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const data = await res.json();
+                for (const value in data.overallStats) {
+                    if (data[value]) {
+                        done(`returned non-zero for ${value}`);
+                    }
+                }
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to get all zeroes for only ignored segments", (done: Done) => {
+        fetch(`${getbaseURL()}/api/userStats?userID=getuserstats_user_02`)
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const data = await res.json();
+                for (const value in data.overallStats) {
+                    if (data[value]) {
+                        done(`returned non-zero for ${value}`);
+                    }
+                }
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not get extra stats if not requested", (done: Done) => {
+        fetch(`${getbaseURL()}/api/userStats?userID=getuserstats_user_01`)
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const data = await res.json();
+                // check for categoryCount
+                if (data.categoryCount || data.actionTypeCount) {
+                    done("returned extra stats");
+                }
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should get parts of extra stats if not requested", (done: Done) => {
+        fetch(`${getbaseURL()}/api/userStats?userID=getuserstats_user_01&fetchActionTypeStats=true`)
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const data = await res.json();
+                // check for categoryCount
+                if (data.categoryCount && !data.actionTypeCount) {
+                    done("returned extra stats");
+                }
+                done();
+            })
+            .catch(err => done(err));
+    });
+});