Merge pull request #435 from mchangrh/categoryLeaderboards

add getTopCategoryUsers

src/app.ts

@@ -45,7 +45,8 @@ import { youtubeApiProxy } from "./routes/youtubeApiProxy";
 import { getChapterNames } from "./routes/getChapterNames";
 import { postRating } from "./routes/ratings/postRating";
 import { getRating } from "./routes/ratings/getRating";
-import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache";
+import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache"
+import { getTopCategoryUsers } from "./routes/getTopCategoryUsers";
 import { addUserAsTempVIP } from "./routes/addUserAsTempVIP";
 
 export function createServer(callback: () => void): Server {
@@ -131,6 +132,7 @@ function setupRoutes(router: Router) {
     router.get("/api/getSavedTimeForUser", getSavedTimeForUser);
 
     router.get("/api/getTopUsers", getTopUsers);
+    router.get("/api/getTopCategoryUsers", getTopCategoryUsers);
 
     //send out totals
     //send the total submissions, total views and total minutes saved

src/routes/getSkipSegments.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from "express";
-import { partition } from "lodash"
+import { partition } from "lodash";
 import { config } from "../config";
 import { db, privateDB } from "../databases/databases";
 import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey } from "../utils/redisKeys";

src/routes/getTopCategoryUsers.ts

@@ -0,0 +1,72 @@
+import { db } from "../databases/databases";
+import { createMemoryCache } from "../utils/createMemoryCache";
+import { config } from "../config";
+import { Request, Response } from "express";
+
+const MILLISECONDS_IN_MINUTE = 60000;
+const getTopCategoryUsersWithCache = createMemoryCache(generateTopCategoryUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
+const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
+
+interface DBSegment {
+    userName: string,
+    viewCount: number,
+    totalSubmissions: number,
+    minutesSaved: number,
+}
+
+async function generateTopCategoryUsersStats(sortBy: string, category: string) {
+    const userNames = [];
+    const viewCounts = [];
+    const totalSubmissions = [];
+    const minutesSaved = [];
+
+    const rows: DBSegment[] = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
+        SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
+        SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
+        LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
+        WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
+        GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
+        ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
+
+    for (const row of rows) {
+        userNames.push(row.userName);
+        viewCounts.push(row.viewCount);
+        totalSubmissions.push(row.totalSubmissions);
+        minutesSaved.push(row.minutesSaved);
+    }
+
+    return {
+        userNames,
+        viewCounts,
+        totalSubmissions,
+        minutesSaved
+    };
+}
+
+export async function getTopCategoryUsers(req: Request, res: Response): Promise<Response> {
+    const sortType = parseInt(req.query.sortType as string);
+    const category = req.query.category as string;
+
+    if (sortType == undefined || !config.categoryList.includes(category) ) {
+        //invalid request
+        return res.sendStatus(400);
+    }
+
+    //setup which sort type to use
+    let sortBy = "";
+    if (sortType == 0) {
+        sortBy = "minutesSaved";
+    } else if (sortType == 1) {
+        sortBy = "viewCount";
+    } else if (sortType == 2) {
+        sortBy = "totalSubmissions";
+    } else {
+        //invalid request
+        return res.sendStatus(400);
+    }
+
+    const stats = await getTopCategoryUsersWithCache(sortBy, category);
+
+    //send this result
+    return res.send(stats);
+}

src/routes/getTopUsers.ts

@@ -36,24 +36,23 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
         GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
         ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]);
 
-    for (let i = 0; i < rows.length; i++) {
+    for (const row of rows) {
-        userNames[i] = rows[i].userName;
+        userNames.push(row.userName);
-
+        viewCounts.push(row.viewCount);
-        viewCounts[i] = rows[i].viewCount;
+        totalSubmissions.push(row.totalSubmissions);
-        totalSubmissions[i] = rows[i].totalSubmissions;
+        minutesSaved.push(row.minutesSaved);
-        minutesSaved[i] = rows[i].minutesSaved;
         if (categoryStatsEnabled) {
-            categoryStats[i] = [
+            categoryStats.push([
-                rows[i].categorySumSponsor,
+                row.categorySumSponsor,
-                rows[i].categorySumIntro,
+                row.categorySumIntro,
-                rows[i].categorySumOutro,
+                row.categorySumOutro,
-                rows[i].categorySumInteraction,
+                row.categorySumInteraction,
-                rows[i].categorySumSelfpromo,
+                row.categorySumSelfpromo,
-                rows[i].categorySumMusicOfftopic,
+                row.categorySumMusicOfftopic,
-                rows[i].categorySumPreview,
+                row.categorySumPreview,
-                rows[i].categorySumHighlight,
+                row.categorySumHighlight,
-                rows[i].categorySumFiller
+                row.categorySumFiller,
-            ];
+            ]);
         }
     }
 

test/cases/getTopCategoryUsers.ts

@@ -0,0 +1,124 @@
+import { db } from "../../src/databases/databases";
+import { getHash } from "../../src/utils/getHash";
+import assert from "assert";
+import { client } from "../utils/httpClient";
+
+const generateSegment = (userid: string, category: string) => ["getTopCategory", 0, 60, 50, `getTopCategoryUUID_${category}`, getHash(userid), 1, 1, category, 0];
+
+describe("getTopCategoryUsers", () => {
+    const endpoint = "/api/getTopCategoryUsers";
+    const user1 = "gettopcategory_1";
+    const user2 = "gettopcategory_2";
+    before(async () => {
+        const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)';
+        await db.prepare("run", insertUserNameQuery, [getHash(user1), user1]);
+        await db.prepare("run", insertUserNameQuery, [getHash(user2), user2]);
+
+        const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "sponsor"));
+        await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "selfpromo"));
+        await db.prepare("run", sponsorTimesQuery, generateSegment(user2, "interaction"));
+    });
+
+    it("Should return 400 if no sortType", (done) => {
+        client.get(endpoint)
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 if invalid sortType provided", (done) => {
+        client.get(endpoint, { params: { sortType: "a" } })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 if invalid category provided", (done) => {
+        client.get(endpoint, { params: { sortType: 1, category: "never_valid_category" } })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to get by all sortTypes", (done) => {
+        client.get(endpoint, { params: {  category: "sponsor", sortType: 0 } })// minutesSaved
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+            })
+            .catch(err => done(err));
+        client.get(endpoint, { params: { category: "sponsor", sortType: 1 } }) // viewCount
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+            })
+            .catch(err => done(err));
+        client.get(endpoint, { params: { category: "sponsor", sortType: 2 } }) // totalSubmissions
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+            })
+            .catch(err => done(err));
+        done();
+    });
+
+    it("Should return accurate sponsor data", (done) => {
+        client.get(endpoint, { params: { sortType: 1, category: "sponsor" } })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(!res.data.userNames.includes(user2), "User 2 should not be present");
+                const user1idx = res.data.userNames.indexOf(user1);
+                assert.ok(user1idx > -1, "User 1 should be present");
+                assert.strictEqual(res.data.viewCounts[user1idx], 1, "User should have 1 view");
+                assert.strictEqual(res.data.totalSubmissions[user1idx], 1, "User should have 1 submission");
+                assert.strictEqual(res.data.minutesSaved[user1idx], 1, "User should have 1 minutes saved");
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return accurate selfpromo data", (done) => {
+        client.get(endpoint, { params: { sortType: 1, category: "selfpromo" } })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(!res.data.userNames.includes(user2), "User 2 should not be present");
+                const user1idx = res.data.userNames.indexOf(user1);
+                assert.ok(user1idx > -1, "User 1 should be present");
+                assert.strictEqual(res.data.viewCounts[user1idx], 1, "User should have 1 view");
+                assert.strictEqual(res.data.totalSubmissions[user1idx], 1, "User should have 1 submission");
+                assert.strictEqual(res.data.minutesSaved[user1idx], 1, "User should have 1 minutes saved");
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return accurate interaction data", (done) => {
+        client.get(endpoint, { params: { sortType: 1, category: "interaction" } })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(!res.data.userNames.includes(user1), "User 1 should not be present");
+                const user1idx = res.data.userNames.indexOf(user2);
+                assert.ok(user1idx > -1, "User 2 should be present");
+                assert.strictEqual(res.data.viewCounts[user1idx], 1, "User should have 1 view");
+                assert.strictEqual(res.data.totalSubmissions[user1idx], 1, "User should have 1 submission");
+                assert.strictEqual(res.data.minutesSaved[user1idx], 1, "User should have 1 minutes saved");
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return accurate outro data", (done) => {
+        client.get(endpoint, { params: { sortType: 1, category: "outro" } })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(!res.data.userNames.includes(user1), "User 1 should not be present");
+                assert.ok(!res.data.userNames.includes(user2), "User 2 should not be present");
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/getTopUsers.ts

@@ -0,0 +1,75 @@
+import { db } from "../../src/databases/databases";
+import { getHash } from "../../src/utils/getHash";
+import assert from "assert";
+import { client } from "../utils/httpClient";
+
+const generateSegment = (userid: string, category: string) => ["getTopUsers", 0, 60, 50, `getTopUserUUID_${category}`, getHash(userid), 1, 1, category, 0];
+
+describe("getTopUsers", () => {
+    const endpoint = "/api/getTopUsers";
+    const user1 = "gettop_1";
+    const user2 = "gettop_2";
+    before(async () => {
+        const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)';
+        await db.prepare("run", insertUserNameQuery, [getHash(user1), user1]);
+        await db.prepare("run", insertUserNameQuery, [getHash(user2), user2]);
+
+        const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "sponsor"));
+        await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "selfpromo"));
+        await db.prepare("run", sponsorTimesQuery, generateSegment(user2, "interaction"));
+    });
+
+    it("Should return 400 if no sortType", (done) => {
+        client.get(endpoint)
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 if invalid sortType provided", (done) => {
+        client.get(endpoint, { params: { sortType: "a" } })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to get by all sortTypes", (done) => {
+        client.get(endpoint, { params: { sortType: 0 } })// minutesSaved
+            .then(res => {
+                // make sure that user1 is before user2
+                assert.strictEqual(res.status, 200);
+                assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`);
+            })
+            .catch(err => done(err));
+        client.get(endpoint, { params: { sortType: 1 } }) // viewCount
+            .then(res => {
+                // make sure that user1 is before user2
+                assert.strictEqual(res.status, 200);
+                assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`);
+            })
+            .catch(err => done(err));
+        client.get(endpoint, { params: { sortType: 2 } }) // totalSubmissions
+            .then(res => {
+                // make sure that user1 is before user2
+                assert.strictEqual(res.status, 200);
+                assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`);
+            })
+            .catch(err => done(err));
+        done();
+    });
+
+    it("Should be able to get - with categoryStats", (done) => {
+        client.get(endpoint, { params: { sortType: 0, categoryStats: true } })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(res.data.categoryStats[0].length > 1);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});