From 0db3240f58a589d25e288e42963b6e24d41d6d27 Mon Sep 17 00:00:00 2001 From: Michael C Date: Mon, 27 Dec 2021 04:48:07 -0500 Subject: [PATCH 1/5] add getTopCategoryUsers --- src/app.ts | 3 ++ src/routes/getTopCategoryUsers.ts | 67 +++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 src/routes/getTopCategoryUsers.ts diff --git a/src/app.ts b/src/app.ts index 23ed171..04415ba 100644 --- a/src/app.ts +++ b/src/app.ts @@ -46,6 +46,8 @@ import { getChapterNames } from "./routes/getChapterNames"; import { postRating } from "./routes/ratings/postRating"; import { getRating } from "./routes/ratings/getRating"; import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache"; +import { getTopCategoryUsers } from "./routes/getTopCategoryUsers"; +import path from "path"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -128,6 +130,7 @@ function setupRoutes(router: Router) { router.get("/api/getSavedTimeForUser", getSavedTimeForUser); router.get("/api/getTopUsers", getTopUsers); + router.get("/api/getTopCategoryUsers", getTopCategoryUsers); //send out totals //send the total submissions, total views and total minutes saved diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts new file mode 100644 index 0000000..5ea135b --- /dev/null +++ b/src/routes/getTopCategoryUsers.ts @@ -0,0 +1,67 @@ +import { db } from "../databases/databases"; +import { createMemoryCache } from "../utils/createMemoryCache"; +import { config } from "../config"; +import { Request, Response } from "express"; + +const MILLISECONDS_IN_MINUTE = 60000; +const getTopCategoryUsersWithCache = createMemoryCache(generateTopCategoryUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); +const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400; + +async function generateTopCategoryUsersStats(sortBy: string, category: string) { + + const userNames = []; + const viewCounts = []; + const totalSubmissions = []; + const minutesSaved = []; + + const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", + SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved", + SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" + LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" + WHERE category = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL + GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 + ORDER BY "${sortBy}" DESC LIMIT 100`, [category, maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]); + + for (let i = 0; i < rows.length; i++) { + userNames[i] = rows[i].userName; + + viewCounts[i] = rows[i].viewCount; + totalSubmissions[i] = rows[i].totalSubmissions; + minutesSaved[i] = rows[i].minutesSaved; + } + + return { + userNames, + viewCounts, + totalSubmissions, + minutesSaved + }; +} + +export async function getTopCategoryUsers(req: Request, res: Response): Promise { + const sortType = parseInt(req.query.sortType as string); + const category = req.query.category as string; + + if (sortType == undefined || config.categoryList.includes(category) ) { + //invalid request + return res.sendStatus(400); + } + + //setup which sort type to use + let sortBy = ""; + if (sortType == 0) { + sortBy = "minutesSaved"; + } else if (sortType == 1) { + sortBy = "viewCount"; + } else if (sortType == 2) { + sortBy = "totalSubmissions"; + } else { + //invalid request + return res.sendStatus(400); + } + + const stats = await getTopCategoryUsersWithCache(sortBy, category); + + //send this result + return res.send(stats); +} From 90f891aee4dd476fe9ab3db481af5481928869ed Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 30 Dec 2021 03:13:55 -0500 Subject: [PATCH 2/5] arr.push instead of add at index --- src/routes/getTopCategoryUsers.ts | 21 ++++++++++++-------- src/routes/getTopUsers.ts | 33 +++++++++++++++---------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index 5ea135b..7f95a24 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -7,14 +7,20 @@ const MILLISECONDS_IN_MINUTE = 60000; const getTopCategoryUsersWithCache = createMemoryCache(generateTopCategoryUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400; -async function generateTopCategoryUsersStats(sortBy: string, category: string) { +interface DBSegment { + userName: string, + viewCount: number, + totalSubmissions: number, + minutesSaved: number, +} +async function generateTopCategoryUsersStats(sortBy: string, category: string) { const userNames = []; const viewCounts = []; const totalSubmissions = []; const minutesSaved = []; - const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", + const rows: DBSegment[] = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved", SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" @@ -22,12 +28,11 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, [category, maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]); - for (let i = 0; i < rows.length; i++) { - userNames[i] = rows[i].userName; - - viewCounts[i] = rows[i].viewCount; - totalSubmissions[i] = rows[i].totalSubmissions; - minutesSaved[i] = rows[i].minutesSaved; + for (const row of rows) { + userNames.push(row.userName); + viewCounts.push(row.viewCount); + totalSubmissions.push(row.totalSubmissions); + minutesSaved.push(row.minutesSaved); } return { diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index bc2b67b..7b287aa 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -36,24 +36,23 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]); - for (let i = 0; i < rows.length; i++) { - userNames[i] = rows[i].userName; - - viewCounts[i] = rows[i].viewCount; - totalSubmissions[i] = rows[i].totalSubmissions; - minutesSaved[i] = rows[i].minutesSaved; + for (const row of rows) { + userNames.push(row.userName); + viewCounts.push(row.viewCount); + totalSubmissions.push(row.totalSubmissions); + minutesSaved.push(row.minutesSaved); if (categoryStatsEnabled) { - categoryStats[i] = [ - rows[i].categorySumSponsor, - rows[i].categorySumIntro, - rows[i].categorySumOutro, - rows[i].categorySumInteraction, - rows[i].categorySumSelfpromo, - rows[i].categorySumMusicOfftopic, - rows[i].categorySumPreview, - rows[i].categorySumHighlight, - rows[i].categorySumFiller - ]; + categoryStats.push([ + row.categorySumSponsor, + row.categorySumIntro, + row.categorySumOutro, + row.categorySumInteraction, + row.categorySumSelfpromo, + row.categorySumMusicOfftopic, + row.categorySumPreview, + row.categorySumHighlight, + row.categorySumFiller, + ]); } } From 98994cee01b2202d752543f03c5410064874c5ff Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 30 Dec 2021 04:07:21 -0500 Subject: [PATCH 3/5] add getTopUsers test --- test/cases/getTopUsers.ts | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 test/cases/getTopUsers.ts diff --git a/test/cases/getTopUsers.ts b/test/cases/getTopUsers.ts new file mode 100644 index 0000000..8808910 --- /dev/null +++ b/test/cases/getTopUsers.ts @@ -0,0 +1,75 @@ +import { db } from "../../src/databases/databases"; +import { getHash } from "../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../utils/httpClient"; + +const generateSegment = (userid: string, category: string) => ["getTopUsers", 0, 60, 50, `getTopUserUUID_${category}`, getHash(userid), 1, 1, category, 0]; + +describe("getTopUsers", () => { + const endpoint = "/api/getTopUsers"; + const user1 = "gettop_1"; + const user2 = "gettop_2"; + before(async () => { + const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)'; + await db.prepare("run", insertUserNameQuery, [getHash(user1), user1]); + await db.prepare("run", insertUserNameQuery, [getHash(user2), user2]); + + const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "sponsor")); + await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "selfpromo")); + await db.prepare("run", sponsorTimesQuery, generateSegment(user2, "interaction")); + }); + + it("Should return 400 if no sortType", (done) => { + client.get(endpoint) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return 400 if invalid sortType provided", (done) => { + client.get(endpoint, { params: { sortType: "a" } }) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to get by all sortTypes", (done) => { + client.get(endpoint, { params: { sortType: 0 } })// minutesSaved + .then(res => { + // make sure that user1 is before user2 + assert.strictEqual(res.status, 200); + assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`); + }) + .catch(err => done(err)); + client.get(endpoint, { params: { sortType: 1 } }) // viewCount + .then(res => { + // make sure that user1 is before user2 + assert.strictEqual(res.status, 200); + assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`); + }) + .catch(err => done(err)); + client.get(endpoint, { params: { sortType: 2 } }) // totalSubmissions + .then(res => { + // make sure that user1 is before user2 + assert.strictEqual(res.status, 200); + assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`); + }) + .catch(err => done(err)); + done(); + }); + + it("Should be able to get - with categoryStats", (done) => { + client.get(endpoint, { params: { sortType: 0, categoryStats: true } }) + .then(res => { + assert.strictEqual(res.status, 200); + assert.ok(res.data.categoryStats[0].length > 1); + done(); + }) + .catch(err => done(err)); + }); +}); From f97af4c433048eba8e931818a78617ed54d7f484 Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 30 Dec 2021 04:07:38 -0500 Subject: [PATCH 4/5] texts for getTopCategoryUsers --- src/routes/getTopCategoryUsers.ts | 6 +- test/cases/getTopCategoryUsers.ts | 124 ++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 test/cases/getTopCategoryUsers.ts diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index 7f95a24..f22fbc1 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -24,9 +24,9 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved", SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" - WHERE category = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL + WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 - ORDER BY "${sortBy}" DESC LIMIT 100`, [category, maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]); + ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]); for (const row of rows) { userNames.push(row.userName); @@ -47,7 +47,7 @@ export async function getTopCategoryUsers(req: Request, res: Response): Promise< const sortType = parseInt(req.query.sortType as string); const category = req.query.category as string; - if (sortType == undefined || config.categoryList.includes(category) ) { + if (sortType == undefined || !config.categoryList.includes(category) ) { //invalid request return res.sendStatus(400); } diff --git a/test/cases/getTopCategoryUsers.ts b/test/cases/getTopCategoryUsers.ts new file mode 100644 index 0000000..f321373 --- /dev/null +++ b/test/cases/getTopCategoryUsers.ts @@ -0,0 +1,124 @@ +import { db } from "../../src/databases/databases"; +import { getHash } from "../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../utils/httpClient"; + +const generateSegment = (userid: string, category: string) => ["getTopCategory", 0, 60, 50, `getTopCategoryUUID_${category}`, getHash(userid), 1, 1, category, 0]; + +describe("getTopCategoryUsers", () => { + const endpoint = "/api/getTopCategoryUsers"; + const user1 = "gettopcategory_1"; + const user2 = "gettopcategory_2"; + before(async () => { + const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)'; + await db.prepare("run", insertUserNameQuery, [getHash(user1), user1]); + await db.prepare("run", insertUserNameQuery, [getHash(user2), user2]); + + const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "sponsor")); + await db.prepare("run", sponsorTimesQuery, generateSegment(user1, "selfpromo")); + await db.prepare("run", sponsorTimesQuery, generateSegment(user2, "interaction")); + }); + + it("Should return 400 if no sortType", (done) => { + client.get(endpoint) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return 400 if invalid sortType provided", (done) => { + client.get(endpoint, { params: { sortType: "a" } }) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return 400 if invalid category provided", (done) => { + client.get(endpoint, { params: { sortType: 1, category: "never_valid_category" } }) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to get by all sortTypes", (done) => { + client.get(endpoint, { params: { category: "sponsor", sortType: 0 } })// minutesSaved + .then(res => { + assert.strictEqual(res.status, 200); + }) + .catch(err => done(err)); + client.get(endpoint, { params: { category: "sponsor", sortType: 1 } }) // viewCount + .then(res => { + assert.strictEqual(res.status, 200); + }) + .catch(err => done(err)); + client.get(endpoint, { params: { category: "sponsor", sortType: 2 } }) // totalSubmissions + .then(res => { + assert.strictEqual(res.status, 200); + }) + .catch(err => done(err)); + done(); + }); + + it("Should return accurate sponsor data", (done) => { + client.get(endpoint, { params: { sortType: 1, category: "sponsor" } }) + .then(res => { + assert.strictEqual(res.status, 200); + assert.ok(!res.data.userNames.includes(user2), "User 2 should not be present"); + const user1idx = res.data.userNames.indexOf(user1); + assert.ok(user1idx > -1, "User 1 should be present"); + assert.strictEqual(res.data.viewCounts[user1idx], 1, "User should have 1 view"); + assert.strictEqual(res.data.totalSubmissions[user1idx], 1, "User should have 1 submission"); + assert.strictEqual(res.data.minutesSaved[user1idx], 1, "User should have 1 minutes saved"); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return accurate selfpromo data", (done) => { + client.get(endpoint, { params: { sortType: 1, category: "selfpromo" } }) + .then(res => { + assert.strictEqual(res.status, 200); + assert.ok(!res.data.userNames.includes(user2), "User 2 should not be present"); + const user1idx = res.data.userNames.indexOf(user1); + assert.ok(user1idx > -1, "User 1 should be present"); + assert.strictEqual(res.data.viewCounts[user1idx], 1, "User should have 1 view"); + assert.strictEqual(res.data.totalSubmissions[user1idx], 1, "User should have 1 submission"); + assert.strictEqual(res.data.minutesSaved[user1idx], 1, "User should have 1 minutes saved"); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return accurate interaction data", (done) => { + client.get(endpoint, { params: { sortType: 1, category: "interaction" } }) + .then(res => { + assert.strictEqual(res.status, 200); + assert.ok(!res.data.userNames.includes(user1), "User 1 should not be present"); + const user1idx = res.data.userNames.indexOf(user2); + assert.ok(user1idx > -1, "User 2 should be present"); + assert.strictEqual(res.data.viewCounts[user1idx], 1, "User should have 1 view"); + assert.strictEqual(res.data.totalSubmissions[user1idx], 1, "User should have 1 submission"); + assert.strictEqual(res.data.minutesSaved[user1idx], 1, "User should have 1 minutes saved"); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return accurate outro data", (done) => { + client.get(endpoint, { params: { sortType: 1, category: "outro" } }) + .then(res => { + assert.strictEqual(res.status, 200); + assert.ok(!res.data.userNames.includes(user1), "User 1 should not be present"); + assert.ok(!res.data.userNames.includes(user2), "User 2 should not be present"); + done(); + }) + .catch(err => done(err)); + }); +}); From 42624a77828be40f979a66c8d59a8a9cdf47ceba Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 30 Dec 2021 04:09:46 -0500 Subject: [PATCH 5/5] minor eslint fixes --- src/app.ts | 1 - src/routes/getSkipSegments.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index 04415ba..a959bb7 100644 --- a/src/app.ts +++ b/src/app.ts @@ -47,7 +47,6 @@ import { postRating } from "./routes/ratings/postRating"; import { getRating } from "./routes/ratings/getRating"; import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache"; import { getTopCategoryUsers } from "./routes/getTopCategoryUsers"; -import path from "path"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 7850661..8dffd9d 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -1,5 +1,5 @@ import { Request, Response } from "express"; -import { partition } from "lodash" +import { partition } from "lodash"; import { config } from "../config"; import { db, privateDB } from "../databases/databases"; import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey } from "../utils/redisKeys";