From b7995832bc4dfc68fc70fdb922a4c4d2e3ad59e1 Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 26 May 2022 21:19:27 -0400 Subject: [PATCH 1/7] add b2 syncing for sqlite base --- .github/workflows/generate-sqlite-base.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate-sqlite-base.yml b/.github/workflows/generate-sqlite-base.yml index 53f1703..97c5cb3 100644 --- a/.github/workflows/generate-sqlite-base.yml +++ b/.github/workflows/generate-sqlite-base.yml @@ -25,4 +25,13 @@ jobs: - uses: actions/upload-artifact@v2 with: name: SponsorTimesDB.db - path: databases/sponsorTimes.db \ No newline at end of file + path: databases/sponsorTimes.db + - uses: mchangrh/s3cmd-sync@v1 + with: + args: --acl-public + env: + S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} + S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + S3_ACCESS_KEY_SECRET: ${{ secrets.S3_ACCESS_KEY_SECRET }} + SOURCE_DIR: 'databases/sponsorTimes.db' \ No newline at end of file From 28448f99d939f4df21add97525dc7bf5f079a738 Mon Sep 17 00:00:00 2001 From: Ajay Date: Mon, 26 Sep 2022 15:17:38 -0400 Subject: [PATCH 2/7] Lock s3cmd-sync version --- .github/workflows/generate-sqlite-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-sqlite-base.yml b/.github/workflows/generate-sqlite-base.yml index 97c5cb3..6a4bf3f 100644 --- a/.github/workflows/generate-sqlite-base.yml +++ b/.github/workflows/generate-sqlite-base.yml @@ -26,7 +26,7 @@ jobs: with: name: SponsorTimesDB.db path: databases/sponsorTimes.db - - uses: mchangrh/s3cmd-sync@v1 + - uses: mchangrh/s3cmd-sync@f4f36b9705bdd9af7ac91964136989ac17e3b513 with: args: --acl-public env: From ea05284b1dc5ddf8fcb54ac9dfa58249d1a0d8eb Mon Sep 17 00:00:00 2001 From: Ajay Date: Mon, 26 Sep 2022 15:23:01 -0400 Subject: [PATCH 3/7] Allow manual dispatch of generate sqlite base --- .github/workflows/generate-sqlite-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/generate-sqlite-base.yml b/.github/workflows/generate-sqlite-base.yml index 6a4bf3f..c320e58 100644 --- a/.github/workflows/generate-sqlite-base.yml +++ b/.github/workflows/generate-sqlite-base.yml @@ -6,6 +6,7 @@ on: - master paths: - databases/** + workflow_dispatch: jobs: make-base-db: From 27f7b6d3c7fc2c72634f2608aeb1484d42655ba2 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Wed, 28 Sep 2022 17:43:04 +0200 Subject: [PATCH 4/7] Add chapter category & action type to getUserStats.ts This also fixes chapters not being counted in the segment counts. --- src/routes/getUserStats.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/routes/getUserStats.ts b/src/routes/getUserStats.ts index 8fca455..38c8764 100644 --- a/src/routes/getUserStats.ts +++ b/src/routes/getUserStats.ts @@ -21,6 +21,7 @@ async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolea SUM(CASE WHEN "category" = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight", SUM(CASE WHEN "category" = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller", SUM(CASE WHEN "category" = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess", + SUM(CASE WHEN "category" = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter", `; } if (fetchActionTypeStats) { @@ -29,15 +30,16 @@ async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolea SUM(CASE WHEN "actionType" = 'mute' THEN 1 ELSE 0 END) as "typeSumMute", SUM(CASE WHEN "actionType" = 'full' THEN 1 ELSE 0 END) as "typeSumFull", SUM(CASE WHEN "actionType" = 'poi' THEN 1 ELSE 0 END) as "typeSumPoi", + SUM(CASE WHEN "actionType" = 'chapter' THEN 1 ELSE 0 END) as "typeSumChapter", `; } try { const row = await db.prepare("get", ` - SELECT SUM(((CASE WHEN "endTime" - "startTime" > ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved", + SELECT SUM(CASE WHEN "actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "endTime" - "startTime" > ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views" END) as "minutesSaved", ${additionalQuery} count(*) as "segmentCount" FROM "sponsorTimes" - WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1 AND "actionType" != 'chapter'`, + WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, userID]); const source = (row.minutesSaved != null) ? row : {}; const handler = { get: (target: Record, name: string) => target?.[name] || 0 }; @@ -60,6 +62,7 @@ async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolea poi_highlight: proxy.categorySumHighlight, filler: proxy.categorySumFiller, exclusive_access: proxy.categorySumExclusiveAccess, + chapter: proxy.categorySumChapter, }; } if (fetchActionTypeStats) { @@ -67,7 +70,8 @@ async function dbGetUserSummary(userID: HashedUserID, fetchCategoryStats: boolea skip: proxy.typeSumSkip, mute: proxy.typeSumMute, full: proxy.typeSumFull, - poi: proxy.typeSumPoi + poi: proxy.typeSumPoi, + chapter: proxy.typeSumChapter, }; } return result; From 771fa18731267b6df77c47da851631858855e305 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Wed, 28 Sep 2022 17:59:04 +0200 Subject: [PATCH 5/7] Modify getUserStats test cases to include chapter counts --- test/cases/getUserStats.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/cases/getUserStats.ts b/test/cases/getUserStats.ts index df096bd..f5d63d3 100644 --- a/test/cases/getUserStats.ts +++ b/test/cases/getUserStats.ts @@ -22,8 +22,7 @@ describe("getUserStats", () => { await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, -2, "skip", "getuserstatsuuid9", getHash("getuserstats_user_02"), 8, 2, "sponsor", 0]); await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "skip", "getuserstatsuuid10", getHash("getuserstats_user_01"), 8, 2, "filler", 0]); await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 0, 0, "full", "getuserstatsuuid11", getHash("getuserstats_user_01"), 8, 2, "exclusive_access", 0]); - - + await db.prepare("run", sponsorTimesQuery, ["getuserstats1", 0, 60, 0, "chapter", "getuserstatsuuid12", getHash("getuserstats_user_01"), 9, 2, "chapter", 0]); }); it("Should be able to get a 400 (No userID parameter)", (done) => { @@ -52,17 +51,19 @@ describe("getUserStats", () => { music_offtopic: 1, poi_highlight: 1, filler: 1, - exclusive_access: 1 + exclusive_access: 1, + chapter: 1, }, actionTypeCount: { mute: 0, skip: 8, full: 1, - poi: 1 + poi: 1, + chapter: 1, }, overallStats: { minutesSaved: 30, - segmentCount: 10 + segmentCount: 11 } }; assert.ok(partialDeepEquals(res.data, expected)); From 601a17c969b2a94ecafdb730c6ea0a4319b33f33 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Wed, 28 Sep 2022 18:07:39 +0200 Subject: [PATCH 6/7] Ignore chapters for saved time calculations in getUserInfo.ts --- src/routes/getUserInfo.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index c0b1955..a5e8ea3 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -14,7 +14,7 @@ const maxRewardTime = config.maxRewardTimePerSegmentInSeconds; async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> { try { const row = await db.prepare("get", - `SELECT SUM(((CASE WHEN "endTime" - "startTime" > ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved", + `SELECT SUM(CASE WHEN "actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "endTime" - "startTime" > ? THEN ? ELSE "endTime" - "startTime" END) / 60) * "views" END) as "minutesSaved", count(*) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [maxRewardTime, maxRewardTime, userID], { useReplica: true }); if (row.minutesSaved != null) { @@ -200,4 +200,4 @@ export async function endpoint(req: Request, res: Response): Promise { return res.status(400).send("Invalid values JSON"); } else return res.sendStatus(500); } -} \ No newline at end of file +} From ed251a047a61ccc28afddbae8964455b81312ad6 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Wed, 28 Sep 2022 18:18:41 +0200 Subject: [PATCH 7/7] Add a test case for getUserInfo.ts ignoring chapters for time saved calc --- test/cases/getUserInfo.ts | 45 ++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index 4f8d3ae..d4676c7 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -10,16 +10,17 @@ describe("getUserInfo", () => { const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)'; await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_01"), "Username user 01"]); - const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; - await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000001", getHash("getuserinfo_user_01"), 1, 10, "sponsor", 0]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000002", getHash("getuserinfo_user_01"), 2, 10, "sponsor", 0]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo1", 1, 11, -1, "uuid000003", getHash("getuserinfo_user_01"), 3, 10, "sponsor", 0]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo1", 1, 11, -2, "uuid000004", getHash("getuserinfo_user_01"), 4, 10, "sponsor", 1]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo2", 1, 11, -5, "uuid000005", getHash("getuserinfo_user_01"), 5, 10, "sponsor", 1]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000007", getHash("getuserinfo_user_02"), 7, 10, "sponsor", 1]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000008", getHash("getuserinfo_user_02"), 8, 10, "sponsor", 1]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 0, 36000, 2,"uuid000009", getHash("getuserinfo_user_03"), 8, 10, "sponsor", 0]); - await db.prepare("run", sponsorTimesQuery, ["getUserInfo3", 1, 11, 2, "uuid000006", getHash("getuserinfo_user_02"), 6, 10, "sponsor", 0]); + const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "actionType", "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000001", getHash("getuserinfo_user_01"), 1, 10, "sponsor", "skip", 0]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000002", getHash("getuserinfo_user_01"), 2, 10, "sponsor", "skip", 0]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo1", 1, 11, -1, "uuid000003", getHash("getuserinfo_user_01"), 3, 10, "sponsor", "skip", 0]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo1", 1, 11, -2, "uuid000004", getHash("getuserinfo_user_01"), 4, 10, "sponsor", "skip", 1]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo2", 1, 11, -5, "uuid000005", getHash("getuserinfo_user_01"), 5, 10, "sponsor", "skip", 1]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000007", getHash("getuserinfo_user_02"), 7, 10, "sponsor", "skip", 1]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 1, 11, 2, "uuid000008", getHash("getuserinfo_user_02"), 8, 10, "sponsor", "skip", 1]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo0", 0, 36000, 2,"uuid000009", getHash("getuserinfo_user_03"), 8, 10, "sponsor", "skip", 0]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo3", 1, 11, 2, "uuid000006", getHash("getuserinfo_user_02"), 6, 10, "sponsor", "skip", 0]); + await db.prepare("run", sponsorTimesQuery, ["getUserInfo4", 1, 11, 2, "uuid000010", getHash("getuserinfo_user_04"), 9, 10, "chapter", "chapter", 0]); const insertWarningQuery = 'INSERT INTO warnings ("userID", "issueTime", "issuerUserID", "enabled", "reason") VALUES (?, ?, ?, ?, ?)'; @@ -307,4 +308,28 @@ describe("getUserInfo", () => { }) .catch(err => done(err)); }); + + it("Should ignore chapters for saved time calculations", (done) => { + client.get(endpoint, { params: { userID: "getuserinfo_user_04" } }) + .then(res => { + assert.strictEqual(res.status, 200); + const expected = { + userName: "f187933817e7b0211a3f6f7d542a63ca9cc289d6cc8a8a79669d69a313671ccf", + userID: "f187933817e7b0211a3f6f7d542a63ca9cc289d6cc8a8a79669d69a313671ccf", + minutesSaved: 0, + viewCount: 10, + ignoredViewCount: 0, + segmentCount: 1, + ignoredSegmentCount: 0, + reputation: 0, + lastSegmentID: "uuid000010", + vip: false, + warnings: 0, + warningReason: "" + }; + assert.deepStrictEqual(res.data, expected); + done(); + }) + .catch(err => done(err)); + }); });