From 14f55c9ee503f5e05ede3096b736e312647c12d2 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Tue, 6 Jul 2021 13:31:44 +0700 Subject: [PATCH 1/7] Add lock category reason --- DatabaseSchema.md | 1 + databases/_upgrade_sponsorTimes_20.sql | 8 ++ src/routes/postLockCategories.ts | 40 +++++-- src/types/segments.model.ts | 7 ++ test/cases/lockCategoriesRecords.ts | 143 ++++++++++++++++++++++--- 5 files changed, 177 insertions(+), 22 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_20.sql diff --git a/DatabaseSchema.md b/DatabaseSchema.md index 6814131..485c4c3 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -92,6 +92,7 @@ | userID | TEXT | not null | | category | TEXT | not null | | hashedVideoID | TEXT | not null, default '' | +| reason | TEXT | not null, default '' | | index | field | | -- | :--: | diff --git a/databases/_upgrade_sponsorTimes_20.sql b/databases/_upgrade_sponsorTimes_20.sql new file mode 100644 index 0000000..a827cb6 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_20.sql @@ -0,0 +1,8 @@ +BEGIN TRANSACTION; + +/* Add hash field */ +ALTER TABLE "lockCategories" ADD "reason" TEXT NOT NULL default ''; + +UPDATE "config" SET value = 20 WHERE key = 'version'; + +COMMIT; diff --git a/src/routes/postLockCategories.ts b/src/routes/postLockCategories.ts index 767fba3..e444b5b 100644 --- a/src/routes/postLockCategories.ts +++ b/src/routes/postLockCategories.ts @@ -10,6 +10,7 @@ export async function postLockCategories(req: Request, res: Response): Promise { + let filteredCategories = categories.filter((category) => { return !!category.match(/^[_a-zA-Z]+$/); - }).filter((category) => { - return noCategoryList.indexOf(category) === -1; + }); + // remove any duplicates + filteredCategories = filteredCategories.filter((category, index) => { + return filteredCategories.indexOf(category) === index; }); - // remove any duplicates - categoriesToMark = categoriesToMark.filter((category, index) => { - return categoriesToMark.indexOf(category) === index; + const categoriesToMark = filteredCategories.filter((category) => { + return noCategoryList.indexOf(category) === -1; }); // calculate hash of videoID @@ -63,7 +65,7 @@ export async function postLockCategories(req: Request, res: Response): Promise { + return noCategoryList.indexOf(category) !== -1; + }); + + for (const category of overlapCategories) { + try { + await db.prepare('run', + 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ?', + [reason, userID, videoID, category]); + } catch (err) { + Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'"); + Logger.error(err); + res.status(500).json({ + message: "Internal Server Error: Could not write marker to the database.", + }); + } + } + } + res.status(200).json({ - submitted: categoriesToMark, + submitted: [...categoriesToMark, ...overlapCategories], }); } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 9d72829..23af6a7 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -92,4 +92,11 @@ export interface SegmentCache { export enum CategoryActionType { Skippable, POI +} + +export interface LockCategory { + category: Category, + reason: string, + videoID: VideoID, + userID: UserID } \ No newline at end of file diff --git a/test/cases/lockCategoriesRecords.ts b/test/cases/lockCategoriesRecords.ts index 2cb9070..e05154f 100644 --- a/test/cases/lockCategoriesRecords.ts +++ b/test/cases/lockCategoriesRecords.ts @@ -2,25 +2,26 @@ import fetch from 'node-fetch'; import {Done, getbaseURL} from '../utils'; import {getHash} from '../../src/utils/getHash'; import {db} from '../../src/databases/databases'; +import {LockCategory} from '../../src/types/segments.model'; describe('lockCategoriesRecords', () => { before(async () => { const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; await db.prepare("run", insertVipUserQuery, [getHash("VIPUser-lockCategories")]); - - const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES (?, ?, ?)'; - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'sponsor']); - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'intro']); - - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'sponsor']); - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'intro']); - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'lockCategoryVideo', 'sponsor']); - - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record', 'sponsor']); - - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'sponsor']); - await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'intro']); + + const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)'; + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'sponsor', 'reason-1']); + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'intro', 'reason-1']); + + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'sponsor', 'reason-2']); + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'intro', 'reason-2']); + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'lockCategoryVideo', 'sponsor', 'reason-3']); + + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record', 'sponsor', 'reason-4']); + + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'sponsor', 'reason-5']); + await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'intro', 'reason-5']); }); it('Should update the database version when starting the application', async () => { @@ -95,11 +96,125 @@ describe('lockCategoriesRecords', () => { }) .then(async res => { if (res.status === 200) { - const result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']); + const result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']) as LockCategory[]; if (result.length !== 4) { done("Expected 4 entrys in db, got " + result.length); } else { + const oldRecordNotChangeReason = result.filter(item => { + return item.reason === 'reason-2' && ['sponsor', 'intro'].includes(item.category); + }); + + const newRecordWithEmptyReason = result.filter(item => { + return item.reason === '' && ['outro', 'shilling'].includes(item.category); + }); + + if (newRecordWithEmptyReason.length !== 2 || oldRecordNotChangeReason.length !== 2) { + done(`Incorrect reason update with oldRecordNotChangeReason=${oldRecordNotChangeReason} instead of 2 or newRecordWithEmptyReason=${newRecordWithEmptyReason} instead of 2`); + } else { + done(); + } + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit categories not in video with reason (http response)', (done: Done) => { + const json = { + videoID: 'no-segments-video-id', + userID: 'VIPUser-lockCategories', + categories: [ + 'outro', + 'shilling', + 'shilling', + 'shil ling', + '', + 'intro', + ], + reason: 'new reason' + }; + + const expected = { + submitted: [ + 'outro', + 'shilling', + 'intro' + ], + }; + + fetch(getbaseURL() + "/api/lockCategories", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(json) + }) + .then(async res => { + if (res.status === 200) { + const data = await res.json(); + if (JSON.stringify(data) === JSON.stringify(expected)) { done(); + } else { + done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit categories not in video with reason (sql check)', (done: Done) => { + const json = { + videoID: 'no-segments-video-id-1', + userID: 'VIPUser-lockCategories', + categories: [ + 'outro', + 'shilling', + 'shilling', + 'shil ling', + '', + 'intro', + ], + reason: 'new reason' + }; + + const expectedWithNewReason = [ + 'outro', + 'shilling', + 'intro' + ]; + + fetch(getbaseURL() + "/api/lockCategories", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(json) + }) + .then(async res => { + if (res.status === 200) { + const result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']) as LockCategory[]; + if (result.length !== 4) { + done("Expected 4 entrys in db, got " + result.length); + } else { + const newRecordWithNewReason = result.filter(item => { + return expectedWithNewReason.includes(item.category) && item.reason === 'new reason'; + }); + + const oldRecordNotChangeReason = result.filter(item => { + return item.reason === 'reason-2'; + }); + + if (newRecordWithNewReason.length !== 3) { + done("Expected 3 entrys in db with new reason, got " + newRecordWithNewReason.length); + } else if (oldRecordNotChangeReason.length !== 1) { + done("Expected 1 entrys in db with old reason, got " + oldRecordNotChangeReason.length); + } else { + done(); + } } } else { done("Status code was " + res.status); From 6fcfeb2889433b0d406d41ff0d82f697b61afaef Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Tue, 6 Jul 2021 13:36:56 +0700 Subject: [PATCH 2/7] Fix deep equal fail --- test/cases/lockCategoriesRecords.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/cases/lockCategoriesRecords.ts b/test/cases/lockCategoriesRecords.ts index e05154f..6f7bb6e 100644 --- a/test/cases/lockCategoriesRecords.ts +++ b/test/cases/lockCategoriesRecords.ts @@ -4,6 +4,10 @@ import {getHash} from '../../src/utils/getHash'; import {db} from '../../src/databases/databases'; import {LockCategory} from '../../src/types/segments.model'; +const deepEquals = (a,b) => { + a.forEach((e) => { if (!b.includes(e)) return false; }); + return true; +}; describe('lockCategoriesRecords', () => { before(async () => { @@ -61,7 +65,7 @@ describe('lockCategoriesRecords', () => { .then(async res => { if (res.status === 200) { const data = await res.json(); - if (JSON.stringify(data) === JSON.stringify(expected)) { + if (deepEquals(data, expected)) { done(); } else { done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); @@ -154,7 +158,7 @@ describe('lockCategoriesRecords', () => { .then(async res => { if (res.status === 200) { const data = await res.json(); - if (JSON.stringify(data) === JSON.stringify(expected)) { + if (deepEquals(data, expected)) { done(); } else { done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); @@ -666,7 +670,7 @@ describe('lockCategoriesRecords', () => { .then(async res => { if (res.status === 200) { const data = await res.json(); - if (JSON.stringify(data) === JSON.stringify(expected)) { + if (deepEquals(data, expected)) { done(); } else { done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); From feba2af9ed7bc35f82ce8a71a2bc62f47e74e923 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Tue, 6 Jul 2021 13:42:08 +0700 Subject: [PATCH 3/7] Fix any type --- test/cases/lockCategoriesRecords.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cases/lockCategoriesRecords.ts b/test/cases/lockCategoriesRecords.ts index 6f7bb6e..246576f 100644 --- a/test/cases/lockCategoriesRecords.ts +++ b/test/cases/lockCategoriesRecords.ts @@ -4,7 +4,7 @@ import {getHash} from '../../src/utils/getHash'; import {db} from '../../src/databases/databases'; import {LockCategory} from '../../src/types/segments.model'; -const deepEquals = (a,b) => { +const deepEquals = (a: string[],b: string[]) => { a.forEach((e) => { if (!b.includes(e)) return false; }); return true; }; @@ -65,7 +65,7 @@ describe('lockCategoriesRecords', () => { .then(async res => { if (res.status === 200) { const data = await res.json(); - if (deepEquals(data, expected)) { + if (deepEquals(data.submitted, expected.submitted)) { done(); } else { done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); @@ -158,7 +158,7 @@ describe('lockCategoriesRecords', () => { .then(async res => { if (res.status === 200) { const data = await res.json(); - if (deepEquals(data, expected)) { + if (deepEquals(data.submitted, expected.submitted)) { done(); } else { done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); @@ -670,7 +670,7 @@ describe('lockCategoriesRecords', () => { .then(async res => { if (res.status === 200) { const data = await res.json(); - if (deepEquals(data, expected)) { + if (deepEquals(data.categories, expected.categories)) { done(); } else { done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(data)); From 301f5e71136bf3e28a811ef3fd2aa1ead44d72d0 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 7 Jul 2021 18:12:52 -0400 Subject: [PATCH 4/7] Include action type in UUID --- src/routes/postSkipSegments.ts | 2 +- src/utils/getSubmissionUUID.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 59d7802..5906c6a 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -519,7 +519,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise Date: Wed, 7 Jul 2021 18:16:06 -0400 Subject: [PATCH 5/7] Add action type when finding UUID in webhook --- src/routes/postSkipSegments.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 5906c6a..b026f7c 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -16,6 +16,7 @@ import { getCategoryActionType } from '../utils/categoryInfo'; import { QueryCacher } from '../utils/queryCacher'; import { getReputation } from '../utils/reputation'; import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model'; +import { UserID } from '../types/user.model'; async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); @@ -164,7 +165,7 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, - submission: { videoID: any; userID: any; segments: any }) { + submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[] }) { if (apiVideoInfo) { const {err, data} = apiVideoInfo; if (err) return false; @@ -234,7 +235,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, const startTime = parseFloat(segments[i].segment[0]); const endTime = parseFloat(segments[i].segment[1]); - const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime); + const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime); // Send to Discord // Note, if this is too spammy. Consider sending all the segments as one Webhook sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); From a471e057f54021f9f27da7cbebed4b1353aa32a3 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 7 Jul 2021 18:21:25 -0400 Subject: [PATCH 6/7] Remove category from segment id --- src/routes/postSkipSegments.ts | 4 ++-- src/utils/getSubmissionUUID.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index b026f7c..2065d35 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -235,7 +235,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, const startTime = parseFloat(segments[i].segment[0]); const endTime = parseFloat(segments[i].segment[1]); - const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime); + const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime); // Send to Discord // Note, if this is too spammy. Consider sending all the segments as one Webhook sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); @@ -520,7 +520,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise Date: Wed, 7 Jul 2021 18:24:16 -0400 Subject: [PATCH 7/7] Fix segment id test --- test/cases/getSubmissionUUID.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/cases/getSubmissionUUID.ts b/test/cases/getSubmissionUUID.ts index 6b3ab3c..e7a0227 100644 --- a/test/cases/getSubmissionUUID.ts +++ b/test/cases/getSubmissionUUID.ts @@ -1,8 +1,10 @@ import {getSubmissionUUID} from '../../src/utils/getSubmissionUUID'; import assert from 'assert'; +import { ActionType, VideoID } from '../../src/types/segments.model'; +import { UserID } from '../../src/types/user.model'; describe('getSubmissionUUID', () => { it('Should return the hashed value', () => { - assert.strictEqual(getSubmissionUUID('video001', 'sponsor', 'testuser001', 13.33337, 42.000001), '1d33d7016aa6482849019bd906d75c08fe6b815e64e823146df35f66c35612dd'); + assert.strictEqual(getSubmissionUUID('video001' as VideoID, 'skip' as ActionType, 'testuser001' as UserID, 13.33337, 42.000001), '3572aa64e0a2d6352c3de14ca45f8a83d193c32635669a7ae0b40c9eb36395872'); }); });