Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into testAssert

DatabaseSchema.md

@@ -92,6 +92,7 @@
 | userID | TEXT | not null |
 | category | TEXT | not null |
 | hashedVideoID | TEXT | not null, default '' |
+| reason | TEXT | not null, default '' |
 
 | index | field |
 | -- | :--: |

databases/_upgrade_sponsorTimes_20.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "lockCategories" ADD "reason" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 20 WHERE key = 'version';
+
+COMMIT;

src/routes/postLockCategories.ts

@@ -10,6 +10,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
     const videoID = req.body.videoID;
     let userID = req.body.userID;
     const categories = req.body.categories;
+    const reason: string = req.body.reason ?? ''; 
 
     // Check input data is valid
     if (!videoID
@@ -46,15 +47,16 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
     }
 
     // get user categories not already submitted that match accepted format
-    let categoriesToMark = categories.filter((category) => {
+    let filteredCategories = categories.filter((category) => {
         return !!category.match(/^[_a-zA-Z]+$/);
-    }).filter((category) => {
+    });
-        return noCategoryList.indexOf(category) === -1;
+    // remove any duplicates
+    filteredCategories = filteredCategories.filter((category, index) => {
+        return filteredCategories.indexOf(category) === index;
     });
 
-    // remove any duplicates
+    const categoriesToMark = filteredCategories.filter((category) => {
-    categoriesToMark = categoriesToMark.filter((category, index) => {
+        return noCategoryList.indexOf(category) === -1;
-        return categoriesToMark.indexOf(category) === index;
     });
 
     // calculate hash of videoID
@@ -63,7 +65,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
     // create database entry
     for (const category of categoriesToMark) {
         try {
-            await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID") VALUES(?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID]);
+            await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason") VALUES(?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason]);
         } catch (err) {
             Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
             Logger.error(err);
@@ -73,7 +75,29 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
         }
     }
 
+    // update reason for existed categories
+    let overlapCategories = [];
+    if (reason.length !== 0) {
+        overlapCategories = filteredCategories.filter((category) => {
+            return noCategoryList.indexOf(category) !== -1;
+        });
+
+        for (const category of overlapCategories) {
+            try {
+                await db.prepare('run', 
+                    'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ?',
+                    [reason, userID, videoID, category]);
+            } catch (err) {
+                Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
+                Logger.error(err);
+                res.status(500).json({
+                    message: "Internal Server Error: Could not write marker to the database.",
+                });
+            }
+        }
+    }
+
     res.status(200).json({
-        submitted: categoriesToMark,
+        submitted: [...categoriesToMark, ...overlapCategories],
     });
 }

src/routes/postSkipSegments.ts

@@ -16,6 +16,7 @@ import { getCategoryActionType } from '../utils/categoryInfo';
 import { QueryCacher } from '../utils/queryCacher';
 import { getReputation } from '../utils/reputation';
 import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
+import { UserID } from '../types/user.model';
 
 async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
     const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -164,7 +165,7 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
 async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, 
-            submission: { videoID: any; userID: any; segments: any }) {
+            submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[] }) {
     if (apiVideoInfo) {
         const {err, data} = apiVideoInfo;
         if (err) return false;
@@ -234,7 +235,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
                     const startTime = parseFloat(segments[i].segment[0]);
                     const endTime = parseFloat(segments[i].segment[1]);
 
-                    const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
+                    const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime);
                     // Send to Discord
                     // Note, if this is too spammy. Consider sending all the segments as one Webhook
                     sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
@@ -519,7 +520,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
             //this can just be a hash of the data
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
-            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
+            const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
             const hashedVideoID = getHash(videoID, 1);
 
             const startingLocked = isVIP ? 1 : 0;

src/types/segments.model.ts

@@ -92,4 +92,11 @@ export interface SegmentCache {
 export enum CategoryActionType {
     Skippable,
     POI
+}
+
+export interface LockCategory {
+    category: Category,
+    reason: string,
+    videoID: VideoID,
+    userID: UserID
 }

src/utils/getSubmissionUUID.ts

@@ -1,6 +1,8 @@
 import {getHash} from './getHash';
 import { HashedValue } from '../types/hash.model';
+import { ActionType, Category, VideoID } from '../types/segments.model';
+import { UserID } from '../types/user.model';
 
-export function getSubmissionUUID(videoID: string, category: string, userID: string, startTime: number, endTime: number): HashedValue{
+export function getSubmissionUUID(videoID: VideoID, actionType: ActionType, userID: UserID, startTime: number, endTime: number): HashedValue{
-    return getHash('v2-categories' + videoID + startTime + endTime + category + userID, 1);
+    return `3${getHash('v3' + videoID + startTime + endTime + userID, 1)}` as HashedValue;
 }

test/cases/getSubmissionUUID.ts

@@ -1,8 +1,10 @@
 import {getSubmissionUUID} from '../../src/utils/getSubmissionUUID';
 import assert from 'assert';
+import { ActionType, VideoID } from '../../src/types/segments.model';
+import { UserID } from '../../src/types/user.model';
 
 describe('getSubmissionUUID', () => {
     it('Should return the hashed value', () => {
-        assert.strictEqual(getSubmissionUUID('video001', 'sponsor', 'testuser001', 13.33337, 42.000001), '1d33d7016aa6482849019bd906d75c08fe6b815e64e823146df35f66c35612dd');
+        assert.strictEqual(getSubmissionUUID('video001' as VideoID, 'skip' as ActionType, 'testuser001' as UserID, 13.33337, 42.000001), '3572aa64e0a2d6352c3de14ca45f8a83d193c32635669a7ae0b40c9eb36395872');
     });
 });

test/cases/lockCategoriesRecords.ts

@@ -3,30 +3,30 @@ import {Done, getbaseURL} from '../utils';
 import {getHash} from '../../src/utils/getHash';
 import {db} from '../../src/databases/databases';
 import assert from 'assert';
+import {LockCategory} from '../../src/types/segments.model';
 
 describe('lockCategoriesRecords', () => {
     before(async () => {
         const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
         await db.prepare("run", insertVipUserQuery, [getHash("VIPUser-lockCategories")]);
- 
+
-        const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES (?, ?, ?)';
+        const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)';
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'sponsor']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'sponsor', 'reason-1']);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'intro']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'intro', 'reason-1']);
- 
+
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'sponsor']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'sponsor', 'reason-2']);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'intro']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'intro', 'reason-2']);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'lockCategoryVideo', 'sponsor']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'lockCategoryVideo', 'sponsor', 'reason-3']);
- 
+
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record', 'sponsor']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record', 'sponsor', 'reason-4']);
- 
+
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'sponsor']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'sponsor', 'reason-5']);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'intro']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'intro', 'reason-5']);
     });
 
     it('Should update the database version when starting the application', async () => {
         const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value;
-        if (version > 1) return;
+        assert.ok(version > 1);
-        else return 'Version isn\'t greater than 1. Version is ' + version;
     });
 
     it('Should be able to submit categories not in video (http response)', (done: Done) => {
@@ -86,8 +86,103 @@ describe('lockCategoriesRecords', () => {
         })
         .then(async res => {
             assert.strictEqual(res.status, 200);
-            const result = await db.prepare('all', 'SELECT * FROM "lockCategories"  WHERE "videoID" = ?', ['no-segments-video-id-1']);
+            const result = await db.prepare('all', 'SELECT * FROM "lockCategories"  WHERE "videoID" = ?', ['no-segments-video-id-1']) as LockCategory[];
             assert.strictEqual(result.length, 4);
+            const oldRecordNotChangeReason = result.filter(item => 
+                item.reason === 'reason-2' && ['sponsor', 'intro'].includes(item.category)
+            );
+
+            const newRecordWithEmptyReason = result.filter(item =>
+                item.reason === '' && ['outro', 'shilling'].includes(item.category)
+            );
+
+            assert.strictEqual(newRecordWithEmptyReason.length, 2);
+            assert.strictEqual(oldRecordNotChangeReason.length, 2);
+            done();
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to submit categories not in video with reason (http response)', (done: Done) => {
+        const json = {
+            videoID: 'no-segments-video-id',
+            userID: 'VIPUser-lockCategories',
+            categories: [
+                'outro',
+                'shilling',
+                'shilling',
+                'shil ling',
+                '',
+                'intro',
+            ],
+            reason: 'new reason'
+        };
+
+        const expected = {
+            submitted: [
+                'outro',
+                'shilling',
+                'intro'
+            ],
+        };
+
+        fetch(getbaseURL() + "/api/lockCategories", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(json)
+        })
+        .then(async res => {
+            assert.strictEqual(res.status, 200);
+            const data = await res.json();
+            assert.deepStrictEqual(data.submitted, expected.submitted);
+            done();
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to submit categories not in video with reason (sql check)', (done: Done) => {
+        const json = {
+            videoID: 'no-segments-video-id-1',
+            userID: 'VIPUser-lockCategories',
+            categories: [
+                'outro',
+                'shilling',
+                'shilling',
+                'shil ling',
+                '',
+                'intro',
+            ],
+            reason: 'new reason'
+        };
+
+        const expectedWithNewReason = [
+            'outro',
+            'shilling',
+            'intro'
+        ];
+
+        fetch(getbaseURL() + "/api/lockCategories", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(json)
+        })
+        .then(async res => {
+            assert.strictEqual(res.status, 200);
+            const result = await db.prepare('all', 'SELECT * FROM "lockCategories"  WHERE "videoID" = ?', ['no-segments-video-id-1']) as LockCategory[];
+            assert.strictEqual(result.length, 4);
+            const newRecordWithNewReason = result.filter(item => 
+                expectedWithNewReason.includes(item.category) && item.reason === 'new reason'
+            );
+            const oldRecordNotChangeReason = result.filter(item =>
+                item.reason === 'reason-2'
+            );
+
+            assert.strictEqual(newRecordWithNewReason.length, 3);
+            assert.strictEqual(oldRecordNotChangeReason.length, 1);
             done();
         })
         .catch(err => done(err));