Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 11:36:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into testAssert

Browse Source
This commit is contained in:
Michael C
2021-07-08 00:51:56 -04:00
parent 178ed1e5af 2105cdf10b
commit 9f05595cd6
8 changed files with 170 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
DatabaseSchema.md
View File

@@ -92,6 +92,7 @@
| userID | TEXT | not null |
| category | TEXT | not null |
| hashedVideoID | TEXT | not null, default '' |
| reason | TEXT | not null, default '' |
| index | field |
| -- | :--: |

8
databases/_upgrade_sponsorTimes_20.sql Normal file
View File

@@ -0,0 +1,8 @@
BEGIN TRANSACTION;
/* Add hash field */
ALTER TABLE "lockCategories" ADD "reason" TEXT NOT NULL default '';
UPDATE "config" SET value = 20 WHERE key = 'version';
COMMIT;

40
src/routes/postLockCategories.ts
View File

@@ -10,6 +10,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
const videoID = req.body.videoID;
let userID = req.body.userID;
const categories = req.body.categories;
const reason: string = req.body.reason ?? '';
// Check input data is valid
if (!videoID
@@ -46,15 +47,16 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
}
// get user categories not already submitted that match accepted format
let categoriesToMark = categories.filter((category) => {
let filteredCategories = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/);
}).filter((category) => {
return noCategoryList.indexOf(category) === -1;
});
// remove any duplicates
filteredCategories = filteredCategories.filter((category, index) => {
return filteredCategories.indexOf(category) === index;
});
// remove any duplicates
categoriesToMark = categoriesToMark.filter((category, index) => {
return categoriesToMark.indexOf(category) === index;
const categoriesToMark = filteredCategories.filter((category) => {
return noCategoryList.indexOf(category) === -1;
});
// calculate hash of videoID
@@ -63,7 +65,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
// create database entry
for (const category of categoriesToMark) {
try {
await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID") VALUES(?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID]);
await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason") VALUES(?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason]);
} catch (err) {
Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
Logger.error(err);
@@ -73,7 +75,29 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
}
}
// update reason for existed categories
let overlapCategories = [];
if (reason.length !== 0) {
overlapCategories = filteredCategories.filter((category) => {
return noCategoryList.indexOf(category) !== -1;
});
for (const category of overlapCategories) {
try {
await db.prepare('run',
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ?',
[reason, userID, videoID, category]);
} catch (err) {
Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
Logger.error(err);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",
});
}
}
}
res.status(200).json({
submitted: categoriesToMark,
submitted: [...categoriesToMark, ...overlapCategories],
});
}

7
src/routes/postSkipSegments.ts
View File

@@ -16,6 +16,7 @@ import { getCategoryActionType } from '../utils/categoryInfo';
import { QueryCacher } from '../utils/queryCacher';
import { getReputation } from '../utils/reputation';
import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
import { UserID } from '../types/user.model';
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -164,7 +165,7 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
submission: { videoID: any; userID: any; segments: any }) {
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[] }) {
if (apiVideoInfo) {
const {err, data} = apiVideoInfo;
if (err) return false;
@@ -234,7 +235,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
@@ -519,7 +520,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
//also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0;

7
src/types/segments.model.ts
View File

@@ -92,4 +92,11 @@ export interface SegmentCache {
export enum CategoryActionType {
Skippable,
POI
}
export interface LockCategory {
category: Category,
reason: string,
videoID: VideoID,
userID: UserID
}

6
src/utils/getSubmissionUUID.ts
View File

@@ -1,6 +1,8 @@
import {getHash} from './getHash';
import { HashedValue } from '../types/hash.model';
import { ActionType, Category, VideoID } from '../types/segments.model';
import { UserID } from '../types/user.model';
export function getSubmissionUUID(videoID: string, category: string, userID: string, startTime: number, endTime: number): HashedValue{
return getHash('v2-categories' + videoID + startTime + endTime + category + userID, 1);
export function getSubmissionUUID(videoID: VideoID, actionType: ActionType, userID: UserID, startTime: number, endTime: number): HashedValue{
return `3${getHash('v3' + videoID + startTime + endTime + userID, 1)}` as HashedValue;
}

4
test/cases/getSubmissionUUID.ts
View File

@@ -1,8 +1,10 @@
import {getSubmissionUUID} from '../../src/utils/getSubmissionUUID';
import assert from 'assert';
import { ActionType, VideoID } from '../../src/types/segments.model';
import { UserID } from '../../src/types/user.model';
describe('getSubmissionUUID', () => {
it('Should return the hashed value', () => {
assert.strictEqual(getSubmissionUUID('video001', 'sponsor', 'testuser001', 13.33337, 42.000001), '1d33d7016aa6482849019bd906d75c08fe6b815e64e823146df35f66c35612dd');
assert.strictEqual(getSubmissionUUID('video001' as VideoID, 'skip' as ActionType, 'testuser001' as UserID, 13.33337, 42.000001), '3572aa64e0a2d6352c3de14ca45f8a83d193c32635669a7ae0b40c9eb36395872');
});
});

127
test/cases/lockCategoriesRecords.ts
View File

@@ -3,30 +3,30 @@ import {Done, getbaseURL} from '../utils';
import {getHash} from '../../src/utils/getHash';
import {db} from '../../src/databases/databases';
import assert from 'assert';
import {LockCategory} from '../../src/types/segments.model';
describe('lockCategoriesRecords', () => {
before(async () => {
const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
await db.prepare("run", insertVipUserQuery, [getHash("VIPUser-lockCategories")]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES (?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'sponsor']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'intro']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'sponsor']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'intro']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'lockCategoryVideo', 'sponsor']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record', 'sponsor']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'sponsor']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'intro']);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'sponsor', 'reason-1']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id', 'intro', 'reason-1']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'sponsor', 'reason-2']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'no-segments-video-id-1', 'intro', 'reason-2']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'lockCategoryVideo', 'sponsor', 'reason-3']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record', 'sponsor', 'reason-4']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'sponsor', 'reason-5']);
await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-lockCategories"), 'delete-record-1', 'intro', 'reason-5']);
});
it('Should update the database version when starting the application', async () => {
const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value;
if (version > 1) return;
else return 'Version isn\'t greater than 1. Version is ' + version;
assert.ok(version > 1);
});
it('Should be able to submit categories not in video (http response)', (done: Done) => {
@@ -86,8 +86,103 @@ describe('lockCategoriesRecords', () => {
})
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']);
const result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']) as LockCategory[];
assert.strictEqual(result.length, 4);
const oldRecordNotChangeReason = result.filter(item =>
item.reason === 'reason-2' && ['sponsor', 'intro'].includes(item.category)
);
const newRecordWithEmptyReason = result.filter(item =>
item.reason === '' && ['outro', 'shilling'].includes(item.category)
);
assert.strictEqual(newRecordWithEmptyReason.length, 2);
assert.strictEqual(oldRecordNotChangeReason.length, 2);
done();
})
.catch(err => done(err));
});
it('Should be able to submit categories not in video with reason (http response)', (done: Done) => {
const json = {
videoID: 'no-segments-video-id',
userID: 'VIPUser-lockCategories',
categories: [
'outro',
'shilling',
'shilling',
'shil ling',
'',
'intro',
],
reason: 'new reason'
};
const expected = {
submitted: [
'outro',
'shilling',
'intro'
],
};
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(json)
})
.then(async res => {
assert.strictEqual(res.status, 200);
const data = await res.json();
assert.deepStrictEqual(data.submitted, expected.submitted);
done();
})
.catch(err => done(err));
});
it('Should be able to submit categories not in video with reason (sql check)', (done: Done) => {
const json = {
videoID: 'no-segments-video-id-1',
userID: 'VIPUser-lockCategories',
categories: [
'outro',
'shilling',
'shilling',
'shil ling',
'',
'intro',
],
reason: 'new reason'
};
const expectedWithNewReason = [
'outro',
'shilling',
'intro'
];
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(json)
})
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']) as LockCategory[];
assert.strictEqual(result.length, 4);
const newRecordWithNewReason = result.filter(item =>
expectedWithNewReason.includes(item.category) && item.reason === 'new reason'
);
const oldRecordNotChangeReason = result.filter(item =>
item.reason === 'reason-2'
);
assert.strictEqual(newRecordWithNewReason.length, 3);
assert.strictEqual(oldRecordNotChangeReason.length, 1);
done();
})
.catch(err => done(err));