From 09eec5a4a57b3703409c973a144d95fee3a6a6c2 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 2 Jan 2022 14:00:54 -0500 Subject: [PATCH 01/14] Add locking by action type --- DatabaseSchema.md | 1 + databases/_upgrade_sponsorTimes_29.sql | 21 +++++ src/routes/postLockCategories.ts | 85 +++++++++++-------- src/routes/postSkipSegments.ts | 4 +- test/cases/lockCategoriesRecords.ts | 109 ++++++++++++++++++++----- test/cases/postSkipSegments.ts | 20 +++++ 6 files changed, 183 insertions(+), 57 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_29.sql diff --git a/DatabaseSchema.md b/DatabaseSchema.md index f0d0785..68daf48 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -94,6 +94,7 @@ | -- | :--: | -- | | videoID | TEXT | not null | | userID | TEXT | not null | +| actionType | TEXT | not null, default 'skip' | | category | TEXT | not null | | hashedVideoID | TEXT | not null, default '' | | reason | TEXT | not null, default '' | diff --git a/databases/_upgrade_sponsorTimes_29.sql b/databases/_upgrade_sponsorTimes_29.sql new file mode 100644 index 0000000..8c5e248 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_29.sql @@ -0,0 +1,21 @@ +BEGIN TRANSACTION; + +CREATE TABLE "sqlb_temp_table_29" ( + "videoID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "actionType" TEXT NOT NULL DEFAULT 'skip', + "category" TEXT NOT NULL, + "hashedVideoID" TEXT NOT NULL default '', + "reason" TEXT NOT NULL default '', + "service" TEXT NOT NULL default 'YouTube' +); + +INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'skip',"category","hashedVideoID","reason","service" FROM "lockCategories"; +INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'mute',"category","hashedVideoID","reason","service" FROM "lockCategories"; + +DROP TABLE "lockCategories"; +ALTER TABLE sqlb_temp_table_29 RENAME TO "lockCategories"; + +UPDATE "config" SET value = 29 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/postLockCategories.ts b/src/routes/postLockCategories.ts index e10eeda..400acd3 100644 --- a/src/routes/postLockCategories.ts +++ b/src/routes/postLockCategories.ts @@ -3,14 +3,15 @@ import { getHashCache } from "../utils/getHashCache"; import { isUserVIP } from "../utils/isUserVIP"; import { db } from "../databases/databases"; import { Request, Response } from "express"; -import { VideoIDHash } from "../types/segments.model"; +import { ActionType, Category, VideoIDHash } from "../types/segments.model"; import { getService } from "../utils/getService"; export async function postLockCategories(req: Request, res: Response): Promise { // Collect user input data const videoID = req.body.videoID; let userID = req.body.userID; - const categories = req.body.categories; + const categories = req.body.categories as Category[]; + const actionTypes = req.body.actionTypes as ActionType[] || [ActionType.Skip, ActionType.Mute]; const reason: string = req.body.reason ?? ""; const service = getService(req.body.service); @@ -20,6 +21,8 @@ export async function postLockCategories(req: Request, res: Response): Promise { - return obj.category; - }); + const existingLocks = (await db.prepare("all", 'SELECT "category", "actionType" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service])) as + { category: Category, actionType: ActionType }[]; + + const filteredCategories = filterData(categories); + const filteredActionTypes = filterData(actionTypes); + + const locksToApply: { category: Category, actionType: ActionType }[] = []; + const overwrittenLocks: { category: Category, actionType: ActionType }[] = []; + for (const category of filteredCategories) { + for (const actionType of filteredActionTypes) { + if (!existingLocks.some((lock) => lock.category === category && lock.actionType === actionType)) { + locksToApply.push({ + category, + actionType + }); + } else { + overwrittenLocks.push({ + category, + actionType + }); + } + } } - // get user categories not already submitted that match accepted format - let filteredCategories = categories.filter((category) => { - return !!category.match(/^[_a-zA-Z]+$/); - }); - // remove any duplicates - filteredCategories = filteredCategories.filter((category, index) => { - return filteredCategories.indexOf(category) === index; - }); - - const categoriesToMark = filteredCategories.filter((category) => { - return noCategoryList.indexOf(category) === -1; - }); - // calculate hash of videoID const hashedVideoID: VideoIDHash = await getHashCache(videoID, 1); // create database entry - for (const category of categoriesToMark) { + for (const lock of locksToApply) { try { - await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason, service]); + await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "actionType", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?, ?)`, [videoID, userID, lock.actionType, lock.category, hashedVideoID, reason, service]); } catch (err) { - Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}' (${service})`); + Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`); Logger.error(err as string); res.status(500).json({ message: "Internal Server Error: Could not write marker to the database.", @@ -78,19 +82,14 @@ export async function postLockCategories(req: Request, res: Response): Promise { - return noCategoryList.indexOf(category) !== -1; - }); - - for (const category of overlapCategories) { + for (const lock of overwrittenLocks) { try { await db.prepare("run", - 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ? AND "service" = ?', - [reason, userID, videoID, category, service]); + 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "actionType" = ? AND "category" = ? AND "service" = ?', + [reason, userID, videoID, lock.actionType, lock.category, service]); } catch (err) { - Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID} (${service})'`); + Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`); Logger.error(err as string); res.status(500).json({ message: "Internal Server Error: Could not write marker to the database.", @@ -100,6 +99,20 @@ export async function postLockCategories(req: Request, res: Response): Promise locksToApply.some((lock) => category === lock.category)))] + : [...filteredCategories], // Legacy + submittedValues: [...locksToApply, ...overwrittenLocks], + }); +} + +function filterData(data: T[]): T[] { + // get user categories not already submitted that match accepted format + const filtered = data.filter((elem) => { + return !!elem.match(/^[_a-zA-Z]+$/); + }); + // remove any duplicates + return filtered.filter((elem, index) => { + return filtered.indexOf(elem) === index; }); } diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 9f6598f..6655236 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -357,7 +357,7 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, } // Reject segment if it's in the locked categories list - const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category); + const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category && segments[i].actionType === c.actionType); if (!isVIP && lockIndex !== -1) { // TODO: Do something about the fradulent submission Logger.warn(`Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}`); @@ -439,7 +439,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array { +const stringDeepEquals = (a: string[], b: string[]): boolean => { let result = true; b.forEach((e) => { if (!a.includes(e)) result = false; @@ -23,18 +24,27 @@ describe("lockCategoriesRecords", () => { const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; await db.prepare("run", insertVipUserQuery, [lockVIPUserHash]); - const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)'; - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "sponsor", "reason-1", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "intro", "reason-1", "YouTube"]); + const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "actionType", "category", "reason", "service") VALUES (?, ?, ?, ?, ?, ?)'; + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "skip", "sponsor", "reason-1", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "mute", "sponsor", "reason-1", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "skip", "intro", "reason-1", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "mute", "intro", "reason-1", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "sponsor", "reason-2", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "intro", "reason-2", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "sponsor", "reason-3", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "skip", "sponsor", "reason-2", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "mute", "sponsor", "reason-2", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "skip", "intro", "reason-2", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "mute", "intro", "reason-2", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "skip", "sponsor", "reason-3", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "mute", "sponsor", "reason-3", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "sponsor", "reason-4", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo-2", "skip", "sponsor", "reason-4", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "sponsor", "reason-5", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "intro", "reason-5", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "skip", "sponsor", "reason-4", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "mute", "sponsor", "reason-4", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "skip", "sponsor", "reason-5", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "mute", "sponsor", "reason-5", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "skip", "intro", "reason-5", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "mute", "intro", "reason-5", "YouTube"]); }); it("Should update the database version when starting the application", async () => { @@ -60,6 +70,32 @@ describe("lockCategoriesRecords", () => { "outro", "shilling", ], + submittedValues: [ + { + actionType: "skip", + category: "outro" + }, + { + actionType: "mute", + category: "outro" + }, + { + actionType: "skip", + category: "shilling" + }, + { + actionType: "mute", + category: "shilling" + }, + { + actionType: "skip", + category: "intro" + }, + { + actionType: "mute", + category: "intro" + } + ] }; client.post(endpoint, json) .then(res => { @@ -88,15 +124,15 @@ describe("lockCategoriesRecords", () => { .then(async res => { assert.strictEqual(res.status, 200); const result = await checkLockCategories(videoID); - assert.strictEqual(result.length, 4); + assert.strictEqual(result.length, 8); const oldRecordNotChangeReason = result.filter(item => item.reason === "reason-2" && ["sponsor", "intro"].includes(item.category) ); const newRecordWithEmptyReason = result.filter(item => item.reason === "" && ["outro", "shilling"].includes(item.category) ); - assert.strictEqual(newRecordWithEmptyReason.length, 2); - assert.strictEqual(oldRecordNotChangeReason.length, 2); + assert.strictEqual(newRecordWithEmptyReason.length, 4); + assert.strictEqual(oldRecordNotChangeReason.length, 4); done(); }) .catch(err => done(err)); @@ -160,7 +196,7 @@ describe("lockCategoriesRecords", () => { .then(async res => { assert.strictEqual(res.status, 200); const result = await checkLockCategories(videoID); - assert.strictEqual(result.length, 4); + assert.strictEqual(result.length, 8); const newRecordWithNewReason = result.filter(item => expectedWithNewReason.includes(item.category) && item.reason === "new reason" ); @@ -168,8 +204,8 @@ describe("lockCategoriesRecords", () => { item.reason === "reason-2" ); - assert.strictEqual(newRecordWithNewReason.length, 3); - assert.strictEqual(oldRecordNotChangeReason.length, 1); + assert.strictEqual(newRecordWithNewReason.length, 6); + assert.strictEqual(oldRecordNotChangeReason.length, 2); done(); }) .catch(err => done(err)); @@ -187,7 +223,7 @@ describe("lockCategoriesRecords", () => { .then(async res => { assert.strictEqual(res.status, 200); const result = await checkLockCategories("underscore"); - assert.strictEqual(result.length, 1); + assert.strictEqual(result.length, 2); done(); }) .catch(err => done(err)); @@ -205,7 +241,7 @@ describe("lockCategoriesRecords", () => { .then(async res => { assert.strictEqual(res.status, 200); const result = await checkLockCategories("bothCases"); - assert.strictEqual(result.length, 1); + assert.strictEqual(result.length, 2); done(); }) .catch(err => done(err)); @@ -231,6 +267,41 @@ describe("lockCategoriesRecords", () => { .catch(err => done(err)); }); + it("Should be able to submit specific action type not in video (sql check)", (done) => { + const videoID = "lockCategoryVideo-2"; + const json = { + videoID, + userID: lockVIPUser, + categories: [ + "sponsor", + ], + actionTypes: [ + "mute" + ], + reason: "custom-reason", + }; + client.post(endpoint, json) + .then(async res => { + assert.strictEqual(res.status, 200); + const result = await checkLockCategories(videoID); + assert.strictEqual(result.length, 2); + assert.ok(partialDeepEquals(result, [ + { + category: "sponsor", + actionType: "skip", + reason: "reason-4", + }, + { + category: "sponsor", + actionType: "mute", + reason: "custom-reason", + } + ])); + done(); + }) + .catch(err => done(err)); + }); + it("Should return 400 for missing params", (done) => { client.post(endpoint, {}) .then(res => { @@ -365,7 +436,7 @@ describe("lockCategoriesRecords", () => { .then(async res => { assert.strictEqual(res.status, 200); const result = await checkLockCategories(videoID); - assert.strictEqual(result.length, 1); + assert.strictEqual(result.length, 2); done(); }) .catch(err => done(err)); diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index c891e51..8bad132 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -901,6 +901,26 @@ describe("postSkipSegments", () => { .catch(err => done(err)); }); + it("Should return not be 403 when submitting with locked category but unlocked actionType", (done) => { + const videoID = "lockedVideo"; + db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") + VALUES(?, ?, ?, ?)`, [getHash("VIPUser-lockCategories"), videoID, "sponsor", "Custom Reason"]) + .then(() => postSkipSegmentJSON({ + userID: submitUserOne, + videoID, + segments: [{ + segment: [1, 10], + category: "sponsor", + actionType: "mute" + }], + })) + .then(res => { + assert.strictEqual(res.status, 200); + done(); + }) + .catch(err => done(err)); + }); + it("Should return 403 for submiting in lockedCategory", (done) => { const videoID = "lockedVideo1"; db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") From 8fef35dbbc1f025a7a8aeb9c8e89696daf3fabd3 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 2 Jan 2022 22:38:06 -0500 Subject: [PATCH 02/14] Allow submitting full sponsors and selfpromo --- src/config.ts | 4 ++-- src/routes/postSkipSegments.ts | 9 ++++++--- src/types/segments.model.ts | 3 ++- test/cases/postSkipSegments.ts | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index c666a90..65b9b10 100644 --- a/src/config.ts +++ b/src/config.ts @@ -21,8 +21,8 @@ addDefaults(config, { webhooks: [], categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], categorySupport: { - sponsor: ["skip", "mute"], - selfpromo: ["skip", "mute"], + sponsor: ["skip", "mute", "full"], + selfpromo: ["skip", "mute", "full"], interaction: ["skip", "mute"], intro: ["skip", "mute"], outro: ["skip", "mute"], diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 6655236..88bc9af 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -383,8 +383,10 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime - || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable && startTime === endTime) - || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime)) { + || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable + && segments[i].actionType !== ActionType.Full && startTime === endTime) + || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime) + || (segments[i].actionType === ActionType.Full && (startTime !== 0 || endTime !== 0))) { //invalid request return { pass: false, errorMessage: "One of your segments times are invalid (too short, startTime before endTime, etc.)", errorCode: 400 }; } @@ -394,7 +396,8 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, return { pass: false, errorMessage: `POI cannot be that early`, errorCode: 400 }; } - if (!isVIP && segments[i].category === "sponsor" && Math.abs(startTime - endTime) < 1) { + if (!isVIP && segments[i].category === "sponsor" + && segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) { // Too short return { pass: false, errorMessage: "Sponsors must be longer than 1 second long", errorCode: 400 }; } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index ba613c4..4bb28d7 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -13,7 +13,8 @@ export type HashedIP = IPAddress & HashedValue; export enum ActionType { Skip = "skip", Mute = "mute", - Chapter = "chapter" + Chapter = "chapter", + Full = "full" } // Uncomment as needed diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 8bad132..bf89dd0 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -1064,6 +1064,40 @@ describe("postSkipSegments", () => { .catch(err => done(err)); }); + it("Should allow submitting full video sponsor", (done) => { + const videoID = "qqwerth"; + postSkipSegmentParam({ + videoID, + startTime: 0, + endTime: 0, + category: "sponsor", + actionType: "full", + userID: submitUserTwo + }) + .then(res => { + assert.strictEqual(res.status, 200); + done(); + }) + .catch(err => done(err)); + }); + + it("Should not allow submitting full video sponsor not at zero seconds", (done) => { + const videoID = "qqwerth"; + postSkipSegmentParam({ + videoID, + startTime: 0, + endTime: 1, + category: "sponsor", + actionType: "full", + userID: submitUserTwo + }) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + it("Should not be able to submit with colons in timestamps", (done) => { const videoID = "colon-1"; postSkipSegmentJSON({ From 89ea13956a37fa19ef856b568d60bb1f7424c1d2 Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 4 Jan 2022 19:27:50 -0500 Subject: [PATCH 03/14] Ensure only one full video label is served --- src/routes/getSkipSegments.ts | 5 +++-- test/cases/getSkipSegmentsByHash.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index f1ecabf..3001414 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -231,7 +231,8 @@ async function chooseSegments(videoID: VideoID, service: Service, segments: DBSe : await fetchData(); // Filter for only 1 item for POI categories - return getWeightedRandomChoice(groups, 1, (choice) => getCategoryActionType(choice.segments[0].category) === CategoryActionType.POI) + return getWeightedRandomChoice(groups, 1, (choice) => choice.segments[0].actionType === ActionType.Full + || getCategoryActionType(choice.segments[0].category) === CategoryActionType.POI) .map(//randomly choose 1 good segment per group and return them group => getWeightedRandomChoice(group.segments, 1)[0] ); @@ -300,7 +301,7 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm group.segments.forEach((segment) => { const bestGroup = result.find((group) => { // At least one segment in the group must have high % overlap or the same action type - // Since POI segments will always have 0 overlap, they will always be in their own groups + // Since POI and Full video segments will always have <= 0 overlap, they will always be in their own groups return group.segments.some((compareSegment) => { const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts index 7cf2081..afbaab5 100644 --- a/test/cases/getSkipSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -18,6 +18,7 @@ describe("getSkipSegmentsByHash", () => { const requiredSegmentHashVidHash = "17bf8d9090e050257772f8bff277293c29c7ce3b25eb969a8fae111a2434504d"; const differentCategoryVidHash = "7fac44d1ee3257ec7f18953e2b5f991828de6854ad57193d1027c530981a89c0"; const nonMusicOverlapVidHash = "306151f778f9bfd19872b3ccfc83cbab37c4f370717436bfd85e0a624cd8ba3c"; + const fullCategoryVidHash = "278fa987eebfe07ae3a4a60cf0663989ad874dd0c1f0430831d63c2001567e6f"; before(async () => { const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "actionType", "service", "hidden", "shadowHidden", "hashedVideoID", "description") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; await db.prepare("run", query, ["getSegmentsByHash-0", 1, 10, 2, 0, "getSegmentsByHash-01", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getSegmentsByHash0Hash, ""]); @@ -50,6 +51,8 @@ describe("getSkipSegmentsByHash", () => { await db.prepare("run", query, ["differentCategoryVid", 60, 70, 2, 1, "differentCategoryVid-2", "testman", 0, 50, "intro", "skip", "YouTube", 0, 0, differentCategoryVidHash, ""]); await db.prepare("run", query, ["nonMusicOverlapVid", 60, 70, 2, 0, "nonMusicOverlapVid-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, nonMusicOverlapVidHash, ""]); await db.prepare("run", query, ["nonMusicOverlapVid", 60, 70, 2, 1, "nonMusicOverlapVid-2", "testman", 0, 50, "music_offtopic", "skip", "YouTube", 0, 0, nonMusicOverlapVidHash, ""]); + await db.prepare("run", query, ["fullCategoryVid", 60, 70, 2, 0, "fullCategoryVid-1", "testman", 0, 50, "sponsor", "full", "YouTube", 0, 0, nonMusicOverlapVidHash, ""]); + await db.prepare("run", query, ["fullCategoryVid", 60, 70, 2, 1, "fullCategoryVid-2", "testman", 0, 50, "selfpromo", "full", "YouTube", 0, 0, fullCategoryVidHash, ""]); }); it("Should be able to get a 200", (done) => { @@ -526,6 +529,19 @@ describe("getSkipSegmentsByHash", () => { .catch(err => done(err)); }); + it("Should only return one segment when fetching full video segments", (done) => { + client.get(`${endpoint}/278f`, { params: { category: ["sponsor", "selfpromo"], actionType: "full" } }) + .then(res => { + assert.strictEqual(res.status, 200); + const data = res.data; + assert.strictEqual(data.length, 1); + assert.strictEqual(data[0].segments.length, 1); + assert.strictEqual(data[0].segments[0].category, "selfpromo"); + done(); + }) + .catch(err => done(err)); + }); + it("Should be able to get specific segments with partial requiredSegments", (done) => { const requiredSegment1 = "fbf0af454059733c8822f6a4ac8ec568e0787f8c0a5ee915dd5b05e0d7a9a388"; const requiredSegment2 = "7e1ebc5194551d2d0a606d64f675e5a14952e4576b2959f8c9d51e316c14f8da"; From 65954520d046e496ce29b21e5403586cdebefc17 Mon Sep 17 00:00:00 2001 From: Ajay Date: Thu, 6 Jan 2022 03:39:46 -0500 Subject: [PATCH 04/14] Treat duplicate full video submission as upvote --- src/routes/postSkipSegments.ts | 36 ++++++++++------- src/routes/voteOnSponsorTime.ts | 68 +++++++++++++++++++-------------- src/types/segments.model.ts | 3 ++ 3 files changed, 66 insertions(+), 41 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 88bc9af..3a4a294 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -9,17 +9,18 @@ import { getIP } from "../utils/getIP"; import { getFormattedTime } from "../utils/getFormattedTime"; import { dispatchEvent } from "../utils/webhookUtils"; import { Request, Response } from "express"; -import { ActionType, Category, CategoryActionType, IncomingSegment, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model"; +import { ActionType, Category, CategoryActionType, IncomingSegment, IPAddress, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model"; import { deleteLockCategories } from "./deleteLockCategories"; import { getCategoryActionType } from "../utils/categoryInfo"; import { QueryCacher } from "../utils/queryCacher"; import { getReputation } from "../utils/reputation"; import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model"; -import { UserID } from "../types/user.model"; +import { HashedUserID, UserID } from "../types/user.model"; import { isUserVIP } from "../utils/isUserVIP"; import { parseUserAgent } from "../utils/userAgent"; import { getService } from "../utils/getService"; import axios from "axios"; +import { vote } from "./voteOnSponsorTime"; type CheckResult = { pass: boolean, @@ -343,7 +344,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming return CHECK_PASS; } -async function checkEachSegmentValid(userID: string, videoID: VideoID, +async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, userID: HashedUserID, videoID: VideoID, segments: IncomingSegment[], service: string, isVIP: boolean, lockedCategoryList: Array): Promise { for (let i = 0; i < segments.length; i++) { @@ -399,14 +400,21 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, if (!isVIP && segments[i].category === "sponsor" && segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) { // Too short - return { pass: false, errorMessage: "Sponsors must be longer than 1 second long", errorCode: 400 }; + return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 }; } //check if this info has already been submitted before - const duplicateCheck2Row = await db.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ? + const duplicateCheck2Row = await db.prepare("get", `SELECT "UUID" FROM "sponsorTimes" WHERE "startTime" = ? and "endTime" = ? and "category" = ? and "actionType" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, segments[i].actionType, videoID, service]); - if (duplicateCheck2Row.count > 0) { - return { pass: false, errorMessage: "Sponsors has already been submitted before.", errorCode: 409 }; + if (duplicateCheck2Row) { + if (segments[i].actionType === ActionType.Full) { + // Forward as vote + vote(rawIP, duplicateCheck2Row.UUID, paramUserID, 1); + segments[i].ignoreSegment = true; + continue; + } else { + return { pass: false, errorMessage: "Segment has already been submitted before.", errorCode: 409 }; + } } } @@ -576,15 +584,15 @@ export async function postSkipSegments(req: Request, res: Response): Promise { + , hashedIP: HashedIP, finalResponse: FinalResponse): Promise<{ status: number, message?: string }> { // Check if they've already made a vote const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); if (usersLastVoteInfo?.category === category) { // Double vote, ignore - return res.sendStatus(finalResponse.finalStatus); + return { status: finalResponse.finalStatus }; } const videoInfo = (await db.prepare("get", `SELECT "category", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; if (!videoInfo) { // Submission doesn't exist - return res.status(400).send("Submission doesn't exist."); + return { status: 400, message: "Submission doesn't exist." }; } if (!config.categoryList.includes(category)) { - return res.status(400).send("Category doesn't exist."); + return { status: 400, message: "Category doesn't exist." }; } if (getCategoryActionType(category) !== CategoryActionType.Skippable) { - return res.status(400).send("Cannot vote for this category"); + return { status: 400, message: "Cannot vote for this category" }; } // Ignore vote if the next category is locked const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [videoInfo.videoID, videoInfo.service, category]); if (nextCategoryLocked && !isVIP) { - return res.sendStatus(200); + return { status: 200 }; } // Ignore vote if the segment is locked if (!isVIP && videoInfo.locked === 1) { - return res.sendStatus(200); + return { status: 200 }; } const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]); @@ -279,7 +279,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i QueryCacher.clearSegmentCache(videoInfo); - return res.sendStatus(finalResponse.finalStatus); + return { status: finalResponse.finalStatus }; } export function getUserID(req: Request): UserID { @@ -289,16 +289,30 @@ export function getUserID(req: Request): UserID { export async function voteOnSponsorTime(req: Request, res: Response): Promise { const UUID = req.query.UUID as SegmentUUID; const paramUserID = getUserID(req); - let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; + const type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; const category = req.query.category as Category; + const ip = getIP(req); + const result = await vote(ip, UUID, paramUserID, type, category); + + const response = res.status(result.status); + if (result.message) { + return response.send(result.message); + } else if (result.json) { + return response.json(result.json); + } else { + return response.send(); + } +} + +export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID, type: number, category?: Category): Promise<{ status: number, message?: string, json?: unknown }> { if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) { //invalid request - return res.sendStatus(400); + return { status: 400 }; } if (paramUserID.length < 30 && config.mode !== "test") { // Ignore this vote, invalid - return res.sendStatus(200); + return { status: 200 }; } //hash the userID @@ -314,9 +328,6 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise