From 6adfb84c0fe7de9f1f26d13baa8df39f8431553b Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 27 Oct 2021 15:55:11 -0400 Subject: [PATCH 01/18] add generate-sqlite-base --- .github/workflows/generate-sqlite-base.yml | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/generate-sqlite-base.yml diff --git a/.github/workflows/generate-sqlite-base.yml b/.github/workflows/generate-sqlite-base.yml new file mode 100644 index 0000000..9f1a68e --- /dev/null +++ b/.github/workflows/generate-sqlite-base.yml @@ -0,0 +1,23 @@ +name: create-sqlite-base +on: + push: + paths: + - databases/** + +jobs: + make-base-db: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + - run: npm install + - name: Set config + run: | + echo '{"mode": "init-db-and-exit"}' > config.json + - name: Run Server + timeout-minutes: 10 + run: npm start + - uses: actions/upload-artifact@v2 + with: + name: SponsorTimesDB.db + path: databases/sponsorTimes.db \ No newline at end of file From 9d761815d8850c6703e1fed362c4a131e6e39bc4 Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 27 Oct 2021 17:03:16 -0400 Subject: [PATCH 02/18] add userID to getSkipSegments --- src/routes/getSkipSegments.ts | 3 ++- test/cases/getSkipSegments.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index b1715a5..c5ae3de 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -53,7 +53,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: UUID: chosenSegment.UUID, locked: chosenSegment.locked, votes: chosenSegment.votes, - videoDuration: chosenSegment.videoDuration + videoDuration: chosenSegment.videoDuration, + userID: chosenSegment.userID, })); } diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index cbdf724..63eb208 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -39,6 +39,7 @@ describe("getSkipSegments", () => { assert.strictEqual(data[0].votes, 1); assert.strictEqual(data[0].locked, 0); assert.strictEqual(data[0].videoDuration, 100); + assert.strictEqual(data[0].userID, "testman"); done(); }) .catch(err => done(err)); From 88a368d0b9f543e93a5007bc8adec79b59f257d4 Mon Sep 17 00:00:00 2001 From: Michael C Date: Sat, 30 Oct 2021 17:58:17 -0400 Subject: [PATCH 03/18] add tests for shadowban --- src/routes/postSkipSegments.ts | 3 +-- test/cases/postSkipSegments.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index ae68983..c65e770 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -620,7 +620,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise { const warnUser03Hash = getHash(warnUser03); const warnUser04 = "warn-user04-qwertyuiopasdfghjklzxcvbnm"; const warnUser04Hash = getHash(warnUser04); + const banUser01 = "ban-user01-loremipsumdolorsitametconsectetur"; + const banUser01Hash = getHash(banUser01); const submitUserOneHash = getHash(submitUserOne); const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`; const warnVideoID = "postSkip2"; const badInputVideoID = "dQw4w9WgXcQ"; + const shadowBanVideoID = "postSkipBan"; const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); const queryDatabaseActionType = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category", "actionType" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); @@ -88,6 +91,9 @@ describe("postSkipSegments", () => { const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; db.prepare("run", insertVipUserQuery, [getHash(submitVIPuser)]); + + // ban user + db.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES(?)`, [banUser01Hash]); }); it("Should be able to submit a single time (Params method)", (done) => { @@ -985,4 +991,28 @@ describe("postSkipSegments", () => { }) .catch(err => done(err)); }); + + it("Should automatically shadowban segments if user is banned", (done) => { + const videoID = shadowBanVideoID; + postSkipSegmentParam({ + videoID, + startTime: 0, + endTime: 10, + category: "sponsor", + userID: banUser01 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const row = await db.prepare("get", `SELECT "startTime", "endTime", "shadowHidden", "userID" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); + const expected = { + startTime: 0, + endTime: 10, + shadowHidden: 1, + userID: banUser01Hash + }; + assert.deepStrictEqual(row, expected); + done(); + }) + .catch(err => done(err)); + }); }); From 32150e4a1dd55a2dd5478ef27602e07a684d65c0 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 00:54:28 -0400 Subject: [PATCH 04/18] Add submitting description for chapters --- ci.json | 1 - databases/_upgrade_sponsorTimes_27.sql | 8 +++ src/config.ts | 3 +- src/routes/postSkipSegments.ts | 19 +++++-- src/types/segments.model.ts | 4 +- test.json | 1 - test/cases/getLockReason.ts | 3 +- test/cases/postSkipSegments.ts | 69 ++++++++++++++++++++++++++ 8 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_27.sql diff --git a/ci.json b/ci.json index 32ef176..2f948b7 100644 --- a/ci.json +++ b/ci.json @@ -46,7 +46,6 @@ ] } ], - "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight"], "maxNumberOfActiveWarnings": 3, "hoursAfterWarningExpires": 24, "rateLimit": { diff --git a/databases/_upgrade_sponsorTimes_27.sql b/databases/_upgrade_sponsorTimes_27.sql new file mode 100644 index 0000000..8887875 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_27.sql @@ -0,0 +1,8 @@ +BEGIN TRANSACTION; + +ALTER TABLE "sponsorTimes" ADD "description" TEXT NOT NULL default ''; +ALTER TABLE "archivedSponsorTimes" ADD "description" TEXT NOT NULL default ''; + +UPDATE "config" SET value = 27 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index d28d2aa..ed6c440 100644 --- a/src/config.ts +++ b/src/config.ts @@ -19,7 +19,7 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight"], + categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight", "chapter"], categorySupport: { sponsor: ["skip", "mute"], selfpromo: ["skip", "mute"], @@ -29,6 +29,7 @@ addDefaults(config, { preview: ["skip"], music_offtopic: ["skip"], poi_highlight: ["skip"], + chapter: ["chapter"] }, maxNumberOfActiveWarnings: 1, hoursAfterWarningExpires: 24, diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index c65e770..09b6c06 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -299,7 +299,7 @@ async function checkUserActiveWarning(userID: string): Promise { return CHECK_PASS; } -function checkInvalidFields(videoID: any, userID: any, segments: Array): CheckResult { +function checkInvalidFields(videoID: VideoID, userID: UserID, segments: IncomingSegment[]): CheckResult { const invalidFields = []; const errors = []; if (typeof videoID !== "string") { @@ -320,6 +320,12 @@ function checkInvalidFields(videoID: any, userID: any, segments: Array): Ch (typeof endTime === "string" && endTime.includes(":"))) { invalidFields.push("segment time"); } + + if (typeof segmentPair.description !== "string" + || (segmentPair.description.length > 60 && segmentPair.actionType === ActionType.Chapter) + || (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) { + invalidFields.push("segment description"); + } } if (invalidFields.length !== 0) { @@ -541,7 +547,8 @@ function preprocessInput(req: Request) { segments = [{ segment: [req.query.startTime as string, req.query.endTime as string], category: req.query.category as Category, - actionType: (req.query.actionType as ActionType) ?? ActionType.Skip + actionType: (req.query.actionType as ActionType) ?? ActionType.Skip, + description: req.query.description as string || "", }]; } // Add default action type @@ -550,6 +557,7 @@ function preprocessInput(req: Request) { segment.actionType = ActionType.Skip; } + segment.description ??= ""; segment.segment = segment.segment.map((time) => typeof segment.segment[0] === "string" ? time?.replace(",", ".") : time); }); @@ -633,9 +641,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise { { category: "outro", locked: 1, reason: "outro-reason", userID: vipUserID2, userName: vipUserName2 }, { category: "preview", locked: 1, reason: "preview-reason", userID: vipUserID1, userName: vipUserName1 }, { category: "music_offtopic", locked: 1, reason: "nonmusic-reason", userID: vipUserID1, userName: vipUserName1 }, - { category: "poi_highlight", locked: 0, reason: "", userID: "", userName: "" } + { category: "poi_highlight", locked: 0, reason: "", userID: "", userName: "" }, + { category: "chapter", locked: 0, reason: "", userID: "", userName: "" } ]; assert.deepStrictEqual(res.data, expected); done(); diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index b4d7f4f..2784d23 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -37,6 +37,7 @@ describe("postSkipSegments", () => { const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); const queryDatabaseActionType = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category", "actionType" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); + const queryDatabaseChapter = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category", "actionType", "description" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); const queryDatabaseDuration = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); const queryDatabaseVideoInfo = (videoID: string) => db.prepare("get", `SELECT * FROM "videoInfo" WHERE "videoID" = ?`, [videoID]); @@ -181,6 +182,34 @@ describe("postSkipSegments", () => { .catch(err => done(err)); }); + it("Should be able to submit a single chapter (JSON method)", (done) => { + const videoID = "postSkipChapter1"; + postSkipSegmentJSON({ + userID: submitUserOne, + videoID, + segments: [{ + segment: [0, 10], + category: "chapter", + actionType: "chapter", + description: "This is a chapter" + }], + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const row = await queryDatabaseChapter(videoID); + const expected = { + startTime: 0, + endTime: 10, + category: "chapter", + actionType: "chapter", + description: "This is a chapter" + }; + assert.ok(partialDeepEquals(row, expected)); + done(); + }) + .catch(err => done(err)); + }); + it("Should not be able to submit an intro with mute action type (JSON method)", (done) => { const videoID = "postSkip4"; postSkipSegmentJSON({ @@ -201,6 +230,46 @@ describe("postSkipSegments", () => { .catch(err => done(err)); }); + it("Should not be able to submit a chapter with skip action type (JSON method)", (done) => { + const videoID = "postSkipChapter2"; + postSkipSegmentJSON({ + userID: submitUserOne, + videoID, + segments: [{ + segment: [0, 10], + category: "chapter", + actionType: "skip" + }], + }) + .then(async res => { + assert.strictEqual(res.status, 400); + const row = await queryDatabaseActionType(videoID); + assert.strictEqual(row, undefined); + done(); + }) + .catch(err => done(err)); + }); + + it("Should not be able to submit a sponsor with a description (JSON method)", (done) => { + const videoID = "postSkipChapter3"; + postSkipSegmentJSON({ + userID: submitUserOne, + videoID, + segments: [{ + segment: [0, 10], + category: "sponsor", + description: "This is a sponsor" + }], + }) + .then(async res => { + assert.strictEqual(res.status, 400); + const row = await queryDatabaseActionType(videoID); + assert.strictEqual(row, undefined); + done(); + }) + .catch(err => done(err)); + }); + it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => { const videoID = "postSkip5"; postSkipSegmentJSON({ From 7eef74a7dc4bb60522424ce786373a414e869642 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 16:09:55 -0400 Subject: [PATCH 05/18] Add new % overlap function used for chapters Kind of #107 --- src/routes/getSkipSegments.ts | 79 +++++++++++++++++++++-- src/types/segments.model.ts | 1 + test/cases/getSkipSegments.ts | 64 ++++++++++++++----- test/cases/getSkipSegmentsByHash.ts | 98 ++++++++++++++++++++++++----- test/utils/partialDeepEquals.ts | 3 +- 5 files changed, 208 insertions(+), 37 deletions(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index c5ae3de..a966cee 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -45,7 +45,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: const filteredSegments = segments.filter((_, index) => shouldFilter[index]); - const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1; + const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? Infinity : 1; return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({ category: chosenSegment.category, actionType: chosenSegment.actionType, @@ -55,6 +55,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: votes: chosenSegment.votes, videoDuration: chosenSegment.videoDuration, userID: chosenSegment.userID, + description: chosenSegment.description })); } @@ -139,7 +140,7 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service const fetchFromDB = () => db .prepare( "all", - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted" FROM "sponsorTimes" + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [`${hashedVideoIDPrefix}%`, service] ) as Promise; @@ -155,7 +156,7 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P const fetchFromDB = () => db .prepare( "all", - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "timeSubmitted" FROM "sponsorTimes" + `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "timeSubmitted", "description" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [videoID, service] ) as Promise; @@ -219,7 +220,7 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise cursor), no other segment will ever fall // inside that group (because they're sorted) so we can create a new one - const overlappingSegmentsGroups: OverlappingSegmentGroup[] = []; + let overlappingSegmentsGroups: OverlappingSegmentGroup[] = []; let currentGroup: OverlappingSegmentGroup; let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created for (const segment of segments) { @@ -261,6 +262,8 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise 0) { + const currentGroup: OverlappingSegmentGroup = { segments: [], votes: 0, reputation: 0, locked: false, required: false }; + + const currentSegment = segmentsLeftToCheck.shift(); + // TODO: extract out this part to be more generic + currentGroup.segments.push(currentSegment); + currentGroup.votes += currentSegment.votes; + currentGroup.reputation += currentSegment.reputation; + currentGroup.locked ||= currentSegment.locked; + currentGroup.required ||= currentSegment.required; + + const currentDuration = currentSegment.endTime - currentSegment.startTime; + for (const [index, compareSegment] of segmentsLeftToCheck.entries()) { + const compareDuration = compareSegment.endTime - compareSegment.startTime; + const overlap = Math.min(currentSegment.endTime, compareSegment.endTime) - Math.max(currentSegment.startTime, compareSegment.startTime); + const overlapPercent = overlap / Math.max(currentDuration, compareDuration); + if (overlapPercent >= percent) { + currentGroup.segments.push(currentSegment); + currentGroup.votes += compareSegment.votes; + currentGroup.reputation += compareSegment.reputation; + currentGroup.locked ||= compareSegment.locked; + currentGroup.required ||= compareSegment.required; + segmentsLeftToCheck.splice(index, 1); + } + } + + result.push(currentGroup); + } + } + + return result; +} + +// function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegmentGroup[] { +// return groups.flatMap((group) => { +// const result: OverlappingSegmentGroup[] = []; +// group.segments.forEach((segment) => { +// const bestGroup = result.find((group) => { +// // At least one segment in the group must have high % overlap or the same action type +// return group.segments.some((compareSegment) => { +// const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); +// const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); +// const overlapPercent = overlap / overallDuration; +// return (segment.actionType === compareSegment.actionType && segment.actionType !== ActionType.Chapter) +// || overlapPercent >= 0.6 +// || (overlapPercent >= 0.8 && segment.actionType === ActionType.Chapter && compareSegment.actionType === ActionType.Chapter); +// }); +// }); + +// if (bestGroup) { +// bestGroup.segments.push(segment); +// bestGroup.votes += segment.votes; +// bestGroup.reputation += segment.reputation; +// bestGroup.locked ||= segment.locked; +// bestGroup.required ||= segment.required; +// } else { +// result.push({ segments: [segment], votes: segment.votes, reputation: segment.reputation, locked: segment.locked, required: segment.required }); +// } +// }); + +// return result; +// }); +// } + /** * * Returns what would be sent to the client. diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 41b9408..e4a8375 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -67,6 +67,7 @@ export interface DBSegment { timeSubmitted: number; userAgent: string; service: Service; + description: string; } export interface OverlappingSegmentGroup { diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index 63eb208..9b1b38c 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -6,23 +6,26 @@ import { client } from "../utils/httpClient"; describe("getSkipSegments", () => { const endpoint = "/api/skipSegments"; before(async () => { - const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "hidden", "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; - await db.prepare("run", query, ["getSkipSegmentID0", 1, 11, 1, 0, "uuid01", "testman", 0, 50, "sponsor", "skip", "YouTube", 100, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID0", 12, 14, 2, 0, "uuid02", "testman", 0, 50, "sponsor", "mute", "YouTube", 100, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID0", 20, 33, 2, 0, "uuid03", "testman", 0, 50, "intro", "skip", "YouTube", 101, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID1", 1, 11, 2, 0, "uuid10", "testman", 0, 50, "sponsor", "skip", "PeerTube", 120, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID2", 1, 11, 2, 1, "uuid20", "testman", 0, 50, "sponsor", "skip", "YouTube", 140, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID3", 1, 11, 2, 0, "uuid30", "testman", 0, 50, "sponsor", "skip", "YouTube", 200, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID3", 7, 22, -3, 0, "uuid31", "testman", 0, 50, "sponsor", "skip", "YouTube", 300, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentMultiple", 1, 11, 2, 0, "uuid40", "testman", 0, 50, "intro", "skip", "YouTube", 400, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentMultiple", 20, 33, 2, 0, "uuid41", "testman", 0, 50, "intro", "skip", "YouTube", 500, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentLocked", 20, 33, 2, 1, "uuid50", "testman", 0, 50, "intro", "skip", "YouTube", 230, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentLocked", 20, 34, 100000, 0, "uuid51", "testman", 0, 50, "intro", "skip", "YouTube", 190, 0, 0]); - await db.prepare("run", query, ["getSkipSegmentID6", 20, 34, 100000, 0, "uuid60", "testman", 0, 50, "sponsor", "skip", "YouTube", 190, 1, 0]); - await db.prepare("run", query, ["requiredSegmentVid", 60, 70, 2, 0, "requiredSegmentVid1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0]); - await db.prepare("run", query, ["requiredSegmentVid", 60, 70, -2, 0, "requiredSegmentVid2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0]); - await db.prepare("run", query, ["requiredSegmentVid", 80, 90, -2, 0, "requiredSegmentVid3", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0]); - await db.prepare("run", query, ["requiredSegmentVid", 80, 90, 2, 0, "requiredSegmentVid4", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0]); + const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "hidden", "shadowHidden", "description") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + await db.prepare("run", query, ["getSkipSegmentID0", 1, 11, 1, 0, "uuid01", "testman", 0, 50, "sponsor", "skip", "YouTube", 100, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID0", 12, 14, 2, 0, "uuid02", "testman", 0, 50, "sponsor", "mute", "YouTube", 100, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID0", 20, 33, 2, 0, "uuid03", "testman", 0, 50, "intro", "skip", "YouTube", 101, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID1", 1, 11, 2, 0, "uuid10", "testman", 0, 50, "sponsor", "skip", "PeerTube", 120, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID2", 1, 11, 2, 1, "uuid20", "testman", 0, 50, "sponsor", "skip", "YouTube", 140, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID3", 1, 11, 2, 0, "uuid30", "testman", 0, 50, "sponsor", "skip", "YouTube", 200, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID3", 7, 22, -3, 0, "uuid31", "testman", 0, 50, "sponsor", "skip", "YouTube", 300, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentMultiple", 1, 11, 2, 0, "uuid40", "testman", 0, 50, "intro", "skip", "YouTube", 400, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentMultiple", 20, 33, 2, 0, "uuid41", "testman", 0, 50, "intro", "skip", "YouTube", 500, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentLocked", 20, 33, 2, 1, "uuid50", "testman", 0, 50, "intro", "skip", "YouTube", 230, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentLocked", 20, 34, 100000, 0, "uuid51", "testman", 0, 50, "intro", "skip", "YouTube", 190, 0, 0, ""]); + await db.prepare("run", query, ["getSkipSegmentID6", 20, 34, 100000, 0, "uuid60", "testman", 0, 50, "sponsor", "skip", "YouTube", 190, 1, 0, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 60, 70, 2, 0, "requiredSegmentVid1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 60, 70, -2, 0, "requiredSegmentVid2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 80, 90, -2, 0, "requiredSegmentVid3", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 80, 90, 2, 0, "requiredSegmentVid4", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, 0, ""]); + await db.prepare("run", query, ["chapterVid", 60, 80, 2, 0, "chapterVid-1", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Chapter 1"]); + await db.prepare("run", query, ["chapterVid", 70, 75, 2, 0, "chapterVid-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Chapter 2"]); + await db.prepare("run", query, ["chapterVid", 71, 76, 2, 0, "chapterVid-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Chapter 3"]); return; }); @@ -388,6 +391,33 @@ describe("getSkipSegments", () => { .catch(err => done(err)); }); + it("Should be able to get overlapping chapter segments if very different", (done) => { + client.get(`${endpoint}?videoID=chapterVid&category=chapter&actionType=chapter`) + .then(res => { + assert.strictEqual(res.status, 200); + const data = res.data; + assert.strictEqual(data.length, 2); + const expected = [{ + UUID: "chapterVid-1", + description: "Chapter 1" + }, { + UUID: "chapterVid-2", + description: "Chapter 2" + }]; + const expected2 = [{ + UUID: "chapterVid-1", + description: "Chapter 1" + }, { + UUID: "chapterVid-3", + description: "Chapter 3" + }]; + + assert.ok(partialDeepEquals(data, expected, false) || partialDeepEquals(data, expected2)); + done(); + }) + .catch(err => done(err)); + }); + it("Should get 400 if no videoID passed in", (done) => { client.get(endpoint) .then(res => { diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts index fb83ec0..e9da156 100644 --- a/test/cases/getSkipSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -16,20 +16,26 @@ describe("getSkipSegmentsByHash", () => { const getSegmentsByHash0Hash = "fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910"; const requiredSegmentVidHash = "d51822c3f681e07aef15a8855f52ad12db9eb9cf059e65b16b64c43359557f61"; before(async () => { - const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "actionType", "service", "hidden", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; - await db.prepare("run", query, ["getSegmentsByHash-0", 1, 10, 2, "getSegmentsByHash-01", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getSegmentsByHash0Hash]); - await db.prepare("run", query, ["getSegmentsByHash-0", 1, 10, 2, "getSegmentsByHash-02", "testman", 0, 50, "sponsor", "skip", "PeerTube", 0, 0, getSegmentsByHash0Hash]); - await db.prepare("run", query, ["getSegmentsByHash-0", 20, 30, 2, "getSegmentsByHash-03", "testman", 100, 150, "intro", "skip", "YouTube", 0, 0, getSegmentsByHash0Hash]); - await db.prepare("run", query, ["getSegmentsByHash-0", 40, 50, 2, "getSegmentsByHash-04", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getSegmentsByHash0Hash]); - await db.prepare("run", query, ["getSegmentsByHash-noMatchHash", 40, 50, 2, "getSegmentsByHash-noMatchHash", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, "fdaffnoMatchHash"]); - await db.prepare("run", query, ["getSegmentsByHash-1", 60, 70, 2, "getSegmentsByHash-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, "3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b"]); - await db.prepare("run", query, ["onlyHidden", 60, 70, 2, "onlyHidden", "testman", 0, 50, "sponsor", "skip", "YouTube", 1, 0, "f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3"]); - await db.prepare("run", query, ["highlightVid", 60, 60, 2, "highlightVid-1", "testman", 0, 50, "poi_highlight", "skip", "YouTube", 0, 0, getHash("highlightVid", 1)]); - await db.prepare("run", query, ["highlightVid", 70, 70, 2, "highlightVid-2", "testman", 0, 50, "poi_highlight", "skip", "YouTube", 0, 0, getHash("highlightVid", 1)]); - await db.prepare("run", query, ["requiredSegmentVid", 60, 70, 2, "requiredSegmentVid-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash]); - await db.prepare("run", query, ["requiredSegmentVid", 60, 70, -2, "requiredSegmentVid-2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash]); - await db.prepare("run", query, ["requiredSegmentVid", 80, 90, -2, "requiredSegmentVid-3", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash]); - await db.prepare("run", query, ["requiredSegmentVid", 80, 90, 2, "requiredSegmentVid-4", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash]); + const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "actionType", "service", "hidden", "shadowHidden", "hashedVideoID", "description") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + await db.prepare("run", query, ["getSegmentsByHash-0", 1, 10, 2, "getSegmentsByHash-01", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getSegmentsByHash0Hash, ""]); + await db.prepare("run", query, ["getSegmentsByHash-0", 1, 10, 2, "getSegmentsByHash-02", "testman", 0, 50, "sponsor", "skip", "PeerTube", 0, 0, getSegmentsByHash0Hash, ""]); + await db.prepare("run", query, ["getSegmentsByHash-0", 20, 30, 2, "getSegmentsByHash-03", "testman", 100, 150, "intro", "skip", "YouTube", 0, 0, getSegmentsByHash0Hash, ""]); + await db.prepare("run", query, ["getSegmentsByHash-0", 40, 50, 2, "getSegmentsByHash-04", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getSegmentsByHash0Hash, ""]); + await db.prepare("run", query, ["getSegmentsByHash-noMatchHash", 40, 50, 2, "getSegmentsByHash-noMatchHash", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, "fdaffnoMatchHash", ""]); + await db.prepare("run", query, ["getSegmentsByHash-1", 60, 70, 2, "getSegmentsByHash-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, "3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b", ""]); + await db.prepare("run", query, ["onlyHidden", 60, 70, 2, "onlyHidden", "testman", 0, 50, "sponsor", "skip", "YouTube", 1, 0, "f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3", ""]); + await db.prepare("run", query, ["highlightVid", 60, 60, 2, "highlightVid-1", "testman", 0, 50, "poi_highlight", "skip", "YouTube", 0, 0, getHash("highlightVid", 1), ""]); + await db.prepare("run", query, ["highlightVid", 70, 70, 2, "highlightVid-2", "testman", 0, 50, "poi_highlight", "skip", "YouTube", 0, 0, getHash("highlightVid", 1), ""]); + await db.prepare("run", query, ["requiredSegmentVid", 60, 70, 2, "requiredSegmentVid-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 60, 70, -2, "requiredSegmentVid-2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 80, 90, -2, "requiredSegmentVid-3", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash, ""]); + await db.prepare("run", query, ["requiredSegmentVid", 80, 90, 2, "requiredSegmentVid-4", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, requiredSegmentVidHash, ""]); + await db.prepare("run", query, ["chapterVid-hash", 60, 80, 2, "chapterVid-hash-1", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, getHash("chapterVid-hash", 1), "Chapter 1"]); //7258 + await db.prepare("run", query, ["chapterVid-hash", 70, 75, 2, "chapterVid-hash-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, getHash("chapterVid-hash", 1), "Chapter 2"]); //7258 + await db.prepare("run", query, ["chapterVid-hash", 71, 76, 2, "chapterVid-hash-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, getHash("chapterVid-hash", 1), "Chapter 3"]); //7258 + await db.prepare("run", query, ["longMuteVid-hash", 40, 45, 2, "longMuteVid-hash-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 + await db.prepare("run", query, ["longMuteVid-hash", 2, 80, 2, "longMuteVid-hash-2", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 + await db.prepare("run", query, ["longMuteVid-hash", 3, 78, 2, "longMuteVid-hash-3", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 }); it("Should be able to get a 200", (done) => { @@ -356,4 +362,68 @@ describe("getSkipSegmentsByHash", () => { }) .catch(err => done(err)); }); + + it("Should be able to get overlapping chapter segments if very different", (done) => { + client.get(`${endpoint}/7258?category=chapter&actionType=chapter`) + .then(res => { + assert.strictEqual(res.status, 200); + const data = res.data; + assert.strictEqual(data.length, 1); + const expected = [{ + segments: [{ + UUID: "chapterVid-hash-1", + description: "Chapter 1" + }, { + UUID: "chapterVid-hash-2", + description: "Chapter 2" + }] + }]; + const expected2 = [{ + segments: [{ + UUID: "chapterVid-hash-1", + description: "Chapter 1" + }, { + UUID: "chapterVid-hash-3", + description: "Chapter 3" + }] + }]; + + assert.ok(partialDeepEquals(data, expected, false) || partialDeepEquals(data, expected2)); + assert.strictEqual(data[0].segments.length, 2); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to get mute segment with small skip segment in middle", (done) => { + client.get(`${endpoint}/6613?actionType=skip&actionType=mute`) + .then(res => { + assert.strictEqual(res.status, 200); + const data = res.data; + assert.strictEqual(data.length, 1); + const expected = [{ + segments: [{ + UUID: "longMuteVid-hash-2", + actionType: "mute" + }, { + UUID: "longMuteVid-hash-1", + actionType: "skip" + }] + }]; + const expected2 = [{ + segments: [{ + UUID: "longMuteVid-hash-3", + actionType: "mute" + }, { + UUID: "longMuteVid-hash-1", + actionType: "skip" + }] + }]; + + assert.ok(partialDeepEquals(data, expected, false) || partialDeepEquals(data, expected2)); + assert.strictEqual(data[0].segments.length, 2); + done(); + }) + .catch(err => done(err)); + }); }); diff --git a/test/utils/partialDeepEquals.ts b/test/utils/partialDeepEquals.ts index 200252a..df651d6 100644 --- a/test/utils/partialDeepEquals.ts +++ b/test/utils/partialDeepEquals.ts @@ -14,8 +14,7 @@ export const partialDeepEquals = (actual: Record, expected: Record< if (print) printActualExpected(actual, expected); return false; } - } - else if (actual?.[key] !== value) { + } else if (actual?.[key] !== value) { if (print) printActualExpected(actual, expected); return false; } From 2733cd6606e0c97994442819e62d820010883438 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 16:11:02 -0400 Subject: [PATCH 06/18] Switch to new method that splits up existing groups instead of making new ones Kind of #107 --- src/routes/getSkipSegments.ts | 89 ++++++++++------------------------- 1 file changed, 26 insertions(+), 63 deletions(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index a966cee..03e2383 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -271,74 +271,37 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise 0) { - const currentGroup: OverlappingSegmentGroup = { segments: [], votes: 0, reputation: 0, locked: false, required: false }; +function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegmentGroup[] { + return groups.flatMap((group) => { + const result: OverlappingSegmentGroup[] = []; + group.segments.forEach((segment) => { + const bestGroup = result.find((group) => { + // At least one segment in the group must have high % overlap or the same action type + return group.segments.some((compareSegment) => { + const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); + const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); + const overlapPercent = overlap / overallDuration; + return (segment.actionType === compareSegment.actionType && segment.actionType !== ActionType.Chapter) + || overlapPercent >= 0.6 + || (overlapPercent >= 0.8 && segment.actionType === ActionType.Chapter && compareSegment.actionType === ActionType.Chapter); + }); + }); - const currentSegment = segmentsLeftToCheck.shift(); - // TODO: extract out this part to be more generic - currentGroup.segments.push(currentSegment); - currentGroup.votes += currentSegment.votes; - currentGroup.reputation += currentSegment.reputation; - currentGroup.locked ||= currentSegment.locked; - currentGroup.required ||= currentSegment.required; - - const currentDuration = currentSegment.endTime - currentSegment.startTime; - for (const [index, compareSegment] of segmentsLeftToCheck.entries()) { - const compareDuration = compareSegment.endTime - compareSegment.startTime; - const overlap = Math.min(currentSegment.endTime, compareSegment.endTime) - Math.max(currentSegment.startTime, compareSegment.startTime); - const overlapPercent = overlap / Math.max(currentDuration, compareDuration); - if (overlapPercent >= percent) { - currentGroup.segments.push(currentSegment); - currentGroup.votes += compareSegment.votes; - currentGroup.reputation += compareSegment.reputation; - currentGroup.locked ||= compareSegment.locked; - currentGroup.required ||= compareSegment.required; - segmentsLeftToCheck.splice(index, 1); - } + if (bestGroup) { + bestGroup.segments.push(segment); + bestGroup.votes += segment.votes; + bestGroup.reputation += segment.reputation; + bestGroup.locked ||= segment.locked; + bestGroup.required ||= segment.required; + } else { + result.push({ segments: [segment], votes: segment.votes, reputation: segment.reputation, locked: segment.locked, required: segment.required }); } + }); - result.push(currentGroup); - } - } - - return result; + return result; + }); } -// function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegmentGroup[] { -// return groups.flatMap((group) => { -// const result: OverlappingSegmentGroup[] = []; -// group.segments.forEach((segment) => { -// const bestGroup = result.find((group) => { -// // At least one segment in the group must have high % overlap or the same action type -// return group.segments.some((compareSegment) => { -// const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); -// const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); -// const overlapPercent = overlap / overallDuration; -// return (segment.actionType === compareSegment.actionType && segment.actionType !== ActionType.Chapter) -// || overlapPercent >= 0.6 -// || (overlapPercent >= 0.8 && segment.actionType === ActionType.Chapter && compareSegment.actionType === ActionType.Chapter); -// }); -// }); - -// if (bestGroup) { -// bestGroup.segments.push(segment); -// bestGroup.votes += segment.votes; -// bestGroup.reputation += segment.reputation; -// bestGroup.locked ||= segment.locked; -// bestGroup.required ||= segment.required; -// } else { -// result.push({ segments: [segment], votes: segment.votes, reputation: segment.reputation, locked: segment.locked, required: segment.required }); -// } -// }); - -// return result; -// }); -// } - /** * * Returns what would be sent to the client. From c0072d5c72d8bfac3a46cba0020ccc1474ca97f2 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 17:28:58 -0400 Subject: [PATCH 07/18] Add another segment to overlap test case and fix for that case --- src/routes/getSkipSegments.ts | 2 +- test/cases/getSkipSegmentsByHash.ts | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 03e2383..b8b0e71 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -281,7 +281,7 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); const overlapPercent = overlap / overallDuration; - return (segment.actionType === compareSegment.actionType && segment.actionType !== ActionType.Chapter) + return (overlapPercent > 0 && segment.actionType === compareSegment.actionType && segment.actionType !== ActionType.Chapter) || overlapPercent >= 0.6 || (overlapPercent >= 0.8 && segment.actionType === ActionType.Chapter && compareSegment.actionType === ActionType.Chapter); }); diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts index e9da156..6fa4a46 100644 --- a/test/cases/getSkipSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -34,8 +34,9 @@ describe("getSkipSegmentsByHash", () => { await db.prepare("run", query, ["chapterVid-hash", 70, 75, 2, "chapterVid-hash-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, getHash("chapterVid-hash", 1), "Chapter 2"]); //7258 await db.prepare("run", query, ["chapterVid-hash", 71, 76, 2, "chapterVid-hash-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, getHash("chapterVid-hash", 1), "Chapter 3"]); //7258 await db.prepare("run", query, ["longMuteVid-hash", 40, 45, 2, "longMuteVid-hash-1", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 - await db.prepare("run", query, ["longMuteVid-hash", 2, 80, 2, "longMuteVid-hash-2", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 - await db.prepare("run", query, ["longMuteVid-hash", 3, 78, 2, "longMuteVid-hash-3", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 + await db.prepare("run", query, ["longMuteVid-hash", 30, 35, 2, "longMuteVid-hash-2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 + await db.prepare("run", query, ["longMuteVid-hash", 2, 80, 2, "longMuteVid-hash-3", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 + await db.prepare("run", query, ["longMuteVid-hash", 3, 78, 2, "longMuteVid-hash-4", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 }); it("Should be able to get a 200", (done) => { @@ -403,8 +404,11 @@ describe("getSkipSegmentsByHash", () => { assert.strictEqual(data.length, 1); const expected = [{ segments: [{ - UUID: "longMuteVid-hash-2", + UUID: "longMuteVid-hash-3", actionType: "mute" + }, { + UUID: "longMuteVid-hash-2", + actionType: "skip" }, { UUID: "longMuteVid-hash-1", actionType: "skip" @@ -412,8 +416,11 @@ describe("getSkipSegmentsByHash", () => { }]; const expected2 = [{ segments: [{ - UUID: "longMuteVid-hash-3", + UUID: "longMuteVid-hash-4", actionType: "mute" + }, { + UUID: "longMuteVid-hash-2", + actionType: "skip" }, { UUID: "longMuteVid-hash-1", actionType: "skip" @@ -421,7 +428,7 @@ describe("getSkipSegmentsByHash", () => { }]; assert.ok(partialDeepEquals(data, expected, false) || partialDeepEquals(data, expected2)); - assert.strictEqual(data[0].segments.length, 2); + assert.strictEqual(data[0].segments.length, 3); done(); }) .catch(err => done(err)); From 0b7904f891e283fd8fd42371fd97656232451a62 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 17:40:26 -0400 Subject: [PATCH 08/18] Add test case for small part that is the same action type --- test/cases/getSkipSegmentsByHash.ts | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts index 6fa4a46..210008c 100644 --- a/test/cases/getSkipSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -37,6 +37,10 @@ describe("getSkipSegmentsByHash", () => { await db.prepare("run", query, ["longMuteVid-hash", 30, 35, 2, "longMuteVid-hash-2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 await db.prepare("run", query, ["longMuteVid-hash", 2, 80, 2, "longMuteVid-hash-3", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 await db.prepare("run", query, ["longMuteVid-hash", 3, 78, 2, "longMuteVid-hash-4", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-hash", 1), ""]); //6613 + await db.prepare("run", query, ["longMuteVid-2-hash", 1, 15, 2, "longMuteVid-2-hash-1", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-2-hash", 1), ""]); //ab0c + await db.prepare("run", query, ["longMuteVid-2-hash", 30, 35, 2, "longMuteVid-2-hash-2", "testman", 0, 50, "sponsor", "skip", "YouTube", 0, 0, getHash("longMuteVid-2-hash", 1), ""]); //ab0c + await db.prepare("run", query, ["longMuteVid-2-hash", 2, 80, 2, "longMuteVid-2-hash-3", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-2-hash", 1), ""]); //ab0c + await db.prepare("run", query, ["longMuteVid-2-hash", 3, 78, 2, "longMuteVid-2-hash-4", "testman", 0, 50, "sponsor", "mute", "YouTube", 0, 0, getHash("longMuteVid-2-hash", 1), ""]); //ab0c }); it("Should be able to get a 200", (done) => { @@ -433,4 +437,45 @@ describe("getSkipSegmentsByHash", () => { }) .catch(err => done(err)); }); + + it("Should be able to get mute segment with small skip segment in middle (2)", (done) => { + client.get(`${endpoint}/ab0c?actionType=skip&actionType=mute`) + .then(res => { + assert.strictEqual(res.status, 200); + const data = res.data; + assert.strictEqual(data.length, 1); + const expected = [{ + segments: [{ + UUID: "longMuteVid-2-hash-1", + actionType: "mute" + }, { + UUID: "longMuteVid-2-hash-2", + actionType: "skip" + }] + }]; + const expected2 = [{ + segments: [{ + UUID: "longMuteVid-2-hash-3", + actionType: "mute" + }, { + UUID: "longMuteVid-2-hash-2", + actionType: "skip" + }] + }]; + const expected3 = [{ + segments: [{ + UUID: "longMuteVid-2-hash-4", + actionType: "mute" + }, { + UUID: "longMuteVid-2-hash-2", + actionType: "skip" + }] + }]; + + assert.ok(partialDeepEquals(data, expected, false) || partialDeepEquals(data, expected2) || partialDeepEquals(data, expected3)); + assert.strictEqual(data[0].segments.length, 2); + done(); + }) + .catch(err => done(err)); + }); }); From c371d35e825c8d441e8fd657b37fb02e98342e19 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 19:56:01 -0400 Subject: [PATCH 09/18] update db schema --- DatabaseSchema.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DatabaseSchema.md b/DatabaseSchema.md index b3484f4..a9f4344 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -44,6 +44,7 @@ | shadowHidden | INTEGER | not null | | hashedVideoID | TEXT | not null, default '', sha256 | | userAgent | TEXT | not null, default '' | +| description | TEXT | not null, default '' | | index | field | | -- | :--: | From 6919b5433b3077e1bd1aa4bdf163fde7a3c737cb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 22:43:03 -0400 Subject: [PATCH 10/18] Add suggested chapter names --- databases/_sponsorTimes_indexes.sql | 5 +++ src/app.ts | 4 ++ src/databases/Postgres.ts | 2 +- src/routes/getChapterNames.ts | 46 ++++++++++++++++++++++ test/cases/getChapterNames.ts | 61 +++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 src/routes/getChapterNames.ts create mode 100644 test/cases/getChapterNames.ts diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 968558f..4a353d0 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -25,6 +25,11 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID" ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST) TABLESPACE pg_default; +CREATE INDEX IF NOT EXISTS "sponsorTimes_description_gin" + ON public."sponsorTimes" USING gin + ("description" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops) + TABLESPACE pg_default; + -- userNames CREATE INDEX IF NOT EXISTS "userNames_userID" diff --git a/src/app.ts b/src/app.ts index 974805e..fbb7d2c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -42,6 +42,7 @@ import { getUserStats } from "./routes/getUserStats"; import ExpressPromiseRouter from "express-promise-router"; import { Server } from "http"; import { youtubeApiProxy } from "./routes/youtubeApiProxy"; +import { getChapterNames } from "./routes/getChapterNames"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -172,6 +173,9 @@ function setupRoutes(router: Router) { // get all segments that match a search router.get("/api/searchSegments", getSearchSegments); + // autocomplete chapter names + router.get("/api/chapterNames", getChapterNames); + // get status router.get("/api/status/:value", getStatus); router.get("/api/status", getStatus); diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 5c2b5c5..79d2adb 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -68,7 +68,7 @@ export class Postgres implements IDatabase { } case "all": { const values = queryResult.rows; - Logger.debug(`result (postgres): ${values}`); + Logger.debug(`result (postgres): ${JSON.stringify(values)}`); return values; } case "run": { diff --git a/src/routes/getChapterNames.ts b/src/routes/getChapterNames.ts new file mode 100644 index 0000000..1b04e5b --- /dev/null +++ b/src/routes/getChapterNames.ts @@ -0,0 +1,46 @@ +import { Logger } from "../utils/logger"; +import { Request, Response } from "express"; +import { db } from "../databases/databases"; +import { Postgres } from "../databases/Postgres"; + +export async function getChapterNames(req: Request, res: Response): Promise { + const description = req.query.description as string; + const channelID = req.query.channelID as string; + + if (!description || typeof(description) !== "string" + || !channelID || typeof(channelID) !== "string") { + return res.sendStatus(400); + } + + if (!(db instanceof Postgres)) { + return res.sendStatus(500).json({ + message: "Not supported on this instance" + }); + } + + try { + const descriptions = await db.prepare("all", ` + SELECT "description" + FROM "sponsorTimes" + WHERE ("votes" > 0 OR ("views" > 100 AND "votes" >= 0)) AND "videoID" IN ( + SELECT "videoID" + FROM "videoInfo" + WHERE "channelID" = ? + ) AND "description" != '' + GROUP BY "description" + ORDER BY SUM("votes"), similarity("description", ?) DESC + LIMIT 5;` + , [channelID, description]) as { description: string }[]; + + if (descriptions?.length > 0) { + return res.status(200).json(descriptions.map(d => ({ + description: d.description + }))); + } + } catch (err) { + Logger.error(err as string); + return res.sendStatus(500); + } + + return res.status(404).json([]); +} diff --git a/test/cases/getChapterNames.ts b/test/cases/getChapterNames.ts new file mode 100644 index 0000000..a1ff03e --- /dev/null +++ b/test/cases/getChapterNames.ts @@ -0,0 +1,61 @@ +import assert from "assert"; +import { db } from "../../src/databases/databases"; +import { Postgres } from "../../src/databases/Postgres"; +import { client } from "../utils/httpClient"; +import { partialDeepEquals } from "../utils/partialDeepEquals"; + +// Only works with Postgres +if (db instanceof Postgres) { + + describe("getChapterNames", function () { + const endpoint = "/api/chapterNames"; + + const chapterNamesVid1 = "chapterNamesVid"; + const chapterChannelID = "chapterChannelID"; + + before(async () => { + const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "hidden", "shadowHidden", "description") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + await db.prepare("run", query, [chapterNamesVid1, 60, 80, 2, 0, "chapterVid-1", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Weird name"]); + await db.prepare("run", query, [chapterNamesVid1, 70, 75, 2, 0, "chapterVid-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "A different one"]); + await db.prepare("run", query, [chapterNamesVid1, 71, 76, 2, 0, "chapterVid-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Something else"]); + + await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID") + SELECT ?, ?`, [ + chapterNamesVid1, chapterChannelID + ]); + }); + + it("Search for 'weird'", async () => { + const result = await client.get(`${endpoint}?description=weird&channelID=${chapterChannelID}`); + const expected = [{ + description: "Weird name", + }]; + + assert.strictEqual(result.status, 200); + assert.strictEqual(result.data.length, 3); + assert.ok(partialDeepEquals(result.data, expected)); + }); + + it("Search for 'different'", async () => { + const result = await client.get(`${endpoint}?description=different&channelID=${chapterChannelID}`); + const expected = [{ + description: "A different one", + }]; + + assert.strictEqual(result.status, 200); + assert.strictEqual(result.data.length, 3); + assert.ok(partialDeepEquals(result.data, expected)); + }); + + it("Search for 'something'", async () => { + const result = await client.get(`${endpoint}?description=something&channelID=${chapterChannelID}`); + const expected = [{ + description: "Something else", + }]; + + assert.strictEqual(result.status, 200); + assert.strictEqual(result.data.length, 3); + assert.ok(partialDeepEquals(result.data, expected)); + }); + }); +} \ No newline at end of file From a2698fb70d75b7bea254e4a640b158b57939be9f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 6 Nov 2021 23:18:28 -0400 Subject: [PATCH 11/18] Set all videoInfo values in chapters name test --- test/cases/getChapterNames.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/cases/getChapterNames.ts b/test/cases/getChapterNames.ts index a1ff03e..0905b47 100644 --- a/test/cases/getChapterNames.ts +++ b/test/cases/getChapterNames.ts @@ -15,13 +15,13 @@ if (db instanceof Postgres) { before(async () => { const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "hidden", "shadowHidden", "description") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; - await db.prepare("run", query, [chapterNamesVid1, 60, 80, 2, 0, "chapterVid-1", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Weird name"]); - await db.prepare("run", query, [chapterNamesVid1, 70, 75, 2, 0, "chapterVid-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "A different one"]); - await db.prepare("run", query, [chapterNamesVid1, 71, 76, 2, 0, "chapterVid-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Something else"]); + await db.prepare("run", query, [chapterNamesVid1, 60, 80, 2, 0, "chapterNamesVid-1", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Weird name"]); + await db.prepare("run", query, [chapterNamesVid1, 70, 75, 2, 0, "chapterNamesVid-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "A different one"]); + await db.prepare("run", query, [chapterNamesVid1, 71, 76, 2, 0, "chapterNamesVid-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Something else"]); - await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID") - SELECT ?, ?`, [ - chapterNamesVid1, chapterChannelID + await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published", "genreUrl") + SELECT ?, ?, ?, ?, ?`, [ + chapterNamesVid1, chapterChannelID, "", 0, "" ]); }); From 1106048b37ddb24b63db332ac2b6031469367d78 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 8 Nov 2021 19:22:17 -0500 Subject: [PATCH 12/18] Allow mute intro, outro, preview and add filler --- src/config.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index ed6c440..b60aa21 100644 --- a/src/config.ts +++ b/src/config.ts @@ -19,14 +19,15 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight", "chapter"], + categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], categorySupport: { sponsor: ["skip", "mute"], selfpromo: ["skip", "mute"], interaction: ["skip", "mute"], - intro: ["skip"], - outro: ["skip"], - preview: ["skip"], + intro: ["skip", "mute"], + outro: ["skip", "mute"], + preview: ["skip", "mute"], + filler: ["skip", "mute"], music_offtopic: ["skip"], poi_highlight: ["skip"], chapter: ["chapter"] From db48953e39a6874774666100c5c4b29ce31bf81b Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 8 Nov 2021 19:22:17 -0500 Subject: [PATCH 13/18] Allow mute intro, outro, preview and add filler --- src/config.ts | 9 +++++---- test/cases/postSkipSegments.ts | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index d28d2aa..d39280f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -19,14 +19,15 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight"], + categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"], categorySupport: { sponsor: ["skip", "mute"], selfpromo: ["skip", "mute"], interaction: ["skip", "mute"], - intro: ["skip"], - outro: ["skip"], - preview: ["skip"], + intro: ["skip", "mute"], + outro: ["skip", "mute"], + preview: ["skip", "mute"], + filler: ["skip", "mute"], music_offtopic: ["skip"], poi_highlight: ["skip"], }, diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index b4d7f4f..ec16dbd 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -181,14 +181,14 @@ describe("postSkipSegments", () => { .catch(err => done(err)); }); - it("Should not be able to submit an intro with mute action type (JSON method)", (done) => { + it("Should not be able to submit an music_offtopic with mute action type (JSON method)", (done) => { const videoID = "postSkip4"; postSkipSegmentJSON({ userID: submitUserOne, videoID, segments: [{ segment: [0, 10], - category: "intro", + category: "music_offtopic", actionType: "mute" }], }) From e85a0d4f28f46522f6fad70f4aaade266fb02e6a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 8 Nov 2021 19:28:27 -0500 Subject: [PATCH 14/18] Fix lock reason test with filler --- test/cases/getLockReason.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cases/getLockReason.ts b/test/cases/getLockReason.ts index a4675c4..c5bdb89 100644 --- a/test/cases/getLockReason.ts +++ b/test/cases/getLockReason.ts @@ -108,6 +108,7 @@ describe("getLockReason", () => { { category: "outro", locked: 1, reason: "outro-reason", userID: vipUserID2, userName: vipUserName2 }, { category: "preview", locked: 1, reason: "preview-reason", userID: vipUserID1, userName: vipUserName1 }, { category: "music_offtopic", locked: 1, reason: "nonmusic-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "filler", locked: 0, reason: "", userID: "", userName: "" }, { category: "poi_highlight", locked: 0, reason: "", userID: "", userName: "" }, { category: "chapter", locked: 0, reason: "", userID: "", userName: "" } ]; From bc6db0d1090b642f557ac3a7413a39c4fa68b2a4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 14 Nov 2021 23:09:12 -0500 Subject: [PATCH 15/18] Add endpoints for rating endpoint (dislikes) https://github.com/ajayyy/SponsorBlock/issues/1039 --- .eslintrc.js | 2 +- DatabaseSchema.md | 33 ++++++++- databases/_private_indexes.sql | 7 ++ databases/_sponsorTimes_indexes.sql | 17 +++++ databases/_upgrade_private_4.sql | 14 ++++ databases/_upgrade_sponsorTimes_28.sql | 13 ++++ src/app.ts | 8 +++ src/config.ts | 6 ++ src/routes/ratings/getRating.ts | 82 ++++++++++++++++++++++ src/routes/ratings/postRating.ts | 62 +++++++++++++++++ src/types/config.model.ts | 1 + src/types/ratings.model.ts | 6 ++ src/utils/getIP.ts | 2 +- src/utils/redisKeys.ts | 7 ++ test/cases/ratings/getRating.ts | 49 +++++++++++++ test/cases/ratings/postRating.ts | 96 ++++++++++++++++++++++++++ test/test.ts | 24 ++++--- 17 files changed, 415 insertions(+), 14 deletions(-) create mode 100644 databases/_upgrade_private_4.sql create mode 100644 databases/_upgrade_sponsorTimes_28.sql create mode 100644 src/routes/ratings/getRating.ts create mode 100644 src/routes/ratings/postRating.ts create mode 100644 src/types/ratings.model.ts create mode 100644 test/cases/ratings/getRating.ts create mode 100644 test/cases/ratings/postRating.ts diff --git a/.eslintrc.js b/.eslintrc.js index e96d7da..c082e3d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -27,6 +27,6 @@ module.exports = { "indent": ["warn", 4, { "SwitchCase": 1 }], "object-curly-spacing": ["warn", "always"], "require-await": "warn", - "no-console": "error" + "no-console": "warn" }, }; diff --git a/DatabaseSchema.md b/DatabaseSchema.md index a9f4344..bcd9949 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -185,6 +185,22 @@ | hashedVideoID | TEXT | not null, default '', sha256 | | userAgent | TEXT | not null, default '' | +### ratings + +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| service | TEXT | not null, default 'YouTube' | +| type | INTEGER | not null | +| count | INTEGER | not null | +| hashedVideoID | TEXT | not null | + +| index | field | +| -- | :--: | +| ratings_hashedVideoID_gin | hashedVideoID | +| ratings_hashedVideoID | hashedVideoID, service | +| ratings_videoID | videoID, service | + # Private [vote](#vote) @@ -238,4 +254,19 @@ | Name | Type | | | -- | :--: | -- | | key | TEXT | not null | -| value | TEXT | not null | \ No newline at end of file +| value | TEXT | not null | + +### ratings + +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| service | TEXT | not null, default 'YouTube' | +| userID | TEXT | not null | +| type | INTEGER | not null | +| timeSubmitted | INTEGER | not null | +| hashedIP | TEXT | not null | + +| index | field | +| -- | :--: | +| ratings_videoID | videoID, service, userID, timeSubmitted | diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql index 48ed490..7ecb3a0 100644 --- a/databases/_private_indexes.sql +++ b/databases/_private_indexes.sql @@ -22,4 +22,11 @@ CREATE INDEX IF NOT EXISTS "votes_userID" CREATE INDEX IF NOT EXISTS "categoryVotes_UUID" ON public."categoryVotes" USING btree ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- ratings + +CREATE INDEX IF NOT EXISTS "ratings_videoID" + ON public."ratings" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 4a353d0..89dd91b 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -86,4 +86,21 @@ CREATE INDEX IF NOT EXISTS "videoInfo_videoID" CREATE INDEX IF NOT EXISTS "videoInfo_channelID" ON public."videoInfo" USING btree ("channelID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- ratings + +CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID_gin" + ON public."ratings" USING gin + ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID" + ON public."ratings" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "ratings_videoID" + ON public."ratings" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_upgrade_private_4.sql b/databases/_upgrade_private_4.sql new file mode 100644 index 0000000..1bc137c --- /dev/null +++ b/databases/_upgrade_private_4.sql @@ -0,0 +1,14 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "ratings" ( + "videoID" TEXT NOT NULL, + "service" TEXT NOT NULL default 'YouTube', + "type" INTEGER NOT NULL, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "hashedIP" TEXT NOT NULL +); + +UPDATE "config" SET value = 4 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_28.sql b/databases/_upgrade_sponsorTimes_28.sql new file mode 100644 index 0000000..031436a --- /dev/null +++ b/databases/_upgrade_sponsorTimes_28.sql @@ -0,0 +1,13 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "ratings" ( + "videoID" TEXT NOT NULL, + "service" TEXT NOT NULL default 'YouTube', + "type" INTEGER NOT NULL, + "count" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL +); + +UPDATE "config" SET value = 28 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index fbb7d2c..3f2b993 100644 --- a/src/app.ts +++ b/src/app.ts @@ -43,6 +43,8 @@ import ExpressPromiseRouter from "express-promise-router"; import { Server } from "http"; import { youtubeApiProxy } from "./routes/youtubeApiProxy"; import { getChapterNames } from "./routes/getChapterNames"; +import { postRating } from "./routes/ratings/postRating"; +import { getRating } from "./routes/ratings/getRating"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -74,9 +76,11 @@ function setupRoutes(router: Router) { // Rate limit endpoint lists const voteEndpoints: RequestHandler[] = [voteOnSponsorTime]; const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime]; + const postRateEndpoints: RequestHandler[] = [postRating]; if (config.rateLimit) { if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID)); if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view)); + if (config.rateLimit.rate) postRateEndpoints.unshift(rateLimitMiddleware(config.rateLimit.rate)); } //add the get function @@ -186,6 +190,10 @@ function setupRoutes(router: Router) { router.get("/api/lockReason", getLockReason); + // ratings + router.get("/api/ratings/rate/:prefix", getRating); + router.post("/api/ratings/rate", postRateEndpoints); + if (config.postgres) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); router.get("/database.json", (req, res) => dumpDatabase(req, res, false)); diff --git a/src/config.ts b/src/config.ts index b60aa21..c666a90 100644 --- a/src/config.ts +++ b/src/config.ts @@ -58,6 +58,12 @@ addDefaults(config, { statusCode: 200, message: "Too many views, please try again later", }, + rate: { + windowMs: 900000, + max: 20, + statusCode: 200, + message: "Success", + } }, userCounterURL: null, newLeafURLs: null, diff --git a/src/routes/ratings/getRating.ts b/src/routes/ratings/getRating.ts new file mode 100644 index 0000000..fe5211f --- /dev/null +++ b/src/routes/ratings/getRating.ts @@ -0,0 +1,82 @@ +import { Request, Response } from "express"; +import { db } from "../../databases/databases"; +import { RatingType } from "../../types/ratings.model"; +import { Service, VideoID, VideoIDHash } from "../../types/segments.model"; +import { getService } from "../../utils/getService"; +import { hashPrefixTester } from "../../utils/hashPrefixTester"; +import { Logger } from "../../utils/logger"; +import { QueryCacher } from "../../utils/queryCacher"; +import { ratingHashKey } from "../../utils/redisKeys"; + +interface DBRating { + videoID: VideoID, + hashedVideoID: VideoIDHash, + service: Service, + type: RatingType, + count: number +} + +export async function getRating(req: Request, res: Response): Promise { + let hashPrefix = req.params.prefix as VideoIDHash; + if (!hashPrefix || !hashPrefixTester(hashPrefix)) { + return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix + } + hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; + + let types: RatingType[] = []; + try { + types = req.query.types + ? JSON.parse(req.query.types as string) + : req.query.type + ? Array.isArray(req.query.type) + ? req.query.type + : [req.query.type] + : [RatingType.Upvote, RatingType.Downvote]; + if (!Array.isArray(types)) { + return res.status(400).send("Categories parameter does not match format requirements."); + } + + types = types.map((type) => parseInt(type as unknown as string, 10)); + } catch(error) { + return res.status(400).send("Bad parameter: categories (invalid JSON)"); + } + + const service: Service = getService(req.query.service, req.body.service); + + try { + const ratings = (await getRatings(hashPrefix, service)) + .filter((rating) => types.includes(rating.type)) + .map((rating) => ({ + videoID: rating.videoID, + hash: rating.hashedVideoID, + service: rating.service, + type: rating.type, + count: rating.count + })); + + if (ratings) { + res.status(200); + } else { + res.status(404); + } + return res.send(ratings ?? []); + } catch (err) { + Logger.error(err as string); + return res.sendStatus(500); + } +} + +async function getRatings(hashPrefix: VideoIDHash, service: Service): Promise { + const fetchFromDB = () => db + .prepare( + "all", + `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`, + [`${hashPrefix}%`, service] + ) as Promise; + + if (hashPrefix.length === 4) { + return await QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service)); + } + + return fetchFromDB(); +} \ No newline at end of file diff --git a/src/routes/ratings/postRating.ts b/src/routes/ratings/postRating.ts new file mode 100644 index 0000000..7e73903 --- /dev/null +++ b/src/routes/ratings/postRating.ts @@ -0,0 +1,62 @@ +import { db, privateDB } from "../../databases/databases"; +import { getHash } from "../../utils/getHash"; +import { Logger } from "../../utils/logger"; +import { Request, Response } from "express"; +import { HashedUserID, UserID } from "../../types/user.model"; +import { HashedIP, IPAddress, VideoID } from "../../types/segments.model"; +import { getIP } from "../../utils/getIP"; +import { getService } from "../../utils/getService"; +import { RatingType, RatingTypes } from "../../types/ratings.model"; +import { config } from "../../config"; + +export async function postRating(req: Request, res: Response): Promise { + const privateUserID = req.body.userID as UserID; + const videoID = req.body.videoID as VideoID; + const service = getService(req.query.service, req.body.service); + const type = req.body.type as RatingType; + const enabled = req.body.enabled ?? true; + + if (privateUserID == undefined || videoID == undefined || service == undefined || type == undefined + || (typeof privateUserID !== "string") || (typeof videoID !== "string") || (typeof service !== "string") + || (typeof type !== "number") || (enabled && (typeof enabled !== "boolean")) || !RatingTypes.includes(type)) { + //invalid request + return res.sendStatus(400); + } + + const hashedIP: HashedIP = getHash(getIP(req) + config.globalSalt as IPAddress, 1); + const hashedUserID: HashedUserID = getHash(privateUserID); + const hashedVideoID = getHash(videoID, 1); + + try { + // Check if this user has voted before + const existingVote = await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]); + if (existingVote.count > 0 && !enabled) { + // Undo the vote + await db.prepare("run", `UPDATE "ratings" SET "count" = "count" - 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + await privateDB.prepare("run", `DELETE FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]); + + return res.sendStatus(200); + } else if (existingVote.count === 0 && enabled) { + // Make sure there hasn't been another vote from this IP + const existingIPVote = (await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "hashedIP" = ?`, [videoID, service, type, hashedIP])) + .count > 0; + if (!existingIPVote) { + // Check if general rating already exists, if so increase it + const rating = await db.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + if (rating.count > 0) { + await db.prepare("run", `UPDATE "ratings" SET "count" = "count" + 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + } else { + await db.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, 1, ?)`, [videoID, service, type, hashedVideoID]); + } + + // Create entry in privateDB + await privateDB.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "userID", "timeSubmitted", "hashedIP") VALUES (?, ?, ?, ?, ?, ?)`, [videoID, service, type, hashedUserID, Date.now(), hashedIP]); + } + } + + return res.sendStatus(200); + } catch (err) { + Logger.error(err as string); + return res.sendStatus(500); + } +} \ No newline at end of file diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 1614c85..bc74a27 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -34,6 +34,7 @@ export interface SBSConfig { rateLimit: { vote: RateLimitConfig; view: RateLimitConfig; + rate: RateLimitConfig; }; mysql?: any; privateMysql?: any; diff --git a/src/types/ratings.model.ts b/src/types/ratings.model.ts new file mode 100644 index 0000000..28b3fbb --- /dev/null +++ b/src/types/ratings.model.ts @@ -0,0 +1,6 @@ +export enum RatingType { + Downvote = 0, + Upvote = 1 +} + +export const RatingTypes = [RatingType.Downvote, RatingType.Upvote]; \ No newline at end of file diff --git a/src/utils/getIP.ts b/src/utils/getIP.ts index 668b9a7..0d2dd90 100644 --- a/src/utils/getIP.ts +++ b/src/utils/getIP.ts @@ -15,6 +15,6 @@ export function getIP(req: Request): IPAddress { case "X-Real-IP": return req.headers["x-real-ip"] as IPAddress; default: - return req.connection.remoteAddress as IPAddress; + return (req.connection?.remoteAddress || req.socket?.remoteAddress) as IPAddress; } } \ No newline at end of file diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index c93e5c1..4d9cd9e 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -16,3 +16,10 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S export function reputationKey(userID: UserID): string { return `reputation.user.${userID}`; } + +export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string { + hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash; + if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`); + + return `rating.${service}.${hashPrefix}`; +} \ No newline at end of file diff --git a/test/cases/ratings/getRating.ts b/test/cases/ratings/getRating.ts new file mode 100644 index 0000000..876ec47 --- /dev/null +++ b/test/cases/ratings/getRating.ts @@ -0,0 +1,49 @@ +import { db } from "../../../src/databases/databases"; +import { getHash } from "../../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../../utils/httpClient"; +import { AxiosResponse } from "axios"; +import { partialDeepEquals } from "../../utils/partialDeepEquals"; + +const endpoint = "/api/ratings/rate/"; +const getRating = (hash: string, params?: unknown): Promise => client.get(endpoint + hash, { params }); + +describe("getRating", () => { + before(async () => { + const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; + await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 0, 5, getHash("some-likes-and-dislikes", 1)]); //b3f0 + await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 1, 10, getHash("some-likes-and-dislikes", 1)]); + }); + + it("Should be able to get dislikes and likes by default", (done) => { + getRating("b3f0") + .then(res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 5, + }, { + type: 1, + count: 10, + }]; + assert.ok(partialDeepEquals(res.data, expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to filter for only dislikes", (done) => { + getRating("b3f0", { type: 0 }) + .then(res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 5, + }]; + assert.ok(partialDeepEquals(res.data, expected)); + + done(); + }) + .catch(err => done(err)); + }); +}); \ No newline at end of file diff --git a/test/cases/ratings/postRating.ts b/test/cases/ratings/postRating.ts new file mode 100644 index 0000000..9ff7ebd --- /dev/null +++ b/test/cases/ratings/postRating.ts @@ -0,0 +1,96 @@ +import { db } from "../../../src/databases/databases"; +import { getHash } from "../../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../../utils/httpClient"; +import { AxiosResponse } from "axios"; +import { partialDeepEquals } from "../../utils/partialDeepEquals"; + +const endpoint = "/api/ratings/rate/"; +const postRating = (body: unknown): Promise => client.post(endpoint, body); +const queryDatabase = (videoID: string) => db.prepare("all", `SELECT * FROM "ratings" WHERE "videoID" = ?`, [videoID]); + +describe("postRating", () => { + before(async () => { + const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; + await db.prepare("run", insertUserNameQuery, ["multiple-rates", "YouTube", 0, 3, getHash("multiple-rates", 1)]); + }); + + it("Should be able to vote on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "normal-video", + type: 0 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + hashedVideoID: getHash("normal-video", 1), + videoID: "normal-video", + type: 0, + count: 1, + service: "YouTube" + }]; + assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to undo a vote on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "normal-video", + type: 0, + enabled: false + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 0 + }]; + assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to vote after someone else on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "multiple-rates", + type: 0 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 4 + }]; + assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to vote a different type than existing votes on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "multiple-rates", + type: 1 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 4 + }, { + type: 1, + count: 1 + }]; + assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected)); + done(); + }) + .catch(err => done(err)); + }); +}); \ No newline at end of file diff --git a/test/test.ts b/test/test.ts index 904ec1d..3484add 100644 --- a/test/test.ts +++ b/test/test.ts @@ -32,19 +32,21 @@ async function init() { // Instantiate a Mocha instance. const mocha = new Mocha(); - const testDir = "./test/cases"; + const testDirs = ["./test/cases", "./test/cases/ratings"]; // Add each .ts file to the mocha instance - fs.readdirSync(testDir) - .filter((file) => - // Only keep the .ts files - file.substr(-3) === ".ts" - ) - .forEach(function(file) { - mocha.addFile( - path.join(testDir, file) - ); - }); + testDirs.forEach(testDir => { + fs.readdirSync(testDir) + .filter((file) => + // Only keep the .ts files + file.substr(-3) === ".ts" + ) + .forEach(function(file) { + mocha.addFile( + path.join(testDir, file) + ); + }); + }); const mockServer = createMockServer(() => { Logger.info("Started mock HTTP Server"); From b9ebd0036526a8bad3ad13c944e81d5952b90078 Mon Sep 17 00:00:00 2001 From: Michael C Date: Mon, 15 Nov 2021 01:17:36 -0500 Subject: [PATCH 16/18] fixed tests, typos and optimized code for ratings --- src/routes/ratings/getRating.ts | 23 +++++--------- test/cases/ratings/getRating.ts | 39 +++++++++++++++++++++--- test/cases/ratings/postRating.ts | 52 +++++++++++++++++++++++--------- 3 files changed, 80 insertions(+), 34 deletions(-) diff --git a/src/routes/ratings/getRating.ts b/src/routes/ratings/getRating.ts index fe5211f..ec7e9f0 100644 --- a/src/routes/ratings/getRating.ts +++ b/src/routes/ratings/getRating.ts @@ -33,12 +33,12 @@ export async function getRating(req: Request, res: Response): Promise : [req.query.type] : [RatingType.Upvote, RatingType.Downvote]; if (!Array.isArray(types)) { - return res.status(400).send("Categories parameter does not match format requirements."); + return res.status(400).send("Types parameter does not match format requirements."); } types = types.map((type) => parseInt(type as unknown as string, 10)); } catch(error) { - return res.status(400).send("Bad parameter: categories (invalid JSON)"); + return res.status(400).send("Bad parameter: types (invalid JSON)"); } const service: Service = getService(req.query.service, req.body.service); @@ -53,20 +53,15 @@ export async function getRating(req: Request, res: Response): Promise type: rating.type, count: rating.count })); - - if (ratings) { - res.status(200); - } else { - res.status(404); - } - return res.send(ratings ?? []); + return res.status((ratings.length) ? 200 : 404) + .send(ratings ?? []); } catch (err) { Logger.error(err as string); return res.sendStatus(500); } } -async function getRatings(hashPrefix: VideoIDHash, service: Service): Promise { +function getRatings(hashPrefix: VideoIDHash, service: Service): Promise { const fetchFromDB = () => db .prepare( "all", @@ -74,9 +69,7 @@ async function getRatings(hashPrefix: VideoIDHash, service: Service): Promise; - if (hashPrefix.length === 4) { - return await QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service)); - } - - return fetchFromDB(); + return (hashPrefix.length === 4) + ? QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service)) + : fetchFromDB(); } \ No newline at end of file diff --git a/test/cases/ratings/getRating.ts b/test/cases/ratings/getRating.ts index 876ec47..ee82f23 100644 --- a/test/cases/ratings/getRating.ts +++ b/test/cases/ratings/getRating.ts @@ -8,15 +8,19 @@ import { partialDeepEquals } from "../../utils/partialDeepEquals"; const endpoint = "/api/ratings/rate/"; const getRating = (hash: string, params?: unknown): Promise => client.get(endpoint + hash, { params }); +const videoOneID = "some-likes-and-dislikes"; +const videoOneIDHash = getHash(videoOneID, 1); +const videoOnePartialHash = videoOneIDHash.substr(0, 4); + describe("getRating", () => { before(async () => { const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; - await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 0, 5, getHash("some-likes-and-dislikes", 1)]); //b3f0 - await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 1, 10, getHash("some-likes-and-dislikes", 1)]); + await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 0, 5, videoOneIDHash]); + await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 1, 10, videoOneIDHash]); }); it("Should be able to get dislikes and likes by default", (done) => { - getRating("b3f0") + getRating(videoOnePartialHash) .then(res => { assert.strictEqual(res.status, 200); const expected = [{ @@ -33,7 +37,7 @@ describe("getRating", () => { }); it("Should be able to filter for only dislikes", (done) => { - getRating("b3f0", { type: 0 }) + getRating(videoOnePartialHash, { type: 0 }) .then(res => { assert.strictEqual(res.status, 200); const expected = [{ @@ -46,4 +50,31 @@ describe("getRating", () => { }) .catch(err => done(err)); }); + + it("Should return 400 for invalid hash", (done) => { + getRating("a") + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return 404 for nonexitent type", (done) => { + getRating(videoOnePartialHash, { type: 100 }) + .then(res => { + assert.strictEqual(res.status, 404); + done(); + }) + .catch(err => done(err)); + }); + + it("Should return 404 for nonexistent videoID", (done) => { + getRating("aaaa") + .then(res => { + assert.strictEqual(res.status, 404); + done(); + }) + .catch(err => done(err)); + }); }); \ No newline at end of file diff --git a/test/cases/ratings/postRating.ts b/test/cases/ratings/postRating.ts index 9ff7ebd..be63f60 100644 --- a/test/cases/ratings/postRating.ts +++ b/test/cases/ratings/postRating.ts @@ -9,37 +9,43 @@ const endpoint = "/api/ratings/rate/"; const postRating = (body: unknown): Promise => client.post(endpoint, body); const queryDatabase = (videoID: string) => db.prepare("all", `SELECT * FROM "ratings" WHERE "videoID" = ?`, [videoID]); +const videoIDOne = "normal-video"; +const videoIDTwo = "multiple-rates"; +const ratingUserID = "rating-testman"; + describe("postRating", () => { before(async () => { const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; - await db.prepare("run", insertUserNameQuery, ["multiple-rates", "YouTube", 0, 3, getHash("multiple-rates", 1)]); + await db.prepare("run", insertUserNameQuery, [videoIDTwo, "YouTube", 0, 3, getHash(videoIDTwo, 1)]); }); it("Should be able to vote on a video", (done) => { + const videoID = videoIDOne; postRating({ - userID: "rating-testman", - videoID: "normal-video", + userID: ratingUserID, + videoID, type: 0 }) .then(async res => { assert.strictEqual(res.status, 200); const expected = [{ - hashedVideoID: getHash("normal-video", 1), - videoID: "normal-video", + hashedVideoID: getHash(videoID, 1), + videoID, type: 0, count: 1, service: "YouTube" }]; - assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected)); + assert.ok(partialDeepEquals(await queryDatabase(videoID), expected)); done(); }) .catch(err => done(err)); }); it("Should be able to undo a vote on a video", (done) => { + const videoID = videoIDOne; postRating({ - userID: "rating-testman", - videoID: "normal-video", + userID: ratingUserID, + videoID, type: 0, enabled: false }) @@ -49,16 +55,17 @@ describe("postRating", () => { type: 0, count: 0 }]; - assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected)); + assert.ok(partialDeepEquals(await queryDatabase(videoID), expected)); done(); }) .catch(err => done(err)); }); it("Should be able to vote after someone else on a video", (done) => { + const videoID = videoIDTwo; postRating({ - userID: "rating-testman", - videoID: "multiple-rates", + userID: ratingUserID, + videoID, type: 0 }) .then(async res => { @@ -67,16 +74,17 @@ describe("postRating", () => { type: 0, count: 4 }]; - assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected)); + assert.ok(partialDeepEquals(await queryDatabase(videoID), expected)); done(); }) .catch(err => done(err)); }); it("Should be able to vote a different type than existing votes on a video", (done) => { + const videoID = videoIDTwo; postRating({ - userID: "rating-testman", - videoID: "multiple-rates", + userID: ratingUserID, + videoID, type: 1 }) .then(async res => { @@ -88,7 +96,21 @@ describe("postRating", () => { type: 1, count: 1 }]; - assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected)); + assert.ok(partialDeepEquals(await queryDatabase(videoID), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should not be able to vote with nonexistent type", (done) => { + const videoID = videoIDOne; + postRating({ + userID: ratingUserID, + videoID, + type: 100 + }) + .then(res => { + assert.strictEqual(res.status, 400); done(); }) .catch(err => done(err)); From 03c95ca15867fca402100836a607ebe4af18f2c8 Mon Sep 17 00:00:00 2001 From: Michael C Date: Mon, 15 Nov 2021 01:48:14 -0500 Subject: [PATCH 17/18] rename clearSegmentCache --- src/routes/postClearCache.ts | 2 +- src/routes/postPurgeAllSegments.ts | 2 +- src/routes/postSkipSegments.ts | 2 +- src/routes/shadowBanUser.ts | 4 ++-- src/routes/voteOnSponsorTime.ts | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/routes/postClearCache.ts b/src/routes/postClearCache.ts index 91d4184..4d8ee82 100644 --- a/src/routes/postClearCache.ts +++ b/src/routes/postClearCache.ts @@ -39,7 +39,7 @@ export async function postClearCache(req: Request, res: Response): Promise `'${c}'`).join(",")})`, [UUID])) .forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { - QueryCacher.clearVideoCache(videoInfo); + QueryCacher.clearSegmentCache(videoInfo); } ); @@ -125,6 +125,6 @@ async function unHideSubmissions(categories: string[], userID: UserID) { // clear cache for all old videos (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID])) .forEach((videoInfo: { category: Category; videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) => { - QueryCacher.clearVideoCache(videoInfo); + QueryCacher.clearSegmentCache(videoInfo); }); } diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 2fa7272..f713caa 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -267,7 +267,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i } } - QueryCacher.clearVideoCache(videoInfo); + QueryCacher.clearSegmentCache(videoInfo); return res.sendStatus(finalResponse.finalStatus); } @@ -473,7 +473,7 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise Date: Mon, 15 Nov 2021 01:50:09 -0500 Subject: [PATCH 18/18] clear rating cache --- src/app.ts | 2 ++ src/routes/ratings/postClearCache.ts | 52 ++++++++++++++++++++++++++++ src/routes/ratings/postRating.ts | 29 ++++++++-------- src/utils/queryCacher.ts | 13 +++++-- test/cases/ratings/postClearCache.ts | 51 +++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 17 deletions(-) create mode 100644 src/routes/ratings/postClearCache.ts create mode 100644 test/cases/ratings/postClearCache.ts diff --git a/src/app.ts b/src/app.ts index 3f2b993..ee0ab8c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -45,6 +45,7 @@ import { youtubeApiProxy } from "./routes/youtubeApiProxy"; import { getChapterNames } from "./routes/getChapterNames"; import { postRating } from "./routes/ratings/postRating"; import { getRating } from "./routes/ratings/getRating"; +import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -193,6 +194,7 @@ function setupRoutes(router: Router) { // ratings router.get("/api/ratings/rate/:prefix", getRating); router.post("/api/ratings/rate", postRateEndpoints); + router.post("/api/ratings/clearCache", ratingPostClearCache); if (config.postgres) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); diff --git a/src/routes/ratings/postClearCache.ts b/src/routes/ratings/postClearCache.ts new file mode 100644 index 0000000..ed3d5d5 --- /dev/null +++ b/src/routes/ratings/postClearCache.ts @@ -0,0 +1,52 @@ +import { Logger } from "../../utils/logger"; +import { HashedUserID, UserID } from "../../types/user.model"; +import { getHash } from "../../utils/getHash"; +import { Request, Response } from "express"; +import { Service, VideoID } from "../../types/segments.model"; +import { QueryCacher } from "../../utils/queryCacher"; +import { isUserVIP } from "../../utils/isUserVIP"; +import { VideoIDHash } from "../../types/segments.model"; +import { getService } from "../..//utils/getService"; + +export async function postClearCache(req: Request, res: Response): Promise { + const videoID = req.query.videoID as VideoID; + const userID = req.query.userID as UserID; + const service = getService(req.query.service as Service); + + const invalidFields = []; + if (typeof videoID !== "string") { + invalidFields.push("videoID"); + } + if (typeof userID !== "string") { + invalidFields.push("userID"); + } + + if (invalidFields.length !== 0) { + // invalid request + const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ", " : "") + c, ""); + return res.status(400).send(`No valid ${fields} field(s) provided`); + } + + // hash the userID as early as possible + const hashedUserID: HashedUserID = getHash(userID); + // hash videoID + const hashedVideoID: VideoIDHash = getHash(videoID, 1); + + // Ensure user is a VIP + if (!(await isUserVIP(hashedUserID))){ + Logger.warn(`Permission violation: User ${hashedUserID} attempted to clear cache for video ${videoID}.`); + return res.status(403).json({ "message": "Not a VIP" }); + } + + try { + QueryCacher.clearRatingCache({ + hashedVideoID, + service + }); + return res.status(200).json({ + message: `Cache cleared on video ${videoID}` + }); + } catch(err) { + return res.sendStatus(500); + } +} diff --git a/src/routes/ratings/postRating.ts b/src/routes/ratings/postRating.ts index 7e73903..f5424b1 100644 --- a/src/routes/ratings/postRating.ts +++ b/src/routes/ratings/postRating.ts @@ -8,6 +8,7 @@ import { getIP } from "../../utils/getIP"; import { getService } from "../../utils/getService"; import { RatingType, RatingTypes } from "../../types/ratings.model"; import { config } from "../../config"; +import { QueryCacher } from "../../utils/queryCacher"; export async function postRating(req: Request, res: Response): Promise { const privateUserID = req.body.userID as UserID; @@ -34,26 +35,26 @@ export async function postRating(req: Request, res: Response): Promise // Undo the vote await db.prepare("run", `UPDATE "ratings" SET "count" = "count" - 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); await privateDB.prepare("run", `DELETE FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]); - - return res.sendStatus(200); } else if (existingVote.count === 0 && enabled) { // Make sure there hasn't been another vote from this IP const existingIPVote = (await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "hashedIP" = ?`, [videoID, service, type, hashedIP])) .count > 0; - if (!existingIPVote) { - // Check if general rating already exists, if so increase it - const rating = await db.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); - if (rating.count > 0) { - await db.prepare("run", `UPDATE "ratings" SET "count" = "count" + 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); - } else { - await db.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, 1, ?)`, [videoID, service, type, hashedVideoID]); - } - - // Create entry in privateDB - await privateDB.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "userID", "timeSubmitted", "hashedIP") VALUES (?, ?, ?, ?, ?, ?)`, [videoID, service, type, hashedUserID, Date.now(), hashedIP]); + if (existingIPVote) { // if exisiting vote, exit early instead + return res.sendStatus(200); + } + // Check if general rating already exists, if so increase it + const rating = await db.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + if (rating.count > 0) { + await db.prepare("run", `UPDATE "ratings" SET "count" = "count" + 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + } else { + await db.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, 1, ?)`, [videoID, service, type, hashedVideoID]); } - } + // Create entry in privateDB + await privateDB.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "userID", "timeSubmitted", "hashedIP") VALUES (?, ?, ?, ?, ?, ?)`, [videoID, service, type, hashedUserID, Date.now(), hashedIP]); + } + // clear rating cache + QueryCacher.clearRatingCache({ hashedVideoID, service }); return res.sendStatus(200); } catch (err) { Logger.error(err as string); diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index c35e9b1..d8ecb96 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -1,6 +1,6 @@ import redis from "../utils/redis"; import { Logger } from "../utils/logger"; -import { skipSegmentsHashKey, skipSegmentsKey, reputationKey } from "./redisKeys"; +import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey } from "./redisKeys"; import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { UserID } from "../types/user.model"; @@ -22,7 +22,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { return data; } -function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void { +function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void { if (videoInfo) { redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); @@ -30,7 +30,14 @@ function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHa } } +function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void { + if (videoInfo) { + redis.delAsync(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)); + } +} + export const QueryCacher = { get, - clearVideoCache + clearSegmentCache, + clearRatingCache }; \ No newline at end of file diff --git a/test/cases/ratings/postClearCache.ts b/test/cases/ratings/postClearCache.ts new file mode 100644 index 0000000..1b2b69a --- /dev/null +++ b/test/cases/ratings/postClearCache.ts @@ -0,0 +1,51 @@ +import { db } from "../../../src/databases/databases"; +import { getHash } from "../../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../../utils/httpClient"; + +const VIPUser = "clearCacheVIP"; +const regularUser = "regular-user"; +const endpoint = "/api/ratings/clearCache"; +const postClearCache = (userID: string, videoID: string) => client({ method: "post", url: endpoint, params: { userID, videoID } }); + +describe("ratings postClearCache", () => { + before(async () => { + await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('${getHash(VIPUser)}')`); + }); + + it("Should be able to clear cache amy video", (done) => { + postClearCache(VIPUser, "dne-video") + .then(res => { + assert.strictEqual(res.status, 200); + done(); + }) + .catch(err => done(err)); + }); + + it("Should get 403 as non-vip", (done) => { + postClearCache(regularUser, "clear-test") + .then(res => { + assert.strictEqual(res.status, 403); + done(); + }) + .catch(err => done(err)); + }); + + it("Should give 400 with missing videoID", (done) => { + client.post(endpoint, { params: { userID: VIPUser } }) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); + + it("Should give 400 with missing userID", (done) => { + client.post(endpoint, { params: { videoID: "clear-test" } }) + .then(res => { + assert.strictEqual(res.status, 400); + done(); + }) + .catch(err => done(err)); + }); +});