From 29d2c9c25efb959a40d1c81bab25e7293e0c920e Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 19 Mar 2021 21:31:16 -0400 Subject: [PATCH 1/6] Add new "Service" option --- databases/_upgrade_sponsorTimes_7.sql | 28 ++++++++++++++ src/routes/getSkipSegments.ts | 21 ++++++---- src/routes/getSkipSegmentsByHash.ts | 9 ++++- src/routes/postSkipSegments.ts | 22 ++++++----- src/types/segments.model.ts | 10 +++++ test/cases/getSkipSegments.ts | 38 ++++++++++++++----- ...entsByHash.ts => getSkipSegmentsByHash.ts} | 30 ++++++++++++--- test/cases/postSkipSegments.ts | 32 ++++++++++++++++ 8 files changed, 155 insertions(+), 35 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_7.sql rename test/cases/{getSegmentsByHash.ts => getSkipSegmentsByHash.ts} (80%) diff --git a/databases/_upgrade_sponsorTimes_7.sql b/databases/_upgrade_sponsorTimes_7.sql new file mode 100644 index 0000000..8f6060b --- /dev/null +++ b/databases/_upgrade_sponsorTimes_7.sql @@ -0,0 +1,28 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_7" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 7 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 65ab4f1..42a44c7 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -4,7 +4,7 @@ import { config } from '../config'; import { db, privateDB } from '../databases/databases'; import { skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; -import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; @@ -47,7 +47,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: })); } -async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise { +async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: Segment[] = []; @@ -59,8 +59,8 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C .prepare( 'all', `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" - WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`, - [videoID] + WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, + [videoID, service] )).reduce((acc: SBRecord, segment: DBSegment) => { acc[segment.category] = acc[segment.category] || []; acc[segment.category].push(segment); @@ -81,7 +81,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C } } -async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise> { +async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise> { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: SBRecord = {}; @@ -95,8 +95,8 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, .prepare( 'all', `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`, - [hashedVideoIDPrefix + '%'] + WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, + [hashedVideoIDPrefix + '%', service] )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { hash: segment.hashedVideoID, @@ -239,6 +239,11 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { + service = Service.YouTube; + } + // Only 404s are cached at the moment const redisResult = await redis.getAsync(skipSegmentsKey(videoID)); @@ -251,7 +256,7 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { + service = Service.YouTube; + } // filter out none string elements, only flat array with strings is valid categories = categories.filter((item: any) => typeof item === "string"); // Get all video id's that match hash prefix - const segments = await getSegmentsByHash(req, hashPrefix, categories); + const segments = await getSegmentsByHash(req, hashPrefix, categories, service); if (!segments) return res.status(404).json([]); diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 6a9c781..1101eac 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -13,6 +13,7 @@ import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; import { skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; +import { Service } from '../types/segments.model'; async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { @@ -45,8 +46,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st }); } -async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) { - if (config.youtubeAPIKey !== null) { +async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { + if (config.youtubeAPIKey !== null && service == Service.YouTube) { const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]); YouTubeAPI.listVideos(videoID, (err: any, data: any) => { @@ -267,7 +268,10 @@ export async function postSkipSegments(req: Request, res: Response) { const videoID = req.query.videoID || req.body.videoID; let userID = req.query.userID || req.body.userID; - + let service: Service = req.query.service ?? req.body.service ?? Service.YouTube; + if (!Object.values(Service).some((val) => val == service)) { + service = Service.YouTube; + } let segments = req.body.segments; if (segments === undefined) { @@ -367,7 +371,7 @@ export async function postSkipSegments(req: Request, res: Response) { //check if this info has already been submitted before const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ? - and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]); + and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]); if (duplicateCheck2Row.count > 0) { res.sendStatus(409); return; @@ -375,7 +379,7 @@ export async function postSkipSegments(req: Request, res: Response) { } // Auto moderator check - if (!isVIP) { + if (!isVIP && service == Service.YouTube) { const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category}); if (autoModerateResult == "Rejected based on NeuralBlock predictions.") { // If NB automod rejects, the submission will start with -2 votes. @@ -491,9 +495,9 @@ export async function postSkipSegments(req: Request, res: Response) { const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" - ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID") - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1), + ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "shadowHidden", "hashedVideoID") + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, shadowBanned, getHash(videoID, 1), ], ); @@ -529,7 +533,7 @@ export async function postSkipSegments(req: Request, res: Response) { res.json(newSegments); for (let i = 0; i < segments.length; i++) { - sendWebhooks(userID, videoID, UUIDs[i], segments[i]); + sendWebhooks(userID, videoID, UUIDs[i], segments[i], service); } } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index e7b6eaf..2478b70 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -8,6 +8,16 @@ export type VideoIDHash = VideoID & HashedValue; export type IPAddress = string & { __ipAddressBrand: unknown }; export type HashedIP = IPAddress & HashedValue; +// Uncomment as needed +export enum Service { + YouTube = 'YouTube', + // Nebula = 'Nebula', + PeerTube = 'PeerTube', + // RSS = 'RSS', + // Corridor = 'Corridor', + // Lbry = 'Lbry' +} + export interface Segment { category: Category; segment: number[]; diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index 85dcfbb..51f1c1b 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -5,16 +5,17 @@ import {getHash} from '../../src/utils/getHash'; describe('getSkipSegments', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')"); + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('testtesttest2', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest,test', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')"); return; }); @@ -37,6 +38,23 @@ describe('getSkipSegments', () => { .catch(err => "Couldn't call endpoint"); }); + it('Should be able to get a time by category for a different service 1', () => { + fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube") + .then(async res => { + if (res.status !== 200) return ("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1") { + return; + } else { + return ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + it('Should be able to get a time by category 2', () => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=intro") .then(async res => { diff --git a/test/cases/getSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts similarity index 80% rename from test/cases/getSegmentsByHash.ts rename to test/cases/getSkipSegmentsByHash.ts index d957e2d..fd45884 100644 --- a/test/cases/getSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -12,11 +12,12 @@ sinonStub.callsFake(YouTubeApiMock.listVideos); describe('getSegmentsByHash', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b }); it('Should be able to get a 200', (done: Done) => { @@ -128,7 +129,24 @@ describe('getSegmentsByHash', () => { if (body.length !== 2) done("expected 2 videos, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length); - else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else if (body[0].segments[0].category !== 'sponsor' + || body[0].segments[0].UUID !== 'getSegmentsByHash-0-0' + || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else done(); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => { + fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube') + .then(async res => { + if (res.status !== 200) done("non 200 status code, was " + res.status); + else { + const body = await res.json(); + if (body.length !== 1) done("expected 2 videos, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); + else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor"); else done(); } }) diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index e31326a..6bc6179 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -97,6 +97,38 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); + it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXcG", + service: "PeerTube", + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('VIP submission should start locked', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { From 5544491728d518faffe6146f6f5b818502c108c7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 19 Mar 2021 22:45:30 -0400 Subject: [PATCH 2/6] Add duration option when submitting and save duration in DB --- databases/_upgrade_sponsorTimes_8.sql | 29 +++ src/routes/postSkipSegments.ts | 338 +++++++++++++------------- src/types/segments.model.ts | 6 + test/cases/postSkipSegments.ts | 68 +++++- 4 files changed, 271 insertions(+), 170 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_8.sql diff --git a/databases/_upgrade_sponsorTimes_8.sql b/databases/_upgrade_sponsorTimes_8.sql new file mode 100644 index 0000000..ccc2ec9 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_8.sql @@ -0,0 +1,29 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_8" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" INTEGER NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 8 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 1101eac..3b6f82d 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -13,8 +13,12 @@ import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; import { skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; -import { Service } from '../types/segments.model'; +import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model'; +interface APIVideoInfo { + err: string | boolean, + data: any +} async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); @@ -46,62 +50,58 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st }); } -async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { - if (config.youtubeAPIKey !== null && service == Service.YouTube) { +async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { + if (apiVideoInfo && service == Service.YouTube) { const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]); - YouTubeAPI.listVideos(videoID, (err: any, data: any) => { - if (err || data.items.length === 0) { - err && Logger.error(err); - return; + const {data, err} = apiVideoInfo; + if (err) return; + + const startTime = parseFloat(segmentInfo.segment[0]); + const endTime = parseFloat(segmentInfo.segment[1]); + sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, { + submissionStart: startTime, + submissionEnd: endTime, + }, segmentInfo); + + // If it is a first time submission + // Then send a notification to discord + if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; + + fetch(config.discordFirstTimeSubmissionsWebhookURL, { + method: 'POST', + body: JSON.stringify({ + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2), + "description": "Submission ID: " + UUID + + "\n\nTimestamp: " + + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + + "\n\nCategory: " + segmentInfo.category, + "color": 10813440, + "author": { + "name": userID, + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + }, + }], + }), + headers: { + 'Content-Type': 'application/json' } - - const startTime = parseFloat(segmentInfo.segment[0]); - const endTime = parseFloat(segmentInfo.segment[1]); - sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, { - submissionStart: startTime, - submissionEnd: endTime, - }, segmentInfo); - - // If it is a first time submission - // Then send a notification to discord - if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; - - fetch(config.discordFirstTimeSubmissionsWebhookURL, { - method: 'POST', - body: JSON.stringify({ - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2), - "description": "Submission ID: " + UUID + - "\n\nTimestamp: " + - getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + - "\n\nCategory: " + segmentInfo.category, - "color": 10813440, - "author": { - "name": userID, - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - }, - }], - }), - headers: { - 'Content-Type': 'application/json' - } - }) - .then(res => { - if (res.status >= 400) { - Logger.error("Error sending first time submission Discord hook"); - Logger.error(JSON.stringify(res)); - Logger.error("\n"); - } - }) - .catch(err => { - Logger.error("Failed to send first time submission Discord hook."); - Logger.error(JSON.stringify(err)); + }) + .then(res => { + if (res.status >= 400) { + Logger.error("Error sending first time submission Discord hook"); + Logger.error(JSON.stringify(res)); Logger.error("\n"); - }); + } + }) + .catch(err => { + Logger.error("Failed to send first time submission Discord hook."); + Logger.error(JSON.stringify(err)); + Logger.error("\n"); }); } } @@ -167,73 +167,98 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. -async function autoModerateSubmission(submission: { videoID: any; userID: any; segments: any }) { - // Get the video information from the youtube API - if (config.youtubeAPIKey !== null) { - const {err, data} = await new Promise((resolve) => { - YouTubeAPI.listVideos(submission.videoID, (err: any, data: any) => resolve({err, data})); - }); +async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, + submission: { videoID: any; userID: any; segments: any }) { + if (apiVideoInfo) { + const {err, data} = apiVideoInfo; + if (err) return false; - if (err) { - return false; - } else { - // Check to see if video exists - if (data.pageInfo.totalResults === 0) { - return "No video exists with id " + submission.videoID; + // Check to see if video exists + if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID; + + const duration = getYouTubeVideoDuration(apiVideoInfo); + const segments = submission.segments; + let nbString = ""; + for (let i = 0; i < segments.length; i++) { + const startTime = parseFloat(segments[i].segment[0]); + const endTime = parseFloat(segments[i].segment[1]); + + if (duration == 0) { + // Allow submission if the duration is 0 (bug in youtube api) + return false; } else { - const segments = submission.segments; - let nbString = ""; - for (let i = 0; i < segments.length; i++) { + if (segments[i].category === "sponsor") { + //Prepare timestamps to send to NB all at once + nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";"; + } + } + } + + // Get all submissions for this user + const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]); + const allSegmentTimes = []; + if (allSubmittedByUser !== undefined) { + //add segments the user has previously submitted + for (const segmentInfo of allSubmittedByUser) { + allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]); + } + } + + //add segments they are trying to add in this submission + for (let i = 0; i < segments.length; i++) { + let startTime = parseFloat(segments[i].segment[0]); + let endTime = parseFloat(segments[i].segment[1]); + allSegmentTimes.push([startTime, endTime]); + } + + //merge all the times into non-overlapping arrays + const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) { + return a[0] - b[0] || a[1] - b[1]; + })); + + let videoDuration = data.items[0].contentDetails.duration; + videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); + if (videoDuration != 0) { + let allSegmentDuration = 0; + //sum all segment times together + allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); + if (allSegmentDuration > (videoDuration / 100) * 80) { + // Reject submission if all segments combine are over 80% of the video + return "Total length of your submitted segments are over 80% of the video."; + } + } + + // Check NeuralBlock + const neuralBlockURL = config.neuralBlockURL; + if (!neuralBlockURL) return false; + const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + + "&segments=" + nbString.substring(0, nbString.length - 1)); + if (!response.ok) return false; + + const nbPredictions = await response.json(); + let nbDecision = false; + let predictionIdx = 0; //Keep track because only sponsor categories were submitted + for (let i = 0; i < segments.length; i++) { + if (segments[i].category === "sponsor") { + if (nbPredictions.probabilities[predictionIdx] < 0.70) { + nbDecision = true; // At least one bad entry const startTime = parseFloat(segments[i].segment[0]); const endTime = parseFloat(segments[i].segment[1]); - let duration = data.items[0].contentDetails.duration; - duration = isoDurations.toSeconds(isoDurations.parse(duration)); - if (duration == 0) { - // Allow submission if the duration is 0 (bug in youtube api) - return false; - } else if ((endTime - startTime) > (duration / 100) * 80) { - // Reject submission if over 80% of the video - return "One of your submitted segments is over 80% of the video."; - } else { - if (segments[i].category === "sponsor") { - //Prepare timestamps to send to NB all at once - nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";"; - } - } - } - // Check NeuralBlock - const neuralBlockURL = config.neuralBlockURL; - if (!neuralBlockURL) return false; - const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + - "&segments=" + nbString.substring(0, nbString.length - 1)); - if (!response.ok) return false; - - const nbPredictions = await response.json(); - let nbDecision = false; - let predictionIdx = 0; //Keep track because only sponsor categories were submitted - for (let i = 0; i < segments.length; i++) { - if (segments[i].category === "sponsor") { - if (nbPredictions.probabilities[predictionIdx] < 0.70) { - nbDecision = true; // At least one bad entry - const startTime = parseFloat(segments[i].segment[0]); - const endTime = parseFloat(segments[i].segment[1]); - - const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime); - // Send to Discord - // Note, if this is too spammy. Consider sending all the segments as one Webhook - sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); - } - predictionIdx++; - } - - } - if (nbDecision) { - return "Rejected based on NeuralBlock predictions."; - } else { - return false; + const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime); + // Send to Discord + // Note, if this is too spammy. Consider sending all the segments as one Webhook + sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); } + predictionIdx++; } + + } + + if (nbDecision) { + return "Rejected based on NeuralBlock predictions."; + } else { + return false; } } else { Logger.debug("Skipped YouTube API"); @@ -244,6 +269,21 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s } } +function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration { + const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration; + return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null; +} + +async function getYouTubeVideoInfo(videoID: VideoID): Promise { + if (config.youtubeAPIKey !== null) { + return new Promise((resolve) => { + YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data})); + }); + } else { + return null; + } +} + function proxySubmission(req: Request) { fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, { method: 'POST', @@ -272,13 +312,14 @@ export async function postSkipSegments(req: Request, res: Response) { if (!Object.values(Service).some((val) => val == service)) { service = Service.YouTube; } + let videoDuration: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration; - let segments = req.body.segments; + let segments = req.body.segments as IncomingSegment[]; if (segments === undefined) { // Use query instead segments = [{ - segment: [req.query.startTime, req.query.endTime], - category: req.query.category, + segment: [req.query.startTime as string, req.query.endTime as string], + category: req.query.category as Category }]; } @@ -378,9 +419,15 @@ export async function postSkipSegments(req: Request, res: Response) { } } + let apiVideoInfo: APIVideoInfo = null; + if (service == Service.YouTube) { + apiVideoInfo = await getYouTubeVideoInfo(videoID); + } + videoDuration = getYouTubeVideoDuration(apiVideoInfo) || videoDuration; + // Auto moderator check if (!isVIP && service == Service.YouTube) { - const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category}); + const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category}); if (autoModerateResult == "Rejected based on NeuralBlock predictions.") { // If NB automod rejects, the submission will start with -2 votes. // Note, if one submission is bad all submissions will be affected. @@ -441,63 +488,18 @@ export async function postSkipSegments(req: Request, res: Response) { let startingVotes = 0 + decreaseVotes; - if (config.youtubeAPIKey !== null) { - let {err, data} = await new Promise((resolve) => { - YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data})); - }); - - if (err) { - Logger.error("Error while submitting when connecting to YouTube API: " + err); - } else { - //get all segments for this video and user - const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [userID, videoID]); - const allSegmentTimes = []; - if (allSubmittedByUser !== undefined) { - //add segments the user has previously submitted - for (const segmentInfo of allSubmittedByUser) { - allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]); - } - } - - //add segments they are trying to add in this submission - for (let i = 0; i < segments.length; i++) { - let startTime = parseFloat(segments[i].segment[0]); - let endTime = parseFloat(segments[i].segment[1]); - allSegmentTimes.push([startTime, endTime]); - } - - //merge all the times into non-overlapping arrays - const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) { - return a[0] - b[0] || a[1] - b[1]; - })); - - let videoDuration = data.items[0].contentDetails.duration; - videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); - if (videoDuration != 0) { - let allSegmentDuration = 0; - //sum all segment times together - allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); - if (allSegmentDuration > (videoDuration / 100) * 80) { - // Reject submission if all segments combine are over 80% of the video - res.status(400).send("Total length of your submitted segments are over 80% of the video."); - return; - } - } - } - } - for (const segmentInfo of segments) { //this can just be a hash of the data //it's better than generating an actual UUID like what was used before //also better for duplication checking - const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]); + const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1])); const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" - ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "shadowHidden", "hashedVideoID") - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, shadowBanned, getHash(videoID, 1), + ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID") + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1), ], ); @@ -508,7 +510,7 @@ export async function postSkipSegments(req: Request, res: Response) { redis.delAsync(skipSegmentsKey(videoID)); } catch (err) { //a DB change probably occurred - res.sendStatus(502); + res.sendStatus(500); Logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err); @@ -533,7 +535,7 @@ export async function postSkipSegments(req: Request, res: Response) { res.json(newSegments); for (let i = 0; i < segments.length; i++) { - sendWebhooks(userID, videoID, UUIDs[i], segments[i], service); + sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service); } } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 2478b70..2bb4e8d 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -3,6 +3,7 @@ import { SBRecord } from "./lib.model"; export type SegmentUUID = string & { __segmentUUIDBrand: unknown }; export type VideoID = string & { __videoIDBrand: unknown }; +export type VideoDuration = number & { __videoDurationBrand: unknown }; export type Category = string & { __categoryBrand: unknown }; export type VideoIDHash = VideoID & HashedValue; export type IPAddress = string & { __ipAddressBrand: unknown }; @@ -18,6 +19,11 @@ export enum Service { // Lbry = 'Lbry' } +export interface IncomingSegment { + category: Category; + segment: string[]; +} + export interface Segment { category: Category; segment: number[]; diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 6bc6179..9ca8333 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -97,6 +97,70 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); + it('Should be able to submit a single time with a duration (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXZX", + videoDuration: 100, + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit a single time with a duration from the API (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "noDuration", + videoDuration: 100, + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["noDuration"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 100) { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { @@ -276,7 +340,7 @@ describe('postSkipSegments', () => { }), }) .then(async res => { - if (res.status === 400) { + if (res.status === 403) { const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["n9rIGdXnSJc"]); let success = true; if (rows.length === 4) { @@ -324,7 +388,7 @@ describe('postSkipSegments', () => { }), }) .then(async res => { - if (res.status === 400) { + if (res.status === 403) { const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["80percent_video"]); let success = rows.length == 2; for (const row of rows) { From 3c89e9c01506e84de46032dc37d04f31a3c5477c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 19 Mar 2021 22:52:23 -0400 Subject: [PATCH 3/6] Send back duration in getSkipSegments request --- src/routes/getSkipSegments.ts | 7 ++++--- src/types/segments.model.ts | 2 ++ test/cases/getSkipSegments.ts | 34 +++++++++++++++++----------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 42a44c7..b6ae72f 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -43,7 +43,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: return chooseSegments(filteredSegments).map((chosenSegment) => ({ category, segment: [chosenSegment.startTime, chosenSegment.endTime], - UUID: chosenSegment.UUID + UUID: chosenSegment.UUID, + videoDuration: chosenSegment.videoDuration })); } @@ -58,7 +59,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C const segmentsByCategory: SBRecord = (await db .prepare( 'all', - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" + `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, [videoID, service] )).reduce((acc: SBRecord, segment: DBSegment) => { @@ -94,7 +95,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, const segmentPerVideoID: SegmentWithHashPerVideoID = (await db .prepare( 'all', - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, [hashedVideoIDPrefix + '%', service] )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 2bb4e8d..6c9cd4a 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -28,6 +28,7 @@ export interface Segment { category: Category; segment: number[]; UUID: SegmentUUID; + videoDuration: VideoDuration; } export enum Visibility { @@ -44,6 +45,7 @@ export interface DBSegment { locked: boolean; shadowHidden: Visibility; videoID: VideoID; + videoDuration: VideoDuration; hashedVideoID: VideoIDHash; } diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index 51f1c1b..674a773 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -5,17 +5,17 @@ import {getHash} from '../../src/utils/getHash'; describe('getSkipSegments', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('testtesttest2', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest,test', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')"); + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, '" + getHash('testtesttest2', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, '" + getHash('testtesttest,test', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, '" + getHash('locked', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, '" + getHash('locked', 1) + "')"); return; }); @@ -28,7 +28,7 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 - && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -45,7 +45,7 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 - && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1") { + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -79,7 +79,7 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 - && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -96,7 +96,7 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 - && data[0].category === "intro" && data[0].UUID === "1-uuid-2") { + && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -117,9 +117,9 @@ describe('getSkipSegments', () => { let success = true; for (const segment of data) { if ((segment.segment[0] !== 20 || segment.segment[1] !== 33 - || segment.category !== "intro" || segment.UUID !== "1-uuid-7") && + || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) && (segment.segment[0] !== 1 || segment.segment[1] !== 11 - || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) { + || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) { success = false; break; } From 02e628f5330403bc578d1e92a54d7d4ab08b54f2 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 20 Mar 2021 01:08:33 -0400 Subject: [PATCH 4/6] Setup csv exports and html status page --- .gitignore | 1 + docker/docker-compose.yml | 1 + src/app.ts | 11 +++++-- src/routes/dumpDatabase.ts | 61 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 src/routes/dumpDatabase.ts diff --git a/.gitignore b/.gitignore index 3cff924..dd78e49 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ test/databases/sponsorTimes.db test/databases/sponsorTimes.db-shm test/databases/sponsorTimes.db-wal test/databases/private.db +docker/database-export # Config files config.json diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a54a3c0..4ee69ee 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,6 +7,7 @@ services: - database.env volumes: - database-data:/var/lib/postgresql/data + - ./database-export/:/opt/exports ports: - 127.0.0.1:5432:5432 redis: diff --git a/src/app.ts b/src/app.ts index 098e5f3..026f74c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter'; import {loggerMiddleware} from './middleware/logger'; import {corsMiddleware} from './middleware/cors'; import {rateLimitMiddleware} from './middleware/requestRateLimit'; +import dumpDatabase from './routes/dumpDatabase'; export function createServer(callback: () => void) { @@ -127,7 +128,11 @@ function setupRoutes(app: Express) { //get if user is a vip app.post('/api/segmentShift', postSegmentShift); - app.get('/database.db', function (req: Request, res: Response) { - res.sendFile("./databases/sponsorTimes.db", {root: "./"}); - }); + if (config.postgres) { + app.get('/database', dumpDatabase); + } else { + app.get('/database.db', function (req: Request, res: Response) { + res.sendFile("./databases/sponsorTimes.db", {root: "./"}); + }); + } } diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts new file mode 100644 index 0000000..fb8363d --- /dev/null +++ b/src/routes/dumpDatabase.ts @@ -0,0 +1,61 @@ +import {db} from '../databases/databases'; +import {Logger} from '../utils/logger'; +import {Request, Response} from 'express'; +import { config } from '../config'; + +const ONE_MINUTE = 1000 * 60; + +const styleHeader = `` + +const licenseHeader = `

The API and database follow CC BY-NC-SA 4.0 unless you have explicit permission.

+

Attribution Template

+

If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.

`; + +const tables = [{ + name: "sponsorTimes", + order: "timeSubmitted" +}, +{ + name: "userNames" +}, +{ + name: "categoryVotes" +}, +{ + name: "noSegments", +}, +{ + name: "warnings", + order: "issueTime" +}, +{ + name: "vipUsers" +}]; + +const links: string = tables.map((table) => `

${table.name}.csv

`) + .reduce((acc, url) => acc + url, ""); + +let lastUpdate = 0; + +export default function dumpDatabase(req: Request, res: Response) { + if (!config.postgres) { + res.status(404).send("Not supported on this instance"); + return; + } + + const now = Date.now(); + const updateQueued = now - lastUpdate > ONE_MINUTE; + + res.status(200).send(`${styleHeader} +

SponsorBlock database dumps

${licenseHeader}${links}
+ ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); + + if (updateQueued) { + lastUpdate = Date.now(); + + for (const table of tables) { + db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) + TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); + } + } +} \ No newline at end of file From 8423165df45a883378245cfb2890dfb40b08c184 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 20 Mar 2021 01:13:16 -0400 Subject: [PATCH 5/6] Add json page for database export --- src/app.ts | 3 ++- src/routes/dumpDatabase.ts | 22 +++++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/app.ts b/src/app.ts index 026f74c..5bc2b2c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -129,7 +129,8 @@ function setupRoutes(app: Express) { app.post('/api/segmentShift', postSegmentShift); if (config.postgres) { - app.get('/database', dumpDatabase); + app.get('/database', (req, res) => dumpDatabase(req, res, true)); + app.get('/database.json', (req, res) => dumpDatabase(req, res, false)); } else { app.get('/database.db', function (req: Request, res: Response) { res.sendFile("./databases/sponsorTimes.db", {root: "./"}); diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index fb8363d..0d0ab02 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -32,12 +32,14 @@ const tables = [{ name: "vipUsers" }]; -const links: string = tables.map((table) => `

${table.name}.csv

`) +const links: string[] = tables.map((table) => `/database/${table.name}.csv`); + +const linksHTML: string = tables.map((table) => `

${table.name}.csv

`) .reduce((acc, url) => acc + url, ""); let lastUpdate = 0; -export default function dumpDatabase(req: Request, res: Response) { +export default function dumpDatabase(req: Request, res: Response, showPage: boolean) { if (!config.postgres) { res.status(404).send("Not supported on this instance"); return; @@ -46,9 +48,19 @@ export default function dumpDatabase(req: Request, res: Response) { const now = Date.now(); const updateQueued = now - lastUpdate > ONE_MINUTE; - res.status(200).send(`${styleHeader} -

SponsorBlock database dumps

${licenseHeader}${links}
- ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); + res.status(200) + + if (showPage) { + res.send(`${styleHeader} +

SponsorBlock database dumps

${licenseHeader}${linksHTML}
+ ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); + } else { + res.send({ + lastUpdated: lastUpdate, + updateQueued, + links + }) + } if (updateQueued) { lastUpdate = Date.now(); From 180d9bfb73d5941b99c7cf76fbca06356e1a1ce1 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 20 Mar 2021 11:46:37 -0400 Subject: [PATCH 6/6] Add explanation to database page --- src/routes/dumpDatabase.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index 0d0ab02..d687f07 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -52,7 +52,12 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool if (showPage) { res.send(`${styleHeader} -

SponsorBlock database dumps

${licenseHeader}${linksHTML}
+

SponsorBlock database dumps

${licenseHeader} +

How this works

+ Send a request to https://sponsor.ajay.app/database.json, or visit this page to trigger the database dump to run. + Then, you can download the csv files below, or use the links returned from the JSON request. +

Links

+ ${linksHTML}
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); } else { res.send({