diff --git a/.gitignore b/.gitignore index 3cff924..dd78e49 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ test/databases/sponsorTimes.db test/databases/sponsorTimes.db-shm test/databases/sponsorTimes.db-wal test/databases/private.db +docker/database-export # Config files config.json diff --git a/README.MD b/README.MD index 8f61c57..e309e54 100644 --- a/README.MD +++ b/README.MD @@ -8,7 +8,7 @@ This is the server backend for it This uses a Postgres or Sqlite database to hold all the timing data. -To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. +To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea. diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql new file mode 100644 index 0000000..2b70fa7 --- /dev/null +++ b/databases/_private_indexes.sql @@ -0,0 +1,32 @@ +-- sponsorTimes + +CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP" + ON public."sponsorTimes" USING btree + ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID" + ON public."sponsorTimes" USING btree + ("videoID" ASC NULLS LAST) +; + +-- votes + +CREATE INDEX IF NOT EXISTS "votes_userID" + ON public.votes USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- shadowBannedUsers + +CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index" + ON public."shadowBannedUsers" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- categoryVotes + +CREATE INDEX IF NOT EXISTS "categoryVotes_UUID" + ON public."categoryVotes" USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql new file mode 100644 index 0000000..c90ef51 --- /dev/null +++ b/databases/_sponsorTimes_indexes.sql @@ -0,0 +1,66 @@ +-- sponsorTimes + +CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted" + ON public."sponsorTimes" USING btree + ("timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTime_userID" + ON public."sponsorTimes" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID" + ON public."sponsorTimes" USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID_gin" + ON public."sponsorTimes" USING gin + ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID" + ON public."sponsorTimes" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +-- userNames + +CREATE INDEX IF NOT EXISTS "userNames_userID" + ON public."userNames" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- vipUsers + +CREATE INDEX IF NOT EXISTS "vipUsers_index" + ON public."vipUsers" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- warnings + +CREATE INDEX IF NOT EXISTS "warnings_index" + ON public.warnings USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "warnings_issueTime" + ON public.warnings USING btree + ("issueTime" ASC NULLS LAST) + TABLESPACE pg_default; + +-- noSegments + +CREATE INDEX IF NOT EXISTS "noSegments_videoID" + ON public."noSegments" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- categoryVotes + +CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public" + ON public."categoryVotes" USING btree + ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_10.sql b/databases/_upgrade_sponsorTimes_10.sql new file mode 100644 index 0000000..174ceaf --- /dev/null +++ b/databases/_upgrade_sponsorTimes_10.sql @@ -0,0 +1,30 @@ +BEGIN TRANSACTION; + +/* Add Hidden field */ +CREATE TABLE "sqlb_temp_table_10" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" REAL NOT NULL DEFAULT '0', + "hidden" INTEGER NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 10 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_7.sql b/databases/_upgrade_sponsorTimes_7.sql new file mode 100644 index 0000000..8f6060b --- /dev/null +++ b/databases/_upgrade_sponsorTimes_7.sql @@ -0,0 +1,28 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_7" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 7 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_8.sql b/databases/_upgrade_sponsorTimes_8.sql new file mode 100644 index 0000000..ccc2ec9 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_8.sql @@ -0,0 +1,29 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_8" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" INTEGER NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 8 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_9.sql b/databases/_upgrade_sponsorTimes_9.sql new file mode 100644 index 0000000..5015a4a --- /dev/null +++ b/databases/_upgrade_sponsorTimes_9.sql @@ -0,0 +1,29 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_9" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" REAL NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 9 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a54a3c0..4ee69ee 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,6 +7,7 @@ services: - database.env volumes: - database-data:/var/lib/postgresql/data + - ./database-export/:/opt/exports ports: - 127.0.0.1:5432:5432 redis: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index daee81f..76a77d3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -2,7 +2,7 @@ worker_processes 8; worker_rlimit_nofile 8192; events { - worker_connections 32768; ## Default: 1024 + worker_connections 132768; ## Default: 1024 } http { @@ -12,19 +12,35 @@ http { upstream backend_GET { least_conn; - server localhost:4442; - server localhost:4443; - server localhost:4444; - server localhost:4445; - server localhost:4446; + server localhost:4441; + server localhost:4442; + #server localhost:4443; + #server localhost:4444; + #server localhost:4445; + #server localhost:4446; #server localhost:4447; #server localhost:4448; + + server 10.0.0.3:4441; + server 10.0.0.3:4442; + + #server 134.209.69.251:80 backup; + + server 116.203.32.253:80 backup; + #server 116.203.32.253:80; } upstream backend_POST { - server localhost:4441; + #server localhost:4441; + #server localhost:4442; + server 10.0.0.3:4441; + #server 10.0.0.3:4442; + } + upstream backend_db { + #server localhost:4441; + server 10.0.0.3:4441; } - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m; + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m; proxy_cache_key "$scheme$request_method$host$request_uri"; add_header X-Cache $upstream_cache_status; @@ -43,14 +59,16 @@ http { # internal; #} + #proxy_send_timeout 120s; + location @myerrordirective_500 { - return 502 "Internal Server Error"; + return 400 "Internal Server Error"; } location @myerrordirective_502 { - return 502 "Bad Gateway"; + return 400 "Bad Gateway"; } location @myerrordirective_504 { - return 502 "Gateway Timeout"; + return 400 "Gateway Timeout"; } @@ -62,17 +80,16 @@ http { return 301 https://sb.ltn.fi; } - location /invidious/ { - proxy_pass https://invidious.fdn.fr/; - } - location /test/ { proxy_pass http://localhost:4440/; #proxy_pass https://sbtest.etcinit.com/; } location /api/skipSegments { - proxy_pass http://backend_$request_method; + #return 200 "[]"; + proxy_pass http://backend_$request_method; + #proxy_cache CACHEZONE; + #proxy_cache_valid 2m; } location /api/getTopUsers { @@ -83,24 +100,43 @@ http { location /api/getTotalStats { proxy_pass http://backend_GET; - } + #return 200 ""; + } location /api/getVideoSponsorTimes { proxy_pass http://backend_GET; } - - location = /database.db { - alias /home/sbadmin/sponsor/databases/sponsorTimes.db; + + location /database { + proxy_pass http://backend_db; } - + + location = /database.db { + #return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals."; + #alias /home/sbadmin/sponsor/databases/sponsorTimes.db; + alias /home/sbadmin/test-db/database.db; + } + + location = /database/sponsorTimes.csv { + alias /home/sbadmin/sponsorTimes.csv; + } + + #location /api/voteOnSponsorTime { + # return 200 "Success"; + #} + + #location /api/viewedVideoSponsorTime { + # return 200 "Success"; + #} + location /api { proxy_pass http://backend_POST; } location / { - root /home/sbadmin/caddy/SponsorBlockSite/public-prod; - + root /home/sbadmin/SponsorBlockSite/public-prod; + ### CORS if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; @@ -132,12 +168,15 @@ http { } - listen 443 ssl; # managed by Certbot + listen 443 default_server ssl; # managed by Certbot + #listen 80; ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + + } diff --git a/package-lock.json b/package-lock.json index dd69f2e..f17b109 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2989,9 +2989,9 @@ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, "y18n": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", - "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", + "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", "dev": true }, "yallist": { diff --git a/src/app.ts b/src/app.ts index 098e5f3..2af8835 100644 --- a/src/app.ts +++ b/src/app.ts @@ -5,7 +5,7 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes'; import {postSegmentShift} from './routes/postSegmentShift'; import {postWarning} from './routes/postWarning'; import {getIsUserVIP} from './routes/getIsUserVIP'; -import {deleteNoSegments} from './routes/deleteNoSegments'; +import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments'; import {postNoSegments} from './routes/postNoSegments'; import {getUserInfo} from './routes/getUserInfo'; import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted'; @@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter'; import {loggerMiddleware} from './middleware/logger'; import {corsMiddleware} from './middleware/cors'; import {rateLimitMiddleware} from './middleware/requestRateLimit'; +import dumpDatabase from './routes/dumpDatabase'; export function createServer(callback: () => void) { @@ -116,7 +117,7 @@ function setupRoutes(app: Express) { //submit video containing no segments app.post('/api/noSegments', postNoSegments); - app.delete('/api/noSegments', deleteNoSegments); + app.delete('/api/noSegments', deleteNoSegmentsEndpoint); //get if user is a vip app.get('/api/isUserVIP', getIsUserVIP); @@ -127,7 +128,12 @@ function setupRoutes(app: Express) { //get if user is a vip app.post('/api/segmentShift', postSegmentShift); - app.get('/database.db', function (req: Request, res: Response) { - res.sendFile("./databases/sponsorTimes.db", {root: "./"}); - }); + if (config.postgres) { + app.get('/database', (req, res) => dumpDatabase(req, res, true)); + app.get('/database.json', (req, res) => dumpDatabase(req, res, false)); + } else { + app.get('/database.db', function (req: Request, res: Response) { + res.sendFile("./databases/sponsorTimes.db", {root: "./"}); + }); + } } diff --git a/src/config.ts b/src/config.ts index 93c6a43..7b15987 100644 --- a/src/config.ts +++ b/src/config.ts @@ -16,7 +16,7 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"], maxNumberOfActiveWarnings: 3, hoursAfterWarningExpires: 24, adminUserID: "", diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index bae2da6..779ebf2 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -23,6 +23,13 @@ export class Postgres implements IDatabase { // Upgrade database if required await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder); + + try { + await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder); + } catch (e) { + Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information."); + Logger.warn(e); + } } } @@ -118,9 +125,18 @@ export class Postgres implements IDatabase { Logger.debug('db update: no file ' + path); } + private async applyIndexes(fileNamePrefix: string, schemaFolder: string) { + const path = schemaFolder + "/_" + fileNamePrefix + "_indexes.sql"; + if (fs.existsSync(path)) { + await this.pool.query(fs.readFileSync(path).toString()); + } else { + Logger.debug('failed to apply indexes to ' + fileNamePrefix); + } + } + private processUpgradeQuery(query: string): string { let result = query; - result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')"); + result = result.replace(/sha256\((.*?)\)/gm, "encode(digest($1, 'sha256'), 'hex')"); result = result.replace(/integer/gmi, "NUMERIC"); return result; diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts index 1335e15..56b1aab 100644 --- a/src/middleware/redisKeys.ts +++ b/src/middleware/redisKeys.ts @@ -1,5 +1,13 @@ -import { Category, VideoID } from "../types/segments.model"; +import { Service, VideoID, VideoIDHash } from "../types/segments.model"; +import { Logger } from "../utils/logger"; export function skipSegmentsKey(videoID: VideoID): string { return "segments-" + videoID; } + +export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { + hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; + if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix); + + return "segments." + service + "." + hashedVideoIDPrefix; +} \ No newline at end of file diff --git a/src/routes/deleteNoSegments.ts b/src/routes/deleteNoSegments.ts index cfa6c78..5584c5a 100644 --- a/src/routes/deleteNoSegments.ts +++ b/src/routes/deleteNoSegments.ts @@ -2,12 +2,14 @@ import {Request, Response} from 'express'; import {isUserVIP} from '../utils/isUserVIP'; import {getHash} from '../utils/getHash'; import {db} from '../databases/databases'; +import { Category, VideoID } from '../types/segments.model'; +import { UserID } from '../types/user.model'; -export async function deleteNoSegments(req: Request, res: Response) { +export async function deleteNoSegmentsEndpoint(req: Request, res: Response) { // Collect user input data - const videoID = req.body.videoID; - let userID = req.body.userID; - const categories = req.body.categories; + const videoID = req.body.videoID as VideoID; + const userID = req.body.userID as UserID; + const categories = req.body.categories as Category[]; // Check input data is valid if (!videoID @@ -23,8 +25,8 @@ export async function deleteNoSegments(req: Request, res: Response) { } // Check if user is VIP - userID = getHash(userID); - const userIsVIP = await isUserVIP(userID); + const hashedUserID = getHash(userID); + const userIsVIP = await isUserVIP(hashedUserID); if (!userIsVIP) { res.status(403).json({ @@ -33,13 +35,22 @@ export async function deleteNoSegments(req: Request, res: Response) { return; } + deleteNoSegments(videoID, categories); + + res.status(200).json({message: 'Removed no segments entrys for video ' + videoID}); +} + +/** + * + * @param videoID + * @param categories If null, will remove all + */ +export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise { const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => { - return (categories.indexOf(entry.category) !== -1); + return categories === null || categories.indexOf(entry.category) !== -1; }); for (const entry of entries) { await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]); } - - res.status(200).json({message: 'Removed no segments entrys for video ' + videoID}); } diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts new file mode 100644 index 0000000..d687f07 --- /dev/null +++ b/src/routes/dumpDatabase.ts @@ -0,0 +1,78 @@ +import {db} from '../databases/databases'; +import {Logger} from '../utils/logger'; +import {Request, Response} from 'express'; +import { config } from '../config'; + +const ONE_MINUTE = 1000 * 60; + +const styleHeader = `` + +const licenseHeader = `

The API and database follow CC BY-NC-SA 4.0 unless you have explicit permission.

+

Attribution Template

+

If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.

`; + +const tables = [{ + name: "sponsorTimes", + order: "timeSubmitted" +}, +{ + name: "userNames" +}, +{ + name: "categoryVotes" +}, +{ + name: "noSegments", +}, +{ + name: "warnings", + order: "issueTime" +}, +{ + name: "vipUsers" +}]; + +const links: string[] = tables.map((table) => `/database/${table.name}.csv`); + +const linksHTML: string = tables.map((table) => `

${table.name}.csv

`) + .reduce((acc, url) => acc + url, ""); + +let lastUpdate = 0; + +export default function dumpDatabase(req: Request, res: Response, showPage: boolean) { + if (!config.postgres) { + res.status(404).send("Not supported on this instance"); + return; + } + + const now = Date.now(); + const updateQueued = now - lastUpdate > ONE_MINUTE; + + res.status(200) + + if (showPage) { + res.send(`${styleHeader} +

SponsorBlock database dumps

${licenseHeader} +

How this works

+ Send a request to https://sponsor.ajay.app/database.json, or visit this page to trigger the database dump to run. + Then, you can download the csv files below, or use the links returned from the JSON request. +

Links

+ ${linksHTML}
+ ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); + } else { + res.send({ + lastUpdated: lastUpdate, + updateQueued, + links + }) + } + + if (updateQueued) { + lastUpdate = Date.now(); + + for (const table of tables) { + db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) + TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); + } + } +} \ No newline at end of file diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 65ab4f1..d36a91a 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -2,9 +2,9 @@ import { Request, Response } from 'express'; import { RedisClient } from 'redis'; import { config } from '../config'; import { db, privateDB } from '../databases/databases'; -import { skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; -import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; @@ -43,11 +43,12 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: return chooseSegments(filteredSegments).map((chosenSegment) => ({ category, segment: [chosenSegment.startTime, chosenSegment.endTime], - UUID: chosenSegment.UUID + UUID: chosenSegment.UUID, + videoDuration: chosenSegment.videoDuration })); } -async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise { +async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: Segment[] = []; @@ -58,9 +59,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C const segmentsByCategory: SBRecord = (await db .prepare( 'all', - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" - WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`, - [videoID] + `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" + WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, + [videoID, service] )).reduce((acc: SBRecord, segment: DBSegment) => { acc[segment.category] = acc[segment.category] || []; acc[segment.category].push(segment); @@ -81,7 +82,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C } } -async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise> { +async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise> { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: SBRecord = {}; @@ -91,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category))); if (categories.length === 0) return null; - const segmentPerVideoID: SegmentWithHashPerVideoID = (await db - .prepare( - 'all', - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`, - [hashedVideoIDPrefix + '%'] - )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { + const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service)) + .filter((segment: DBSegment) => categories.includes(segment?.category)) + .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { hash: segment.hashedVideoID, segmentPerCategory: {}, @@ -130,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, } } +async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { + const fetchFromDB = () => db + .prepare( + 'all', + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" + WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, + [hashedVideoIDPrefix + '%', service] + ); + + if (hashedVideoIDPrefix.length === 4) { + const key = skipSegmentsHashKey(hashedVideoIDPrefix, service); + const {err, reply} = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug("Got data from redis: " + reply); + return JSON.parse(reply); + } catch (e) { + // If all else, continue on to fetching from the database + } + } + + const data = await fetchFromDB(); + + redis.setAsync(key, JSON.stringify(data)); + return data; + } + + return await fetchFromDB(); +} + //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique @@ -239,6 +267,11 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { + service = Service.YouTube; + } + // Only 404s are cached at the moment const redisResult = await redis.getAsync(skipSegmentsKey(videoID)); @@ -251,7 +284,7 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { + service = Service.YouTube; + } // filter out none string elements, only flat array with strings is valid categories = categories.filter((item: any) => typeof item === "string"); // Get all video id's that match hash prefix - const segments = await getSegmentsByHash(req, hashPrefix, categories); + const segments = await getSegmentsByHash(req, hashPrefix, categories, service); if (!segments) return res.status(404).json([]); diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 6a9c781..1f380da 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -11,9 +11,15 @@ import {getFormattedTime} from '../utils/getFormattedTime'; import {isUserTrustworthy} from '../utils/isUserTrustworthy'; import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; -import { skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; +import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; +import { deleteNoSegments } from './deleteNoSegments'; +interface APIVideoInfo { + err: string | boolean, + data: any +} async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); @@ -45,62 +51,58 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st }); } -async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) { - if (config.youtubeAPIKey !== null) { +async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { + if (apiVideoInfo && service == Service.YouTube) { const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]); - YouTubeAPI.listVideos(videoID, (err: any, data: any) => { - if (err || data.items.length === 0) { - err && Logger.error(err); - return; + const {data, err} = apiVideoInfo; + if (err) return; + + const startTime = parseFloat(segmentInfo.segment[0]); + const endTime = parseFloat(segmentInfo.segment[1]); + sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, { + submissionStart: startTime, + submissionEnd: endTime, + }, segmentInfo); + + // If it is a first time submission + // Then send a notification to discord + if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; + + fetch(config.discordFirstTimeSubmissionsWebhookURL, { + method: 'POST', + body: JSON.stringify({ + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2), + "description": "Submission ID: " + UUID + + "\n\nTimestamp: " + + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + + "\n\nCategory: " + segmentInfo.category, + "color": 10813440, + "author": { + "name": userID, + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + }, + }], + }), + headers: { + 'Content-Type': 'application/json' } - - const startTime = parseFloat(segmentInfo.segment[0]); - const endTime = parseFloat(segmentInfo.segment[1]); - sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, { - submissionStart: startTime, - submissionEnd: endTime, - }, segmentInfo); - - // If it is a first time submission - // Then send a notification to discord - if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; - - fetch(config.discordFirstTimeSubmissionsWebhookURL, { - method: 'POST', - body: JSON.stringify({ - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2), - "description": "Submission ID: " + UUID + - "\n\nTimestamp: " + - getFormattedTime(startTime) + " to " + getFormattedTime(endTime) + - "\n\nCategory: " + segmentInfo.category, - "color": 10813440, - "author": { - "name": userID, - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - }, - }], - }), - headers: { - 'Content-Type': 'application/json' - } - }) - .then(res => { - if (res.status >= 400) { - Logger.error("Error sending first time submission Discord hook"); - Logger.error(JSON.stringify(res)); - Logger.error("\n"); - } - }) - .catch(err => { - Logger.error("Failed to send first time submission Discord hook."); - Logger.error(JSON.stringify(err)); + }) + .then(res => { + if (res.status >= 400) { + Logger.error("Error sending first time submission Discord hook"); + Logger.error(JSON.stringify(res)); Logger.error("\n"); - }); + } + }) + .catch(err => { + Logger.error("Failed to send first time submission Discord hook."); + Logger.error(JSON.stringify(err)); + Logger.error("\n"); }); } } @@ -166,73 +168,98 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. -async function autoModerateSubmission(submission: { videoID: any; userID: any; segments: any }) { - // Get the video information from the youtube API - if (config.youtubeAPIKey !== null) { - const {err, data} = await new Promise((resolve) => { - YouTubeAPI.listVideos(submission.videoID, (err: any, data: any) => resolve({err, data})); - }); +async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, + submission: { videoID: any; userID: any; segments: any }) { + if (apiVideoInfo) { + const {err, data} = apiVideoInfo; + if (err) return false; - if (err) { - return false; - } else { - // Check to see if video exists - if (data.pageInfo.totalResults === 0) { - return "No video exists with id " + submission.videoID; + // Check to see if video exists + if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID; + + const duration = getYouTubeVideoDuration(apiVideoInfo); + const segments = submission.segments; + let nbString = ""; + for (let i = 0; i < segments.length; i++) { + const startTime = parseFloat(segments[i].segment[0]); + const endTime = parseFloat(segments[i].segment[1]); + + if (duration == 0) { + // Allow submission if the duration is 0 (bug in youtube api) + return false; } else { - const segments = submission.segments; - let nbString = ""; - for (let i = 0; i < segments.length; i++) { + if (segments[i].category === "sponsor") { + //Prepare timestamps to send to NB all at once + nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";"; + } + } + } + + // Get all submissions for this user + const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]); + const allSegmentTimes = []; + if (allSubmittedByUser !== undefined) { + //add segments the user has previously submitted + for (const segmentInfo of allSubmittedByUser) { + allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]); + } + } + + //add segments they are trying to add in this submission + for (let i = 0; i < segments.length; i++) { + let startTime = parseFloat(segments[i].segment[0]); + let endTime = parseFloat(segments[i].segment[1]); + allSegmentTimes.push([startTime, endTime]); + } + + //merge all the times into non-overlapping arrays + const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) { + return a[0] - b[0] || a[1] - b[1]; + })); + + let videoDuration = data.items[0].contentDetails.duration; + videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); + if (videoDuration != 0) { + let allSegmentDuration = 0; + //sum all segment times together + allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); + if (allSegmentDuration > (videoDuration / 100) * 80) { + // Reject submission if all segments combine are over 80% of the video + return "Total length of your submitted segments are over 80% of the video."; + } + } + + // Check NeuralBlock + const neuralBlockURL = config.neuralBlockURL; + if (!neuralBlockURL) return false; + const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + + "&segments=" + nbString.substring(0, nbString.length - 1)); + if (!response.ok) return false; + + const nbPredictions = await response.json(); + let nbDecision = false; + let predictionIdx = 0; //Keep track because only sponsor categories were submitted + for (let i = 0; i < segments.length; i++) { + if (segments[i].category === "sponsor") { + if (nbPredictions.probabilities[predictionIdx] < 0.70) { + nbDecision = true; // At least one bad entry const startTime = parseFloat(segments[i].segment[0]); const endTime = parseFloat(segments[i].segment[1]); - let duration = data.items[0].contentDetails.duration; - duration = isoDurations.toSeconds(isoDurations.parse(duration)); - if (duration == 0) { - // Allow submission if the duration is 0 (bug in youtube api) - return false; - } else if ((endTime - startTime) > (duration / 100) * 80) { - // Reject submission if over 80% of the video - return "One of your submitted segments is over 80% of the video."; - } else { - if (segments[i].category === "sponsor") { - //Prepare timestamps to send to NB all at once - nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";"; - } - } - } - // Check NeuralBlock - const neuralBlockURL = config.neuralBlockURL; - if (!neuralBlockURL) return false; - const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID + - "&segments=" + nbString.substring(0, nbString.length - 1)); - if (!response.ok) return false; - - const nbPredictions = await response.json(); - let nbDecision = false; - let predictionIdx = 0; //Keep track because only sponsor categories were submitted - for (let i = 0; i < segments.length; i++) { - if (segments[i].category === "sponsor") { - if (nbPredictions.probabilities[predictionIdx] < 0.70) { - nbDecision = true; // At least one bad entry - const startTime = parseFloat(segments[i].segment[0]); - const endTime = parseFloat(segments[i].segment[1]); - - const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime); - // Send to Discord - // Note, if this is too spammy. Consider sending all the segments as one Webhook - sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); - } - predictionIdx++; - } - - } - if (nbDecision) { - return "Rejected based on NeuralBlock predictions."; - } else { - return false; + const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime); + // Send to Discord + // Note, if this is too spammy. Consider sending all the segments as one Webhook + sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); } + predictionIdx++; } + + } + + if (nbDecision) { + return "Rejected based on NeuralBlock predictions."; + } else { + return false; } } else { Logger.debug("Skipped YouTube API"); @@ -243,6 +270,21 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s } } +function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration { + const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration; + return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null; +} + +async function getYouTubeVideoInfo(videoID: VideoID): Promise { + if (config.youtubeAPIKey !== null) { + return new Promise((resolve) => { + YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data})); + }); + } else { + return null; + } +} + function proxySubmission(req: Request) { fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, { method: 'POST', @@ -267,14 +309,18 @@ export async function postSkipSegments(req: Request, res: Response) { const videoID = req.query.videoID || req.body.videoID; let userID = req.query.userID || req.body.userID; + let service: Service = req.query.service ?? req.body.service ?? Service.YouTube; + if (!Object.values(Service).some((val) => val == service)) { + service = Service.YouTube; + } + let videoDuration: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration; - - let segments = req.body.segments; + let segments = req.body.segments as IncomingSegment[]; if (segments === undefined) { // Use query instead segments = [{ - segment: [req.query.startTime, req.query.endTime], - category: req.query.category, + segment: [req.query.startTime as string, req.query.endTime as string], + category: req.query.category as Category }]; } @@ -312,7 +358,7 @@ export async function postSkipSegments(req: Request, res: Response) { return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?'); } - const noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => { + let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => { return list.category; }); @@ -321,6 +367,32 @@ export async function postSkipSegments(req: Request, res: Response) { const decreaseVotes = 0; + let apiVideoInfo: APIVideoInfo = null; + if (service == Service.YouTube) { + apiVideoInfo = await getYouTubeVideoInfo(videoID); + } + const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo); + if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) { + // If api duration is far off, take that one instead (it is only precise to seconds, not millis) + videoDuration = apiVideoDuration || 0 as VideoDuration; + } + + const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 + AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as + {videoDuration: VideoDuration, UUID: SegmentUUID}[]; + // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden + const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); + if (videoDurationChanged) { + // Hide all previous submissions + for (const submission of previousSubmissions) { + await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]); + } + + // Reset no segments + noSegmentList = []; + deleteNoSegments(videoID, null); + } + // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { @@ -343,7 +415,7 @@ export async function postSkipSegments(req: Request, res: Response) { + segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n " + (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "") - + "If you believe this is incorrect, please contact someone on Discord.", + + "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app", ); return; } @@ -367,7 +439,7 @@ export async function postSkipSegments(req: Request, res: Response) { //check if this info has already been submitted before const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ? - and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]); + and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]); if (duplicateCheck2Row.count > 0) { res.sendStatus(409); return; @@ -375,8 +447,8 @@ export async function postSkipSegments(req: Request, res: Response) { } // Auto moderator check - if (!isVIP) { - const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category}); + if (!isVIP && service == Service.YouTube) { + const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category}); if (autoModerateResult == "Rejected based on NeuralBlock predictions.") { // If NB automod rejects, the submission will start with -2 votes. // Note, if one submission is bad all submissions will be affected. @@ -437,63 +509,19 @@ export async function postSkipSegments(req: Request, res: Response) { let startingVotes = 0 + decreaseVotes; - if (config.youtubeAPIKey !== null) { - let {err, data} = await new Promise((resolve) => { - YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data})); - }); - - if (err) { - Logger.error("Error while submitting when connecting to YouTube API: " + err); - } else { - //get all segments for this video and user - const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [userID, videoID]); - const allSegmentTimes = []; - if (allSubmittedByUser !== undefined) { - //add segments the user has previously submitted - for (const segmentInfo of allSubmittedByUser) { - allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]); - } - } - - //add segments they are trying to add in this submission - for (let i = 0; i < segments.length; i++) { - let startTime = parseFloat(segments[i].segment[0]); - let endTime = parseFloat(segments[i].segment[1]); - allSegmentTimes.push([startTime, endTime]); - } - - //merge all the times into non-overlapping arrays - const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) { - return a[0] - b[0] || a[1] - b[1]; - })); - - let videoDuration = data.items[0].contentDetails.duration; - videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); - if (videoDuration != 0) { - let allSegmentDuration = 0; - //sum all segment times together - allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); - if (allSegmentDuration > (videoDuration / 100) * 80) { - // Reject submission if all segments combine are over 80% of the video - res.status(400).send("Total length of your submitted segments are over 80% of the video."); - return; - } - } - } - } - for (const segmentInfo of segments) { //this can just be a hash of the data //it's better than generating an actual UUID like what was used before //also better for duplication checking - const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]); + const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1])); + const hashedVideoID = getHash(videoID, 1); const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" - ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID") - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1), + ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID") + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID, ], ); @@ -502,9 +530,10 @@ export async function postSkipSegments(req: Request, res: Response) { // Clear redis cache for this video redis.delAsync(skipSegmentsKey(videoID)); + redis.delAsync(skipSegmentsHashKey(hashedVideoID, service)); } catch (err) { //a DB change probably occurred - res.sendStatus(502); + res.sendStatus(500); Logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " + segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err); @@ -529,7 +558,7 @@ export async function postSkipSegments(req: Request, res: Response) { res.json(newSegments); for (let i = 0; i < segments.length; i++) { - sendWebhooks(userID, videoID, UUIDs[i], segments[i]); + sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service); } } diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index a1e81e7..240baf9 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash'; import {config} from '../config'; import { UserID } from '../types/user.model'; import redis from '../utils/redis'; -import { skipSegmentsKey } from '../middleware/redisKeys'; -import { VideoID } from '../types/segments.model'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; +import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; const voteTypes = { normal: 0, @@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) { } } -async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string - , hashedIP: string, finalResponse: FinalResponse, res: Response) { +async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category + , hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) { // Check if they've already made a vote const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); @@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS return; } - const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]); - if (!currentCategory) { + const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`, + [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service}; + if (!videoInfo) { // Submission doesn't exist res.status(400).send("Submission doesn't exist."); return; @@ -198,7 +199,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS } // See if the submissions category is ready to change - const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]); + const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]); const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); @@ -210,9 +211,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS // Add submission as vote if (!currentCategoryInfo && submissionInfo) { - await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]); + await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]); - await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]); + await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]); } const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; @@ -225,6 +226,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS } } + clearRedisCache(videoInfo); + res.sendStatus(finalResponse.finalStatus); } @@ -233,10 +236,10 @@ export function getUserID(req: Request): UserID { } export async function voteOnSponsorTime(req: Request, res: Response) { - const UUID = req.query.UUID as string; + const UUID = req.query.UUID as SegmentUUID; const paramUserID = getUserID(req); let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; - const category = req.query.category as string; + const category = req.query.category as Category; if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) { //invalid request @@ -258,7 +261,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { const ip = getIP(req); //hash the ip 5000 times so no one can get it from the database - const hashedIP = getHash(ip + config.globalSalt); + const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress); //check if this user is on the vip list const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0; @@ -353,13 +356,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) { } //check if the increment amount should be multiplied (downvotes have more power if there have been many views) - const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as - {videoID: VideoID, votes: number, views: number}; + const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as + {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number}; if (voteTypeEnum === voteTypes.normal) { if ((isVIP || isOwnSubmission) && incrementAmount < 0) { //this user is a vip and a downvote - incrementAmount = -(row.votes + 2 - oldIncrementAmount); + incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount); type = incrementAmount; } } else if (voteTypeEnum == voteTypes.incorrect) { @@ -403,8 +406,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]); } - // Clear redis cache for this video - redis.delAsync(skipSegmentsKey(row?.videoID)); + clearRedisCache(videoInfo); //for each positive vote, see if a hidden submission can be shown again if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { @@ -441,7 +443,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { voteTypeEnum, isVIP, isOwnSubmission, - row, + row: videoInfo, category, incrementAmount, oldIncrementAmount, @@ -453,4 +455,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) { res.status(500).json({error: 'Internal error creating segment vote'}); } -} \ No newline at end of file +} + +function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) { + if (videoInfo) { + redis.delAsync(skipSegmentsKey(videoInfo.videoID)); + redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); + } +} diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index e7b6eaf..a95bac3 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -3,15 +3,33 @@ import { SBRecord } from "./lib.model"; export type SegmentUUID = string & { __segmentUUIDBrand: unknown }; export type VideoID = string & { __videoIDBrand: unknown }; +export type VideoDuration = number & { __videoDurationBrand: unknown }; export type Category = string & { __categoryBrand: unknown }; export type VideoIDHash = VideoID & HashedValue; export type IPAddress = string & { __ipAddressBrand: unknown }; export type HashedIP = IPAddress & HashedValue; +// Uncomment as needed +export enum Service { + YouTube = 'YouTube', + PeerTube = 'PeerTube', + // Twitch = 'Twitch', + // Nebula = 'Nebula', + // RSS = 'RSS', + // Corridor = 'Corridor', + // Lbry = 'Lbry' +} + +export interface IncomingSegment { + category: Category; + segment: string[]; +} + export interface Segment { category: Category; segment: number[]; UUID: SegmentUUID; + videoDuration: VideoDuration; } export enum Visibility { @@ -28,6 +46,7 @@ export interface DBSegment { locked: boolean; shadowHidden: Visibility; videoID: VideoID; + videoDuration: VideoDuration; hashedVideoID: VideoIDHash; } diff --git a/test.json b/test.json index add6ebf..58eb72a 100644 --- a/test.json +++ b/test.json @@ -49,7 +49,7 @@ ] } ], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], + "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"], "maxNumberOfActiveWarnings": 3, "hoursAfterWarningExpires": 24, "rateLimit": { diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index 85dcfbb..f40203c 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -5,17 +5,19 @@ import {getHash} from '../../src/utils/getHash'; describe('getSkipSegments', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')"); - + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 0, '" + getHash('testtesttest,test', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, 0, '" + getHash('locked', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, 0, '" + getHash('locked', 1) + "')"); + await db.prepare("run", startOfQuery + "('onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', 0, 50, 'sponsor', 'YouTube', 190, 1, 0, '" + getHash('onlyHiddenSegments', 1) + "')"); + return; }); @@ -27,7 +29,24 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 - && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) { + return; + } else { + return ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to get a time by category for a different service 1', () => { + fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube") + .then(async res => { + if (res.status !== 200) return ("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -61,7 +80,7 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 - && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -78,7 +97,23 @@ describe('getSkipSegments', () => { else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 - && data[0].category === "intro" && data[0].UUID === "1-uuid-2") { + && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) { + return; + } else { + return ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => ("Couldn't call endpoint")); + }); + + it('Should be empty if all submissions are hidden', () => { + fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments") + .then(async res => { + if (res.status !== 200) return ("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 0) { return; } else { return ("Received incorrect body: " + (await res.text())); @@ -99,9 +134,9 @@ describe('getSkipSegments', () => { let success = true; for (const segment of data) { if ((segment.segment[0] !== 20 || segment.segment[1] !== 33 - || segment.category !== "intro" || segment.UUID !== "1-uuid-7") && + || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) && (segment.segment[0] !== 1 || segment.segment[1] !== 11 - || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) { + || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) { success = false; break; } diff --git a/test/cases/getSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts similarity index 74% rename from test/cases/getSegmentsByHash.ts rename to test/cases/getSkipSegmentsByHash.ts index d957e2d..23d9d49 100644 --- a/test/cases/getSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -12,11 +12,13 @@ sinonStub.callsFake(YouTubeApiMock.listVideos); describe('getSegmentsByHash', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "hidden", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b + await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3 }); it('Should be able to get a 200', (done: Done) => { @@ -54,6 +56,19 @@ describe('getSegmentsByHash', () => { .catch(err => done("Couldn't call endpoint")); }); + it('Should be able to get an empty array if only hidden videos', (done: Done) => { + fetch(getbaseURL() + '/api/skipSegments/f3a1?categories=["sponsor"]') + .then(async res => { + if (res.status !== 404) done("non 404 status code, was " + res.status); + else { + const body = await res.text(); + if (JSON.parse(body).length === 0 && body === '[]') done(); // pass + else done("non empty array returned"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + it('Should return 400 prefix too short', (done: Done) => { fetch(getbaseURL() + '/api/skipSegments/11?categories=["shilling"]') .then(res => { @@ -128,7 +143,24 @@ describe('getSegmentsByHash', () => { if (body.length !== 2) done("expected 2 videos, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length); - else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else if (body[0].segments[0].category !== 'sponsor' + || body[0].segments[0].UUID !== 'getSegmentsByHash-0-0' + || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else done(); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => { + fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube') + .then(async res => { + if (res.status !== 200) done("non 200 status code, was " + res.status); + else { + const body = await res.json(); + if (body.length !== 1) done("expected 2 videos, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); + else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor"); else done(); } }) diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index e31326a..512c2ee 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -6,6 +6,7 @@ import {db} from '../../src/databases/databases'; import {ImportMock} from 'ts-mock-imports'; import * as YouTubeAPIModule from '../../src/utils/youtubeApi'; import {YouTubeApiMock} from '../youtubeMock'; +import e from 'express'; const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, 'YouTubeAPI'); const sinonStub = mockManager.mock('listVideos'); @@ -97,6 +98,177 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); + it('Should be able to submit a single time with a duration from the YouTube API (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXZX", + videoDuration: 100, + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXZH", + videoDuration: 5010.20, + segments: [{ + segment: [1, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]); + if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit a single time with a duration in the body (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "noDuration", + videoDuration: 100, + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["noDuration"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 100) { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + + it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => { + await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") + VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')"); + + try { + const res = await fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "noDuration", + videoDuration: 100, + segments: [{ + segment: [1, 10], + category: "sponsor", + }], + }), + }); + + if (res.status === 200) { + const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]); + const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" + FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]); + const videoRow = videoRows[0]; + const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" + FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]); + if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10 + && videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) { + return; + } else { + return "Submitted times were not saved. Actual submission: " + JSON.stringify(videoRow); + } + } else { + return "Status code was " + res.status; + } + } catch (e) { + return e; + } + }); + + it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXcG", + service: "PeerTube", + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('VIP submission should start locked', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { @@ -244,7 +416,7 @@ describe('postSkipSegments', () => { }), }) .then(async res => { - if (res.status === 400) { + if (res.status === 403) { const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["n9rIGdXnSJc"]); let success = true; if (rows.length === 4) { @@ -292,7 +464,7 @@ describe('postSkipSegments', () => { }), }) .then(async res => { - if (res.status === 400) { + if (res.status === 403) { const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["80percent_video"]); let success = rows.length == 2; for (const row of rows) {