diff --git a/.gitignore b/.gitignore
index 3cff924..dd78e49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
test/databases/sponsorTimes.db-shm
test/databases/sponsorTimes.db-wal
test/databases/private.db
+docker/database-export
# Config files
config.json
diff --git a/README.MD b/README.MD
index 8f61c57..e309e54 100644
--- a/README.MD
+++ b/README.MD
@@ -8,7 +8,7 @@ This is the server backend for it
This uses a Postgres or Sqlite database to hold all the timing data.
-To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
+To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.
diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql
new file mode 100644
index 0000000..2b70fa7
--- /dev/null
+++ b/databases/_private_indexes.sql
@@ -0,0 +1,32 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
+ ON public."sponsorTimes" USING btree
+ ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID"
+ ON public."sponsorTimes" USING btree
+ ("videoID" ASC NULLS LAST)
+;
+
+-- votes
+
+CREATE INDEX IF NOT EXISTS "votes_userID"
+ ON public.votes USING btree
+ ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- shadowBannedUsers
+
+CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
+ ON public."shadowBannedUsers" USING btree
+ ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
+ ON public."categoryVotes" USING btree
+ ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
\ No newline at end of file
diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql
new file mode 100644
index 0000000..c90ef51
--- /dev/null
+++ b/databases/_sponsorTimes_indexes.sql
@@ -0,0 +1,66 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted"
+ ON public."sponsorTimes" USING btree
+ ("timeSubmitted" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_userID"
+ ON public."sponsorTimes" USING btree
+ ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
+ ON public."sponsorTimes" USING btree
+ ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID_gin"
+ ON public."sponsorTimes" USING gin
+ ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
+ TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"
+ ON public."sponsorTimes" USING btree
+ ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- userNames
+
+CREATE INDEX IF NOT EXISTS "userNames_userID"
+ ON public."userNames" USING btree
+ ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- vipUsers
+
+CREATE INDEX IF NOT EXISTS "vipUsers_index"
+ ON public."vipUsers" USING btree
+ ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- warnings
+
+CREATE INDEX IF NOT EXISTS "warnings_index"
+ ON public.warnings USING btree
+ ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
+ TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "warnings_issueTime"
+ ON public.warnings USING btree
+ ("issueTime" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- noSegments
+
+CREATE INDEX IF NOT EXISTS "noSegments_videoID"
+ ON public."noSegments" USING btree
+ ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
+ ON public."categoryVotes" USING btree
+ ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+ TABLESPACE pg_default;
\ No newline at end of file
diff --git a/databases/_upgrade_sponsorTimes_10.sql b/databases/_upgrade_sponsorTimes_10.sql
new file mode 100644
index 0000000..174ceaf
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_10.sql
@@ -0,0 +1,30 @@
+BEGIN TRANSACTION;
+
+/* Add Hidden field */
+CREATE TABLE "sqlb_temp_table_10" (
+ "videoID" TEXT NOT NULL,
+ "startTime" REAL NOT NULL,
+ "endTime" REAL NOT NULL,
+ "votes" INTEGER NOT NULL,
+ "locked" INTEGER NOT NULL default '0',
+ "incorrectVotes" INTEGER NOT NULL default '1',
+ "UUID" TEXT NOT NULL UNIQUE,
+ "userID" TEXT NOT NULL,
+ "timeSubmitted" INTEGER NOT NULL,
+ "views" INTEGER NOT NULL,
+ "category" TEXT NOT NULL DEFAULT 'sponsor',
+ "service" TEXT NOT NULL DEFAULT 'YouTube',
+ "videoDuration" REAL NOT NULL DEFAULT '0',
+ "hidden" INTEGER NOT NULL DEFAULT '0',
+ "shadowHidden" INTEGER NOT NULL,
+ "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 10 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/databases/_upgrade_sponsorTimes_7.sql b/databases/_upgrade_sponsorTimes_7.sql
new file mode 100644
index 0000000..8f6060b
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_7.sql
@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_7" (
+ "videoID" TEXT NOT NULL,
+ "startTime" REAL NOT NULL,
+ "endTime" REAL NOT NULL,
+ "votes" INTEGER NOT NULL,
+ "locked" INTEGER NOT NULL default '0',
+ "incorrectVotes" INTEGER NOT NULL default '1',
+ "UUID" TEXT NOT NULL UNIQUE,
+ "userID" TEXT NOT NULL,
+ "timeSubmitted" INTEGER NOT NULL,
+ "views" INTEGER NOT NULL,
+ "category" TEXT NOT NULL DEFAULT 'sponsor',
+ "service" TEXT NOT NULL DEFAULT 'YouTube',
+ "shadowHidden" INTEGER NOT NULL,
+ "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/databases/_upgrade_sponsorTimes_8.sql b/databases/_upgrade_sponsorTimes_8.sql
new file mode 100644
index 0000000..ccc2ec9
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_8.sql
@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_8" (
+ "videoID" TEXT NOT NULL,
+ "startTime" REAL NOT NULL,
+ "endTime" REAL NOT NULL,
+ "votes" INTEGER NOT NULL,
+ "locked" INTEGER NOT NULL default '0',
+ "incorrectVotes" INTEGER NOT NULL default '1',
+ "UUID" TEXT NOT NULL UNIQUE,
+ "userID" TEXT NOT NULL,
+ "timeSubmitted" INTEGER NOT NULL,
+ "views" INTEGER NOT NULL,
+ "category" TEXT NOT NULL DEFAULT 'sponsor',
+ "service" TEXT NOT NULL DEFAULT 'YouTube',
+ "videoDuration" INTEGER NOT NULL DEFAULT '0',
+ "shadowHidden" INTEGER NOT NULL,
+ "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/databases/_upgrade_sponsorTimes_9.sql b/databases/_upgrade_sponsorTimes_9.sql
new file mode 100644
index 0000000..5015a4a
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_9.sql
@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_9" (
+ "videoID" TEXT NOT NULL,
+ "startTime" REAL NOT NULL,
+ "endTime" REAL NOT NULL,
+ "votes" INTEGER NOT NULL,
+ "locked" INTEGER NOT NULL default '0',
+ "incorrectVotes" INTEGER NOT NULL default '1',
+ "UUID" TEXT NOT NULL UNIQUE,
+ "userID" TEXT NOT NULL,
+ "timeSubmitted" INTEGER NOT NULL,
+ "views" INTEGER NOT NULL,
+ "category" TEXT NOT NULL DEFAULT 'sponsor',
+ "service" TEXT NOT NULL DEFAULT 'YouTube',
+ "videoDuration" REAL NOT NULL DEFAULT '0',
+ "shadowHidden" INTEGER NOT NULL,
+ "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 9 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index a54a3c0..4ee69ee 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -7,6 +7,7 @@ services:
- database.env
volumes:
- database-data:/var/lib/postgresql/data
+ - ./database-export/:/opt/exports
ports:
- 127.0.0.1:5432:5432
redis:
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index daee81f..76a77d3 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -2,7 +2,7 @@ worker_processes 8;
worker_rlimit_nofile 8192;
events {
- worker_connections 32768; ## Default: 1024
+ worker_connections 132768; ## Default: 1024
}
http {
@@ -12,19 +12,35 @@ http {
upstream backend_GET {
least_conn;
- server localhost:4442;
- server localhost:4443;
- server localhost:4444;
- server localhost:4445;
- server localhost:4446;
+ server localhost:4441;
+ server localhost:4442;
+ #server localhost:4443;
+ #server localhost:4444;
+ #server localhost:4445;
+ #server localhost:4446;
#server localhost:4447;
#server localhost:4448;
+
+ server 10.0.0.3:4441;
+ server 10.0.0.3:4442;
+
+ #server 134.209.69.251:80 backup;
+
+ server 116.203.32.253:80 backup;
+ #server 116.203.32.253:80;
}
upstream backend_POST {
- server localhost:4441;
+ #server localhost:4441;
+ #server localhost:4442;
+ server 10.0.0.3:4441;
+ #server 10.0.0.3:4442;
+ }
+ upstream backend_db {
+ #server localhost:4441;
+ server 10.0.0.3:4441;
}
- proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m;
+ proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
proxy_cache_key "$scheme$request_method$host$request_uri";
add_header X-Cache $upstream_cache_status;
@@ -43,14 +59,16 @@ http {
# internal;
#}
+ #proxy_send_timeout 120s;
+
location @myerrordirective_500 {
- return 502 "Internal Server Error";
+ return 400 "Internal Server Error";
}
location @myerrordirective_502 {
- return 502 "Bad Gateway";
+ return 400 "Bad Gateway";
}
location @myerrordirective_504 {
- return 502 "Gateway Timeout";
+ return 400 "Gateway Timeout";
}
@@ -62,17 +80,16 @@ http {
return 301 https://sb.ltn.fi;
}
- location /invidious/ {
- proxy_pass https://invidious.fdn.fr/;
- }
-
location /test/ {
proxy_pass http://localhost:4440/;
#proxy_pass https://sbtest.etcinit.com/;
}
location /api/skipSegments {
- proxy_pass http://backend_$request_method;
+ #return 200 "[]";
+ proxy_pass http://backend_$request_method;
+ #proxy_cache CACHEZONE;
+ #proxy_cache_valid 2m;
}
location /api/getTopUsers {
@@ -83,24 +100,43 @@ http {
location /api/getTotalStats {
proxy_pass http://backend_GET;
- }
+ #return 200 "";
+ }
location /api/getVideoSponsorTimes {
proxy_pass http://backend_GET;
}
-
- location = /database.db {
- alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
+
+ location /database {
+ proxy_pass http://backend_db;
}
-
+
+ location = /database.db {
+ #return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
+ #alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
+ alias /home/sbadmin/test-db/database.db;
+ }
+
+ location = /database/sponsorTimes.csv {
+ alias /home/sbadmin/sponsorTimes.csv;
+ }
+
+ #location /api/voteOnSponsorTime {
+ # return 200 "Success";
+ #}
+
+ #location /api/viewedVideoSponsorTime {
+ # return 200 "Success";
+ #}
+
location /api {
proxy_pass http://backend_POST;
}
location / {
- root /home/sbadmin/caddy/SponsorBlockSite/public-prod;
-
+ root /home/sbadmin/SponsorBlockSite/public-prod;
+
### CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
@@ -132,12 +168,15 @@ http {
}
- listen 443 ssl; # managed by Certbot
+ listen 443 default_server ssl; # managed by Certbot
+ #listen 80;
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+
+
}
diff --git a/package-lock.json b/package-lock.json
index dd69f2e..f17b109 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2989,9 +2989,9 @@
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"y18n": {
- "version": "4.0.0",
- "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
- "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz",
+ "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==",
"dev": true
},
"yallist": {
diff --git a/src/app.ts b/src/app.ts
index 098e5f3..2af8835 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -5,7 +5,7 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
import {postSegmentShift} from './routes/postSegmentShift';
import {postWarning} from './routes/postWarning';
import {getIsUserVIP} from './routes/getIsUserVIP';
-import {deleteNoSegments} from './routes/deleteNoSegments';
+import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments';
import {postNoSegments} from './routes/postNoSegments';
import {getUserInfo} from './routes/getUserInfo';
import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
@@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter';
import {loggerMiddleware} from './middleware/logger';
import {corsMiddleware} from './middleware/cors';
import {rateLimitMiddleware} from './middleware/requestRateLimit';
+import dumpDatabase from './routes/dumpDatabase';
export function createServer(callback: () => void) {
@@ -116,7 +117,7 @@ function setupRoutes(app: Express) {
//submit video containing no segments
app.post('/api/noSegments', postNoSegments);
- app.delete('/api/noSegments', deleteNoSegments);
+ app.delete('/api/noSegments', deleteNoSegmentsEndpoint);
//get if user is a vip
app.get('/api/isUserVIP', getIsUserVIP);
@@ -127,7 +128,12 @@ function setupRoutes(app: Express) {
//get if user is a vip
app.post('/api/segmentShift', postSegmentShift);
- app.get('/database.db', function (req: Request, res: Response) {
- res.sendFile("./databases/sponsorTimes.db", {root: "./"});
- });
+ if (config.postgres) {
+ app.get('/database', (req, res) => dumpDatabase(req, res, true));
+ app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
+ } else {
+ app.get('/database.db', function (req: Request, res: Response) {
+ res.sendFile("./databases/sponsorTimes.db", {root: "./"});
+ });
+ }
}
diff --git a/src/config.ts b/src/config.ts
index 93c6a43..7b15987 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -16,7 +16,7 @@ addDefaults(config, {
privateDBSchema: "./databases/_private.db.sql",
readOnly: false,
webhooks: [],
- categoryList: ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+ categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"],
maxNumberOfActiveWarnings: 3,
hoursAfterWarningExpires: 24,
adminUserID: "",
diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts
index bae2da6..779ebf2 100644
--- a/src/databases/Postgres.ts
+++ b/src/databases/Postgres.ts
@@ -23,6 +23,13 @@ export class Postgres implements IDatabase {
// Upgrade database if required
await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+
+ try {
+ await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+ } catch (e) {
+ Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
+ Logger.warn(e);
+ }
}
}
@@ -118,9 +125,18 @@ export class Postgres implements IDatabase {
Logger.debug('db update: no file ' + path);
}
+ private async applyIndexes(fileNamePrefix: string, schemaFolder: string) {
+ const path = schemaFolder + "/_" + fileNamePrefix + "_indexes.sql";
+ if (fs.existsSync(path)) {
+ await this.pool.query(fs.readFileSync(path).toString());
+ } else {
+ Logger.debug('failed to apply indexes to ' + fileNamePrefix);
+ }
+ }
+
private processUpgradeQuery(query: string): string {
let result = query;
- result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
+ result = result.replace(/sha256\((.*?)\)/gm, "encode(digest($1, 'sha256'), 'hex')");
result = result.replace(/integer/gmi, "NUMERIC");
return result;
diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts
index 1335e15..56b1aab 100644
--- a/src/middleware/redisKeys.ts
+++ b/src/middleware/redisKeys.ts
@@ -1,5 +1,13 @@
-import { Category, VideoID } from "../types/segments.model";
+import { Service, VideoID, VideoIDHash } from "../types/segments.model";
+import { Logger } from "../utils/logger";
export function skipSegmentsKey(videoID: VideoID): string {
return "segments-" + videoID;
}
+
+export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
+ hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
+ if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
+
+ return "segments." + service + "." + hashedVideoIDPrefix;
+}
\ No newline at end of file
diff --git a/src/routes/deleteNoSegments.ts b/src/routes/deleteNoSegments.ts
index cfa6c78..5584c5a 100644
--- a/src/routes/deleteNoSegments.ts
+++ b/src/routes/deleteNoSegments.ts
@@ -2,12 +2,14 @@ import {Request, Response} from 'express';
import {isUserVIP} from '../utils/isUserVIP';
import {getHash} from '../utils/getHash';
import {db} from '../databases/databases';
+import { Category, VideoID } from '../types/segments.model';
+import { UserID } from '../types/user.model';
-export async function deleteNoSegments(req: Request, res: Response) {
+export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
// Collect user input data
- const videoID = req.body.videoID;
- let userID = req.body.userID;
- const categories = req.body.categories;
+ const videoID = req.body.videoID as VideoID;
+ const userID = req.body.userID as UserID;
+ const categories = req.body.categories as Category[];
// Check input data is valid
if (!videoID
@@ -23,8 +25,8 @@ export async function deleteNoSegments(req: Request, res: Response) {
}
// Check if user is VIP
- userID = getHash(userID);
- const userIsVIP = await isUserVIP(userID);
+ const hashedUserID = getHash(userID);
+ const userIsVIP = await isUserVIP(hashedUserID);
if (!userIsVIP) {
res.status(403).json({
@@ -33,13 +35,22 @@ export async function deleteNoSegments(req: Request, res: Response) {
return;
}
+ deleteNoSegments(videoID, categories);
+
+ res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
+}
+
+/**
+ *
+ * @param videoID
+ * @param categories If null, will remove all
+ */
+export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise {
const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
- return (categories.indexOf(entry.category) !== -1);
+ return categories === null || categories.indexOf(entry.category) !== -1;
});
for (const entry of entries) {
await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
}
-
- res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
}
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
new file mode 100644
index 0000000..d687f07
--- /dev/null
+++ b/src/routes/dumpDatabase.ts
@@ -0,0 +1,78 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import { config } from '../config';
+
+const ONE_MINUTE = 1000 * 60;
+
+const styleHeader = ``
+
+const licenseHeader = `The API and database follow CC BY-NC-SA 4.0 unless you have explicit permission.
+Attribution Template
+If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.
`;
+
+const tables = [{
+ name: "sponsorTimes",
+ order: "timeSubmitted"
+},
+{
+ name: "userNames"
+},
+{
+ name: "categoryVotes"
+},
+{
+ name: "noSegments",
+},
+{
+ name: "warnings",
+ order: "issueTime"
+},
+{
+ name: "vipUsers"
+}];
+
+const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
+
+const linksHTML: string = tables.map((table) => `${table.name}.csv
`)
+ .reduce((acc, url) => acc + url, "");
+
+let lastUpdate = 0;
+
+export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
+ if (!config.postgres) {
+ res.status(404).send("Not supported on this instance");
+ return;
+ }
+
+ const now = Date.now();
+ const updateQueued = now - lastUpdate > ONE_MINUTE;
+
+ res.status(200)
+
+ if (showPage) {
+ res.send(`${styleHeader}
+ SponsorBlock database dumps
${licenseHeader}
+ How this works
+ Send a request to https://sponsor.ajay.app/database.json, or visit this page to trigger the database dump to run.
+ Then, you can download the csv files below, or use the links returned from the JSON request.
+ Links
+ ${linksHTML}
+ ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+ } else {
+ res.send({
+ lastUpdated: lastUpdate,
+ updateQueued,
+ links
+ })
+ }
+
+ if (updateQueued) {
+ lastUpdate = Date.now();
+
+ for (const table of tables) {
+ db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
+ TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts
index 65ab4f1..d36a91a 100644
--- a/src/routes/getSkipSegments.ts
+++ b/src/routes/getSkipSegments.ts
@@ -2,9 +2,9 @@ import { Request, Response } from 'express';
import { RedisClient } from 'redis';
import { config } from '../config';
import { db, privateDB } from '../databases/databases';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { SBRecord } from '../types/lib.model';
-import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
+import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { getHash } from '../utils/getHash';
import { getIP } from '../utils/getIP';
import { Logger } from '../utils/logger';
@@ -43,11 +43,12 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
return chooseSegments(filteredSegments).map((chosenSegment) => ({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
- UUID: chosenSegment.UUID
+ UUID: chosenSegment.UUID,
+ videoDuration: chosenSegment.videoDuration
}));
}
-async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise {
+async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: Segment[] = [];
@@ -58,9 +59,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
const segmentsByCategory: SBRecord = (await db
.prepare(
'all',
- `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes"
- WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
- [videoID]
+ `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes"
+ WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
+ [videoID, service]
)).reduce((acc: SBRecord, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
acc[segment.category].push(segment);
@@ -81,7 +82,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
}
}
-async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise> {
+async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: SBRecord = {};
@@ -91,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null;
- const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
- .prepare(
- 'all',
- `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
- WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
- [hashedVideoIDPrefix + '%']
- )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
+ const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
+ .filter((segment: DBSegment) => categories.includes(segment?.category))
+ .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
hash: segment.hashedVideoID,
segmentPerCategory: {},
@@ -130,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}
}
+async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise {
+ const fetchFromDB = () => db
+ .prepare(
+ 'all',
+ `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
+ WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
+ [hashedVideoIDPrefix + '%', service]
+ );
+
+ if (hashedVideoIDPrefix.length === 4) {
+ const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
+ const {err, reply} = await redis.getAsync(key);
+
+ if (!err && reply) {
+ try {
+ Logger.debug("Got data from redis: " + reply);
+ return JSON.parse(reply);
+ } catch (e) {
+ // If all else, continue on to fetching from the database
+ }
+ }
+
+ const data = await fetchFromDB();
+
+ redis.setAsync(key, JSON.stringify(data));
+ return data;
+ }
+
+ return await fetchFromDB();
+}
+
//gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique
@@ -239,6 +267,11 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) {
+ service = Service.YouTube;
+ }
+
// Only 404s are cached at the moment
const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
@@ -251,7 +284,7 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) {
+ service = Service.YouTube;
+ }
// filter out none string elements, only flat array with strings is valid
categories = categories.filter((item: any) => typeof item === "string");
// Get all video id's that match hash prefix
- const segments = await getSegmentsByHash(req, hashPrefix, categories);
+ const segments = await getSegmentsByHash(req, hashPrefix, categories, service);
if (!segments) return res.status(404).json([]);
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 6a9c781..1f380da 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -11,9 +11,15 @@ import {getFormattedTime} from '../utils/getFormattedTime';
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
import {dispatchEvent} from '../utils/webhookUtils';
import {Request, Response} from 'express';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis';
+import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
+import { deleteNoSegments } from './deleteNoSegments';
+interface APIVideoInfo {
+ err: string | boolean,
+ data: any
+}
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -45,62 +51,58 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
});
}
-async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) {
- if (config.youtubeAPIKey !== null) {
+async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
+ if (apiVideoInfo && service == Service.YouTube) {
const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
- YouTubeAPI.listVideos(videoID, (err: any, data: any) => {
- if (err || data.items.length === 0) {
- err && Logger.error(err);
- return;
+ const {data, err} = apiVideoInfo;
+ if (err) return;
+
+ const startTime = parseFloat(segmentInfo.segment[0]);
+ const endTime = parseFloat(segmentInfo.segment[1]);
+ sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
+ submissionStart: startTime,
+ submissionEnd: endTime,
+ }, segmentInfo);
+
+ // If it is a first time submission
+ // Then send a notification to discord
+ if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
+
+ fetch(config.discordFirstTimeSubmissionsWebhookURL, {
+ method: 'POST',
+ body: JSON.stringify({
+ "embeds": [{
+ "title": data.items[0].snippet.title,
+ "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
+ "description": "Submission ID: " + UUID +
+ "\n\nTimestamp: " +
+ getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
+ "\n\nCategory: " + segmentInfo.category,
+ "color": 10813440,
+ "author": {
+ "name": userID,
+ },
+ "thumbnail": {
+ "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+ },
+ }],
+ }),
+ headers: {
+ 'Content-Type': 'application/json'
}
-
- const startTime = parseFloat(segmentInfo.segment[0]);
- const endTime = parseFloat(segmentInfo.segment[1]);
- sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
- submissionStart: startTime,
- submissionEnd: endTime,
- }, segmentInfo);
-
- // If it is a first time submission
- // Then send a notification to discord
- if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
-
- fetch(config.discordFirstTimeSubmissionsWebhookURL, {
- method: 'POST',
- body: JSON.stringify({
- "embeds": [{
- "title": data.items[0].snippet.title,
- "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
- "description": "Submission ID: " + UUID +
- "\n\nTimestamp: " +
- getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
- "\n\nCategory: " + segmentInfo.category,
- "color": 10813440,
- "author": {
- "name": userID,
- },
- "thumbnail": {
- "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
- },
- }],
- }),
- headers: {
- 'Content-Type': 'application/json'
- }
- })
- .then(res => {
- if (res.status >= 400) {
- Logger.error("Error sending first time submission Discord hook");
- Logger.error(JSON.stringify(res));
- Logger.error("\n");
- }
- })
- .catch(err => {
- Logger.error("Failed to send first time submission Discord hook.");
- Logger.error(JSON.stringify(err));
+ })
+ .then(res => {
+ if (res.status >= 400) {
+ Logger.error("Error sending first time submission Discord hook");
+ Logger.error(JSON.stringify(res));
Logger.error("\n");
- });
+ }
+ })
+ .catch(err => {
+ Logger.error("Failed to send first time submission Discord hook.");
+ Logger.error(JSON.stringify(err));
+ Logger.error("\n");
});
}
}
@@ -166,73 +168,98 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
-async function autoModerateSubmission(submission: { videoID: any; userID: any; segments: any }) {
- // Get the video information from the youtube API
- if (config.youtubeAPIKey !== null) {
- const {err, data} = await new Promise((resolve) => {
- YouTubeAPI.listVideos(submission.videoID, (err: any, data: any) => resolve({err, data}));
- });
+async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
+ submission: { videoID: any; userID: any; segments: any }) {
+ if (apiVideoInfo) {
+ const {err, data} = apiVideoInfo;
+ if (err) return false;
- if (err) {
- return false;
- } else {
- // Check to see if video exists
- if (data.pageInfo.totalResults === 0) {
- return "No video exists with id " + submission.videoID;
+ // Check to see if video exists
+ if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
+
+ const duration = getYouTubeVideoDuration(apiVideoInfo);
+ const segments = submission.segments;
+ let nbString = "";
+ for (let i = 0; i < segments.length; i++) {
+ const startTime = parseFloat(segments[i].segment[0]);
+ const endTime = parseFloat(segments[i].segment[1]);
+
+ if (duration == 0) {
+ // Allow submission if the duration is 0 (bug in youtube api)
+ return false;
} else {
- const segments = submission.segments;
- let nbString = "";
- for (let i = 0; i < segments.length; i++) {
+ if (segments[i].category === "sponsor") {
+ //Prepare timestamps to send to NB all at once
+ nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
+ }
+ }
+ }
+
+ // Get all submissions for this user
+ const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]);
+ const allSegmentTimes = [];
+ if (allSubmittedByUser !== undefined) {
+ //add segments the user has previously submitted
+ for (const segmentInfo of allSubmittedByUser) {
+ allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
+ }
+ }
+
+ //add segments they are trying to add in this submission
+ for (let i = 0; i < segments.length; i++) {
+ let startTime = parseFloat(segments[i].segment[0]);
+ let endTime = parseFloat(segments[i].segment[1]);
+ allSegmentTimes.push([startTime, endTime]);
+ }
+
+ //merge all the times into non-overlapping arrays
+ const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
+ return a[0] - b[0] || a[1] - b[1];
+ }));
+
+ let videoDuration = data.items[0].contentDetails.duration;
+ videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
+ if (videoDuration != 0) {
+ let allSegmentDuration = 0;
+ //sum all segment times together
+ allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
+ if (allSegmentDuration > (videoDuration / 100) * 80) {
+ // Reject submission if all segments combine are over 80% of the video
+ return "Total length of your submitted segments are over 80% of the video.";
+ }
+ }
+
+ // Check NeuralBlock
+ const neuralBlockURL = config.neuralBlockURL;
+ if (!neuralBlockURL) return false;
+ const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
+ "&segments=" + nbString.substring(0, nbString.length - 1));
+ if (!response.ok) return false;
+
+ const nbPredictions = await response.json();
+ let nbDecision = false;
+ let predictionIdx = 0; //Keep track because only sponsor categories were submitted
+ for (let i = 0; i < segments.length; i++) {
+ if (segments[i].category === "sponsor") {
+ if (nbPredictions.probabilities[predictionIdx] < 0.70) {
+ nbDecision = true; // At least one bad entry
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
- let duration = data.items[0].contentDetails.duration;
- duration = isoDurations.toSeconds(isoDurations.parse(duration));
- if (duration == 0) {
- // Allow submission if the duration is 0 (bug in youtube api)
- return false;
- } else if ((endTime - startTime) > (duration / 100) * 80) {
- // Reject submission if over 80% of the video
- return "One of your submitted segments is over 80% of the video.";
- } else {
- if (segments[i].category === "sponsor") {
- //Prepare timestamps to send to NB all at once
- nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
- }
- }
- }
- // Check NeuralBlock
- const neuralBlockURL = config.neuralBlockURL;
- if (!neuralBlockURL) return false;
- const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
- "&segments=" + nbString.substring(0, nbString.length - 1));
- if (!response.ok) return false;
-
- const nbPredictions = await response.json();
- let nbDecision = false;
- let predictionIdx = 0; //Keep track because only sponsor categories were submitted
- for (let i = 0; i < segments.length; i++) {
- if (segments[i].category === "sponsor") {
- if (nbPredictions.probabilities[predictionIdx] < 0.70) {
- nbDecision = true; // At least one bad entry
- const startTime = parseFloat(segments[i].segment[0]);
- const endTime = parseFloat(segments[i].segment[1]);
-
- const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
- // Send to Discord
- // Note, if this is too spammy. Consider sending all the segments as one Webhook
- sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
- }
- predictionIdx++;
- }
-
- }
- if (nbDecision) {
- return "Rejected based on NeuralBlock predictions.";
- } else {
- return false;
+ const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
+ // Send to Discord
+ // Note, if this is too spammy. Consider sending all the segments as one Webhook
+ sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
}
+ predictionIdx++;
}
+
+ }
+
+ if (nbDecision) {
+ return "Rejected based on NeuralBlock predictions.";
+ } else {
+ return false;
}
} else {
Logger.debug("Skipped YouTube API");
@@ -243,6 +270,21 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s
}
}
+function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
+ const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
+ return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
+}
+
+async function getYouTubeVideoInfo(videoID: VideoID): Promise {
+ if (config.youtubeAPIKey !== null) {
+ return new Promise((resolve) => {
+ YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
+ });
+ } else {
+ return null;
+ }
+}
+
function proxySubmission(req: Request) {
fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, {
method: 'POST',
@@ -267,14 +309,18 @@ export async function postSkipSegments(req: Request, res: Response) {
const videoID = req.query.videoID || req.body.videoID;
let userID = req.query.userID || req.body.userID;
+ let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+ if (!Object.values(Service).some((val) => val == service)) {
+ service = Service.YouTube;
+ }
+ let videoDuration: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration;
-
- let segments = req.body.segments;
+ let segments = req.body.segments as IncomingSegment[];
if (segments === undefined) {
// Use query instead
segments = [{
- segment: [req.query.startTime, req.query.endTime],
- category: req.query.category,
+ segment: [req.query.startTime as string, req.query.endTime as string],
+ category: req.query.category as Category
}];
}
@@ -312,7 +358,7 @@ export async function postSkipSegments(req: Request, res: Response) {
return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
}
- const noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
+ let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
return list.category;
});
@@ -321,6 +367,32 @@ export async function postSkipSegments(req: Request, res: Response) {
const decreaseVotes = 0;
+ let apiVideoInfo: APIVideoInfo = null;
+ if (service == Service.YouTube) {
+ apiVideoInfo = await getYouTubeVideoInfo(videoID);
+ }
+ const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
+ if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
+ // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
+ videoDuration = apiVideoDuration || 0 as VideoDuration;
+ }
+
+ const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0
+ AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as
+ {videoDuration: VideoDuration, UUID: SegmentUUID}[];
+ // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
+ const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
+ if (videoDurationChanged) {
+ // Hide all previous submissions
+ for (const submission of previousSubmissions) {
+ await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
+ }
+
+ // Reset no segments
+ noSegmentList = [];
+ deleteNoSegments(videoID, null);
+ }
+
// Check if all submissions are correct
for (let i = 0; i < segments.length; i++) {
if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@@ -343,7 +415,7 @@ export async function postSkipSegments(req: Request, res: Response) {
+ segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n "
+ (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " +
"Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "")
- + "If you believe this is incorrect, please contact someone on Discord.",
+ + "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app",
);
return;
}
@@ -367,7 +439,7 @@ export async function postSkipSegments(req: Request, res: Response) {
//check if this info has already been submitted before
const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ?
- and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]);
+ and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]);
if (duplicateCheck2Row.count > 0) {
res.sendStatus(409);
return;
@@ -375,8 +447,8 @@ export async function postSkipSegments(req: Request, res: Response) {
}
// Auto moderator check
- if (!isVIP) {
- const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
+ if (!isVIP && service == Service.YouTube) {
+ const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected.
@@ -437,63 +509,19 @@ export async function postSkipSegments(req: Request, res: Response) {
let startingVotes = 0 + decreaseVotes;
- if (config.youtubeAPIKey !== null) {
- let {err, data} = await new Promise((resolve) => {
- YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
- });
-
- if (err) {
- Logger.error("Error while submitting when connecting to YouTube API: " + err);
- } else {
- //get all segments for this video and user
- const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [userID, videoID]);
- const allSegmentTimes = [];
- if (allSubmittedByUser !== undefined) {
- //add segments the user has previously submitted
- for (const segmentInfo of allSubmittedByUser) {
- allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
- }
- }
-
- //add segments they are trying to add in this submission
- for (let i = 0; i < segments.length; i++) {
- let startTime = parseFloat(segments[i].segment[0]);
- let endTime = parseFloat(segments[i].segment[1]);
- allSegmentTimes.push([startTime, endTime]);
- }
-
- //merge all the times into non-overlapping arrays
- const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
- return a[0] - b[0] || a[1] - b[1];
- }));
-
- let videoDuration = data.items[0].contentDetails.duration;
- videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
- if (videoDuration != 0) {
- let allSegmentDuration = 0;
- //sum all segment times together
- allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
- if (allSegmentDuration > (videoDuration / 100) * 80) {
- // Reject submission if all segments combine are over 80% of the video
- res.status(400).send("Total length of your submitted segments are over 80% of the video.");
- return;
- }
- }
- }
- }
-
for (const segmentInfo of segments) {
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
//also better for duplication checking
- const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
+ const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
+ const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0;
try {
await db.prepare('run', `INSERT INTO "sponsorTimes"
- ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID")
- VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
- videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
+ ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+ videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
],
);
@@ -502,9 +530,10 @@ export async function postSkipSegments(req: Request, res: Response) {
// Clear redis cache for this video
redis.delAsync(skipSegmentsKey(videoID));
+ redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
} catch (err) {
//a DB change probably occurred
- res.sendStatus(502);
+ res.sendStatus(500);
Logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " +
segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err);
@@ -529,7 +558,7 @@ export async function postSkipSegments(req: Request, res: Response) {
res.json(newSegments);
for (let i = 0; i < segments.length; i++) {
- sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
+ sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service);
}
}
diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts
index a1e81e7..240baf9 100644
--- a/src/routes/voteOnSponsorTime.ts
+++ b/src/routes/voteOnSponsorTime.ts
@@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash';
import {config} from '../config';
import { UserID } from '../types/user.model';
import redis from '../utils/redis';
-import { skipSegmentsKey } from '../middleware/redisKeys';
-import { VideoID } from '../types/segments.model';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
+import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
const voteTypes = {
normal: 0,
@@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) {
}
}
-async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string
- , hashedIP: string, finalResponse: FinalResponse, res: Response) {
+async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category
+ , hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) {
// Check if they've already made a vote
const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]);
@@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
return;
}
- const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]);
- if (!currentCategory) {
+ const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
+ [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
+ if (!videoInfo) {
// Submission doesn't exist
res.status(400).send("Submission doesn't exist.");
return;
@@ -198,7 +199,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
}
// See if the submissions category is ready to change
- const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
+ const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
@@ -210,9 +211,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
// Add submission as vote
if (!currentCategoryInfo && submissionInfo) {
- await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
+ await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
- await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
+ await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
}
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
@@ -225,6 +226,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
}
}
+ clearRedisCache(videoInfo);
+
res.sendStatus(finalResponse.finalStatus);
}
@@ -233,10 +236,10 @@ export function getUserID(req: Request): UserID {
}
export async function voteOnSponsorTime(req: Request, res: Response) {
- const UUID = req.query.UUID as string;
+ const UUID = req.query.UUID as SegmentUUID;
const paramUserID = getUserID(req);
let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
- const category = req.query.category as string;
+ const category = req.query.category as Category;
if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
//invalid request
@@ -258,7 +261,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
const ip = getIP(req);
//hash the ip 5000 times so no one can get it from the database
- const hashedIP = getHash(ip + config.globalSalt);
+ const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress);
//check if this user is on the vip list
const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0;
@@ -353,13 +356,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
}
//check if the increment amount should be multiplied (downvotes have more power if there have been many views)
- const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
- {videoID: VideoID, votes: number, views: number};
+ const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
+ {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
if (voteTypeEnum === voteTypes.normal) {
if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
//this user is a vip and a downvote
- incrementAmount = -(row.votes + 2 - oldIncrementAmount);
+ incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount);
type = incrementAmount;
}
} else if (voteTypeEnum == voteTypes.incorrect) {
@@ -403,8 +406,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
}
- // Clear redis cache for this video
- redis.delAsync(skipSegmentsKey(row?.videoID));
+ clearRedisCache(videoInfo);
//for each positive vote, see if a hidden submission can be shown again
if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
@@ -441,7 +443,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
voteTypeEnum,
isVIP,
isOwnSubmission,
- row,
+ row: videoInfo,
category,
incrementAmount,
oldIncrementAmount,
@@ -453,4 +455,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
res.status(500).json({error: 'Internal error creating segment vote'});
}
-}
\ No newline at end of file
+}
+
+function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
+ if (videoInfo) {
+ redis.delAsync(skipSegmentsKey(videoInfo.videoID));
+ redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
+ }
+}
diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts
index e7b6eaf..a95bac3 100644
--- a/src/types/segments.model.ts
+++ b/src/types/segments.model.ts
@@ -3,15 +3,33 @@ import { SBRecord } from "./lib.model";
export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
export type VideoID = string & { __videoIDBrand: unknown };
+export type VideoDuration = number & { __videoDurationBrand: unknown };
export type Category = string & { __categoryBrand: unknown };
export type VideoIDHash = VideoID & HashedValue;
export type IPAddress = string & { __ipAddressBrand: unknown };
export type HashedIP = IPAddress & HashedValue;
+// Uncomment as needed
+export enum Service {
+ YouTube = 'YouTube',
+ PeerTube = 'PeerTube',
+ // Twitch = 'Twitch',
+ // Nebula = 'Nebula',
+ // RSS = 'RSS',
+ // Corridor = 'Corridor',
+ // Lbry = 'Lbry'
+}
+
+export interface IncomingSegment {
+ category: Category;
+ segment: string[];
+}
+
export interface Segment {
category: Category;
segment: number[];
UUID: SegmentUUID;
+ videoDuration: VideoDuration;
}
export enum Visibility {
@@ -28,6 +46,7 @@ export interface DBSegment {
locked: boolean;
shadowHidden: Visibility;
videoID: VideoID;
+ videoDuration: VideoDuration;
hashedVideoID: VideoIDHash;
}
diff --git a/test.json b/test.json
index add6ebf..58eb72a 100644
--- a/test.json
+++ b/test.json
@@ -49,7 +49,7 @@
]
}
],
- "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+ "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"],
"maxNumberOfActiveWarnings": 3,
"hoursAfterWarningExpires": 24,
"rateLimit": {
diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts
index 85dcfbb..f40203c 100644
--- a/test/cases/getSkipSegments.ts
+++ b/test/cases/getSkipSegments.ts
@@ -5,17 +5,19 @@ import {getHash} from '../../src/utils/getHash';
describe('getSkipSegments', () => {
before(async () => {
- let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
- await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
- await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
- await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
- await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
- await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
- await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
- await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
- await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
- await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
-
+ let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+ await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')");
+ await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')");
+ await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')");
+ await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 0, '" + getHash('testtesttest,test', 1) + "')");
+ await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('test3', 1) + "')");
+ await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('test3', 1) + "')");
+ await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')");
+ await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, 0, '" + getHash('multiple', 1) + "')");
+ await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, 0, '" + getHash('locked', 1) + "')");
+ await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, 0, '" + getHash('locked', 1) + "')");
+ await db.prepare("run", startOfQuery + "('onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', 0, 50, 'sponsor', 'YouTube', 190, 1, 0, '" + getHash('onlyHiddenSegments', 1) + "')");
+
return;
});
@@ -27,7 +29,24 @@ describe('getSkipSegments', () => {
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
- && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+ && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
+ return;
+ } else {
+ return ("Received incorrect body: " + (await res.text()));
+ }
+ }
+ })
+ .catch(err => "Couldn't call endpoint");
+ });
+
+ it('Should be able to get a time by category for a different service 1', () => {
+ fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube")
+ .then(async res => {
+ if (res.status !== 200) return ("Status code was: " + res.status);
+ else {
+ const data = await res.json();
+ if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+ && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
@@ -61,7 +80,7 @@ describe('getSkipSegments', () => {
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
- && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+ && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
@@ -78,7 +97,23 @@ describe('getSkipSegments', () => {
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
- && data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
+ && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) {
+ return;
+ } else {
+ return ("Received incorrect body: " + (await res.text()));
+ }
+ }
+ })
+ .catch(err => ("Couldn't call endpoint"));
+ });
+
+ it('Should be empty if all submissions are hidden', () => {
+ fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments")
+ .then(async res => {
+ if (res.status !== 200) return ("Status code was: " + res.status);
+ else {
+ const data = await res.json();
+ if (data.length === 0) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
@@ -99,9 +134,9 @@ describe('getSkipSegments', () => {
let success = true;
for (const segment of data) {
if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
- || segment.category !== "intro" || segment.UUID !== "1-uuid-7") &&
+ || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) &&
(segment.segment[0] !== 1 || segment.segment[1] !== 11
- || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) {
+ || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) {
success = false;
break;
}
diff --git a/test/cases/getSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts
similarity index 74%
rename from test/cases/getSegmentsByHash.ts
rename to test/cases/getSkipSegmentsByHash.ts
index d957e2d..23d9d49 100644
--- a/test/cases/getSegmentsByHash.ts
+++ b/test/cases/getSkipSegmentsByHash.ts
@@ -12,11 +12,13 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
describe('getSegmentsByHash', () => {
before(async () => {
- let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
- await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
- await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
- await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
- await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+ let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+ await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+ await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+ await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+ await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+ await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+ await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3
});
it('Should be able to get a 200', (done: Done) => {
@@ -54,6 +56,19 @@ describe('getSegmentsByHash', () => {
.catch(err => done("Couldn't call endpoint"));
});
+ it('Should be able to get an empty array if only hidden videos', (done: Done) => {
+ fetch(getbaseURL() + '/api/skipSegments/f3a1?categories=["sponsor"]')
+ .then(async res => {
+ if (res.status !== 404) done("non 404 status code, was " + res.status);
+ else {
+ const body = await res.text();
+ if (JSON.parse(body).length === 0 && body === '[]') done(); // pass
+ else done("non empty array returned");
+ }
+ })
+ .catch(err => done("Couldn't call endpoint"));
+ });
+
it('Should return 400 prefix too short', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/11?categories=["shilling"]')
.then(res => {
@@ -128,7 +143,24 @@ describe('getSegmentsByHash', () => {
if (body.length !== 2) done("expected 2 videos, got " + body.length);
else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
- else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+ else if (body[0].segments[0].category !== 'sponsor'
+ || body[0].segments[0].UUID !== 'getSegmentsByHash-0-0'
+ || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+ else done();
+ }
+ })
+ .catch(err => done("Couldn't call endpoint"));
+ });
+
+ it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => {
+ fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube')
+ .then(async res => {
+ if (res.status !== 200) done("non 200 status code, was " + res.status);
+ else {
+ const body = await res.json();
+ if (body.length !== 1) done("expected 2 videos, got " + body.length);
+ else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
+ else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor");
else done();
}
})
diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts
index e31326a..512c2ee 100644
--- a/test/cases/postSkipSegments.ts
+++ b/test/cases/postSkipSegments.ts
@@ -6,6 +6,7 @@ import {db} from '../../src/databases/databases';
import {ImportMock} from 'ts-mock-imports';
import * as YouTubeAPIModule from '../../src/utils/youtubeApi';
import {YouTubeApiMock} from '../youtubeMock';
+import e from 'express';
const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, 'YouTubeAPI');
const sinonStub = mockManager.mock('listVideos');
@@ -97,6 +98,177 @@ describe('postSkipSegments', () => {
.catch(err => done(err));
});
+ it('Should be able to submit a single time with a duration from the YouTube API (JSON method)', (done: Done) => {
+ fetch(getbaseURL()
+ + "/api/postVideoSponsorTimes", {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ userID: "test",
+ videoID: "dQw4w9WgXZX",
+ videoDuration: 100,
+ segments: [{
+ segment: [0, 10],
+ category: "sponsor",
+ }],
+ }),
+ })
+ .then(async res => {
+ if (res.status === 200) {
+ const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]);
+ if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) {
+ done();
+ } else {
+ done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+ }
+ } else {
+ done("Status code was " + res.status);
+ }
+ })
+ .catch(err => done(err));
+ });
+
+ it('Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)', (done: Done) => {
+ fetch(getbaseURL()
+ + "/api/postVideoSponsorTimes", {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ userID: "test",
+ videoID: "dQw4w9WgXZH",
+ videoDuration: 5010.20,
+ segments: [{
+ segment: [1, 10],
+ category: "sponsor",
+ }],
+ }),
+ })
+ .then(async res => {
+ if (res.status === 200) {
+ const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]);
+ if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) {
+ done();
+ } else {
+ done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+ }
+ } else {
+ done("Status code was " + res.status);
+ }
+ })
+ .catch(err => done(err));
+ });
+
+ it('Should be able to submit a single time with a duration in the body (JSON method)', (done: Done) => {
+ fetch(getbaseURL()
+ + "/api/postVideoSponsorTimes", {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ userID: "test",
+ videoID: "noDuration",
+ videoDuration: 100,
+ segments: [{
+ segment: [0, 10],
+ category: "sponsor",
+ }],
+ }),
+ })
+ .then(async res => {
+ if (res.status === 200) {
+ const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["noDuration"]);
+ if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 100) {
+ done();
+ } else {
+ done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+ }
+ } else {
+ done("Status code was " + res.status);
+ }
+ })
+ .catch(err => done(err));
+ });
+
+ it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
+ await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category")
+ VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')");
+
+ try {
+ const res = await fetch(getbaseURL()
+ + "/api/postVideoSponsorTimes", {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ userID: "test",
+ videoID: "noDuration",
+ videoDuration: 100,
+ segments: [{
+ segment: [1, 10],
+ category: "sponsor",
+ }],
+ }),
+ });
+
+ if (res.status === 200) {
+ const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]);
+ const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
+ FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
+ const videoRow = videoRows[0];
+ const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
+ FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
+ if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
+ && videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
+ return;
+ } else {
+ return "Submitted times were not saved. Actual submission: " + JSON.stringify(videoRow);
+ }
+ } else {
+ return "Status code was " + res.status;
+ }
+ } catch (e) {
+ return e;
+ }
+ });
+
+ it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
+ fetch(getbaseURL()
+ + "/api/postVideoSponsorTimes", {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ userID: "test",
+ videoID: "dQw4w9WgXcG",
+ service: "PeerTube",
+ segments: [{
+ segment: [0, 10],
+ category: "sponsor",
+ }],
+ }),
+ })
+ .then(async res => {
+ if (res.status === 200) {
+ const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]);
+ if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") {
+ done();
+ } else {
+ done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+ }
+ } else {
+ done("Status code was " + res.status);
+ }
+ })
+ .catch(err => done(err));
+ });
+
it('VIP submission should start locked', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
@@ -244,7 +416,7 @@ describe('postSkipSegments', () => {
}),
})
.then(async res => {
- if (res.status === 400) {
+ if (res.status === 403) {
const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["n9rIGdXnSJc"]);
let success = true;
if (rows.length === 4) {
@@ -292,7 +464,7 @@ describe('postSkipSegments', () => {
}),
})
.then(async res => {
- if (res.status === 400) {
+ if (res.status === 403) {
const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["80percent_video"]);
let success = rows.length == 2;
for (const row of rows) {