Merge remote-tracking branch 'upstream/master' into fix/prepare-statements

README.MD

@@ -32,10 +32,6 @@ Run the server with `npm start`.
 
 If you want to make changes, run `npm run dev` to automatically reload the server and run tests whenever a file is saved.
 
-# Privacy Policy
-
-If you set the `youtubeAPIKey` option in `config.json`, you must follow [Google's Privacy Policy](https://policies.google.com/privacy) and [YouTube's Terms of Service](https://www.youtube.com/t/terms)
-
 # API Docs
 
 Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs)

config.json.example

@@ -4,7 +4,7 @@
     "port": 80,
     "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]",
     "adminUserID": "[the hashed id of the user who can perform admin actions]",
-    "youtubeAPIKey": null, //get this from Google Cloud Platform [optional]
+    "newLeafURLs": ["http://localhost:3241"],
     "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
     "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
     "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
@@ -22,7 +22,7 @@
     "mode": "development",
     "readOnly": false,
     "webhooks": [],
-    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected
+    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "preview", "music_offtopic", "highlight"], // List of supported categories any other category will be rejected
     "getTopUsersCacheTimeMinutes": 5, // cacheTime for getTopUsers result in minutes
     "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire
     "hoursAfterWarningExpire": 24,

databases/_private.db.sql

@@ -1,8 +1,5 @@
 BEGIN TRANSACTION;
 
-CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
-	"userID"	TEXT NOT NULL
-);
 CREATE TABLE IF NOT EXISTS "votes" (
 	"UUID"	TEXT NOT NULL,
 	"userID"	TEXT NOT NULL,

databases/_sponsorTimes_indexes.sql

@@ -51,10 +51,10 @@ CREATE INDEX IF NOT EXISTS "warnings_issueTime"
     ("issueTime" ASC NULLS LAST)
     TABLESPACE pg_default;
 
--- noSegments
+-- lockCategories
 
 CREATE INDEX IF NOT EXISTS "noSegments_videoID"
-    ON public."noSegments" USING btree
+    ON public."lockCategories" USING btree
     ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
 
@@ -63,4 +63,11 @@ CREATE INDEX IF NOT EXISTS "noSegments_videoID"
 CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
     ON public."categoryVotes" USING btree
     ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- shadowBannedUsers
+
+CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
+    ON public."shadowBannedUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;

databases/_upgrade_sponsorTimes_12.sql

@@ -0,0 +1,31 @@
+BEGIN TRANSACTION;
+
+/* Add reputation field */
+CREATE TABLE "sqlb_temp_table_12" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+    "videoDuration"	REAL NOT NULL DEFAULT '0',
+    "hidden"	INTEGER NOT NULL DEFAULT '0',
+    "reputation" REAL NOT NULL DEFAULT 0,
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 12 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_13.sql

@@ -0,0 +1,17 @@
+BEGIN TRANSACTION;
+
+/* Add locked field */
+CREATE TABLE "sqlb_temp_table_13" (
+	"userID"	TEXT NOT NULL,
+	"userName"	TEXT NOT NULL,
+	"locked" INTEGER NOT NULL default '0'
+);
+
+INSERT INTO sqlb_temp_table_13 SELECT "userID", "userName", 0 FROM "userNames";
+
+DROP TABLE "userNames";
+ALTER TABLE sqlb_temp_table_13 RENAME TO "userNames";
+
+UPDATE "config" SET value = 13 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_14.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
+	"userID"	TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 14 WHERE key = 'version';
+
+COMMIT;

docker/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
     ports:
       - 5432:5432
-    restart: always
+    restart: unless-stopped
   redis:
     container_name: redis
     image: redis
@@ -19,7 +19,15 @@ services:
       - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
     ports:
       - 32773:6379
-    restart: always
+    restart: unless-stopped
+  newleaf:
+    image: abeltramo/newleaf:latest
+    container_name: newleaf
+    restart: unless-stopped
+    ports:
+      - 3241:3000
+    volumes:
+      - ./newleaf/configuration.py:/workdir/configuration.py
 
 volumes:
   database-data:

docker/newleaf/configuration.py

@@ -0,0 +1,17 @@
+# ==============================
+#  You MUST set these settings.
+# ==============================
+
+# A URL that this site can be accessed on. Do not include a trailing slash.
+website_origin = "http://newleaf:3000"
+
+
+# ==============================
+#  These settings are optional.
+# ==============================
+
+# The address of the interface to bind to.
+#bind_host = "0.0.0.0"
+
+# The port to bind to.
+#bind_port = 3000

docker/redis/redis.conf

@@ -1,2 +1,5 @@
 maxmemory-policy allkeys-lru
-maxmemory 1000mb
+maxmemory 2000mb
+
+appendonly no
+save ""

nginx/nginx.conf

@@ -1,8 +1,8 @@
 worker_processes 8;
-worker_rlimit_nofile 8192;
+worker_rlimit_nofile 65536;
 
 events {
-  worker_connections  132768;  ## Default: 1024
+  worker_connections  432768;  ## Default: 1024
 }
 
 http {
@@ -13,7 +13,7 @@ http {
     upstream backend_GET {
 	least_conn;        
         server localhost:4441;
-	server localhost:4442;
+	#server localhost:4442;
         #server localhost:4443;
         #server localhost:4444;
         #server localhost:4445;
@@ -48,9 +48,9 @@ http {
         server_name sponsor.ajay.app api.sponsor.ajay.app;
 
 	error_page 404 /404.html;
-	error_page 500 @myerrordirective_500;
+	#error_page 500 @myerrordirective_500;
-	error_page 502 @myerrordirective_502;
+	#error_page 502 @myerrordirective_502;
-	error_page 504 @myerrordirective_504;
+	#error_page 504 @myerrordirective_504;
 	#location = /404 {
 	#	root /home/sbadmin/caddy/SponsorBlockSite/public-prod;
 	#	internal;
@@ -58,15 +58,15 @@ http {
 
 	#proxy_send_timeout 120s;
 
-	location @myerrordirective_500 {
+	#location @myerrordirective_500 {
- 	    return 400 "Internal Server Error";
+ 	#    return 400 "Internal Server Error";
-	}
+	#}
-	location @myerrordirective_502 {
+	#location @myerrordirective_502 {
-            return 400 "Bad Gateway";
+        #    return 400 "Bad Gateway";
-        }
+        #}
-	location @myerrordirective_504 {
+	#location @myerrordirective_504 {
-            return 400 "Gateway Timeout";
+        #    return 400 "Gateway Timeout";
-        }
+        #}
 
 
 	location /news {
@@ -106,8 +106,11 @@ http {
         }
      
 	location /download/ {
+	    gzip on;
+	    gzip_types      text/plain application/json;
 	     #alias /home/sbadmin/sponsor/docker/database-export/;
-	     return 307 https://cdnsponsor.ajay.app$request_uri;
+	      alias /home/sbadmin/sponsor/docker/database-export/;
+	     #return 307 https://cdnsponsor.ajay.app$request_uri;
 	}
 	location /database {
             proxy_pass http://backend_db;
@@ -139,35 +142,6 @@ http {
         
         location / {
             root /home/sbadmin/SponsorBlockSite/public-prod;            
-
-            ### CORS
-            if ($request_method = 'OPTIONS') {
-                add_header 'Access-Control-Allow-Origin' '*';
-                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                #
-                # Custom headers and headers various browsers *should* be OK with but aren't
-                #
-                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                #
-                # Tell client that this pre-flight info is valid for 20 days
-                #
-                add_header 'Access-Control-Max-Age' 1728000;
-                add_header 'Content-Type' 'text/plain; charset=utf-8';
-                add_header 'Content-Length' 0;
-                return 204;
-             }
-             if ($request_method = 'POST') {
-                add_header 'Access-Control-Allow-Origin' '*';
-                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
-             }
-             if ($request_method = 'GET') {
-                add_header 'Access-Control-Allow-Origin' '*';
-                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
-             }
         }
         
     
@@ -202,36 +176,7 @@ http {
 	}
 
 	location / {
-            root /home/sbadmin/SponsorBlockSite/public-prod;            
+            root /home/sbadmin/SponsorBlockSite/public-prod;
-
-            ### CORS
-            if ($request_method = 'OPTIONS') {
-                add_header 'Access-Control-Allow-Origin' '*';
-                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                #
-                # Custom headers and headers various browsers *should* be OK with but aren't
-                #
-                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                #
-                # Tell client that this pre-flight info is valid for 20 days
-                #
-                add_header 'Access-Control-Max-Age' 1728000;
-                add_header 'Content-Type' 'text/plain; charset=utf-8';
-                add_header 'Content-Length' 0;
-                return 204;
-             }
-             if ($request_method = 'POST') {
-                add_header 'Access-Control-Allow-Origin' '*';
-                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
-             }
-             if ($request_method = 'GET') {
-                add_header 'Access-Control-Allow-Origin' '*';
-                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
-                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
-             }
         }
     
 
@@ -250,7 +195,7 @@ http {
     server {
 
 	 access_log off;
-        error_log /dev/null;
+        error_log /etc/nginx/logs/log.txt;
 	
 
     if ($host = api.sponsor.ajay.app) {

package.json

@@ -13,6 +13,7 @@
   "author": "Ajay Ramachandran",
   "license": "MIT",
   "dependencies": {
+    "abort-controller": "^3.0.0",
     "better-sqlite3": "^7.1.5",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
@@ -21,22 +22,21 @@
     "iso8601-duration": "^1.2.0",
     "node-fetch": "^2.6.0",
     "pg": "^8.5.1",
-    "redis": "^3.0.2",
+    "redis": "^3.1.1",
     "sync-mysql": "^3.0.1",
-    "uuid": "^3.3.2",
+    "uuid": "^3.3.2"
-    "youtube-api": "^3.0.1"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^5.4.0",
     "@types/express": "^4.17.8",
     "@types/express-rate-limit": "^5.1.0",
-    "@types/mocha": "^8.0.3",
+    "@types/mocha": "^8.2.2",
     "@types/node": "^14.11.9",
     "@types/node-fetch": "^2.5.7",
     "@types/pg": "^7.14.10",
     "@types/redis": "^2.8.28",
     "@types/request": "^2.48.5",
-    "mocha": "^7.1.1",
+    "mocha": "^8.4.0",
     "nodemon": "^2.0.2",
     "sinon": "^9.2.0",
     "ts-mock-imports": "^1.3.0",

src/app.ts

@@ -25,9 +25,11 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments';
 import {userCounter} from './middleware/userCounter';
 import {loggerMiddleware} from './middleware/logger';
 import {corsMiddleware} from './middleware/cors';
+import {apiCspMiddleware} from './middleware/apiCsp';
 import {rateLimitMiddleware} from './middleware/requestRateLimit';
 import dumpDatabase, {redirectLink} from './routes/dumpDatabase';
-
+import {endpoint as getSegmentInfo} from './routes/getSegmentInfo';
+import {postClearCache} from './routes/postClearCache';
 
 export function createServer(callback: () => void) {
     // Create a service (the app object is just a callback).
@@ -36,6 +38,7 @@ export function createServer(callback: () => void) {
     //setup CORS correctly
     app.use(corsMiddleware);
     app.use(loggerMiddleware);
+    app.use("/api/", apiCspMiddleware);
     app.use(express.json());
 
     if (config.userCounterURL) app.use(userCounter);
@@ -110,6 +113,7 @@ function setupRoutes(app: Express) {
     app.get('/api/getTotalStats', getTotalStats);
 
     app.get('/api/getUserInfo', getUserInfo);
+    app.get('/api/userInfo', getUserInfo);
 
     //send out a formatted time saved total
     app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
@@ -130,6 +134,12 @@ function setupRoutes(app: Express) {
     //get if user is a vip
     app.post('/api/segmentShift', postSegmentShift);
 
+    //get segment info
+    app.get('/api/segmentInfo', getSegmentInfo);
+
+    //clear cache as VIP
+    app.post('/api/clearCache', postClearCache)
+
     if (config.postgres) {
         app.get('/database', (req, res) => dumpDatabase(req, res, true));
         app.get('/database.json', (req, res) => dumpDatabase(req, res, false));

src/config.ts

@@ -16,14 +16,15 @@ addDefaults(config, {
     privateDBSchema: "./databases/_private.db.sql",
     readOnly: false,
     webhooks: [],
-    categoryList: ["sponsor", "selfpromo", "interaction",  "intro", "outro", "preview", "music_offtopic"],
+    categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"],
     maxNumberOfActiveWarnings: 3,
     hoursAfterWarningExpires: 24,
     adminUserID: "",
-    discordCompletelyIncorrectReportWebhookURL: "",
+    discordCompletelyIncorrectReportWebhookURL: null,
-    discordFirstTimeSubmissionsWebhookURL: "",
+    discordFirstTimeSubmissionsWebhookURL: null,
-    discordNeuralBlockRejectWebhookURL: "",
+    discordNeuralBlockRejectWebhookURL: null,
-    discordReportChannelWebhookURL: "",
+    discordFailedReportChannelWebhookURL: null,
+    discordReportChannelWebhookURL: null,
     getTopUsersCacheTimeMinutes: 0,
     globalSalt: null,
     mode: "",
@@ -44,7 +45,7 @@ addDefaults(config, {
         },
     },
     userCounterURL: null,
-    youtubeAPIKey: null,
+    newLeafURLs: null,
     maxRewardTimePerSegmentInSeconds: 86400,
     postgres: null,
     dumpDatabase: {

src/databases/IDatabase.ts

@@ -1,5 +1,5 @@
 export interface IDatabase {
-    async init(): Promise<void>;
+    init(): Promise<void>;
 
     prepare(type: QueryType, query: string, params?: any[]): Promise<any | any[] | void>;
 }

src/middleware/apiCsp.ts

@@ -0,0 +1,6 @@
+import {NextFunction, Request, Response} from 'express';
+
+export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) {
+    res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
+    next();
+}

src/middleware/cors.ts

@@ -2,6 +2,7 @@ import {NextFunction, Request, Response} from 'express';
 
 export function corsMiddleware(req: Request, res: Response, next: NextFunction) {
     res.header("Access-Control-Allow-Origin", "*");
-    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+    res.header("Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Accept");
+    res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE")
     next();
 }

src/routes/dumpDatabase.ts

@@ -143,6 +143,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
             ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
     } else {
         res.send({
+            dbVersion: await getDbVersion(),
             lastUpdated: lastUpdate,
             updateQueued,
             links: latestDumpFiles.map((item:any) => {
@@ -158,6 +159,12 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
     await queueDump();
 }
 
+async function getDbVersion(): Promise<number> {
+    const row = await db.prepare('get', `SELECT "value" FROM "config" WHERE "key" = 'version'`);
+    if (row === undefined) return 0;
+    return row.value;
+}
+
 export async function redirectLink(req: Request, res: Response): Promise<void> {
     if (!config?.dumpDatabase?.enabled) {
         res.status(404).send("Database dump is disabled");
@@ -210,4 +217,4 @@ async function queueDump(): Promise<void> {
         updateRunning = false;
         lastUpdate = startTime;
     }
-}
+}

src/routes/getSegmentInfo.ts

@@ -0,0 +1,79 @@
+import { Request, Response } from 'express';
+import { db } from '../databases/databases';
+import { DBSegment, SegmentUUID } from "../types/segments.model";
+
+const isValidSegmentUUID = (str: string): Boolean => /^([a-f0-9]{64}|[a-f0-9]{8}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{12})/.test(str)
+
+async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise<DBSegment> {
+    try {
+        return await db.prepare('get',
+            `SELECT "videoID", "startTime", "endTime", "votes", "locked",
+                "UUID", "userID", "timeSubmitted", "views", "category",
+                "service", "videoDuration", "hidden", "reputation", "shadowHidden" FROM "sponsorTimes"
+            WHERE "UUID" = ?`, [UUID]);
+    } catch (err) {
+        return null;
+    }
+}
+
+async function getSegmentsByUUID(UUIDs: SegmentUUID[]): Promise<DBSegment[]> {
+    const DBSegments: DBSegment[] = [];
+    for (let UUID of UUIDs) {
+        // if UUID is invalid, skip
+        if (!isValidSegmentUUID(UUID)) continue;
+        DBSegments.push(await getSegmentFromDBByUUID(UUID as SegmentUUID));
+    }
+    return DBSegments;
+}
+
+async function handleGetSegmentInfo(req: Request, res: Response) {
+    // If using params instead of JSON, only one UUID can be pulled
+    let UUIDs = req.query.UUIDs
+    ? JSON.parse(req.query.UUIDs as string)
+    : req.query.UUID
+        ? [req.query.UUID]
+        : null;
+    // deduplicate with set
+    UUIDs = [ ...new Set(UUIDs)];
+    // if more than 10 entries, slice
+    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
+    if (!Array.isArray(UUIDs) || !UUIDs) {
+        res.status(400).send("UUIDs parameter does not match format requirements.");
+        return false;
+    }
+    const DBSegments = await getSegmentsByUUID(UUIDs);
+    // all uuids failed lookup
+    if (!DBSegments?.length) {
+        res.sendStatus(400);
+        return false;
+    }
+    // uuids valid but not found
+    if (DBSegments[0] === null || DBSegments[0] === undefined) {
+        res.sendStatus(400);
+        return false;
+    }
+    return DBSegments;
+}
+
+async function endpoint(req: Request, res: Response): Promise<void> {
+    try {
+        const DBSegments = await handleGetSegmentInfo(req, res);
+
+        // If false, res.send has already been called
+        if (DBSegments) {
+            //send result
+            res.send(DBSegments);
+        }
+    } catch (err) {
+        if (err instanceof SyntaxError) { // catch JSON.parse error
+            res.status(400).send("UUIDs parameter does not match format requirements.");
+        } else res.status(500).send();
+    }
+}
+
+export {
+    getSegmentFromDBByUUID,
+    getSegmentsByUUID,
+    handleGetSegmentInfo,
+    endpoint
+};

src/routes/getSkipSegments.ts

@@ -1,14 +1,15 @@
 import { Request, Response } from 'express';
-import { RedisClient } from 'redis';
 import { config } from '../config';
 import { db, privateDB } from '../databases/databases';
-import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys';
 import { SBRecord } from '../types/lib.model';
-import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
+import { Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
+import { getCategoryActionType } from '../utils/categoryInfo';
 import { getHash } from '../utils/getHash';
 import { getIP } from '../utils/getIP';
 import { Logger } from '../utils/logger';
-import redis from '../utils/redis';
+import { QueryCacher } from '../utils/queryCacher'
+import { getReputation } from '../utils/reputation';
 
 
 async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
@@ -40,7 +41,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
     
     const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
 
-    return chooseSegments(filteredSegments).map((chosenSegment) => ({
+    const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1
+    return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({
         category,
         segment: [chosenSegment.startTime, chosenSegment.endTime],
         UUID: chosenSegment.UUID,
@@ -48,7 +50,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
     }));
 }
 
-async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
+async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories: Category[], service: Service): Promise<Segment[]> {
     const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
     const segments: Segment[] = [];
 
@@ -56,13 +58,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
         categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
         if (categories.length === 0) return null;
 
-        const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
+        const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await getSegmentsFromDBByVideoID(videoID, service))
-            .prepare(
+            .filter((segment: DBSegment) => categories.includes(segment?.category))
-                'all',
+            .reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
-                `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" 
-                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
-                [videoID, service]
-            )).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
                 acc[segment.category] = acc[segment.category] || [];
                 acc[segment.category].push(segment);
 
@@ -70,7 +68,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
             }, {});
 
         for (const [category, categorySegments] of Object.entries(segmentsByCategory)) {
-            segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache)));
+            segments.push(...(await prepareCategorySegments(req, videoID, category as Category, categorySegments, cache)));
         }
 
         return segments;
@@ -92,7 +90,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
         categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
         if (categories.length === 0) return null;
 
-        const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
+        const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
             .filter((segment: DBSegment) => categories.includes(segment?.category))
             .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
@@ -127,37 +125,34 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
     }
 }
 
-async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
+async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
     const fetchFromDB = () => db
         .prepare(
             'all',
-            `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
+            `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
             WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
             [hashedVideoIDPrefix + '%', service]
-        );
+        ) as Promise<DBSegment[]>;
 
     if (hashedVideoIDPrefix.length === 4) {
-        const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
+        return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service))
-        const {err, reply} = await redis.getAsync(key);
-    
-        if (!err && reply) {
-            try {
-                Logger.debug("Got data from redis: " + reply);
-                return JSON.parse(reply);
-            } catch (e) {
-                // If all else, continue on to fetching from the database
-            }
-        }
-
-        const data = await fetchFromDB();
-
-        redis.setAsync(key, JSON.stringify(data));
-        return data;
     }
 
     return await fetchFromDB();
 }
 
+async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
+    const fetchFromDB = () => db
+        .prepare(
+            'all',
+            `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden" FROM "sponsorTimes" 
+            WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
+            [videoID, service]
+        ) as Promise<DBSegment[]>;
+
+    return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service))
+}
+
 //gets a weighted random choice from the choices array based on their `votes` property.
 //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
 //choices are unique
@@ -174,10 +169,12 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
     //assign a weight to each choice
     let totalWeight = 0;
     let choicesWithWeights: TWithWeight[] = choices.map(choice => {
+        const boost = Math.min(choice.reputation, 4);
+
         //The 3 makes -2 the minimum votes before being ignored completely
         //this can be changed if this system increases in popularity.
-        const weight = Math.exp((choice.votes + 3));
+        const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost);
-        totalWeight += weight;
+        totalWeight += Math.max(weight, 0);
 
         return {...choice, weight};
     });
@@ -206,7 +203,7 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
 //Only one similar time will be returned, randomly generated based on the sqrt of votes.
 //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
 //Segments with less than -1 votes are already ignored before this function is called
-function chooseSegments(segments: DBSegment[]): DBSegment[] {
+async function chooseSegments(segments: DBSegment[], max: number): Promise<DBSegment[]> {
     //Create groups of segments that are similar to eachother
     //Segments must be sorted by their startTime so that we can build groups chronologically:
     //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
@@ -215,9 +212,9 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] {
     const overlappingSegmentsGroups: OverlappingSegmentGroup[] = [];
     let currentGroup: OverlappingSegmentGroup;
     let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
-    segments.forEach(segment => {
+    for (const segment of segments) {
         if (segment.startTime > cursor) {
-            currentGroup = {segments: [], votes: 0, locked: false};
+            currentGroup = {segments: [], votes: 0, reputation: 0, locked: false};
             overlappingSegmentsGroups.push(currentGroup);
         }
 
@@ -227,21 +224,28 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] {
             currentGroup.votes += segment.votes;
         }
 
+        if (segment.userID) segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID));
+        if (segment.reputation > 0) {
+            currentGroup.reputation += segment.reputation;
+        }
+
         if (segment.locked) {
             currentGroup.locked = true;
         }
 
         cursor = Math.max(cursor, segment.endTime);
-    });
+    };
 
     overlappingSegmentsGroups.forEach((group) => {
         if (group.locked) {
             group.segments = group.segments.filter((segment) => segment.locked);
         }
+
+        group.reputation = group.reputation / group.segments.length;
     });
 
-    //if there are too many groups, find the best 8
+    //if there are too many groups, find the best ones
-    return getWeightedRandomChoice(overlappingSegmentsGroups, 32).map(
+    return getWeightedRandomChoice(overlappingSegmentsGroups, max).map(
         //randomly choose 1 good segment per group and return them
         group => getWeightedRandomChoice(group.segments, 1)[0],
     );
@@ -266,24 +270,16 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
         : req.query.category
             ? [req.query.category]
             : ['sponsor'];
+    if (!Array.isArray(categories)) {
+        res.status(400).send("Categories parameter does not match format requirements.");
+        return false;
+    }
 
     let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
     if (!Object.values(Service).some((val) => val == service)) {
         service = Service.YouTube;
     }
 
-    // Only 404s are cached at the moment
-    const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
-    
-    if (redisResult.reply) {
-        const redisSegments = JSON.parse(redisResult.reply);
-        if (redisSegments?.length === 0) {
-            res.sendStatus(404);
-            Logger.debug("Using segments from cache for " + videoID);
-            return false;
-        }
-    }
-
     const segments = await getSegmentsByVideoID(req, videoID, categories, service);
 
     if (segments === null || segments === undefined) {
@@ -294,9 +290,6 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
     if (segments.length === 0) {
         res.sendStatus(404);
 
-        // Save in cache
-        if (categories.length == 7) redis.setAsync(skipSegmentsKey(videoID), JSON.stringify(segments));
-
         return false;
     }
 
@@ -313,7 +306,9 @@ async function endpoint(req: Request, res: Response): Promise<void> {
             res.send(segments);
         }
     } catch (err) {
-        res.status(500).send();
+        if (err instanceof SyntaxError) {
+            res.status(400).send("Categories parameter does not match format requirements.");
+        } else res.status(500).send();
     }
 }
 

src/routes/getTopUsers.ts

@@ -21,17 +21,18 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
             SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as "categorySumOutro",
             SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as "categorySumInteraction",
             SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as "categorySelfpromo",
-            SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic", `;
+            SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic",
+            SUM(CASE WHEN category = 'preview' THEN 1 ELSE 0 END) as "categorySumPreview", `;
     }
 
     const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
         SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
         SUM("votes") as "userVotes", ` +
         additionalFields +
-        `IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
+        `COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
         LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID"
-        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL
+        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
-        GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
+        GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
         ORDER BY ? DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, sortBy]);
 
     for (let i = 0; i < rows.length; i++) {
@@ -48,6 +49,7 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
                 rows[i].categorySumInteraction,
                 rows[i].categorySelfpromo,
                 rows[i].categoryMusicOfftopic,
+                rows[i].categorySumPreview
             ];
         }
     }
@@ -71,10 +73,6 @@ export async function getTopUsers(req: Request, res: Response) {
         return;
     }
 
-    //TODO: remove. This is broken for now
-    res.status(200).send();
-    return;
-
     //setup which sort type to use
     let sortBy = '';
     if (sortType == 0) {

src/routes/getUserInfo.ts

@@ -4,6 +4,8 @@ import {isUserVIP} from '../utils/isUserVIP';
 import {Request, Response} from 'express';
 import {Logger} from '../utils/logger';
 import { HashedUserID, UserID } from '../types/user.model';
+import { getReputation } from '../utils/reputation';
+import { SegmentUUID } from "../types/segments.model";
 
 async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
     try {
@@ -26,6 +28,15 @@ async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ min
     }
 }
 
+async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<number> {
+    try {
+        let row = await db.prepare("get", `SELECT COUNT(*) as "ignoredSegmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]);
+        return row?.ignoredSegmentCount ?? 0
+    } catch (err) {
+        return null;
+    }
+}
+
 async function dbGetUsername(userID: HashedUserID) {
     try {
         let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -49,6 +60,15 @@ async function dbGetViewsForUser(userID: HashedUserID) {
     }
 }
 
+async function dbGetIgnoredViewsForUser(userID: HashedUserID) {
+    try {
+        let row = await db.prepare('get', `SELECT SUM("views") as "ignoredViewCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]);
+        return row?.ignoredViewCount ?? 0;
+    } catch (err) {
+        return false;
+    }
+}
+
 async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
     try {
         let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]);
@@ -59,18 +79,25 @@ async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
     }
 }
 
-export async function getUserInfo(req: Request, res: Response) {
+async function dbGetLastSegmentForUser(userID: HashedUserID): Promise<SegmentUUID> {
-    let userID = req.query.userID as UserID;
+    try {
+        let row = await db.prepare('get', `SELECT "UUID" FROM "sponsorTimes" WHERE "userID" = ? ORDER BY "timeSubmitted" DESC LIMIT 1`, [userID]);
+        return row?.UUID ?? null;
+    } catch (err) {
+        return null;
+    }
+}
 
-    if (userID == undefined) {
+export async function getUserInfo(req: Request, res: Response) {
+    const userID = req.query.userID as UserID;
+    const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID;
+
+    if (hashedUserID == undefined) {
         //invalid request
         res.status(400).send('Parameters are not valid');
         return;
     }
 
-    //hash the userID
-    const hashedUserID: HashedUserID = getHash(userID);
-
     const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID);
     if (segmentsSummary) {
         res.send({
@@ -78,9 +105,13 @@ export async function getUserInfo(req: Request, res: Response) {
             userName: await dbGetUsername(hashedUserID),
             minutesSaved: segmentsSummary.minutesSaved,
             segmentCount: segmentsSummary.segmentCount,
+            ignoredSegmentCount: await dbGetIgnoredSegmentCount(hashedUserID),
             viewCount: await dbGetViewsForUser(hashedUserID),
+            ignoredViewCount: await dbGetIgnoredViewsForUser(hashedUserID),
             warnings: await dbGetWarningsForUser(hashedUserID),
+            reputation: await getReputation(hashedUserID),
             vip: await isUserVIP(hashedUserID),
+            lastSegmentID: await dbGetLastSegmentForUser(hashedUserID),
         });
     } else {
         res.status(400).send();

src/routes/postClearCache.ts

@@ -0,0 +1,55 @@
+import { Logger } from '../utils/logger';
+import { HashedUserID, UserID } from '../types/user.model';
+import { getHash } from '../utils/getHash';
+import { Request, Response } from 'express';
+import { Service, VideoID } from '../types/segments.model';
+import { QueryCacher } from '../utils/queryCacher';
+import { isUserVIP } from '../utils/isUserVIP';
+import { VideoIDHash } from "../types/segments.model";
+
+export async function postClearCache(req: Request, res: Response) {
+    const videoID = req.query.videoID as VideoID;
+    let userID = req.query.userID as UserID;
+    const service = req.query.service as Service ?? Service.YouTube;
+
+    const invalidFields = [];
+    if (typeof videoID !== 'string') {
+      invalidFields.push('videoID');
+    }
+    if (typeof userID !== 'string') {
+      invalidFields.push('userID');
+    }
+
+    if (invalidFields.length !== 0) {
+      // invalid request
+      const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, '');
+      res.status(400).send(`No valid ${fields} field(s) provided`);
+      return false;
+    }
+
+    // hash the userID as early as possible
+    const hashedUserID: HashedUserID = getHash(userID);
+    // hash videoID
+    const hashedVideoID: VideoIDHash = getHash(videoID, 1);
+
+    // Ensure user is a VIP
+    if (!(await isUserVIP(hashedUserID))){
+        Logger.warn("Permission violation: User " + hashedUserID + " attempted to clear cache for video " + videoID + ".");
+        res.status(403).json({"message": "Not a VIP"});
+        return false;
+    }
+
+    try {
+        QueryCacher.clearVideoCache({
+            videoID,
+            hashedVideoID,
+            service
+        });
+        res.status(200).json({
+            message: "Cache cleared on video " + videoID
+        });
+    } catch(err) {
+        res.status(500).send()
+        return false;
+    }
+}

src/routes/postSkipSegments.ts

@@ -1,30 +1,28 @@
 import {config} from '../config';
 import {Logger} from '../utils/logger';
 import {db, privateDB} from '../databases/databases';
-import {YouTubeAPI} from '../utils/youtubeApi';
+import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi';
 import {getSubmissionUUID} from '../utils/getSubmissionUUID';
 import fetch from 'node-fetch';
-import isoDurations from 'iso8601-duration';
+import isoDurations, { end } from 'iso8601-duration';
 import {getHash} from '../utils/getHash';
 import {getIP} from '../utils/getIP';
 import {getFormattedTime} from '../utils/getFormattedTime';
 import {isUserTrustworthy} from '../utils/isUserTrustworthy';
 import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
-import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys';
 import redis from '../utils/redis';
-import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
+import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
 import { deleteLockCategories } from './deleteLockCategories';
+import { getCategoryActionType } from '../utils/categoryInfo';
+import { QueryCacher } from '../utils/queryCacher';
+import { getReputation } from '../utils/reputation';
+import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
 
-interface APIVideoInfo {
+async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
-    err: string | boolean,
-    data: any
-}
-
-async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
     const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
     const userName = row !== undefined ? row.userName : null;
-    const video = youtubeData.items[0];
 
     let scopeName = "submissions.other";
     if (submissionCount <= 1) {
@@ -34,8 +32,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
     dispatchEvent(scopeName, {
         "video": {
             "id": videoID,
-            "title": video.snippet.title,
+            "title": youtubeData?.title,
-            "thumbnail": video.snippet.thumbnails.maxres ? video.snippet.thumbnails.maxres : null,
+            "thumbnail": getMaxResThumbnail(youtubeData) || null,
             "url": "https://www.youtube.com/watch?v=" + videoID,
         },
         "submission": {
@@ -73,7 +71,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
             method: 'POST',
             body: JSON.stringify({
                 "embeds": [{
-                    "title": data.items[0].snippet.title,
+                    "title": data?.title,
                     "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
                     "description": "Submission ID: " + UUID +
                         "\n\nTimestamp: " +
@@ -84,7 +82,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
                         "name": userID,
                     },
                     "thumbnail": {
-                        "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                        "url": getMaxResThumbnail(data) || "",
                     },
                 }],
             }),
@@ -174,10 +172,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
         const {err, data} = apiVideoInfo;
         if (err) return false;
 
-        // Check to see if video exists
+        const duration = apiVideoInfo?.data?.lengthSeconds;
-        if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
-
-        const duration = getYouTubeVideoDuration(apiVideoInfo);
         const segments = submission.segments;
         let nbString = "";
         for (let i = 0; i < segments.length; i++) {
@@ -217,8 +212,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
             return a[0] - b[0] || a[1] - b[1];
         }));
 
-        let videoDuration = data.items[0].contentDetails.duration;
+        const videoDuration = data?.lengthSeconds;
-        videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
         if (videoDuration != 0) {
             let allSegmentDuration = 0;
             //sum all segment times together
@@ -270,16 +264,9 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
     }
 }
 
-function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
+async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
-    const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
+    if (config.newLeafURLs !== null) {
-    return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
+        return YouTubeAPI.listVideos(videoID, ignoreCache);
-}
-
-async function getYouTubeVideoInfo(videoID: VideoID): Promise<APIVideoInfo> {
-    if (config.youtubeAPIKey !== null) {
-        return new Promise((resolve) => {
-            YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
-        });
     } else {
         return null;
     }
@@ -291,14 +278,10 @@ function proxySubmission(req: Request) {
             body: req.body,
         })
         .then(async res => {
-            if (config.mode === 'development') {
+            Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')');
-                Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')');
-            }
         })
         .catch(err => {
-            if (config.mode === 'development') {
+            Logger.error("Proxy Submission: Failed to make call");
-                Logger.error("Proxy Submission: Failed to make call");
-            }
         });
 }
 
@@ -367,22 +350,24 @@ export async function postSkipSegments(req: Request, res: Response) {
 
     const decreaseVotes = 0;
 
+    const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 
+                    AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as 
+                        {videoDuration: VideoDuration, UUID: SegmentUUID}[];
+    // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
+    const videoDurationChanged = (videoDuration: number) => videoDuration != 0 && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
+
     let apiVideoInfo: APIVideoInfo = null;
     if (service == Service.YouTube) {
-        apiVideoInfo = await getYouTubeVideoInfo(videoID);
+        // Don't use cache if we don't know the video duraton, or the client claims that it has changed
+        apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration));
     }
-    const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
+    const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
     if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
         // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
         videoDuration = apiVideoDuration || 0 as VideoDuration;
     }
 
-    const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 
+    if (videoDurationChanged(videoDuration)) {
-                    AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as 
-                        {videoDuration: VideoDuration, UUID: SegmentUUID}[];
-    // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
-    const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
-    if (videoDurationChanged) {
         // Hide all previous submissions
         for (const submission of previousSubmissions) {
             await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
@@ -411,10 +396,10 @@ export async function postSkipSegments(req: Request, res: Response) {
             // TODO: Do something about the fradulent submission
             Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
             res.status(403).send(
-                "Request rejected by auto moderator: New submissions are not allowed for the following category: '"
+                "New submissions are not allowed for the following category: '"
                 + segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n "
                 + (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " +
-                "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "")
+                "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n" : "")
                 + "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app",
             );
             return;
@@ -425,7 +410,9 @@ export async function postSkipSegments(req: Request, res: Response) {
         let endTime = parseFloat(segments[i].segment[1]);
 
         if (isNaN(startTime) || isNaN(endTime)
-            || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) {
+                || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime
+                || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable && startTime === endTime) 
+                || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime)) {
             //invalid request
             res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)");
             return;
@@ -498,7 +485,7 @@ export async function postSkipSegments(req: Request, res: Response) {
         }
 
         //check to see if this user is shadowbanned
-        const shadowBanRow = await privateDB.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
+        const shadowBanRow = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
 
         let shadowBanned = shadowBanRow.userCount;
 
@@ -508,6 +495,7 @@ export async function postSkipSegments(req: Request, res: Response) {
         }
 
         let startingVotes = 0 + decreaseVotes;
+        const reputation = await getReputation(userID);
 
         for (const segmentInfo of segments) {
             //this can just be a hash of the data
@@ -519,9 +507,9 @@ export async function postSkipSegments(req: Request, res: Response) {
             const startingLocked = isVIP ? 1 : 0;
             try {
                 await db.prepare('run', `INSERT INTO "sponsorTimes" 
-                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
+                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID")
-                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, reputation, shadowBanned, hashedVideoID,
                     ],
                 );
 
@@ -529,8 +517,12 @@ export async function postSkipSegments(req: Request, res: Response) {
                 await privateDB.prepare('run', `INSERT INTO "sponsorTimes" VALUES(?, ?, ?)`, [videoID, hashedIP, timeSubmitted]);
             
                 // Clear redis cache for this video
-                redis.delAsync(skipSegmentsKey(videoID));
+                QueryCacher.clearVideoCache({
-                redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
+                    videoID,
+                    hashedVideoID,
+                    service,
+                    userID
+                });
             } catch (err) {
                 //a DB change probably occurred
                 res.sendStatus(500);

src/routes/postWarning.ts

@@ -32,7 +32,7 @@ export async function postWarning(req: Request, res: Response) {
             return;
         }
     } else {
-        await db.prepare('run', 'UPDATE "warnings" SET "enabled" = 0 WHERE "userID" = ? AND "issuerUserID" = ?', [userID, issuerUserID]);
+        await db.prepare('run', 'UPDATE "warnings" SET "enabled" = 0 WHERE "userID" = ?', [userID]);
         resultStatus = "removed from";
     }
 

src/routes/setUsername.ts

@@ -40,6 +40,19 @@ export async function setUsername(req: Request, res: Response) {
         userID = getHash(userID);
     }
 
+    try {
+        const row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ? AND "locked" = '1'`, [userID]);
+        if (adminUserIDInput === undefined && row.count > 0) {
+            res.sendStatus(200);
+            return;
+        }
+    }
+    catch (error) {
+        Logger.error(error);
+        res.sendStatus(500);
+        return;
+    }
+
     try {
         //check if username is already set
         let row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -49,7 +62,7 @@ export async function setUsername(req: Request, res: Response) {
             await db.prepare('run', `UPDATE "userNames" SET "userName" = ? WHERE "userID" = ?`, [userName, userID]);
         } else {
             //add to the db
-            await db.prepare('run', `INSERT INTO "userNames" VALUES(?, ?)`, [userID, userName]);
+            await db.prepare('run', `INSERT INTO "userNames"("userID", "userName") VALUES(?, ?)`, [userID, userName]);
         }
 
         res.sendStatus(200);

src/routes/shadowBanUser.ts

@@ -1,6 +1,10 @@
-import {db, privateDB} from '../databases/databases';
+import {db} from '../databases/databases';
 import {getHash} from '../utils/getHash';
 import {Request, Response} from 'express';
+import { config } from '../config';
+import { Category, Service, VideoID, VideoIDHash } from '../types/segments.model';
+import { UserID } from '../types/user.model';
+import { QueryCacher } from '../utils/queryCacher';
 
 export async function shadowBanUser(req: Request, res: Response) {
     const userID = req.query.userID as string;
@@ -8,12 +12,15 @@ export async function shadowBanUser(req: Request, res: Response) {
     let adminUserIDInput = req.query.adminUserID as string;
 
     const enabled = req.query.enabled === undefined
-        ? false
+        ? true
         : req.query.enabled === 'true';
 
     //if enabled is false and the old submissions should be made visible again
     const unHideOldSubmissions = req.query.unHideOldSubmissions !== "false";
 
+    const categories: string[] = req.query.categories ? JSON.parse(req.query.categories as string) : config.categoryList;
+    categories.filter((category) => typeof category === "string" && !(/[^a-z|_|-]/.test(category)));
+
     if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined)) {
         //invalid request
         res.sendStatus(400);
@@ -32,23 +39,30 @@ export async function shadowBanUser(req: Request, res: Response) {
 
     if (userID) {
         //check to see if this user is already shadowbanned
-        const row = await privateDB.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
+        const row = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
 
         if (enabled && row.userCount == 0) {
             //add them to the shadow ban list
 
             //add it to the table
-            await privateDB.prepare('run', `INSERT INTO "shadowBannedUsers" VALUES(?)`, [userID]);
+            await db.prepare('run', `INSERT INTO "shadowBannedUsers" VALUES(?)`, [userID]);
 
             //find all previous submissions and hide them
             if (unHideOldSubmissions) {
-                await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ?
+                await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
                                 AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
                                 "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
+               
+                // clear cache for all old videos
+                (await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]))
+                    .forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => {
+                        QueryCacher.clearVideoCache(videoInfo);
+                    }
+                );
             }
         } else if (!enabled && row.userCount > 0) {
             //remove them from the shadow ban list
-            await privateDB.prepare('run', `DELETE FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
+            await db.prepare('run', `DELETE FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
 
             //find all previous submissions and unhide them
             if (unHideOldSubmissions) {
@@ -60,8 +74,15 @@ export async function shadowBanUser(req: Request, res: Response) {
 
                 await Promise.all(allSegments.filter((item: {uuid: string}) => {
                     return segmentsToIgnore.indexOf(item) === -1;
-                }).map((UUID: string) => {
+                }).map(async (UUID: string) => {
-                    return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ?`, [UUID]);
+                    // collect list for unshadowbanning
+                    (await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ? AND "shadowHidden" = 1 AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]))
+                        .forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => {
+                            QueryCacher.clearVideoCache(videoInfo);  
+                        }
+                    );
+
+                    return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]);
                 }));
             }
         }
@@ -85,7 +106,7 @@ export async function shadowBanUser(req: Request, res: Response) {
             }
         } /*else if (!enabled && row.userCount > 0) {
             // //remove them from the shadow ban list
-            // await privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]);
+            // await db.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]);
 
             // //find all previous submissions and unhide them
             // if (unHideOldSubmissions) {

src/routes/voteOnSponsorTime.ts

@@ -2,27 +2,34 @@ import {Request, Response} from 'express';
 import {Logger} from '../utils/logger';
 import {isUserVIP} from '../utils/isUserVIP';
 import fetch from 'node-fetch';
-import {YouTubeAPI} from '../utils/youtubeApi';
+import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi';
 import {db, privateDB} from '../databases/databases';
 import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from '../utils/webhookUtils';
-import {isUserTrustworthy} from '../utils/isUserTrustworthy';
 import {getFormattedTime} from '../utils/getFormattedTime';
 import {getIP} from '../utils/getIP';
 import {getHash} from '../utils/getHash';
 import {config} from '../config';
 import { UserID } from '../types/user.model';
-import redis from '../utils/redis';
+import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
-import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
+import { getCategoryActionType } from '../utils/categoryInfo';
-import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
+import { QueryCacher } from '../utils/queryCacher';
 
 const voteTypes = {
     normal: 0,
     incorrect: 1,
 };
 
+enum VoteWebhookType {
+    Normal,
+    Rejected
+}
+
 interface FinalResponse {
+    blockVote: boolean,
     finalStatus: number
-    finalMessage: string
+    finalMessage: string,
+    webhookType: VoteWebhookType,
+    webhookMessage: string
 }
 
 interface VoteData {
@@ -53,96 +60,102 @@ async function sendWebhooks(voteData: VoteData) {
     if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) {
         let webhookURL: string = null;
         if (voteData.voteTypeEnum === voteTypes.normal) {
-            webhookURL = config.discordReportChannelWebhookURL;
+            switch (voteData.finalResponse.webhookType) {
+                case VoteWebhookType.Normal:
+                    webhookURL = config.discordReportChannelWebhookURL;
+                    break;
+                case VoteWebhookType.Rejected:
+                    webhookURL = config.discordFailedReportChannelWebhookURL;
+                    break;
+            }
+            
         } else if (voteData.voteTypeEnum === voteTypes.incorrect) {
             webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
         }
 
-        if (config.youtubeAPIKey !== null) {
+        if (config.newLeafURLs !== null) {
-            YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => {
+            const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID);
-                if (err || data.items.length === 0) {
+            if (err) return;
-                    err && Logger.error(err.toString());
+
-                    return;
+            const isUpvote = voteData.incrementAmount > 0;
-                }
+            // Send custom webhooks
-                const isUpvote = voteData.incrementAmount > 0;
+            dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
-                // Send custom webhooks
+                "user": {
-                dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
+                    "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
+                },
+                "video": {
+                    "id": submissionInfoRow.videoID,
+                    "title": data?.title,
+                    "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
+                    "thumbnail": getMaxResThumbnail(data) || null,
+                },
+                "submission": {
+                    "UUID": voteData.UUID,
+                    "views": voteData.row.views,
+                    "category": voteData.category,
+                    "startTime": submissionInfoRow.startTime,
+                    "endTime": submissionInfoRow.endTime,
                     "user": {
-                        "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
+                        "UUID": submissionInfoRow.userID,
-                    },
+                        "username": submissionInfoRow.userName,
-                    "video": {
+                        "submissions": {
-                        "id": submissionInfoRow.videoID,
+                            "total": submissionInfoRow.count,
-                        "title": data.items[0].snippet.title,
+                            "ignored": submissionInfoRow.disregarded,
-                        "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
-                        "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
-                    },
-                    "submission": {
-                        "UUID": voteData.UUID,
-                        "views": voteData.row.views,
-                        "category": voteData.category,
-                        "startTime": submissionInfoRow.startTime,
-                        "endTime": submissionInfoRow.endTime,
-                        "user": {
-                            "UUID": submissionInfoRow.userID,
-                            "username": submissionInfoRow.userName,
-                            "submissions": {
-                                "total": submissionInfoRow.count,
-                                "ignored": submissionInfoRow.disregarded,
-                            },
                         },
                     },
-                    "votes": {
+                },
-                        "before": voteData.row.votes,
+                "votes": {
-                        "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount),
+                    "before": voteData.row.votes,
-                    },
+                    "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount),
-                });
+                },
-
-                // Send discord message
-                if (webhookURL !== null && !isUpvote) {
-                    fetch(webhookURL, {
-                        method: 'POST',
-                        body: JSON.stringify({
-                            "embeds": [{
-                                "title": data.items[0].snippet.title,
-                                "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
-                                    + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
-                                "description": "**" + voteData.row.votes + " Votes Prior | " +
-                                    (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views
-                                    + " Views**\n\n**Submission ID:** " + voteData.UUID
-                                    + "\n**Category:** " + submissionInfoRow.category
-                                    + "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID
-                                    + "\n\n**Total User Submissions:** " + submissionInfoRow.count
-                                    + "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded
-                                    + "\n\n**Timestamp:** " +
-                                    getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
-                                "color": 10813440,
-                                "author": {
-                                    "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
-                                },
-                                "thumbnail": {
-                                    "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
-                                },
-                            }],
-                        }),
-                        headers: {
-                            'Content-Type': 'application/json'
-                        }
-                    })
-                    .then(async res => {
-                        if (res.status >= 400) {
-                            Logger.error("Error sending reported submission Discord hook");
-                            Logger.error(JSON.stringify((await res.text())));
-                            Logger.error("\n");
-                        }
-                    })
-                    .catch(err => {
-                        Logger.error("Failed to send reported submission Discord hook.");
-                        Logger.error(JSON.stringify(err));
-                        Logger.error("\n");
-                    });
-                }
-
             });
+
+            // Send discord message
+            if (webhookURL !== null && !isUpvote) {
+                fetch(webhookURL, {
+                    method: 'POST',
+                    body: JSON.stringify({
+                        "embeds": [{
+                            "title": data?.title,
+                            "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
+                                + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
+                            "description": "**" + voteData.row.votes + " Votes Prior | " +
+                                (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views
+                                + " Views**\n\n**Submission ID:** " + voteData.UUID
+                                + "\n**Category:** " + submissionInfoRow.category
+                                + "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID
+                                + "\n\n**Total User Submissions:** " + submissionInfoRow.count
+                                + "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded
+                                + "\n\n**Timestamp:** " +
+                                getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
+                            "color": 10813440,
+                            "author": {
+                                "name": voteData.finalResponse?.webhookMessage ??
+                                        voteData.finalResponse?.finalMessage ?? 
+                                        getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
+                            },
+                            "thumbnail": {
+                                "url": getMaxResThumbnail(data) || "",
+                            },
+                        }],
+                    }),
+                    headers: {
+                        'Content-Type': 'application/json'
+                    }
+                })
+                .then(async res => {
+                    if (res.status >= 400) {
+                        Logger.error("Error sending reported submission Discord hook");
+                        Logger.error(JSON.stringify((await res.text())));
+                        Logger.error("\n");
+                    }
+                })
+                .catch(err => {
+                    Logger.error("Failed to send reported submission Discord hook.");
+                    Logger.error(JSON.stringify(err));
+                    Logger.error("\n");
+                });
+            }
         }
     }
 }
@@ -158,8 +171,8 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
         return;
     }
 
-    const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
+    const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`,
-                             [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
+                             [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID};
     if (!videoInfo) {
         // Submission doesn't exist
         res.status(400).send("Submission doesn't exist.");
@@ -170,6 +183,10 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
         res.status(400).send("Category doesn't exist.");
         return;
     }
+    if (getCategoryActionType(category) !== CategoryActionType.Skippable) {
+        res.status(400).send("Cannot vote for this category");
+        return;
+    }
 
     const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]);
 
@@ -226,7 +243,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
         }
     }
 
-    clearRedisCache(videoInfo);
+    QueryCacher.clearVideoCache(videoInfo);
 
     res.sendStatus(finalResponse.finalStatus);
 }
@@ -253,8 +270,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
 
     // To force a non 200, change this early
     let finalResponse: FinalResponse = {
+        blockVote: false,
         finalStatus: 200,
-        finalMessage: null
+        finalMessage: null,
+        webhookType: VoteWebhookType.Normal,
+        webhookMessage: null
     }
 
     //x-forwarded-for if this server is behind a proxy
@@ -277,8 +297,9 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
                                     ' where "UUID" = ?', [UUID]));
 
         if (await isSegmentLocked() || await isVideoLocked()) {
-            finalResponse.finalStatus = 403;
+            finalResponse.blockVote = true;
-            finalResponse.finalMessage = "Vote rejected: A moderator has decided that this segment is correct"
+            finalResponse.webhookType = VoteWebhookType.Rejected
+            finalResponse.webhookMessage = "Vote rejected: A moderator has decided that this segment is correct"
         }
     }
 
@@ -312,7 +333,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         return res.status(403).send('Vote rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
     }
 
-    const voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;
+    const voteTypeEnum = (type == 0 || type == 1 || type == 20) ? voteTypes.normal : voteTypes.incorrect;
 
     try {
         //check if vote has already happened
@@ -362,8 +383,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         }
 
         //check if the increment amount should be multiplied (downvotes have more power if there have been many views)
-        const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as 
+        const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as 
-                        {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
+                        {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number, userID: UserID};
 
         if (voteTypeEnum === voteTypes.normal) {
             if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
@@ -381,9 +402,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
 
         // Only change the database if they have made a submission before and haven't voted recently
         const ableToVote = isVIP
-            || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
+            || (!(isOwnSubmission && incrementAmount > 0)
-                && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
+                && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
+                && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
                 && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
+                && !finalResponse.blockVote
                 && finalResponse.finalStatus === 200;
 
         if (ableToVote) {
@@ -403,7 +426,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
 
             //update the vote count on this sponsorTime
             //oldIncrementAmount will be zero is row is null
-            await db.prepare('run', 'UPDATE "sponsorTimes" SET ' + columnName + ' = ' + columnName + ' + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]);
+            await db.prepare('run', 'UPDATE "sponsorTimes" SET "' + columnName + '" = "' + columnName + '" + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]);
             if (isVIP && incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
                 // Lock this submission
                 await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 1 WHERE "UUID" = ?', [UUID]);
@@ -412,32 +435,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
                  await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
             }
 
-            clearRedisCache(videoInfo);
+            QueryCacher.clearVideoCache(videoInfo);
-
-            //for each positive vote, see if a hidden submission can be shown again
-            if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
-                //find the UUID that submitted the submission that was voted on
-                const submissionUserIDInfo = await db.prepare('get', 'SELECT "userID" FROM "sponsorTimes" WHERE "UUID" = ?', [UUID]);
-                if (!submissionUserIDInfo) {
-                    // They are voting on a non-existent submission
-                    res.status(400).send("Voting on a non-existent submission");
-                    return;
-                }
-
-                const submissionUserID = submissionUserIDInfo.userID;
-
-                //check if any submissions are hidden
-                const hiddenSubmissionsRow = await db.prepare('get', 'SELECT count(*) as "hiddenSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" > 0', [submissionUserID]);
-
-                if (hiddenSubmissionsRow.hiddenSubmissions > 0) {
-                    //see if some of this users submissions should be visible again
-
-                    if (await isUserTrustworthy(submissionUserID)) {
-                        //they are trustworthy again, show 2 of their submissions again, if there are two to show
-                        await db.prepare('run', 'UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE ROWID IN (SELECT ROWID FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = 1 LIMIT 2)', [submissionUserID]);
-                    }
-                }
-            }
         }
 
         res.status(finalResponse.finalStatus).send(finalResponse.finalMessage ?? undefined);
@@ -461,11 +459,4 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
 
         res.status(500).json({error: 'Internal error creating segment vote'});
     }
-}
+}
-
-function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
-    if (videoInfo) {
-        redis.delAsync(skipSegmentsKey(videoInfo.videoID));
-        redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
-    }
-}

src/types/config.model.ts

@@ -6,8 +6,9 @@ export interface SBSConfig {
     mockPort?: number;
     globalSalt: string;
     adminUserID: string;
-    youtubeAPIKey?: string;
+    newLeafURLs?: string[];
     discordReportChannelWebhookURL?: string;
+    discordFailedReportChannelWebhookURL?: string;
     discordFirstTimeSubmissionsWebhookURL?: string;
     discordCompletelyIncorrectReportWebhookURL?: string;
     neuralBlockURL?: string;

src/types/segments.model.ts

@@ -1,10 +1,11 @@
 import { HashedValue } from "./hash.model";
 import { SBRecord } from "./lib.model";
+import { UserID } from "./user.model";
 
 export type SegmentUUID = string  & { __segmentUUIDBrand: unknown };
 export type VideoID = string & { __videoIDBrand: unknown };
 export type VideoDuration = number & { __videoDurationBrand: unknown };
-export type Category = string & { __categoryBrand: unknown };
+export type Category = ("sponsor" | "selfpromo" | "interaction" | "intro" | "outro" | "preview" | "music_offtopic" | "highlight") & { __categoryBrand: unknown };
 export type VideoIDHash = VideoID & HashedValue;
 export type IPAddress = string & { __ipAddressBrand: unknown };
 export type HashedIP = IPAddress & HashedValue;
@@ -42,11 +43,13 @@ export interface DBSegment {
     startTime: number;
     endTime: number;
     UUID: SegmentUUID;
+    userID: UserID;
     votes: number;
     locked: boolean;
     shadowHidden: Visibility;
     videoID: VideoID;
     videoDuration: VideoDuration;
+    reputation: number;
     hashedVideoID: VideoIDHash;
 }
 
@@ -54,10 +57,12 @@ export interface OverlappingSegmentGroup {
     segments: DBSegment[],
     votes: number;
     locked: boolean; // Contains a locked segment
+    reputation: number;
 }
 
 export interface VotableObject {
     votes: number;
+    reputation: number;
 }
 
 export interface VotableObjectWithWeight extends VotableObject {
@@ -72,4 +77,9 @@ export interface VideoData {
 export interface SegmentCache {
     shadowHiddenSegmentIPs: SBRecord<VideoID, {hashedIP: HashedIP}[]>,
     userHashedIP?: HashedIP
+}
+
+export enum CategoryActionType {
+    Skippable,
+    POI
 }

src/types/youtubeApi.model.ts

@@ -0,0 +1,111 @@
+export interface APIVideoData {
+    "title": string,
+    "videoId": string,
+    "videoThumbnails": [
+        {
+        "quality": string,
+        "url": string,
+        second__originalUrl: string,
+        "width": number,
+        "height": number
+        }
+    ],
+    
+    "description": string,
+    "descriptionHtml": string,
+    "published": number,
+    "publishedText": string,
+    
+    "keywords": string[],
+    "viewCount": number,
+    "likeCount": number,
+    "dislikeCount": number,
+    
+    "paid": boolean,
+    "premium": boolean,
+    "isFamilyFriendly": boolean,
+    "allowedRegions": string[],
+    "genre": string,
+    "genreUrl": string,
+    
+    "author": string,
+    "authorId": string,
+    "authorUrl": string,
+    "authorThumbnails": [
+        {
+        "url": string,
+        "width": number,
+        "height": number
+        }
+    ],
+    
+    "subCountText": string,
+    "lengthSeconds": number,
+    "allowRatings": boolean,
+    "rating": number,
+    "isListed": boolean,
+    "liveNow": boolean,
+    "isUpcoming": boolean,
+    "premiereTimestamp"?: number,
+    
+    "hlsUrl"?: string,
+    "adaptiveFormats": [
+        {
+        "index": string,
+        "bitrate": string,
+        "init": string,
+        "url": string,
+        "itag": string,
+        "type": string,
+        "clen": string,
+        "lmt": string,
+        "projectionType": number,
+        "container": string,
+        "encoding": string,
+        "qualityLabel"?: string,
+        "resolution"?: string
+        }
+    ],
+    "formatStreams": [
+        {
+        "url": string,
+        "itag": string,
+        "type": string,
+        "quality": string,
+        "container": string,
+        "encoding": string,
+        "qualityLabel": string,
+        "resolution": string,
+        "size": string
+        }
+    ],
+    "captions": [
+        {
+        "label": string,
+        "languageCode": string,
+        "url": string
+        }
+    ],
+    "recommendedVideos": [
+        {
+        "videoId": string,
+        "title": string,
+        "videoThumbnails": [
+            {
+            "quality": string,
+            "url": string,
+            "width": number,
+            "height": number
+            }
+        ],
+        "author": string,
+        "lengthSeconds": number,
+        "viewCountText": string
+        }
+    ]
+}
+
+export interface APIVideoInfo {
+    err: string | boolean,
+    data?: APIVideoData
+}

src/utils/categoryInfo.ts

@@ -0,0 +1,10 @@
+import { Category, CategoryActionType } from "../types/segments.model";
+
+export function getCategoryActionType(category: Category): CategoryActionType {
+    switch (category) {
+        case "highlight":
+            return CategoryActionType.POI;
+        default:
+            return CategoryActionType.Skippable;
+    }
+}

src/utils/queryCacher.ts

@@ -0,0 +1,36 @@
+import redis from "../utils/redis";
+import { Logger } from "../utils/logger";
+import { skipSegmentsHashKey, skipSegmentsKey, reputationKey } from "./redisKeys";
+import { Service, VideoID, VideoIDHash } from "../types/segments.model";
+import { UserID } from "../types/user.model";
+
+async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
+    const {err, reply} = await redis.getAsync(key);
+
+    if (!err && reply) {
+        try {
+            Logger.debug("Got data from redis: " + reply);
+            return JSON.parse(reply);
+        } catch (e) {
+            // If all else, continue on to fetching from the database
+        }
+    }
+
+    const data = await fetchFromDB();
+
+    redis.setAsync(key, JSON.stringify(data));
+    return data;
+}
+
+function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }) {
+    if (videoInfo) {
+        redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
+        redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
+        if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID));
+    }
+}
+
+export const QueryCacher = {
+    get,
+    clearVideoCache
+}

src/utils/redisKeys.ts

@@ -1,8 +1,9 @@
 import { Service, VideoID, VideoIDHash } from "../types/segments.model";
-import { Logger } from "../utils/logger";
+import { UserID } from "../types/user.model";
+import { Logger } from "./logger";
 
-export function skipSegmentsKey(videoID: VideoID): string {
+export function skipSegmentsKey(videoID: VideoID, service: Service): string {
-    return "segments-" + videoID;
+    return "segments." + service + ".videoID." + videoID;
 } 
 
 export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
@@ -10,4 +11,8 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
     if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
     
     return "segments." + service + "." + hashedVideoIDPrefix;
-} 
+} 
+
+export function reputationKey(userID: UserID): string {
+    return "reputation.user." + userID;
+}

src/utils/reputation.ts

@@ -0,0 +1,59 @@
+import { db } from "../databases/databases";
+import { UserID } from "../types/user.model";
+import { QueryCacher } from "./queryCacher";
+import { reputationKey } from "./redisKeys";
+
+interface ReputationDBResult {
+    totalSubmissions: number,
+    downvotedSubmissions: number,
+    nonSelfDownvotedSubmissions: number,
+    upvotedSum: number,
+    lockedSum: number,
+    oldUpvotedSubmissions: number
+}
+
+export async function getReputation(userID: UserID): Promise<number> {
+    const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
+    // 1596240000000 is August 1st 2020, a little after auto upvote was disabled
+    const fetchFromDB = () => db.prepare("get", 
+            `SELECT COUNT(*) AS "totalSubmissions",
+                SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions",
+                SUM(CASE WHEN "votes" < 0 AND "videoID" NOT IN 
+                    (SELECT b."videoID" FROM "sponsorTimes" as b 
+					 	WHERE b."userID" = ?
+                            AND b."votes" > 0 AND b."category" = "a"."category" AND b."videoID" = "a"."videoID" LIMIT 1)
+					THEN 1 ELSE 0 END) AS "nonSelfDownvotedSubmissions",
+                SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum",
+                SUM(locked) AS "lockedSum",
+                SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions"
+            FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, pastDate, userID]) as Promise<ReputationDBResult>;
+
+    const result = await QueryCacher.get(fetchFromDB, reputationKey(userID));
+    
+    // Grace period
+    if (result.totalSubmissions < 5) {
+        return 0;
+    }
+
+    const downvoteRatio = result.downvotedSubmissions / result.totalSubmissions;
+    if (downvoteRatio > 0.3) {
+        return convertRange(Math.min(downvoteRatio, 0.7), 0.3, 0.7, -0.5, -2.5);
+    }
+
+    const nonSelfDownvoteRatio = result.nonSelfDownvotedSubmissions / result.totalSubmissions;
+    if (nonSelfDownvoteRatio > 0.05) {
+        return convertRange(Math.min(nonSelfDownvoteRatio, 0.4), 0.05, 0.4, -0.5, -2.5);
+    }
+
+    if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) {
+        return 0;
+    }
+
+    return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20);
+}
+
+function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number {
+    const currentRange = currentMax - currentMin;  
+    const targetRange = targetMax - targetMin;
+    return ((value - currentMin) / currentRange) * targetRange + targetMin;
+}

src/utils/webhookUtils.ts

@@ -1,6 +1,7 @@
 import {config} from '../config';
 import {Logger} from '../utils/logger';
 import fetch from 'node-fetch';
+import AbortController from "abort-controller";
 
 function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string {
     if (isOwnSubmission) {
@@ -30,7 +31,8 @@ function dispatchEvent(scope: string, data: any): void {
     let webhooks = config.webhooks;
     if (webhooks === undefined || webhooks.length === 0) return;
     Logger.debug("Dispatching webhooks");
-    webhooks.forEach(webhook => {
+
+    for (const webhook of webhooks) {
         let webhookURL = webhook.url;
         let authKey = webhook.key;
         let scopes = webhook.scopes || [];
@@ -43,13 +45,13 @@ function dispatchEvent(scope: string, data: any): void {
                 "Authorization": authKey,
                 "Event-Type": scope, // Maybe change this in the future?
                 'Content-Type': 'application/json'
-            },
+            }
         })
         .catch(err => {
             Logger.warn('Couldn\'t send webhook to ' + webhook.url);
             Logger.warn(err);
         });
-    });
+    }
 }
 
 export {

src/utils/youtubeApi.ts

@@ -1,52 +1,56 @@
+import fetch from 'node-fetch';
 import {config} from '../config';
 import {Logger} from './logger';
 import redis from './redis';
-// @ts-ignore
+import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
-import _youTubeAPI from 'youtube-api';
-
-_youTubeAPI.authenticate({
-    type: "key",
-    key: config.youtubeAPIKey,
-});
 
 export class YouTubeAPI {
-    static listVideos(videoID: string, callback: (err: string | boolean, data: any) => void) {
+    static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
-        const part = 'contentDetails,snippet';
         if (!videoID || videoID.length !== 11 || videoID.includes(".")) {
-            callback("Invalid video ID", undefined);
+            return { err: "Invalid video ID" };
-            return;
         }
 
-        const redisKey = "youtube.video." + videoID;
+        const redisKey = "yt.newleaf.video." + videoID;
-        redis.get(redisKey, (getErr, result) => {
+        if (!ignoreCache) {
-            if (getErr || !result) {
+            const {err, reply} =  await redis.getAsync(redisKey);
+
+            if (!err && reply) {
                 Logger.debug("redis: no cache for video information: " + videoID);
-                _youTubeAPI.videos.list({
+
-                    part,
+                return { err: err?.message, data: JSON.parse(reply) }
-                    id: videoID,
+            }
-                }, (ytErr: boolean | string, { data }: any) => {
+        }
-                    if (!ytErr) {
+
-                        // Only set cache if data returned
+        if (!config.newLeafURLs || config.newLeafURLs.length <= 0) return {err: "NewLeaf URL not found", data: null};
-                        if (data.items.length > 0) {
+
-                            redis.set(redisKey, JSON.stringify(data), (setErr) => {
+        try {
-                                if (setErr) {
+            const result = await fetch(config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)] + "/api/v1/videos/" + videoID, { method: "GET" });
-                                    Logger.warn(setErr.message);
+
-                                } else {
+            if (result.ok) {
-                                    Logger.debug("redis: video information cache set for: " + videoID);
+                const data = await result.json();
-                                }
+                if (data.error) {
-                                callback(false, data); // don't fail
+                    Logger.warn("NewLeaf API Error for " + videoID + ": " + data.error)
-                            });
+                    return { err: data.error, data: null };
-                        } else {
+                }
-                            callback(false, data); // don't fail
+
-                        }
+                redis.setAsync(redisKey, JSON.stringify(data)).then((result) => {
+                    if (result?.err) {
+                        Logger.warn(result?.err.message);
                     } else {
-                        callback(ytErr, data);
+                        Logger.debug("redis: video information cache set for: " + videoID);
                     }
                 });
+                
+                return { err: false, data };
             } else {
-                Logger.debug("redis: fetched video information from cache: " + videoID);
+                return { err: result.statusText, data: null };
-                callback(getErr?.message, JSON.parse(result));
             }
-        });
+        } catch (err) {
-    };
+            return {err, data: null}
+        }       
+    }
 }
+
+export function getMaxResThumbnail(apiInfo: APIVideoData): string | void {
+    return apiInfo?.videoThumbnails?.find((elem) => elem.quality === "maxres")?.second__originalUrl;
+}

test.json

@@ -2,8 +2,8 @@
     "port": 8080,
     "mockPort": 8081,
     "globalSalt": "testSalt",
-    "adminUserID": "testUserId",
+    "adminUserID": "4bdfdc9cddf2c7d07a8a87b57bf6d25389fb75d1399674ee0e0938a6a60f4c3b",
-    "youtubeAPIKey": "",
+    "newLeafURLs": ["placeholder"],
     "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
     "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
     "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",
@@ -40,16 +40,9 @@
                 "vote.up",
                 "vote.down"
             ]
-        }, {
-            "url": "http://unresolvable.host:8081/FailedWebhook",
-            "key": "superSecretKey",
-            "scopes": [
-                "vote.up",
-                "vote.down"
-            ]
         }
     ],
-    "categoryList": ["sponsor", "selfpromo", "interaction",  "intro", "outro", "preview", "music_offtopic"],
+    "categoryList": ["sponsor", "selfpromo", "interaction",  "intro", "outro", "preview", "music_offtopic", "highlight"],
     "maxNumberOfActiveWarnings": 3,
     "hoursAfterWarningExpires": 24,
     "rateLimit": {

test/cases/getSegmentInfo.ts

@@ -0,0 +1,312 @@
+import fetch from 'node-fetch';
+import {db} from '../../src/databases/databases';
+import {Done, getbaseURL} from '../utils';
+import {getHash} from '../../src/utils/getHash';
+
+const ENOENTID =        "0000000000000000000000000000000000000000000000000000000000000000"
+const upvotedID =       "a000000000000000000000000000000000000000000000000000000000000000"
+const downvotedID =     "b000000000000000000000000000000000000000000000000000000000000000"
+const lockedupID =      "c000000000000000000000000000000000000000000000000000000000000000"
+const infvotesID =      "d000000000000000000000000000000000000000000000000000000000000000"
+const shadowhiddenID =  "e000000000000000000000000000000000000000000000000000000000000000"
+const lockeddownID =    "f000000000000000000000000000000000000000000000000000000000000000"
+const hiddenID =        "1000000000000000000000000000000000000000000000000000000000000000"
+const fillerID1 =       "1100000000000000000000000000000000000000000000000000000000000000"
+const fillerID2 =       "1200000000000000000000000000000000000000000000000000000000000000"
+const fillerID3 =       "1300000000000000000000000000000000000000000000000000000000000000"
+const fillerID4 =       "1400000000000000000000000000000000000000000000000000000000000000"
+const fillerID5 =       "1500000000000000000000000000000000000000000000000000000000000000"
+const oldID =           "a0000000-0000-0000-0000-000000000000"
+
+describe('getSegmentInfo', () => {
+    before(async () => {
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('upvoted', 1, 10, 2, 0, '" + upvotedID+ "', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('upvoted', 1) + "')");
+        await db.prepare("run", startOfQuery + "('downvoted', 1, 10, -2, 0, '" + downvotedID+ "', 'testman', 0, 50, 'sponsor', 'YouTube', 120, 0, 0, '" + getHash('downvoted', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked-up', 1, 10, 2, 1, '"+ lockedupID +"', 'testman', 0, 50, 'sponsor', 'YouTube', 101, 0, 0, '" + getHash('locked-up', 1) + "')");
+        await db.prepare("run", startOfQuery + "('infvotes', 1, 10, 100000, 0, '"+infvotesID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 101, 0, 0, '" + getHash('infvotes', 1) + "')");
+        await db.prepare("run", startOfQuery + "('hidden', 1, 10, 2, 0, '"+hiddenID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 1, 0, '" + getHash('hidden', 1) + "')");
+        await db.prepare("run", startOfQuery + "('shadowhidden', 1, 10, 2, 0, '"+shadowhiddenID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 1, '" + getHash('shadowhidden', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked-down', 1, 10, -2, 1, '"+lockeddownID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('locked-down', 1) + "')");
+        await db.prepare("run", startOfQuery + "('oldID', 1, 10, 1, 0, '"+oldID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('oldID', 1) + "')");
+
+        await db.prepare("run", startOfQuery + "('filler', 1, 2, 1, 0, '"+fillerID1+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')");
+        await db.prepare("run", startOfQuery + "('filler', 2, 3, 1, 0, '"+fillerID2+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')");
+        await db.prepare("run", startOfQuery + "('filler', 3, 4, 1, 0, '"+fillerID3+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')");
+        await db.prepare("run", startOfQuery + "('filler', 4, 5, 1, 0, '"+fillerID4+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')");
+        await db.prepare("run", startOfQuery + "('filler', 5, 6, 1, 0, '"+fillerID5+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')");
+    });
+
+    it('Should be able to retreive upvoted segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${upvotedID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "upvoted" && data[0].votes === 2) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive downvoted segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${downvotedID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "downvoted" && data[0].votes === -2) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive locked up segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${lockedupID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "locked-up" && data[0].locked === 1 && data[0].votes === 2) {
+                    done();
+                } else {
+                    done ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive infinite vote segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${infvotesID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "infvotes" && data[0].votes === 100000) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive shadowhidden segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${shadowhiddenID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "shadowhidden" && data[0].shadowHidden === 1) {
+                    done();
+                } else {
+                    done ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive locked down segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${lockeddownID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "locked-down" && data[0].votes === -2 && data[0].locked === 1) {
+                    done();
+                } else {
+                    done ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive hidden segment', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${hiddenID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "hidden" && data[0].hidden === 1) {
+                    done();
+                } else {
+                    done ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive segment with old ID', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${oldID}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "oldID" && data[0].votes === 1) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive single segment in array', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}"]`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 1 && data[0].videoID === "upvoted" && data[0].votes === 2) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to retreive multiple segments in array', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}", "${downvotedID}"]`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 2 &&
+                    (data[0].videoID === "upvoted" && data[0].votes === 2) &&
+                    (data[1].videoID === "downvoted" && data[1].votes === -2)) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be possible to send unexpected query parameters', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${upvotedID}&fakeparam=hello&category=sponsor`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data[0].videoID === "upvoted" && data[0].votes === 2) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should return 400 if array passed to UUID', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=["${upvotedID}", "${downvotedID}"]`)
+        .then(res => {
+            if (res.status !== 400) done("non 400 respone code: " + res.status);
+            else done(); // pass
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should return 400 if bad array passed to UUIDs', (done: Done) => {
+        fetch(getbaseURL() + "/api/segmentInfo?UUIDs=[not-quoted,not-quoted]")
+        .then(res => {
+            if (res.status !== 400) done("non 404 respone code: " + res.status);
+            else done(); // pass
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should return 400 if bad UUID passed', (done: Done) => {
+        fetch(getbaseURL() + "/api/segmentInfo?UUID=notarealuuid")
+        .then(res => {
+            if (res.status !== 400) done("non 400 respone code: " + res.status);
+            else done(); // pass
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should return 400 if bad UUIDs passed in array', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["notarealuuid", "anotherfakeuuid"]`)
+        .then(res => {
+            if (res.status !== 400) done("non 400 respone code: " + res.status);
+            else done(); // pass
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should return good UUID when mixed with bad UUIDs', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}", "anotherfakeuuid"]`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 1 && data[0].videoID === "upvoted" && data[0].votes === 2) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should cut off array at 10', (done: Done) => {
+        const filledIDArray = `["${upvotedID}", "${downvotedID}", "${lockedupID}", "${shadowhiddenID}", "${lockeddownID}", "${hiddenID}", "${fillerID1}", "${fillerID2}", "${fillerID3}", "${fillerID4}", "${fillerID5}"]`
+        fetch(getbaseURL() + `/api/segmentInfo?UUIDs=${filledIDArray}`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                // last segment should be fillerID4
+                if (data.length === 10 && data[0].videoID === "upvoted" && data[0].votes === 2 && data[9].videoID === "filler" && data[9].UUID === fillerID4) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should not duplicate reponses', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${downvotedID}"]`)
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 2 && data[0].videoID === "upvoted" && data[0].votes === 2 && data[1].videoID === "downvoted" && data[1].votes === -2) {
+                    done();
+                } else {
+                    done("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
+    it('Should return 400 if UUID not found', (done: Done) => {
+        fetch(getbaseURL() + `/api/segmentInfo?UUID=${ENOENTID}`)
+        .then(res => {
+            if (res.status !== 400) done("non 400 respone code: " + res.status);
+            else done(); // pass
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+});

test/cases/getSkipSegments.ts

@@ -21,111 +21,104 @@ describe('getSkipSegments', () => {
     });
 
 
-    it('Should be able to get a time by category 1', () => {
+    it('Should be able to get a time by category 1', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
                     && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + (await res.text()));
+                    done("Received incorrect body: " + (await res.text()));
                 }
             }
         })
         .catch(err => "Couldn't call endpoint");
     });
 
-    it('Should be able to get a time by category for a different service 1', () => {
+    it('Should be able to get a time by category for a different service 1', (done: Done) => {
-        fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube")
+        fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest2&category=sponsor&service=PeerTube")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
                     && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + (await res.text()));
+                    done("Received incorrect body: " + (await res.text()));
                 }
             }
         })
         .catch(err => "Couldn't call endpoint");
     });
 
-    it('Should be able to get a time by category 2', () => {
+    it('Should be able to get a time by category 2', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=intro")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
                     && data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + (await res.text()));
+                    done("Received incorrect body: " + (await res.text()));
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should be able to get a time by categories array', () => {
+    it('Should be able to get a time by categories array', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[\"sponsor\"]")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
                     && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + (await res.text()));
+                    done("Received incorrect body: " + (await res.text()));
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should be able to get a time by categories array 2', () => {
+    it('Should be able to get a time by categories array 2', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[\"intro\"]")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
                     && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + (await res.text()));
+                    done("Received incorrect body: " + (await res.text()));
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should be empty if all submissions are hidden', () => {
+    it('Should return 404 if all submissions are hidden', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments")
-        .then(async res => {
+        .then(res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 404) done("non 404 respone code: " + res.status);
-            else {
+            else done(); // pass
-                const data = await res.json();
-                if (data.length === 0) {
-                    return;
-                } else {
-                    return ("Received incorrect body: " + (await res.text()));
-                }
-            }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should be able to get multiple times by category', () => {
+    it('Should be able to get multiple times by category', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=multiple&categories=[\"intro\"]")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200)done("Status code was: " + res.status);
             else {
                 const body = await res.text();
                 const data = JSON.parse(body);
@@ -133,28 +126,28 @@ describe('getSkipSegments', () => {
                     let success = true;
                     for (const segment of data) {
                         if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
-                            || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) &&
+                            || segment.category !== "intro" || segment.UUID !== "1-uuid-7") &&
                             (segment.segment[0] !== 1 || segment.segment[1] !== 11
-                                || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) {
+                                || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) {
                             success = false;
                             break;
                         }
                     }
 
-                    if (success) return;
+                    if (success) done();
-                    else return ("Received incorrect body: " + body);
+                    else done("Received incorrect body: " + body);
                 } else {
-                    return ("Received incorrect body: " + body);
+                    done("Received incorrect body: " + body);
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint\n\n" + err));
     });
 
-    it('Should be able to get multiple times by multiple categories', () => {
+    it('Should be able to get multiple times by multiple categories', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[\"sponsor\", \"intro\"]")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const body = await res.text();
                 const data = JSON.parse(body);
@@ -171,91 +164,99 @@ describe('getSkipSegments', () => {
                         }
                     }
 
-                    if (success) return;
+                    if (success) done();
-                    else return ("Received incorrect body: " + body);
+                    else done("Received incorrect body: " + body);
                 } else {
-                    return ("Received incorrect body: " + body);
+                    done("Received incorrect body: " + body);
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should be possible to send unexpected query parameters', () => {
+    it('Should be possible to send unexpected query parameters', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&fakeparam=hello&category=sponsor")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const body = await res.text();
                 const data = JSON.parse(body);
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
                     && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + body);
+                    done("Received incorrect body: " + body);
                 }
             }
         })
-        .catch(err => ("Couldn't call endpoint"));
+        .catch(err => done("Couldn't call endpoint"));
     });
 
-    it('Low voted submissions should be hidden', () => {
+    it('Low voted submissions should be hidden', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=test3&category=sponsor")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done("Status code was: " + res.status);
             else {
                 const body = await res.text();
                 const data = JSON.parse(body);
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
                     && data[0].category === "sponsor" && data[0].UUID === "1-uuid-4") {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + body);
+                    done("Received incorrect body: " + body);
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should return 404 if no segment found', () => {
+    it('Should return 404 if no segment found', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=notarealvideo")
         .then(res => {
-            if (res.status !== 404) return ("non 404 respone code: " + res.status);
+            if (res.status !== 404) done("non 404 respone code: " + res.status);
-            else return; // pass
+            else done(); // pass
         })
         .catch(err => ("couldn't call endpoint"));
     });
 
+    it('Should return 400 if bad categories argument', (done: Done) => {
+        fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[not-quoted,not-quoted]")
+        .then(res => {
+            if (res.status !== 400) done("non 400 respone code: " + res.status);
+            else done(); // pass
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
 
-    it('Should be able send a comma in a query param', () => {
+    it('Should be able send a comma in a query param', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest,test&category=sponsor")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done ("Status code was: " + res.status);
             else {
                 const body = await res.text();
                 const data = JSON.parse(body);
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
                     && data[0].category === "sponsor" && data[0].UUID === "1-uuid-1") {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + body);
+                    done("Received incorrect body: " + body);
                 }
             }
         })
         .catch(err => ("Couldn't call endpoint"));
     });
 
-    it('Should always get locked segment', () => {
+    it('Should always get locked segment', (done: Done) => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=locked&category=intro")
         .then(async res => {
-            if (res.status !== 200) return ("Status code was: " + res.status);
+            if (res.status !== 200) done ("Status code was: " + res.status);
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
                     && data[0].category === "intro" && data[0].UUID === "1-uuid-locked-8") {
-                    return;
+                    done();
                 } else {
-                    return ("Received incorrect body: " + (await res.text()));
+                    done("Received incorrect body: " + (await res.text()));
                 }
             }
         })

test/cases/getSkipSegmentsByHash.ts

@@ -19,6 +19,8 @@ describe('getSegmentsByHash', () => {
         await db.prepare("run", query, ['getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash']);
         await db.prepare("run", query, ['getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b']);
         await db.prepare("run", query, ['onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, 'f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3']);
+        await db.prepare("run", query, ['highlightVid', 60, 60, 2, 'highlightVid-1', 'testman', 0, 50, 'highlight', 'YouTube', 0, 0, getHash('highlightVid', 1)]);
+        await db.prepare("run", query, ['highlightVid', 70, 70, 2, 'highlightVid-2', 'testman', 0, 50, 'highlight', 'YouTube', 0, 0, getHash('highlightVid', 1)]);
     });
 
     it('Should be able to get a 200', (done: Done) => {
@@ -158,7 +160,7 @@ describe('getSegmentsByHash', () => {
             if (res.status !== 200) done("non 200 status code, was " + res.status);
             else {
                 const body = await res.json();
-                if (body.length !== 1) done("expected 2 videos, got " + body.length);
+                if (body.length !== 1) done("expected 1 video, got " + body.length);
                 else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
                 else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor");
                 else done();
@@ -167,6 +169,20 @@ describe('getSegmentsByHash', () => {
         .catch(err => done("Couldn't call endpoint"));
     });
 
+    it('Should only return one segment when fetching highlight segments', (done: Done) => {
+        fetch(getbaseURL() + '/api/skipSegments/c962?category=highlight')
+        .then(async res => {
+            if (res.status !== 200) done("non 200 status code, was " + res.status);
+            else {
+                const body = await res.json();
+                if (body.length !== 1) done("expected 1 video, got " + body.length);
+                else if (body[0].segments.length !== 1) done("expected 1 segment, got " + body[0].segments.length);
+                else done();
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
     it('Should be able to post a segment and get it using endpoint', (done: Done) => {
         let testID = 'abc123goodVideo';
         fetch(getbaseURL() + "/api/postVideoSponsorTimes", {

test/cases/getUserInfo.ts

@@ -9,14 +9,14 @@ describe('getUserInfo', () => {
         await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_01"), 'Username user 01']);
 
         const sponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
-        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000001', getHash("getuserinfo_user_01"), 0, 10, 'sponsor', 0]);
+        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000001', getHash("getuserinfo_user_01"), 1, 10, 'sponsor', 0]);
-        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000002', getHash("getuserinfo_user_01"), 0, 10, 'sponsor', 0]);
+        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000002', getHash("getuserinfo_user_01"), 2, 10, 'sponsor', 0]);
-        await db.prepare("run", sponsorTimesQuery, ['yyyxxxzzz', 1, 11, -1, 'uuid000003', getHash("getuserinfo_user_01"), 0, 10, 'sponsor', 0]);
+        await db.prepare("run", sponsorTimesQuery, ['yyyxxxzzz', 1, 11, -1, 'uuid000003', getHash("getuserinfo_user_01"), 3, 10, 'sponsor', 0]);
-        await db.prepare("run", sponsorTimesQuery, ['yyyxxxzzz', 1, 11, -2, 'uuid000004', getHash("getuserinfo_user_01"), 0, 10, 'sponsor', 1]);
+        await db.prepare("run", sponsorTimesQuery, ['yyyxxxzzz', 1, 11, -2, 'uuid000004', getHash("getuserinfo_user_01"), 4, 10, 'sponsor', 1]);
-        await db.prepare("run", sponsorTimesQuery, ['xzzzxxyyy', 1, 11, -5, 'uuid000005', getHash("getuserinfo_user_01"), 0, 10, 'sponsor', 1]);
+        await db.prepare("run", sponsorTimesQuery, ['xzzzxxyyy', 1, 11, -5, 'uuid000005', getHash("getuserinfo_user_01"), 5, 10, 'sponsor', 1]);
-        await db.prepare("run", sponsorTimesQuery, ['zzzxxxyyy', 1, 11, 2, 'uuid000006', getHash("getuserinfo_user_02"), 0, 10, 'sponsor', 0]);
+        await db.prepare("run", sponsorTimesQuery, ['zzzxxxyyy', 1, 11, 2, 'uuid000006', getHash("getuserinfo_user_02"), 6, 10, 'sponsor', 0]);
-        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000007', getHash("getuserinfo_user_02"), 0, 10, 'sponsor', 1]);
+        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000007', getHash("getuserinfo_user_02"), 7, 10, 'sponsor', 1]);
-        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000008', getHash("getuserinfo_user_02"), 0, 10, 'sponsor', 1]);
+        await db.prepare("run", sponsorTimesQuery, ['xxxyyyzzz', 1, 11, 2, 'uuid000008', getHash("getuserinfo_user_02"), 8, 10, 'sponsor', 1]);
 
         const insertWarningQuery = 'INSERT INTO warnings ("userID", "issueTime", "issuerUserID", "enabled") VALUES (?, ?, ?, ?)';
         await db.prepare("run", insertWarningQuery, [getHash('getuserinfo_warning_0'), 10, 'getuserinfo_vip', 1]);
@@ -25,7 +25,7 @@ describe('getUserInfo', () => {
     });
 
     it('Should be able to get a 200', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_01')
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_user_01')
         .then(res => {
             if (res.status !== 200) done('non 200 (' + res.status + ')');
             else done(); // pass
@@ -34,7 +34,7 @@ describe('getUserInfo', () => {
     });
 
     it('Should be able to get a 400 (No userID parameter)', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo')
+        fetch(getbaseURL() + '/api/userInfo')
         .then(res => {
             if (res.status !== 400) done('non 400 (' + res.status + ')');
             else done(); // pass
@@ -42,8 +42,8 @@ describe('getUserInfo', () => {
         .catch(err => done('couldn\'t call endpoint'));
     });
 
-    it('Should done(info', (done: Done) => {
+    it('Should be able to get user info', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_01')
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_user_01')
         .then(async res => {
             if (res.status !== 200) {
                 done("non 200");
@@ -55,8 +55,16 @@ describe('getUserInfo', () => {
                     done('Returned incorrect minutesSaved "' + data.minutesSaved + '"');
                 } else if (data.viewCount !== 30) {
                     done('Returned incorrect viewCount "' + data.viewCount + '"');
+                } else if (data.ignoredViewCount !== 20) {
+                    done('Returned incorrect ignoredViewCount "' + data.ignoredViewCount + '"');
                 } else if (data.segmentCount !== 3) {
                     done('Returned incorrect segmentCount "' + data.segmentCount + '"');
+                } else if (data.ignoredSegmentCount !== 2) {
+                    done('Returned incorrect ignoredSegmentCount "' + data.ignoredSegmentCount + '"');
+                } else if (data.reputation !== -2) {
+                    done('Returned incorrect reputation "' + data.reputation + '"');
+                } else if (data.lastSegmentID !== "uuid000005") {
+                    done('Returned incorrect last segment "' + data.lastSegmentID + '"');
                 } else {
                     done(); // pass
                 }
@@ -66,7 +74,7 @@ describe('getUserInfo', () => {
     });
 
     it('Should get warning data', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_0')
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_0')
         .then(async res => {
             if (res.status !== 200) {
                 done('non 200 (' + res.status + ')');
@@ -79,8 +87,22 @@ describe('getUserInfo', () => {
         .catch(err => ("couldn't call endpoint"));
     });
 
+    it('Should get warning data with public ID', (done: Done) => {
+        fetch(getbaseURL() + '/api/userInfo?publicUserID=' + getHash("getuserinfo_warning_0"))
+        .then(async res => {
+            if (res.status !== 200) {
+                done('non 200 (' + res.status + ')');
+            } else {
+                const data = await res.json();
+                if (data.warnings !== 1) done('wrong number of warnings: ' + data.warnings + ', not ' + 1);
+                else done();
+            }
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
+
     it('Should get multiple warnings', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_1')
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_1')
         .then(async res => {
             if (res.status !== 200) {
                 done('non 200 (' + res.status + ')');
@@ -93,8 +115,8 @@ describe('getUserInfo', () => {
         .catch(err => ("couldn't call endpoint"));
     });
 
-    it('Should not get warnings if noe', (done: Done) => {
+    it('Should not get warnings if none', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_2')
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_2')
         .then(async res => {
             if (res.status !== 200) {
                 done('non 200 (' + res.status + ')');
@@ -108,7 +130,7 @@ describe('getUserInfo', () => {
     });
 
     it('Should done(userID for userName (No userName set)', (done: Done) => {
-        fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_02')
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_user_02')
         .then(async res => {
             if (res.status !== 200) {
                 done('non 200 (' + res.status + ')');
@@ -122,4 +144,18 @@ describe('getUserInfo', () => {
         })
         .catch(err => ('couldn\'t call endpoint'));
     });
+
+    it('Should return null segment if none', (done: Done) => {
+        fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_null')
+        .then(async res => {
+            if (res.status !== 200) {
+                done('non 200 (' + res.status + ')');
+            } else {
+                const data = await res.json();
+                if (data.lastSegmentID !== null) done('returned segment ' + data.warnings + ', not ' + null);
+                else done(); // pass
+            }
+        })
+        .catch(err => ("couldn't call endpoint"));
+    });
 });

test/cases/postClearCache.ts

@@ -0,0 +1,72 @@
+import fetch from 'node-fetch';
+import {Done, getbaseURL} from '../utils';
+import {db} from '../../src/databases/databases';
+import {getHash} from '../../src/utils/getHash';
+
+describe('postClearCache', () => {
+    before(async () => {
+        await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("clearing-vip") + "')");
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('clear-test', 0, 1, 2, 'clear-uuid', 'testman', 0, 50, 'sponsor', 0, '" + getHash('clear-test', 1) + "')");
+    });
+
+    it('Should be able to clear cache for existing video', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/clearCache?userID=clearing-vip&videoID=clear-test", {
+            method: 'POST'
+        })
+        .then(res => {
+            if (res.status === 200) done();
+            else done("Status code was " + res.status);
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to clear cache for nonexistent video', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/clearCache?userID=clearing-vip&videoID=dne-video", {
+            method: 'POST'
+        })
+        .then(res => {
+            if (res.status === 200) done();
+            else done("Status code was " + res.status);
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should get 403 as non-vip', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/clearCache?userID=regular-user&videoID=clear-tes", {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 403) done('non 403 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should give 400 with missing videoID', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/clearCache?userID=clearing-vip", {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 400) done('non 400 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should give 400 with missing userID', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/clearCache?userID=clearing-vip", {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 400) done('non 400 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(err => done(err));
+    });
+});

test/cases/postSkipSegments.ts

@@ -118,7 +118,7 @@ describe('postSkipSegments', () => {
         .then(async res => {
             if (res.status === 200) {
                 const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]);
-                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) {
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 4980) {
                     done();
                 } else {
                     done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
@@ -140,7 +140,7 @@ describe('postSkipSegments', () => {
             body: JSON.stringify({
                 userID: "test",
                 videoID: "dQw4w9WgXZH",
-                videoDuration: 5010.20,
+                videoDuration: 4980.20,
                 segments: [{
                     segment: [1, 10],
                     category: "sponsor",
@@ -150,7 +150,7 @@ describe('postSkipSegments', () => {
         .then(async res => {
             if (res.status === 200) {
                 const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]);
-                if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) {
+                if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 4980.20) {
                     done();
                 } else {
                     done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
@@ -237,6 +237,21 @@ describe('postSkipSegments', () => {
         }
     });
 
+    it('Should still not be allowed if youtube thinks duration is 0', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", {
+                method: 'POST',
+        })
+        .then(async res => {
+            if (res.status === 403) done(); // pass
+            else {
+                const body = await res.text();
+                done("non 403 status code: " + res.status + " (" + body + ")");
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
     it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {
@@ -500,6 +515,51 @@ describe('postSkipSegments', () => {
         .catch(err => done("Couldn't call endpoint"));
     });
 
+    it('Should be rejected if segment starts and ends at the same time', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/skipSegments?videoID=qqwerty&startTime=90&endTime=90&userID=testing&category=intro", {
+            method: 'POST',
+        })
+        .then(async res => {
+            if (res.status === 400) done(); // pass
+            else {
+                const body = await res.text();
+                done("non 400 status code: " + res.status + " (" + body + ")");
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be accepted if highlight segment starts and ends at the same time', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30&userID=testing&category=highlight", {
+            method: 'POST',
+        })
+        .then(async res => {
+            if (res.status === 200) done(); // pass
+            else {
+                const body = await res.text();
+                done("non 200 status code: " + res.status + " (" + body + ")");
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be rejected if highlight segment doesn\'t start and end at the same time', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=highlight", {
+            method: 'POST',
+        })
+        .then(async res => {
+            if (res.status === 400) done(); // pass
+            else {
+                const body = await res.text();
+                done("non 400 status code: " + res.status + " (" + body + ")");
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
     it('Should be rejected if a sponsor is less than 1 second', (done: Done) => {
         fetch(getbaseURL()
             + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", {
@@ -621,34 +681,6 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
-    it('Should be allowed if youtube thinks duration is 0', (done: Done) => {
-        fetch(getbaseURL()
-            + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", {
-                method: 'POST',
-        })
-        .then(async res => {
-            if (res.status === 200) done(); // pass
-            else {
-                const body = await res.text();
-                done("non 200 status code: " + res.status + " (" + body + ")");
-            }
-        })
-        .catch(err => done("Couldn't call endpoint"));
-    });
-
-    it('Should be rejected if not a valid videoID', (done: Done) => {
-        fetch(getbaseURL()
-            + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing")
-        .then(async res => {
-            if (res.status === 403) done(); // pass
-            else {
-                const body = await res.text();
-                done("non 403 status code: " + res.status + " (" + body + ")");
-            }
-        })
-        .catch(err => done("Couldn't call endpoint"));
-    });
-
     it('Should return 400 for missing params (Params method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", {

test/cases/postWarning.ts

@@ -101,7 +101,7 @@ describe('postWarning', () => {
         .catch(err => done(err));
     });
 
-    it('Should not be able to create warning if vip (exp 403)', (done: Done) => {
+    it('Should not be able to create warning if not vip (exp 403)', (done: Done) => {
         let json = {
             issuerUserID: 'warning-not-vip',
             userID: 'warning-1',

test/cases/reputation.ts

@@ -0,0 +1,123 @@
+import assert from 'assert';
+import { db } from '../../src/databases/databases';
+import { UserID } from '../../src/types/user.model';
+import { getHash } from '../../src/utils/getHash';
+import { getReputation } from '../../src/utils/reputation';
+
+const userIDLowSubmissions = "reputation-lowsubmissions" as UserID;
+const userIDHighDownvotes = "reputation-highdownvotes" as UserID;
+const userIDHighNonSelfDownvotes = "reputation-highnonselfdownvotes" as UserID;
+const userIDNewSubmissions = "reputation-newsubmissions" as UserID;
+const userIDLowSum = "reputation-lowsum" as UserID;
+const userIDHighRepBeforeManualVote = "reputation-oldhighrep" as UserID;
+const userIDHighRep = "reputation-highrep" as UserID;
+const userIDHighRepAndLocked = "reputation-highlockedrep" as UserID;
+
+describe('reputation', () => {
+    before(async () => {
+        const videoID = "reputation-videoID";
+
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-0', '${getHash(userIDLowSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-1', '${getHash(userIDLowSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 100, 0, 'reputation-0-uuid-2', '${getHash(userIDLowSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-1-uuid-0', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-1', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-2', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-3', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-4', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-uuid-5', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-6', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-7', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+
+        // Each downvote is on a different video (ie. they didn't resubmit to fix their downvote)
+        await db.prepare("run", startOfQuery + `('${videoID}A', 1, 11, 2, 0, 'reputation-1-1-uuid-0', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        // Different category, same video
+        await db.prepare("run", startOfQuery + `('${videoID}A', 1, 11, -2, 0, 'reputation-1-1-uuid-1', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'intro', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-2', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-3', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-4', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-1-uuid-5', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-6', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-7', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+    
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-0', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-1', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-2', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-3', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-4', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-2-uuid-5', '${getHash(userIDNewSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-6', '${getHash(userIDNewSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-7', '${getHash(userIDNewSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-3-uuid-0', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-1', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-2', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-3', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-4', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-3-uuid-5', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-6', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-7', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-0', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-1', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-2', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-3', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-4', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-4-uuid-5', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-6', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-7', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-0', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-1', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-2', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-3', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-4', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-5-uuid-5', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-5-uuid-6', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-5-uuid-7', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+    
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-0', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-1', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-2', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-3', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-6-uuid-4', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-6-uuid-5', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-6-uuid-6', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+        await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-6-uuid-7', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
+    });
+
+    it("user in grace period", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDLowSubmissions)), 0);
+    });
+
+    it("user with high downvote ratio", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDHighDownvotes)), -2.125);
+    });
+
+    it("user with high non self downvote ratio", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDHighNonSelfDownvotes)), -1.6428571428571428);
+    });
+
+    it("user with mostly new submissions", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDNewSubmissions)), 0);
+    });
+
+    it("user with not enough vote sum", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDLowSum)), 0);
+    });
+
+    it("user with lots of old votes (before autovote was disabled) ", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDHighRepBeforeManualVote)), 0);
+    });
+
+    it("user with high reputation", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDHighRep)), 0.24137931034482757);
+    });
+
+    it("user with high reputation and locked segments", async () => {
+        assert.strictEqual(await getReputation(getHash(userIDHighRepAndLocked)), 1.8413793103448277);
+    });
+
+});

test/cases/setUsername.ts

@@ -0,0 +1,184 @@
+import fetch from 'node-fetch';
+import { Done, getbaseURL } from '../utils';
+import { db } from '../../src/databases/databases';
+import { getHash } from '../../src/utils/getHash';
+
+const adminPrivateUserID = 'testUserId';
+const user01PrivateUserID = 'setUsername_01';
+const username01 = 'Username 01';
+const user02PrivateUserID = 'setUsername_02';
+const username02 = 'Username 02';
+const user03PrivateUserID = 'setUsername_03';
+const username03 = 'Username 03';
+const user04PrivateUserID = 'setUsername_04';
+const username04 = 'Username 04';
+const user05PrivateUserID = 'setUsername_05';
+const username05 = 'Username 05';
+const user06PrivateUserID = 'setUsername_06';
+const username06 = 'Username 06';
+const user07PrivateUserID = 'setUsername_07';
+const username07 = 'Username 07';
+
+async function addUsername(userID: string, userName: string, locked = 0) {
+    await db.prepare('run', 'INSERT INTO "userNames" ("userID", "userName", "locked") VALUES(?, ?, ?)', [userID, userName, locked]);
+}
+
+async function getUsername(userID: string) {
+    const row = await db.prepare('get', 'SELECT "userName" FROM "userNames" WHERE userID = ?', [userID]);
+    if (!row) {
+        return null;
+    }
+    return row.userName;
+}
+
+describe('setUsername', () => {
+    before(async () => {
+        await addUsername(getHash(user01PrivateUserID), username01, 0);
+        await addUsername(getHash(user02PrivateUserID), username02, 0);
+        await addUsername(getHash(user03PrivateUserID), username03, 0);
+        await addUsername(getHash(user04PrivateUserID), username04, 1);
+        await addUsername(getHash(user05PrivateUserID), username05, 0);
+        await addUsername(getHash(user06PrivateUserID), username06, 0);
+        await addUsername(getHash(user07PrivateUserID), username07, 1);
+    });
+
+    it('Should return 200', (done: Done) => {
+        fetch(`${getbaseURL()}/api/setUsername?userID=${user01PrivateUserID}&username=Changed%20Username`, {
+            method: 'POST',
+        })
+        .then(res => {
+            if (res.status !== 200) done(`Status code was ${res.status}`);
+            else done(); // pass
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should return 400 for missing param "userID"', (done: Done) => {
+        fetch(`${getbaseURL()}/api/setUsername?username=MyUsername`, {
+            method: 'POST',
+        })
+        .then(res => {
+            if (res.status !== 400) done(`Status code was ${res.status}`);
+            else done(); // pass
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should return 400 for missing param "username"', (done: Done) => {
+        fetch(`${getbaseURL()}/api/setUsername?userID=test`, {
+            method: 'POST',
+        })
+        .then(res => {
+            if (res.status !== 400) done(`Status code was ${res.status}`);
+            else done(); // pass
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should return 400 for "username" longer then 64 characters', (done: Done) => {
+        const username65 = '0000000000000000000000000000000000000000000000000000000000000000X';
+        fetch(`${getbaseURL()}/api/setUsername?userID=test&username=${encodeURIComponent(username65)}`, {
+            method: 'POST',
+        })
+        .then(res => {
+            if (res.status !== 400) done(`Status code was ${res.status}`);
+            else  done(); // pass
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should not change username if it contains "discord"', (done: Done) => {
+        const newUsername = 'discord.me';
+        fetch(`${getbaseURL()}/api/setUsername?userID=${user02PrivateUserID}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            if (res.status !== 200) done(`Status code was ${res.status}`);
+            else {
+                const userName = await getUsername(getHash(user02PrivateUserID));
+                if (userName === newUsername) {
+                    done(`Username '${username02}' got changed to '${newUsername}'`);
+                }
+                else done();
+            }
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should be able to change username', (done: Done) => {
+        const newUsername = 'newUsername';
+        fetch(`${getbaseURL()}/api/setUsername?userID=${user03PrivateUserID}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            const username = await getUsername(getHash(user03PrivateUserID));
+            if (username !== newUsername) done(`Username did not change`);
+            else done();
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should not be able to change locked username', (done: Done) => {
+        const newUsername = 'newUsername';
+        fetch(`${getbaseURL()}/api/setUsername?userID=${user04PrivateUserID}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            const username = await getUsername(getHash(user04PrivateUserID));
+            if (username === newUsername) done(`Username '${username04}' got changed to '${username}'`);
+            else done();
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Should filter out unicode control characters', (done: Done) => {
+        const newUsername = 'This\nUsername+has\tInvalid+Characters';
+        fetch(`${getbaseURL()}/api/setUsername?userID=${user05PrivateUserID}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            const username = await getUsername(getHash(user05PrivateUserID));
+            if (username === newUsername) done(`Username contains unicode control characters`);
+            else  done();
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Incorrect adminUserID should return 403', (done: Done) => {
+        const newUsername = 'New Username';
+        fetch(`${getbaseURL()}/api/setUsername?adminUserID=invalidAdminID&userID=${getHash(user06PrivateUserID)}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            if (res.status !== 403) done(`Status code was ${res.status}`);
+            else done();
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Admin should be able to change username', (done: Done) => {
+        const newUsername = 'New Username';
+        fetch(`${getbaseURL()}/api/setUsername?adminUserID=${adminPrivateUserID}&userID=${getHash(user06PrivateUserID)}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            const username = await getUsername(getHash(user06PrivateUserID));
+            if (username !== newUsername) done(`Failed to change username from '${username06}' to '${newUsername}'`);
+            else done();
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+
+    it('Admin should be able to change locked username', (done: Done) => {
+        const newUsername = 'New Username';
+        fetch(`${getbaseURL()}/api/setUsername?adminUserID=${adminPrivateUserID}&userID=${getHash(user07PrivateUserID)}&username=${encodeURIComponent(newUsername)}`, {
+            method: 'POST',
+        })
+        .then(async res => {
+            const username = await getUsername(getHash(user06PrivateUserID));
+            if (username !== newUsername) done(`Failed to change username from '${username06}' to '${newUsername}'`);
+            else done();
+        })
+        .catch(err => done(`couldn't call endpoint`));
+    });
+});

test/cases/shadowBanUser.ts

@@ -0,0 +1,126 @@
+import fetch from 'node-fetch';
+import {db, privateDB} from '../../src/databases/databases';
+import {Done, getbaseURL} from '../utils';
+import {getHash} from '../../src/utils/getHash';
+
+describe('shadowBanUser', () => {
+    before(() => {
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+        db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-1-uuid-0', 'shadowBanned', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')");
+        db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-1-uuid-0-1', 'shadowBanned', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')");
+        db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-1-uuid-2', 'shadowBanned', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')");
+
+        db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-2-uuid-0', 'shadowBanned2', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')");
+        db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-2-uuid-0-1', 'shadowBanned2', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')");
+        db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-2-uuid-2', 'shadowBanned2', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')");
+
+        db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-3-uuid-0', 'shadowBanned3', 0, 50, 'sponsor', 'YouTube', 100, 0, 1, '" + getHash('testtesttest', 1) + "')");
+        db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-3-uuid-0-1', 'shadowBanned3', 0, 50, 'sponsor', 'PeerTube', 120, 0, 1, '" + getHash('testtesttest2', 1) + "')");
+        db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-3-uuid-2', 'shadowBanned3', 0, 50, 'intro', 'YouTube', 101, 0, 1, '" + getHash('testtesttest', 1) + "')");
+        db.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('shadowBanned3')`);
+
+        db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("shadow-ban-vip") + "')");
+    });
+
+
+    it('Should be able to ban user and hide submissions', (done: Done) => {
+        fetch(getbaseURL() + "/api/shadowBanUser?userID=shadowBanned&adminUserID=shadow-ban-vip", {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned", 1]);
+                const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]);
+
+                if (shadowRow && videoRow?.length === 3) {
+                    done();
+                } else {
+                    done("Ban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow));
+                }
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be able to unban user without unhiding submissions', (done: Done) => {
+        fetch(getbaseURL() + "/api/shadowBanUser?userID=shadowBanned&adminUserID=shadow-ban-vip&enabled=false&unHideOldSubmissions=false", {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned", 1]);
+                const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]);
+
+                if (!shadowRow && videoRow?.length === 3) {
+                    done();
+                } else {
+                    done("Unban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow));
+                }
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be able to ban user and hide submissions from only some categories', (done: Done) => {
+        fetch(getbaseURL() + '/api/shadowBanUser?userID=shadowBanned2&adminUserID=shadow-ban-vip&categories=["sponsor"]', {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const videoRow: {category: string, shadowHidden: number}[] = (await db.prepare('all', `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned2", 1]));
+                const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]);
+
+                if (shadowRow && 2 == videoRow?.length && 2 === videoRow?.filter((elem) => elem?.category === "sponsor")?.length) {
+                    done();
+                } else {
+                    done("Ban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow));
+                }
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be able to unban user and unhide submissions', (done: Done) => {
+        fetch(getbaseURL() + "/api/shadowBanUser?userID=shadowBanned2&adminUserID=shadow-ban-vip&enabled=false", {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned2", 1]);
+                const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]);
+
+                if (!shadowRow && videoRow?.length === 0) {
+                    done();
+                } else {
+                    done("Unban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow));
+                }
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be able to unban user and unhide some submissions', (done: Done) => {
+        fetch(getbaseURL() + `/api/shadowBanUser?userID=shadowBanned3&adminUserID=shadow-ban-vip&enabled=false&categories=["sponsor"]`, {
+            method: 'POST'
+        })
+        .then(async res => {
+            if (res.status !== 200) done("Status code was: " + res.status);
+            else {
+                const videoRow = await db.prepare('all', `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned3", 1]);
+                const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned3"]);
+
+                if (!shadowRow && videoRow?.length === 1 && videoRow[0]?.category === "intro") {
+                    done();
+                } else {
+                    done("Unban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow));
+                }
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+});

test/cases/unBan.ts

@@ -1,14 +1,14 @@
 import fetch from 'node-fetch';
 import * as utils from  '../utils';
 import { getHash } from '../../src/utils/getHash';
-import { db, privateDB } from '../../src/databases/databases';
+import { db } from '../../src/databases/databases';
 
 describe('unBan', () => {
   before(async () => {
     const insertShadowBannedUserQuery = 'INSERT INTO "shadowBannedUsers" VALUES(?)';
-    await privateDB.prepare("run", insertShadowBannedUserQuery, ['testMan-unBan']);
+    await db.prepare("run", insertShadowBannedUserQuery, ['testMan-unBan']);
-    await privateDB.prepare("run", insertShadowBannedUserQuery, ['testWoman-unBan']);
+    await db.prepare("run", insertShadowBannedUserQuery, ['testWoman-unBan']);
-    await privateDB.prepare("run", insertShadowBannedUserQuery, ['testEntity-unBan']);
+    await db.prepare("run", insertShadowBannedUserQuery, ['testEntity-unBan']);
 
     const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
     await db.prepare("run", insertVipUserQuery, [getHash("VIPUser-unBan")]);

test/cases/voteOnSponsorTime.ts

@@ -1,6 +1,6 @@
 import fetch from 'node-fetch';
 import {config} from '../../src/config';
-import {db, privateDB} from '../../src/databases/databases';
+import {db} from '../../src/databases/databases';
 import {Done, getbaseURL} from '../utils';
 import {getHash} from '../../src/utils/getHash';
 import {ImportMock} from 'ts-mock-imports';
@@ -57,7 +57,7 @@ describe('voteOnSponsorTime', () => {
 
 
         await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [getHash("VIPUser")]);
-        await privateDB.prepare("run", 'INSERT INTO "shadowBannedUsers" ("userID") VALUES (?)', [getHash("randomID4")]);
+        await db.prepare("run", 'INSERT INTO "shadowBannedUsers" ("userID") VALUES (?)', [getHash("randomID4")]);
 
         await db.prepare("run", 'INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES (?, ?, ?)', ['no-sponsor-segments-video', 'someUser', 'sponsor']);
     });
@@ -263,6 +263,24 @@ describe('voteOnSponsorTime', () => {
         .catch(err => done(err));
     });
 
+    it('Should not able to change to highlight category', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=highlight")
+        .then(async res => {
+            if (res.status === 400) {
+                let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["incorrect-category"]);
+                if (row.category === "sponsor") {
+                    done();
+                } else {
+                    done("Vote did not succeed. Submission went from sponsor to " + row.category);
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
     it('Should be able to change your vote for a category and it should add your vote to the database', (done: Done) => {
         fetch(getbaseURL()
             + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro")
@@ -427,10 +445,10 @@ describe('voteOnSponsorTime', () => {
             + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0")
         .then(async res => {
             let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
-            if (res.status === 403 && row.votes === 2) {
+            if (res.status === 200 && row.votes === 2) {
                 done();
             } else {
-                done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
+                done("Status code was " + res.status + " instead of 200, row was " + JSON.stringify(row));
             }
         })
         .catch(err => done(err));
@@ -455,10 +473,10 @@ describe('voteOnSponsorTime', () => {
             + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro")
         .then(async res => {
             let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
-            if (res.status === 403 && row.category === "sponsor") {
+            if (res.status === 200 && row.category === "sponsor") {
                 done();
             } else {
-                done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
+                done("Status code was " + res.status + " instead of 200, row was " + JSON.stringify(row));
             }
         })
         .catch(err => done(err));

test/utils.ts

@@ -7,4 +7,4 @@ export function getbaseURL() {
 /**
  * Duplicated from Mocha types. TypeScript doesn't infer that type by itself for some reason.
  */
-export type Done = (err?: any) => void;
+export type Done = (err?: any) => void;

test/youtubeMock.ts

@@ -1,69 +1,51 @@
-/*
+import { APIVideoData, APIVideoInfo } from "../src/types/youtubeApi.model";
-YouTubeAPI.videos.list({
-  part: "snippet",
-  id: videoID
-}, function (err, data) {});
- */
-
-// https://developers.google.com/youtube/v3/docs/videos
-
 
 export class YouTubeApiMock {
-    static listVideos(videoID: string, callback: (ytErr: any, data: any) => void) {
+    static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
         const obj = {
             id: videoID
         };
 
         if (obj.id === "knownWrongID") {
-            callback(undefined, {
+            return {
-                pageInfo: {
+                err: "No video found"
-                    totalResults: 0,
+            };
-                },
-                items: [],
-            });
         }
+
         if (obj.id === "noDuration") {
-            callback(undefined, {
+            return {
-                pageInfo: {
+                err: null,
-                    totalResults: 1,
+                data: {
-                },
+                    title: "Example Title",
-                items: [
+                    lengthSeconds: 0,
-                    {
+                    videoThumbnails: [
-                        contentDetails: {
+                        {
-                            duration: "PT0S",
+                           quality: "maxres",
+                           url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
+                           second__originalUrl:"https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
+                           width: 1280,
+                           height: 720
                         },
-                        snippet: {
+                    ]
-                            title: "Example Title",
+                } as APIVideoData
-                            thumbnails: {
+            };
-                                maxres: {
-                                    url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                                },
-                            },
-                        },
-                    },
-                ],
-            });
         } else {
-            callback(undefined, {
+            return {
-                pageInfo: {
+                err: null,
-                    totalResults: 1,
+                data: {
-                },
+                    title: "Example Title",
-                items: [
+                    lengthSeconds: 4980,
-                    {
+                    videoThumbnails: [
-                        contentDetails: {
+                        {
-                            duration: "PT1H23M30S",
+                           quality: "maxres",
+                           url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
+                           second__originalUrl:"https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
+                           width: 1280,
+                           height: 720
                         },
-                        snippet: {
+                    ]
-                            title: "Example Title",
+                } as APIVideoData
-                            thumbnails: {
+            };
-                                maxres: {
-                                    url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
-                                },
-                            },
-                        },
-                    },
-                ],
-            });
         }
     }
 }