Merge branch 'master' into fix/general-fixes

.eslintrc.js

@@ -28,5 +28,6 @@ module.exports = {
         "quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
         "require-await": "warn",
         "semi": "warn",
+        "no-console": "error"
     },
 };

databases/_sponsorTimes.db.sql

@@ -38,5 +38,6 @@ CREATE TABLE IF NOT EXISTS "config" (
 );
 
 CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
+CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore
 
 COMMIT;

docker/docker-compose.yml

@@ -28,6 +28,15 @@ services:
       - 3241:3000
     volumes:
       - ./newleaf/configuration.py:/workdir/configuration.py
+  rsync:
+    image: mchangrh/rsync:latest
+    container_name: rsync
+    restart: always
+    ports:
+      - 873:873
+    volumes:
+      - ./rsync/rsyncd.conf:/etc/rsyncd.conf
+      - ./database-export/:/mirror
 
 volumes:
   database-data:

docker/rsync/rsyncd.conf

@@ -0,0 +1,15 @@
+pid file = /var/run/rsyncd.pid
+lock file = /var/run/rsync.lock
+log file = /var/log/rsync.log
+# replace with user accessing the files
+
+[sponsorblock]
+use chroot = no
+max connections = 10
+# path to mirrored files
+path = /mirror
+comment = sponsorblock-database
+read only = true
+refuse options = c delete zl
+# disallow checksumming and compression level to reduce CPU/IO load
+# disallow deleting files clientside

nginx/nginx.conf

@@ -17,6 +17,10 @@ http {
 	include		/etc/nginx/mime.types;
 	include		/etc/nginx/proxy.conf;
 	include		/etc/nginx/fastcgi.conf;
+	## Custom MIME definition
+	types {
+		text/csv	csv;
+	}
 
 	upstream backend_GET {
 		least_conn;
@@ -32,6 +36,9 @@ http {
 		server		10.0.0.3:4441;
 		server		10.0.0.3:4442;
 
+		server		10.0.0.5:4441;
+		server		10.0.0.5:4442;
+
 		#server		134.209.69.251:80	backup;
 
 		#server		116.203.32.253:80	backup;
@@ -88,8 +95,8 @@ http {
 		}
 
 		location /test/ {
-			return			404 "";
+			# return			404 "";
-			#proxy_pass		http://localhost:4440/;
+			proxy_pass		http://10.0.0.5:4445/;
 			#proxy_pass		https://sbtest.etcinit.com/;
 		}
 
@@ -124,7 +131,8 @@ http {
 		location /download/ {
 			access_log /etc/nginx/logs/download.log no_ip;
 			gzip on;
-			gzip_types	text/plain application/json;
+			gzip_types	text/csv;
+			gzip_comp_level 1;
 			alias		/home/sbadmin/sponsor/docker/database-export/;
 			#return 307 https://cdnsponsor.ajay.app$request_uri;
 		}

src/databases/databases.ts

@@ -4,7 +4,6 @@ import { Mysql } from "./Mysql";
 import { Postgres } from "./Postgres";
 import { IDatabase } from "./IDatabase";
 
-
 let db: IDatabase;
 let privateDB: IDatabase;
 if (config.mysql) {
@@ -68,6 +67,15 @@ async function initDb(): Promise<void> {
         // Attach private db to main db
         (db as Sqlite).attachDatabase(config.privateDB, "privateDB");
     }
+
+    if (config.mode === "mirror" && db instanceof Postgres) {
+        const tables = config?.dumpDatabase?.tables ?? [];
+        const tableNames = tables.map(table => table.name);
+        for (const table of tableNames) {
+            const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`;
+            await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
 }
 
 export {

src/index.ts

@@ -7,11 +7,15 @@ import { getCommit } from "./utils/getCommit";
 
 async function init() {
     process.on("unhandledRejection", (error: any) => {
+        // eslint-disable-next-line no-console
         console.dir(error?.stack);
         process.exit(1);
     });
 
     await initDb();
+    // edge case clause for creating compatible .db files, do not enable
+    if (config.mode === "init-db-and-exit") process.exit(0);
+    // do not enable init-db-only mode for usage.
     (global as any).HEADCOMMIT = config.mode === "development" ? "development"
         : config.mode === "test" ? "test"
             : getCommit() as string;

src/routes/dumpDatabase.ts

@@ -118,6 +118,12 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
             Then, you can download the csv files below, or use the links returned from the JSON request. 
             A dump will also be triggered by making a request to one of these urls.
 
+            <h3>Keeping your dump up to date</h3>
+
+            If you want a live dump, please do not continually fetch this url. 
+            Please instead use the <a href="https://github.com/mchangrh/sb-mirror">sb-mirror</a> project.
+            This can automatically fetch new data and will not require a redownload each time, saving bandwith.
+
             <h3>Links</h3>
             <table>
                 <thead>
@@ -184,7 +190,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
         res.sendStatus(404);
     }
 
-    await queueDump();
+    if (req.query.generate !== "false") await queueDump();
 }
 
 function updateQueueTime(): void {

src/routes/getLockReason.ts

@@ -8,7 +8,9 @@ const possibleCategoryList = config.categoryList;
 interface lockArray {
     category: Category;
     locked: number,
-    reason: string
+    reason: string,
+    userID: string,
+    userName: string,
 }
 
 export async function getLockReason(req: Request, res: Response): Promise<Response> {
@@ -38,31 +40,49 @@ export async function getLockReason(req: Request, res: Response): Promise<Respon
 
     try {
         // Get existing lock categories markers
-        const row = await db.prepare("all", 'SELECT "category", "reason" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string}[];
+        const row = await db.prepare("all", 'SELECT "category", "reason", "userID" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string, userID: string }[];
         // map to object array
         const locks = [];
-        const lockedCategories = [] as string[];
+        const userIDs = new Set();
         // get all locks for video, check if requested later
         for (const lock of row) {
             locks.push({
                 category: lock.category,
                 locked: 1,
-                reason: lock.reason
+                reason: lock.reason,
+                userID: lock?.userID || "",
+                userName: "",
             } as lockArray);
-            lockedCategories.push(lock.category);
+            userIDs.add(lock.userID);
         }
+        // all userName from userIDs
+        const userNames = await db.prepare(
+            "all",
+            `SELECT "userName", "userID" FROM "userNames" WHERE "userID" IN (${Array.from("?".repeat(userIDs.size)).join()}) LIMIT ?`,
+            [...userIDs, userIDs.size]
+        ) as { userName: string, userID: string }[];
+
+        const results = [];
+        for (const category of searchCategories)  {
+            const lock = locks.find(l => l.category === category);
+            if (lock?.userID) {
+                // mapping userName to locks
+                const user = userNames.find(u => u.userID === lock.userID);
+                lock.userName = user?.userName || "";
+                results.push(lock);
+            } else {
                 // add empty locks for categories requested but not locked
-        const noLockCategories = searchCategories.filter(x => !lockedCategories.includes(x));
+                results.push({
-        for (const noLock of noLockCategories) {
+                    category,
-            locks.push({
-                category: noLock,
                     locked: 0,
-                reason: ""
+                    reason: "",
+                    userID: "",
+                    userName: "",
                 } as lockArray);
             }
-        // return real and fake locks that were requested
+        }
-        const filtered = locks.filter(lock => searchCategories.includes(lock.category));
+
-        return res.send(filtered);
+        return res.send(results);
     } catch (err) {
         Logger.error(err as string);
         return res.sendStatus(500);

src/routes/getStatus.ts

@@ -1,6 +1,7 @@
 import { db } from "../databases/databases";
 import { Logger } from "../utils/logger";
 import { Request, Response } from "express";
+import os from "os";
 
 export async function getStatus(req: Request, res: Response): Promise<Response> {
     const startTime = Date.now();
@@ -14,8 +15,9 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
             db: Number(dbVersion),
             startTime,
             processTime: Date.now() - startTime,
+            loadavg: os.loadavg().slice(1) // only return 5 & 15 minute load average
         };
-        return value ? res.send(String(statusValues[value])) : res.send(statusValues);
+        return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues);
     } catch (err) {
         Logger.error(err as string);
         return res.sendStatus(500);

src/routes/postSkipSegments.ts

@@ -6,7 +6,6 @@ import { getSubmissionUUID } from "../utils/getSubmissionUUID";
 import { getHash } from "../utils/getHash";
 import { getIP } from "../utils/getIP";
 import { getFormattedTime } from "../utils/getFormattedTime";
-import { isUserTrustworthy } from "../utils/isUserTrustworthy";
 import { dispatchEvent } from "../utils/webhookUtils";
 import { Request, Response } from "express";
 import { ActionType, Category, CategoryActionType, IncomingSegment, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model";
@@ -622,13 +621,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
         //check to see if this user is shadowbanned
         const shadowBanRow = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID]);
 
-        let shadowBanned = shadowBanRow.userCount;
-
-        if (!(await isUserTrustworthy(userID))) {
-            //hide this submission as this user is untrustworthy
-            shadowBanned = 1;
-        }
-
         const startingVotes = 0 + decreaseVotes;
         const reputation = await getReputation(userID);
 
@@ -644,7 +636,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
                 await db.prepare("run", `INSERT INTO "sponsorTimes" 
                     ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "userAgent")
                     VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                    videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, segmentInfo.actionType, service, videoDuration, reputation, shadowBanned, hashedVideoID, userAgent
+                    videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, segmentInfo.actionType, service, videoDuration, reputation, 0, hashedVideoID, userAgent
                 ],
                 );
 

src/utils/isUserTrustworthy.ts

@@ -1,20 +0,0 @@
-import { db } from "../databases/databases";
-
-/**
- * Returns true if the user is considered trustworthy. This happens after a user has made 5 submissions and has less than 60% downvoted submissions
- * @param userID
- */
-export async function isUserTrustworthy(userID: string): Promise<boolean> {
-    //check to see if this user how many submissions this user has submitted
-    const totalSubmissionsRow = await db.prepare("get", `SELECT count(*) as "totalSubmissions", sum(votes) as "voteSum" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
-
-    if (totalSubmissionsRow.totalSubmissions > 5) {
-        //check if they have a high downvote ratio
-        const downvotedSubmissionsRow = await db.prepare("get", `SELECT count(*) as "downvotedSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND (votes < 0 OR "shadowHidden" > 0)`, [userID]);
-
-        return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 ||
-            (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions);
-    }
-
-    return true;
-}

src/utils/logger.ts

@@ -79,6 +79,7 @@ class Logger {
         if (levelStr.length === 4) {
             levelStr += " ";  // ensure logs are aligned
         }
+        // eslint-disable-next-line no-console
         console.log(colors.Dim, `${levelStr} ${new Date().toISOString()}: `, color, str, colors.Reset);
     }
 }

test/cases/getLockCategories.ts

@@ -2,6 +2,7 @@ import { getHash } from "../../src/utils/getHash";
 import { db } from "../../src/databases/databases";
 import assert from "assert";
 import { client } from "../utils/httpClient";
+import { mixedDeepEquals } from "../utils/partialDeepEquals";
 const endpoint = "/api/lockCategories";
 const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } });
 const getLockCategoriesWithService = (videoID: string, service: string) => client.get(endpoint, { params: { videoID, service } });
@@ -12,13 +13,13 @@ describe("getLockCategories", () => {
         await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]);
 
         const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)';
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "sponsor", "1-short", "YouTube"]);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "interaction", "1-longer-reason", "YouTube"]);
 
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory2", "preview", "2-reason", "YouTube"]);
 
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason", "PeerTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "nonmusic", "3-reason", "PeerTube"]);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "sponsor", "3-reason", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "sponsor", "3-reason", "YouTube"]);
     });
 
     it("Should update the database version when starting the application", async () => {
@@ -27,7 +28,7 @@ describe("getLockCategories", () => {
     });
 
     it("Should be able to get multiple locks", (done) => {
-        getLockCategories("getLock1")
+        getLockCategories("getLockCategory1")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -37,14 +38,14 @@ describe("getLockCategories", () => {
                     ],
                     reason: "1-longer-reason"
                 };
-                assert.deepStrictEqual(res.data, expected);
+                assert.ok(mixedDeepEquals(res.data, expected));
                 done();
             })
             .catch(err => done(err));
     });
 
     it("Should be able to get single locks", (done) => {
-        getLockCategories("getLock2")
+        getLockCategories("getLockCategory2")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -60,7 +61,7 @@ describe("getLockCategories", () => {
     });
 
     it("should return 404 if no lock exists", (done) => {
-        getLockCategories("getLockNull")
+        getLockCategories("getLockCategoryNull")
             .then(res => {
                 assert.strictEqual(res.status, 404);
                 done();
@@ -78,7 +79,7 @@ describe("getLockCategories", () => {
     });
 
     it("Should be able to get multiple locks with service", (done) => {
-        getLockCategoriesWithService("getLock1", "YouTube")
+        getLockCategoriesWithService("getLockCategory1", "YouTube")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -88,14 +89,14 @@ describe("getLockCategories", () => {
                     ],
                     reason: "1-longer-reason"
                 };
-                assert.deepStrictEqual(res.data, expected);
+                assert.ok(mixedDeepEquals(res.data, expected));
                 done();
             })
             .catch(err => done(err));
     });
 
     it("Should be able to get single locks with service", (done) => {
-        getLockCategoriesWithService("getLock3", "PeerTube")
+        getLockCategoriesWithService("getLockCategory3", "PeerTube")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -111,7 +112,7 @@ describe("getLockCategories", () => {
     });
 
     it("Should be able to get single locks with service", (done) => {
-        getLockCategoriesWithService("getLock3", "Youtube")
+        getLockCategoriesWithService("getLockCategory3", "Youtube")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -127,7 +128,7 @@ describe("getLockCategories", () => {
     });
 
     it("should return result from Youtube service if service not match", (done) => {
-        getLockCategoriesWithService("getLock3", "Dailymotion")
+        getLockCategoriesWithService("getLockCategory3", "Dailymotion")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {

test/cases/getLockReason.ts

@@ -5,19 +5,33 @@ import { client } from "../utils/httpClient";
 
 const endpoint = "/api/lockReason";
 
+const vipUserName1 = "getLockReason-vipUserName_1";
+const vipUserID1 = getHash("getLockReason-vipUserID_1");
+const vipUserName2 = "getLockReason-vipUserName_2";
+const vipUserID2 = getHash("getLockReason-vipUserID_2");
+
 describe("getLockReason", () => {
     before(async () => {
-        const vipUserID = "getLockReasonVIP";
-        const vipUserHash = getHash(vipUserID);
         const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
-        await db.prepare("run", insertVipUserQuery, [vipUserHash]);
+        await db.prepare("run", insertVipUserQuery, [vipUserID1]);
-        await db.prepare("run", insertVipUserQuery, [vipUserHash]);
+        await db.prepare("run", insertVipUserQuery, [vipUserID2]);
+
+        const insertVipUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES (?, ?)';
+        await db.prepare("run", insertVipUserNameQuery, [vipUserID1, vipUserName1]);
+        await db.prepare("run", insertVipUserNameQuery, [vipUserID2, vipUserName2]);
 
         const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)';
-        await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "sponsor", "sponsor-reason"]);
+        await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "sponsor", "sponsor-reason"]);
-        await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "interaction", "interaction-reason"]);
+        await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "interaction", "interaction-reason"]);
-        await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "preview", "preview-reason"]);
+        await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "preview", "preview-reason"]);
-        await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "music_offtopic", "nonmusic-reason"]);
+        await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "music_offtopic", "nonmusic-reason"]);
+        await db.prepare("run", insertLockCategoryQuery, [vipUserID2, "getLockReason", "outro", "outro-reason"]);
+    });
+
+    after(async () => {
+        const deleteUserNameQuery = 'DELETE FROM "userNames" WHERE  "userID" = ? AND  "userName" = ? LIMIT 1';
+        await db.prepare("run", deleteUserNameQuery, [vipUserID1, vipUserName1]);
+        await db.prepare("run", deleteUserNameQuery, [vipUserID2, vipUserName2]);
     });
 
     it("Should update the database version when starting the application", async () => {
@@ -31,7 +45,7 @@ describe("getLockReason", () => {
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = [
-                    { category: "sponsor", locked: 1, reason: "sponsor-reason" }
+                    { category: "sponsor", locked: 1, reason: "sponsor-reason", userID: vipUserID1, userName: vipUserName1 }
                 ];
                 assert.deepStrictEqual(res.data, expected);
                 done();
@@ -44,7 +58,7 @@ describe("getLockReason", () => {
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = [
-                    { category: "intro", locked: 0, reason: "" }
+                    { category: "intro", locked: 0, reason: "", userID: "", userName: "" }
                 ];
                 assert.deepStrictEqual(res.data, expected);
                 done();
@@ -53,12 +67,13 @@ describe("getLockReason", () => {
     });
 
     it("should get multiple locks with array", (done) => {
-        client.get(endpoint, { params: { videoID: "getLockReason", categories: `["intro","sponsor"]` } })
+        client.get(endpoint, { params: { videoID: "getLockReason", categories: `["intro","sponsor","outro"]` } })
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = [
-                    { category: "sponsor", locked: 1, reason: "sponsor-reason" },
+                    { category: "intro", locked: 0, reason: "", userID: "", userName: "" },
-                    { category: "intro", locked: 0, reason: "" }
+                    { category: "sponsor", locked: 1, reason: "sponsor-reason", userID: vipUserID1, userName: vipUserName1 },
+                    { category: "outro", locked: 1, reason: "outro-reason", userID: vipUserID2, userName: vipUserName2 }
                 ];
                 assert.deepStrictEqual(res.data, expected);
                 done();
@@ -71,9 +86,9 @@ describe("getLockReason", () => {
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = [
-                    { category: "interaction", locked: 1, reason: "interaction-reason" },
+                    { category: "interaction", locked: 1, reason: "interaction-reason", userID: vipUserID1, userName: vipUserName1  },
-                    { category: "music_offtopic", locked: 1, reason: "nonmusic-reason" },
+                    { category: "music_offtopic", locked: 1, reason: "nonmusic-reason", userID: vipUserID1, userName: vipUserName1  },
-                    { category: "intro", locked: 0, reason: "" }
+                    { category: "intro", locked: 0, reason: "", userID: "", userName: "" }
                 ];
                 assert.deepStrictEqual(res.data, expected);
                 done();
@@ -86,14 +101,14 @@ describe("getLockReason", () => {
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = [
-                    { category: "sponsor", locked: 1, reason: "sponsor-reason" },
+                    { category: "sponsor", locked: 1, reason: "sponsor-reason", userID: vipUserID1, userName: vipUserName1 },
-                    { category: "interaction", locked: 1, reason: "interaction-reason" },
+                    { category: "selfpromo", locked: 0, reason: "", userID: "", userName: "" },
-                    { category: "preview", locked: 1, reason: "preview-reason" },
+                    { category: "interaction", locked: 1, reason: "interaction-reason", userID: vipUserID1, userName: vipUserName1 },
-                    { category: "music_offtopic", locked: 1, reason: "nonmusic-reason" },
+                    { category: "intro", locked: 0, reason: "", userID: "", userName: "" },
-                    { category: "selfpromo", locked: 0, reason: "" },
+                    { category: "outro", locked: 1, reason: "outro-reason", userID: vipUserID2, userName: vipUserName2 },
-                    { category: "intro", locked: 0, reason: "" },
+                    { category: "preview", locked: 1, reason: "preview-reason", userID: vipUserID1, userName: vipUserName1 },
-                    { category: "outro", locked: 0, reason: "" },
+                    { category: "music_offtopic", locked: 1, reason: "nonmusic-reason", userID: vipUserID1, userName: vipUserName1 },
-                    { category: "poi_highlight", locked: 0, reason: "" }
+                    { category: "poi_highlight", locked: 0, reason: "", userID: "", userName: "" }
                 ];
                 assert.deepStrictEqual(res.data, expected);
                 done();

test/cases/getStatus.ts

@@ -19,6 +19,7 @@ describe("getStatus", () => {
                 assert.strictEqual(data.db, Number(dbVersion));
                 assert.ok(data.startTime);
                 assert.ok(data.processTime >= 0);
+                assert.ok(data.loadavg.length == 2);
                 done();
             })
             .catch(err => done(err));
@@ -74,4 +75,15 @@ describe("getStatus", () => {
             })
             .catch(err => done(err));
     });
+
+    it("Should be able to get loadavg only", (done) => {
+        client.get(`${endpoint}/loadavg`)
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(Number(res.data[0]) >= 0);
+                assert.ok(Number(res.data[1]) >= 0);
+                done();
+            })
+            .catch(err => done(err));
+    });
 });

test/cases/postSkipSegments.ts

@@ -1,6 +1,6 @@
 import { config } from "../../src/config";
 import { getHash } from "../../src/utils/getHash";
-import { partialDeepEquals } from "../utils/partialDeepEquals";
+import { partialDeepEquals, arrayDeepEquals } from "../utils/partialDeepEquals";
 import { db } from "../../src/databases/databases";
 import { ImportMock } from "ts-mock-imports";
 import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
@@ -29,7 +29,7 @@ describe("postSkipSegments", () => {
 
     const submitUserOneHash = getHash(submitUserOne);
     const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`;
-    const warnVideoID = "dQw4w9WgXcF";
+    const warnVideoID = "postSkip2";
     const badInputVideoID = "dQw4w9WgXcQ";
 
     const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
@@ -91,7 +91,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time (Params method)", (done) => {
-        const videoID = "dQw4w9WgXcR";
+        const videoID = "postSkip1";
         postSkipSegmentParam({
             videoID,
             startTime: 2,
@@ -125,7 +125,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcF";
+        const videoID = "postSkip2";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -150,7 +150,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time with an action type (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcV";
+        const videoID = "postSkip3";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -176,7 +176,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should not be able to submit an intro with mute action type (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXpQ";
+        const videoID = "postSkip4";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -196,7 +196,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXZX";
+        const videoID = "postSkip5";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -222,7 +222,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXZH";
+        const videoID = "postSkip6";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -331,7 +331,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time under a different service (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcG";
+        const videoID = "postSkip7";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -383,7 +383,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit multiple times (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcT";
+        const videoID = "postSkip11";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -407,14 +407,14 @@ describe("postSkipSegments", () => {
                     endTime: 60,
                     category: "intro"
                 }];
-                assert.deepStrictEqual(rows, expected);
+                assert.ok(arrayDeepEquals(rows, expected));
                 done();
             })
             .catch(err => done(err));
     }).timeout(5000);
 
     it("Should allow multiple times if total is under 80% of video(JSON method)", (done) => {
-        const videoID = "L_jWHffIx5E";
+        const videoID = "postSkip9";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -452,7 +452,7 @@ describe("postSkipSegments", () => {
                     endTime: 170,
                     category: "sponsor"
                 }];
-                assert.deepStrictEqual(rows, expected);
+                assert.ok(arrayDeepEquals(rows, expected));
                 done();
             })
             .catch(err => done(err));
@@ -505,20 +505,20 @@ describe("postSkipSegments", () => {
             .then(async res => {
                 assert.strictEqual(res.status, 403);
                 const expected = [{
-                    category: "sponsor",
+                    category: "interaction",
-                    startTime: 2000,
+                    startTime: 0,
-                    endTime: 4000
+                    endTime: 1000
                 }, {
-                    category: "sponsor",
+                    category: "interaction",
-                    startTime: 1500,
+                    startTime: 1001,
-                    endTime: 2750
+                    endTime: 1005
                 }, {
-                    category: "sponsor",
+                    category: "interaction",
-                    startTime: 4050,
+                    startTime: 0,
-                    endTime: 4750
+                    endTime: 5000
                 }];
-                const rows = await queryDatabase(videoID);
+                const rows = await db.prepare("all", `SELECT "category", "startTime", "endTime" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
-                assert.notDeepStrictEqual(rows, expected);
+                assert.ok(arrayDeepEquals(rows, expected));
                 done();
             })
             .catch(err => done(err));

test/utils/partialDeepEquals.ts

@@ -22,3 +22,34 @@ export const partialDeepEquals = (actual: Record<string, any>, expected: Record<
     }
     return true;
 };
+
+export const arrayDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
+    if (actual.length !== expected.length) return false;
+    let flag = true;
+    const actualString = JSON.stringify(actual);
+    const expectedString = JSON.stringify(expected);
+    // check every value in arr1 for match in arr2
+    actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; });
+    // check arr2 for match in arr1
+    expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; });
+
+    if (!flag && print) printActualExpected(actual, expected);
+    return flag;
+};
+
+export const mixedDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
+    for (const [ key, value ] of Object.entries(expected)) {
+        // if value is object or array, recurse
+        if (Array.isArray(value)) {
+            if (!arrayDeepEquals(actual?.[key], value, false)) {
+                if (print) printActualExpected(actual, expected);
+                return false;
+            }
+        }
+        else if (actual?.[key] !== value) {
+            if (print) printActualExpected(actual, expected);
+            return false;
+        }
+    }
+    return true;
+};