From 29d2c9c25efb959a40d1c81bab25e7293e0c920e Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 19 Mar 2021 21:31:16 -0400
Subject: [PATCH 01/32] Add new "Service" option

---
 databases/_upgrade_sponsorTimes_7.sql         | 28 ++++++++++++++
 src/routes/getSkipSegments.ts                 | 21 ++++++----
 src/routes/getSkipSegmentsByHash.ts           |  9 ++++-
 src/routes/postSkipSegments.ts                | 22 ++++++-----
 src/types/segments.model.ts                   | 10 +++++
 test/cases/getSkipSegments.ts                 | 38 ++++++++++++++-----
 ...entsByHash.ts => getSkipSegmentsByHash.ts} | 30 ++++++++++++---
 test/cases/postSkipSegments.ts                | 32 ++++++++++++++++
 8 files changed, 155 insertions(+), 35 deletions(-)
 create mode 100644 databases/_upgrade_sponsorTimes_7.sql
 rename test/cases/{getSegmentsByHash.ts => getSkipSegmentsByHash.ts} (80%)

diff --git a/databases/_upgrade_sponsorTimes_7.sql b/databases/_upgrade_sponsorTimes_7.sql
new file mode 100644
index 0000000..8f6060b
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_7.sql
@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_7" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts
index 65ab4f1..42a44c7 100644
--- a/src/routes/getSkipSegments.ts
+++ b/src/routes/getSkipSegments.ts
@@ -4,7 +4,7 @@ import { config } from '../config';
 import { db, privateDB } from '../databases/databases';
 import { skipSegmentsKey } from '../middleware/redisKeys';
 import { SBRecord } from '../types/lib.model';
-import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
+import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
 import { getHash } from '../utils/getHash';
 import { getIP } from '../utils/getIP';
 import { Logger } from '../utils/logger';
@@ -47,7 +47,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
     }));
 }
 
-async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise<Segment[]> {
+async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
     const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
     const segments: Segment[] = [];
 
@@ -59,8 +59,8 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
             .prepare(
                 'all',
                 `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" 
-                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
-                [videoID]
+                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
+                [videoID, service]
             )).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
                 acc[segment.category] = acc[segment.category] || [];
                 acc[segment.category].push(segment);
@@ -81,7 +81,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
     }
 }
 
-async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise<SBRecord<VideoID, VideoData>> {
+async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise<SBRecord<VideoID, VideoData>> {
     const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
     const segments: SBRecord<VideoID, VideoData> = {};
 
@@ -95,8 +95,8 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
             .prepare(
                 'all',
                 `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
-                WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
-                [hashedVideoIDPrefix + '%']
+                WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
+                [hashedVideoIDPrefix + '%', service]
             )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
                     hash: segment.hashedVideoID,
@@ -239,6 +239,11 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
             ? [req.query.category]
             : ['sponsor'];
 
+    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    if (!Object.values(Service).some((val) => val == service)) {
+        service = Service.YouTube;
+    }
+
     // Only 404s are cached at the moment
     const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
     
@@ -251,7 +256,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
         }
     }
 
-    const segments = await getSegmentsByVideoID(req, videoID, categories);
+    const segments = await getSegmentsByVideoID(req, videoID, categories, service);
 
     if (segments === null || segments === undefined) {
         res.sendStatus(500);
diff --git a/src/routes/getSkipSegmentsByHash.ts b/src/routes/getSkipSegmentsByHash.ts
index eb9429c..29194d4 100644
--- a/src/routes/getSkipSegmentsByHash.ts
+++ b/src/routes/getSkipSegmentsByHash.ts
@@ -1,7 +1,7 @@
 import {hashPrefixTester} from '../utils/hashPrefixTester';
 import {getSegmentsByHash} from './getSkipSegments';
 import {Request, Response} from 'express';
-import { Category, VideoIDHash } from '../types/segments.model';
+import { Category, Service, VideoIDHash } from '../types/segments.model';
 
 export async function getSkipSegmentsByHash(req: Request, res: Response) {
     let hashPrefix = req.params.prefix as VideoIDHash;
@@ -25,12 +25,17 @@ export async function getSkipSegmentsByHash(req: Request, res: Response) {
     catch(error) {
         return res.status(400).send("Bad parameter: categories (invalid JSON)");
     }
+
+    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    if (!Object.values(Service).some((val) => val == service)) {
+        service = Service.YouTube;
+    }
     
     // filter out none string elements, only flat array with strings is valid
     categories = categories.filter((item: any) => typeof item === "string");
 
     // Get all video id's that match hash prefix
-    const segments = await getSegmentsByHash(req, hashPrefix, categories);
+    const segments = await getSegmentsByHash(req, hashPrefix, categories, service);
 
     if (!segments) return res.status(404).json([]);
 
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 6a9c781..1101eac 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -13,6 +13,7 @@ import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
 import { skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
+import { Service } from '../types/segments.model';
 
 
 async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
@@ -45,8 +46,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
     });
 }
 
-async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) {
-    if (config.youtubeAPIKey !== null) {
+async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
+    if (config.youtubeAPIKey !== null && service == Service.YouTube) {
         const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
 
         YouTubeAPI.listVideos(videoID, (err: any, data: any) => {
@@ -267,7 +268,10 @@ export async function postSkipSegments(req: Request, res: Response) {
 
     const videoID = req.query.videoID || req.body.videoID;
     let userID = req.query.userID || req.body.userID;
-
+    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    if (!Object.values(Service).some((val) => val == service)) {
+        service = Service.YouTube;
+    }
 
     let segments = req.body.segments;
     if (segments === undefined) {
@@ -367,7 +371,7 @@ export async function postSkipSegments(req: Request, res: Response) {
 
         //check if this info has already been submitted before
         const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ?
-            and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]);
+            and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]);
         if (duplicateCheck2Row.count > 0) {
             res.sendStatus(409);
             return;
@@ -375,7 +379,7 @@ export async function postSkipSegments(req: Request, res: Response) {
     }
 
     // Auto moderator check
-    if (!isVIP) {
+    if (!isVIP && service == Service.YouTube) {
         const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
         if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
             // If NB automod rejects, the submission will start with -2 votes.
@@ -491,9 +495,9 @@ export async function postSkipSegments(req: Request, res: Response) {
             const startingLocked = isVIP ? 1 : 0;
             try {
                 await db.prepare('run', `INSERT INTO "sponsorTimes" 
-                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID")
-                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
+                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "shadowHidden", "hashedVideoID")
+                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, shadowBanned, getHash(videoID, 1),
                     ],
                 );
 
@@ -529,7 +533,7 @@ export async function postSkipSegments(req: Request, res: Response) {
     res.json(newSegments);
 
     for (let i = 0; i < segments.length; i++) {
-        sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
+        sendWebhooks(userID, videoID, UUIDs[i], segments[i], service);
     }
 }
 
diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts
index e7b6eaf..2478b70 100644
--- a/src/types/segments.model.ts
+++ b/src/types/segments.model.ts
@@ -8,6 +8,16 @@ export type VideoIDHash = VideoID & HashedValue;
 export type IPAddress = string & { __ipAddressBrand: unknown };
 export type HashedIP = IPAddress & HashedValue;
 
+// Uncomment as needed
+export enum Service {
+    YouTube = 'YouTube',
+    // Nebula = 'Nebula',
+    PeerTube = 'PeerTube',
+    // RSS = 'RSS',
+    // Corridor = 'Corridor',
+    // Lbry = 'Lbry'
+}
+
 export interface Segment { 
     category: Category; 
     segment: number[]; 
diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts
index 85dcfbb..51f1c1b 100644
--- a/test/cases/getSkipSegments.ts
+++ b/test/cases/getSkipSegments.ts
@@ -5,16 +5,17 @@ import {getHash} from '../../src/utils/getHash';
 
 describe('getSkipSegments', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('testtesttest2', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest,test', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')");
     
         return;
     });
@@ -37,6 +38,23 @@ describe('getSkipSegments', () => {
         .catch(err => "Couldn't call endpoint");
     });
 
+    it('Should be able to get a time by category for a different service 1', () => {
+        fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube")
+        .then(async res => {
+            if (res.status !== 200) return ("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1") {
+                    return;
+                } else {
+                    return ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
     it('Should be able to get a time by category 2', () => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=intro")
         .then(async res => {
diff --git a/test/cases/getSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts
similarity index 80%
rename from test/cases/getSegmentsByHash.ts
rename to test/cases/getSkipSegmentsByHash.ts
index d957e2d..fd45884 100644
--- a/test/cases/getSegmentsByHash.ts
+++ b/test/cases/getSkipSegmentsByHash.ts
@@ -12,11 +12,12 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
 
 describe('getSegmentsByHash', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
     });
 
     it('Should be able to get a 200', (done: Done) => {
@@ -128,7 +129,24 @@ describe('getSegmentsByHash', () => {
                 if (body.length !== 2) done("expected 2 videos, got " + body.length);
                 else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
                 else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
-                else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+                else if (body[0].segments[0].category !== 'sponsor' 
+                        || body[0].segments[0].UUID !== 'getSegmentsByHash-0-0' 
+                        || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+                else done();
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => {
+        fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube')
+        .then(async res => {
+            if (res.status !== 200) done("non 200 status code, was " + res.status);
+            else {
+                const body = await res.json();
+                if (body.length !== 1) done("expected 2 videos, got " + body.length);
+                else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
+                else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor");
                 else done();
             }
         })
diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts
index e31326a..6bc6179 100644
--- a/test/cases/postSkipSegments.ts
+++ b/test/cases/postSkipSegments.ts
@@ -97,6 +97,38 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
+    it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "dQw4w9WgXcG",
+                service: "PeerTube",
+                segments: [{
+                    segment: [0, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]);
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
     it('VIP submission should start locked', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {

From 5544491728d518faffe6146f6f5b818502c108c7 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 19 Mar 2021 22:45:30 -0400
Subject: [PATCH 02/32] Add duration option when submitting and save duration
 in DB

---
 databases/_upgrade_sponsorTimes_8.sql |  29 +++
 src/routes/postSkipSegments.ts        | 338 +++++++++++++-------------
 src/types/segments.model.ts           |   6 +
 test/cases/postSkipSegments.ts        |  68 +++++-
 4 files changed, 271 insertions(+), 170 deletions(-)
 create mode 100644 databases/_upgrade_sponsorTimes_8.sql

diff --git a/databases/_upgrade_sponsorTimes_8.sql b/databases/_upgrade_sponsorTimes_8.sql
new file mode 100644
index 0000000..ccc2ec9
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_8.sql
@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_8" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+    "videoDuration"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 1101eac..3b6f82d 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -13,8 +13,12 @@ import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
 import { skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
-import { Service } from '../types/segments.model';
+import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
 
+interface APIVideoInfo {
+    err: string | boolean,
+    data: any
+}
 
 async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
     const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -46,62 +50,58 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
     });
 }
 
-async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
-    if (config.youtubeAPIKey !== null && service == Service.YouTube) {
+async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
+    if (apiVideoInfo && service == Service.YouTube) {
         const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
 
-        YouTubeAPI.listVideos(videoID, (err: any, data: any) => {
-            if (err || data.items.length === 0) {
-                err && Logger.error(err);
-                return;
+        const {data, err} = apiVideoInfo;
+        if (err) return;
+
+        const startTime = parseFloat(segmentInfo.segment[0]);
+        const endTime = parseFloat(segmentInfo.segment[1]);
+        sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
+            submissionStart: startTime,
+            submissionEnd: endTime,
+        }, segmentInfo);
+
+        // If it is a first time submission
+        // Then send a notification to discord
+        if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
+        
+        fetch(config.discordFirstTimeSubmissionsWebhookURL, {
+            method: 'POST',
+            body: JSON.stringify({
+                "embeds": [{
+                    "title": data.items[0].snippet.title,
+                    "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
+                    "description": "Submission ID: " + UUID +
+                        "\n\nTimestamp: " +
+                        getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
+                        "\n\nCategory: " + segmentInfo.category,
+                    "color": 10813440,
+                    "author": {
+                        "name": userID,
+                    },
+                    "thumbnail": {
+                        "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                    },
+                }],
+            }),
+            headers: {
+                'Content-Type': 'application/json'
             }
-
-            const startTime = parseFloat(segmentInfo.segment[0]);
-            const endTime = parseFloat(segmentInfo.segment[1]);
-            sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
-                submissionStart: startTime,
-                submissionEnd: endTime,
-            }, segmentInfo);
-
-            // If it is a first time submission
-            // Then send a notification to discord
-            if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
-            
-            fetch(config.discordFirstTimeSubmissionsWebhookURL, {
-                method: 'POST',
-                body: JSON.stringify({
-                    "embeds": [{
-                        "title": data.items[0].snippet.title,
-                        "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
-                        "description": "Submission ID: " + UUID +
-                            "\n\nTimestamp: " +
-                            getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
-                            "\n\nCategory: " + segmentInfo.category,
-                        "color": 10813440,
-                        "author": {
-                            "name": userID,
-                        },
-                        "thumbnail": {
-                            "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
-                        },
-                    }],
-                }),
-                headers: {
-                    'Content-Type': 'application/json'
-                }
-            })
-            .then(res => {
-                if (res.status >= 400) {
-                    Logger.error("Error sending first time submission Discord hook");
-                    Logger.error(JSON.stringify(res));
-                    Logger.error("\n");
-                }
-            })
-            .catch(err => {
-                Logger.error("Failed to send first time submission Discord hook.");
-                Logger.error(JSON.stringify(err));
+        })
+        .then(res => {
+            if (res.status >= 400) {
+                Logger.error("Error sending first time submission Discord hook");
+                Logger.error(JSON.stringify(res));
                 Logger.error("\n");
-            });
+            }
+        })
+        .catch(err => {
+            Logger.error("Failed to send first time submission Discord hook.");
+            Logger.error(JSON.stringify(err));
+            Logger.error("\n");
         });
     }
 }
@@ -167,73 +167,98 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
 // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
-async function autoModerateSubmission(submission: { videoID: any; userID: any; segments: any }) {
-    // Get the video information from the youtube API
-    if (config.youtubeAPIKey !== null) {
-        const {err, data} = await new Promise((resolve) => {
-            YouTubeAPI.listVideos(submission.videoID, (err: any, data: any) => resolve({err, data}));
-        });
+async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, 
+            submission: { videoID: any; userID: any; segments: any }) {
+    if (apiVideoInfo) {
+        const {err, data} = apiVideoInfo;
+        if (err) return false;
 
-        if (err) {
-            return false;
-        } else {
-            // Check to see if video exists
-            if (data.pageInfo.totalResults === 0) {
-                return "No video exists with id " + submission.videoID;
+        // Check to see if video exists
+        if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
+
+        const duration = getYouTubeVideoDuration(apiVideoInfo);
+        const segments = submission.segments;
+        let nbString = "";
+        for (let i = 0; i < segments.length; i++) {
+            const startTime = parseFloat(segments[i].segment[0]);
+            const endTime = parseFloat(segments[i].segment[1]);
+
+            if (duration == 0) {
+                // Allow submission if the duration is 0 (bug in youtube api)
+                return false;
             } else {
-                const segments = submission.segments;
-                let nbString = "";
-                for (let i = 0; i < segments.length; i++) {
+                if (segments[i].category === "sponsor") {
+                    //Prepare timestamps to send to NB all at once
+                    nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
+                }
+            }
+        }
+
+        // Get all submissions for this user
+        const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]);
+        const allSegmentTimes = [];
+        if (allSubmittedByUser !== undefined) {
+            //add segments the user has previously submitted
+            for (const segmentInfo of allSubmittedByUser) {
+                allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
+            }
+        }
+
+        //add segments they are trying to add in this submission
+        for (let i = 0; i < segments.length; i++) {
+            let startTime = parseFloat(segments[i].segment[0]);
+            let endTime = parseFloat(segments[i].segment[1]);
+            allSegmentTimes.push([startTime, endTime]);
+        }
+
+        //merge all the times into non-overlapping arrays
+        const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
+            return a[0] - b[0] || a[1] - b[1];
+        }));
+
+        let videoDuration = data.items[0].contentDetails.duration;
+        videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
+        if (videoDuration != 0) {
+            let allSegmentDuration = 0;
+            //sum all segment times together
+            allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
+            if (allSegmentDuration > (videoDuration / 100) * 80) {
+                // Reject submission if all segments combine are over 80% of the video
+                return "Total length of your submitted segments are over 80% of the video.";
+            }
+        }
+
+        // Check NeuralBlock
+        const neuralBlockURL = config.neuralBlockURL;
+        if (!neuralBlockURL) return false;
+        const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
+            "&segments=" + nbString.substring(0, nbString.length - 1));
+        if (!response.ok) return false;
+
+        const nbPredictions = await response.json();
+        let nbDecision = false;
+        let predictionIdx = 0; //Keep track because only sponsor categories were submitted
+        for (let i = 0; i < segments.length; i++) {
+            if (segments[i].category === "sponsor") {
+                if (nbPredictions.probabilities[predictionIdx] < 0.70) {
+                    nbDecision = true; // At least one bad entry
                     const startTime = parseFloat(segments[i].segment[0]);
                     const endTime = parseFloat(segments[i].segment[1]);
 
-                    let duration = data.items[0].contentDetails.duration;
-                    duration = isoDurations.toSeconds(isoDurations.parse(duration));
-                    if (duration == 0) {
-                        // Allow submission if the duration is 0 (bug in youtube api)
-                        return false;
-                    } else if ((endTime - startTime) > (duration / 100) * 80) {
-                        // Reject submission if over 80% of the video
-                        return "One of your submitted segments is over 80% of the video.";
-                    } else {
-                        if (segments[i].category === "sponsor") {
-                            //Prepare timestamps to send to NB all at once
-                            nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
-                        }
-                    }
-                }
-                // Check NeuralBlock
-                const neuralBlockURL = config.neuralBlockURL;
-                if (!neuralBlockURL) return false;
-                const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
-                    "&segments=" + nbString.substring(0, nbString.length - 1));
-                if (!response.ok) return false;
-
-                const nbPredictions = await response.json();
-                let nbDecision = false;
-                let predictionIdx = 0; //Keep track because only sponsor categories were submitted
-                for (let i = 0; i < segments.length; i++) {
-                    if (segments[i].category === "sponsor") {
-                        if (nbPredictions.probabilities[predictionIdx] < 0.70) {
-                            nbDecision = true; // At least one bad entry
-                            const startTime = parseFloat(segments[i].segment[0]);
-                            const endTime = parseFloat(segments[i].segment[1]);
-
-                            const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
-                            // Send to Discord
-                            // Note, if this is too spammy. Consider sending all the segments as one Webhook
-                            sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
-                        }
-                        predictionIdx++;
-                    }
-
-                }
-                if (nbDecision) {
-                    return "Rejected based on NeuralBlock predictions.";
-                } else {
-                    return false;
+                    const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
+                    // Send to Discord
+                    // Note, if this is too spammy. Consider sending all the segments as one Webhook
+                    sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
                 }
+                predictionIdx++;
             }
+
+        }
+
+        if (nbDecision) {
+            return "Rejected based on NeuralBlock predictions.";
+        } else {
+            return false;
         }
     } else {
         Logger.debug("Skipped YouTube API");
@@ -244,6 +269,21 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s
     }
 }
 
+function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
+    const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
+    return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
+}
+
+async function getYouTubeVideoInfo(videoID: VideoID): Promise<APIVideoInfo> {
+    if (config.youtubeAPIKey !== null) {
+        return new Promise((resolve) => {
+            YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
+        });
+    } else {
+        return null;
+    }
+}
+
 function proxySubmission(req: Request) {
     fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, {
             method: 'POST',
@@ -272,13 +312,14 @@ export async function postSkipSegments(req: Request, res: Response) {
     if (!Object.values(Service).some((val) => val == service)) {
         service = Service.YouTube;
     }
+    let videoDuration: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration;
 
-    let segments = req.body.segments;
+    let segments = req.body.segments as IncomingSegment[];
     if (segments === undefined) {
         // Use query instead
         segments = [{
-            segment: [req.query.startTime, req.query.endTime],
-            category: req.query.category,
+            segment: [req.query.startTime as string, req.query.endTime as string],
+            category: req.query.category as Category
         }];
     }
 
@@ -378,9 +419,15 @@ export async function postSkipSegments(req: Request, res: Response) {
         }
     }
 
+    let apiVideoInfo: APIVideoInfo = null;
+    if (service == Service.YouTube) {
+        apiVideoInfo = await getYouTubeVideoInfo(videoID);
+    }
+    videoDuration = getYouTubeVideoDuration(apiVideoInfo) || videoDuration;
+
     // Auto moderator check
     if (!isVIP && service == Service.YouTube) {
-        const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
+        const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category});
         if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
             // If NB automod rejects, the submission will start with -2 votes.
             // Note, if one submission is bad all submissions will be affected.
@@ -441,63 +488,18 @@ export async function postSkipSegments(req: Request, res: Response) {
 
         let startingVotes = 0 + decreaseVotes;
 
-        if (config.youtubeAPIKey !== null) {
-            let {err, data} = await new Promise((resolve) => {
-                YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
-            });
-
-            if (err) {
-                Logger.error("Error while submitting when connecting to YouTube API: " + err);
-            } else {
-                //get all segments for this video and user
-                const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [userID, videoID]);
-                const allSegmentTimes = [];
-                if (allSubmittedByUser !== undefined) {
-                    //add segments the user has previously submitted
-                    for (const segmentInfo of allSubmittedByUser) {
-                        allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
-                    }
-                }
-
-                //add segments they are trying to add in this submission
-                for (let i = 0; i < segments.length; i++) {
-                    let startTime = parseFloat(segments[i].segment[0]);
-                    let endTime = parseFloat(segments[i].segment[1]);
-                    allSegmentTimes.push([startTime, endTime]);
-                }
-
-                //merge all the times into non-overlapping arrays
-                const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
-                    return a[0] - b[0] || a[1] - b[1];
-                }));
-
-                let videoDuration = data.items[0].contentDetails.duration;
-                videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
-                if (videoDuration != 0) {
-                    let allSegmentDuration = 0;
-                    //sum all segment times together
-                    allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
-                    if (allSegmentDuration > (videoDuration / 100) * 80) {
-                        // Reject submission if all segments combine are over 80% of the video
-                        res.status(400).send("Total length of your submitted segments are over 80% of the video.");
-                        return;
-                    }
-                }
-            }
-        }
-
         for (const segmentInfo of segments) {
             //this can just be a hash of the data
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
-            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
+            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
 
             const startingLocked = isVIP ? 1 : 0;
             try {
                 await db.prepare('run', `INSERT INTO "sponsorTimes" 
-                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "shadowHidden", "hashedVideoID")
-                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, shadowBanned, getHash(videoID, 1),
+                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
+                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1),
                     ],
                 );
 
@@ -508,7 +510,7 @@ export async function postSkipSegments(req: Request, res: Response) {
                 redis.delAsync(skipSegmentsKey(videoID));
             } catch (err) {
                 //a DB change probably occurred
-                res.sendStatus(502);
+                res.sendStatus(500);
                 Logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " +
                     segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err);
 
@@ -533,7 +535,7 @@ export async function postSkipSegments(req: Request, res: Response) {
     res.json(newSegments);
 
     for (let i = 0; i < segments.length; i++) {
-        sendWebhooks(userID, videoID, UUIDs[i], segments[i], service);
+        sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service);
     }
 }
 
diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts
index 2478b70..2bb4e8d 100644
--- a/src/types/segments.model.ts
+++ b/src/types/segments.model.ts
@@ -3,6 +3,7 @@ import { SBRecord } from "./lib.model";
 
 export type SegmentUUID = string  & { __segmentUUIDBrand: unknown };
 export type VideoID = string & { __videoIDBrand: unknown };
+export type VideoDuration = number & { __videoDurationBrand: unknown };
 export type Category = string & { __categoryBrand: unknown };
 export type VideoIDHash = VideoID & HashedValue;
 export type IPAddress = string & { __ipAddressBrand: unknown };
@@ -18,6 +19,11 @@ export enum Service {
     // Lbry = 'Lbry'
 }
 
+export interface IncomingSegment { 
+    category: Category; 
+    segment: string[]; 
+}
+
 export interface Segment { 
     category: Category; 
     segment: number[]; 
diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts
index 6bc6179..9ca8333 100644
--- a/test/cases/postSkipSegments.ts
+++ b/test/cases/postSkipSegments.ts
@@ -97,6 +97,70 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
+    it('Should be able to submit a single time with a duration (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "dQw4w9WgXZX",
+                videoDuration: 100,
+                segments: [{
+                    segment: [0, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]);
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to submit a single time with a duration from the API (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "noDuration",
+                videoDuration: 100,
+                segments: [{
+                    segment: [0, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["noDuration"]);
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 100) {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
     it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {
@@ -276,7 +340,7 @@ describe('postSkipSegments', () => {
                 }),
             })
         .then(async res => {
-            if (res.status === 400) {
+            if (res.status === 403) {
                 const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["n9rIGdXnSJc"]);
                 let success = true;
                 if (rows.length === 4) {
@@ -324,7 +388,7 @@ describe('postSkipSegments', () => {
             }),
         })
         .then(async res => {
-            if (res.status === 400) {
+            if (res.status === 403) {
                 const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["80percent_video"]);
                 let success = rows.length == 2;
                 for (const row of rows) {

From 3c89e9c01506e84de46032dc37d04f31a3c5477c Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 19 Mar 2021 22:52:23 -0400
Subject: [PATCH 03/32] Send back duration in getSkipSegments request

---
 src/routes/getSkipSegments.ts |  7 ++++---
 src/types/segments.model.ts   |  2 ++
 test/cases/getSkipSegments.ts | 34 +++++++++++++++++-----------------
 3 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts
index 42a44c7..b6ae72f 100644
--- a/src/routes/getSkipSegments.ts
+++ b/src/routes/getSkipSegments.ts
@@ -43,7 +43,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
     return chooseSegments(filteredSegments).map((chosenSegment) => ({
         category,
         segment: [chosenSegment.startTime, chosenSegment.endTime],
-        UUID: chosenSegment.UUID
+        UUID: chosenSegment.UUID,
+        videoDuration: chosenSegment.videoDuration
     }));
 }
 
@@ -58,7 +59,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
         const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
             .prepare(
                 'all',
-                `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" 
+                `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" 
                 WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
                 [videoID, service]
             )).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
@@ -94,7 +95,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
         const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
             .prepare(
                 'all',
-                `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
+                `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
                 WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
                 [hashedVideoIDPrefix + '%', service]
             )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts
index 2bb4e8d..6c9cd4a 100644
--- a/src/types/segments.model.ts
+++ b/src/types/segments.model.ts
@@ -28,6 +28,7 @@ export interface Segment {
     category: Category; 
     segment: number[]; 
     UUID: SegmentUUID;
+    videoDuration: VideoDuration;
 }
 
 export enum Visibility {
@@ -44,6 +45,7 @@ export interface DBSegment {
     locked: boolean;
     shadowHidden: Visibility;
     videoID: VideoID;
+    videoDuration: VideoDuration;
     hashedVideoID: VideoIDHash;
 }
 
diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts
index 51f1c1b..674a773 100644
--- a/test/cases/getSkipSegments.ts
+++ b/test/cases/getSkipSegments.ts
@@ -5,17 +5,17 @@ import {getHash} from '../../src/utils/getHash';
 
 describe('getSkipSegments', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('testtesttest2', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest,test', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')");
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, '" + getHash('testtesttest2', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, '" + getHash('testtesttest,test', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, '" + getHash('locked', 1) + "')");
     
         return;
     });
@@ -28,7 +28,7 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
-                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -45,7 +45,7 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
-                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1") {
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -79,7 +79,7 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
-                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -96,7 +96,7 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
-                    && data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
+                    && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -117,9 +117,9 @@ describe('getSkipSegments', () => {
                     let success = true;
                     for (const segment of data) {
                         if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
-                            || segment.category !== "intro" || segment.UUID !== "1-uuid-7") &&
+                            || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) &&
                             (segment.segment[0] !== 1 || segment.segment[1] !== 11
-                                || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) {
+                                || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) {
                             success = false;
                             break;
                         }

From 02e628f5330403bc578d1e92a54d7d4ab08b54f2 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 20 Mar 2021 01:08:33 -0400
Subject: [PATCH 04/32] Setup csv exports and html status page

---
 .gitignore                 |  1 +
 docker/docker-compose.yml  |  1 +
 src/app.ts                 | 11 +++++--
 src/routes/dumpDatabase.ts | 61 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+), 3 deletions(-)
 create mode 100644 src/routes/dumpDatabase.ts

diff --git a/.gitignore b/.gitignore
index 3cff924..dd78e49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
 test/databases/sponsorTimes.db-shm
 test/databases/sponsorTimes.db-wal
 test/databases/private.db
+docker/database-export
 
 # Config files
 config.json
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index a54a3c0..4ee69ee 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -7,6 +7,7 @@ services:
       - database.env
     volumes:
       - database-data:/var/lib/postgresql/data
+      - ./database-export/:/opt/exports
     ports:
       - 127.0.0.1:5432:5432
   redis:
diff --git a/src/app.ts b/src/app.ts
index 098e5f3..026f74c 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter';
 import {loggerMiddleware} from './middleware/logger';
 import {corsMiddleware} from './middleware/cors';
 import {rateLimitMiddleware} from './middleware/requestRateLimit';
+import dumpDatabase from './routes/dumpDatabase';
 
 
 export function createServer(callback: () => void) {
@@ -127,7 +128,11 @@ function setupRoutes(app: Express) {
     //get if user is a vip
     app.post('/api/segmentShift', postSegmentShift);
 
-    app.get('/database.db', function (req: Request, res: Response) {
-        res.sendFile("./databases/sponsorTimes.db", {root: "./"});
-    });
+    if (config.postgres) {
+        app.get('/database', dumpDatabase);
+    } else {
+        app.get('/database.db', function (req: Request, res: Response) {
+            res.sendFile("./databases/sponsorTimes.db", {root: "./"});
+        });
+    }
 }
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
new file mode 100644
index 0000000..fb8363d
--- /dev/null
+++ b/src/routes/dumpDatabase.ts
@@ -0,0 +1,61 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import { config } from '../config';
+
+const ONE_MINUTE = 1000 * 60;
+
+const styleHeader = `<style>body{font-family: sans-serif}</style>`
+
+const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
+<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
+<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
+
+const tables = [{
+    name: "sponsorTimes",
+    order: "timeSubmitted"
+},
+{
+    name: "userNames"
+},
+{
+    name: "categoryVotes"
+},
+{
+    name: "noSegments",
+},
+{
+    name: "warnings",
+    order: "issueTime"
+},
+{
+    name: "vipUsers"
+}];
+
+const links: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
+                        .reduce((acc, url) => acc + url, "");
+
+let lastUpdate = 0;
+
+export default function dumpDatabase(req: Request, res: Response) {
+    if (!config.postgres) {
+        res.status(404).send("Not supported on this instance");
+        return;
+    }
+
+    const now = Date.now();
+    const updateQueued = now - lastUpdate > ONE_MINUTE;
+
+    res.status(200).send(`${styleHeader}
+                <h1>SponsorBlock database dumps</h1>${licenseHeader}${links}<br/>
+                ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+
+    if (updateQueued) {
+        lastUpdate = Date.now();
+
+        for (const table of tables) {
+            db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
+                    TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
+}
\ No newline at end of file

From 8423165df45a883378245cfb2890dfb40b08c184 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 20 Mar 2021 01:13:16 -0400
Subject: [PATCH 05/32] Add json page for database export

---
 src/app.ts                 |  3 ++-
 src/routes/dumpDatabase.ts | 22 +++++++++++++++++-----
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/app.ts b/src/app.ts
index 026f74c..5bc2b2c 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -129,7 +129,8 @@ function setupRoutes(app: Express) {
     app.post('/api/segmentShift', postSegmentShift);
 
     if (config.postgres) {
-        app.get('/database', dumpDatabase);
+        app.get('/database', (req, res) => dumpDatabase(req, res, true));
+        app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
     } else {
         app.get('/database.db', function (req: Request, res: Response) {
             res.sendFile("./databases/sponsorTimes.db", {root: "./"});
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
index fb8363d..0d0ab02 100644
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -32,12 +32,14 @@ const tables = [{
     name: "vipUsers"
 }];
 
-const links: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
+const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
+
+const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
                         .reduce((acc, url) => acc + url, "");
 
 let lastUpdate = 0;
 
-export default function dumpDatabase(req: Request, res: Response) {
+export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
     if (!config.postgres) {
         res.status(404).send("Not supported on this instance");
         return;
@@ -46,9 +48,19 @@ export default function dumpDatabase(req: Request, res: Response) {
     const now = Date.now();
     const updateQueued = now - lastUpdate > ONE_MINUTE;
 
-    res.status(200).send(`${styleHeader}
-                <h1>SponsorBlock database dumps</h1>${licenseHeader}${links}<br/>
-                ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+    res.status(200)
+    
+    if (showPage) {
+        res.send(`${styleHeader}
+            <h1>SponsorBlock database dumps</h1>${licenseHeader}${linksHTML}<br/>
+            ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+    } else {
+        res.send({
+            lastUpdated: lastUpdate,
+            updateQueued,
+            links
+        })
+    }
 
     if (updateQueued) {
         lastUpdate = Date.now();

From 180d9bfb73d5941b99c7cf76fbca06356e1a1ce1 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 20 Mar 2021 11:46:37 -0400
Subject: [PATCH 06/32] Add explanation to database page

---
 src/routes/dumpDatabase.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
index 0d0ab02..d687f07 100644
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -52,7 +52,12 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
     
     if (showPage) {
         res.send(`${styleHeader}
-            <h1>SponsorBlock database dumps</h1>${licenseHeader}${linksHTML}<br/>
+            <h1>SponsorBlock database dumps</h1>${licenseHeader}
+            <h3>How this works</h3>
+            Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
+            Then, you can download the csv files below, or use the links returned from the JSON request.
+            <h3>Links</h3>
+            ${linksHTML}<br/>
             ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
     } else {
         res.send({

From cbf043ac7e01a1566c8c0bf44aa1cc3bde356b5f Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 20 Mar 2021 11:54:50 -0400
Subject: [PATCH 07/32] Add twitch

---
 src/types/segments.model.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts
index 6c9cd4a..a95bac3 100644
--- a/src/types/segments.model.ts
+++ b/src/types/segments.model.ts
@@ -12,8 +12,9 @@ export type HashedIP = IPAddress & HashedValue;
 // Uncomment as needed
 export enum Service {
     YouTube = 'YouTube',
-    // Nebula = 'Nebula',
     PeerTube = 'PeerTube',
+    // Twitch = 'Twitch',
+    // Nebula = 'Nebula',
     // RSS = 'RSS',
     // Corridor = 'Corridor',
     // Lbry = 'Lbry'

From 84b86bb6a1fd19a3a5ac75664df16190294ddfc5 Mon Sep 17 00:00:00 2001
From: Nanobyte <nanobyte@ymail.com>
Date: Sun, 21 Mar 2021 22:40:57 +0100
Subject: [PATCH 08/32] Make dumpDatabase configurable

---
 config.json.example        | 25 +++++++++++++++++++++++++
 src/config.ts              | 27 ++++++++++++++++++++++++++-
 src/routes/dumpDatabase.ts | 37 ++++++++++++++-----------------------
 3 files changed, 65 insertions(+), 24 deletions(-)

diff --git a/config.json.example b/config.json.example
index f8548c1..d859e4d 100644
--- a/config.json.example
+++ b/config.json.example
@@ -38,5 +38,30 @@
         "max": 20, // 20 requests in 15min time window
         "statusCode": 200
       }
+    },
+    "dumpDatabase": {
+        "enabled": true,
+        "minTimeBetweenMs": 60000, // 1 minute between dumps
+        "exportPath": "/opt/exports",
+        "tables": [{
+            "name": "sponsorTimes",
+            "order": "timeSubmitted"
+        },
+        {
+            "name": "userNames"
+        },
+        {
+            "name": "categoryVotes"
+        },
+        {
+            "name": "noSegments",
+        },
+        {
+            "name": "warnings",
+            "order": "issueTime"
+        },
+        {
+            "name": "vipUsers"
+        }]
     }
 }
diff --git a/src/config.ts b/src/config.ts
index 93c6a43..477aa72 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -45,7 +45,32 @@ addDefaults(config, {
     },
     userCounterURL: null,
     youtubeAPIKey: null,
-    postgres: null
+    postgres: null,
+    dumpDatabase: {
+        enabled: true,
+        minTimeBetweenMs: 60000,
+        exportPath: '/opt/exports',
+        tables: [{
+            name: "sponsorTimes",
+            order: "timeSubmitted"
+        },
+        {
+            name: "userNames"
+        },
+        {
+            name: "categoryVotes"
+        },
+        {
+            name: "noSegments",
+        },
+        {
+            name: "warnings",
+            order: "issueTime"
+        },
+        {
+            name: "vipUsers"
+        }]
+    }
 });
 
 // Add defaults
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
index d687f07..ec3e801 100644
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -11,26 +11,13 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
 <p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
 <p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
 
-const tables = [{
-    name: "sponsorTimes",
-    order: "timeSubmitted"
-},
-{
-    name: "userNames"
-},
-{
-    name: "categoryVotes"
-},
-{
-    name: "noSegments",
-},
-{
-    name: "warnings",
-    order: "issueTime"
-},
-{
-    name: "vipUsers"
-}];
+const tables = config?.dumpDatabase?.tables ?? [];
+const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
+const exportPath = config?.dumpDatabase?.exportPath ?? '/opt/exports';
+
+if (tables.length === 0) {
+    Logger.warn('[dumpDatabase] No tables configured');
+}
 
 const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
 
@@ -40,13 +27,17 @@ const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.n
 let lastUpdate = 0;
 
 export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
+    if (config?.dumpDatabase?.enabled === false) {
+        res.status(404).send("Database dump is disabled");
+        return;
+    }
     if (!config.postgres) {
         res.status(404).send("Not supported on this instance");
         return;
     }
 
     const now = Date.now();
-    const updateQueued = now - lastUpdate > ONE_MINUTE;
+    const updateQueued = now - lastUpdate > MILLISECONDS_BETWEEN_DUMPS;
 
     res.status(200)
     
@@ -72,7 +63,7 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
 
         for (const table of tables) {
             db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
-                    TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+                    TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
         }
     }
-}
\ No newline at end of file
+}

From 514ea03655f043fa8a65b1f1666b3ffea91247b8 Mon Sep 17 00:00:00 2001
From: Nanobyte <nanobyte@ymail.com>
Date: Sun, 21 Mar 2021 22:59:16 +0100
Subject: [PATCH 09/32] Add TS for dumpDatabase config

---
 src/types/config.model.ts | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/types/config.model.ts b/src/types/config.model.ts
index b27cfaa..c0611b7 100644
--- a/src/types/config.model.ts
+++ b/src/types/config.model.ts
@@ -38,6 +38,7 @@ export interface SBSConfig {
     maximumPrefix?: string;
     redis?: redis.ClientOpts;
     postgres?: PoolConfig;
+    dumpDatabase?: DumpDatabase;
 }
 
 export interface WebhookConfig {
@@ -61,4 +62,16 @@ export interface PostgresConfig {
     createDbIfNotExists: boolean;
     enableWalCheckpointNumber: boolean;
     postgres: PoolConfig;
-}
\ No newline at end of file
+}
+
+export interface DumpDatabase {
+    enabled: boolean;
+    minTimeBetweenMs: number;
+    exportPath: string;
+    tables: DumpDatabaseTable[];
+}
+
+export interface DumpDatabaseTable {
+    name: string;
+    order?: string;
+}

From 11b4f642a60c973a4060e0416487e4a60a55014c Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sun, 21 Mar 2021 19:16:56 -0400
Subject: [PATCH 10/32] Apply indexes after upgrades

---
 databases/_private_indexes.sql      | 32 ++++++++++++++
 databases/_sponsorTimes_indexes.sql | 66 +++++++++++++++++++++++++++++
 src/databases/Postgres.ts           | 11 +++++
 3 files changed, 109 insertions(+)
 create mode 100644 databases/_private_indexes.sql
 create mode 100644 databases/_sponsorTimes_indexes.sql

diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql
new file mode 100644
index 0000000..d7fb746
--- /dev/null
+++ b/databases/_private_indexes.sql
@@ -0,0 +1,32 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "idx_16928_sponsorTimes_hashedIP"
+    ON public."sponsorTimes" USING btree
+    ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
+    ON public."sponsorTimes" USING btree
+    ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- votes
+
+CREATE INDEX IF NOT EXISTS "votes_userID"
+    ON public.votes USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- shadowBannedUsers
+
+CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
+    ON public."shadowBannedUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
+    ON public."categoryVotes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
\ No newline at end of file
diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql
new file mode 100644
index 0000000..33f1913
--- /dev/null
+++ b/databases/_sponsorTimes_indexes.sql
@@ -0,0 +1,66 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "sponsorTiems_timeSubmitted"
+    ON public."sponsorTimes" USING btree
+    ("timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_userID"
+    ON public."sponsorTimes" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
+    ON public."sponsorTimes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID"
+    ON public."sponsorTimes" USING btree
+    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "startTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"
+    ON public."sponsorTimes" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- userNames
+
+CREATE INDEX IF NOT EXISTS "userNames_userID"
+    ON public."userNames" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- vipUsers
+
+CREATE INDEX IF NOT EXISTS "vipUsers_index"
+    ON public."vipUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- warnings
+
+CREATE INDEX IF NOT EXISTS warnings_index
+    ON public.warnings USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "warnings_issueTime"
+    ON public.warnings USING btree
+    ("issueTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- noSegments
+
+CREATE INDEX IF NOT EXISTS "noSegments_videoID"
+    ON public."noSegments" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
+    ON public."categoryVotes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
\ No newline at end of file
diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts
index bae2da6..1f87e8f 100644
--- a/src/databases/Postgres.ts
+++ b/src/databases/Postgres.ts
@@ -23,6 +23,8 @@ export class Postgres implements IDatabase {
 
             // Upgrade database if required
             await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+
+            await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
         }
     }
 
@@ -118,6 +120,15 @@ export class Postgres implements IDatabase {
         Logger.debug('db update: no file ' + path);
     }
 
+    private async applyIndexes(fileNamePrefix: string, schemaFolder: string) {
+        const path = schemaFolder + "/_" + fileNamePrefix + "_indexes.sql";
+        if (fs.existsSync(path)) {
+            await this.pool.query(fs.readFileSync(path).toString());
+        } else {
+            Logger.debug('failed to apply indexes to ' + fileNamePrefix);
+        }
+    }
+
     private processUpgradeQuery(query: string): string {
         let result = query;
         result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");

From 8219b0398e43230bdbefaeaf22302943f508dfbb Mon Sep 17 00:00:00 2001
From: Nanobyte <nanobyte@ymail.com>
Date: Mon, 22 Mar 2021 01:35:45 +0100
Subject: [PATCH 11/32] Fix invalid json in example config

---
 config.json.example | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.json.example b/config.json.example
index d859e4d..65f9b42 100644
--- a/config.json.example
+++ b/config.json.example
@@ -54,7 +54,7 @@
             "name": "categoryVotes"
         },
         {
-            "name": "noSegments",
+            "name": "noSegments"
         },
         {
             "name": "warnings",

From 2c3dde0d2ed1beb80197bfb60f5f55b6f46c69c5 Mon Sep 17 00:00:00 2001
From: Nanobyte <nanobyte@ymail.com>
Date: Mon, 22 Mar 2021 22:18:23 +0100
Subject: [PATCH 12/32] Timestamp based dump filenames and garbage collection

---
 config.json.example        |   3 +-
 src/config.ts              |   3 +-
 src/routes/dumpDatabase.ts | 120 ++++++++++++++++++++++++++++++++++---
 src/types/config.model.ts  |   3 +-
 4 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/config.json.example b/config.json.example
index 65f9b42..580b8fb 100644
--- a/config.json.example
+++ b/config.json.example
@@ -42,7 +42,8 @@
     "dumpDatabase": {
         "enabled": true,
         "minTimeBetweenMs": 60000, // 1 minute between dumps
-        "exportPath": "/opt/exports",
+        "appExportPath": "/opt/exports",
+        "postgresExportPath": "/opt/exports",
         "tables": [{
             "name": "sponsorTimes",
             "order": "timeSubmitted"
diff --git a/src/config.ts b/src/config.ts
index 477aa72..0e985dc 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -49,7 +49,8 @@ addDefaults(config, {
     dumpDatabase: {
         enabled: true,
         minTimeBetweenMs: 60000,
-        exportPath: '/opt/exports',
+        appExportPath: '/opt/exports',
+        postgresExportPath: '/opt/exports',
         tables: [{
             name: "sponsorTimes",
             order: "timeSubmitted"
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
index ec3e801..b7f2726 100644
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -2,10 +2,29 @@ import {db} from '../databases/databases';
 import {Logger} from '../utils/logger';
 import {Request, Response} from 'express';
 import { config } from '../config';
+const util = require('util');
+const fs = require('fs');
+const path = require('path');
+const unlink = util.promisify(fs.unlink);
+const fstat = util.promisify(fs.fstat);
 
 const ONE_MINUTE = 1000 * 60;
 
-const styleHeader = `<style>body{font-family: sans-serif}</style>`
+const styleHeader = `<style>
+    body {
+        font-family: sans-serif
+    }
+    table th,
+    table td {
+        padding: 2px 4px;
+    }
+    table th {
+        text-align: left;
+    }
+    table tbody tr:nth-child(odd) {
+        background: #efefef;
+    }
+</style>`
 
 const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
 <p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
@@ -13,7 +32,15 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
 
 const tables = config?.dumpDatabase?.tables ?? [];
 const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
-const exportPath = config?.dumpDatabase?.exportPath ?? '/opt/exports';
+const appExportPath = config?.dumpDatabase?.appExportPath ?? '/opt/exports';
+const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? '/opt/exports';
+const tableNames = tables.map(table => table.name);
+
+interface TableDumpList {
+    fileName: string;
+    tableName: string;
+};
+let latestDumpFiles: TableDumpList[] = [];
 
 if (tables.length === 0) {
     Logger.warn('[dumpDatabase] No tables configured');
@@ -26,7 +53,50 @@ const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.n
 
 let lastUpdate = 0;
 
-export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
+function removeOutdatedDumps(exportPath: string): Promise<any> {
+    return new Promise((resolve, reject) => {
+        // Get list of table names
+        // Create array for each table
+        const tableFiles = tableNames.reduce((obj: any, tableName) => {
+            obj[tableName] = [];
+            return obj;
+        }, {});
+        // read files in export directory
+        fs.readdir(exportPath, (err: any, files: string[]) => {
+            if (err) Logger.error(err);
+            if (err) return resolve();
+            files.forEach(file => {
+                // we only care about files that start with "<tablename>_" and ends with .csv
+                tableNames.forEach(tableName => {
+                    if (file.startsWith(`${tableName}_`) && file.endsWith('.csv')) {
+                        // extract the timestamp from the filename
+                        // we could also use the fs.stat mtime
+                        const timestamp = Number(file.split('_')[1].replace('.csv', ''));
+                        tableFiles[tableName].push({
+                            file: path.join(exportPath, file),
+                            timestamp,
+                        });
+                    }
+                });
+            });
+            const outdatedTime = Math.floor(Date.now() - (MILLISECONDS_BETWEEN_DUMPS * 1.5));
+            for (let tableName in tableFiles) {
+                const files = tableFiles[tableName];
+                files.forEach(async (item: any) => {
+                    if (item.timestamp < outdatedTime) {
+                        // remove old file
+                        await unlink(item.file).catch((error: any) => {
+                            Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
+                        });
+                    }
+                });
+            }
+            resolve();
+        });
+    });
+}
+
+export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) {
     if (config?.dumpDatabase?.enabled === false) {
         res.status(404).send("Database dump is disabled");
         return;
@@ -48,22 +118,58 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
             Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
             Then, you can download the csv files below, or use the links returned from the JSON request.
             <h3>Links</h3>
-            ${linksHTML}<br/>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Table</th>
+                        <th>CSV</th>
+                    </tr>
+                </thead>
+                <tbody>
+                ${latestDumpFiles.map((item:any) => {
+                    return `
+                    <tr>
+                        <td>${item.tableName}</td>
+                        <td><a href="/download/${item.fileName}">${item.fileName}</a></td>
+                    </tr>
+                    `;
+                }).join('')}
+                ${latestDumpFiles.length === 0 ? '<tr><td colspan="2">Please wait: Generating files</td></tr>' : ''}
+                </tbody>
+            </table>
+            <hr/>
             ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
     } else {
         res.send({
             lastUpdated: lastUpdate,
             updateQueued,
-            links
+            links: latestDumpFiles.map((item:any) => {
+                return {
+                    table: item.tableName,
+                    url: `/download/${item.fileName}`,
+                    size: item.fileSize,
+                };
+            }),
         })
     }
 
     if (updateQueued) {
         lastUpdate = Date.now();
+        
+        await removeOutdatedDumps(appExportPath);
+        
+        const dumpFiles = [];
 
         for (const table of tables) {
-            db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
-                    TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+            const fileName = `${table.name}_${lastUpdate}.csv`;
+            const file = `${postgresExportPath}/${fileName}`;
+            await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
+                    TO '${file}' WITH (FORMAT CSV, HEADER true);`);
+            dumpFiles.push({
+                fileName,
+                tableName: table.name,
+            });
         }
+        latestDumpFiles = [...dumpFiles];
     }
 }
diff --git a/src/types/config.model.ts b/src/types/config.model.ts
index c0611b7..f46cc17 100644
--- a/src/types/config.model.ts
+++ b/src/types/config.model.ts
@@ -67,7 +67,8 @@ export interface PostgresConfig {
 export interface DumpDatabase {
     enabled: boolean;
     minTimeBetweenMs: number;
-    exportPath: string;
+    appExportPath: string;
+    postgresExportPath: string;
     tables: DumpDatabaseTable[];
 }
 

From 27c2562a7f7319c122bbbf68e6196f361635a037 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Tue, 23 Mar 2021 23:46:46 -0400
Subject: [PATCH 13/32] Add more indexes

---
 databases/_private_indexes.sql      | 10 +++++-----
 databases/_sponsorTimes_indexes.sql |  8 ++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql
index d7fb746..2b4781e 100644
--- a/databases/_private_indexes.sql
+++ b/databases/_private_indexes.sql
@@ -1,15 +1,15 @@
 -- sponsorTimes
 
-CREATE INDEX IF NOT EXISTS "idx_16928_sponsorTimes_hashedIP"
-    ON public."sponsorTimes" USING btree
-    ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
-
 CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
     ON public."sponsorTimes" USING btree
     ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
 
+CREATE INDEX "privateDB_sponsorTimes_videoID"
+    ON public."sponsorTimes" USING btree
+    ("videoID" ASC NULLS LAST)
+;
+
 -- votes
 
 CREATE INDEX IF NOT EXISTS "votes_userID"
diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql
index 33f1913..6d4d91d 100644
--- a/databases/_sponsorTimes_indexes.sql
+++ b/databases/_sponsorTimes_indexes.sql
@@ -14,10 +14,10 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
     ON public."sponsorTimes" USING btree
     ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
-
-CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID"
-    ON public."sponsorTimes" USING btree
-    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "startTime" ASC NULLS LAST)
+    
+CREATE INDEX "sponsorTimes_hashedVideoID_gin"
+    ON public."sponsorTimes" USING gin
+    ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
     TABLESPACE pg_default;
 
 CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"

From 5c827baa1adc2d2d9f568a740c5167bdfce9055f Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Thu, 25 Mar 2021 18:48:44 -0400
Subject: [PATCH 14/32] Update db link

---
 README.MD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.MD b/README.MD
index 8f61c57..e309e54 100644
--- a/README.MD
+++ b/README.MD
@@ -8,7 +8,7 @@ This is the server backend for it
 
 This uses a Postgres or Sqlite database to hold all the timing data.
 
-To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
+To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
 
 Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.
 

From c7eb5fed35fa520ae35d8fdef56a4e76ffa9ed2a Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 26 Mar 2021 18:35:25 -0400
Subject: [PATCH 15/32] Fix video duration precision and use submitted one when
 possible

---
 databases/_upgrade_sponsorTimes_9.sql | 29 +++++++++++++++++++++++++++
 src/routes/postSkipSegments.ts        |  6 +++++-
 2 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 databases/_upgrade_sponsorTimes_9.sql

diff --git a/databases/_upgrade_sponsorTimes_9.sql b/databases/_upgrade_sponsorTimes_9.sql
new file mode 100644
index 0000000..5015a4a
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_9.sql
@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_9" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+    "videoDuration"	REAL NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 9 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 3b6f82d..2f847bf 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -423,7 +423,11 @@ export async function postSkipSegments(req: Request, res: Response) {
     if (service == Service.YouTube) {
         apiVideoInfo = await getYouTubeVideoInfo(videoID);
     }
-    videoDuration = getYouTubeVideoDuration(apiVideoInfo) || videoDuration;
+    const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
+    if (!apiVideoDuration || Math.abs(videoDuration - apiVideoDuration) > 2) {
+        // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
+        videoDuration = apiVideoDuration;
+    }
 
     // Auto moderator check
     if (!isVIP && service == Service.YouTube) {

From 46524e4298e3536a534c90173043290cad213993 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 26 Mar 2021 19:02:32 -0400
Subject: [PATCH 16/32] Fix indexes

---
 databases/_private_indexes.sql      | 2 +-
 databases/_sponsorTimes_indexes.sql | 8 ++++----
 src/databases/Postgres.ts           | 7 ++++++-
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql
index 2b4781e..2b70fa7 100644
--- a/databases/_private_indexes.sql
+++ b/databases/_private_indexes.sql
@@ -5,7 +5,7 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
     ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
 
-CREATE INDEX "privateDB_sponsorTimes_videoID"
+CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID"
     ON public."sponsorTimes" USING btree
     ("videoID" ASC NULLS LAST)
 ;
diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql
index 6d4d91d..c90ef51 100644
--- a/databases/_sponsorTimes_indexes.sql
+++ b/databases/_sponsorTimes_indexes.sql
@@ -1,6 +1,6 @@
 -- sponsorTimes
 
-CREATE INDEX IF NOT EXISTS "sponsorTiems_timeSubmitted"
+CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted"
     ON public."sponsorTimes" USING btree
     ("timeSubmitted" ASC NULLS LAST)
     TABLESPACE pg_default;
@@ -14,8 +14,8 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
     ON public."sponsorTimes" USING btree
     ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
-    
-CREATE INDEX "sponsorTimes_hashedVideoID_gin"
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID_gin"
     ON public."sponsorTimes" USING gin
     ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
     TABLESPACE pg_default;
@@ -41,7 +41,7 @@ CREATE INDEX IF NOT EXISTS "vipUsers_index"
 
 -- warnings
 
-CREATE INDEX IF NOT EXISTS warnings_index
+CREATE INDEX IF NOT EXISTS "warnings_index"
     ON public.warnings USING btree
     ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
     TABLESPACE pg_default;
diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts
index 1f87e8f..e7e16f4 100644
--- a/src/databases/Postgres.ts
+++ b/src/databases/Postgres.ts
@@ -24,7 +24,12 @@ export class Postgres implements IDatabase {
             // Upgrade database if required
             await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
 
-            await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+            try {
+                await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+            } catch (e) {
+                Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
+                Logger.warn(e);
+            }
         }
     }
 

From 37a07ace72d13f18c281b25f29c2c2c3fd27cd86 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 26 Mar 2021 19:03:30 -0400
Subject: [PATCH 17/32] Cache data for getting hash-prefix segments

---
 src/middleware/redisKeys.ts     | 10 ++++++-
 src/routes/getSkipSegments.ts   | 43 ++++++++++++++++++++++++------
 src/routes/postSkipSegments.ts  |  6 +++--
 src/routes/voteOnSponsorTime.ts | 47 ++++++++++++++++++++-------------
 4 files changed, 76 insertions(+), 30 deletions(-)

diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts
index 1335e15..56b1aab 100644
--- a/src/middleware/redisKeys.ts
+++ b/src/middleware/redisKeys.ts
@@ -1,5 +1,13 @@
-import { Category, VideoID } from "../types/segments.model";
+import { Service, VideoID, VideoIDHash } from "../types/segments.model";
+import { Logger } from "../utils/logger";
 
 export function skipSegmentsKey(videoID: VideoID): string {
     return "segments-" + videoID;
 } 
+
+export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
+    hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
+    if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
+    
+    return "segments." + service + "." + hashedVideoIDPrefix;
+} 
\ No newline at end of file
diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts
index b6ae72f..c2a9155 100644
--- a/src/routes/getSkipSegments.ts
+++ b/src/routes/getSkipSegments.ts
@@ -2,7 +2,7 @@ import { Request, Response } from 'express';
 import { RedisClient } from 'redis';
 import { config } from '../config';
 import { db, privateDB } from '../databases/databases';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
 import { SBRecord } from '../types/lib.model';
 import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
 import { getHash } from '../utils/getHash';
@@ -92,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
         categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
         if (categories.length === 0) return null;
 
-        const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
-            .prepare(
-                'all',
-                `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
-                WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
-                [hashedVideoIDPrefix + '%', service]
-            )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
+        const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
+            .filter((segment: DBSegment) => categories.includes(segment?.category))
+            .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
                     hash: segment.hashedVideoID,
                     segmentPerCategory: {},
@@ -131,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
     }
 }
 
+async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
+    const fetchFromDB = () => db
+        .prepare(
+            'all',
+            `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
+            WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`,
+            [hashedVideoIDPrefix + '%', service]
+        );
+
+    if (hashedVideoIDPrefix.length === 4) {
+        const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
+        const {err, reply} = await redis.getAsync(key);
+    
+        if (!err && reply) {
+            try {
+                Logger.debug("Got data from redis: " + reply);
+                return JSON.parse(reply);
+            } catch (e) {
+                // If all else, continue on to fetching from the database
+            }
+        }
+
+        const data = await fetchFromDB();
+
+        redis.setAsync(key, JSON.stringify(data));
+        return data;
+    }
+
+    return await fetchFromDB();
+}
+
 //gets a weighted random choice from the choices array based on their `votes` property.
 //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
 //choices are unique
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 2f847bf..75b572d 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -11,7 +11,7 @@ import {getFormattedTime} from '../utils/getFormattedTime';
 import {isUserTrustworthy} from '../utils/isUserTrustworthy';
 import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
-import { skipSegmentsKey } from '../middleware/redisKeys';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
 import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
 
@@ -497,13 +497,14 @@ export async function postSkipSegments(req: Request, res: Response) {
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
             const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
+            const hashedVideoID = getHash(videoID, 1);
 
             const startingLocked = isVIP ? 1 : 0;
             try {
                 await db.prepare('run', `INSERT INTO "sponsorTimes" 
                     ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
                     VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1),
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
                     ],
                 );
 
@@ -512,6 +513,7 @@ export async function postSkipSegments(req: Request, res: Response) {
             
                 // Clear redis cache for this video
                 redis.delAsync(skipSegmentsKey(videoID));
+                redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
             } catch (err) {
                 //a DB change probably occurred
                 res.sendStatus(500);
diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts
index 29f0e66..c0aaac0 100644
--- a/src/routes/voteOnSponsorTime.ts
+++ b/src/routes/voteOnSponsorTime.ts
@@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash';
 import {config} from '../config';
 import { UserID } from '../types/user.model';
 import redis from '../utils/redis';
-import { skipSegmentsKey } from '../middleware/redisKeys';
-import { VideoID } from '../types/segments.model';
+import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
+import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
 
 const voteTypes = {
     normal: 0,
@@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) {
     }
 }
 
-async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string
-            , hashedIP: string, finalResponse: FinalResponse, res: Response) {
+async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category
+            , hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) {
     // Check if they've already made a vote
     const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]);
 
@@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
         return;
     }
 
-    const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]);
-    if (!currentCategory) {
+    const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
+                             [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
+    if (!videoInfo) {
         // Submission doesn't exist
         res.status(400).send("Submission doesn't exist.");
         return;
@@ -196,7 +197,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
     }
 
     // See if the submissions category is ready to change
-    const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
+    const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
 
     const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
     const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
@@ -208,9 +209,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
 
     // Add submission as vote
     if (!currentCategoryInfo && submissionInfo) {
-        await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
+        await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
 
-        await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
+        await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
     }
 
     const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
@@ -222,6 +223,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
         await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
     }
 
+    clearRedisCache(videoInfo);
+
     res.sendStatus(finalResponse.finalStatus);
 }
 
@@ -230,10 +233,10 @@ export function getUserID(req: Request): UserID {
 }
 
 export async function voteOnSponsorTime(req: Request, res: Response) {
-    const UUID = req.query.UUID as string;
+    const UUID = req.query.UUID as SegmentUUID;
     const paramUserID = getUserID(req);
     let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
-    const category = req.query.category as string;
+    const category = req.query.category as Category;
 
     if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
         //invalid request
@@ -255,7 +258,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
     const ip = getIP(req);
 
     //hash the ip 5000 times so no one can get it from the database
-    const hashedIP = getHash(ip + config.globalSalt);
+    const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress);
 
     //check if this user is on the vip list
     const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0;
@@ -350,13 +353,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         }
 
         //check if the increment amount should be multiplied (downvotes have more power if there have been many views)
-        const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as 
-                        {videoID: VideoID, votes: number, views: number};
+        const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as 
+                        {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
 
         if (voteTypeEnum === voteTypes.normal) {
             if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
                 //this user is a vip and a downvote
-                incrementAmount = -(row.votes + 2 - oldIncrementAmount);
+                incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount);
                 type = incrementAmount;
             }
         } else if (voteTypeEnum == voteTypes.incorrect) {
@@ -399,8 +402,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
                  await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
             }
 
-            // Clear redis cache for this video
-            redis.delAsync(skipSegmentsKey(row?.videoID));
+            clearRedisCache(videoInfo);
 
             //for each positive vote, see if a hidden submission can be shown again
             if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
@@ -437,7 +439,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
                 voteTypeEnum,
                 isVIP,
                 isOwnSubmission,
-                row,
+                row: videoInfo,
                 category,
                 incrementAmount,
                 oldIncrementAmount,
@@ -449,4 +451,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
 
         res.status(500).json({error: 'Internal error creating segment vote'});
     }
-}
\ No newline at end of file
+}
+
+function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
+    if (videoInfo) {
+        redis.delAsync(skipSegmentsKey(videoInfo.videoID));
+        redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
+    }
+}

From 5152d7e6495b40472bb21c9ba893fc5ec3eb2c08 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 26 Mar 2021 19:13:52 -0400
Subject: [PATCH 18/32] Fixed tests

---
 src/routes/postSkipSegments.ts |  4 ++--
 test/cases/postSkipSegments.ts | 36 ++++++++++++++++++++++++++++++++--
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 75b572d..7738653 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -424,9 +424,9 @@ export async function postSkipSegments(req: Request, res: Response) {
         apiVideoInfo = await getYouTubeVideoInfo(videoID);
     }
     const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
-    if (!apiVideoDuration || Math.abs(videoDuration - apiVideoDuration) > 2) {
+    if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
         // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
-        videoDuration = apiVideoDuration;
+        videoDuration = apiVideoDuration || 0 as VideoDuration;
     }
 
     // Auto moderator check
diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts
index 9ca8333..399e922 100644
--- a/test/cases/postSkipSegments.ts
+++ b/test/cases/postSkipSegments.ts
@@ -97,7 +97,7 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
-    it('Should be able to submit a single time with a duration (JSON method)', (done: Done) => {
+    it('Should be able to submit a single time with a duration from the YouTube API (JSON method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {
             method: 'POST',
@@ -129,7 +129,39 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
-    it('Should be able to submit a single time with a duration from the API (JSON method)', (done: Done) => {
+    it('Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "dQw4w9WgXZH",
+                videoDuration: 5010.20,
+                segments: [{
+                    segment: [1, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]);
+                if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to submit a single time with a duration in the body (JSON method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {
             method: 'POST',

From c17f0b1e6e21a648a1d508810ec1235535065e7b Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Mon, 29 Mar 2021 18:39:55 -0400
Subject: [PATCH 19/32] Add invite link

---
 src/routes/postSkipSegments.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 7738653..0ab6de8 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -388,7 +388,7 @@ export async function postSkipSegments(req: Request, res: Response) {
                 + segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n "
                 + (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " +
                 "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "")
-                + "If you believe this is incorrect, please contact someone on Discord.",
+                + "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app",
             );
             return;
         }

From c9a8dc21b15226e32d521a8dbde0b0517e90647a Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Mon, 29 Mar 2021 19:16:18 -0400
Subject: [PATCH 20/32] Unlock videos and hide segments if duration changed

---
 databases/_upgrade_sponsorTimes_10.sql | 30 ++++++++++++++++++
 src/app.ts                             |  4 +--
 src/routes/deleteNoSegments.ts         | 29 +++++++++++------
 src/routes/getSkipSegments.ts          |  4 +--
 src/routes/postSkipSegments.ts         | 40 ++++++++++++++++-------
 test/cases/getSkipSegments.ts          | 41 +++++++++++++++++-------
 test/cases/getSkipSegmentsByHash.ts    | 26 +++++++++++----
 test/cases/postSkipSegments.ts         | 44 ++++++++++++++++++++++++++
 8 files changed, 175 insertions(+), 43 deletions(-)
 create mode 100644 databases/_upgrade_sponsorTimes_10.sql

diff --git a/databases/_upgrade_sponsorTimes_10.sql b/databases/_upgrade_sponsorTimes_10.sql
new file mode 100644
index 0000000..174ceaf
--- /dev/null
+++ b/databases/_upgrade_sponsorTimes_10.sql
@@ -0,0 +1,30 @@
+BEGIN TRANSACTION;
+
+/* Add Hidden field */
+CREATE TABLE "sqlb_temp_table_10" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+    "videoDuration"	REAL NOT NULL DEFAULT '0',
+    "hidden"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 10 WHERE key = 'version';
+
+COMMIT;
\ No newline at end of file
diff --git a/src/app.ts b/src/app.ts
index 5bc2b2c..2af8835 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -5,7 +5,7 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
 import {postSegmentShift} from './routes/postSegmentShift';
 import {postWarning} from './routes/postWarning';
 import {getIsUserVIP} from './routes/getIsUserVIP';
-import {deleteNoSegments} from './routes/deleteNoSegments';
+import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments';
 import {postNoSegments} from './routes/postNoSegments';
 import {getUserInfo} from './routes/getUserInfo';
 import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
@@ -117,7 +117,7 @@ function setupRoutes(app: Express) {
     //submit video containing no segments
     app.post('/api/noSegments', postNoSegments);
 
-    app.delete('/api/noSegments', deleteNoSegments);
+    app.delete('/api/noSegments', deleteNoSegmentsEndpoint);
 
     //get if user is a vip
     app.get('/api/isUserVIP', getIsUserVIP);
diff --git a/src/routes/deleteNoSegments.ts b/src/routes/deleteNoSegments.ts
index cfa6c78..5584c5a 100644
--- a/src/routes/deleteNoSegments.ts
+++ b/src/routes/deleteNoSegments.ts
@@ -2,12 +2,14 @@ import {Request, Response} from 'express';
 import {isUserVIP} from '../utils/isUserVIP';
 import {getHash} from '../utils/getHash';
 import {db} from '../databases/databases';
+import { Category, VideoID } from '../types/segments.model';
+import { UserID } from '../types/user.model';
 
-export async function deleteNoSegments(req: Request, res: Response) {
+export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
     // Collect user input data
-    const videoID = req.body.videoID;
-    let userID = req.body.userID;
-    const categories = req.body.categories;
+    const videoID = req.body.videoID as VideoID;
+    const userID = req.body.userID as UserID;
+    const categories = req.body.categories as Category[];
 
     // Check input data is valid
     if (!videoID
@@ -23,8 +25,8 @@ export async function deleteNoSegments(req: Request, res: Response) {
     }
 
     // Check if user is VIP
-    userID = getHash(userID);
-    const userIsVIP = await isUserVIP(userID);
+    const hashedUserID = getHash(userID);
+    const userIsVIP = await isUserVIP(hashedUserID);
 
     if (!userIsVIP) {
         res.status(403).json({
@@ -33,13 +35,22 @@ export async function deleteNoSegments(req: Request, res: Response) {
         return;
     }
 
+    deleteNoSegments(videoID, categories);  
+
+    res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
+}
+
+/**
+ * 
+ * @param videoID 
+ * @param categories If null, will remove all
+ */
+export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> {
     const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
-        return (categories.indexOf(entry.category) !== -1);
+        return categories === null || categories.indexOf(entry.category) !== -1;
     });
 
     for (const entry of entries) {
         await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
     }
-
-    res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
 }
diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts
index c2a9155..d36a91a 100644
--- a/src/routes/getSkipSegments.ts
+++ b/src/routes/getSkipSegments.ts
@@ -60,7 +60,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
             .prepare(
                 'all',
                 `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" 
-                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
+                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
                 [videoID, service]
             )).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
                 acc[segment.category] = acc[segment.category] || [];
@@ -132,7 +132,7 @@ async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Serv
         .prepare(
             'all',
             `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
-            WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`,
+            WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
             [hashedVideoIDPrefix + '%', service]
         );
 
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 0ab6de8..37c4985 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -13,7 +13,8 @@ import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
 import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
-import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
+import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
+import { deleteNoSegments } from './deleteNoSegments';
 
 interface APIVideoInfo {
     err: string | boolean,
@@ -357,7 +358,7 @@ export async function postSkipSegments(req: Request, res: Response) {
         return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
     }
 
-    const noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
+    let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
         return list.category;
     });
 
@@ -366,6 +367,31 @@ export async function postSkipSegments(req: Request, res: Response) {
 
     const decreaseVotes = 0;
 
+    let apiVideoInfo: APIVideoInfo = null;
+    if (service == Service.YouTube) {
+        apiVideoInfo = await getYouTubeVideoInfo(videoID);
+    }
+    const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
+    if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
+        // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
+        videoDuration = apiVideoDuration || 0 as VideoDuration;
+    }
+
+    const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 AND "shadowHidden" = 0 AND "votes" >= 0`, [videoID, service]) as 
+                        {videoDuration: VideoDuration, UUID: SegmentUUID}[];
+    // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
+    const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
+    if (videoDurationChanged) {
+        // Hide all previous submissions
+        for (const submission of previousSubmissions) {
+            await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
+        }
+
+        // Reset no segments
+        noSegmentList = [];
+        deleteNoSegments(videoID, null);
+    }
+
     // Check if all submissions are correct
     for (let i = 0; i < segments.length; i++) {
         if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@@ -419,16 +445,6 @@ export async function postSkipSegments(req: Request, res: Response) {
         }
     }
 
-    let apiVideoInfo: APIVideoInfo = null;
-    if (service == Service.YouTube) {
-        apiVideoInfo = await getYouTubeVideoInfo(videoID);
-    }
-    const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
-    if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
-        // If api duration is far off, take that one instead (it is only precise to seconds, not millis)
-        videoDuration = apiVideoDuration || 0 as VideoDuration;
-    }
-
     // Auto moderator check
     if (!isVIP && service == Service.YouTube) {
         const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category});
diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts
index 674a773..f40203c 100644
--- a/test/cases/getSkipSegments.ts
+++ b/test/cases/getSkipSegments.ts
@@ -5,18 +5,19 @@ import {getHash} from '../../src/utils/getHash';
 
 describe('getSkipSegments', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, '" + getHash('testtesttest2', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, '" + getHash('testtesttest,test', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, '" + getHash('locked', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, '" + getHash('locked', 1) + "')");
-    
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 0, '" + getHash('testtesttest,test', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', 0, 50, 'sponsor', 'YouTube', 190, 1, 0, '" + getHash('onlyHiddenSegments', 1) + "')");
+
         return;
     });
 
@@ -106,6 +107,22 @@ describe('getSkipSegments', () => {
         .catch(err => ("Couldn't call endpoint"));
     });
 
+    it('Should be empty if all submissions are hidden', () => {
+        fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments")
+        .then(async res => {
+            if (res.status !== 200) return ("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 0) {
+                    return;
+                } else {
+                    return ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => ("Couldn't call endpoint"));
+    });
+
     it('Should be able to get multiple times by category', () => {
         fetch(getbaseURL() + "/api/skipSegments?videoID=multiple&categories=[\"intro\"]")
         .then(async res => {
diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts
index fd45884..23d9d49 100644
--- a/test/cases/getSkipSegmentsByHash.ts
+++ b/test/cases/getSkipSegmentsByHash.ts
@@ -12,12 +12,13 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
 
 describe('getSegmentsByHash', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "hidden", "shadowHidden", "hashedVideoID") VALUES';
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+        await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3
     });
 
     it('Should be able to get a 200', (done: Done) => {
@@ -55,6 +56,19 @@ describe('getSegmentsByHash', () => {
         .catch(err => done("Couldn't call endpoint"));
     });
 
+    it('Should be able to get an empty array if only hidden videos', (done: Done) => {
+        fetch(getbaseURL() + '/api/skipSegments/f3a1?categories=["sponsor"]')
+        .then(async res => {
+            if (res.status !== 404) done("non 404 status code, was " + res.status);
+            else {
+                const body = await res.text();
+                if (JSON.parse(body).length === 0 && body === '[]') done(); // pass
+                else done("non empty array returned");
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
     it('Should return 400 prefix too short', (done: Done) => {
         fetch(getbaseURL() + '/api/skipSegments/11?categories=["shilling"]')
         .then(res => {
diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts
index 399e922..512c2ee 100644
--- a/test/cases/postSkipSegments.ts
+++ b/test/cases/postSkipSegments.ts
@@ -6,6 +6,7 @@ import {db} from '../../src/databases/databases';
 import {ImportMock} from 'ts-mock-imports';
 import * as YouTubeAPIModule from '../../src/utils/youtubeApi';
 import {YouTubeApiMock} from '../youtubeMock';
+import e from 'express';
 
 const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, 'YouTubeAPI');
 const sinonStub = mockManager.mock('listVideos');
@@ -193,6 +194,49 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
+    it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
+        await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") 
+            VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')");
+
+        try {
+            const res = await fetch(getbaseURL()
+                + "/api/postVideoSponsorTimes", {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    userID: "test",
+                    videoID: "noDuration",
+                    videoDuration: 100,
+                    segments: [{
+                        segment: [1, 10],
+                        category: "sponsor",
+                    }],
+                }),
+            });
+
+            if (res.status === 200) {
+                const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]);
+                const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" 
+                    FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
+                const videoRow = videoRows[0];
+                const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" 
+                    FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
+                if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10 
+                        && videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
+                    return;
+                } else {
+                    return "Submitted times were not saved. Actual submission: " + JSON.stringify(videoRow);
+                }
+            } else {
+                return "Status code was " + res.status;
+            }
+        } catch (e) {
+            return e;
+        }
+    });
+
     it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {

From 93d724202196917d48770e7962dd80252928bd64 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Tue, 30 Mar 2021 22:50:09 -0400
Subject: [PATCH 21/32] Add preview category

---
 src/config.ts | 2 +-
 test.json     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/config.ts b/src/config.ts
index 93c6a43..7b15987 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -16,7 +16,7 @@ addDefaults(config, {
     privateDBSchema: "./databases/_private.db.sql",
     readOnly: false,
     webhooks: [],
-    categoryList: ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+    categoryList: ["sponsor", "selfpromo", "interaction",  "intro", "outro", "preview", "music_offtopic"],
     maxNumberOfActiveWarnings: 3,
     hoursAfterWarningExpires: 24,
     adminUserID: "",
diff --git a/test.json b/test.json
index add6ebf..58eb72a 100644
--- a/test.json
+++ b/test.json
@@ -49,7 +49,7 @@
             ]
         }
     ],
-    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+    "categoryList": ["sponsor", "selfpromo", "interaction",  "intro", "outro", "preview", "music_offtopic"],
     "maxNumberOfActiveWarnings": 3,
     "hoursAfterWarningExpires": 24,
     "rateLimit": {

From 1eca55d96c4a327c3dbd89af6d4cc74ad24afc22 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 31 Mar 2021 21:52:03 +0000
Subject: [PATCH 22/32] Bump y18n from 4.0.0 to 4.0.1

Bumps [y18n](https://github.com/yargs/y18n) from 4.0.0 to 4.0.1.
- [Release notes](https://github.com/yargs/y18n/releases)
- [Changelog](https://github.com/yargs/y18n/blob/master/CHANGELOG.md)
- [Commits](https://github.com/yargs/y18n/commits)

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index dd69f2e..f17b109 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2989,9 +2989,9 @@
       "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
     },
     "y18n": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
-      "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz",
+      "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==",
       "dev": true
     },
     "yallist": {

From cfbf8a47d7664ea212392bf016c8cec1612fa295 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 2 Apr 2021 12:03:21 -0400
Subject: [PATCH 23/32] Fix hashing function

---
 src/databases/Postgres.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts
index e7e16f4..779ebf2 100644
--- a/src/databases/Postgres.ts
+++ b/src/databases/Postgres.ts
@@ -136,7 +136,7 @@ export class Postgres implements IDatabase {
 
     private processUpgradeQuery(query: string): string {
         let result = query;
-        result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
+        result = result.replace(/sha256\((.*?)\)/gm, "encode(digest($1, 'sha256'), 'hex')");
         result = result.replace(/integer/gmi, "NUMERIC");
 
         return result;

From bc688a3d8dcc765a12618559ff20819bbd7d58a8 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Fri, 2 Apr 2021 16:40:51 -0400
Subject: [PATCH 24/32] FIx duration issue

---
 src/routes/postSkipSegments.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
index 37c4985..1f380da 100644
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -377,7 +377,8 @@ export async function postSkipSegments(req: Request, res: Response) {
         videoDuration = apiVideoDuration || 0 as VideoDuration;
     }
 
-    const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 AND "shadowHidden" = 0 AND "votes" >= 0`, [videoID, service]) as 
+    const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 
+                    AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as 
                         {videoDuration: VideoDuration, UUID: SegmentUUID}[];
     // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
     const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);

From 77a4c2fe34d67002b2427d921c8599cd224ab03d Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Tue, 13 Apr 2021 03:04:02 +0200
Subject: [PATCH 25/32] Update nginx config

---
 nginx/nginx.conf | 87 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 63 insertions(+), 24 deletions(-)

diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index daee81f..76a77d3 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -2,7 +2,7 @@ worker_processes 8;
 worker_rlimit_nofile 8192;
 
 events {
-  worker_connections  32768;  ## Default: 1024
+  worker_connections  132768;  ## Default: 1024
 }
 
 http {
@@ -12,19 +12,35 @@ http {
   
     upstream backend_GET {
 	least_conn;        
-        server localhost:4442;
-        server localhost:4443;
-        server localhost:4444;
-        server localhost:4445;
-        server localhost:4446;
+        server localhost:4441;
+	server localhost:4442;
+        #server localhost:4443;
+        #server localhost:4444;
+        #server localhost:4445;
+        #server localhost:4446;
         #server localhost:4447;
         #server localhost:4448;
+
+	server 10.0.0.3:4441;        
+	server 10.0.0.3:4442;
+
+	#server 134.209.69.251:80 backup;
+
+        server 116.203.32.253:80 backup;
+        #server 116.203.32.253:80;
     }
     upstream backend_POST {
-        server localhost:4441;
+        #server localhost:4441;
+        #server localhost:4442;
+        server 10.0.0.3:4441;
+	#server 10.0.0.3:4442;
+    }
+    upstream backend_db {
+        #server localhost:4441;
+	server 10.0.0.3:4441;
     }
 
-    proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m;
+    proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
     proxy_cache_key "$scheme$request_method$host$request_uri";
     add_header X-Cache $upstream_cache_status;
 
@@ -43,14 +59,16 @@ http {
 	#	internal;
 	#}
 
+	#proxy_send_timeout 120s;
+
 	location @myerrordirective_500 {
- 	    return 502 "Internal Server Error";
+ 	    return 400 "Internal Server Error";
 	}
 	location @myerrordirective_502 {
-            return 502 "Bad Gateway";
+            return 400 "Bad Gateway";
         }
 	location @myerrordirective_504 {
-            return 502 "Gateway Timeout";
+            return 400 "Gateway Timeout";
         }
 
 
@@ -62,17 +80,16 @@ http {
 	    return 301 https://sb.ltn.fi;
 	}
 
-	location /invidious/ {
-	    proxy_pass https://invidious.fdn.fr/;
-	}
-
 	location /test/ {
 	    proxy_pass http://localhost:4440/;
             #proxy_pass https://sbtest.etcinit.com/;
 	}
 
         location /api/skipSegments {
-            proxy_pass http://backend_$request_method;
+            #return 200 "[]";
+	    proxy_pass http://backend_$request_method;
+            #proxy_cache CACHEZONE;
+            #proxy_cache_valid  2m;
         }
 
         location /api/getTopUsers {
@@ -83,24 +100,43 @@ http {
 
         location /api/getTotalStats {
             proxy_pass http://backend_GET;
-        }
+            #return 200 "";
+	}
 
         
         location /api/getVideoSponsorTimes {
             proxy_pass http://backend_GET;
         }
-        
-        location = /database.db {
-            alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
+     
+	location /database {
+            proxy_pass http://backend_db;
         }
-        
+   
+        location = /database.db {
+	    #return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
+            #alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
+            alias /home/sbadmin/test-db/database.db;
+	}
+       
+	location = /database/sponsorTimes.csv {
+	    alias /home/sbadmin/sponsorTimes.csv;
+	}
+
+	#location /api/voteOnSponsorTime {
+	#    return 200 "Success";
+	#}
+ 
+	#location /api/viewedVideoSponsorTime {
+        #    return 200 "Success";
+        #}
+
         location /api {
             proxy_pass http://backend_POST;
         }
         
         location / {
-            root /home/sbadmin/caddy/SponsorBlockSite/public-prod;
-            
+            root /home/sbadmin/SponsorBlockSite/public-prod;            
+
             ### CORS
             if ($request_method = 'OPTIONS') {
                 add_header 'Access-Control-Allow-Origin' '*';
@@ -132,12 +168,15 @@ http {
         }
         
     
-    listen 443 ssl; # managed by Certbot
+    listen 443 default_server ssl; # managed by Certbot
+    #listen 80;
     ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot
     ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot
     include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
     ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
 
+
+
 }
     
     

From 5b2f05741ed8e35d114123fab4cc2717b0c97bb4 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 17 Apr 2021 16:23:45 -0400
Subject: [PATCH 26/32] Fix no segments votes being allowed

---
 src/routes/voteOnSponsorTime.ts | 76 +++++++++++++++++----------------
 test/cases/voteOnSponsorTime.ts | 21 +++++----
 2 files changed, 52 insertions(+), 45 deletions(-)

diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts
index 29f0e66..a1e81e7 100644
--- a/src/routes/voteOnSponsorTime.ts
+++ b/src/routes/voteOnSponsorTime.ts
@@ -175,51 +175,54 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
     const timeSubmitted = Date.now();
 
     const voteAmount = isVIP ? 500 : 1;
+    const ableToVote = isVIP || finalResponse.finalStatus === 200 || true;
 
-    // Add the vote
-    if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
-        // Update the already existing db entry
-        await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
-    } else {
-        // Add a db entry
-        await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
-    }
+    if (ableToVote) {
+        // Add the vote
+        if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
+            // Update the already existing db entry
+            await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
+        } else {
+            // Add a db entry
+            await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
+        }
 
-    // Add the info into the private db
-    if (usersLastVoteInfo?.votes > 0) {
-        // Reverse the previous vote
-        await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
+        // Add the info into the private db
+        if (usersLastVoteInfo?.votes > 0) {
+            // Reverse the previous vote
+            await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
 
-        await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
-    } else {
-        await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
-    }
+            await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
+        } else {
+            await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
+        }
 
-    // See if the submissions category is ready to change
-    const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
+        // See if the submissions category is ready to change
+        const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
 
-    const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
-    const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
-    const startingVotes = isSubmissionVIP ? 10000 : 1;
+        const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
+        const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
+        const startingVotes = isSubmissionVIP ? 10000 : 1;
 
-    // Change this value from 1 in the future to make it harder to change categories
-    // Done this way without ORs incase the value is zero
-    const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
+        // Change this value from 1 in the future to make it harder to change categories
+        // Done this way without ORs incase the value is zero
+        const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
 
-    // Add submission as vote
-    if (!currentCategoryInfo && submissionInfo) {
-        await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
+        // Add submission as vote
+        if (!currentCategoryInfo && submissionInfo) {
+            await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
 
-        await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
-    }
+            await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
+        }
 
-    const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
+        const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
 
-    //TODO: In the future, raise this number from zero to make it harder to change categories
-    // VIPs change it every time
-    if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
-        // Replace the category
-        await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
+        //TODO: In the future, raise this number from zero to make it harder to change categories
+        // VIPs change it every time
+        if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
+            // Replace the category
+            await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
+        }
     }
 
     res.sendStatus(finalResponse.finalStatus);
@@ -371,7 +374,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         const ableToVote = isVIP
             || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
                 && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
-                && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined);
+                && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
+                && finalResponse.finalStatus === 200;
 
         if (ableToVote) {
             //update the votes table
diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts
index 47a1f19..2aae65f 100644
--- a/test/cases/voteOnSponsorTime.ts
+++ b/test/cases/voteOnSponsorTime.ts
@@ -410,12 +410,13 @@ describe('voteOnSponsorTime', () => {
 
     it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => {
         fetch(getbaseURL()
-            + "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=0")
+            + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0")
         .then(async res => {
-            if (res.status === 403) {
+            let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
+            if (res.status === 403 && row.votes === 2) {
                 done();
             } else {
-                done("Status code was " + res.status + " instead of 403");
+                done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
             }
         })
         .catch(err => done(err));
@@ -423,12 +424,13 @@ describe('voteOnSponsorTime', () => {
 
     it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => {
         fetch(getbaseURL()
-            + "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=1")
+            + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=1")
         .then(async res => {
-            if (res.status === 200) {
+            let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
+            if (res.status === 200 && row.votes === 3) {
                 done();
             } else {
-                done("Status code was " + res.status + " instead of 200");
+                done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
             }
         })
         .catch(err => done(err));
@@ -436,12 +438,13 @@ describe('voteOnSponsorTime', () => {
 
     it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => {
         fetch(getbaseURL()
-            + "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&category=outro")
+            + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro")
         .then(async res => {
-            if (res.status === 403) {
+            let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
+            if (res.status === 403 && row.category === "sponsor") {
                 done();
             } else {
-                done("Status code was " + res.status + " instead of 403");
+                done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
             }
         })
         .catch(err => done(err));

From 9d06bda4f8d93942e39ec6aa1c7eb2f6eeedfdcd Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 17 Apr 2021 16:37:39 -0400
Subject: [PATCH 27/32] Don't allow downvoting dead submissions

---
 src/routes/voteOnSponsorTime.ts | 12 +++++++++---
 test/cases/voteOnSponsorTime.ts | 19 +++++++++++++++++--
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts
index 240baf9..4cb7690 100644
--- a/src/routes/voteOnSponsorTime.ts
+++ b/src/routes/voteOnSponsorTime.ts
@@ -286,13 +286,19 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
         return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res);
     }
 
-    if (type == 1 && !isVIP && !isOwnSubmission) {
+    if (type !== undefined && !isVIP && !isOwnSubmission) {
         // Check if upvoting hidden segment
         const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
 
         if (voteInfo && voteInfo.votes <= -2) {
-            res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
-            return;
+            if (type == 1) {
+                res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
+                return;
+            } else if (type == 0) {
+                // Already downvoted enough, ignore
+                res.status(200).send();
+                return;
+            }
         }
     }
 
diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts
index 2aae65f..86c3233 100644
--- a/test/cases/voteOnSponsorTime.ts
+++ b/test/cases/voteOnSponsorTime.ts
@@ -368,10 +368,25 @@ describe('voteOnSponsorTime', () => {
         fetch(getbaseURL()
             + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1")
         .then(async res => {
-            if (res.status === 403) {
+            let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
+            if (res.status === 403 && row.votes === -3) {
                 done();
             } else {
-                done("Status code was " + res.status + " instead of 403");
+                done("Status code was " + res.status + ", row is " + JSON.stringify(row));
+            }
+        })
+        .catch(err => done(err));
+    });
+
+    it('Non-VIP should not be able to downvote "dead" submission', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=0")
+        .then(async res => {
+            let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
+            if (res.status === 200 && row.votes === -3) {
+                done();
+            } else {
+                done("Status code was " + res.status + ", row is " + JSON.stringify(row));
             }
         })
         .catch(err => done(err));

From 058c05a1f72f77276c21c53f07c4adc8d43f0cf3 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sun, 18 Apr 2021 04:49:05 +0200
Subject: [PATCH 28/32] Fix permission issues

---
 docker/docker-compose.yml |  8 ++++----
 nginx/nginx.conf          | 10 +++++++---
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 4ee69ee..f7408d3 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -7,9 +7,9 @@ services:
       - database.env
     volumes:
       - database-data:/var/lib/postgresql/data
-      - ./database-export/:/opt/exports
+      - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
     ports:
-      - 127.0.0.1:5432:5432
+      - 5432:5432
   redis:
     container_name: redis
     image: redis
@@ -17,7 +17,7 @@ services:
     volumes:
       - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
     ports:
-      - 127.0.0.1:32773:6379
+      - 32773:6379
 
 volumes:
-  database-data:
\ No newline at end of file
+  database-data:
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index 76a77d3..6302691 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -108,6 +108,9 @@ http {
             proxy_pass http://backend_GET;
         }
      
+	location /database/ {
+	     alias /home/sbadmin/sponsor/docker/database-export/;
+	}
 	location /database {
             proxy_pass http://backend_db;
         }
@@ -118,9 +121,10 @@ http {
             alias /home/sbadmin/test-db/database.db;
 	}
        
-	location = /database/sponsorTimes.csv {
-	    alias /home/sbadmin/sponsorTimes.csv;
-	}
+	#location = /database/sponsorTimes.csv {
+	#    alias /home/sbadmin/sponsorTimes.csv;
+	#}
+	
 
 	#location /api/voteOnSponsorTime {
 	#    return 200 "Success";

From a06ab724adab93c082cb7a972c859768304b8c24 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 17 Apr 2021 23:06:39 -0400
Subject: [PATCH 29/32] Fix file locations + formatting

---
 src/config.ts              |  2 +-
 src/routes/dumpDatabase.ts | 28 +++++++++++++---------------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/config.ts b/src/config.ts
index 0e985dc..6139a6e 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -49,7 +49,7 @@ addDefaults(config, {
     dumpDatabase: {
         enabled: true,
         minTimeBetweenMs: 60000,
-        appExportPath: '/opt/exports',
+        appExportPath: './docker/database-export',
         postgresExportPath: '/opt/exports',
         tables: [{
             name: "sponsorTimes",
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
index b7f2726..8e33440 100644
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -2,11 +2,10 @@ import {db} from '../databases/databases';
 import {Logger} from '../utils/logger';
 import {Request, Response} from 'express';
 import { config } from '../config';
-const util = require('util');
-const fs = require('fs');
-const path = require('path');
+import util from 'util';
+import fs from 'fs';
+import path from 'path';
 const unlink = util.promisify(fs.unlink);
-const fstat = util.promisify(fs.fstat);
 
 const ONE_MINUTE = 1000 * 60;
 
@@ -16,7 +15,7 @@ const styleHeader = `<style>
     }
     table th,
     table td {
-        padding: 2px 4px;
+        padding: 7px;
     }
     table th {
         text-align: left;
@@ -32,7 +31,7 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
 
 const tables = config?.dumpDatabase?.tables ?? [];
 const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
-const appExportPath = config?.dumpDatabase?.appExportPath ?? '/opt/exports';
+const appExportPath = config?.dumpDatabase?.appExportPath ?? './docker/database-export';
 const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? '/opt/exports';
 const tableNames = tables.map(table => table.name);
 
@@ -46,14 +45,9 @@ if (tables.length === 0) {
     Logger.warn('[dumpDatabase] No tables configured');
 }
 
-const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
-
-const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
-                        .reduce((acc, url) => acc + url, "");
-
 let lastUpdate = 0;
 
-function removeOutdatedDumps(exportPath: string): Promise<any> {
+function removeOutdatedDumps(exportPath: string): Promise<void> {
     return new Promise((resolve, reject) => {
         // Get list of table names
         // Create array for each table
@@ -61,10 +55,12 @@ function removeOutdatedDumps(exportPath: string): Promise<any> {
             obj[tableName] = [];
             return obj;
         }, {});
+
         // read files in export directory
         fs.readdir(exportPath, (err: any, files: string[]) => {
             if (err) Logger.error(err);
             if (err) return resolve();
+
             files.forEach(file => {
                 // we only care about files that start with "<tablename>_" and ends with .csv
                 tableNames.forEach(tableName => {
@@ -79,6 +75,7 @@ function removeOutdatedDumps(exportPath: string): Promise<any> {
                     }
                 });
             });
+
             const outdatedTime = Math.floor(Date.now() - (MILLISECONDS_BETWEEN_DUMPS * 1.5));
             for (let tableName in tableFiles) {
                 const files = tableFiles[tableName];
@@ -91,13 +88,14 @@ function removeOutdatedDumps(exportPath: string): Promise<any> {
                     }
                 });
             }
+
             resolve();
         });
     });
 }
 
 export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) {
-    if (config?.dumpDatabase?.enabled === false) {
+    if (!config?.dumpDatabase?.enabled) {
         res.status(404).send("Database dump is disabled");
         return;
     }
@@ -130,7 +128,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
                     return `
                     <tr>
                         <td>${item.tableName}</td>
-                        <td><a href="/download/${item.fileName}">${item.fileName}</a></td>
+                        <td><a href="/database/${item.fileName}">${item.fileName}</a></td>
                     </tr>
                     `;
                 }).join('')}
@@ -146,7 +144,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
             links: latestDumpFiles.map((item:any) => {
                 return {
                     table: item.tableName,
-                    url: `/download/${item.fileName}`,
+                    url: `/database/${item.fileName}`,
                     size: item.fileSize,
                 };
             }),

From 911ebddd69acca9bf5a485e5b5e6d1178934bfc8 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 17 Apr 2021 23:09:21 -0400
Subject: [PATCH 30/32] Fix example config

---
 config.json.example | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.json.example b/config.json.example
index 580b8fb..5319b66 100644
--- a/config.json.example
+++ b/config.json.example
@@ -42,7 +42,7 @@
     "dumpDatabase": {
         "enabled": true,
         "minTimeBetweenMs": 60000, // 1 minute between dumps
-        "appExportPath": "/opt/exports",
+        "appExportPath": "./docker/database-export",
         "postgresExportPath": "/opt/exports",
         "tables": [{
             "name": "sponsorTimes",

From 2814ce7b7f95f4c4e0fcba2ed801341b31049489 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sun, 18 Apr 2021 12:28:47 -0400
Subject: [PATCH 31/32] Change dump deletion logic

---
 src/routes/dumpDatabase.ts | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
index 8e33440..7610589 100644
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -41,6 +41,11 @@ interface TableDumpList {
 };
 let latestDumpFiles: TableDumpList[] = [];
 
+interface TableFile {
+    file: string,
+    timestamp: number
+};
+
 if (tables.length === 0) {
     Logger.warn('[dumpDatabase] No tables configured');
 }
@@ -51,42 +56,37 @@ function removeOutdatedDumps(exportPath: string): Promise<void> {
     return new Promise((resolve, reject) => {
         // Get list of table names
         // Create array for each table
-        const tableFiles = tableNames.reduce((obj: any, tableName) => {
+        const tableFiles: Record<string, TableFile[]> = tableNames.reduce((obj: any, tableName) => {
             obj[tableName] = [];
             return obj;
         }, {});
 
         // read files in export directory
-        fs.readdir(exportPath, (err: any, files: string[]) => {
+        fs.readdir(exportPath, async (err: any, files: string[]) => {
             if (err) Logger.error(err);
             if (err) return resolve();
 
             files.forEach(file => {
                 // we only care about files that start with "<tablename>_" and ends with .csv
                 tableNames.forEach(tableName => {
-                    if (file.startsWith(`${tableName}_`) && file.endsWith('.csv')) {
-                        // extract the timestamp from the filename
-                        // we could also use the fs.stat mtime
-                        const timestamp = Number(file.split('_')[1].replace('.csv', ''));
+                    if (file.startsWith(`${tableName}`) && file.endsWith('.csv')) {
+                        const filePath = path.join(exportPath, file);
                         tableFiles[tableName].push({
-                            file: path.join(exportPath, file),
-                            timestamp,
+                            file: filePath,
+                            timestamp: fs.statSync(filePath).mtime.getTime()
                         });
                     }
                 });
             });
 
-            const outdatedTime = Math.floor(Date.now() - (MILLISECONDS_BETWEEN_DUMPS * 1.5));
             for (let tableName in tableFiles) {
-                const files = tableFiles[tableName];
-                files.forEach(async (item: any) => {
-                    if (item.timestamp < outdatedTime) {
-                        // remove old file
-                        await unlink(item.file).catch((error: any) => {
-                            Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
-                        });
-                    }
-                });
+                const files = tableFiles[tableName].sort((a, b) => b.timestamp - a.timestamp);
+                for (let i = 2; i < files.length; i++) {
+                    // remove old file
+                    await unlink(files[i].file).catch((error: any) => {
+                        Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
+                    });
+                }
             }
 
             resolve();

From b6660d656f0e5867e1d09c8e08362322ebecf631 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sun, 18 Apr 2021 21:10:43 +0200
Subject: [PATCH 32/32] Add cdn redirect

---
 nginx/nginx.conf | 86 +++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 77 insertions(+), 9 deletions(-)

diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index 6302691..d420877 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -36,8 +36,8 @@ http {
 	#server 10.0.0.3:4442;
     }
     upstream backend_db {
-        #server localhost:4441;
-	server 10.0.0.3:4441;
+        server localhost:4441;
+	#server 10.0.0.3:4441;
     }
 
     proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
@@ -47,9 +47,6 @@ http {
     server {
         server_name sponsor.ajay.app api.sponsor.ajay.app;
 
-	access_log off;
-        error_log /dev/null;
-
 	error_page 404 /404.html;
 	error_page 500 @myerrordirective_500;
 	error_page 502 @myerrordirective_502;
@@ -109,7 +106,7 @@ http {
         }
      
 	location /database/ {
-	     alias /home/sbadmin/sponsor/docker/database-export/;
+	     return 307 https://cdnsponsor.ajay.app$request_uri;
 	}
 	location /database {
             proxy_pass http://backend_db;
@@ -172,17 +169,74 @@ http {
         }
         
     
-    listen 443 default_server ssl; # managed by Certbot
+    listen 443 default_server ssl http2; # managed by Certbot
+    #listen 443 http3 reuseport;
+    ssl_protocols TLSv1.2 TLSv1.3;
     #listen 80;
-    ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot
-    ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
     include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
     ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
 
 
 
+
+
+
+
 }
     
+    server {
+        server_name cdnsponsor.ajay.app;
+
+	error_page 404 /404.html;
+	
+	location /database/ {
+	     alias /home/sbadmin/sponsor/docker/database-export/;
+	}
+
+	location / {
+            root /home/sbadmin/SponsorBlockSite/public-prod;            
+
+            ### CORS
+            if ($request_method = 'OPTIONS') {
+                add_header 'Access-Control-Allow-Origin' '*';
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+                #
+                # Custom headers and headers various browsers *should* be OK with but aren't
+                #
+                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+                #
+                # Tell client that this pre-flight info is valid for 20 days
+                #
+                add_header 'Access-Control-Max-Age' 1728000;
+                add_header 'Content-Type' 'text/plain; charset=utf-8';
+                add_header 'Content-Length' 0;
+                return 204;
+             }
+             if ($request_method = 'POST') {
+                add_header 'Access-Control-Allow-Origin' '*';
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
+             }
+             if ($request_method = 'GET') {
+                add_header 'Access-Control-Allow-Origin' '*';
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+                add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
+             }
+        }
+    
+
+    listen 443 ssl; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+
+
+}
     
 
 
@@ -191,6 +245,7 @@ http {
 
 	 access_log off;
         error_log /dev/null;
+	
 
     if ($host = api.sponsor.ajay.app) {
         return 301 https://$host$request_uri;
@@ -209,4 +264,17 @@ http {
 
 
 
+}
+    
+    server {
+    if ($host = cdnsponsor.ajay.app) {
+        return 301 https://$host$request_uri;
+    } # managed by Certbot
+
+
+        server_name cdnsponsor.ajay.app;
+    listen 80;
+    return 404; # managed by Certbot
+
+
 }}