Merge pull request #185 from LoganDark/patch-1

Correct indentation in _private.db.sql

.github/workflows/ci.yml

@@ -0,0 +1,18 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+
+  build:
+    name: Run Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      # Initialization
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v1
+      - run: npm install
+
+      - name: Run Tests
+        run: npm test

.gitignore

@@ -88,4 +88,18 @@ typings/
 .dynamodb/
 
 # Databases
-databases
+databases/sponsorTimes.db
+databases/sponsorTimes.db-shm
+databases/sponsorTimes.db-wal
+databases/private.db
+databases/sponsorTimesReal.db
+test/databases/sponsorTimes.db
+test/databases/sponsorTimes.db-shm
+test/databases/sponsorTimes.db-wal
+test/databases/private.db
+
+# Config files
+config.json
+
+# Mac files
+.DS_Store

Dockerfile

@@ -0,0 +1,11 @@
+FROM node:12
+WORKDIR /usr/src/app
+COPY package.json .
+RUN npm install
+COPY index.js .
+COPY src src
+RUN mkdir databases
+COPY databases/*.sql databases/
+COPY entrypoint.sh .
+EXPOSE 8080
+CMD ./entrypoint.sh

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Ajay Ramachandran
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.MD

@@ -8,10 +8,30 @@ This is the server backend for it
 
 This is a simple Sqlite database that will hold all the timing data.
 
-To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. So, you can download a backup or get archive.org to take a backup if you do desire.
+To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
 
 Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.
 
 # Client
 
 The client web browser extension is available here: https://github.com/ajayyy/SponsorBlock
+
+# Build Yourself
+
+This is a node.js server, so clone this repo and run `npm install` to install all dependencies.
+
+Make sure to put the database files in the `./databases` folder if you want to use a pre-existing database. Otherwise, a fresh database will be created.
+
+Rename `config.json.example` to `config.json` and fill the parameters inside. Make sure to remove the comments as comments are not supported in JSON.
+
+Ensure all the tests pass with `npm test`
+
+Run the server with `npm start`.
+
+# Developing
+
+If you want to make changes, run `npm run dev` to automatically reload the server and run tests whenever a file is saved.
+
+# API Docs
+
+Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs)

config.json.example

@@ -0,0 +1,42 @@
+{
+    // Remove all comments from the config when using it!
+
+    "port": 80,
+    "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]",
+    "adminUserID": "[the hashed id of the user who can perform admin actions]",
+    "youtubeAPIKey": null, //get this from Google Cloud Platform [optional]
+    "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
+    "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
+    "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
+    "neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app
+    "discordNeuralBlockRejectWebhookURL": null, //URL from discord if you would like notifications when NeuralBlock rejects a submission [optional]
+    "userCounterURL": null, // For user counting. URL to instance of https://github.com/ajayyy/PrivacyUserCount
+    "proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash)
+    "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For"
+    "db": "./databases/sponsorTimes.db",
+    "privateDB": "./databases/private.db",
+    "createDatabaseIfNotExist": true, //This will run on startup every time (unless readOnly is true) - so ensure "create table if not exists" is used in the schema
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    "mode": "development",
+    "readOnly": false,
+    "webhooks": [],
+    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected
+    "getTopUsersCacheTimeMinutes": 5, // cacheTime for getTopUsers result in minutes
+    "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire
+    "hoursAfterWarningExpire": 24,
+    "rateLimit": {
+      "vote": {
+        "windowMs": 900000, // 15 minutes
+        "max": 20, // 20 requests in 15min time window
+        "message": "Too many votes, please try again later",
+        "statusCode": 429
+      },
+      "view": {
+        "windowMs": 900000, // 15 minutes
+        "max": 20, // 20 requests in 15min time window
+        "statusCode": 200
+      }
+    }
+}

databases/_private.db.sql

@@ -0,0 +1,35 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
+	"userID"	TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "votes" (
+	"UUID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"type"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "categoryVotes" (
+	"UUID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "sponsorTimes" (
+	"videoID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "config" (
+	"key" TEXT NOT NULL,
+	"value" TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS sponsorTimes_hashedIP on sponsorTimes(hashedIP);
+CREATE INDEX IF NOT EXISTS votes_userID on votes(UUID);
+
+COMMIT;

databases/_sponsorTimes.db.sql

@@ -0,0 +1,36 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "vipUsers" (
+	"userID"	TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "sponsorTimes" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+    "category" TEXT NOT NULL,
+	"shadowHidden"	INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "userNames" (
+	"userID"	TEXT NOT NULL,
+	"userName"	TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "categoryVotes" (
+	"UUID"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL,
+	"votes"	INTEGER NOT NULL default '0'
+);
+
+CREATE TABLE IF NOT EXISTS "config" (
+    "key" TEXT NOT NULL UNIQUE,
+	"value" TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID);
+CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID);
+
+COMMIT;

databases/_upgrade_private_1.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* for testing the db upgrade, don't remove because it looks empty */
+
+/* Add version to config */
+INSERT INTO config (key, value) VALUES("version", 1);
+
+COMMIT;

databases/_upgrade_sponsorTimes_1.sql

@@ -0,0 +1,25 @@
+BEGIN TRANSACTION;
+
+/* Add incorrectVotes field */
+CREATE TABLE "sqlb_temp_table_1" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT "sponsor",
+	"shadowHidden"	INTEGER NOT NULL
+);
+INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
+
+DROP TABLE sponsorTimes;
+ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
+
+/* Add version to config */
+INSERT INTO config (key, value) VALUES("version", 1);
+
+COMMIT;

databases/_upgrade_sponsorTimes_2.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+/* Add new table: noSegments */
+CREATE TABLE "noSegments" (
+	"videoID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL
+);
+
+/* Add version to config */
+UPDATE config SET value = 2 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_3.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' 
+/* Add hash field */
+ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default "";
+UPDATE sponsorTimes SET hashedVideoID = sha256(videoID);
+
+CREATE INDEX IF NOT EXISTS sponsorTimes_hashedVideoID on sponsorTimes(hashedVideoID);
+
+/* Bump version in config */
+UPDATE config SET value = 3 WHERE key = "version";
+
+COMMIT;

databases/_upgrade_sponsorTimes_4.sql

@@ -0,0 +1,12 @@
+BEGIN TRANSACTION;
+
+/* Create warnings table */
+CREATE TABLE "warnings" (
+    userID TEXT NOT NULL,
+    issueTime INTEGER NOT NULL,
+    issuerUserID TEXT NOT NULL
+);
+
+UPDATE config SET value = 4 WHERE key = "version";
+
+COMMIT;

entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+echo 'Entrypoint script'
+cd /usr/src/app
+cp /etc/sponsorblock/config.json . || cat <<EOF > config.json
+{
+    "port": 8080,
+    "globalSalt": "[CHANGE THIS]",
+    "adminUserID": "[CHANGE THIS]",
+    "youtubeAPIKey": null,
+    "discordReportChannelWebhookURL": null, 
+    "discordFirstTimeSubmissionsWebhookURL": null, 
+    "discordAutoModWebhookURL": null,
+    "proxySubmission": null,
+    "behindProxy": "X-Forwarded-For",
+    "db": "./databases/sponsorTimes.db",
+    "privateDB": "./databases/private.db",
+    "createDatabaseIfNotExist": true,
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    "mode": "development",
+    "readOnly": false
+}
+EOF
+node index.js

index.js

@@ -1,497 +1,6 @@
-var express = require('express');
-var http = require('http');
-
-// Create a service (the app object is just a callback).
-var app = express();
-
-//uuid service
-var uuidv1 = require('uuid/v1');
-
-//hashing service
-var crypto = require('crypto');
-
-//load database
-var sqlite3 = require('sqlite3').verbose();
-var db = new sqlite3.Database('./databases/sponsorTimes.db');
-//where the more sensitive data such as IP addresses are stored
-var privateDB = new sqlite3.Database('./databases/private.db');
-
-// Create an HTTP service.
-http.createServer(app).listen(80);
-
-//global salt that is added to every ip before hashing to
-//  make it even harder for someone to decode the ip
-var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb";
-
-//setup CORS correctly
-app.use(function(req, res, next) {
-    res.header("Access-Control-Allow-Origin", "*");
-    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-    next();
+var config = require('./src/config.js');
+var createServer = require('./src/app.js');
+const logger = require('./src/utils/logger.js');
+var server = createServer(() => {
+  logger.info("Server started on port " + config.port + ".");
 });
-
-//add the get function
-app.get('/api/getVideoSponsorTimes', function (req, res) {
-    let videoID = req.query.videoID;
-
-    let sponsorTimes = [];
-    let votes = []
-    let UUIDs = [];
-
-    db.prepare("SELECT startTime, endTime, votes, UUID FROM sponsorTimes WHERE videoID = ? ORDER BY startTime").all(videoID, function(err, rows) {
-        if (err) console.log(err);
-
-        for (let i = 0; i < rows.length; i++) {
-            //check if votes are above -2
-            if (rows[i].votes < -2) {
-                //too untrustworthy, just ignore it
-                continue;
-            }
-            sponsorTimes.push([]);
-            
-            let index = sponsorTimes.length - 1;
-    
-            sponsorTimes[index][0] = rows[i].startTime;
-            sponsorTimes[index][1] = rows[i].endTime;
-
-            votes[index] = rows[i].votes;
-            UUIDs[index] = rows[i].UUID;
-        }
-
-        if (sponsorTimes.length == 0) {
-            res.sendStatus(404);
-            return;
-        }
-
-        organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs);
-        sponsorTimes = organisedData.sponsorTimes;
-        UUIDs = organisedData.UUIDs;
-
-        if (sponsorTimes.length == 0) {
-            res.sendStatus(404);
-        } else {
-            //send result
-            res.send({
-                sponsorTimes: sponsorTimes,
-                UUIDs: UUIDs
-            })
-        }
-    });
-});
-
-//add the post function
-app.get('/api/postVideoSponsorTimes', function (req, res) {
-    let videoID = req.query.videoID;
-    let startTime = req.query.startTime;
-    let endTime = req.query.endTime;
-    let userID = req.query.userID;
-
-    //check if all correct inputs are here and the length is 1 second or more
-    if (videoID == undefined || startTime == undefined || endTime == undefined || userID == undefined
-            || Math.abs(startTime - endTime) < 1) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    //hash the userID
-    userID = getHashedUserID(userID);
-
-    //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-
-    //hash the ip so no one can get it from the database
-    let hashedIP = ip + globalSalt;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha256');
-        hashedIP = hashCreator.update(hashedIP).digest('hex');
-    }
-
-    startTime = parseFloat(startTime);
-    endTime = parseFloat(endTime);
-
-    if (isNaN(startTime) || isNaN(endTime)) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    if (startTime > endTime) {
-        //time can't go backwards
-        res.sendStatus(400);
-        return;
-    }
-
-    //this can just be a hash of the data
-    //it's better than generating an actual UUID like what was used before
-    //also better for duplication checking
-    let hashCreator = crypto.createHash('sha256');
-    let UUID = hashCreator.update(videoID + startTime + endTime + userID).digest('hex');
-
-    //get current time
-    let timeSubmitted = Date.now();
-
-    let yesterday = timeSubmitted - 86400000;
-    
-    //check to see if this ip has submitted too many sponsors today
-    privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday], function(err, row) {
-        if (row.count >= 10) {
-            //too many sponsors for the same video from the same ip address
-            res.sendStatus(429);
-        } else {
-            //check to see if the user has already submitted sponsors for this video
-            db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID], function(err, row) {
-                if (row.count >= 4) {
-                    //too many sponsors for the same video from the same user
-                    res.sendStatus(429);
-                } else {
-                    //check if this info has already been submitted first
-                    db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID], function(err, row) {
-                        if (err) console.log(err);
-                        
-                        if (row == null) {
-                            //not a duplicate, execute query
-                            db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, 0, UUID, userID, timeSubmitted, 0);
-
-                            //add to private db as well
-                            privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted);
-
-                            res.sendStatus(200);
-                        } else {
-                            res.sendStatus(409);
-                        }
-                    });
-                }
-            });
-        }
-    });
-});
-
-//voting endpoint
-app.get('/api/voteOnSponsorTime', function (req, res) {
-    let UUID = req.query.UUID;
-    let userID = req.query.userID;
-    let type = req.query.type;
-
-    if (UUID == undefined || userID == undefined || type == undefined) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    //hash the userID
-    userID = getHashedUserID(userID + UUID);
-
-    //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-
-    //hash the ip so no one can get it from the database
-    let hashedIP = ip + globalSalt;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha256');
-        hashedIP = hashCreator.update(hashedIP).digest('hex');
-    }
-
-    //check if vote has already happened
-    privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID, function(err, row) {
-        if (err) console.log(err);
-                
-        if (row != undefined && row.type == type) {
-            //they have already done this exact vote
-            res.status(405).send("Duplicate Vote");
-            return;
-        }
-
-        //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
-        let incrementAmount = 0;
-        let oldIncrementAmount = 0;
-
-        if (type == 1) {
-            //upvote
-            incrementAmount = 1;
-        } else if (type == 0) {
-            //downvote
-            incrementAmount = -1;
-        } else {
-            //unrecongnised type of vote
-            res.sendStatus(400);
-            return;
-        }
-        if (row != undefined) {
-            if (row.type == 1) {
-                //upvote
-                oldIncrementAmount = 1;
-            } else if (row.type == 0) {
-                //downvote
-                oldIncrementAmount = -1;
-            }
-        }
-
-        //update the votes table
-        if (row != undefined) {
-            privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID);
-        } else {
-            privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type);
-        }
-
-        //update the vote count on this sponsorTime
-        //oldIncrementAmount will be zero is row is null
-        db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID);
-
-        //added to db
-        res.sendStatus(200);
-    });
-});
-
-//Endpoint when a sponsorTime is used up
-app.get('/api/viewedVideoSponsorTime', function (req, res) {
-    let UUID = req.query.UUID;
-
-    if (UUID == undefined) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    //up the view count by one
-    db.prepare("UPDATE sponsorTimes SET views = views + 1 WHERE UUID = ?").run(UUID);
-
-    res.sendStatus(200);
-});
-
-//Gets all the views added up for one userID
-//Useful to see how much one user has contributed
-app.get('/api/getViewsForUser', function (req, res) {
-    let userID = req.query.userID;
-
-    if (userID == undefined) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    //hash the userID
-    userID = getHashedUserID(userID);
-
-    //up the view count by one
-    db.prepare("SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?").get(userID, function(err, row) {
-        if (err) console.log(err);
-
-        if (row.viewCount != null) {
-            res.send({
-                viewCount: row.viewCount
-            });
-        } else {
-            res.sendStatus(404);
-        }
-    });
-});
-
-app.get('/database.db', function (req, res) {
-    res.sendFile("./databases/sponsorTimes.db", { root: __dirname });
-});
-
-function getHashedUserID(userID) {
-    //hash the userID so no one can get it from the database
-    let hashedUserID = userID;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha256');
-        hashedUserID = hashCreator.update(hashedUserID).digest('hex');
-    }
-
-    return hashedUserID;
-}
-
-//This function will find sponsor times that are contained inside of eachother, called similar sponsor times
-//Only one similar time will be returned, randomly generated based on the sqrt of votes.
-//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
-//Sponsor times with less than -2 votes are already ignored before this function is called
-function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
-    //list of sponsors that are contained inside eachother
-    let similarSponsors = [];
-
-    for (let i = 0; i < sponsorTimes.length; i++) {
-        //see if the start time is located between the start and end time of the other sponsor time.
-        for (let j = 0; j < sponsorTimes.length; j++) {
-            if (sponsorTimes[j][0] > sponsorTimes[i][0] && sponsorTimes[j][0] < sponsorTimes[i][1]) {
-                //sponsor j is contained in sponsor i
-                similarSponsors.push([i, j]);
-            }
-        }
-    }
-
-    let similarSponsorsGroups = [];
-    //once they have been added to a group, they don't need to be dealt with anymore
-    let dealtWithSimilarSponsors = [];
-
-    //create lists of all the similar groups (if 1 and 2 are similar, and 2 and 3 are similar, the group is 1, 2, 3)
-    for (let i = 0; i < similarSponsors.length; i++) {
-        if (dealtWithSimilarSponsors.includes(i)) {
-            //dealt with already
-            continue;
-        }
-
-        //this is the group of indexes that are similar
-        let group = similarSponsors[i];
-        for (let j = 0; j < similarSponsors.length; j++) {
-            if (group.includes(similarSponsors[j][0]) || group.includes(similarSponsors[j][1])) {
-                //this is a similar group
-                group.push(similarSponsors[j][0]);
-                group.push(similarSponsors[j][1]);
-                dealtWithSimilarSponsors.push(j);
-            }
-        }
-        similarSponsorsGroups.push(group);
-    }
-
-    //remove duplicate indexes in group arrays
-    for (let i = 0; i < similarSponsorsGroups.length; i++) {
-        uniqueArray = similarSponsorsGroups[i].filter(function(item, pos, self) {
-            return self.indexOf(item) == pos;
-        });
-
-        similarSponsorsGroups[i] = uniqueArray;
-    }
-
-    let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes);
-
-    let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices;
-    //the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added
-    let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith;
-
-    let voteSums = weightedRandomIndexes.weightSums;
-    //convert these into the votes
-    for (let i = 0; i < voteSums.length; i++) {
-        if (voteSums[i] != undefined) {
-            //it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor.
-            votes[finalSponsorTimeIndexes[i]] = voteSums;
-        }
-    }
-
-    //find the indexes never dealt with and add them
-    for (let i = 0; i < sponsorTimes.length; i++) {
-        if (!finalSponsorTimeIndexesDealtWith.includes(i)) {
-            finalSponsorTimeIndexes.push(i)
-        }
-    }
-
-    //if there are too many indexes, find the best 4
-    if (finalSponsorTimeIndexes.length > 4) {
-        finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 4).finalChoices;
-    }
-
-    //convert this to a final array to return
-    let finalSponsorTimes = [];
-    for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
-        finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]);
-    }
-
-    //convert this to a final array of UUIDs as well
-    let finalUUIDs = [];
-    for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
-        finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]);
-    }
-
-    return {
-        sponsorTimes: finalSponsorTimes,
-        UUIDs: finalUUIDs
-    };
-}
-
-//gets the getWeightedRandomChoice for each group in an array of groups
-function getWeightedRandomChoiceForArray(choiceGroups, weights) {
-    let finalChoices = [];
-    //the indexes either chosen to be added to final indexes or chosen not to be added
-    let choicesDealtWith = [];
-    //for each choice group, what are the sums of the weights
-    let weightSums = [];
-
-    for (let i = 0; i < choiceGroups.length; i++) {
-        //find weight sums for this group
-        weightSums.push(0);
-        for (let j = 0; j < choiceGroups[i].length; j++) {
-            //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
-            if (weights[choiceGroups[i][j]] > 0) {
-                weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]];
-            }
-        }
-
-        //create a random choice for this group
-        let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1)
-        finalChoices.push(randomChoice.finalChoices);
-
-        for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) {
-            choicesDealtWith.push(randomChoice.choicesDealtWith[j])
-        }
-    }
-
-    return {
-        finalChoices: finalChoices,
-        choicesDealtWith: choicesDealtWith,
-        weightSums: weightSums
-    };
-}
-
-//gets a weighted random choice from the indexes array based on the weights.
-//amountOfChoices speicifies the amount of choices to return, 1 or more.
-//choices are unique
-function getWeightedRandomChoice(choices, weights, amountOfChoices) {
-    if (amountOfChoices > choices.length) {
-        //not possible, since all choices must be unique
-        return null;
-    }
-
-    let finalChoices = [];
-    let choicesDealtWith = [];
-
-    let sqrtWeightsList = [];
-    //the total of all the weights run through the cutom sqrt function
-    let totalSqrtWeights = 0;
-    for (let j = 0; j < choices.length; j++) {
-        //multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
-        //The 3 makes -2 the minimum votes before being ignored completely
-        //https://www.desmos.com/calculator/ljftxolg9j
-        //this can be changed if this system increases in popularity.
-        let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10);
-        sqrtWeightsList.push(sqrtVote)
-        totalSqrtWeights += sqrtVote;
-
-        //this index has now been deat with
-        choicesDealtWith.push(choices[j]);
-    }
-
-    //iterate and find amountOfChoices choices
-    let randomNumber = Math.random();
-    //this array will keep adding to this variable each time one sqrt vote has been dealt with
-    //this is the sum of all the sqrtVotes under this index
-    let currentVoteNumber = 0;
-    for (let j = 0; j < sqrtWeightsList.length; j++) {
-        if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) {
-            //this one was randomly generated
-            finalChoices.push(choices[j]);
-            //remove that from original array, for next recursion pass if it happens
-            choices.splice(j, 1);
-            break;
-        }
-
-        //add on to the count
-        currentVoteNumber += sqrtWeightsList[j];
-    }
-    
-    //add on the other choices as well using recursion
-    if (amountOfChoices > 1) {
-        let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices;
-        //add all these choices to the finalChoices array being returned
-        for (let i = 0; i < otherChoices.length; i++) {
-            finalChoices.push(otherChoices[i]);
-        }
-    }
-
-    return {
-        finalChoices: finalChoices,
-        choicesDealtWith: choicesDealtWith
-    };
-}

package.json

@@ -4,15 +4,27 @@
   "description": "Server that holds the SponsorBlock database",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "node test.js",
+    "dev": "nodemon -x \"(npm test || echo test failed) && npm start\"",
+    "dev:bash": "nodemon -x 'npm test ; npm start'",
     "start": "node index.js"
   },
   "author": "Ajay Ramachandran",
   "license": "MIT",
   "dependencies": {
+    "better-sqlite3": "^5.4.3",
     "express": "^4.17.1",
+    "express-rate-limit": "^5.1.3",
     "http": "0.0.0",
-    "sqlite3": "^4.0.9",
-    "uuid": "^3.3.2"
+    "iso8601-duration": "^1.2.0",
+    "node-fetch": "^2.6.0",
+    "redis": "^3.0.2",
+    "sync-mysql": "^3.0.1",
+    "uuid": "^3.3.2",
+    "youtube-api": "^2.0.10"
+  },
+  "devDependencies": {
+    "mocha": "^7.1.1",
+    "nodemon": "^2.0.2"
   }
 }

src/app.js

@@ -0,0 +1,137 @@
+var express = require('express');
+// Create a service (the app object is just a callback).
+var app = express();
+var config = require('./config.js');
+var redis = require('./utils/redis.js');
+const getIP = require('./utils/getIP.js');
+const getHash = require('./utils/getHash.js');
+
+// Middleware 
+const rateLimitMiddleware = require('./middleware/requestRateLimit.js');
+var corsMiddleware = require('./middleware/cors.js');
+var loggerMiddleware = require('./middleware/logger.js');
+const userCounter = require('./middleware/userCounter.js');
+
+// Routes
+var getSkipSegments = require('./routes/getSkipSegments.js').endpoint;
+var postSkipSegments = require('./routes/postSkipSegments.js');
+var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js');
+var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js');
+var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js');
+var setUsername = require('./routes/setUsername.js');
+var getUsername = require('./routes/getUsername.js');
+var shadowBanUser = require('./routes/shadowBanUser.js');
+var addUserAsVIP = require('./routes/addUserAsVIP.js');
+var getSavedTimeForUser = require('./routes/getSavedTimeForUser.js');
+var getViewsForUser = require('./routes/getViewsForUser.js');
+var getTopUsers = require('./routes/getTopUsers.js');
+var getTotalStats = require('./routes/getTotalStats.js');
+var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js');
+var getUserInfo = require('./routes/getUserInfo.js');
+var postNoSegments = require('./routes/postNoSegments.js');
+var deleteNoSegments = require('./routes/deleteNoSegments.js');
+var getIsUserVIP = require('./routes/getIsUserVIP.js');
+var warnUser = require('./routes/postWarning.js');
+var postSegmentShift = require('./routes/postSegmentShift.js');
+
+// Old Routes
+var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js');
+var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js');
+
+// Rate limit endpoint lists
+let voteEndpoints = [voteOnSponsorTime.endpoint];
+let viewEndpoints = [viewedVideoSponsorTime];
+if (config.rateLimit) {
+    if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote));
+    if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
+}
+
+//setup CORS correctly
+app.use(corsMiddleware);
+app.use(loggerMiddleware);
+app.use(express.json())
+
+if (config.userCounterURL) app.use(userCounter);
+
+// Setup pretty JSON
+if (config.mode === "development") app.set('json spaces', 2);
+
+// Set production mode
+app.set('env', config.mode || 'production');
+
+//add the get function
+app.get('/api/getVideoSponsorTimes', oldGetVideoSponsorTimes);
+
+//add the oldpost function
+app.get('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
+app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
+
+//add the skip segments functions
+app.get('/api/skipSegments', getSkipSegments);
+app.post('/api/skipSegments', postSkipSegments);
+
+// add the privacy protecting skip segments functions
+app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash);
+
+//voting endpoint
+app.get('/api/voteOnSponsorTime', ...voteEndpoints);
+app.post('/api/voteOnSponsorTime', ...voteEndpoints);
+
+//Endpoint when a submission is skipped
+app.get('/api/viewedVideoSponsorTime', ...viewEndpoints);
+app.post('/api/viewedVideoSponsorTime', ...viewEndpoints);
+
+//To set your username for the stats view
+app.post('/api/setUsername', setUsername);
+
+//get what username this user has
+app.get('/api/getUsername', getUsername);
+
+//Endpoint used to hide a certain user's data
+app.post('/api/shadowBanUser', shadowBanUser);
+
+//Endpoint used to make a user a VIP user with special privileges
+app.post('/api/addUserAsVIP', addUserAsVIP);
+
+//Gets all the views added up for one userID
+//Useful to see how much one user has contributed
+app.get('/api/getViewsForUser', getViewsForUser);
+
+//Gets all the saved time added up (views * sponsor length) for one userID
+//Useful to see how much one user has contributed
+//In minutes
+app.get('/api/getSavedTimeForUser', getSavedTimeForUser);
+
+app.get('/api/getTopUsers', getTopUsers);
+
+//send out totals
+//send the total submissions, total views and total minutes saved
+app.get('/api/getTotalStats', getTotalStats);
+
+app.get('/api/getUserInfo', getUserInfo);
+
+//send out a formatted time saved total
+app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
+
+//submit video containing no segments
+app.post('/api/noSegments', postNoSegments);
+
+app.delete('/api/noSegments', deleteNoSegments);
+
+//get if user is a vip
+app.get('/api/isUserVIP', getIsUserVIP);
+
+//sent user a warning
+app.post('/api/warnUser', warnUser);
+
+//get if user is a vip
+app.post('/api/segmentShift', postSegmentShift);
+
+app.get('/database.db', function (req, res) {
+    res.sendFile("./databases/sponsorTimes.db", { root: "./" });
+});
+
+// Create an HTTP service.
+module.exports = function createServer (callback) {
+    return app.listen(config.port, callback);
+}

src/config.js

@@ -0,0 +1,37 @@
+
+const fs = require('fs');
+let config = {};
+
+// Check to see if launched in test mode
+if (process.env.npm_lifecycle_script === 'node test.js') {
+  config = JSON.parse(fs.readFileSync('test.json'));
+} else {
+  config = JSON.parse(fs.readFileSync('config.json'));
+}
+
+addDefaults(config, {
+  "port": 80,
+  "behindProxy": "X-Forwarded-For",
+  "db": "./databases/sponsorTimes.db",
+  "privateDB": "./databases/private.db",
+  "createDatabaseIfNotExist": true,
+  "schemaFolder": "./databases",
+  "dbSchema": "./databases/_sponsorTimes.db.sql",
+  "privateDBSchema": "./databases/_private.db.sql",
+  "readOnly": false,
+  "webhooks": [],
+  "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+  "maxNumberOfActiveWarnings": 3,
+  "hoursAfterWarningExpires": 24
+})
+
+module.exports = config;
+
+// Add defaults
+function addDefaults(config, defaults) {
+  for (const key in defaults) {
+      if(!config.hasOwnProperty(key)) {
+        config[key] = defaults[key];
+      }
+  }
+};

src/databases/Mysql.js

@@ -0,0 +1,28 @@
+var MysqlInterface = require('sync-mysql');
+const logger = require('../utils/logger.js');
+
+class Mysql {
+  constructor(msConfig) {
+    this.connection = new MysqlInterface(msConfig);
+  }
+
+  exec(query) {
+    this.prepare('run', query, []);
+  }
+
+  prepare (type, query, params) {
+    logger.debug("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params);
+    if (type === 'get') {
+      return this.connection.query(query, params)[0];
+    } else if (type === 'run') {
+      this.connection.query(query, params);
+    } else if (type === 'all') {
+      return this.connection.query(query, params);
+    } else {
+      logger.warn('returning undefined...');
+      return undefined;
+    }
+  }
+}
+
+module.exports = Mysql;

src/databases/Sqlite.js

@@ -0,0 +1,33 @@
+const { db } = require("./databases");
+var config = require('../config.js');
+const logger = require('../utils/logger.js');
+
+class Sqlite {
+  constructor(connection) {
+    this.connection = connection;
+  }
+
+  getConnection() {
+    return this.connection;
+  }
+
+  prepare(type, query, params) {
+    if (type === 'get') {
+      return this.connection.prepare(query).get(...params);
+    } else if (type === 'run') {
+      this.connection.prepare(query).run(...params);
+    } else if (type === 'all') {
+      return this.connection.prepare(query).all(...params); 
+    } else {
+      logger.debug('sqlite query: returning undefined')
+      logger.debug("prepare:        type: " + type + ", query: " + query + ", params: " + params);
+      return undefined;
+    }
+  }
+
+  exec(query) {
+    return this.connection.exec(query);
+  }
+}
+
+module.exports = Sqlite;

src/databases/databases.js

@@ -0,0 +1,81 @@
+var config = require('../config.js');
+var Sqlite3 = require('better-sqlite3');
+var fs = require('fs');
+var path = require('path');
+var Sqlite = require('./Sqlite.js')
+var Mysql = require('./Mysql.js');
+const logger = require('../utils/logger.js');
+const getHash = require('../utils/getHash.js');
+
+let options = {
+  readonly: config.readOnly,
+  fileMustExist: !config.createDatabaseIfNotExist
+};
+
+if (config.mysql) {
+  module.exports = {
+    db: new Mysql(config.mysql),
+    privateDB: new Mysql(config.privateMysql)
+  };
+} else {
+  // Make dirs if required
+  if (!fs.existsSync(path.join(config.db, "../"))) {
+    fs.mkdirSync(path.join(config.db, "../"));
+  }
+  if (!fs.existsSync(path.join(config.privateDB, "../"))) {
+    fs.mkdirSync(path.join(config.privateDB, "../"));
+  }
+
+  var db = new Sqlite3(config.db, options);
+  var privateDB = new Sqlite3(config.privateDB, options);
+
+  if (config.createDatabaseIfNotExist && !config.readOnly) {
+    if (fs.existsSync(config.dbSchema)) db.exec(fs.readFileSync(config.dbSchema).toString());
+    if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString());
+  }
+
+  if (!config.readOnly) {
+    db.function("sha256", (string) => {
+      return getHash(string, 1);
+    });
+    
+    // Upgrade database if required
+    ugradeDB(db, "sponsorTimes");
+    ugradeDB(privateDB, "private")
+  }
+
+  // Attach private db to main db
+  db.prepare("ATTACH ? as privateDB").run(config.privateDB);
+
+  // Enable WAL mode checkpoint number
+  if (!config.readOnly && config.mode === "production") {
+    db.exec("PRAGMA journal_mode=WAL;");
+    db.exec("PRAGMA wal_autocheckpoint=1;");
+  }
+
+  // Enable Memory-Mapped IO
+  db.exec("pragma mmap_size= 500000000;");
+  privateDB.exec("pragma mmap_size= 500000000;");
+
+  module.exports = {
+    db: new Sqlite(db),
+    privateDB: new Sqlite(privateDB)
+  };
+
+  function ugradeDB(db, prefix) {
+    let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
+    let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
+
+    let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
+    logger.debug('db update: trying ' + path);
+    while (fs.existsSync(path)) {
+      logger.debug('db update: updating ' + path);
+      db.exec(fs.readFileSync(path).toString());
+
+      versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
+      path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
+      logger.debug('db update: trying ' + path);
+    }
+    logger.debug('db update: no file ' + path);
+  }
+}

src/middleware/cors.js

@@ -0,0 +1,5 @@
+module.exports = function corsMiddleware(req, res, next) {
+  res.header("Access-Control-Allow-Origin", "*");
+  res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+  next();
+}

src/middleware/logger.js

@@ -0,0 +1,6 @@
+const log = require('../utils/logger.js'); // log not logger to not interfere with function name
+
+module.exports = function logger (req, res, next) {
+  log.info("Request recieved: " + req.method + " " + req.url);
+  next();
+}

src/middleware/requestRateLimit.js

@@ -0,0 +1,18 @@
+const getIP = require('../utils/getIP.js');
+const getHash = require('../utils/getHash.js');
+const rateLimit = require('express-rate-limit');
+
+module.exports = (limitConfig) => rateLimit({
+  windowMs: limitConfig.windowMs,
+  max: limitConfig.max,
+  message: limitConfig.message,
+  statusCode: limitConfig.statusCode,
+  headers: false,
+  keyGenerator: (req /*, res*/) => {
+    return getHash(getIP(req), 1);
+  },
+  skip: (/*req, res*/) => {
+    // skip rate limit if running in test mode
+    return process.env.npm_lifecycle_script === 'node test.js';
+  }
+});

src/middleware/userCounter.js

@@ -0,0 +1,13 @@
+const fetch = require('node-fetch');
+
+const config = require('../config.js');
+const getIP = require('../utils/getIP.js');
+const getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+
+module.exports = function userCounter(req, res, next) {
+    fetch(config.userCounterURL + "/api/v1/addIP?hashedIP=" + getHash(getIP(req), 1), { method: "POST" })
+        .catch(() => logger.debug("Failing to connect to user counter at: " + config.userCounterURL))
+
+    next();
+}

src/routes/addUserAsVIP.js

@@ -0,0 +1,45 @@
+var fs = require('fs');
+var config = require('../config.js');
+
+var db = require('../databases/databases.js').db;
+var getHash = require('../utils/getHash.js');
+
+module.exports = async function addUserAsVIP (req, res) {
+    let userID = req.query.userID;
+    let adminUserIDInput = req.query.adminUserID;
+
+    let enabled = req.query.enabled;
+    if (enabled === undefined){
+        enabled = true;
+    } else {
+        enabled = enabled === "true";
+    }
+
+    if (userID == undefined || adminUserIDInput == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    adminUserIDInput = getHash(adminUserIDInput);
+
+    if (adminUserIDInput !== config.adminUserID) {
+        //not authorized
+        res.sendStatus(403);
+        return;
+    }
+
+    //check to see if this user is already a vip
+    let row = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]);
+
+    if (enabled && row.userCount == 0) {
+        //add them to the vip list
+        db.prepare('run', "INSERT INTO vipUsers VALUES(?)", [userID]);
+    } else if (!enabled && row.userCount > 0) {
+        //remove them from the shadow ban list
+        db.prepare('run', "DELETE FROM vipUsers WHERE userID = ?", [userID]);
+    }
+
+    res.sendStatus(200);
+}

src/routes/deleteNoSegments.js

@@ -0,0 +1,43 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+module.exports = (req, res) => {
+    // Collect user input data
+    let videoID = req.body.videoID;
+    let userID = req.body.userID;
+    let categories = req.body.categories;
+
+    // Check input data is valid
+    if (!videoID 
+        || !userID 
+        || !categories 
+        || !Array.isArray(categories) 
+        || categories.length === 0
+    ) {
+        res.status(400).json({
+            message: 'Bad Format'
+        });
+        return;
+    }
+
+    // Check if user is VIP
+    userID = getHash(userID);
+    let userIsVIP = isUserVIP(userID);
+
+    if (!userIsVIP) {
+        res.status(403).json({
+            message: 'Must be a VIP to mark videos.'
+        });
+        return;
+    }
+
+    db.prepare("all", 'SELECT * FROM noSegments WHERE videoID = ?', [videoID]).filter((entry) => {
+        return (categories.indexOf(entry.category) !== -1);
+    }).forEach((entry) => {
+        db.prepare('run', 'DELETE FROM noSegments WHERE videoID = ? AND category = ?', [videoID, entry.category]);
+    });
+
+    res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
+};

src/routes/getDaysSavedFormatted.js

@@ -0,0 +1,12 @@
+var db = require('../databases/databases.js').db;
+
+module.exports = function getDaysSavedFormatted (req, res) {
+  let row = db.prepare('get', "SELECT SUM((endTime - startTime) / 60 / 60 / 24 * views) as daysSaved from sponsorTimes where shadowHidden != 1", []);
+      
+  if (row !== undefined) {
+      //send this result
+      res.send({
+          daysSaved: row.daysSaved.toFixed(2)
+      });
+  }
+}

src/routes/getIsUserVIP.js

@@ -0,0 +1,31 @@
+var db = require('../databases/databases.js').db;
+
+var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+
+module.exports = (req, res) => {
+    let userID = req.query.userID;
+
+    if (userID == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    try {
+        let vipState = isUserVIP(userID);
+        res.status(200).json({
+            hashedUserID: userID,
+            vip: vipState
+        });
+    } catch (err) {
+        logger.error(err);
+        res.sendStatus(500);
+
+        return;
+    }
+}

src/routes/getSavedTimeForUser.js

@@ -0,0 +1,32 @@
+var db = require('../databases/databases.js').db;
+var getHash = require('../utils/getHash.js');
+
+module.exports = function getSavedTimeForUser (req, res) {
+  let userID = req.query.userID;
+
+  if (userID == undefined) {
+      //invalid request
+      res.sendStatus(400);
+      return;
+  }
+
+  //hash the userID
+  userID = getHash(userID);
+
+  try {
+      let row = db.prepare("get", "SELECT SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE userID = ? AND votes > -1 AND shadowHidden != 1 ", [userID]);
+
+      if (row.minutesSaved != null) {
+          res.send({
+              timeSaved: row.minutesSaved
+          });
+      } else {
+          res.sendStatus(404);
+      }
+  } catch (err) {
+      console.log(err);
+      res.sendStatus(500);
+
+      return;
+  }
+}

src/routes/getSkipSegments.js

@@ -0,0 +1,189 @@
+var config = require('../config.js');
+
+var databases = require('../databases/databases.js');
+var db = databases.db;
+var privateDB = databases.privateDB;
+
+var logger = require('../utils/logger.js');
+var getHash = require('../utils/getHash.js');
+var getIP = require('../utils/getIP.js');
+
+function cleanGetSegments(req, videoID, categories) {
+    let userHashedIP, shadowHiddenSegments;
+
+    let segments = [];
+
+    try {
+        for (const category of categories) {
+            const categorySegments = db
+                .prepare(
+                    'all',
+                    'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
+                    [videoID, category]
+                )
+                .filter(segment => {
+                    if (segment.votes < -1) {
+                        return false; //too untrustworthy, just ignore it
+                    }
+
+                    //check if shadowHidden
+                    //this means it is hidden to everyone but the original ip that submitted it
+                    if (segment.shadowHidden != 1) {
+                        return true;
+                    }
+
+                    if (shadowHiddenSegments === undefined) {
+                        shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
+                    }
+
+                    //if this isn't their ip, don't send it to them
+                    return shadowHiddenSegments.some(shadowHiddenSegment => {
+                        if (userHashedIP === undefined) {
+                            //hash the IP only if it's strictly necessary
+                            userHashedIP = getHash(getIP(req) + config.globalSalt);
+                        }
+                        return shadowHiddenSegment.hashedIP === userHashedIP;
+                    });
+                });
+
+            chooseSegments(categorySegments).forEach(chosenSegment => {
+                segments.push({
+                    category,
+                    segment: [chosenSegment.startTime, chosenSegment.endTime],
+                    UUID: chosenSegment.UUID,
+                });
+            });
+        }
+
+        return segments;
+    } catch (err) {
+        if (err) {
+            logger.error(err);
+            return null;
+        }
+    }
+}
+
+//gets a weighted random choice from the choices array based on their `votes` property.
+//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
+//choices are unique
+function getWeightedRandomChoice(choices, amountOfChoices) {
+    //trivial case: no need to go through the whole process
+    if (amountOfChoices >= choices.length) {
+        return choices;
+    }
+
+    //assign a weight to each choice
+    let totalWeight = 0;
+    choices = choices.map(choice => {
+        //The 3 makes -2 the minimum votes before being ignored completely
+        //https://www.desmos.com/calculator/c1duhfrmts
+        //this can be changed if this system increases in popularity.
+        const weight = Math.exp((choice.votes + 3), 0.85);
+        totalWeight += weight;
+
+        return { ...choice, weight };
+    });
+
+    //iterate and find amountOfChoices choices
+    const chosen = [];
+    while (amountOfChoices-- > 0) {
+        //weighted random draw of one element of choices
+        const randomNumber = Math.random() * totalWeight;
+        let stackWeight = choices[0].weight;
+        let i = 0;
+        while (stackWeight < randomNumber) {
+            stackWeight += choices[++i].weight;
+        }
+
+        //add it to the chosen ones and remove it from the choices before the next iteration
+        chosen.push(choices[i]);
+        totalWeight -= choices[i].weight;
+        choices.splice(i, 1);
+    }
+
+    return chosen;
+}
+
+//This function will find segments that are contained inside of eachother, called similar segments
+//Only one similar time will be returned, randomly generated based on the sqrt of votes.
+//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
+//Segments with less than -1 votes are already ignored before this function is called
+function chooseSegments(segments) {
+    //Create groups of segments that are similar to eachother
+    //Segments must be sorted by their startTime so that we can build groups chronologically:
+    //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
+    //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
+    //   inside that group (because they're sorted) so we can create a new one
+    const similarSegmentsGroups = [];
+    let currentGroup;
+    let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
+    segments.forEach(segment => {
+        if (segment.startTime > cursor) {
+            currentGroup = { segments: [], votes: 0 };
+            similarSegmentsGroups.push(currentGroup);
+        }
+
+        currentGroup.segments.push(segment);
+        //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
+        if (segment.votes > 0) {
+            currentGroup.votes += segment.votes;
+        }
+
+        cursor = Math.max(cursor, segment.endTime);
+    });
+
+    //if there are too many groups, find the best 8
+    return getWeightedRandomChoice(similarSegmentsGroups, 32).map(
+        //randomly choose 1 good segment per group and return them
+        group => getWeightedRandomChoice(group.segments, 1)[0]
+    );
+}
+
+/**
+ *
+ * Returns what would be sent to the client.
+ * Will respond with errors if required. Returns false if it errors.
+ *
+ * @param req
+ * @param res
+ *
+ * @returns
+ */
+function handleGetSegments(req, res) {
+    const videoID = req.query.videoID;
+    // Default to sponsor
+    // If using params instead of JSON, only one category can be pulled
+    const categories = req.query.categories
+        ? JSON.parse(req.query.categories)
+        : req.query.category
+        ? [req.query.category]
+        : ['sponsor'];
+
+    let segments = cleanGetSegments(req, videoID, categories);
+
+    if (segments === null || segments === undefined) {
+        res.sendStatus(500);
+        return false;
+    }
+
+    if (segments.length == 0) {
+        res.sendStatus(404);
+        return false;
+    }
+
+    return segments;
+}
+
+module.exports = {
+    handleGetSegments,
+    cleanGetSegments,
+    endpoint: function (req, res) {
+        let segments = handleGetSegments(req, res);
+
+        if (segments) {
+            //send result
+            res.send(segments);
+        }
+    },
+};

src/routes/getSkipSegmentsByHash.js

@@ -0,0 +1,33 @@
+const hashPrefixTester = require('../utils/hashPrefixTester.js');
+const getSegments = require('./getSkipSegments.js').cleanGetSegments;
+
+const databases = require('../databases/databases.js');
+const logger = require('../utils/logger.js');
+const db = databases.db;
+
+module.exports = async function (req, res) {
+    let hashPrefix = req.params.prefix;
+    if (!hashPrefixTester(req.params.prefix)) {
+        res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+        return;
+    }
+
+    const categories = req.query.categories
+    ? JSON.parse(req.query.categories)
+    : req.query.category
+    ? [req.query.category]
+    : ['sponsor'];
+
+    // Get all video id's that match hash prefix
+    const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
+
+    let segments = videoIds.map((video) => {
+        return {
+            videoID: video.videoID,
+            hash: video.hashedVideoID,
+            segments: getSegments(req, video.videoID, categories)
+        };
+    });
+
+    res.status((segments.length === 0) ? 404 : 200).json(segments);
+}

src/routes/getTopUsers.js

@@ -0,0 +1,92 @@
+var db = require('../databases/databases.js').db;
+const logger = require('../utils/logger.js');
+const createMemoryCache = require('../utils/createMemoryCache.js');
+const config = require('../config.js');
+
+const MILLISECONDS_IN_MINUTE = 60000;
+const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
+
+function generateTopUsersStats(sortBy, categoryStatsEnabled = false) {
+  return new Promise((resolve, reject) => {
+    const userNames = [];
+    const viewCounts = [];
+    const totalSubmissions = [];
+    const minutesSaved = [];
+    const categoryStats = categoryStatsEnabled ? [] : undefined;
+    
+    let additionalFields = '';
+    if (categoryStatsEnabled) {
+      additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " +
+                          "SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " +
+                          "SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " +
+                          "SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " +
+                          "SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " +
+                          "SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, ";
+    }
+
+    const rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + 
+                    "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " +
+                    "SUM(votes) as userVotes, " +
+                    additionalFields +
+                    "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " +
+                    "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " +
+                    "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " +
+                    "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " +
+                    "ORDER BY " + sortBy + " DESC LIMIT 100", []);
+    
+    for (let i = 0; i < rows.length; i++) {
+        userNames[i] = rows[i].userName;
+
+        viewCounts[i] = rows[i].viewCount;
+        totalSubmissions[i] = rows[i].totalSubmissions;
+        minutesSaved[i] = rows[i].minutesSaved;
+        if (categoryStatsEnabled) {
+          categoryStats[i] = [
+            rows[i].categorySponsor,
+            rows[i].categorySumIntro,
+            rows[i].categorySumOutro,
+            rows[i].categorySumInteraction,
+            rows[i].categorySelfpromo,
+            rows[i].categoryMusicOfftopic,
+          ];
+        }
+    }
+    
+    resolve({
+      userNames,
+      viewCounts,
+      totalSubmissions,
+      minutesSaved,
+      categoryStats
+    });
+  });
+}
+
+module.exports = async function getTopUsers (req, res) {
+  let sortType = req.query.sortType;
+  let categoryStatsEnabled = req.query.categoryStats;
+
+  if (sortType == undefined) {
+      //invalid request
+      res.sendStatus(400);
+      return;
+  }
+  
+  //setup which sort type to use
+  let sortBy = '';
+  if (sortType == 0) {
+    sortBy = 'minutesSaved';
+  } else if (sortType == 1) {
+    sortBy = 'viewCount';
+  } else if (sortType == 2) {
+    sortBy = 'totalSubmissions';
+  } else {
+    //invalid request
+    return res.sendStatus(400);
+  }
+  
+  const stats = await getTopUsersWithCache(sortBy, categoryStatsEnabled);
+  
+  //send this result
+  res.send(stats);
+}

src/routes/getTotalStats.js

@@ -0,0 +1,69 @@
+const db = require('../databases/databases.js').db;
+const request = require('request');
+const config = require('../config.js');
+
+// A cache of the number of chrome web store users
+let chromeUsersCache = null;
+let firefoxUsersCache = null;
+
+// By the privacy friendly user counter
+let apiUsersCache = null;
+
+let lastUserCountCheck = 0;
+
+module.exports = function getTotalStats (req, res) {
+    let row = db.prepare('get', "SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, " +
+              "SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE shadowHidden != 1 AND votes >= 0", []);
+
+    if (row !== undefined) {
+        let extensionUsers = chromeUsersCache + firefoxUsersCache;
+
+        //send this result
+        res.send({
+            userCount: row.userCount,
+            activeUsers: extensionUsers,
+            apiUsers: Math.max(apiUsersCache, extensionUsers),
+            viewCount: row.viewCount,
+            totalSubmissions: row.totalSubmissions,
+            minutesSaved: row.minutesSaved
+        });
+
+        // Check if the cache should be updated (every ~14 hours)
+        let now = Date.now();
+        if (now - lastUserCountCheck > 5000000) {
+            lastUserCountCheck = now;
+
+            updateExtensionUsers();
+        }
+    }
+}
+
+function updateExtensionUsers() {
+    if (config.userCounterURL) {
+        request.get(config.userCounterURL + "/api/v1/userCount", (err, response, body) => {
+            apiUsersCache = Math.max(apiUsersCache, JSON.parse(body).userCount);
+        });
+    }
+
+    request.get("https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/", function (err, firefoxResponse, body) {
+        try {
+            firefoxUsersCache = parseInt(JSON.parse(body).average_daily_users);
+
+            request.get("https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone", function(err, chromeResponse, body) {
+                if (body !== undefined) {
+                    try {
+                        chromeUsersCache = parseInt(body.match(/(?<=\<span class=\"e-f-ih\" title=\").*?(?= users\">)/)[0].replace(",", ""));
+                    } catch (error) {
+                        // Re-check later
+                        lastUserCountCheck = 0;
+                    }
+                } else {
+                    lastUserCountCheck = 0;
+                }
+            });
+        } catch (error) {
+            // Re-check later
+            lastUserCountCheck = 0;
+        }
+    });
+}

src/routes/getUserInfo.js

@@ -0,0 +1,82 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+
+function dbGetSubmittedSegmentSummary (userID) {
+  try {
+    let row = db.prepare("get", "SELECT SUM(((endTime - startTime) / 60) * views) as minutesSaved, count(*) as segmentCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]);
+    if (row.minutesSaved != null) {
+      return {
+        minutesSaved: row.minutesSaved,
+        segmentCount: row.segmentCount,
+      };
+    } else {
+      return {
+        minutesSaved: 0,
+        segmentCount: 0,
+      };
+    }
+  } catch (err) {
+    return false;
+  }
+}
+
+function dbGetUsername (userID) {
+  try {
+    let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]);
+    if (row !== undefined) {
+      return row.userName;
+    } else {
+      //no username yet, just send back the userID
+      return userID;
+    }
+  } catch (err) {
+    return false;
+  }
+}
+
+function dbGetViewsForUser (userID) {
+  try {
+    let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]);
+    //increase the view count by one
+    if (row.viewCount != null) {
+      return row.viewCount;
+    } else {
+      return 0;
+    }
+  } catch (err) {
+    return false;
+  }
+}
+
+function dbGetWarningsForUser (userID) {
+  try {
+    let rows = db.prepare('all', "SELECT * FROM warnings WHERE userID = ?", [userID]);
+    return rows.length;
+  } catch (err) {
+    logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0') ;
+    return 0;
+  }
+}
+
+module.exports = function getUserInfo (req, res) {
+  let userID = req.query.userID;
+
+  if (userID == undefined) {
+    //invalid request
+    res.status(400).send('Parameters are not valid');
+    return;
+  }
+
+  //hash the userID
+  userID = getHash(userID);
+  
+  const segmentsSummary = dbGetSubmittedSegmentSummary(userID);
+  res.send({
+    userID,
+    userName: dbGetUsername(userID),
+    minutesSaved: segmentsSummary.minutesSaved,
+    segmentCount: segmentsSummary.segmentCount,
+    viewCount: dbGetViewsForUser(userID),
+    warnings: dbGetWarningsForUser(userID)
+  });
+}

src/routes/getUsername.js

@@ -0,0 +1,37 @@
+var db = require('../databases/databases.js').db;
+
+var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+
+module.exports = function getUsername (req, res) {
+    let userID = req.query.userID;
+
+    if (userID == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    try {
+        let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]);
+
+        if (row !== undefined) {
+            res.send({
+                userName: row.userName
+            });
+        } else {
+            //no username yet, just send back the userID
+            res.send({
+                userName: userID
+            });
+        }
+    } catch (err) {
+        logger.error(err);
+        res.sendStatus(500);
+
+        return;
+    }
+}

src/routes/getViewsForUser.js

@@ -0,0 +1,33 @@
+var db = require('../databases/databases.js').db;
+var getHash = require('../utils/getHash.js');
+var logger = require('../utils/logger.js');
+module.exports = function getViewsForUser(req, res) {
+  let userID = req.query.userID;
+
+  if (userID == undefined) {
+      //invalid request
+      res.sendStatus(400);
+      return;
+  }
+
+  //hash the userID
+  userID = getHash(userID);
+
+  try {
+      let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?", [userID]);
+
+      //increase the view count by one
+      if (row.viewCount != null) {
+          res.send({
+              viewCount: row.viewCount
+          });
+      } else {
+          res.sendStatus(404);
+      }
+  } catch (err) {
+      logger.error(err);
+      res.sendStatus(500);
+
+      return;
+  }
+}

src/routes/oldGetVideoSponsorTimes.js

@@ -0,0 +1,26 @@
+var getSkipSegments = require("./getSkipSegments.js")
+
+
+module.exports = function (req, res) {
+    let videoID = req.query.videoID;
+
+    let segments = getSkipSegments.handleGetSegments(req, res);
+
+    if (segments) {
+        // Convert to old outputs
+        let sponsorTimes = [];
+        let UUIDs = [];
+
+        for (const segment of segments) {
+            sponsorTimes.push(segment.segment);
+            UUIDs.push(segment.UUID);
+        }
+
+        res.send({
+            sponsorTimes,
+            UUIDs
+        })
+    }
+
+    // Error has already been handled in the other method
+}

src/routes/oldSubmitSponsorTimes.js

@@ -0,0 +1,7 @@
+var postSkipSegments = require('./postSkipSegments.js');
+
+module.exports = async function submitSponsorTimes(req, res) {
+    req.query.category = "sponsor";
+
+    return postSkipSegments(req, res);
+}

src/routes/postNoSegments.js

@@ -0,0 +1,74 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+module.exports = (req, res) => {
+    // Collect user input data
+    let videoID = req.body.videoID;
+    let userID = req.body.userID;
+    let categories = req.body.categories;
+
+    // Check input data is valid
+    if (!videoID 
+        || !userID 
+        || !categories 
+        || !Array.isArray(categories) 
+        || categories.length === 0
+    ) {
+        res.status(400).json({
+            message: 'Bad Format'
+        });
+        return;
+    }
+
+    // Check if user is VIP
+    userID = getHash(userID);
+    let userIsVIP = isUserVIP(userID);
+
+    if (!userIsVIP) {
+        res.status(403).json({
+            message: 'Must be a VIP to mark videos.'
+        });
+        return;
+    }
+
+    // Get existing no segment markers
+    let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]);
+    if (!noSegmentList || noSegmentList.length === 0) {
+        noSegmentList = [];
+    } else {
+        noSegmentList = noSegmentList.map((obj) => {
+            return obj.category;
+        });
+    }
+
+    // get user categories not already submitted that match accepted format
+    let categoriesToMark = categories.filter((category) => {
+        return !!category.match(/^[_a-zA-Z]+$/);
+    }).filter((category) => {
+        return noSegmentList.indexOf(category) === -1;
+    });
+
+    // remove any duplicates
+    categoriesToMark = categoriesToMark.filter((category, index) => {
+        return categoriesToMark.indexOf(category) === index;
+    });
+
+    // create database entry
+    categoriesToMark.forEach((category) => {
+        try {
+            db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]);
+        } catch (err) {
+            logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'");
+            logger.error(err);
+            res.status(500).json({
+                message: "Internal Server Error: Could not write marker to the database."
+            });
+        }
+    });
+
+    res.status(200).json({
+        submitted: categoriesToMark
+    }); 
+};

src/routes/postSegmentShift.js

@@ -0,0 +1,101 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+const ACTION_NONE = Symbol('none');
+const ACTION_UPDATE = Symbol('update');
+const ACTION_REMOVE = Symbol('remove');
+
+function shiftSegment(segment, shift) {
+  if (segment.startTime >= segment.endTime) return {action: ACTION_NONE, segment};
+  if (shift.startTime >= shift.endTime) return {action: ACTION_NONE, segment};
+  const duration = shift.endTime - shift.startTime;
+  if (shift.endTime < segment.startTime) {
+    // Scenario #1 cut before segment
+    segment.startTime -= duration;
+    segment.endTime -= duration;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (shift.startTime > segment.endTime) {
+    // Scenario #2 cut after segment
+    return {action: ACTION_NONE, segment};
+  }
+  if (segment.startTime < shift.startTime && segment.endTime > shift.endTime) {
+    // Scenario #3 cut inside segment
+    segment.endTime -= duration;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (segment.startTime >= shift.startTime && segment.endTime > shift.endTime) {
+    // Scenario #4 cut overlap startTime
+    segment.startTime = shift.startTime;
+    segment.endTime -= duration;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (segment.startTime < shift.startTime && segment.endTime <= shift.endTime) {
+    // Scenario #5 cut overlap endTime
+    segment.endTime = shift.startTime;
+    return {action: ACTION_UPDATE, segment};
+  }
+  if (segment.startTime >= shift.startTime && segment.endTime <= shift.endTime) {
+    // Scenario #6 cut overlap startTime and endTime
+    return {action: ACTION_REMOVE, segment};
+  }
+  return {action: ACTION_NONE, segment};
+}
+
+module.exports = (req, res) => {
+  // Collect user input data
+  const videoID = req.body.videoID;
+  const startTime = req.body.startTime;
+  const endTime = req.body.endTime;
+  let userID = req.body.userID;
+
+  // Check input data is valid
+  if (!videoID
+    || !userID
+    || !startTime
+    || !endTime
+  ) {
+    res.status(400).json({
+      message: 'Bad Format'
+    });
+    return;
+  }
+
+  // Check if user is VIP
+  userID = getHash(userID);
+  const userIsVIP = isUserVIP(userID);
+
+  if (!userIsVIP) {
+    res.status(403).json({
+      message: 'Must be a VIP to perform this action.'
+    });
+    return;
+  }
+  
+  try {
+    const segments = db.prepare('all', 'SELECT startTime, endTime, UUID FROM sponsorTimes WHERE videoID = ?', [videoID]);
+    const shift = {
+      startTime,
+      endTime,
+    };
+    segments.forEach(segment => {
+      const result = shiftSegment(segment, shift);
+        switch (result.action) {
+          case ACTION_UPDATE:
+            db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]);
+          break;
+          case ACTION_REMOVE:
+            db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ?, votes = -2 WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]);
+          break;
+        }
+    });
+  }
+  catch(err) {
+    logger.error(err);
+    res.sendStatus(500);
+  }
+  
+  res.sendStatus(200);
+};

src/routes/postSkipSegments.js

@@ -0,0 +1,530 @@
+const config = require('../config.js');
+
+const databases = require('../databases/databases.js');
+const db = databases.db;
+const privateDB = databases.privateDB;
+const YouTubeAPI = require('../utils/youtubeAPI.js');
+const logger = require('../utils/logger.js');
+const getSubmissionUUID = require('../utils/getSubmissionUUID.js');
+const request = require('request');
+const isoDurations = require('iso8601-duration');
+const fetch = require('node-fetch');
+
+const getHash = require('../utils/getHash.js');
+const getIP = require('../utils/getIP.js');
+const getFormattedTime = require('../utils/getFormattedTime.js');
+const isUserTrustworthy = require('../utils/isUserTrustworthy.js')
+const { dispatchEvent } = require('../utils/webhookUtils.js');
+
+function sendWebhookNotification(userID, videoID, UUID, submissionCount, youtubeData, {submissionStart, submissionEnd}, segmentInfo) {
+    let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]);
+    let userName = row !== undefined ? row.userName : null;
+    let video = youtubeData.items[0];
+
+    let scopeName = "submissions.other";
+    if (submissionCount <= 1) {
+        scopeName = "submissions.new";
+    }
+
+    dispatchEvent(scopeName, {
+        "video": {
+            "id": videoID,
+            "title": video.snippet.title,
+            "thumbnail": video.snippet.thumbnails.maxres ? video.snippet.thumbnails.maxres : null,
+            "url": "https://www.youtube.com/watch?v=" + videoID
+        },
+        "submission": {
+            "UUID": UUID,
+            "category": segmentInfo.category,
+            "startTime": submissionStart,
+            "endTime": submissionEnd,
+            "user": {
+                "UUID": userID,
+                "username": userName
+            }
+        }
+    });
+}
+
+function sendWebhooks(userID, videoID, UUID, segmentInfo) {
+    if (config.youtubeAPIKey !== null) {
+        let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]);
+
+        YouTubeAPI.listVideos(videoID, (err, data) => {
+            if (err || data.items.length === 0) {
+                err && logger.error(err);
+                return;
+            }
+
+            let startTime = parseFloat(segmentInfo.segment[0]);
+            let endTime = parseFloat(segmentInfo.segment[1]);
+            sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {submissionStart: startTime, submissionEnd: endTime}, segmentInfo);
+
+            // If it is a first time submission
+            // Then send a notification to discord
+            if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
+            request.post(config.discordFirstTimeSubmissionsWebhookURL, {
+                json: {
+                    "embeds": [{
+                        "title": data.items[0].snippet.title,
+                        "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2),
+                        "description": "Submission ID: " + UUID +
+                                "\n\nTimestamp: " +
+                                getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
+                                "\n\nCategory: " + segmentInfo.category,
+                        "color": 10813440,
+                        "author": {
+                            "name": userID
+                        },
+                        "thumbnail": {
+                            "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                        }
+                    }]
+                }
+            }, (err, res) => {
+                    if (err) {
+                        logger.error("Failed to send first time submission Discord hook.");
+                        logger.error(JSON.stringify(err));
+                        logger.error("\n");
+                    } else if (res && res.statusCode >= 400) {
+                        logger.error("Error sending first time submission Discord hook");
+                        logger.error(JSON.stringify(res));
+                        logger.error("\n");
+                    }
+            });
+        });
+    }
+}
+
+function sendWebhooksNB(userID, videoID, UUID, startTime, endTime, category, probability, ytData) {
+    let submissionInfoRow = db.prepare('get', "SELECT " +
+        "(select count(1) from sponsorTimes where userID = ?) count, " +
+        "(select count(1) from sponsorTimes where userID = ? and votes <= -2) disregarded, " +
+        "coalesce((select userName FROM userNames WHERE userID = ?), ?) userName",
+        [userID, userID, userID, userID]);
+
+    let submittedBy = "";
+    // If a userName was created then show both
+    if (submissionInfoRow.userName !== userID){
+        submittedBy = submissionInfoRow.userName + "\n " + userID;
+    } else {
+        submittedBy = userID;
+    }
+
+    // Send discord message
+    if (config.discordNeuralBlockRejectWebhookURL === null) return;
+    request.post(config.discordNeuralBlockRejectWebhookURL, {
+        json: {
+            "embeds": [{
+                "title": ytData.items[0].snippet.title,
+                "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (startTime.toFixed(0) - 2),
+                "description": "**Submission ID:** " + UUID +
+                        "\n**Timestamp:** " + getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
+                        "\n**Predicted Probability:** " + probability +
+                        "\n**Category:** " + category +
+                        "\n**Submitted by:** "+ submittedBy +
+                        "\n**Total User Submissions:** "+submissionInfoRow.count +
+                        "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded,
+                "color": 10813440,
+                "thumbnail": {
+                    "url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
+                }
+            }]
+        }
+    }, (err, res) => {
+            if (err) {
+                logger.error("Failed to send NeuralBlock Discord hook.");
+                logger.error(JSON.stringify(err));
+                logger.error("\n");
+            } else if (res && res.statusCode >= 400) {
+                logger.error("Error sending NeuralBlock Discord hook");
+                logger.error(JSON.stringify(res));
+                logger.error("\n");
+            }
+    });
+}
+
+// callback:  function(reject: "String containing reason the submission was rejected")
+// returns: string when an error, false otherwise
+
+// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
+//   false for a pass - it was confusing and lead to this bug - any use of this function in
+//   the future could have the same problem.
+async function autoModerateSubmission(submission) {
+    // Get the video information from the youtube API
+    if (config.youtubeAPIKey !== null) {
+        let {err, data} = await new Promise((resolve, reject) => {
+            YouTubeAPI.listVideos(submission.videoID, (err, data) => resolve({err, data}));
+        });
+
+        if (err) {
+            return false;
+        } else {
+            // Check to see if video exists
+            if (data.pageInfo.totalResults === 0) {
+                return "No video exists with id " + submission.videoID;
+            } else {
+                let segments = submission.segments;
+                let nbString = "";
+                for (let i = 0; i < segments.length; i++) {
+                    let startTime = parseFloat(segments[i].segment[0]);
+                    let endTime = parseFloat(segments[i].segment[1]);
+
+                    let duration = data.items[0].contentDetails.duration;
+                    duration = isoDurations.toSeconds(isoDurations.parse(duration));
+                    if (duration == 0) {
+                        // Allow submission if the duration is 0 (bug in youtube api)
+                        return false;
+                    } else if ((endTime - startTime) > (duration/100)*80) {
+                        // Reject submission if over 80% of the video
+                        return "One of your submitted segments is over 80% of the video.";
+                    } else {
+                        if (segments[i].category === "sponsor") {
+                          //Prepare timestamps to send to NB all at once
+                          nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
+                        }
+                    }
+                }
+                // Check NeuralBlock
+                let neuralBlockURL = config.neuralBlockURL;
+                if (!neuralBlockURL) return false;
+                let response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
+                        "&segments=" + nbString.substring(0,nbString.length-1));
+                if (!response.ok) return false;
+
+                let nbPredictions = await response.json();
+                nbDecision = false;
+                let predictionIdx = 0; //Keep track because only sponsor categories were submitted
+                for (let i = 0; i < segments.length; i++){
+                    if (segments[i].category === "sponsor"){
+                        if (nbPredictions.probabilities[predictionIdx] < 0.70){
+                          nbDecision = true; // At least one bad entry
+                          startTime = parseFloat(segments[i].segment[0]);
+                          endTime = parseFloat(segments[i].segment[1]);
+
+                          const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
+                          // Send to Discord
+                          // Note, if this is too spammy. Consider sending all the segments as one Webhook
+                          sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
+                        }
+                        predictionIdx++;
+                    }
+
+                }
+                if (nbDecision){
+                    return "Rejected based on NeuralBlock predictions.";
+                } else {
+                    return false;
+                }
+            }
+        }
+    } else {
+        logger.debug("Skipped YouTube API");
+
+        // Can't moderate the submission without calling the youtube API
+        // so allow by default.
+        return false;
+    }
+}
+
+function proxySubmission(req) {
+    request.post(config.proxySubmission + '/api/skipSegments?userID='+req.query.userID+'&videoID='+req.query.videoID, {json: req.body}, (err, result) => {
+        if (config.mode === 'development') {
+            if (!err) {
+                logger.debug('Proxy Submission: ' + result.statusCode + ' ('+result.body+')');
+            } else {
+                logger.error("Proxy Submission: Failed to make call");
+            }
+        }
+    });
+}
+
+module.exports = async function postSkipSegments(req, res) {
+    if (config.proxySubmission) {
+        proxySubmission(req);
+    }
+
+    let videoID = req.query.videoID || req.body.videoID;
+    let userID = req.query.userID || req.body.userID;
+
+    let segments = req.body.segments;
+
+    if (segments === undefined) {
+        // Use query instead
+        segments = [{
+            segment: [req.query.startTime, req.query.endTime],
+            category: req.query.category
+        }];
+    }
+
+    const invalidFields = [];
+    if (typeof videoID !== 'string') {
+      invalidFields.push('videoID');
+    }
+    if (typeof userID !== 'string') {
+      invalidFields.push('userID');
+    }
+    if (!Array.isArray(segments) || segments.length < 1) {
+      invalidFields.push('segments');
+    }
+
+    if (invalidFields.length !== 0) {
+      // invalid request
+      const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, '');
+      res.status(400).send(`No valid ${fields} field(s) provided`);
+      return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(getIP(req) + config.globalSalt);
+    
+    const MILLISECONDS_IN_HOUR = 3600000;
+    const now = Date.now();
+    let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?",
+      [userID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))]
+    ).count;
+    
+    if (warningsCount >= config.maxNumberOfActiveWarnings) {
+      return res.status(403).send('Submission blocked. Too many active warnings!');
+    }
+
+    let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
+    
+    //check if this user is on the vip list
+    let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0;
+
+    let decreaseVotes = 0;
+
+    // Check if all submissions are correct
+    for (let i = 0; i < segments.length; i++) {
+        if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
+            //invalid request
+            res.status(400).send("One of your segments are invalid");
+            return;
+        }
+        
+        if (!config.categoryList.includes(segments[i].category)) {
+          res.status("400").send("Category doesn't exist.");
+          return;
+        }
+
+        // Reject segemnt if it's in the no segments list
+        if (!isVIP && noSegmentList.indexOf(segments[i].category) !== -1) {
+            // TODO: Do something about the fradulent submission
+            logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
+            res.status(403).send(
+              "Request rejected by auto moderator: New submissions are not allowed for the following category: '"
+              + segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n "
+              + (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + 
+                        "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "")
+              +  "If you believe this is incorrect, please contact someone on Discord."
+            );
+            return;
+        }
+        
+
+        let startTime = parseFloat(segments[i].segment[0]);
+        let endTime = parseFloat(segments[i].segment[1]);
+
+        if (isNaN(startTime) || isNaN(endTime)
+                || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) {
+            //invalid request
+            res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)");
+            return;
+        }
+
+        if (segments[i].category === "sponsor" && Math.abs(startTime - endTime) < 1) {
+            // Too short
+            res.status(400).send("Sponsors must be longer than 1 second long");
+            return;
+        }
+
+        //check if this info has already been submitted before
+        let duplicateCheck2Row = db.prepare('get', "SELECT COUNT(*) as count FROM sponsorTimes WHERE startTime = ? " +
+            "and endTime = ? and category = ? and videoID = ?", [startTime, endTime, segments[i].category, videoID]);
+        if (duplicateCheck2Row.count > 0) {
+            res.sendStatus(409);
+            return;
+        }
+    }
+
+    // Auto moderator check
+    if (!isVIP) {
+        let autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
+        if (autoModerateResult == "Rejected based on NeuralBlock predictions."){
+            // If NB automod rejects, the submission will start with -2 votes.
+            // Note, if one submission is bad all submissions will be affected.
+            // However, this behavior is consistent with other automod functions
+            // already in place.
+            //decreaseVotes = -2; //Disable for now
+        } else if (autoModerateResult) {
+            //Normal automod behavior
+            res.status(403).send("Request rejected by auto moderator: " + autoModerateResult + " If this is an issue, send a message on Discord.");
+            return;
+        }
+    }
+    // Will be filled when submitting
+    let UUIDs = [];
+
+    try {
+        //get current time
+        let timeSubmitted = Date.now();
+
+        let yesterday = timeSubmitted - 86400000;
+
+        // Disable IP ratelimiting for now
+        if (false) {
+            //check to see if this ip has submitted too many sponsors today
+            let rateLimitCheckRow = privateDB.prepare('get', "SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?", [hashedIP, videoID, yesterday]);
+
+            if (rateLimitCheckRow.count >= 10) {
+                //too many sponsors for the same video from the same ip address
+                res.sendStatus(429);
+
+                return;
+            }
+        }
+
+        // Disable max submissions for now
+        if (false) {
+            //check to see if the user has already submitted sponsors for this video
+            let duplicateCheckRow = db.prepare('get', "SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?", [userID, videoID]);
+
+            if (duplicateCheckRow.count >= 16) {
+                //too many sponsors for the same video from the same user
+                res.sendStatus(429);
+
+                return;
+            }
+        }
+
+        //check to see if this user is shadowbanned
+        let shadowBanRow = privateDB.prepare('get', "SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?", [userID]);
+
+        let shadowBanned = shadowBanRow.userCount;
+
+        if (!(await isUserTrustworthy(userID))) {
+            //hide this submission as this user is untrustworthy
+            shadowBanned = 1;
+        }
+
+        let startingVotes = 0 + decreaseVotes;
+        if (isVIP) {
+            //this user is a vip, start them at a higher approval rating
+            startingVotes = 10000;
+        }
+
+        if (config.youtubeAPIKey !== null) {
+            let {err, data} = await new Promise((resolve, reject) => {
+                YouTubeAPI.listVideos(videoID, (err, data) => resolve({err, data}));
+            });
+
+            if (err) {
+                logger.error("Error while submitting when connecting to YouTube API: " + err);
+            } else {
+                //get all segments for this video and user
+                let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ? and votes > -1", [userID, videoID]);
+                let allSegmentTimes = [];
+                if (allSubmittedByUser !== undefined) {
+                    //add segments the user has previously submitted
+                    for (const segmentInfo of allSubmittedByUser) {
+                        allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)])
+                    }
+                }
+
+                //add segments they are trying to add in this submission
+                for (let i = 0; i < segments.length; i++) {
+                    let startTime = parseFloat(segments[i].segment[0]);
+                    let endTime = parseFloat(segments[i].segment[1]);
+                    allSegmentTimes.push([startTime, endTime]);
+                }
+
+                //merge all the times into non-overlapping arrays
+                const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function(a, b) { return a[0]-b[0] || a[1]-b[1] }));
+
+                let videoDuration = data.items[0].contentDetails.duration;
+                videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
+                if (videoDuration != 0) {
+                    let allSegmentDuration = 0;
+                    //sum all segment times together
+                    allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
+                    if (allSegmentDuration > (videoDuration/100)*80) {
+                        // Reject submission if all segments combine are over 80% of the video
+                        res.status(400).send("Total length of your submitted segments are over 80% of the video.");
+                        return;
+                    }
+                }
+            }
+        }
+
+        for (const segmentInfo of segments) {
+            //this can just be a hash of the data
+            //it's better than generating an actual UUID like what was used before
+            //also better for duplication checking
+            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
+
+            try {
+                db.prepare('run', "INSERT INTO sponsorTimes " + 
+                    "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
+                    "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1)
+                    ]
+                );
+            
+                //add to private db as well
+                privateDB.prepare('run', "INSERT INTO sponsorTimes VALUES(?, ?, ?)", [videoID, hashedIP, timeSubmitted]);
+            } catch (err) {
+                //a DB change probably occurred
+                res.sendStatus(502);
+                logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " +
+                    segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err);
+
+                return;
+            }
+
+            UUIDs.push(UUID);
+        }
+    } catch (err) {
+        logger.error(err);
+
+        res.sendStatus(500);
+
+        return;
+    }
+
+    res.sendStatus(200);
+
+    for (let i = 0; i < segments.length; i++) {
+        sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
+    }
+}
+
+// Takes an array of arrays: 
+// ex) 
+// [
+//     [3, 40], 
+//     [50, 70], 
+//     [60, 80], 
+//     [100, 150]
+// ] 
+// => transforms to combining overlapping segments
+// [
+//     [3, 40],
+//     [50, 80], 
+//     [100, 150]
+// ]
+function mergeTimeSegments(ranges) {
+    var result = [], last;
+
+    ranges.forEach(function (r) {
+        if (!last || r[0] > last[1])
+            result.push(last = r);
+        else if (r[1] > last[1])
+            last[1] = r[1];
+    });
+
+    return result;
+}

src/routes/postWarning.js

@@ -0,0 +1,24 @@
+const db = require('../databases/databases.js').db;
+const getHash = require('../utils/getHash.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+const logger = require('../utils/logger.js');
+
+module.exports = (req, res) => {
+    // Collect user input data
+    let issuerUserID = getHash(req.body.issuerUserID);
+    let userID = getHash(req.body.userID);
+    let issueTime = new Date().getTime();
+
+    // Ensure user is a VIP
+    if (!isUserVIP(issuerUserID)) {
+        logger.debug("Permission violation: User " + issuerUserID + " attempted to warn user " + userID + "."); // maybe warn?
+        res.status(403).json({"message": "Not a VIP"});
+        return;
+    }
+
+    db.prepare('run', 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES (?, ?, ?)', [userID, issueTime, issuerUserID]);
+    res.status(200).json({
+        message: "Warning issued to user '" + userID + "'."
+    });
+
+};

src/routes/setUsername.js

@@ -0,0 +1,60 @@
+
+var config = require('../config.js');
+
+var db = require('../databases/databases.js').db;
+var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+
+
+module.exports = function setUsername(req, res) {
+  let userID = req.query.userID;
+  let userName = req.query.username;
+
+  let adminUserIDInput = req.query.adminUserID;
+
+  if (userID == undefined || userName == undefined || userID === "undefined" || userName.length > 64) {
+      //invalid request
+      res.sendStatus(400);
+      return;
+  }
+
+  if (userName.includes("discord")) {
+      // Don't allow
+      res.sendStatus(200);
+      return;
+  }
+
+  if (adminUserIDInput != undefined) {
+      //this is the admin controlling the other users account, don't hash the controling account's ID
+      adminUserIDInput = getHash(adminUserIDInput);
+
+      if (adminUserIDInput != config.adminUserID) {
+          //they aren't the admin
+          res.sendStatus(403);
+          return;
+      }
+  } else {
+      //hash the userID
+      userID = getHash(userID);
+  }
+
+  try {
+      //check if username is already set
+      let row = db.prepare('get', "SELECT count(*) as count FROM userNames WHERE userID = ?", [userID]);
+
+      if (row.count > 0) {
+          //already exists, update this row
+          db.prepare('run', "UPDATE userNames SET userName = ? WHERE userID = ?", [userName, userID]);
+      } else {
+          //add to the db
+          db.prepare('run', "INSERT INTO userNames VALUES(?, ?)", [userID, userName]);
+      }
+
+      res.sendStatus(200);
+  } catch (err) {
+      logger.error(err);
+      res.sendStatus(500);
+
+      return;
+  }
+}

src/routes/shadowBanUser.js

@@ -0,0 +1,100 @@
+var databases = require('../databases/databases.js');
+var db = databases.db;
+var privateDB = databases.privateDB;
+
+var getHash = require('../utils/getHash.js');
+const logger = require('../utils/logger.js');
+
+module.exports = async function shadowBanUser(req, res) {
+  let userID = req.query.userID;
+  let hashedIP = req.query.hashedIP;
+  let adminUserIDInput = req.query.adminUserID;
+
+  let enabled = req.query.enabled;
+  if (enabled === undefined){
+      enabled = true;
+  } else {
+      enabled = enabled === "true";
+  }
+
+  //if enabled is false and the old submissions should be made visible again
+  let unHideOldSubmissions = req.query.unHideOldSubmissions !== "false";
+
+  if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined)) {
+      //invalid request
+      res.sendStatus(400);
+      return;
+  }
+
+  //hash the userID
+  adminUserIDInput = getHash(adminUserIDInput);
+
+  let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [adminUserIDInput]).userCount > 0;
+  if (!isVIP) {
+      //not authorized
+      res.sendStatus(403);
+      return;
+  }
+
+  if (userID) {
+    //check to see if this user is already shadowbanned
+    let row = privateDB.prepare('get', "SELECT count(*) as userCount FROM shadowBannedUsers WHERE userID = ?", [userID]);
+
+    if (enabled && row.userCount == 0) {
+        //add them to the shadow ban list
+
+        //add it to the table
+        privateDB.prepare('run', "INSERT INTO shadowBannedUsers VALUES(?)", [userID]);
+
+        //find all previous submissions and hide them
+        if (unHideOldSubmissions) {
+            db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 1 WHERE userID = ?", [userID]);
+        }
+    } else if (!enabled && row.userCount > 0) {
+        //remove them from the shadow ban list
+        privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]);
+
+        //find all previous submissions and unhide them
+        if (unHideOldSubmissions) {
+            let segmentsToIgnore = db.prepare('all', "SELECT uuid FROM sponsorTimes st JOIN noSegments ns on st.videoID = ns.videoID AND st.category = ns.category WHERE st.userID = ?", [userID]).map((item) => item.UUID);
+            let allSegments = db.prepare('all', "SELECT uuid FROM sponsorTimes st WHERE st.userID = ?", [userID]).map((item) => item.UUID);
+            
+            allSegments.filter((item) => {
+                return segmentsToIgnore.indexOf(item) === -1;
+            }).forEach((uuid) => {
+                db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 0 WHERE uuid = ?", [uuid]);
+            });
+        }
+    }
+  } else if (hashedIP) {
+    //check to see if this user is already shadowbanned
+    // let row = privateDB.prepare('get', "SELECT count(*) as userCount FROM shadowBannedIPs WHERE hashedIP = ?", [hashedIP]);
+
+    // if (enabled && row.userCount == 0) {
+    if (enabled) {
+        //add them to the shadow ban list
+
+        //add it to the table
+        // privateDB.prepare('run', "INSERT INTO shadowBannedIPs VALUES(?)", [hashedIP]);
+
+        
+
+        //find all previous submissions and hide them
+        if (unHideOldSubmissions) {
+            db.prepare('run',  "UPDATE sponsorTimes SET shadowHidden = 1 WHERE timeSubmitted IN " +
+                "(SELECT privateDB.timeSubmitted FROM sponsorTimes LEFT JOIN privateDB.sponsorTimes as privateDB ON sponsorTimes.timeSubmitted=privateDB.timeSubmitted " +
+                "WHERE privateDB.hashedIP = ?)", [hashedIP]);
+        }
+    } else if (!enabled && row.userCount > 0) {
+        // //remove them from the shadow ban list
+        // privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]);
+
+        // //find all previous submissions and unhide them
+        // if (unHideOldSubmissions) {
+        //     db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 0 WHERE userID = ?", [userID]);
+        // }
+    }
+  }
+
+  res.sendStatus(200);
+}

src/routes/viewedVideoSponsorTime.js

@@ -0,0 +1,16 @@
+var db = require('../databases/databases.js').db;
+
+module.exports = function viewedVideoSponsorTime(req, res) {
+  let UUID = req.query.UUID;
+
+  if (UUID == undefined) {
+      //invalid request
+      res.sendStatus(400);
+      return;
+  }
+
+  //up the view count by one
+  db.prepare('run', "UPDATE sponsorTimes SET views = views + 1 WHERE UUID = ?", [UUID]);
+
+  res.sendStatus(200);
+}

src/routes/voteOnSponsorTime.js

@@ -0,0 +1,403 @@
+var config = require('../config.js');
+
+var getHash = require('../utils/getHash.js');
+var getIP = require('../utils/getIP.js');
+var getFormattedTime = require('../utils/getFormattedTime.js');
+var isUserTrustworthy = require('../utils/isUserTrustworthy.js');
+const { getVoteAuthor, getVoteAuthorRaw, dispatchEvent } = require('../utils/webhookUtils.js');
+
+var databases = require('../databases/databases.js');
+var db = databases.db;
+var privateDB = databases.privateDB;
+var YouTubeAPI = require('../utils/youtubeAPI.js');
+var request = require('request');
+const logger = require('../utils/logger.js');
+const isUserVIP = require('../utils/isUserVIP.js');
+
+const voteTypes = {
+    normal: 0,
+    incorrect: 1
+}
+
+/**
+ * @param {Object} voteData
+ * @param {string} voteData.UUID
+ * @param {string} voteData.nonAnonUserID
+ * @param {number} voteData.voteTypeEnum
+ * @param {boolean} voteData.isVIP
+ * @param {boolean} voteData.isOwnSubmission
+ * @param voteData.row
+ * @param {string} voteData.category
+ * @param {number} voteData.incrementAmount
+ * @param {number} voteData.oldIncrementAmount
+ */
+function sendWebhooks(voteData) {
+    let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " +
+        "(select count(1) from sponsorTimes where userID = s.userID) count, " +
+        "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
+        "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?",
+    [voteData.UUID]);
+
+    let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [voteData.nonAnonUserID]);
+
+    if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) {
+        let webhookURL = null;
+        if (voteData.voteTypeEnum === voteTypes.normal) {
+            webhookURL = config.discordReportChannelWebhookURL;
+        } else if (voteData.voteTypeEnum === voteTypes.incorrect) {
+            webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
+        }
+
+        if (config.youtubeAPIKey !== null) {
+            YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => {
+                if (err || data.items.length === 0) {
+                    err && logger.error(err);
+                    return;
+                }
+                let isUpvote = voteData.incrementAmount > 0;
+                // Send custom webhooks
+                dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
+                    "user": {
+                        "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission)
+                    },
+                    "video": {
+                        "id": submissionInfoRow.videoID,
+                        "title": data.items[0].snippet.title,
+                        "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
+                        "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : ""
+                    },
+                    "submission": {
+                        "UUID": voteData.UUID,
+                        "views": voteData.row.views,
+                        "category": voteData.category,
+                        "startTime": submissionInfoRow.startTime,
+                        "endTime": submissionInfoRow.endTime,
+                        "user": {
+                            "UUID": submissionInfoRow.userID,
+                            "username": submissionInfoRow.userName,
+                            "submissions": {
+                                "total": submissionInfoRow.count,
+                                "ignored": submissionInfoRow.disregarded
+                            }
+                        }
+                    },
+                    "votes": {
+                        "before": voteData.row.votes,
+                        "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)
+                    }
+                });
+                
+                // Send discord message
+                if (webhookURL !== null && !isUpvote) {
+                    request.post(webhookURL, {
+                        json: {
+                            "embeds": [{
+                                "title": data.items[0].snippet.title,
+                                "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID 
+                                    + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
+                                "description": "**" + voteData.row.votes + " Votes Prior | " + 
+                                    (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views 
+                                    + " Views**\n\n**Submission ID:** " + voteData.UUID 
+                                    + "\n**Category:** " + submissionInfoRow.category
+                                    + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID 
+                                    + "\n\n**Total User Submissions:** "+submissionInfoRow.count
+                                    + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded
+                                    +"\n\n**Timestamp:** " + 
+                                        getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
+                                "color": 10813440,
+                                "author": {
+                                    "name": getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission)
+                                },
+                                "thumbnail": {
+                                    "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                                }
+                            }]
+                        }
+                    }, (err, res) => {
+                        if (err) {
+                            logger.error("Failed to send reported submission Discord hook.");
+                            logger.error(JSON.stringify(err));
+                            logger.error("\n");
+                        } else if (res && res.statusCode >= 400) {
+                            logger.error("Error sending reported submission Discord hook");
+                            logger.error(JSON.stringify(res));
+                            logger.error("\n");
+                        }
+                    });
+                }
+
+            });
+        }
+    }
+}
+
+function categoryVote(UUID, userID, isVIP, category, hashedIP, res) {
+    // Check if they've already made a vote
+    let previousVoteInfo = privateDB.prepare('get', "select count(*) as votes, category from categoryVotes where UUID = ? and userID = ?", [UUID, userID]);
+
+    if (previousVoteInfo !== undefined && previousVoteInfo.category === category) {
+        // Double vote, ignore
+        res.sendStatus(200);
+        return;
+    }
+
+    let currentCategory = db.prepare('get', "select category from sponsorTimes where UUID = ?", [UUID]);
+    if (!currentCategory) {
+        // Submission doesn't exist
+        res.status("400").send("Submission doesn't exist.");
+        return;
+    }
+    
+    if (!config.categoryList.includes(category)) {
+      res.status("400").send("Category doesn't exist.");
+      return;
+    }
+
+    let timeSubmitted = Date.now();
+
+    let voteAmount = isVIP ? 500 : 1;
+
+    // Add the vote
+    if (db.prepare('get', "select count(*) as count from categoryVotes where UUID = ? and category = ?", [UUID, category]).count > 0) {
+        // Update the already existing db entry
+        db.prepare('run', "update categoryVotes set votes = votes + ? where UUID = ? and category = ?", [voteAmount, UUID, category]);
+    } else {
+        // Add a db entry
+        db.prepare('run', "insert into categoryVotes (UUID, category, votes) values (?, ?, ?)", [UUID, category, voteAmount]);
+    }
+
+    // Add the info into the private db
+    if (previousVoteInfo  !== undefined) {
+        // Reverse the previous vote
+        db.prepare('run', "update categoryVotes set votes = votes - ? where UUID = ? and category = ?", [voteAmount, UUID, previousVoteInfo.category]);
+
+        privateDB.prepare('run', "update categoryVotes set category = ?, timeSubmitted = ?, hashedIP = ? where userID = ?", [category, timeSubmitted, hashedIP, userID]);
+    } else {
+        privateDB.prepare('run', "insert into categoryVotes (UUID, userID, hashedIP, category, timeSubmitted) values (?, ?, ?, ?, ?)", [UUID, userID, hashedIP, category, timeSubmitted]);
+    }
+
+    // See if the submissions category is ready to change
+    let currentCategoryInfo = db.prepare("get", "select votes from categoryVotes where UUID = ? and category = ?", [UUID, currentCategory.category]);
+
+    let submissionInfo = db.prepare("get", "SELECT userID, timeSubmitted, votes FROM sponsorTimes WHERE UUID = ?", [UUID]);
+    let isSubmissionVIP = submissionInfo && isUserVIP(submissionInfo.userID);
+    let startingVotes = isSubmissionVIP ? 10000 : 1;
+
+    // Change this value from 1 in the future to make it harder to change categories
+    // Done this way without ORs incase the value is zero
+    let currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
+
+    // Add submission as vote
+    if (!currentCategoryInfo && submissionInfo) {
+        db.prepare("run", "insert into categoryVotes (UUID, category, votes) values (?, ?, ?)", [UUID, currentCategory.category, currentCategoryCount]);
+
+        privateDB.prepare("run", "insert into categoryVotes (UUID, userID, hashedIP, category, timeSubmitted) values (?, ?, ?, ?, ?)", [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
+    }
+
+    let nextCategoryCount = (previousVoteInfo.votes || 0) + voteAmount;
+
+    //TODO: In the future, raise this number from zero to make it harder to change categories
+    // VIPs change it every time
+    if (nextCategoryCount - currentCategoryCount >= (submissionInfo ? Math.max(Math.ceil(submissionInfo.votes / 2), 1) : 1) || isVIP) {
+        // Replace the category
+        db.prepare('run', "update sponsorTimes set category = ? where UUID = ?", [category, UUID]);
+    }
+
+    res.sendStatus(200);
+}
+
+async function voteOnSponsorTime(req, res) {
+    let UUID = req.query.UUID;
+    let userID = req.query.userID;
+    let type = req.query.type;
+    let category = req.query.category;
+
+    if (UUID === undefined || userID === undefined || (type === undefined && category === undefined)) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    let nonAnonUserID = getHash(userID);
+    userID = getHash(userID + UUID);
+
+    //x-forwarded-for if this server is behind a proxy
+    let ip = getIP(req);
+
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(ip + config.globalSalt);
+
+    //check if this user is on the vip list
+    let isVIP = db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [nonAnonUserID]).userCount > 0;
+
+    //check if user voting on own submission
+    let isOwnSubmission = db.prepare("get", "SELECT UUID as submissionCount FROM sponsorTimes where userID = ? AND UUID = ?", [nonAnonUserID, UUID]) !== undefined;
+        
+    if (type === undefined && category !== undefined) {
+        return categoryVote(UUID, nonAnonUserID, isVIP, category, hashedIP, res);
+    }
+
+    if (type == 1 && !isVIP && !isOwnSubmission) {
+        // Check if upvoting hidden segment
+        let voteInfo = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", [UUID]);
+
+        if (voteInfo && voteInfo.votes <= -2) {
+            res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.")
+            return;
+        }
+    }
+    
+    const MILLISECONDS_IN_HOUR = 3600000;
+    const now = Date.now();
+    let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?",
+      [nonAnonUserID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))]
+    ).count;
+    
+    if (warningsCount >= config.maxNumberOfActiveWarnings) {
+      return res.status(403).send('Vote blocked. Too many active warnings!');
+    }
+
+    let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;
+
+    try {
+        //check if vote has already happened
+        let votesRow = privateDB.prepare('get', "SELECT type FROM votes WHERE userID = ? AND UUID = ?", [userID, UUID]);
+        
+        //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
+        let incrementAmount = 0;
+        let oldIncrementAmount = 0;
+
+        if (type == 1 || type == 11) {
+            //upvote
+            incrementAmount = 1;
+        } else if (type == 0 || type == 10) {
+            //downvote
+            incrementAmount = -1;
+        } else if (type == 20) {
+            //undo/cancel vote
+            incrementAmount = 0;
+        } else {
+            //unrecongnised type of vote
+            res.sendStatus(400);
+            return;
+        }
+        if (votesRow != undefined) {
+            if (votesRow.type === 1 || type === 11) {
+                //upvote
+                oldIncrementAmount = 1;
+            } else if (votesRow.type === 0 || type === 10) {
+                //downvote
+                oldIncrementAmount = -1;
+            } else if (votesRow.type === 2) {
+                //extra downvote
+                oldIncrementAmount = -4;
+            } else if (votesRow.type === 20) {
+                //undo/cancel vote
+                oldIncrementAmount = 0;
+            } else if (votesRow.type < 0) {
+                //vip downvote
+                oldIncrementAmount = votesRow.type;
+            } else if (votesRow.type === 12) {
+                // VIP downvote for completely incorrect
+                oldIncrementAmount = -500;
+            } else if (votesRow.type === 13) {
+                // VIP upvote for completely incorrect
+                oldIncrementAmount = 500;
+            }
+        }
+
+        //check if the increment amount should be multiplied (downvotes have more power if there have been many views)
+        let row = db.prepare('get', "SELECT votes, views FROM sponsorTimes WHERE UUID = ?", [UUID]);
+
+        if (voteTypeEnum === voteTypes.normal) {
+            if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
+                //this user is a vip and a downvote
+                incrementAmount = - (row.votes + 2 - oldIncrementAmount);
+                type = incrementAmount;
+            }
+        } else if (voteTypeEnum == voteTypes.incorrect) {
+            if (isVIP || isOwnSubmission) {
+                //this user is a vip and a downvote
+                incrementAmount = 500 * incrementAmount;
+                type = incrementAmount < 0 ? 12 : 13;
+            }
+        }
+
+        // Only change the database if they have made a submission before and haven't voted recently
+        let ableToVote = isVIP 
+                        || (db.prepare("get", "SELECT userID FROM sponsorTimes WHERE userID = ?", [nonAnonUserID]) !== undefined
+                        && privateDB.prepare("get", "SELECT userID FROM shadowBannedUsers WHERE userID = ?", [nonAnonUserID]) === undefined
+                        && privateDB.prepare("get", "SELECT UUID FROM votes WHERE UUID = ? AND hashedIP = ? AND userID != ?", [UUID, hashedIP, userID]) === undefined);
+
+        if (ableToVote) {
+            //update the votes table
+            if (votesRow != undefined) {
+                privateDB.prepare('run', "UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?", [type, userID, UUID]);
+            } else {
+                privateDB.prepare('run', "INSERT INTO votes VALUES(?, ?, ?, ?)", [UUID, userID, hashedIP, type]);
+            }
+
+            let columnName = "";
+            if (voteTypeEnum === voteTypes.normal) {
+                columnName = "votes";
+            } else if (voteTypeEnum === voteTypes.incorrect) {
+                columnName = "incorrectVotes";
+            }
+
+            //update the vote count on this sponsorTime
+            //oldIncrementAmount will be zero is row is null
+            db.prepare('run', "UPDATE sponsorTimes SET " + columnName + " = " + columnName + " + ? WHERE UUID = ?", [incrementAmount - oldIncrementAmount, UUID]);
+
+            //for each positive vote, see if a hidden submission can be shown again
+            if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
+                //find the UUID that submitted the submission that was voted on
+                let submissionUserIDInfo = db.prepare('get', "SELECT userID FROM sponsorTimes WHERE UUID = ?", [UUID]);
+                if (!submissionUserIDInfo) {
+                    // They are voting on a non-existent submission
+                    res.status(400).send("Voting on a non-existent submission");
+                    return;
+                }
+
+                let submissionUserID = submissionUserIDInfo.userID;
+
+                //check if any submissions are hidden
+                let hiddenSubmissionsRow = db.prepare('get', "SELECT count(*) as hiddenSubmissions FROM sponsorTimes WHERE userID = ? AND shadowHidden > 0", [submissionUserID]);
+
+                if (hiddenSubmissionsRow.hiddenSubmissions > 0) {
+                    //see if some of this users submissions should be visible again
+                    
+                    if (await isUserTrustworthy(submissionUserID)) {
+                        //they are trustworthy again, show 2 of their submissions again, if there are two to show
+                        db.prepare('run', "UPDATE sponsorTimes SET shadowHidden = 0 WHERE ROWID IN (SELECT ROWID FROM sponsorTimes WHERE userID = ? AND shadowHidden = 1 LIMIT 2)", [submissionUserID]);
+                    }
+                }
+            }
+        }
+
+        res.sendStatus(200);
+
+        if (incrementAmount - oldIncrementAmount !== 0) {
+            sendWebhooks({
+                UUID,
+                nonAnonUserID,
+                voteTypeEnum,
+                isVIP,
+                isOwnSubmission,
+                row,
+                category,
+                incrementAmount,
+                oldIncrementAmount
+            });
+        }
+    } catch (err) {
+        logger.error(err);
+
+        res.status(500).json({error: 'Internal error creating segment vote'});
+    }
+}
+
+module.exports = {
+  voteOnSponsorTime,
+  endpoint: voteOnSponsorTime
+};

src/utils/createMemoryCache.js

@@ -0,0 +1,44 @@
+module.exports = function createMemoryCache(memoryFn, cacheTimeMs) {
+  if (isNaN(cacheTimeMs)) cacheTimeMs = 0;
+
+  // holds the promise results
+  const cache = new Map();
+  // holds the promises that are not fulfilled
+  const promiseMemory = new Map();
+  return (...args) => {
+    // create cacheKey by joining arguments as string
+    const cacheKey = args.join('.');
+    // check if promising is already running
+    if (promiseMemory.has(cacheKey)) {
+      return promiseMemory.get(cacheKey);
+    }
+    else {
+      // check if result is in cache
+      if (cache.has(cacheKey)) {
+        const cacheItem = cache.get(cacheKey);
+        const now = Date.now();
+        // check if cache is valid
+        if (!(cacheItem.cacheTime + cacheTimeMs < now)) {
+          return Promise.resolve(cacheItem.result);
+        }
+      }
+      // create new promise
+      const promise = new Promise(async (resolve, reject) => {
+        resolve((await memoryFn(...args)));
+      });
+      // store promise reference until fulfilled
+      promiseMemory.set(cacheKey, promise);
+      return promise.then(result => {
+        // store promise result in cache
+        cache.set(cacheKey, {
+          result,
+          cacheTime: Date.now(),
+        });
+        // remove fulfilled promise from memory
+        promiseMemory.delete(cacheKey);
+        // return promise result
+        return result;
+      });
+    }
+  };
+};

src/utils/getFormattedTime.js

@@ -0,0 +1,14 @@
+//converts time in seconds to minutes:seconds
+module.exports = function getFormattedTime(totalSeconds) {
+  let minutes = Math.floor(totalSeconds / 60);
+  let seconds = totalSeconds - minutes * 60;
+  let secondsDisplay = seconds.toFixed(3);
+  if (seconds < 10) {
+      //add a zero
+      secondsDisplay = "0" + secondsDisplay;
+  }
+
+  let formatted = minutes+ ":" + secondsDisplay;
+
+  return formatted;
+}

src/utils/getHash.js

@@ -0,0 +1,12 @@
+var crypto = require('crypto');
+
+module.exports = function (value, times=5000) {
+  if (times <= 0) return "";
+
+  for (let i = 0; i < times; i++) {
+      let hashCreator = crypto.createHash('sha256');
+      value = hashCreator.update(value).digest('hex');
+  }
+
+  return value;
+}

src/utils/getIP.js

@@ -0,0 +1,16 @@
+var config = require('../config.js');
+
+module.exports = function getIP(req) {
+  if (config.behindProxy === true || config.behindProxy === "true") config.behindProxy = "X-Forwarded-For";
+
+  switch (config.behindProxy) {
+    case "X-Forwarded-For":
+      return req.headers['x-forwarded-for'];
+    case "Cloudflare":
+      return req.headers['cf-connecting-ip'];
+    case "X-Real-IP":
+      return req.headers['x-real-ip'];
+    default:
+      return req.connection.remoteAddress;
+  }
+}

src/utils/getSubmissionUUID.js

@@ -0,0 +1,7 @@
+const getHash = require('./getHash.js');
+
+module.exports = function getSubmissionUUID(videoID, category, userID,
+  startTime, endTime) {
+  return getHash('v2-categories' + videoID + startTime + endTime + category +
+    userID, 1);
+};

src/utils/hashPrefixTester.js

@@ -0,0 +1,10 @@
+const config = require('../config.js');
+
+const minimumPrefix = config.minimumPrefix || '3';
+const maximumPrefix = config.maximumPrefix || '32'; // Half the hash.
+
+const prefixChecker = new RegExp('^[\\da-f]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i');
+
+module.exports = (prefix) => {
+    return prefixChecker.test(prefix);
+};

src/utils/isUserTrustworthy.js

@@ -0,0 +1,20 @@
+var databases = require('../databases/databases.js');
+var db = databases.db;
+
+//returns true if the user is considered trustworthy
+//this happens after a user has made 5 submissions and has less than 60% downvoted submissions
+
+module.exports = async (userID) => {
+    //check to see if this user how many submissions this user has submitted
+    let totalSubmissionsRow = db.prepare('get', "SELECT count(*) as totalSubmissions, sum(votes) as voteSum FROM sponsorTimes WHERE userID = ?", [userID]);
+
+    if (totalSubmissionsRow.totalSubmissions > 5) {
+        //check if they have a high downvote ratio
+        let downvotedSubmissionsRow = db.prepare('get', "SELECT count(*) as downvotedSubmissions FROM sponsorTimes WHERE userID = ? AND (votes < 0 OR shadowHidden > 0)", [userID]);
+        
+        return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || 
+                (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions);
+    }
+
+    return true;
+}

src/utils/isUserVIP.js

@@ -0,0 +1,8 @@
+const databases = require('../databases/databases.js');
+const db = databases.db;
+
+module.exports = (userID) => {
+    return db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0;
+}
+
+

src/utils/logger.js

@@ -0,0 +1,70 @@
+const config = require('../config.js');
+
+const levels = {
+  ERROR: "ERROR",
+  WARN: "WARN",
+  INFO: "INFO",
+  DEBUG: "DEBUG"
+};
+
+const colors = {
+  Reset: "\x1b[0m",
+  Bright: "\x1b[1m",
+  Dim: "\x1b[2m",
+  Underscore: "\x1b[4m",
+  Blink: "\x1b[5m",
+  Reverse: "\x1b[7m",
+  Hidden: "\x1b[8m",
+
+  FgBlack: "\x1b[30m",
+  FgRed: "\x1b[31m",
+  FgGreen: "\x1b[32m",
+  FgYellow: "\x1b[33m",
+  FgBlue: "\x1b[34m",
+  FgMagenta: "\x1b[35m",
+  FgCyan: "\x1b[36m",
+  FgWhite: "\x1b[37m",
+
+  BgBlack: "\x1b[40m",
+  BgRed: "\x1b[41m",
+  BgGreen: "\x1b[42m",
+  BgYellow: "\x1b[43m",
+  BgBlue: "\x1b[44m",
+  BgMagenta: "\x1b[45m",
+  BgCyan: "\x1b[46m",
+  BgWhite: "\x1b[47m",
+}
+
+const settings = {
+  ERROR: true,
+  WARN: true,
+  INFO: false,
+  DEBUG: false
+};
+
+if (config.mode === 'development') {
+  settings.INFO = true;
+  settings.DEBUG = true;
+} else if (config.mode === 'test') {
+  settings.WARN = false;
+}
+
+function log(level, string) {
+  if (!!settings[level]) {
+    let color = colors.Bright;
+    if (level === levels.ERROR) color = colors.FgRed;
+    if (level === levels.WARN) color = colors.FgYellow;
+
+    if (level.length === 4) {level = level + " "}; // ensure logs are aligned
+    console.log(colors.Dim, level + " " + new Date().toISOString() + ": ", color, string, colors.Reset);
+  }
+}
+
+module.exports = {
+  levels,
+  log,
+  error: (string) => {log(levels.ERROR, string)},
+  warn: (string) => {log(levels.WARN, string)},
+  info: (string) => {log(levels.INFO, string)},
+  debug: (string) => {log(levels.DEBUG, string)},
+};

src/utils/redis.js

@@ -0,0 +1,18 @@
+const config = require('../config.js');
+const logger = require('./logger.js');
+
+if (config.redis) {
+    const redis = require('redis');
+    logger.info('Connected to redis');
+    const client = redis.createClient(config.redis);
+    module.exports = client;
+} else {
+    module.exports = {
+        get: (key, callback) => {
+            callback(false);
+        },
+        set: (key, value, callback) => {
+            callback(false);
+        }
+    };
+}

src/utils/webhookUtils.js

@@ -0,0 +1,52 @@
+const config = require('../config.js');
+const logger = require('../utils/logger.js');
+const request = require('request');
+
+function getVoteAuthorRaw(submissionCount, isVIP, isOwnSubmission) {
+    if (isOwnSubmission) {
+        return "self";
+    } else if (isVIP) {
+        return "vip";
+    } else if (submissionCount === 0) {
+        return "new";
+    } else {
+        return "other";
+    };
+};
+
+function getVoteAuthor(submissionCount, isVIP, isOwnSubmission) {
+    if (submissionCount === 0) {
+        return "Report by New User";
+    } else if (isOwnSubmission) {
+        return "Report by Submitter";
+    } else if (isVIP) {
+        return "Report by VIP User";
+    }
+
+    return "";
+}
+
+function dispatchEvent(scope, data) {
+    let webhooks = config.webhooks;
+    if (webhooks === undefined || webhooks.length === 0) return;
+    logger.debug("Dispatching webhooks");
+    webhooks.forEach(webhook => {
+        let webhookURL = webhook.url;
+        let authKey = webhook.key;
+        let scopes = webhook.scopes || [];
+        if (!scopes.includes(scope.toLowerCase())) return;
+        request.post(webhookURL, {json: data, headers: {
+            "Authorization": authKey,
+            "Event-Type": scope // Maybe change this in the future? 
+        }}).on('error', (e) => {
+            logger.warn('Couldn\'t send webhook to ' + webhook.url);
+            logger.warn(e);
+        });
+    });
+}
+
+module.exports = {
+    getVoteAuthorRaw,
+    getVoteAuthor,
+    dispatchEvent
+}

src/utils/youtubeAPI.js

@@ -0,0 +1,62 @@
+var config = require('../config.js');
+
+// YouTube API
+const YouTubeAPI = require("youtube-api");
+const redis = require('./redis.js');
+const logger = require('./logger.js');
+
+var exportObject;
+// If in test mode, return a mocked youtube object
+//   otherwise return an authenticated youtube api
+if (config.mode === "test") {
+    exportObject = require("../../test/youtubeMock.js");
+} else {
+    YouTubeAPI.authenticate({
+        type: "key",
+        key: config.youtubeAPIKey
+    });
+    exportObject = YouTubeAPI;
+
+    // YouTubeAPI.videos.list wrapper with cacheing
+    exportObject.listVideos = (videoID, callback) => {
+        let part = 'contentDetails,snippet';
+        if (videoID.length !== 11 || videoID.includes(".")) {
+            callback("Invalid video ID");
+            return;
+        }
+
+        let redisKey = "youtube.video." + videoID;
+        redis.get(redisKey, (getErr, result) => {
+            if (getErr || !result) {
+                logger.debug("redis: no cache for video information: " + videoID);
+                YouTubeAPI.videos.list({
+                    part,
+                    id: videoID
+                }, (ytErr, data) => {
+                    if (!ytErr) {
+                        // Only set cache if data returned
+                        if (data.items.length > 0) {
+                            redis.set(redisKey, JSON.stringify(data), (setErr) => {
+                                if(setErr) {
+                                    logger.warn(setErr);
+                                } else {
+                                    logger.debug("redis: video information cache set for: " + videoID);
+                                }
+                                callback(false, data); // don't fail
+                            });
+                        } else {
+                            callback(false, data); // don't fail
+                        }
+                    } else {
+                        callback(ytErr, data)
+                    }
+                });
+            } else {
+                logger.debug("redis: fetched video information from cache: " + videoID);
+                callback(getErr, JSON.parse(result));
+            }
+        });
+    };
+}
+
+module.exports = exportObject;

test.js

@@ -0,0 +1,41 @@
+var Mocha = require('mocha'),
+    fs = require('fs'),
+    path = require('path');
+
+var config = require('./src/config.js');
+// delete old test database
+if (fs.existsSync(config.db)) fs.unlinkSync(config.db);
+if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB);
+
+var createServer = require('./src/app.js');
+var createMockServer = require('./test/mocks.js');
+const logger = require('./src/utils/logger.js');
+
+// Instantiate a Mocha instance.
+var mocha = new Mocha();
+
+var testDir = './test/cases'
+
+// Add each .js file to the mocha instance
+fs.readdirSync(testDir).filter(function(file) {
+    // Only keep the .js files
+    return file.substr(-3) === '.js';
+
+}).forEach(function(file) {
+    mocha.addFile(
+        path.join(testDir, file)
+    );
+});
+
+var mockServer = createMockServer(() => {
+  logger.info("Started mock HTTP Server");
+  var server = createServer(() => {
+    logger.info("Started main HTTP server");
+    // Run the tests.
+    mocha.run((failures) => {
+      mockServer.close();
+      server.close();
+      process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures
+    });
+  });
+});

test.json

@@ -0,0 +1,68 @@
+{
+    "port": 8080,
+    "mockPort": 8081,
+    "globalSalt": "testSalt",
+    "adminUserID": "testUserId",
+    "youtubeAPIKey": "",
+    "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
+    "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
+    "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",
+    "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
+    "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
+    "behindProxy": true,
+    "db": "./test/databases/sponsorTimes.db",
+    "privateDB": "./test/databases/private.db",
+    "createDatabaseIfNotExist": true,
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    "mode": "test",
+    "readOnly": false,
+    "webhooks": [
+        {
+            "url": "http://127.0.0.1:8081/CustomWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8081/FailedWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8099/WrongPort",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://unresolvable.host:8081/FailedWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }
+    ],
+    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
+    "maxNumberOfActiveWarnings": 3,
+    "hoursAfterWarningExpires": 24,
+    "rateLimit": {
+        "vote": {
+          "windowMs": 900000,
+          "max": 20,
+          "message": "Too many votes, please try again later",
+          "statusCode": 429
+        },
+        "view": {
+          "windowMs": 900000,
+          "max": 20,
+          "statusCode": 200
+        }
+    }
+}

test/cases/dbUpgrade.js

@@ -0,0 +1,12 @@
+const databases = require('../../src/databases/databases.js');
+const db = databases.db;
+const privateDB = databases.privateDB;
+
+describe('dbUpgrade', () => {
+  it('Should update the database version when starting the application', (done) => {
+    let dbVersion = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    let privateVersion = privateDB.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    if (dbVersion >= 1 && privateVersion >= 1) done();
+    else done('Versions are not at least 1. db is ' + dbVersion + ', private is ' + privateVersion);
+  });
+});

test/cases/getHash.js

@@ -0,0 +1,33 @@
+var getHash = require('../../src/utils/getHash.js');
+
+var assert = require('assert');
+describe('getHash', () => {
+  it('Should not output the input string', () => {
+    assert(getHash("test") !== "test");
+    assert(getHash("test", -1) !== "test");
+    assert(getHash("test", 0) !== "test");
+    assert(getHash("test", null) !== "test");
+  });
+
+  it('Should return a hashed value', () => {
+    assert.equal(getHash("test"), "2f327ef967ade1ebf4319163f7debbda9cc17bb0c8c834b00b30ca1cf1c256ee");
+  });
+
+  it ('Should be able to output the same has the DB upgrade script will output', () => {
+    assert.equal(getHash("vid", 1), "1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663");
+  });
+
+  it ('Should take a variable number of passes', () => {
+    assert.equal(getHash("test", 1), "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08");
+    assert.equal(getHash("test", 2), "7b3d979ca8330a94fa7e9e1b466d8b99e0bcdea1ec90596c0dcc8d7ef6b4300c");
+    assert.equal(getHash("test", 3), "5b24f7aa99f1e1da5698a4f91ae0f4b45651a1b625c61ed669dd25ff5b937972");
+  });
+
+  it ('Should default to 5000 passes', () => {
+    assert.equal(getHash("test"), getHash("test", 5000));
+  });
+
+  it ('Should not take a negative number of passes', () => {
+    assert.equal(getHash("test", -1), "");
+  });
+});

test/cases/getIsUserVIP.js

@@ -0,0 +1,57 @@
+var request = require('request');
+var utils = require('../utils.js');
+var db = require('../../src/databases/databases.js').db;
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getIsUserVIP', () => {
+  before(() => {
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("supertestman") + "')");
+  }); 
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP?userID=supertestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+
+  it('Should get a 400 if no userID', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+  it('Should say a VIP is a VIP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP?userID=supertestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else {
+          if (JSON.parse(body).vip === true) done(); // pass
+          else done("Result was non-vip when should have been vip");
+        }
+      });
+  });
+
+  it('Should say a normal user is not a VIP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/isUserVIP?userID=regulartestman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200: " + res.statusCode);
+        else {
+          if (JSON.parse(body).vip === false) done(); // pass
+          else done("Result was vip when should have been non-vip");
+        }
+      });
+  });
+});

test/cases/getSavedTimeForUser.js

@@ -0,0 +1,21 @@
+var request = require('request');
+var utils = require('../utils.js');
+var db = require('../../src/databases/databases.js').db;
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getSavedTimeForUser', () => {
+  before(() => {
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0, '" + getHash('getSavedTimeForUser', 0) + "')");
+  }); 
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getSavedTimeForUser?userID=testman", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200");
+        else done(); // pass
+      });
+  });
+});

test/cases/getSegmentsByHash.js

@@ -0,0 +1,190 @@
+var request = require('request');
+var db = require('../../src/databases/databases.js').db;
+var utils = require('../utils.js');
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getSegmentsByHash', () => {
+  before(() => {
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    db.exec(startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+    db.exec(startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+    db.exec(startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+    db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+  }); 
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          done();
+        } // pass
+      });
+  });
+
+  it('Should be able to get a 200 with empty segments for video but no matching categories', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/3272f?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          if (JSON.parse(body) && JSON.parse(body).length > 0 && JSON.parse(body)[0].segments.length === 0) {
+            done(); // pass
+          } else {
+            done("response had segments");
+          }
+        }
+      });
+  });
+
+  it('Should be able to get an empty array if no videos', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/11111?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode);
+        else {
+          if (JSON.parse(body).length === 0 && body === '[]') done(); // pass
+          else done("non empty array returned");
+        }
+      });
+  });
+
+  it('Should return 400 prefix too short', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/11?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 400 prefix too long', (done) => {
+    let prefix = new Array(50).join('1');
+    if (prefix.length <= 32) { // default value, config can change this
+      done('failed to generate a long enough string for the test ' + prefix.length);
+      return;
+    }
+
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/'+prefix+'?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should not return 400 prefix in range', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/11111?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 400) done("prefix length 5 gave 400 " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 404 for no hash', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/?categories=["shilling"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 404) done("expected 404, got " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should return 500 for bad format categories', (done) => { // should probably be 400
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/?categories=shilling', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 500) done("expected 500 got " + res.statusCode);
+        else {
+          done(); // pass
+        }
+      });
+  });
+
+  it('Should be able to get multiple videos', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          body = JSON.parse(body);
+          if (body.length !== 2) done("expected 2 video, got " + body.length);
+          else if (body[0].segments.length !== 2) done("expected 2 segments for first video, got " + body[0].segments.length);
+          else if (body[1].segments.length !== 1) done("expected 1 segment for second video, got " + body[1].segments.length);
+          else done();
+        }
+      });
+  });
+
+  it('Should be able to get 200 for no categories (default sponsor)', (done) => {
+    request.get(utils.getbaseURL() 
+     + '/api/skipSegments/fdaf', null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else {
+          body = JSON.parse(body);
+          if (body.length !== 2) done("expected 2 videos, got " + body.length);
+          else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
+          else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
+          else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+          else done();
+        }
+      });
+  });
+
+  it('Should be able to post a segment and get it using endpoint', (done) => {
+    let testID = 'abc123goodVideo';
+    request.post(utils.getbaseURL() 
+      + "/api/postVideoSponsorTimes", {
+        json: {
+          userID: "test",
+          videoID: testID,
+          segments: [{
+            segment: [13, 17],
+            category: "sponsor"
+          }]
+        }
+      }, 
+      (err, res, body) => {
+        if (err) done('(post) ' + err);
+        else if (res.statusCode === 200) {
+          request.get(utils.getbaseURL() 
+            + '/api/skipSegments/'+getHash(testID, 1).substring(0,3), null, 
+              (err, res, body) => {
+                if (err) done("(get) Couldn't call endpoint");
+                else if (res.statusCode !== 200) done("(get) non 200 status code, was " + res.statusCode);
+                else {
+                  body = JSON.parse(body);
+                  if (body.length !== 1) done("(get) expected 1 video, got " + body.length);
+                  else if (body[0].segments.length !== 1) done("(get) expected 1 segments for first video, got " + body[0].segments.length);
+                  else if (body[0].segments[0].category !== 'sponsor') done("(get) segment should be sponsor, was "+body[0].segments[0].category);
+                  else done();
+                }
+              });
+        } else {
+          done("(post) non 200 status code, was " + res.statusCode);
+        }
+      }
+    );
+  });
+});

test/cases/getSkipSegments.js

@@ -0,0 +1,217 @@
+var request = require('request');
+var db = require('../../src/databases/databases.js').db;
+var utils = require('../utils.js');
+var getHash = require('../../src/utils/getHash.js');
+
+
+describe('getSkipSegments', () => {
+  before(() => {
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
+    db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
+    db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
+    db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
+    db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
+    db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
+    db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
+  }); 
+  
+
+  it('Should be able to get a time by category 1', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/skipSegments?videoID=testtesttest&category=sponsor", null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    done();
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should be able to get a time by category 2', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/skipSegments?videoID=testtesttest&category=intro", null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
+                && data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
+                    done();
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should be able to get a time by categories array', (done) => {
+    request.get(utils.getbaseURL() 
+    + "/api/skipSegments?videoID=testtesttest&categories=[\"sponsor\"]", null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    done();
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should be able to get a time by categories array 2', (done) => {
+    request.get(utils.getbaseURL() 
+    + "/api/skipSegments?videoID=testtesttest&categories=[\"intro\"]", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
+                && data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
+                    done();
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should be able to get multiple times by category', (done) => {
+    request.get(utils.getbaseURL() 
+    + "/api/skipSegments?videoID=multiple&categories=[\"intro\"]", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 2) {
+
+                let success = true;
+                for (const segment of data) {
+                    if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
+                        || segment.category !== "intro" || segment.UUID !== "1-uuid-7") &&
+                        (segment.segment[0] !== 1 || segment.segment[1] !== 11
+                            || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) {
+                        success = false;
+                        break;
+                    }
+                }
+
+                if (success) done();
+                else done("Received incorrect body: " + res.body);
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should be able to get multiple times by multiple categories', (done) => {
+    request.get(utils.getbaseURL() 
+      + "/api/skipSegments?videoID=testtesttest&categories=[\"sponsor\", \"intro\"]", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 2) {
+
+                let success = true;
+                for (const segment of data) {
+                    if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
+                        || segment.category !== "intro" || segment.UUID !== "1-uuid-2") &&
+                        (segment.segment[0] !== 1 || segment.segment[1] !== 11
+                            || segment.category !== "sponsor" || segment.UUID !== "1-uuid-0")) {
+                        success = false;
+                        break;
+                    }
+                }
+
+                if (success) done();
+                else done("Received incorrect body: " + res.body);
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should be possible to send unexpected query parameters', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/skipSegments?videoID=testtesttest&fakeparam=hello&category=sponsor", null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    done();
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Low voted submissions should be hidden', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/skipSegments?videoID=test3&category=sponsor", null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+        else {
+            let data = JSON.parse(res.body);
+            if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                && data[0].category === "sponsor" && data[0].UUID === "1-uuid-4") {
+                    done();
+            } else {
+                done("Received incorrect body: " + res.body);
+            }
+        }
+      });
+  });
+
+  it('Should return 404 if no segment found', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/skipSegments?videoID=notarealvideo", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 404) done("non 404 respone code: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+  
+
+  it('Should be able send a comma in a query param', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/skipSegments?videoID=testtesttest,test&category=sponsor", null, 
+      (err, res, body) => {
+            if (err) done("Couldn't call endpoint");
+            else if (res.statusCode !== 200) done("Status code was: " + res.statusCode);
+            else {
+                let data = JSON.parse(res.body);
+                if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-1") {
+                        done();
+                } else {
+                    done("Received incorrect body: " + res.body);
+                }
+            }
+      });
+  });
+
+});

test/cases/getSubmissionUUID.js

@@ -0,0 +1,8 @@
+const getSubmissionUUID = require('../../src/utils/getSubmissionUUID.js');
+const assert = require('assert');
+
+describe('getSubmissionUUID', () => {
+  it('Should return the hashed value', () => {
+    assert.equal(getSubmissionUUID('video001', 'sponsor', 'testuser001', 13.33337, 42.000001), '1d33d7016aa6482849019bd906d75c08fe6b815e64e823146df35f66c35612dd');
+  });
+});

test/cases/getUserInfo.js

@@ -0,0 +1,167 @@
+var request = require('request');
+var utils = require('../utils.js');
+var db = require('../../src/databases/databases.js').db;
+var getHash = require('../../src/utils/getHash.js');
+
+describe('getUserInfo', () => {
+  before(() => {
+    let startOfUserNamesQuery = "INSERT INTO userNames (userID, userName) VALUES";
+    db.exec(startOfUserNamesQuery + "('" + getHash("getuserinfo_user_01") + "', 'Username user 01')");
+    let startOfSponsorTimesQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES";
+    db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000001', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)");
+    db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000002', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)");
+    db.exec(startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -1, 'uuid000003', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)");
+    db.exec(startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -2, 'uuid000004', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)");
+    db.exec(startOfSponsorTimesQuery + "('xzzzxxyyy', 1, 11, -5, 'uuid000005', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)");
+    db.exec(startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 0)");
+    db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)");
+    db.exec(startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)");
+
+
+    db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_0') + "', 10, 'getuserinfo_vip')");
+    db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_1') + "', 10, 'getuserinfo_vip')");
+    db.exec("INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES ('" + getHash('getuserinfo_warning_1') + "', 10, 'getuserinfo_vip')");
+  });
+
+  it('Should be able to get a 200', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo?userID=getuserinfo_user_01', null,
+      (err, res, body) => {
+        if (err) {
+          done('couldn\'t call endpoint');
+        } else {
+          if (res.statusCode !== 200) {
+            done('non 200 (' + res.statusCode + ')');
+          } else {
+            done(); // pass
+          }
+        }
+      });
+  });
+
+  it('Should be able to get a 400 (No userID parameter)', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo', null,
+      (err, res, body) => {
+        if (err) {
+          done('couldn\'t call endpoint');
+        } else {
+          if (res.statusCode !== 400) {
+            done('non 400');
+          } else {
+            done(); // pass
+          }
+        }
+      });
+  });
+
+  it('Should return info', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo?userID=getuserinfo_user_01', null,
+      (err, res, body) => {
+        if (err) {
+          done("couldn't call endpoint");
+        } else {
+          if (res.statusCode !== 200) {
+            done("non 200");
+          } else {
+            const data = JSON.parse(body);
+            if (data.userName !== 'Username user 01') {
+              done('Returned incorrect userName "' + data.userName + '"');
+            } else if (data.minutesSaved !== 5) {
+              done('Returned incorrect minutesSaved "' + data.minutesSaved + '"');
+            } else if (data.viewCount !== 30) {
+              done('Returned incorrect viewCount "' + data.viewCount + '"');
+            } else if (data.segmentCount !== 3) {
+              done('Returned incorrect segmentCount "' + data.segmentCount + '"');
+            }  else {
+              done(); // pass
+            }
+          }
+        }
+      });
+  });
+
+  it('Should get warning data', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo?userID=getuserinfo_warning_0', null,
+      (err, res, body) => {
+        if (err) {
+          done("couldn't call endpoint");
+        } else {
+          if (res.statusCode !== 200) {
+            done("non 200");
+          } else {
+            const data = JSON.parse(body);
+            if (data.warnings !== 1) {
+              done('wrong number of warnings: ' + data.warnings + ', not ' + 1);
+            } else {
+              done(); // pass
+            }
+          }
+        }
+      });
+  });
+
+  it('Should get multiple warnings', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo?userID=getuserinfo_warning_1', null,
+      (err, res, body) => {
+        if (err) {
+          done("couldn't call endpoint");
+        } else {
+          if (res.statusCode !== 200) {
+            done("non 200");
+          } else {
+            const data = JSON.parse(body);
+            if (data.warnings !== 2) {
+              done('wrong number of warnings: ' + data.warnings + ', not ' + 2);
+            } else {
+              done(); // pass
+            }
+          }
+        }
+      });
+  });
+
+  it('Should not get warnings if noe', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo?userID=getuserinfo_warning_2', null,
+      (err, res, body) => {
+        if (err) {
+          done("couldn't call endpoint");
+        } else {
+          if (res.statusCode !== 200) {
+            done("non 200");
+          } else {
+            const data = JSON.parse(body);
+            if (data.warnings !== 0) {
+              done('wrong number of warnings: ' + data.warnings + ', not ' + 0);
+            } else {
+              done(); // pass
+            }
+          }
+        }
+      });
+  });
+
+  it('Should return userID for userName (No userName set)', (done) => {
+    request.get(utils.getbaseURL()
+     + '/api/getUserInfo?userID=getuserinfo_user_02', null,
+      (err, res, body) => {
+        if (err) {
+          done('couldn\'t call endpoint');
+        } else {
+          if (res.statusCode !== 200) {
+            done('non 200');
+          } else {
+            const data = JSON.parse(body);
+            if (data.userName !== 'c2a28fd225e88f74945794ae85aef96001d4a1aaa1022c656f0dd48ac0a3ea0f') {
+              return done('Did not return userID for userName');
+            }
+            done(); // pass
+          }
+        }
+      });
+  });
+});

test/cases/noSegmentRecords.js

@@ -0,0 +1,466 @@
+var request = require('request');
+
+var utils = require('../utils.js');
+const getHash = require('../../src/utils/getHash.js');
+
+var databases = require('../../src/databases/databases.js');
+const logger = require('../../src/utils/logger.js');
+var db = databases.db;
+
+describe('noSegmentRecords', () => {
+  before(() => {
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'delete-record', 'sponsor')");
+
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'sponsor')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'intro')");
+  });
+
+  it('Should update the database version when starting the application', (done) => {
+    let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value;
+    if (version > 1) done();
+    else done('Version isn\'t greater than 1. Version is ' + version);
+  });
+
+  it('Should be able to submit categories not in video (http response)', (done) => {
+    let json = {
+      videoID: 'no-segments-video-id',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'outro',
+        'shilling',
+        'shilling',
+        'shil ling',
+        '',
+        'intro'
+      ]
+    };
+
+    let expected = {
+      submitted: [
+        'outro',
+        'shilling'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          if (JSON.stringify(body) === JSON.stringify(expected)) {
+            done();
+          } else {
+            done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(body));
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categories not in video (sql check)', (done) => {
+    let json = {
+      videoID: 'no-segments-video-id-1',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'outro',
+        'shilling',
+        'shilling',
+        'shil ling',
+        '',
+        'intro'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['no-segments-video-id-1']);
+          if (result.length !== 4) {
+            console.log(result);
+            done("Expected 4 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categories with _ in the category', (done) => {
+    let json = {
+      videoID: 'underscore',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'word_word',
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['underscore']);
+          if (result.length !== 1) {
+            console.log(result);
+            done("Expected 1 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit categories with upper and lower case in the category', (done) => {
+    let json = {
+      videoID: 'bothCases',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'wordWord',
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['bothCases']);
+          if (result.length !== 1) {
+            console.log(result);
+            done("Expected 1 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should not be able to submit categories with $ in the category', (done) => {
+    let json = {
+      videoID: 'specialChar',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'word&word',
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['specialChar']);
+          if (result.length !== 0) {
+            console.log(result);
+            done("Expected 0 entrys in db, got " + result.length);
+          } else {
+            done();
+          }
+        } else {
+          console.log(body);
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for missing params', (done) => {
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json: {}}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no categories', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: []
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no userID', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: null,
+      categories: ['sponsor']
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for no videoID', (done) => {
+    let json = {
+      videoID: null,
+      userID: 'test',
+      categories: ['sponsor']
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 object categories)', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: {}
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 bad format categories', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: 'sponsor'
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done()
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+  
+  it('Should return 403 if user is not VIP', (done) => {
+    let json = {
+      videoID: 'test',
+      userID: 'test',
+      categories: [
+        'sponsor'
+      ]
+    };
+
+    request.post(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 403) {
+          done();
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to delete a noSegment record', (done) => {
+    let json = {
+      videoID: 'delete-record',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'sponsor'
+      ]
+    };
+
+    request.delete(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['delete-record']);
+          if (result.length === 0) {
+            done();
+          } else {
+            done("Didn't delete record");
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to delete one noSegment record without removing another', (done) => {
+    let json = {
+      videoID: 'delete-record-1',
+      userID: 'VIPUser-noSegments',
+      categories: [
+        'sponsor'
+      ]
+    };
+
+    request.delete(utils.getbaseURL() 
+     + "/api/noSegments", {json}, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['delete-record-1']);
+          if (result.length === 1) {
+            done();
+          } else {
+            done("Didn't delete record");
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+
+  /*
+   * Submission tests in this file do not check database records, only status codes.
+   * To test the submission code properly see ./test/cases/postSkipSegments.js
+   */
+
+  it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "sponsor"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 403) {
+        done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+  it('Should not be able to submit segments to a video where any of the submissions with a no-segment record', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "sponsor"
+         },{
+          segment: [50, 60],
+          category: "intro"
+        }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 403) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+
+  it('Should  be able to submit a segment to a video with a different no-segment record', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "noSubmitVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "intro"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 200) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+
+  it('Should be able to submit a segment to a video with no no-segment records', (done) => {
+    request.post(utils.getbaseURL() 
+    + "/api/postVideoSponsorTimes", {
+      json: {
+         userID: "testman42",
+         videoID: "normalVideo",
+         segments: [{
+           segment: [20, 40],
+           category: "intro"
+         }]
+      }
+    }, 
+     (err, res, body) => {
+       if (err) done(err);
+       else if (res.statusCode === 200) {
+           done()
+       } else {
+         done("Status code was " + res.statusCode);
+       }
+     });
+  });
+});

test/cases/oldGetSponsorTime.js

@@ -0,0 +1,88 @@
+var request = require('request');
+var db = require('../../src/databases/databases.js').db;
+var utils = require('../utils.js');
+var getHash = require('../../src/utils/getHash.js');
+
+
+/*
+ *CREATE TABLE IF NOT EXISTS "sponsorTimes" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"shadowHidden"	INTEGER NOT NULL
+);
+ */
+
+describe('getVideoSponsorTime (Old get method)', () => {
+  before(() => {
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 1) + "')");
+    db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 1) + "')");
+  }); 
+
+  it('Should be able to get a time', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getVideoSponsorTimes?videoID=old-testtesttest", null, 
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200");
+        else done(); // pass
+      });
+  });
+
+  it('Should return 404 if no segment found', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getVideoSponsorTimes?videoID=notarealvideo", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 404) done("non 404 respone code: " + res.statusCode);
+        else done(); // pass
+      });
+  });
+
+
+  it('Should be possible to send unexpected query parameters', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getVideoSponsorTimes?videoID=old-testtesttest&fakeparam=hello", null, 
+      (err, res, body) => {
+        if (err) done("couldn't callendpoint");
+        else if (res.statusCode !== 200) done("non 200");
+        else done(); // pass
+      });
+  });
+
+  it('Should be able send a comma in a query param', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getVideoSponsorTimes?videoID=old-testtesttest,test", null, 
+      (err, res, body) => {
+        if (err) done("couln't call endpoint");
+        else if (res.statusCode !== 200) done("non 200 response: " + res.statusCode);
+        else if (JSON.parse(body).UUIDs[0] === 'uuid-1') done(); // pass
+        else done("couldn't parse response");
+      });
+  });
+
+  it('Should be able to get the correct time', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/getVideoSponsorTimes?videoID=old-testtesttest", null, 
+      (err, res, body) => {
+        if (err) done("couldn't call endpoint");
+        else if (res.statusCode !== 200) done("non 200");
+        else {
+          let parsedBody = JSON.parse(body);
+          if (parsedBody.sponsorTimes[0][0] === 1
+            && parsedBody.sponsorTimes[0][1] === 11
+            && parsedBody.UUIDs[0] === 'uuid-0') {
+            done(); // pass
+          } else {
+            done("Wrong data was returned + " + parsedBody);
+          }
+        };
+      });
+  });
+});

test/cases/oldSubmitSponsorTimes.js

@@ -0,0 +1,55 @@
+var assert = require('assert');
+var request = require('request');
+
+var utils = require('../utils.js');
+
+var databases = require('../../src/databases/databases.js');
+var db = databases.db;
+
+describe('postVideoSponsorTime (Old submission method)', () => {
+  it('Should be able to submit a time (GET)', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcQ&startTime=1&endTime=10&userID=test", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcQ"]);
+          if (row.startTime === 1 && row.endTime === 10 && row.category === "sponsor") {
+            done()
+          } else {
+            done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit a time (POST)', (done) => {
+    request.post(utils.getbaseURL() 
+     + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcE&startTime=1&endTime=11&userID=test", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcE"]);
+          if (row.startTime === 1 && row.endTime === 11 && row.category === "sponsor") {
+            done()
+          } else {
+            done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should return 400 for missing params', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/postVideoSponsorTimes?startTime=1&endTime=10&userID=test", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        if (res.statusCode === 400) done();
+        else done("Status code was: " + res.statusCode);
+      });
+  });
+});

test/cases/postSkipSegments.js

@@ -0,0 +1,455 @@
+var assert = require('assert');
+var request = require('request');
+var config = require('../../src/config.js');
+var getHash = require('../../src/utils/getHash.js');
+
+var utils = require('../utils.js');
+
+var databases = require('../../src/databases/databases.js');
+var db = databases.db;
+
+describe('postSkipSegments', () => {
+  before(() => {
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    
+    db.exec(startOfQuery + "('80percent_video', 0, 1000, 0, '80percent-uuid-0', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')");
+    db.exec(startOfQuery + "('80percent_video', 1001, 1005, 0, '80percent-uuid-1', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')");
+    db.exec(startOfQuery + "('80percent_video', 0, 5000, -2, '80percent-uuid-2', '" + getHash("test") + "', 0, 0, 'interaction', 0, '80percent_video')");
+
+    const now = Date.now();
+    const warnVip01Hash = getHash("warn-vip01");
+    const warnUser01Hash = getHash("warn-user01");
+    const warnUser02Hash = getHash("warn-user02");
+    const MILLISECONDS_IN_HOUR = 3600000;
+    const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires;
+    const startOfWarningQuery = 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES';
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + now + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-1000) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-2000) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-3601000) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 1000)) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 2000)) + "', '" + warnVip01Hash + "')");
+  });
+  
+  it('Should be able to submit a single time (Params method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes?videoID=dQw4w9WgXcR&startTime=2&endTime=10&userID=test&category=sponsor", null,
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcR"]);
+          if (row.startTime === 2 && row.endTime === 10 && row.category === "sponsor") {
+            done()
+          } else {
+            done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit a single time (JSON method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "dQw4w9WgXcF",
+          segments: [{
+            segment: [0, 10],
+            category: "sponsor"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcF"]);
+          if (row.startTime === 0 && row.endTime === 10 && row.category === "sponsor") {
+            done()
+          } else {
+            done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be able to submit multiple times (JSON method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "dQw4w9WgXcQ",
+          segments: [{
+            segment: [3, 10],
+            category: "sponsor"
+          }, {
+            segment: [30, 60],
+            category: "intro"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcR"]);
+          let success = true;
+          if (rows.length === 2) {
+            for (const row of rows) {
+              if ((row.startTime !== 3 || row.endTime !== 10 || row.category !== "sponsor") &&
+                  (row.startTime !== 30 || row.endTime !== 60 || row.category !== "intro")) {
+                success = false;
+                break;
+              }
+            }
+          }
+
+          if (success) done();
+          else done("Submitted times were not saved. Actual submissions: " + JSON.stringify(row));
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  }).timeout(5000);
+
+  it('Should allow multiple times if total is under 80% of video(JSON method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "L_jWHffIx5E",
+          segments: [{
+            segment: [3, 3000],
+            category: "sponsor"
+          },{
+            segment: [3002, 3050],
+            category: "intro"
+          },{
+            segment: [45, 100],
+            category: "interaction"
+          },{
+            segment: [99, 170],
+            category: "sponsor"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["L_jWHffIx5E"]);
+          let success = true;
+          if (rows.length === 4) {
+            for (const row of rows) {
+              if ((row.startTime !== 3 || row.endTime !== 3000 || row.category !== "sponsor") &&
+                  (row.startTime !== 3002 || row.endTime !== 3050 || row.category !== "intro") &&
+                  (row.startTime !== 45 || row.endTime !== 100 || row.category !== "interaction") &&
+                  (row.startTime !== 99 || row.endTime !== 170 || row.category !== "sponsor")) {
+                success = false;
+                break;
+              }
+            }
+          }
+
+          if (success) done();
+          else done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows));
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  }).timeout(5000);
+
+  it('Should reject multiple times if total is over 80% of video (JSON method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "n9rIGdXnSJc",
+          segments: [{
+            segment: [0, 2000],
+            category: "interaction"
+          },{
+            segment: [3000, 4000],
+            category: "sponsor"
+          },{
+            segment: [1500, 2750],
+            category: "sponsor"
+          },{
+            segment: [4050, 4750],
+            category: "intro"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["n9rIGdXnSJc"]);
+          let success = true;
+          if (rows.length === 4) {
+            for (const row of rows) {
+              if ((row.startTime === 0 || row.endTime === 2000 || row.category === "interaction") ||
+                  (row.startTime === 3000 || row.endTime === 4000 || row.category === "sponsor") ||
+                  (row.startTime === 1500 || row.endTime === 2750 || row.category === "sponsor") ||
+                  (row.startTime === 4050 || row.endTime === 4750 || row.category === "intro")) {
+                success = false;
+                break;
+              }
+            }
+          }
+
+          if (success) done();
+          else 
+          done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows));
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  }).timeout(5000);
+
+  it('Should reject multiple times if total is over 80% of video including previosuly submitted times(JSON method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "80percent_video",
+          segments: [{
+            segment: [2000, 4000],
+            category: "sponsor"
+          },{
+            segment: [1500, 2750],
+            category: "sponsor"
+          },{
+            segment: [4050, 4750],
+            category: "sponsor"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          let rows = db.prepare('all', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ? and votes > -1", ["80percent_video"]);
+          let success = true && rows.length == 2;
+          for (const row of rows) {
+            if ((row.startTime === 2000 || row.endTime === 4000 || row.category === "sponsor") ||
+                (row.startTime === 1500 || row.endTime === 2750 || row.category === "sponsor") ||
+                (row.startTime === 4050 || row.endTime === 4750 || row.category === "sponsor")) {
+              success = false;
+              break;
+            }
+          }
+          if (success) done();
+          else 
+          done("Submitted times were not saved. Actual submissions: " + JSON.stringify(rows));
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  }).timeout(5000);
+
+  it('Should be accepted if a non-sponsor is less than 1 second', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=intro", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 200) done(); // pass
+        else done("non 200 status code: " + res.statusCode + " ("+body+")");
+      });
+  });
+
+  it('Should be rejected if a sponsor is less than 1 second', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 400) done(); // pass
+        else done("non 403 status code: " + res.statusCode + " ("+body+")");
+      });
+  });
+
+  it('Should be rejected if over 80% of the video', (done) => {
+    request.get(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes?videoID=qqwerty&startTime=30&endTime=1000000&userID=testing", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 403) done(); // pass
+        else done("non 403 status code: " + res.statusCode + " ("+body+")");
+      });
+  });
+
+  it("Should be rejected if NB's predicted probability is <70%.", (done) => {
+    request.get(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes?videoID=LevkAjUE6d4&startTime=40&endTime=60&userID=testing", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 200) done(); // pass
+        else done("non 200 status code: " + res.statusCode + " ("+body+")");
+      });
+  });
+
+  it('Should be rejected if user has to many active warnings', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "warn-user01",
+          videoID: "dQw4w9WgXcF",
+          segments: [{
+            segment: [0, 10],
+            category: "sponsor"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 403) {
+          done(); // success
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+      });
+  });
+
+  it('Should be accepted if user has some active warnings', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "warn-user02",
+          videoID: "dQw4w9WgXcF",
+          segments: [{
+            segment: [50, 60],
+            category: "sponsor"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          done(); // success
+        } else {
+          done("Status code was " + res.statusCode + " " + body);
+        }
+      });
+  });
+
+  it('Should be allowed if youtube thinks duration is 0', (done) => {
+    request.get(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 200) done(); // pass
+        else done("non 200 status code: " + res.statusCode + " ("+body+")");
+      });
+  });
+
+  it('Should be rejected if not a valid videoID', (done) => {
+    request.get(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing", null,
+      (err, res, body) => {
+        if (err) done("Couldn't call endpoint");
+        else if (res.statusCode === 403) done(); // pass
+        else done("non 403 status code: " + res.statusCode + " ("+body+")");
+      });
+  });
+
+  it('Should return 400 for missing params (Params method)', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", null,
+      (err, res, body) => {
+        if (err) done(true);
+        if (res.statusCode === 400) done();
+        else done(true);
+      });
+  });
+
+  it('Should return 400 for missing params (JSON method) 1', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          segments: [{
+            segment: [9, 10],
+            category: "sponsor"
+          }, {
+            segment: [31, 60],
+            category: "intro"
+          }]
+       }
+     },
+      (err, res, body) => {
+        if (err) done(true);
+        else if (res.statusCode === 400) done();
+        else done(true);
+      });
+  });
+  it('Should return 400 for missing params (JSON method) 2', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+        userID: "test",
+        videoID: "dQw4w9WgXcQ"
+       }
+      },
+      (err, res, body) => {
+        if (err) done(true);
+        else if (res.statusCode === 400) done();
+        else done(true);
+      });
+  });
+  it('Should return 400 for missing params (JSON method) 3', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "dQw4w9WgXcQ",
+          segments: [{
+            segment: [0],
+            category: "sponsor"
+          }, {
+            segment: [31, 60],
+            category: "intro"
+          }]
+       }
+      },
+      (err, res, body) => {
+        if (err) done(true);
+        else if (res.statusCode === 400) done();
+        else done(true);
+      });
+  });
+  it('Should return 400 for missing params (JSON method) 4', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "dQw4w9WgXcQ",
+          segments: [{
+            segment: [9, 10]
+          }, {
+            segment: [31, 60],
+            category: "intro"
+          }]
+       }
+      },
+      (err, res, body) => {
+        if (err) done(true);
+        else if (res.statusCode === 400) done();
+        else done(true);
+      });
+  });
+  it('Should return 400 for missing params (JSON method) 5', (done) => {
+    request.post(utils.getbaseURL()
+     + "/api/postVideoSponsorTimes", {
+       json: {
+          userID: "test",
+          videoID: "dQw4w9WgXcQ"
+       }
+      },
+      (err, res, body) => {
+        if (err) done(true);
+        else if (res.statusCode === 400) done();
+        else done(true);
+      });
+  });
+});

test/cases/postWarning.js

@@ -0,0 +1,47 @@
+var request = require('request');
+var utils = require('../utils.js');
+var db = require('../../src/databases/databases.js').db;
+var getHash = require('../../src/utils/getHash.js');
+
+describe('postWarning', () => {
+    before(() => {
+      db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("warning-vip") + "')");
+    });
+  
+    it('Should be able to create warning if vip (exp 200)', (done) => {
+        let json = {
+            issuerUserID: 'warning-vip',
+            userID: 'warning-0'
+        };
+
+        request.post(utils.getbaseURL() 
+          + "/api/warnUser", {json}, 
+          (err, res, body) => {
+            if (err) done(err);
+            else if (res.statusCode === 200) {
+                done();
+            } else {
+                console.log(body);
+                done("Status code was " + res.statusCode);
+            }
+        });
+    });
+    it('Should not be able to create warning if vip (exp 403)', (done) => {
+        let json = {
+            issuerUserID: 'warning-not-vip',
+            userID: 'warning-1'
+        };
+
+        request.post(utils.getbaseURL() 
+          + "/api/warnUser", {json}, 
+          (err, res, body) => {
+            if (err) done(err);
+            else if (res.statusCode === 403) {
+                done();
+            } else {
+                console.log(body);
+                done("Status code was " + res.statusCode);
+            }
+        });
+    });
+});

test/cases/segmentShift.js

@@ -0,0 +1,273 @@
+const request = require('request');
+const utils = require('../utils.js');
+const { db } = require('../../src/databases/databases.js');
+const getHash = require('../../src/utils/getHash.js');
+
+function dbSponsorTimesAdd(db, videoID, startTime, endTime, UUID, category) {
+  const votes = 0,
+    userID = 0,
+    timeSubmitted = 0,
+    views = 0,
+    shadowHidden = 0,
+    hashedVideoID = `hash_${UUID}`;
+  db.exec(`INSERT INTO
+    sponsorTimes (videoID, startTime, endTime, votes, UUID,
+    userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)
+  VALUES
+    ('${videoID}', ${startTime}, ${endTime}, ${votes}, '${UUID}',
+    '${userID}', ${timeSubmitted}, ${views}, '${category}', ${shadowHidden}, '${hashedVideoID}')
+  `);
+}
+
+function dbSponsorTimesSetByUUID(db, UUID, startTime, endTime) {
+  db.prepare('run', `UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?`, [startTime, endTime, UUID]);
+}
+
+function dbSponsorTimesCompareExpect(db, expect) {
+  for (let i=0, len=expect.length; i<len; i++) {
+    const expectSeg = expect[i];
+    let seg = db.prepare('get', "SELECT startTime, endTime FROM sponsorTimes WHERE UUID = ?", [expectSeg.UUID]);
+    if ('removed' in expect) {
+      if (expect.removed === true && seg.votes === -2) {
+        return;
+      }
+      else {
+        return `${expectSeg.UUID} doesnt got removed`;
+      }
+    }
+    if (seg.startTime !== expectSeg.startTime) {
+      return `${expectSeg.UUID} startTime is incorrect. seg.startTime is ${seg.startTime} expected ${expectSeg.startTime}`;
+    }
+    if (seg.endTime !== expectSeg.endTime) {
+      return `${expectSeg.UUID} endTime is incorrect. seg.endTime is ${seg.endTime} expected ${expectSeg.endTime}`;
+    }
+  }
+  return;
+}
+
+describe('segmentShift', function() {
+  const privateVipUserID = 'VIPUser-segmentShift';
+  const vipUserID = getHash(privateVipUserID);
+  const baseURL = utils.getbaseURL();
+  
+  before(function(done) {
+    // startTime and endTime get set in beforeEach for consistency
+    dbSponsorTimesAdd(db, 'vsegshift01', 0, 0, 'vsegshifttest01uuid01', 'intro');
+    dbSponsorTimesAdd(db, 'vsegshift01', 0, 0, 'vsegshifttest01uuid02', 'sponsor');
+    dbSponsorTimesAdd(db, 'vsegshift01', 0, 0, 'vsegshifttest01uuid03', 'interaction');
+    dbSponsorTimesAdd(db, 'vsegshift01', 0, 0, 'vsegshifttest01uuid04', 'outro');
+    db.exec(`INSERT INTO vipUsers (userID) VALUES ('${vipUserID}')`);
+    done();
+  });
+  
+  beforeEach(function(done) {
+    // resetting startTime and endTime to reuse them
+    dbSponsorTimesSetByUUID(db, 'vsegshifttest01uuid01',   0,  10);
+    dbSponsorTimesSetByUUID(db, 'vsegshifttest01uuid02',  60,  90);
+    dbSponsorTimesSetByUUID(db, 'vsegshifttest01uuid03',  40,  45);
+    dbSponsorTimesSetByUUID(db, 'vsegshifttest01uuid04', 120, 140);
+    done();
+  });
+  
+  it('Reject none VIP user', function(done) {
+    request.post(`${baseURL}/api/segmentShift`, {
+      json: {
+        videoID: 'vsegshift01',
+        userID: 'segshift_randomuser001',
+        startTime: 20,
+        endTime: 30,
+      }
+    }, (err, res, body) => {
+      if (err) return done(err);
+      return done(res.statusCode === 403 ? undefined : res.statusCode);
+    });
+  });
+  
+  it('Shift is outside segments', function(done) {
+    request.post(`${baseURL}/api/segmentShift`, {
+      json: {
+        videoID: 'vsegshift01',
+        userID: privateVipUserID,
+        startTime: 20,
+        endTime: 30,
+      }
+    }, (err, res, body) => {
+      if (err) return done(err);
+      if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`);
+      const expect = [
+        {
+          UUID: 'vsegshifttest01uuid01',
+          startTime: 0,
+          endTime: 10,
+        },
+        {
+          UUID: 'vsegshifttest01uuid02',
+          startTime: 50,
+          endTime: 80,
+        },
+        {
+          UUID: 'vsegshifttest01uuid03',
+          startTime: 30,
+          endTime: 35,
+        },
+        {
+          UUID: 'vsegshifttest01uuid04',
+          startTime: 110,
+          endTime: 130,
+        },
+      ];
+      done(dbSponsorTimesCompareExpect(db, expect));
+    });
+  });
+  
+  it('Shift is inside segment', function(done) {
+    request.post(`${baseURL}/api/segmentShift`, {
+      json: {
+        videoID: 'vsegshift01',
+        userID: privateVipUserID,
+        startTime: 65,
+        endTime: 75,
+      }
+    }, (err, res, body) => {
+      if (err) return done(err);
+      if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`);
+      const expect = [
+        {
+          UUID: 'vsegshifttest01uuid01',
+          startTime: 0,
+          endTime: 10,
+        },
+        {
+          UUID: 'vsegshifttest01uuid02',
+          startTime: 60,
+          endTime: 80,
+        },
+        {
+          UUID: 'vsegshifttest01uuid03',
+          startTime: 40,
+          endTime: 45,
+        },
+        {
+          UUID: 'vsegshifttest01uuid04',
+          startTime: 110,
+          endTime: 130,
+        },
+      ];
+      done(dbSponsorTimesCompareExpect(db, expect));
+    });
+  });
+  
+  it('Shift is overlaping startTime of segment', function(done) {
+    request.post(`${baseURL}/api/segmentShift`, {
+      json: {
+        videoID: 'vsegshift01',
+        userID: privateVipUserID,
+        startTime: 32,
+        endTime: 42,
+      }
+    }, (err, res, body) => {
+      if (err) return done(err);
+      if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`);
+      const expect = [
+        {
+          UUID: 'vsegshifttest01uuid01',
+          startTime: 0,
+          endTime: 10,
+        },
+        {
+          UUID: 'vsegshifttest01uuid02',
+          startTime: 50,
+          endTime: 80,
+        },
+        {
+          UUID: 'vsegshifttest01uuid03',
+          startTime: 32,
+          endTime: 35,
+        },
+        {
+          UUID: 'vsegshifttest01uuid04',
+          startTime: 110,
+          endTime: 130,
+        },
+      ];
+      done(dbSponsorTimesCompareExpect(db, expect));
+    });
+  });
+
+  it('Shift is overlaping endTime of segment', function(done) {
+    request.post(`${baseURL}/api/segmentShift`, {
+      json: {
+        videoID: 'vsegshift01',
+        userID: privateVipUserID,
+        startTime: 85,
+        endTime: 95,
+      }
+    }, (err, res, body) => {
+      if (err) return done(err);
+      if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`);
+      const expect = [
+        {
+          UUID: 'vsegshifttest01uuid01',
+          startTime: 0,
+          endTime: 10,
+        },
+        {
+          UUID: 'vsegshifttest01uuid02',
+          startTime: 60,
+          endTime: 85,
+        },
+        {
+          UUID: 'vsegshifttest01uuid03',
+          startTime: 40,
+          endTime: 45,
+        },
+        {
+          UUID: 'vsegshifttest01uuid04',
+          startTime: 110,
+          endTime: 130,
+        },
+      ];
+      done(dbSponsorTimesCompareExpect(db, expect));
+    });
+  });
+  
+  it('Shift is overlaping segment', function(done) {
+    request.post(`${baseURL}/api/segmentShift`, {
+      json: {
+        videoID: 'vsegshift01',
+        userID: privateVipUserID,
+        startTime: 35,
+        endTime: 55,
+      }
+    }, (err, res, body) => {
+      if (err) return done(err);
+      if (res.statusCode !== 200) return done(`Status code was ${res.statusCode}`);
+      const expect = [
+        {
+          UUID: 'vsegshifttest01uuid01',
+          startTime: 0,
+          endTime: 10,
+        },
+        {
+          UUID: 'vsegshifttest01uuid02',
+          startTime: 40,
+          endTime: 70,
+        },
+        {
+          UUID: 'vsegshifttest01uuid03',
+          startTime: 40,
+          endTime: 45,
+          removed: true,
+        },
+        {
+          UUID: 'vsegshifttest01uuid04',
+          startTime: 100,
+          endTime: 120,
+        },
+      ];
+      done(dbSponsorTimesCompareExpect(db, expect));
+    });
+  });
+  
+  
+});

test/cases/unBan.js

@@ -0,0 +1,79 @@
+var request = require('request');
+
+var utils = require('../utils.js');
+const getHash = require('../../src/utils/getHash.js');
+
+var databases = require('../../src/databases/databases.js');
+const logger = require('../../src/utils/logger.js');
+var db = databases.db;
+
+describe('unBan', () => {
+  before(() => {
+    db.exec("INSERT INTO shadowBannedUsers VALUES('testMan-unBan')");
+    db.exec("INSERT INTO shadowBannedUsers VALUES('testWoman-unBan')");
+    db.exec("INSERT INTO shadowBannedUsers VALUES('testEntity-unBan')");
+
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-unBan") + "')");
+    db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')");
+
+    let startOfInsertSegmentQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    db.exec(startOfInsertSegmentQuery + "('unBan-videoID-0', 1, 11, 2, 'unBan-uuid-0', 'testMan-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-0', 1) + "')");
+    db.exec(startOfInsertSegmentQuery + "('unBan-videoID-1', 1, 11, 2, 'unBan-uuid-1', 'testWoman-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-1', 1) + "')");
+    db.exec(startOfInsertSegmentQuery + "('unBan-videoID-1', 1, 11, 2, 'unBan-uuid-2', 'testEntity-unBan', 0, 60, 'sponsor', 1, '" + getHash('unBan-videoID-1', 1) + "')");
+    db.exec(startOfInsertSegmentQuery + "('unBan-videoID-2', 1, 11, 2, 'unBan-uuid-3', 'testEntity-unBan', 0, 60, 'sponsor', 1, '" + getHash('unBan-videoID-2', 1) + "')");
+  });
+
+  it('Should be able to unban a user and re-enable shadow banned segments', (done) => {
+    request.post(utils.getbaseURL() + "/api/shadowBanUser?userID=testMan-unBan&adminUserID=VIPUser-unBan&enabled=false", null, (err, res, body) => {
+      if (err) done(err);
+      else if (res.statusCode === 200) {
+        let result = db.prepare('all', 'SELECT * FROM sponsorTimes WHERE videoID = ? AND userID = ? AND shadowHidden = ?', ['unBan-videoID-0', 'testMan-unBan', 1]);
+        if (result.length !== 0) {
+          console.log(result);
+          done("Expected 0 banned entrys in db, got " + result.length);
+        } else {
+          done();
+        }
+      } else {
+        console.log(body);
+        done("Status code was " + res.statusCode);
+      }
+    });
+  });
+
+  it('Should be able to unban a user and re-enable shadow banned segments without noSegment entrys', (done) => {
+    request.post(utils.getbaseURL() + "/api/shadowBanUser?userID=testWoman-unBan&adminUserID=VIPUser-unBan&enabled=false", null, (err, res, body) => {
+      if (err) done(err);
+      else if (res.statusCode === 200) {
+        let result = db.prepare('all', 'SELECT * FROM sponsorTimes WHERE videoID = ? AND userID = ? AND shadowHidden = ?', ['unBan-videoID-1', 'testWoman-unBan', 1]);
+        if (result.length !== 1) {
+          console.log(result);
+          done("Expected 1 banned entry1 in db, got " + result.length);
+        } else {
+          done();
+        }
+      } else {
+        console.log(body);
+        done("Status code was " + res.statusCode);
+      }
+    });
+  }); 
+
+  it('Should be able to unban a user and re-enable shadow banned segments with a mix of noSegment entrys', (done) => {
+    request.post(utils.getbaseURL() + "/api/shadowBanUser?userID=testEntity-unBan&adminUserID=VIPUser-unBan&enabled=false", null, (err, res, body) => {
+      if (err) done(err);
+      else if (res.statusCode === 200) {
+        let result = db.prepare('all', 'SELECT * FROM sponsorTimes WHERE  userID = ? AND shadowHidden = ?', ['testEntity-unBan', 1]);
+        if (result.length !== 1) {
+          console.log(result);
+          done("Expected 1 banned entry1 in db, got " + result.length);
+        } else {
+          done();
+        }
+      } else {
+        console.log(body);
+        done("Status code was " + res.statusCode);
+      }
+    });
+  }); 
+});

test/cases/voteOnSponsorTime.js

@@ -0,0 +1,371 @@
+const request = require('request');
+const config = require('../../src/config.js');
+const { db, privateDB } = require('../../src/databases/databases.js');
+const utils = require('../utils.js');
+const getHash = require('../../src/utils/getHash.js');
+
+describe('voteOnSponsorTime', () => {
+  before(() => {
+    const now = Date.now();
+    const warnVip01Hash = getHash("warn-vip01");
+    const warnUser01Hash = getHash("warn-voteuser01");
+    const warnUser02Hash = getHash("warn-voteuser02");
+    const MILLISECONDS_IN_HOUR = 3600000;
+    const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires;
+    let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
+    const startOfWarningQuery = 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES';
+    
+    db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')");
+    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest2', 1) + "')");
+    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '" + getHash('vote-testtesttest2', 1) + "')");
+    db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '" + getHash('vote-testtesttest2', 1) + "')");
+    db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-testtesttest3', 1) + "')");
+    db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest,test', 1) + "')");
+    db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-test3', 1) + "')");
+    db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-test3', 1) + "')");
+    db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-multiple', 1) + "')");
+    db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-multiple', 1) + "')");
+    db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '" + getHash('voter-submitter', 1) + "')");
+    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '" + getHash('voter-submitter2', 1) + "')");
+    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '" + getHash('voter-submitter2', 1) + "')");
+    db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '" + getHash('voter-submitter2', 1) + "')");
+    db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '" + getHash('own-submission-video', 1) + "')");
+    db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('not-own-submission-video', 1) + "')");
+    db.exec(startOfQuery + "('incorrect-category', 1, 11, 500, 'incorrect-category', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category', 1) + "')");
+    db.exec(startOfQuery + "('incorrect-category-change', 1, 11, 500, 'incorrect-category-change', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '" + getHash('incorrect-category-change', 1) + "')");
+    db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'warnvote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 1) + "')");
+    
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + now + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-1000) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-2000) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser01Hash + "', '" + (now-3601000) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + now + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 1000)) + "', '" + warnVip01Hash + "')");
+    db.exec(startOfWarningQuery + "('" + warnUser02Hash + "', '" + (now-(warningExpireTime + 2000)) + "', '" + warnVip01Hash + "')");
+
+
+    db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
+    privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");
+  }); 
+
+  it('Should be able to upvote a segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID&UUID=vote-uuid-0&type=1", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-0"]);
+          if (row.votes === 3) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 2 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('Should be able to downvote a segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-2&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-2"]);
+          if (row.votes < 10) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 10 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('Should not be able to downvote the same segment when voting from a different user on the same IP', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID3&UUID=vote-uuid-2&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-2"]);
+          if (row.votes === 9) {
+            done()
+          } else {
+            done("Vote did not fail. Submission went from 9 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it("Should not be able to downvote a segment if the user is shadow banned", (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID4&UUID=vote-uuid-1.6&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-1.6"]);
+          if (row.votes === 10) {
+            done()
+          } else {
+            done("Vote did not fail. Submission went from 10 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it("Should not be able to upvote a segment if the user hasn't submitted yet", (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=hasNotSubmittedID&UUID=vote-uuid-1&type=1", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-1"]);
+          if (row.votes === 2) {
+            done()
+          } else {
+            done("Vote did not fail. Submission went from 2 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it("Should not be able to downvote a segment if the user hasn't submitted yet", (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=hasNotSubmittedID&UUID=vote-uuid-1.5&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-1.5"]);
+          if (row.votes === 10) {
+            done()
+          } else {
+            done("Vote did not fail. Submission went from 10 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('VIP should be able to completely downvote a segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-3&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-3"]);
+          if (row.votes <= -2) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 100 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('should be able to completely downvote your own segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=own-submission-id&UUID=own-submission-uuid&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["own-submission-uuid"]);
+          if (row.votes <= -2) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 500 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('should not be able to completely downvote somebody elses segment', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=not-own-submission-uuid&type=0", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["not-own-submission-uuid"]);
+          if (row.votes === 499) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from 500 votes to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  /* Raised the limit for votes - test update needed
+  
+  it('Should be able to vote for a category and it should immediately change (for now)', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=intro", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-4"]);
+          if (row.category === "intro") {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from sponsor to " + row.category);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });*/
+
+  it('Should not able to change to an invalid category', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=fakecategory", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category"]);
+          if (row.category === "sponsor") {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from sponsor to " + row.category);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  /*  Raised the limit for votes - test update needed
+  
+  it('Should be able to change your vote for a category and it should immediately change (for now)', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-4"]);
+          if (row.category === "outro") {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from intro to " + row.category);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });*/
+
+
+  it('Should not be able to change your vote to an invalid category', (done) => {
+    const vote = (inputCat, assertCat, callback) => {
+      request.get(utils.getbaseURL() 
+        + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category-change&category="+inputCat, null, 
+        (err) => {
+        if (err) done(err);
+        else{
+          let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category-change"]);
+          if (row.category === assertCat) {
+            callback();
+          } else {
+            done("Vote did not succeed. Submission went from sponsor to " + row.category);
+          }
+        }
+      });
+    };
+    vote("sponsor", "sponsor", () => {
+      vote("fakeCategory", "sponsor", done);
+    });
+  });
+
+
+  it('VIP should be able to vote for a category and it should immediately change', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&category=outro", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-5"]);
+          let row2 = db.prepare('get', "SELECT votes FROM categoryVotes WHERE UUID = ? and category = ?", ["vote-uuid-5", "outro"]);
+          if (row.category === "outro" && row2.votes === 500) {
+            done()
+          } else {
+            done("Vote did not succeed. Submission went from intro to " + row.category + ". Category votes are " + row2.votes + " and should be 500.");
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('Should not be able to category-vote on an invalid UUID submission', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID3&UUID=invalid-uuid&category=intro", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 400) {
+          done();
+        } else {
+          done("Status code was " + res.statusCode + " instead of 400.");
+        }
+    });
+  });
+
+  it('Non-VIP should not be able to upvote "dead" submission', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 403) {
+          done();
+        } else {
+          done("Status code was " + res.statusCode + " instead of 403");
+        }
+    });
+  });
+
+  it('VIP should be able to upvote "dead" submission', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&type=1", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 200) {
+          let row = db.prepare('get', "SELECT votes FROM sponsorTimes WHERE UUID = ?", ["vote-uuid-5"]);
+          if (row.votes > -3) {
+            done()
+          } else {
+            done("Vote did not succeed. Votes raised from -3 to " + row.votes);
+          }
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+  it('Should not be able to upvote a segment (Too many warning)', (done) => {
+    request.get(utils.getbaseURL() 
+     + "/api/voteOnSponsorTime?userID=warn-voteuser01&UUID=warnvote-uuid-0&type=1", null, 
+      (err, res, body) => {
+        if (err) done(err);
+        else if (res.statusCode === 403) {
+          done(); // success
+        } else {
+          done("Status code was " + res.statusCode);
+        }
+    });
+  });
+
+});

test/mocks.js

@@ -0,0 +1,51 @@
+var express = require('express');
+var app = express();
+
+var config = require('../src/config.js');
+
+app.post('/ReportChannelWebhook', (req, res) => {
+  res.sendStatus(200);
+});
+
+app.post('/FirstTimeSubmissionsWebhook', (req, res) => {
+  res.sendStatus(200);
+});
+
+app.post('/CompletelyIncorrectReportWebhook', (req, res) => {
+  res.sendStatus(200);
+});
+
+// Testing NeuralBlock
+app.post('/NeuralBlockRejectWebhook', (req, res) => {
+  res.sendStatus(200);
+});
+
+app.get('/NeuralBlock/api/checkSponsorSegments', (req, res) => {
+  if (req.query.vid === "LevkAjUE6d4") {
+    res.json({
+      probabilities: [0.69]
+    });
+    return;
+  }
+  res.sendStatus(500);
+});
+
+//getSponsorSegments is no longer being used for automod
+app.get('/NeuralBlock/api/getSponsorSegments', (req, res) => {
+  if (req.query.vid === "LevkAjUE6d4") {
+    res.json({
+      sponsorSegments: [[0.47,7.549],[264.023,317.293]]
+    });
+    return;
+  }
+  res.sendStatus(500);
+});
+
+// Testing webhooks
+app.post('/CustomWebhook', (req, res) => {
+  res.sendStatus(200);
+});
+
+module.exports = function createMockServer(callback) {
+  return app.listen(config.mockPort, callback);
+}

test/utils.js

@@ -0,0 +1,7 @@
+var config = require('../src/config.js');
+
+module.exports = {
+  getbaseURL: () => {
+    return "http://localhost:" + config.port;
+  }
+};

test/youtubeMock.js

@@ -0,0 +1,72 @@
+/*
+YouTubeAPI.videos.list({
+  part: "snippet",
+  id: videoID
+}, function (err, data) {});
+ */
+
+ // https://developers.google.com/youtube/v3/docs/videos
+
+const YouTubeAPI = {
+  listVideos: (id, callback) => {
+    YouTubeAPI.videos.list({
+      id: id
+    }, callback);
+  },
+  videos: {
+    list: (obj, callback) => {
+      if (obj.id === "knownWrongID") {
+        callback(undefined, {
+          pageInfo: {
+            totalResults: 0
+          },
+          items: []
+        });
+      } if (obj.id === "noDuration") {
+        callback(undefined, {
+          pageInfo: {
+            totalResults: 1
+          },
+          items: [
+            {
+              contentDetails: {
+                duration: "PT0S"
+              },
+              snippet: {
+                title: "Example Title",
+                thumbnails: {
+                  maxres: {
+                    url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png"
+                  }
+                }
+              }
+            }
+          ]
+        });
+      } else  {
+        callback(undefined, {
+          pageInfo: {
+            totalResults: 1
+          },
+          items: [
+            {
+              contentDetails: {
+                duration: "PT1H23M30S"
+              },
+              snippet: {
+                title: "Example Title",
+                thumbnails: {
+                  maxres: {
+                    url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png"
+                  }
+                }
+              }
+            }
+          ]
+        });
+      }
+    }
+  }
+};
+
+module.exports = YouTubeAPI;