Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into export

2025-12-13 23:17:02 +03:00 · 2021-05-23 11:17:10 -04:00
parent f6f5570d0c e71399f5af
commit 590ed037dd
29 changed files with 5190 additions and 324 deletions
--- a/src/app.ts
+++ b/src/app.ts
@@ -5,8 +5,8 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
 import {postSegmentShift} from './routes/postSegmentShift';
 import {postWarning} from './routes/postWarning';
 import {getIsUserVIP} from './routes/getIsUserVIP';
-import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments';
-import {postNoSegments} from './routes/postNoSegments';
+import {deleteLockCategoriesEndpoint} from './routes/deleteLockCategories';
+import {postLockCategories} from './routes/postLockCategories';
 import {getUserInfo} from './routes/getUserInfo';
 import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
 import {getTotalStats} from './routes/getTotalStats';
@@ -25,8 +25,9 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments';
 import {userCounter} from './middleware/userCounter';
 import {loggerMiddleware} from './middleware/logger';
 import {corsMiddleware} from './middleware/cors';
+import {apiCspMiddleware} from './middleware/apiCsp';
 import {rateLimitMiddleware} from './middleware/requestRateLimit';
-import dumpDatabase from './routes/dumpDatabase';
+import dumpDatabase, {redirectLink} from './routes/dumpDatabase';


 export function createServer(callback: () => void) {
@@ -36,6 +37,7 @@ export function createServer(callback: () => void) {
    //setup CORS correctly
    app.use(corsMiddleware);
    app.use(loggerMiddleware);
+    app.use("/api/", apiCspMiddleware);
    app.use(express.json());

    if (config.userCounterURL) app.use(userCounter);
@@ -114,10 +116,12 @@ function setupRoutes(app: Express) {
    //send out a formatted time saved total
    app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);

-    //submit video containing no segments
-    app.post('/api/noSegments', postNoSegments);
+    //submit video to lock categories
+    app.post('/api/noSegments', postLockCategories);
+    app.post('/api/lockCategories', postLockCategories);

-    app.delete('/api/noSegments', deleteNoSegmentsEndpoint);
+    app.delete('/api/noSegments', deleteLockCategoriesEndpoint);
+    app.delete('/api/lockCategories', deleteLockCategoriesEndpoint);

    //get if user is a vip
    app.get('/api/isUserVIP', getIsUserVIP);
@@ -131,6 +135,7 @@ function setupRoutes(app: Express) {
    if (config.postgres) {
        app.get('/database', (req, res) => dumpDatabase(req, res, true));
        app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
+        app.get('/database/*', redirectLink)
    } else {
        app.get('/database.db', function (req: Request, res: Response) {
            res.sendFile("./databases/sponsorTimes.db", {root: "./"});
--- a/src/config.ts
+++ b/src/config.ts
@@ -45,7 +45,34 @@ addDefaults(config, {
    },
    userCounterURL: null,
    youtubeAPIKey: null,
-    postgres: null
+    maxRewardTimePerSegmentInSeconds: 86400,
+    postgres: null,
+    dumpDatabase: {
+        enabled: false,
+        minTimeBetweenMs: 60000,
+        appExportPath: './docker/database-export',
+        postgresExportPath: '/opt/exports',
+        tables: [{
+            name: "sponsorTimes",
+            order: "timeSubmitted"
+        },
+        {
+            name: "userNames"
+        },
+        {
+            name: "categoryVotes"
+        },
+        {
+            name: "lockCategories",
+        },
+        {
+            name: "warnings",
+            order: "issueTime"
+        },
+        {
+            name: "vipUsers"
+        }]
+    }
 });

 // Add defaults
--- a/src/middleware/apiCsp.ts
+++ b/src/middleware/apiCsp.ts
@@ -0,0 +1,6 @@
+import {NextFunction, Request, Response} from 'express';
+
+export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) {
+    res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
+    next();
+}
--- a/src/routes/deleteLockCategories.ts
+++ b/src/routes/deleteLockCategories.ts
@@ -5,7 +5,7 @@ import {db} from '../databases/databases';
 import { Category, VideoID } from '../types/segments.model';
 import { UserID } from '../types/user.model';

-export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
+export async function deleteLockCategoriesEndpoint(req: Request, res: Response) {
    // Collect user input data
    const videoID = req.body.videoID as VideoID;
    const userID = req.body.userID as UserID;
@@ -35,9 +35,9 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
        return;
    }

-    deleteNoSegments(videoID, categories);  
+    deleteLockCategories(videoID, categories);  

-    res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
+    res.status(200).json({message: 'Removed lock categories entrys for video ' + videoID});
 }

 /**
@@ -45,12 +45,12 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
 * @param videoID 
 * @param categories If null, will remove all
 */
-export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> {
-    const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
+export async function deleteLockCategories(videoID: VideoID, categories: Category[]): Promise<void> {
+    const entries = (await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
        return categories === null || categories.indexOf(entry.category) !== -1;
    });

    for (const entry of entries) {
-        await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
+        await db.prepare('run', 'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
    }
 }
--- a/src/routes/dumpDatabase.ts
+++ b/src/routes/dumpDatabase.ts
@@ -2,51 +2,111 @@ import {db} from '../databases/databases';
 import {Logger} from '../utils/logger';
 import {Request, Response} from 'express';
 import { config } from '../config';
+import util from 'util';
+import fs from 'fs';
+import path from 'path';
+const unlink = util.promisify(fs.unlink);

 const ONE_MINUTE = 1000 * 60;

-const styleHeader = `<style>body{font-family: sans-serif}</style>`
+const styleHeader = `<style>
+    body {
+        font-family: sans-serif
+    }
+    table th,
+    table td {
+        padding: 7px;
+    }
+    table th {
+        text-align: left;
+    }
+    table tbody tr:nth-child(odd) {
+        background: #efefef;
+    }
+</style>`

 const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
 <p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
 <p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;

-const tables = [{
-    name: "sponsorTimes",
-    order: "timeSubmitted"
-},
-{
-    name: "userNames"
-},
-{
-    name: "categoryVotes"
-},
-{
-    name: "noSegments",
-},
-{
-    name: "warnings",
-    order: "issueTime"
-},
-{
-    name: "vipUsers"
-}];
+const tables = config?.dumpDatabase?.tables ?? [];
+const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
+const appExportPath = config?.dumpDatabase?.appExportPath ?? './docker/database-export';
+const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? '/opt/exports';
+const tableNames = tables.map(table => table.name);

-const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
+interface TableDumpList {
+    fileName: string;
+    tableName: string;
+};
+let latestDumpFiles: TableDumpList[] = [];

-const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
-                        .reduce((acc, url) => acc + url, "");
+interface TableFile {
+    file: string,
+    timestamp: number
+};
+
+if (tables.length === 0) {
+    Logger.warn('[dumpDatabase] No tables configured');
+}

 let lastUpdate = 0;
+let updateQueued = false;
+let updateRunning = false;

-export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
+function removeOutdatedDumps(exportPath: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+        // Get list of table names
+        // Create array for each table
+        const tableFiles: Record<string, TableFile[]> = tableNames.reduce((obj: any, tableName) => {
+            obj[tableName] = [];
+            return obj;
+        }, {});
+
+        // read files in export directory
+        fs.readdir(exportPath, async (err: any, files: string[]) => {
+            if (err) Logger.error(err);
+            if (err) return resolve();
+
+            files.forEach(file => {
+                // we only care about files that start with "<tablename>_" and ends with .csv
+                tableNames.forEach(tableName => {
+                    if (file.startsWith(`${tableName}`) && file.endsWith('.csv')) {
+                        const filePath = path.join(exportPath, file);
+                        tableFiles[tableName].push({
+                            file: filePath,
+                            timestamp: fs.statSync(filePath).mtime.getTime()
+                        });
+                    }
+                });
+            });
+
+            for (let tableName in tableFiles) {
+                const files = tableFiles[tableName].sort((a, b) => b.timestamp - a.timestamp);
+                for (let i = 2; i < files.length; i++) {
+                    // remove old file
+                    await unlink(files[i].file).catch((error: any) => {
+                        Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
+                    });
+                }
+            }
+
+            resolve();
+        });
+    });
+}
+
+export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) {
+    if (!config?.dumpDatabase?.enabled) {
+        res.status(404).send("Database dump is disabled");
+        return;
+    }
    if (!config.postgres) {
        res.status(404).send("Not supported on this instance");
        return;
    }

-    const now = Date.now();
-    const updateQueued = now - lastUpdate > ONE_MINUTE;
+    updateQueueTime();

    res.status(200)
    
@@ -54,25 +114,100 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
        res.send(`${styleHeader}
            <h1>SponsorBlock database dumps</h1>${licenseHeader}
            <h3>How this works</h3>
-            Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
-            Then, you can download the csv files below, or use the links returned from the JSON request.
+
+            Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to get a list of urls and the update status database dump to run.
+            Then, you can download the csv files below, or use the links returned from the JSON request. 
+            A dump will also be triggered by making a request to one of these urls.
+
            <h3>Links</h3>
-            ${linksHTML}<br/>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Table</th>
+                        <th>CSV</th>
+                    </tr>
+                </thead>
+                <tbody>
+                ${latestDumpFiles.map((item:any) => {
+                    return `
+                    <tr>
+                        <td>${item.tableName}</td>
+                        <td><a href="/database/${item.tableName}.csv">${item.tableName}.csv</a></td>
+                    </tr>
+                    `;
+                }).join('')}
+                ${latestDumpFiles.length === 0 ? '<tr><td colspan="2">Please wait: Generating files</td></tr>' : ''}
+                </tbody>
+            </table>
+            <hr/>
            ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
    } else {
        res.send({
            lastUpdated: lastUpdate,
            updateQueued,
-            links
+            links: latestDumpFiles.map((item:any) => {
+                return {
+                    table: item.tableName,
+                    url: `/database/${item.tableName}.csv`,
+                    size: item.fileSize,
+                };
+            }),
        })
    }

-    if (updateQueued) {
-        lastUpdate = Date.now();
+    await queueDump();
+}
+
+export async function redirectLink(req: Request, res: Response): Promise<void> {
+    if (!config?.dumpDatabase?.enabled) {
+        res.status(404).send("Database dump is disabled");
+        return;
+    }
+    if (!config.postgres) {
+        res.status(404).send("Not supported on this instance");
+        return;
+    }
+    
+    const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
+
+    updateQueueTime();
+
+    if (file) {
+        res.redirect("/download/" + file.fileName);
+    } else {
+        res.status(404).send();
+    }
+
+    await queueDump();
+}
+
+function updateQueueTime(): void {
+    updateQueued ||= Date.now() - lastUpdate > MILLISECONDS_BETWEEN_DUMPS;
+}
+
+async function queueDump(): Promise<void> {
+    if (updateQueued && !updateRunning) {
+        const startTime = Date.now();
+        updateRunning = true;
+
+        await removeOutdatedDumps(appExportPath);
+        
+        const dumpFiles = [];

        for (const table of tables) {
-            db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
-                    TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+            const fileName = `${table.name}_${startTime}.csv`;
+            const file = `${postgresExportPath}/${fileName}`;
+            await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
+                    TO '${file}' WITH (FORMAT CSV, HEADER true);`);
+            dumpFiles.push({
+                fileName,
+                tableName: table.name,
+            });
        }
+        latestDumpFiles = [...dumpFiles];
+
+        updateQueued = false;
+        updateRunning = false;
+        lastUpdate = startTime;
    }
 }
--- a/src/routes/getSavedTimeForUser.ts
+++ b/src/routes/getSavedTimeForUser.ts
@@ -1,8 +1,11 @@
 import {db} from '../databases/databases';
 import {Request, Response} from 'express';
 import {getHash} from '../utils/getHash';
+import {config} from '../config';
 import { Logger } from '../utils/logger';

+const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
+
 export async function getSavedTimeForUser(req: Request, res: Response) {
    let userID = req.query.userID as string;

@@ -16,7 +19,7 @@ export async function getSavedTimeForUser(req: Request, res: Response) {
    userID = getHash(userID);

    try {
-        let row = await db.prepare("get", 'SELECT SUM(("endTime" - "startTime") / 60 * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]);
+        let row = await db.prepare("get", 'SELECT SUM(((CASE WHEN "endTime" - "startTime" > ' + maxRewardTimePerSegmentInSeconds + ' THEN ' + maxRewardTimePerSegmentInSeconds + ' ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]);

        if (row.minutesSaved != null) {
            res.send({
--- a/src/routes/getSkipSegments.ts
+++ b/src/routes/getSkipSegments.ts
@@ -268,6 +268,10 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
        : req.query.category
            ? [req.query.category]
            : ['sponsor'];
+    if (!Array.isArray(categories)) {
+        res.status(400).send("Categories parameter does not match format requirements.");
+        return false;
+    }

    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
    if (!Object.values(Service).some((val) => val == service)) {
--- a/src/routes/getTopUsers.ts
+++ b/src/routes/getTopUsers.ts
@@ -5,6 +5,7 @@ import {Request, Response} from 'express';

 const MILLISECONDS_IN_MINUTE = 60000;
 const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
+const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;

 async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) {
    const userNames = [];
@@ -24,14 +25,14 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
    }

    const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
-        SUM(("sponsorTimes"."endTime" - "sponsorTimes"."startTime") / 60 * "sponsorTimes"."views") as "minutesSaved",
+        SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ${maxRewardTimePerSegmentInSeconds} THEN ${maxRewardTimePerSegmentInSeconds} ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
        SUM("votes") as "userVotes", ` +
        additionalFields +
        `IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
        LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID"
        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL
        GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
-        ORDER BY "` + sortBy + `" DESC LIMIT 100`, []);
+        ORDER BY "${sortBy}" DESC LIMIT 100`, []);

    for (let i = 0; i < rows.length; i++) {
        userNames[i] = rows[i].userName;
@@ -70,6 +71,10 @@ export async function getTopUsers(req: Request, res: Response) {
        return;
    }

+    //TODO: remove. This is broken for now
+    res.status(200).send();
+    return;
+
    //setup which sort type to use
    let sortBy = '';
    if (sortType == 0) {
--- a/src/routes/getUserInfo.ts
+++ b/src/routes/getUserInfo.ts
@@ -1,11 +1,15 @@
 import {db} from '../databases/databases';
 import {getHash} from '../utils/getHash';
+import {isUserVIP} from '../utils/isUserVIP';
 import {Request, Response} from 'express';
-import {Logger} from '../utils/logger'
+import {Logger} from '../utils/logger';
+import { HashedUserID, UserID } from '../types/user.model';

-async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSaved: number, segmentCount: number }> {
+async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
    try {
-        let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(*) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
+        let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved",
+                                            count(*) as "segmentCount" FROM "sponsorTimes" 
+                                            WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
        if (row.minutesSaved != null) {
            return {
                minutesSaved: row.minutesSaved,
@@ -22,7 +26,7 @@ async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSa
    }
 }

-async function dbGetUsername(userID: string) {
+async function dbGetUsername(userID: HashedUserID) {
    try {
        let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
        if (row !== undefined) {
@@ -36,24 +40,19 @@ async function dbGetUsername(userID: string) {
    }
 }

-async function dbGetViewsForUser(userID: string) {
+async function dbGetViewsForUser(userID: HashedUserID) {
    try {
        let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
-        //increase the view count by one
-        if (row.viewCount != null) {
-            return row.viewCount;
-        } else {
-            return 0;
-        }
+        return row?.viewCount ?? 0;
    } catch (err) {
        return false;
    }
 }

-async function dbGetWarningsForUser(userID: string): Promise<number> {
+async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
    try {
-        let rows = await db.prepare('all', `SELECT * FROM "warnings" WHERE "userID" = ?`, [userID]);
-        return rows.length;
+        let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]);
+        return row?.total ?? 0;
    } catch (err) {
        Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0');
        return 0;
@@ -61,7 +60,7 @@ async function dbGetWarningsForUser(userID: string): Promise<number> {
 }

 export async function getUserInfo(req: Request, res: Response) {
-    let userID = req.query.userID as string;
+    let userID = req.query.userID as UserID;

    if (userID == undefined) {
        //invalid request
@@ -70,17 +69,18 @@ export async function getUserInfo(req: Request, res: Response) {
    }

    //hash the userID
-    userID = getHash(userID);
+    const hashedUserID: HashedUserID = getHash(userID);

-    const segmentsSummary = await dbGetSubmittedSegmentSummary(userID);
+    const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID);
    if (segmentsSummary) {
        res.send({
-            userID,
-            userName: await dbGetUsername(userID),
+            userID: hashedUserID,
+            userName: await dbGetUsername(hashedUserID),
            minutesSaved: segmentsSummary.minutesSaved,
            segmentCount: segmentsSummary.segmentCount,
-            viewCount: await dbGetViewsForUser(userID),
-            warnings: await dbGetWarningsForUser(userID),
+            viewCount: await dbGetViewsForUser(hashedUserID),
+            warnings: await dbGetWarningsForUser(hashedUserID),
+            vip: await isUserVIP(hashedUserID),
        });
    } else {
        res.status(400).send();
--- a/src/routes/postLockCategories.ts
+++ b/src/routes/postLockCategories.ts
@@ -4,7 +4,7 @@ import {isUserVIP} from '../utils/isUserVIP';
 import {db} from '../databases/databases';
 import {Request, Response} from 'express';

-export async function postNoSegments(req: Request, res: Response) {
+export async function postLockCategories(req: Request, res: Response) {
    // Collect user input data
    let videoID = req.body.videoID;
    let userID = req.body.userID;
@@ -34,12 +34,12 @@ export async function postNoSegments(req: Request, res: Response) {
        return;
    }

-    // Get existing no segment markers
-    let noSegmentList = await db.prepare('all', 'SELECT "category" from "noSegments" where "videoID" = ?', [videoID]);
-    if (!noSegmentList || noSegmentList.length === 0) {
-        noSegmentList = [];
+    // Get existing lock categories markers
+    let noCategoryList = await db.prepare('all', 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]);
+    if (!noCategoryList || noCategoryList.length === 0) {
+        noCategoryList = [];
    } else {
-        noSegmentList = noSegmentList.map((obj: any) => {
+        noCategoryList = noCategoryList.map((obj: any) => {
            return obj.category;
        });
    }
@@ -48,7 +48,7 @@ export async function postNoSegments(req: Request, res: Response) {
    let categoriesToMark = categories.filter((category) => {
        return !!category.match(/^[_a-zA-Z]+$/);
    }).filter((category) => {
-        return noSegmentList.indexOf(category) === -1;
+        return noCategoryList.indexOf(category) === -1;
    });

    // remove any duplicates
@@ -59,9 +59,9 @@ export async function postNoSegments(req: Request, res: Response) {
    // create database entry
    for (const category of categoriesToMark) {
        try {
-            await db.prepare('run', `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]);
+            await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]);
        } catch (err) {
-            Logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'");
+            Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
            Logger.error(err);
            res.status(500).json({
                message: "Internal Server Error: Could not write marker to the database.",
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -14,7 +14,7 @@ import {Request, Response} from 'express';
 import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
 import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
-import { deleteNoSegments } from './deleteNoSegments';
+import { deleteLockCategories } from './deleteLockCategories';
 import { getCategoryActionType } from '../utils/categoryInfo';

 interface APIVideoInfo {
@@ -357,7 +357,7 @@ export async function postSkipSegments(req: Request, res: Response) {
        return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
    }

-    let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
+    let lockedCategoryList = (await db.prepare('all', 'SELECT category from "lockCategories" where "videoID" = ?', [videoID])).map((list: any) => {
        return list.category;
    });

@@ -389,9 +389,9 @@ export async function postSkipSegments(req: Request, res: Response) {
            await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
        }

-        // Reset no segments
-        noSegmentList = [];
-        deleteNoSegments(videoID, null);
+        // Reset lock categories
+        lockedCategoryList = [];
+        deleteLockCategories(videoID, null);
    }

    // Check if all submissions are correct
@@ -407,8 +407,8 @@ export async function postSkipSegments(req: Request, res: Response) {
            return;
        }

-        // Reject segemnt if it's in the no segments list
-        if (!isVIP && noSegmentList.indexOf(segments[i].category) !== -1) {
+        // Reject segment if it's in the locked categories list
+        if (!isVIP && lockedCategoryList.indexOf(segments[i].category) !== -1) {
            // TODO: Do something about the fradulent submission
            Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
            res.status(403).send(
--- a/src/routes/setUsername.ts
+++ b/src/routes/setUsername.ts
@@ -21,6 +21,10 @@ export async function setUsername(req: Request, res: Response) {
        res.sendStatus(200);
        return;
    }
+    
+    // remove unicode control characters from username (example: \n, \r, \t etc.)
+    // source: https://en.wikipedia.org/wiki/Control_character#In_Unicode
+    userName = userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, '');

    if (adminUserIDInput != undefined) {
        //this is the admin controlling the other users account, don't hash the controling account's ID
@@ -35,6 +39,12 @@ export async function setUsername(req: Request, res: Response) {
        //hash the userID
        userID = getHash(userID);
    }
+    
+    if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148"].includes(userID)) {
+        // Don't allow
+        res.sendStatus(200);
+        return;   
+    }

    try {
        //check if username is already set
--- a/src/routes/shadowBanUser.ts
+++ b/src/routes/shadowBanUser.ts
@@ -43,8 +43,8 @@ export async function shadowBanUser(req: Request, res: Response) {
            //find all previous submissions and hide them
            if (unHideOldSubmissions) {
                await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ?
-                                AND NOT EXISTS ( SELECT "videoID", "category" FROM "noSegments" WHERE
-                                "sponsorTimes"."videoID" = "noSegments"."videoID" AND "sponsorTimes"."category" = "noSegments"."category")`, [userID]);
+                                AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
+                                "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
            }
        } else if (!enabled && row.userCount > 0) {
            //remove them from the shadow ban list
@@ -53,7 +53,7 @@ export async function shadowBanUser(req: Request, res: Response) {
            //find all previous submissions and unhide them
            if (unHideOldSubmissions) {
                let segmentsToIgnore = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st
-                                JOIN "noSegments" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
+                                JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
                                    , [userID])).map((item: {UUID: string}) => item.UUID);
                let allSegments = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID]))
                                        .map((item: {UUID: string}) => item.UUID);
--- a/src/routes/voteOnSponsorTime.ts
+++ b/src/routes/voteOnSponsorTime.ts
@@ -180,51 +180,54 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
    const timeSubmitted = Date.now();

    const voteAmount = isVIP ? 500 : 1;
+    const ableToVote = isVIP || finalResponse.finalStatus === 200 || true;

-    // Add the vote
-    if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
-        // Update the already existing db entry
-        await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
-    } else {
-        // Add a db entry
-        await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
-    }
+    if (ableToVote) {
+        // Add the vote
+        if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
+            // Update the already existing db entry
+            await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
+        } else {
+            // Add a db entry
+            await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
+        }

-    // Add the info into the private db
-    if (usersLastVoteInfo?.votes > 0) {
-        // Reverse the previous vote
-        await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
+        // Add the info into the private db
+        if (usersLastVoteInfo?.votes > 0) {
+            // Reverse the previous vote
+            await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);

-        await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
-    } else {
-        await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
-    }
+            await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
+        } else {
+            await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
+        }

-    // See if the submissions category is ready to change
-    const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
+        // See if the submissions category is ready to change
+        const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);

-    const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
-    const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
-    const startingVotes = isSubmissionVIP ? 10000 : 1;
+        const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
+        const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
+        const startingVotes = isSubmissionVIP ? 10000 : 1;

-    // Change this value from 1 in the future to make it harder to change categories
-    // Done this way without ORs incase the value is zero
-    const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
+        // Change this value from 1 in the future to make it harder to change categories
+        // Done this way without ORs incase the value is zero
+        const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;

-    // Add submission as vote
-    if (!currentCategoryInfo && submissionInfo) {
-        await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
+        // Add submission as vote
+        if (!currentCategoryInfo && submissionInfo) {
+            await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);

-        await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
-    }
+            await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
+        }

-    const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
+        const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;

-    //TODO: In the future, raise this number from zero to make it harder to change categories
-    // VIPs change it every time
-    if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
-        // Replace the category
-        await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
+        //TODO: In the future, raise this number from zero to make it harder to change categories
+        // VIPs change it every time
+        if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
+            // Replace the category
+            await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
+        }
    }

    clearRedisCache(videoInfo);
@@ -273,8 +276,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
    // If not upvote
    if (!isVIP && type !== 1) {
        const isSegmentLocked = async () => !!(await db.prepare('get', `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked; 
-        const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "noSegments".category from "noSegments" left join "sponsorTimes"' + 
-                                ' on ("noSegments"."videoID" = "sponsorTimes"."videoID" and "noSegments".category = "sponsorTimes".category)' + 
+        const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes"' + 
+                                ' on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and "lockCategories".category = "sponsorTimes".category)' + 
                                    ' where "UUID" = ?', [UUID]));

        if (await isSegmentLocked() || await isVideoLocked()) {
@@ -287,13 +290,19 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
        return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res);
    }

-    if (type == 1 && !isVIP && !isOwnSubmission) {
+    if (type !== undefined && !isVIP && !isOwnSubmission) {
        // Check if upvoting hidden segment
        const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);

        if (voteInfo && voteInfo.votes <= -2) {
-            res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
-            return;
+            if (type == 1) {
+                res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
+                return;
+            } else if (type == 0) {
+                // Already downvoted enough, ignore
+                res.status(200).send();
+                return;
+            }
        }
    }

@@ -378,7 +387,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
        const ableToVote = isVIP
            || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
                && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
-                && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined);
+                && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
+                && finalResponse.finalStatus === 200;

        if (ableToVote) {
            //update the votes table
--- a/src/types/config.model.ts
+++ b/src/types/config.model.ts
@@ -37,7 +37,9 @@ export interface SBSConfig {
    minimumPrefix?: string;
    maximumPrefix?: string;
    redis?: redis.ClientOpts;
+    maxRewardTimePerSegmentInSeconds?: number;
    postgres?: PoolConfig;
+    dumpDatabase?: DumpDatabase;
 }

 export interface WebhookConfig {
@@ -61,4 +63,17 @@ export interface PostgresConfig {
    createDbIfNotExists: boolean;
    enableWalCheckpointNumber: boolean;
    postgres: PoolConfig;
-}
+}
+
+export interface DumpDatabase {
+    enabled: boolean;
+    minTimeBetweenMs: number;
+    appExportPath: string;
+    postgresExportPath: string;
+    tables: DumpDatabaseTable[];
+}
+
+export interface DumpDatabaseTable {
+    name: string;
+    order?: string;
+}
--- a/src/utils/redis.ts
+++ b/src/utils/redis.ts
@@ -29,6 +29,10 @@ if (config.redis) {
    exportObject.getAsync = (key) => new Promise((resolve) => client.get(key, (err, reply) => resolve({err, reply})));
    exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({err, reply})));
    exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
+
+    client.on("error", function(error) {
+        Logger.error(error);
+    });
 }

 export default exportObject;