Add config for min

add additional poi_highlight tests

.editorconfig

@@ -0,0 +1,17 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{js,json,ts,tsx}]
+charset = utf-8
+indent_style = space
+indent_size = 4
+
+[package.json]
+indent_size = 2

.eslintrc.js

@@ -0,0 +1,32 @@
+module.exports = {
+    env: {
+        browser: false,
+        es2021: true,
+        node: true,
+    },
+    extends: [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended",
+    ],
+    parser: "@typescript-eslint/parser",
+    parserOptions: {
+        ecmaVersion: 12,
+        sourceType: "module",
+    },
+    plugins: ["@typescript-eslint"],
+    rules: {
+        // TODO: Remove warn rules when not needed anymore
+        "@typescript-eslint/no-empty-interface": "off",
+        "@typescript-eslint/no-explicit-any": "off",
+        "indent": ["warn", 4, { "SwitchCase": 1 }],
+        "no-multiple-empty-lines": ["error", { max: 2, maxEOF: 0 }],
+        "no-self-assign": "off",
+        "no-trailing-spaces": "warn",
+        "object-curly-spacing": ["warn", "always"],
+        "prefer-template": "warn",
+        "quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
+        "require-await": "warn",
+        "semi": "warn",
+        "no-console": "warn"
+    },
+};

.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: SQLite CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  test:
+    name: Run Tests with SQLite
+    runs-on: ubuntu-latest
+
+    steps:
+      # Initialization
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Run Tests
+        timeout-minutes: 5
+        run: npm test

.github/workflows/db-backup.yml

@@ -0,0 +1,18 @@
+name: Docker image builds
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - containers/backup-db/**
+  workflow_dispatch:
+
+jobs:
+  backup-db:
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      name: "db-backup"
+      username: "ajayyy"
+      folder: "./containers/backup-db"
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docker-build.yml

@@ -0,0 +1,46 @@
+# Based on https://github.com/ajayyy/sb-mirror/blob/main/.github/workflows/docker-build.yml
+name: multi-build-docker
+on: 
+  workflow_call:
+    inputs:
+      name:
+        required: true
+        type: string
+      username:
+        required: true
+        type: string
+      folder:
+        required: true
+        type: string
+    secrets:
+      GH_TOKEN:
+        required: true
+
+jobs:
+  build_container:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/${{ inputs.username }}/${{ inputs.name }}
+          tags: |
+            type-raw,value=alpine
+          flavor: |
+            latest=true
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GH_TOKEN }}
+      - name: push
+        uses: docker/build-push-action@v3
+        with:
+          context: ${{ inputs.folder }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/eslint.yml

@@ -0,0 +1,21 @@
+name: Linting
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  lint:
+    name: Lint with ESLint
+    runs-on: ubuntu-latest
+
+    steps:
+      # Initialization
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Run Tests
+        timeout-minutes: 5
+        run: npm run lint

.github/workflows/generate-sqlite-base.yml

@@ -0,0 +1,28 @@
+name: create-sqlite-base
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - databases/**
+
+jobs:
+  make-base-db:
+    name: Generate SQLite base .db
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Set config
+        run: |
+          echo '{"mode": "init-db-and-exit"}' > config.json
+      - name: Run Server
+        timeout-minutes: 10
+        run: npm start
+      - uses: actions/upload-artifact@v2
+        with:
+          name: SponsorTimesDB.db
+          path: databases/sponsorTimes.db

.github/workflows/postgres-redis-ci.yml

@@ -0,0 +1,29 @@
+name: PostgreSQL + Redis CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  test:
+    name: Run Tests with PostgreSQL and Redis
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the docker-compose stack
+        env:
+          PG_USER: ci_db_user
+          PG_PASS: ci_db_pass
+        run: docker-compose -f docker/docker-compose-ci.yml up -d
+      - name: Check running containers
+        run: docker ps
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Run Tests
+        env:
+          TEST_POSTGRES: true
+        timeout-minutes: 5
+        run: npm test

.github/workflows/sb-server.yml

@@ -0,0 +1,25 @@
+name: Docker image builds
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  sb-server:
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      name: "sb-server"
+      username: "ajayyy"
+      folder: "."
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  rsync-host:
+    needs: sb-server
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      name: "rsync-host"
+      username: "ajayyy"
+      folder: "./containers/rsync"
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/take-action.yml

@@ -0,0 +1,12 @@
+name: Assign issue to contributor
+on: [issue_comment]
+
+jobs:
+  assign:
+    name: Take an issue
+    runs-on: ubuntu-latest
+    steps:
+      - name: take the issue
+        uses: bdougie/take-action@main
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.gitignore

@@ -2,47 +2,9 @@
 logs
 *.log
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
-
-# Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
 
 # Dependency directories
 node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
 
 # TypeScript cache
 *.tsbuildinfo
@@ -53,39 +15,33 @@ typings/
 # Optional eslint cache
 .eslintcache
 
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
 # dotenv environment variables file
 .env
 .env.test
 
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# next.js build output
-.next
-
-# nuxt.js build output
-.nuxt
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-# DynamoDB Local files
-.dynamodb/
-
 # Databases
-databases
+databases/sponsorTimes.db
+databases/sponsorTimes.db-shm
+databases/sponsorTimes.db-wal
+databases/private.db
+databases/private.db-shm
+databases/private.db-wal
+databases/sponsorTimesReal.db
+test/databases/sponsorTimes.db
+test/databases/sponsorTimes.db-shm
+test/databases/sponsorTimes.db-wal
+test/databases/private.db
+databases/cache
+docker/database-export
+
+# Config files
+config.json
+docker/database.env
+
+# Other
+working
+
+# Mac files
+.DS_Store
+/.idea/
+/dist/

DatabaseSchema.md

@@ -0,0 +1,283 @@
+# SponsorTimesDB
+
+[vipUsers](#vipUsers)  
+[sponsorTimes](#sponsorTimes)  
+[userNames](#userNames)  
+[categoryVotes](#categoryVotes)  
+[lockCategories](#lockCategories)  
+[warnings](#warnings)  
+[shadowBannedUsers](#shadowBannedUsers)  
+[unlistedVideos](#unlistedVideos)  
+[config](#config)  
+[archivedSponsorTimes](#archivedSponsorTimes)
+
+### vipUsers
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| vipUsers_index | userID |
+
+### sponsorTimes  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| startTime | REAL | not null |
+| endTime | REAL | not null |
+| votes | INTEGER | not null |
+| locked | INTEGER | not null, default '0' |
+| incorrectVotes | INTEGER | not null, default 1 |
+| UUID | TEXT | not null, unique |
+| userID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| views | INTEGER | not null |
+| category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
+| service | TEXT | not null, default 'YouTube' |
+| videoDuration | INTEGER | not null, default '0' |
+| hidden | INTEGER | not null, default '0' |
+| reputation | REAL | not null, default '0' |
+| shadowHidden | INTEGER | not null |
+| hashedVideoID | TEXT | not null, default '', sha256 |
+| userAgent | TEXT | not null, default '' |
+| description | TEXT | not null, default '' |
+
+| index | field |
+| -- | :--: |
+| sponsorTime_timeSubmitted | timeSubmitted |
+| sponsorTime_userID | userID |
+| sponsorTimes_UUID | UUID |
+| sponsorTimes_hashedVideoID | hashedVideoID, category |
+| sponsorTimes_videoID | videoID, service, category, timeSubmitted |
+
+### userNames
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+| userName | TEXT | not null |
+| locked | INTEGER | not nul, default '0' |
+
+| index | field |
+| -- | :--: |
+| userNames_userID | userID |
+
+### categoryVotes
+
+| Name | Type | |
+| -- | :--: | -- |
+| UUID | TEXT | not null |
+| category | TEXT | not null |
+| votes | INTEGER | not null, default 0 |
+
+| index | field |
+| -- | :--: |
+| categoryVotes_UUID_public | UUID, category |
+
+### lockCategories  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| userID | TEXT | not null |
+| actionType | TEXT | not null, default 'skip' |
+| category | TEXT | not null |
+| hashedVideoID | TEXT | not null, default '' |
+| reason | TEXT | not null, default '' |
+| service | TEXT | not null, default 'YouTube' |
+
+| index | field |
+| -- | :--: |
+| lockCategories_videoID | videoID, service, category |
+
+### warnings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+| issueTime | INTEGER | not null |
+| issuerUserID | TEXT | not null |
+| enabled | INTEGER | not null |
+| reason | TEXT | not null, default '' |
+
+| index | field |
+| -- | :--: |
+| warnings_index | userID |
+| warnings_issueTime | issueTime |
+
+### shadowBannedUsers  
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| shadowBannedUsers_index | userID |
+
+### videoInfo  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| channelID | TEXT | not null |
+| title | TEXT | not null |
+| published | REAL | not null |
+| genreUrl | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| videoInfo_videoID | timeSubmitted |
+| videoInfo_channelID | userID |
+
+### unlistedVideos  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| year | TEXT | not null |
+| views | TEXT | not null |
+| channelID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| service | TEXT | not null, default 'YouTube' |
+
+### config
+
+| Name | Type | |
+| -- | :--: | -- |
+| key | TEXT | not null, unique |
+| value | TEXT | not null |
+
+### archivedSponsorTimes
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| startTime | REAL | not null |
+| endTime | REAL | not null |
+| votes | INTEGER | not null |
+| locked | INTEGER | not null, default '0' |
+| incorrectVotes | INTEGER | not null, default 1 |
+| UUID | TEXT | not null, unique |
+| userID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| views | INTEGER | not null |
+| category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
+| service | TEXT | not null, default 'YouTube' |
+| videoDuration | INTEGER | not null, default '0' |
+| hidden | INTEGER | not null, default '0' |
+| reputation | REAL | not null, default '0' |
+| shadowHidden | INTEGER | not null |
+| hashedVideoID | TEXT | not null, default '', sha256 |
+| userAgent | TEXT | not null, default '' |
+
+### ratings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| service | TEXT | not null, default 'YouTube' |
+| type | INTEGER | not null |
+| count | INTEGER | not null |
+| hashedVideoID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| ratings_hashedVideoID_gin | hashedVideoID |
+| ratings_hashedVideoID | hashedVideoID, service |
+| ratings_videoID | videoID, service |
+
+# Private 
+
+[votes](#votes)  
+[categoryVotes](#categoryVotes)  
+[sponsorTimes](#sponsorTimes)  
+[config](#config)  
+[ratings](#ratings)  
+[tempVipLog](#tempVipLog)  
+[userNameLogs](#userNameLogs)  
+
+### votes
+
+| Name | Type | |
+| -- | :--: | -- |
+| UUID | TEXT | not null |
+| userID | TEXT | not null |
+| hashedIP | TEXT | not null |
+| type | INTEGER | not null |
+
+| index | field |
+| -- | :--: |
+| votes_userID | UUID |
+
+### categoryVotes
+
+| Name | Type | |
+| -- | :--: | -- |
+| UUID | TEXT | not null |
+| userID | TEXT | not null |
+| hashedIP | TEXT | not null |
+| category | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+
+| index | field |
+| -- | :--: |
+| categoryVotes_UUID | UUID, userID, hasedIP, category |
+
+### sponsorTimes  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| hashedIP | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| service | TEXT | not null, default 'YouTube' |
+
+| index | field |
+| -- | :--: |
+| sponsorTimes_hashedIP | hashedIP |
+| privateDB_sponsorTimes_videoID_v2 | videoID, service |
+
+### config  
+
+| Name | Type | |
+| -- | :--: | -- |
+| key | TEXT | not null |
+| value | TEXT | not null |
+
+### ratings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| service | TEXT | not null, default 'YouTube' |
+| userID | TEXT | not null |
+| type | INTEGER | not null |
+| timeSubmitted | INTEGER | not null |
+| hashedIP | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| ratings_videoID | videoID, service, userID, timeSubmitted |
+
+### tempVipLog
+| Name | Type | |
+| -- | :--: | -- |
+| issuerUserID | TEXT | not null |
+| targetUserID | TEXT | not null |
+| enabled | BOOLEAN | not null |
+| updatedAt | INTEGER | not null |
+
+### userNameLogs
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+| newUserName | TEXT | not null |
+| oldUserName | TEXT | not null |
+| updatedByAdmin | BOOLEAN | not null |
+| updatedAt | INTEGER | not null |

Dockerfile

@@ -0,0 +1,16 @@
+FROM node:16-alpine as builder
+RUN apk add --no-cache --virtual .build-deps python3 make g++
+COPY package.json package-lock.json tsconfig.json entrypoint.sh ./
+COPY src src
+RUN npm ci && npm run tsc
+
+FROM node:16-alpine as app
+WORKDIR /usr/src/app
+RUN apk add git postgresql-client
+COPY --from=builder ./node_modules ./node_modules
+COPY --from=builder ./dist ./dist
+COPY ./.git ./.git
+COPY entrypoint.sh .
+COPY databases/*.sql databases/
+EXPOSE 8080
+CMD ./entrypoint.sh

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Ajay Ramachandran
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.MD

@@ -1,17 +1,37 @@
-# SponsorBlocker Server
+# SponsorBlock Server
 
-SponsorBlocker is an extension that will skip over sponsored segments of YouTube videos. SponsorBlocker is a crowdsourced browser extension that let's anyone submit the start and end time's of sponsored segments of YouTube videos. Once one person submits this information, everyone else with this extension will skip right over the sponsored segment.
+SponsorBlock is an extension that will skip over sponsored segments of YouTube videos. SponsorBlock is a crowdsourced browser extension that let's anyone submit the start and end time's of sponsored segments of YouTube videos. Once one person submits this information, everyone else with this extension will skip right over the sponsored segment.
 
 This is the server backend for it
 
 # Server
 
-This is a simple Sqlite database that will hold all the timing data.
+This uses a Postgres or Sqlite database to hold all the timing data.
 
-To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. So, you can download a backup or get archive.org to take a backup if you do desire.
+To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
 
 Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.
 
 # Client
 
-The client web browser extension is available here: https://github.com/ajayyy/SponsorBlocker
+The client web browser extension is available here: https://github.com/ajayyy/SponsorBlock
+
+# Build Yourself
+
+This is a node.js server, so clone this repo and run `npm install` to install all dependencies.
+
+Make sure to put the database files in the `./databases` folder if you want to use a pre-existing database. Otherwise, a fresh database will be created.
+
+Rename `config.json.example` to `config.json` and fill the parameters inside. Make sure to remove the comments as comments are not supported in JSON.
+
+Ensure all the tests pass with `npm test`
+
+Run the server with `npm start`.
+
+# Developing
+
+If you want to make changes, run `npm run dev` to automatically reload the server and run tests whenever a file is saved.
+
+# API Docs
+
+Available [here](https://wiki.sponsor.ajay.app/index.php/API_Docs)

ci.json

@@ -0,0 +1,71 @@
+{
+    "port": 8080,
+    "mockPort": 8081,
+    "globalSalt": "testSalt",
+    "adminUserID": "4bdfdc9cddf2c7d07a8a87b57bf6d25389fb75d1399674ee0e0938a6a60f4c3b",
+    "newLeafURLs": ["placeholder"],
+    "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
+    "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
+    "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",
+    "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
+    "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
+    "behindProxy": true,
+    "postgres": {
+        "user": "ci_db_user",
+        "password": "ci_db_pass",
+        "host": "localhost",
+        "port": 5432
+    },
+    "redis": {
+        "socket": {
+            "host": "localhost",
+            "port": 6379
+        }
+    },
+    "createDatabaseIfNotExist": true,
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    "categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
+    "mode": "test",
+    "readOnly": false,
+    "webhooks": [
+        {
+            "url": "http://127.0.0.1:8081/CustomWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8081/FailedWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8099/WrongPort",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }
+    ],
+    "maxNumberOfActiveWarnings": 3,
+    "hoursAfterWarningExpires": 24,
+    "rateLimit": {
+        "vote": {
+          "windowMs": 900000,
+          "max": 20,
+          "message": "Too many votes, please try again later",
+          "statusCode": 429
+        },
+        "view": {
+          "windowMs": 900000,
+          "max": 20,
+          "statusCode": 200
+        }
+    }
+}

config.json.example

@@ -0,0 +1,70 @@
+{
+    // Remove all comments from the config when using it!
+
+    "port": 80,
+    "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]",
+    "adminUserID": "[the hashed id of the user who can perform admin actions]",
+    "newLeafURLs": ["http://localhost:3241"],
+    "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
+    "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
+    "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
+    "neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app
+    "discordNeuralBlockRejectWebhookURL": null, //URL from discord if you would like notifications when NeuralBlock rejects a submission [optional]
+    "userCounterURL": null, // For user counting. URL to instance of https://github.com/ajayyy/PrivacyUserCount
+    "proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash)
+    "behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For"
+    "db": "./databases/sponsorTimes.db",
+    "privateDB": "./databases/private.db",
+    "createDatabaseIfNotExist": true, //This will run on startup every time (unless readOnly is true) - so ensure "create table if not exists" is used in the schema
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    // when using redis, add `"enable_offline_queue": false` to force commands to fail instead of timing out
+    "mode": "development",
+    "readOnly": false,
+    "webhooks": [],
+    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "preview", "music_offtopic", "poi_highlight"], // List of supported categories any other category will be rejected
+    "getTopUsersCacheTimeMinutes": 5, // cacheTime for getTopUsers result in minutes
+    "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire
+    "hoursAfterWarningExpire": 24,
+    "rateLimit": {
+      "vote": {
+        "windowMs": 900000, // 15 minutes
+        "max": 20, // 20 requests in 15min time window
+        "message": "Too many votes, please try again later",
+        "statusCode": 429
+      },
+      "view": {
+        "windowMs": 900000, // 15 minutes
+        "max": 20, // 20 requests in 15min time window
+        "statusCode": 200
+      }
+    },
+    "maxRewardTimePerSegmentInSeconds": 86400, // maximum time a user get rewarded in the leaderboard for a single segment
+    "dumpDatabase": {
+        "enabled": true,
+        "minTimeBetweenMs": 60000, // 1 minute between dumps
+        "appExportPath": "./docker/database-export",
+        "postgresExportPath": "/opt/exports",
+        "tables": [{
+            "name": "sponsorTimes",
+            "order": "timeSubmitted"
+        },
+        {
+            "name": "userNames"
+        },
+        {
+            "name": "categoryVotes"
+        },
+        {
+            "name": "lockCategories"
+        },
+        {
+            "name": "warnings",
+            "order": "issueTime"
+        },
+        {
+            "name": "vipUsers"
+        }]
+    }
+}

containers/backup-db/Dockerfile

@@ -0,0 +1,13 @@
+FROM alpine
+RUN apk add postgresql-client
+RUN apk add restic --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community/
+
+COPY ./backup.sh /usr/src/app/backup.sh
+RUN chmod +x /usr/src/app/backup.sh
+COPY ./backup.sh /usr/src/app/forget.sh
+RUN chmod +x /usr/src/app/forget.sh
+
+RUN echo '30  *  *  *  *    /usr/src/app/backup.sh' >> /etc/crontabs/root
+RUN echo '10  0  *  *  1    /usr/src/app/forget.sh' >> /etc/crontabs/root
+
+CMD crond -l 2 -f

containers/backup-db/backup.sh

@@ -0,0 +1,6 @@
+mkdir ./dump
+
+pg_dump -f ./dump/sponsorTimes.dump sponsorTimes
+pg_dump -f ./dump/privateDB.dump privateDB
+
+restic backup ./dump

containers/backup-db/forget.sh

@@ -0,0 +1 @@
+restic forget --prune --keep-last 48 --keep-daily 7 --keep-weekly 8

containers/rsync/Dockerfile

@@ -0,0 +1,6 @@
+FROM ghcr.io/ajayyy/sb-server:latest
+EXPOSE 873/tcp
+RUN apk add rsync>3.2.4-r0
+RUN mkdir /usr/src/app/database-export
+
+CMD rsync --no-detach --daemon & ./entrypoint.sh

databases/_private.db.sql

@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "votes" (
+	"UUID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"type"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "categoryVotes" (
+	"UUID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "sponsorTimes" (
+	"videoID"	TEXT NOT NULL,
+	"hashedIP"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "config" (
+	"key" TEXT NOT NULL,
+	"value" TEXT NOT NULL
+);
+
+COMMIT;

databases/_private_indexes.sql

@@ -0,0 +1,26 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_v4"
+    ON public."sponsorTimes" USING btree
+    ("videoID" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST);
+
+-- votes
+
+CREATE INDEX IF NOT EXISTS "votes_userID"
+    ON public.votes USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
+    ON public."categoryVotes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- ratings
+
+CREATE INDEX IF NOT EXISTS "ratings_videoID"
+    ON public."ratings" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;

databases/_sponsorTimes.db.sql

@@ -0,0 +1,43 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "vipUsers" (
+	"userID"	TEXT NOT NULL
+);
+
+COMMIT;
+
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "sponsorTimes" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category" TEXT NOT NULL,
+	"shadowHidden"	INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "userNames" (
+	"userID"	TEXT NOT NULL,
+	"userName"	TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS "categoryVotes" (
+	"UUID"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL,
+	"votes"	INTEGER NOT NULL default 0
+);
+
+CREATE TABLE IF NOT EXISTS "config" (
+	"key" TEXT NOT NULL UNIQUE,
+	"value" TEXT NOT NULL
+);
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
+CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore
+
+COMMIT;

databases/_sponsorTimes_indexes.sql

@@ -0,0 +1,106 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted"
+    ON public."sponsorTimes" USING btree
+    ("timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_userID"
+    ON public."sponsorTimes" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
+    ON public."sponsorTimes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID"
+    ON public."sponsorTimes" USING btree
+    (service COLLATE pg_catalog."default" ASC NULLS LAST, "hashedVideoID" text_pattern_ops ASC NULLS LAST, "startTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"
+    ON public."sponsorTimes" USING btree
+    (service COLLATE pg_catalog."default" ASC NULLS LAST, "videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "startTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_description_gin"
+    ON public."sponsorTimes" USING gin
+    ("description" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
+    TABLESPACE pg_default;
+
+-- userNames
+
+CREATE INDEX IF NOT EXISTS "userNames_userID"
+    ON public."userNames" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- vipUsers
+
+CREATE INDEX IF NOT EXISTS "vipUsers_index"
+    ON public."vipUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- warnings
+
+CREATE INDEX IF NOT EXISTS "warnings_index"
+    ON public.warnings USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "warnings_issueTime"
+    ON public.warnings USING btree
+    ("issueTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- lockCategories
+
+CREATE INDEX IF NOT EXISTS "lockCategories_videoID"
+    ON public."lockCategories" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
+    ON public."categoryVotes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- shadowBannedUsers
+
+CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
+    ON public."shadowBannedUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- videoInfo
+CREATE INDEX IF NOT EXISTS "videoInfo_videoID"
+    ON public."videoInfo" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "videoInfo_channelID"
+    ON public."videoInfo" USING btree
+    ("channelID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- ratings
+
+CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID_gin"
+    ON public."ratings" USING gin
+    ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID"
+    ON public."ratings" USING btree
+    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "ratings_videoID"
+    ON public."ratings" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;

databases/_upgrade_private_1.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* for testing the db upgrade, don't remove because it looks empty */
+
+/* Add version to config */
+INSERT INTO config (key, value) VALUES('version', 1);
+
+COMMIT;

databases/_upgrade_private_2.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "userNameLogs" (
+	"userID"	TEXT NOT NULL,
+	"newUserName"	TEXT NOT NULL,
+	"oldUserName"	TEXT NOT NULL,
+	"updatedByAdmin"	BOOLEAN NOT NULL,
+	"updatedAt"	INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 2 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_3.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "sponsorTimes" ADD "service" TEXT NOT NULL default 'YouTube';
+-- UPDATE "sponsorTimes" SET "service" = "YouTube";
+
+DROP INDEX IF EXISTS "privateDB_sponsorTimes_videoID";
+
+UPDATE "config" SET value = 3 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_4.sql

@@ -0,0 +1,14 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "ratings" (
+	"videoID"		TEXT NOT NULL,
+	"service"		TEXT NOT NULL default 'YouTube',
+	"type"			INTEGER NOT NULL,
+	"userID"		TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"hashedIP"		TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 4 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_5.sql

@@ -0,0 +1,12 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "tempVipLog" (
+	"issuerUserID"	TEXT NOT NULL,
+	"targetUserID"	TEXT NOT NULL,
+	"enabled"				BOOLEAN NOT NULL,
+	"updatedAt"			INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 5 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_6.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+DROP INDEX IF EXISTS "sponsorTimes_hashedIP", "privateDB_sponsorTimes_videoID_v2"; --!sqlite-ignore
+
+UPDATE "config" SET value = 6 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_7.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "votes" ADD "normalUserID" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_8.sql

@@ -0,0 +1,15 @@
+BEGIN TRANSACTION;
+
+-- Add primary keys
+
+ALTER TABLE "userNameLogs" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "categoryVotes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "sponsorTimes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "config" ADD PRIMARY KEY ("key"); --!sqlite-ignore
+ALTER TABLE "ratings" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "tempVipLog" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "votes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_9.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+-- Add primary keys
+
+DROP INDEX IF EXISTS "privateDB_sponsorTimes_v3"; --!sqlite-ignore
+
+UPDATE "config" SET value = 9 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_1.sql

@@ -0,0 +1,25 @@
+BEGIN TRANSACTION;
+
+/* Add incorrectVotes field */
+CREATE TABLE "sqlb_temp_table_1" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"incorrectVotes" INTEGER NOT NULL default 1,
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"shadowHidden"	INTEGER NOT NULL
+);
+INSERT INTO sqlb_temp_table_1 SELECT "videoID","startTime","endTime","votes",'1',"UUID","userID","timeSubmitted","views","category","shadowHidden" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
+
+/* Add version to config */
+INSERT INTO config (key, value) VALUES('version', 1);
+
+COMMIT;

databases/_upgrade_sponsorTimes_10.sql

@@ -0,0 +1,30 @@
+BEGIN TRANSACTION;
+
+/* Add Hidden field */
+CREATE TABLE "sqlb_temp_table_10" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"hidden"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 10 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_11.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* Rename table: noSegments to lockCategories */
+ALTER TABLE "noSegments" RENAME TO "lockCategories";
+
+UPDATE "config" SET value = 11 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_12.sql

@@ -0,0 +1,31 @@
+BEGIN TRANSACTION;
+
+/* Add reputation field */
+CREATE TABLE "sqlb_temp_table_12" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"hidden"	INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT 0,
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 12 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_13.sql

@@ -0,0 +1,17 @@
+BEGIN TRANSACTION;
+
+/* Add locked field */
+CREATE TABLE "sqlb_temp_table_13" (
+	"userID"	TEXT NOT NULL,
+	"userName"	TEXT NOT NULL,
+	"locked" INTEGER NOT NULL default '0'
+);
+
+INSERT INTO sqlb_temp_table_13 SELECT "userID", "userName", 0 FROM "userNames";
+
+DROP TABLE "userNames";
+ALTER TABLE sqlb_temp_table_13 RENAME TO "userNames";
+
+UPDATE "config" SET value = 13 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_14.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
+	"userID"	TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 14 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_15.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "unlistedVideos" (
+	"videoID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 15 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_16.sql

@@ -0,0 +1,15 @@
+BEGIN TRANSACTION;
+
+DROP TABLE "unlistedVideos";
+
+CREATE TABLE IF NOT EXISTS "unlistedVideos" (
+	"videoID"	TEXT NOT NULL,
+	"year"	TEXT NOT NULL,
+	"views"	TEXT NOT NULL,
+	"channelID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 16 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_17.sql

@@ -0,0 +1,19 @@
+BEGIN TRANSACTION;
+
+/* Add reason field */
+CREATE TABLE "sqlb_temp_table_17" (
+	"userID" TEXT NOT NULL,
+	"issueTime" INTEGER NOT NULL,
+	"issuerUserID" TEXT NOT NULL,
+	enabled INTEGER NOT NULL,
+	"reason" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_17 SELECT "userID","issueTime","issuerUserID","enabled", '' FROM "warnings";
+
+DROP TABLE warnings;
+ALTER TABLE sqlb_temp_table_17 RENAME TO "warnings";
+
+UPDATE "config" SET value = 17 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_18.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "lockCategories" ADD "hashedVideoID" TEXT NOT NULL default '';
+UPDATE "lockCategories" SET "hashedVideoID" = sha256("videoID");
+
+UPDATE "config" SET value = 18 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_19.sql

@@ -0,0 +1,32 @@
+BEGIN TRANSACTION;
+
+/* Add actionType field */
+CREATE TABLE "sqlb_temp_table_19" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"actionType"	TEXT NOT NULL DEFAULT 'skip',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"hidden"	INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT 0,
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_19 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'skip',"service","videoDuration","hidden","reputation","shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_19 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 19 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_2.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+/* Add new table: noSegments */
+CREATE TABLE "noSegments" (
+	"videoID"	TEXT NOT NULL,
+	"userID"	TEXT NOT NULL,
+	"category"	TEXT NOT NULL
+);
+
+/* Add version to config */
+UPDATE "config" SET value = 2 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_20.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "lockCategories" ADD "reason" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 20 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_21.sql

@@ -0,0 +1,26 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" (
+	"videoID" TEXT NOT NULL,
+	"startTime" REAL NOT NULL,
+	"endTime" REAL NOT NULL,
+	"votes" INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL DEFAULT '0',
+	"incorrectVotes" INTEGER NOT NULL DEFAULT 1,
+	"UUID" TEXT NOT NULL UNIQUE,
+	"userID" TEXT NOT NULL,
+	"timeSubmitted" INTEGER NOT NULL,
+	"views" INTEGER NOT NULL,
+	"category" TEXT NOT NULL DEFAULT 'sponsor',
+	"service" TEXT NOT NULL DEFAULT 'Youtube',
+	"actionType" TEXT NOT NULL DEFAULT 'skip',
+	"videoDuration" INTEGER NOT NULL DEFAULT '0',
+	"hidden" INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT '0',
+	"shadowHidden" INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL DEFAULT ''
+);
+
+UPDATE "config" SET value = 21 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_22.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "sponsorTimes" ADD "userAgent" TEXT NOT NULL default '';
+
+ALTER TABLE "archivedSponsorTimes" ADD "userAgent" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 22 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_23.sql

@@ -0,0 +1,12 @@
+BEGIN TRANSACTION;
+
+DELETE FROM "userNames" WHERE ctid NOT IN ( --!sqlite-ignore
+    SELECT MIN(ctid) FROM "userNames"       --!sqlite-ignore
+    GROUP BY "userID"                       --!sqlite-ignore
+);                                          --!sqlite-ignore
+
+ALTER TABLE "userNames" ADD UNIQUE("userID");   --!sqlite-ignore
+
+UPDATE "config" SET value = 23 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_24.sql

@@ -0,0 +1,21 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "lockCategories" ADD "service" TEXT NOT NULL default 'YouTube';
+
+UPDATE "lockCategories" 
+SET "service" = "sponsorTimes"."service" 
+FROM "sponsorTimes" 
+WHERE "lockCategories"."videoID" = "sponsorTimes"."videoID";
+
+ALTER TABLE "unlistedVideos" ADD "service" TEXT NOT NULL default 'YouTube';
+
+UPDATE "unlistedVideos" 
+SET "service" = "sponsorTimes"."service" 
+FROM "sponsorTimes"
+WHERE "unlistedVideos"."videoID" = "sponsorTimes"."videoID";
+
+DROP INDEX IF EXISTS "noSegments_videoID";
+
+UPDATE "config" SET value = 24 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_25.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "videoInfo" (
+	"videoID"	TEXT PRIMARY KEY NOT NULL,
+	"channelID"	TEXT NOT NULL,
+	"title"	    TEXT NOT NULL,
+	"published"	REAL NOT NULL,
+	"genreUrl"	REAL NOT NULL
+);
+
+UPDATE "config" SET value = 25 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_26.sql

@@ -0,0 +1,18 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "sqlb_temp_table_26" (
+	"videoID"	TEXT PRIMARY KEY NOT NULL,
+	"channelID"	TEXT NOT NULL,
+	"title"	    TEXT NOT NULL,
+	"published"	REAL NOT NULL,
+	"genreUrl"	TEXT NOT NULL
+);
+
+INSERT INTO sqlb_temp_table_26 SELECT "videoID", "channelID", "title", "published", '' FROM "videoInfo";
+
+DROP TABLE "videoInfo";
+ALTER TABLE sqlb_temp_table_26 RENAME TO "videoInfo";
+
+UPDATE "config" SET value = 26 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_27.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "sponsorTimes" ADD "description" TEXT NOT NULL default '';
+ALTER TABLE "archivedSponsorTimes" ADD "description" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 27 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_28.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "ratings" (
+	"videoID"	    TEXT NOT NULL,
+	"service"	    TEXT NOT NULL default 'YouTube',
+	"type"	        INTEGER NOT NULL,
+	"count"	        INTEGER NOT NULL,
+	"hashedVideoID"	TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 28 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_29.sql

@@ -0,0 +1,21 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "sqlb_temp_table_29" (
+	"videoID"		TEXT NOT NULL,
+	"userID"		TEXT NOT NULL,
+	"actionType"	TEXT NOT NULL DEFAULT 'skip',
+	"category"		TEXT NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default '',
+	"reason" 		TEXT NOT NULL default '',
+	"service"		TEXT NOT NULL default 'YouTube'
+);
+
+INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'skip',"category","hashedVideoID","reason","service" FROM "lockCategories";
+INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'mute',"category","hashedVideoID","reason","service" FROM "lockCategories";
+
+DROP TABLE "lockCategories";
+ALTER TABLE sqlb_temp_table_29 RENAME TO "lockCategories";
+
+UPDATE "config" SET value = 29 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_3.sql

@@ -0,0 +1,11 @@
+BEGIN TRANSACTION;
+
+/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */
+/* Add hash field */
+ALTER TABLE "sponsorTimes" ADD "hashedVideoID" TEXT NOT NULL default '';
+UPDATE "sponsorTimes" SET "hashedVideoID" = sha256("videoID");
+
+/* Bump version in config */
+UPDATE "config" SET value = 3 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_30.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+UPDATE "sponsorTimes" SET "actionType" = 'poi' WHERE "category" = 'poi_highlight';
+
+UPDATE "config" SET value = 30 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_31.sql

@@ -0,0 +1,25 @@
+BEGIN TRANSACTION;
+
+/* START lockCategory migrations
+no sponsor migrations
+no selfpromo migrations */
+
+/* exclusive_access migrations */
+DELETE FROM "lockCategories" WHERE "category" = 'exclusive_access' AND "actionType" != 'full';
+/* delete all full locks on categories without full */
+DELETE FROM "lockCategories" WHERE "actionType" = 'full' AND "category" in ('interaction', 'intro', 'outro', 'preview', 'filler', 'music_offtopic', 'poi_highlight');
+/* delete all non-skip music_offtopic locks */
+DELETE FROM "lockCategories" WHERE "category" = 'music_offtopic' AND "actionType" != 'skip';
+/* convert all poi_highlight to actionType poi */
+UPDATE "lockCategories" SET "actionType" = 'poi' WHERE "category" = 'poi_highlight' AND "actionType" = 'skip';
+/* delete all non-skip poi_highlight locks */
+DELETE FROM "lockCategories" WHERE "category" = 'poi_highlight' AND "actionType" != 'poi';
+
+/* END lockCategory migrations */
+
+/* delete all redundant userName entries */
+DELETE FROM "userNames" WHERE "userName" = "userID" AND "locked" = 0;
+
+UPDATE "config" SET value = 31 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_32.sql

@@ -0,0 +1,19 @@
+BEGIN TRANSACTION;
+
+-- Add primary keys
+
+ALTER TABLE "sponsorTimes" ADD PRIMARY KEY ("UUID"); --!sqlite-ignore
+ALTER TABLE "vipUsers" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
+ALTER TABLE "userNames" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
+ALTER TABLE "categoryVotes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "lockCategories" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "warnings" ADD PRIMARY KEY ("userID", "issueTime"); --!sqlite-ignore
+ALTER TABLE "shadowBannedUsers" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
+ALTER TABLE "unlistedVideos" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "config" ADD PRIMARY KEY ("key"); --!sqlite-ignore
+ALTER TABLE "archivedSponsorTimes" ADD PRIMARY KEY ("UUID"); --!sqlite-ignore
+ALTER TABLE "ratings" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+
+UPDATE "config" SET value = 32 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_4.sql

@@ -0,0 +1,12 @@
+BEGIN TRANSACTION;
+
+/* Create warnings table */
+CREATE TABLE "warnings" (
+    "userID" TEXT NOT NULL,
+    "issueTime" INTEGER NOT NULL,
+    "issuerUserID" TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 4 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_5.sql

@@ -0,0 +1,17 @@
+BEGIN TRANSACTION;
+
+/* Add enabled field */
+CREATE TABLE "sqlb_temp_table_5" (
+	"userID" TEXT NOT NULL,
+    "issueTime" INTEGER NOT NULL,
+    "issuerUserID" TEXT NOT NULL,
+    enabled INTEGER NOT NULL
+);
+INSERT INTO sqlb_temp_table_5 SELECT "userID","issueTime","issuerUserID",1 FROM "warnings";
+
+DROP TABLE warnings;
+ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";;
+
+UPDATE "config" SET value = 5 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_6.sql

@@ -0,0 +1,27 @@
+BEGIN TRANSACTION;
+
+/* Add new voting field */
+CREATE TABLE "sqlb_temp_table_6" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_6 SELECT "videoID","startTime","endTime","votes",'0',"incorrectVotes","UUID","userID","timeSubmitted","views","category","shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_6 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 6 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_7.sql

@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_7" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_8.sql

@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_8" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_9.sql

@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_9" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 9 WHERE key = 'version';
+
+COMMIT;

docker/database.env.example

@@ -0,0 +1,3 @@
+POSTGRES_USER=sponsorblock
+POSTGRES_PASSWORD=<choose-a-password>
+POSTGRES_DB=sponsorTimes

docker/docker-compose-ci.yml

@@ -0,0 +1,13 @@
+version: '3'
+services:
+  postgres:
+    image: postgres:alpine
+    environment:
+      - POSTGRES_USER=${PG_USER}
+      - POSTGRES_PASSWORD=${PG_PASS}
+    ports:
+      - 5432:5432
+  redis:
+    image: redis:alpine
+    ports:
+      - 6379:6379

docker/docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3'
+services:
+  database:
+    container_name: database
+    image: postgres:13
+    env_file: 
+      - database.env
+    volumes:
+      - database-data:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+    restart: always
+  redis:
+    container_name: redis
+    image: redis:6.0
+    command: /usr/local/etc/redis/redis.conf
+    volumes:
+      - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
+    ports:
+      - 32773:6379
+    restart: always
+  newleaf:
+    image: abeltramo/newleaf:latest
+    container_name: newleaf
+    restart: always
+    ports:
+      - 3241:3000
+    volumes:
+      - ./newleaf/configuration.py:/workdir/configuration.py
+  rsync:
+    image: mchangrh/rsync:latest
+    container_name: rsync
+    restart: always
+    ports:
+      - 873:873
+    volumes:
+      - ./rsync/rsyncd.conf:/etc/rsyncd.conf
+      - ./database-export/:/mirror
+
+volumes:
+  database-data:

docker/migrate/database.pgload.txt

@@ -0,0 +1,5 @@
+load database
+     from ./database.db
+     into postgresql://sponsorblock:pw@127.0.0.1:5432/sponsorTimes
+
+with quote identifiers;

docker/newleaf/configuration.py

@@ -0,0 +1,17 @@
+# ==============================
+#  You MUST set these settings.
+# ==============================
+
+# A URL that this site can be accessed on. Do not include a trailing slash.
+website_origin = "http://newleaf:3000"
+
+
+# ==============================
+#  These settings are optional.
+# ==============================
+
+# The address of the interface to bind to.
+#bind_host = "0.0.0.0"
+
+# The port to bind to.
+#bind_port = 3000

docker/redis/redis.conf

@@ -0,0 +1,4 @@
+maxmemory-policy allkeys-lru
+maxmemory 6500mb
+
+appendonly no

docker/rsync/rsyncd.conf

@@ -0,0 +1,15 @@
+pid file = /var/run/rsyncd.pid
+lock file = /var/run/rsync.lock
+log file = /var/log/rsync.log
+# replace with user accessing the files
+
+[sponsorblock]
+use chroot = no
+max connections = 10
+# path to mirrored files
+path = /mirror
+comment = sponsorblock-database
+read only = true
+refuse options = c delete zl
+# disallow checksumming and compression level to reduce CPU/IO load
+# disallow deleting files clientside

entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+echo 'Entrypoint script'
+cd /usr/src/app
+
+# blank config, use defaults
+cat <<EOF > config.json
+{
+}
+EOF
+
+node dist/src/index.js

index.js

@@ -1,359 +0,0 @@
-var express = require('express');
-var http = require('http');
-
-// Create a service (the app object is just a callback).
-var app = express();
-
-//uuid service
-var uuidv1 = require('uuid/v1');
-
-//hashing service
-var crypto = require('crypto');
-
-//load database
-var sqlite3 = require('sqlite3').verbose();
-var db = new sqlite3.Database('./databases/sponsorTimes.db');
-
-// Create an HTTP service.
-http.createServer(app).listen(80);
-
-//global salt that is added to every ip before hashing to
-//  make it even harder for someone to decode the ip
-var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb";
-
-//setup CORS correctly
-app.use(function(req, res, next) {
-    res.header("Access-Control-Allow-Origin", "*");
-    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-    next();
-});
-
-//add the get function
-app.get('/api/getVideoSponsorTimes', function (req, res) {
-    let videoID = req.query.videoID;
-
-    let sponsorTimes = [];
-    let votes = []
-    let UUIDs = [];
-
-    db.prepare("SELECT startTime, endTime, votes, UUID FROM sponsorTimes WHERE videoID = ?").all(videoID, function(err, rows) {
-        if (err) console.log(err);
-
-        for (let i = 0; i < rows.length; i++) {
-            //check if votes are above -2
-            if (rows[i].votes < -2) {
-                //too untrustworthy, just ignore it
-                continue;
-            }
-            sponsorTimes.push([]);
-            
-            let index = sponsorTimes.length - 1;
-    
-            sponsorTimes[index][0] = rows[i].startTime;
-            sponsorTimes[index][1] = rows[i].endTime;
-
-            votes[index] = rows[i].votes;
-            UUIDs[index] = rows[i].UUID;
-        }
-
-        if (sponsorTimes.length == 0) {
-            res.sendStatus(404);
-            return;
-        }
-
-        organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs);
-        sponsorTimes = organisedData.sponsorTimes;
-        UUIDs = organisedData.UUIDs;
-
-        if (sponsorTimes.length == 0) {
-            res.sendStatus(404);
-        } else {
-            //send result
-            res.send({
-                sponsorTimes: sponsorTimes,
-                UUIDs: UUIDs
-            })
-        }
-    });
-});
-
-//add the post function
-app.get('/api/postVideoSponsorTimes', function (req, res) {
-    let videoID = req.query.videoID;
-    let startTime = req.query.startTime;
-    let endTime = req.query.endTime;
-    let userID = req.query.userID;
-
-    if (typeof videoID != 'string' || startTime == undefined || endTime == undefined || userID == undefined) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-
-    //hash the ip so no one can get it from the database
-    let hashCreator = crypto.createHash('sha256');
-    let hashedIP = hashCreator.update(ip + globalSalt).digest('hex');
-
-    startTime = parseFloat(startTime);
-    endTime = parseFloat(endTime);
-
-    let UUID = uuidv1();
-
-    //get current time
-    let timeSubmitted = Date.now();
-
-    //check to see if the user has already submitted sponsors for this video
-    db.prepare("SELECT UUID FROM sponsorTimes WHERE userID = ? and videoID = ?").all([userID, videoID], function(err, rows) {
-        if (rows.length >= 4) {
-            //too many sponsors for the same video from the same user
-            res.sendStatus(429);
-        } else {
-            //check if this info has already been submitted first
-            db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID], function(err, row) {
-                if (err) console.log(err);
-                
-                if (row == null) {
-                    //not a duplicate, execute query
-                    db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, 0, UUID, userID, hashedIP, timeSubmitted);
-
-                    res.sendStatus(200);
-                } else {
-                    res.sendStatus(409);
-                }
-            });
-        }
-    });
-});
-
-//voting endpoint
-app.get('/api/voteOnSponsorTime', function (req, res) {
-    let UUID = req.query.UUID;
-    let userID = req.query.userID;
-    let type = req.query.type;
-
-    if (UUID == undefined || userID == undefined || type == undefined) {
-        //invalid request
-        res.sendStatus(400);
-        return;
-    }
-
-    //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
-    let incrementAmount = 0;
-
-    //don't use userID for now, and just add the vote
-    if (type == 1) {
-        //upvote
-        incrementAmount = 1;
-    } else if (type == 0) {
-        //downvote
-        incrementAmount = -1;
-    } else {
-        //unrecongnised type of vote
-        req.sendStatus(400);
-        return;
-    }
-
-    db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount, UUID);
-
-    //added to db
-    res.sendStatus(200);
-});
-
-app.get('/database.db', function (req, res) {
-    res.sendFile("./databases/sponsorTimes.db", { root: __dirname });
-});
-
-
-//This function will find sponsor times that are contained inside of eachother, called similar sponsor times
-//Only one similar time will be returned, randomly generated based on the sqrt of votes.
-//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
-//Sponsor times with less than -2 votes are already ignored before this function is called
-function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
-    //list of sponsors that are contained inside eachother
-    let similarSponsors = [];
-
-    for (let i = 0; i < sponsorTimes.length; i++) {
-        //see if the start time is located between the start and end time of the other sponsor time.
-        for (let j = 0; j < sponsorTimes.length; j++) {
-            if (sponsorTimes[j][0] > sponsorTimes[i][0] && sponsorTimes[j][0] < sponsorTimes[i][1]) {
-                //sponsor j is contained in sponsor i
-                similarSponsors.push([i, j]);
-            }
-        }
-    }
-
-    let similarSponsorsGroups = [];
-    //once they have been added to a group, they don't need to be dealt with anymore
-    let dealtWithSimilarSponsors = [];
-
-    //create lists of all the similar groups (if 1 and 2 are similar, and 2 and 3 are similar, the group is 1, 2, 3)
-    for (let i = 0; i < similarSponsors.length; i++) {
-        if (dealtWithSimilarSponsors.includes(i)) {
-            //dealt with already
-            continue;
-        }
-
-        //this is the group of indexes that are similar
-        let group = similarSponsors[i];
-        for (let j = 0; j < similarSponsors.length; j++) {
-            if (group.includes(similarSponsors[j][0]) || group.includes(similarSponsors[j][1])) {
-                //this is a similar group
-                group.push(similarSponsors[j][0]);
-                group.push(similarSponsors[j][1]);
-                dealtWithSimilarSponsors.push(j);
-            }
-        }
-        similarSponsorsGroups.push(group);
-    }
-
-    //remove duplicate indexes in group arrays
-    for (let i = 0; i < similarSponsorsGroups.length; i++) {
-        uniqueArray = similarSponsorsGroups[i].filter(function(item, pos, self) {
-            return self.indexOf(item) == pos;
-        });
-
-        similarSponsorsGroups[i] = uniqueArray;
-    }
-
-    let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes);
-
-    let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices;
-    //the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added
-    let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith;
-
-    let voteSums = weightedRandomIndexes.weightSums;
-    //convert these into the votes
-    for (let i = 0; i < voteSums.length; i++) {
-        if (voteSums[i] != undefined) {
-            //it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor.
-            votes[finalSponsorTimeIndexes[i]] = voteSums;
-        }
-    }
-
-    //find the indexes never dealt with and add them
-    for (let i = 0; i < sponsorTimes.length; i++) {
-        if (!finalSponsorTimeIndexesDealtWith.includes(i)) {
-            finalSponsorTimeIndexes.push(i)
-        }
-    }
-
-    //if there are too many indexes, find the best 4
-    if (finalSponsorTimeIndexes.length > 4) {
-        finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 4).finalChoices;
-    }
-
-    //convert this to a final array to return
-    let finalSponsorTimes = [];
-    for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
-        finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]);
-    }
-
-    //convert this to a final array of UUIDs as well
-    let finalUUIDs = [];
-    for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
-        finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]);
-    }
-
-    return {
-        sponsorTimes: finalSponsorTimes,
-        UUIDs: finalUUIDs
-    };
-}
-
-//gets the getWeightedRandomChoice for each group in an array of groups
-function getWeightedRandomChoiceForArray(choiceGroups, weights) {
-    let finalChoices = [];
-    //the indexes either chosen to be added to final indexes or chosen not to be added
-    let choicesDealtWith = [];
-    //for each choice group, what are the sums of the weights
-    let weightSums = [];
-
-    for (let i = 0; i < choiceGroups.length; i++) {
-        //find weight sums for this group
-        weightSums.push(0);
-        for (let j = 0; j < choiceGroups[i].length; j++) {
-            //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
-            if (weights[choiceGroups[i][j]] > 0) {
-                weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]];
-            }
-        }
-
-        //create a random choice for this group
-        let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1)
-        finalChoices.push(randomChoice.finalChoices);
-
-        for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) {
-            choicesDealtWith.push(randomChoice.choicesDealtWith[j])
-        }
-    }
-
-    return {
-        finalChoices: finalChoices,
-        choicesDealtWith: choicesDealtWith,
-        weightSums: weightSums
-    };
-}
-
-//gets a weighted random choice from the indexes array based on the weights.
-//amountOfChoices speicifies the amount of choices to return, 1 or more.
-//choices are unique
-function getWeightedRandomChoice(choices, weights, amountOfChoices) {
-    if (amountOfChoices > choices.length) {
-        //not possible, since all choices must be unique
-        return null;
-    }
-
-    let finalChoices = [];
-    let choicesDealtWith = [];
-
-    let sqrtWeightsList = [];
-    //the total of all the weights run through the cutom sqrt function
-    let totalSqrtWeights = 0;
-    for (let j = 0; j < choices.length; j++) {
-        //multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
-        //The 3 makes -2 the minimum votes before being ignored completely
-        //https://www.desmos.com/calculator/ljftxolg9j
-        //this can be changed if this system increases in popularity.
-        let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10);
-        sqrtWeightsList.push(sqrtVote)
-        totalSqrtWeights += sqrtVote;
-
-        //this index has now been deat with
-        choicesDealtWith.push(choices[j]);
-    }
-
-    //iterate and find amountOfChoices choices
-    let randomNumber = Math.random();
-    //this array will keep adding to this variable each time one sqrt vote has been dealt with
-    //this is the sum of all the sqrtVotes under this index
-    let currentVoteNumber = 0;
-    for (let j = 0; j < sqrtWeightsList.length; j++) {
-        if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) {
-            //this one was randomly generated
-            finalChoices.push(choices[j]);
-            //remove that from original array, for next recursion pass if it happens
-            choices.splice(j, 1);
-            break;
-        }
-
-        //add on to the count
-        currentVoteNumber += sqrtWeightsList[j];
-    }
-    
-    //add on the other choices as well using recursion
-    if (amountOfChoices > 1) {
-        let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices;
-        //add all these choices to the finalChoices array being returned
-        for (let i = 0; i < otherChoices.length; i++) {
-            finalChoices.push(otherChoices[i]);
-        }
-    }
-
-    return {
-        finalChoices: finalChoices,
-        choicesDealtWith: choicesDealtWith
-    };
-}

nginx/cors.conf

@@ -0,0 +1,11 @@
+if ($request_method = 'OPTIONS') {
+  add_header 'Access-Control-Allow-Origin' '*';
+  add_header 'Access-Control-Allow-Methods' 'GET, POST, DELETE';
+  add_header 'Access-Control-Allow-Headers' 'Content-Type';
+  # cache CORS for 24 hours
+  add_header 'Access-Control-Max-Age' 86400;
+  # return empty response for preflight
+  add_header 'Content-Type' 'text/plain; charset=UTF-8';
+  add_header 'Content-Length' 0;
+  return 204;
+}

nginx/error.conf

@@ -0,0 +1,7 @@
+error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html;
+
+location = /error.html {
+  ssi on;
+  internal;
+  root /etc/nginx/error;
+}

nginx/error/error.html

@@ -0,0 +1 @@
+<!--# echo var="status"--> <!--# echo var="status_text"--> https://status.sponsor.ajay.app

nginx/error_map.conf

@@ -0,0 +1,15 @@
+map $status $status_text {
+  400 'Bad Request';
+  401 'Unauthorized';
+  403 'Forbidden';
+  404 'Not Found';
+  405 'Method Not Allowed';
+  408 'Request Timeout';
+  409 'Conflict';
+  429 'Too Many Requests';
+  500 'Internal Server Error';
+  502 'Bad Gateway';
+  503 'Service Unavailable';
+  504 'Gateway Timeout';
+  505 'HTTP Version Not Supported';
+}

nginx/nginx.conf

@@ -0,0 +1,317 @@
+worker_processes 2;
+worker_rlimit_nofile 500000;
+worker_shutdown_timeout 10;
+
+events {
+	worker_connections 100000; # Default: 1024
+	#use epoll;
+	#multi_accept on;
+}
+
+http {
+	log_format no_ip    '$remote_user [$time_local] '
+                        '"$request" $status $body_bytes_sent '
+                        '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+    log_format user_agent '[$time_local] '
+                           '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+	#limit_req_zone $binary_remote_addr zone=mylimit:10m rate=10r/s;
+	limit_req_log_level warn;
+	
+	include		/etc/nginx/mime.types;
+	include		/etc/nginx/proxy.conf;
+	
+	# error_map has to be at http level
+	include	  /etc/nginx/error_map.conf;
+	# Custom MIME definition
+	types {
+		text/csv	csv;
+	}
+	# keepalive settings
+	#keepalive_requests		10;
+	keepalive_timeout		10s;
+	http2_idle_timeout		20s; # replaced by keepalive_timeout in 1.19.7	
+
+	access_log off;
+	#error_log /etc/nginx/logs/error.log warn;
+	error_log /dev/null crit;
+
+	upstream backend_GET {
+		least_conn;
+		
+		#keepalive 	5;
+		#server		localhost:4441;
+		#server		localhost:4442;
+		#server		localhost:4443;
+		#server		localhost:4444;
+		#server		localhost:4445;
+		#server		localhost:4446;
+		#server		localhost:4447;
+		#server		localhost:4448;
+		#server 		10.0.0.4:4441 max_fails=25 fail_timeout=20s;
+
+		#server		10.0.0.3:4441 max_fails=25 fail_timeout=20s;
+		#server		10.0.0.3:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.5:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.5:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.6:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.6:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.9:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.9:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.12:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.12:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.10:4441 max_fails=25 fail_timeout=20s;
+        server		10.0.0.10:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.13:4441 max_fails=25 fail_timeout=20s;
+        server		10.0.0.13:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.14:4441 max_fails=25 fail_timeout=20s;
+        server		10.0.0.14:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.11:4441 max_fails=25 fail_timeout=20s;
+        	server		10.0.0.11:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.16:4441 max_fails=25 fail_timeout=20s;
+        	server		10.0.0.16:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.17:4441 max_fails=25 fail_timeout=20s;
+        	server		10.0.0.17:4442 max_fails=25 fail_timeout=20s;
+
+		#server		134.209.69.251:80	backup;
+
+		#server		116.203.32.253:80	backup;
+		#server		116.203.32.253:80;
+	}
+	upstream backend_POST {
+		#server		localhost:4441;
+		#server		localhost:4442;
+		server		10.0.0.3:4441 max_fails=25 fail_timeout=15s;
+		server		10.0.0.4:4441 max_fails=25 fail_timeout=15s;
+		#server		10.0.0.3:4442;
+	}
+	upstream backend_db {
+		server		10.0.0.4:4441 max_fails=1 fail_timeout=3s;
+		#server		10.0.0.3:4441;
+		#server		10.0.0.4; 
+	}
+	upstream backend_db_dl {
+		server		10.0.0.4;
+	}
+
+	proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
+	proxy_cache_key "$scheme$request_method$host$request_uri";
+	add_header X-Cache $upstream_cache_status;
+
+	server {
+		server_name sponsor.ajay.app api.sponsor.ajay.app;
+
+		include		  /etc/nginx/error.conf;
+		set_real_ip_from 10.0.0.0/24;
+		real_ip_header proxy_protocol;
+
+		location /news {
+			return	301		https://blog.ajay.app/sponsorblock;
+		}
+
+		location /viewer {
+			return	301		https://sb.ltn.fi;
+		}
+
+		location /test/ {
+			# return			404 "";
+			proxy_pass		http://10.0.0.4:4445/;
+			#proxy_pass		https://sbtest.etcinit.com/;
+		}
+
+		#access_log /etc/nginx/logs/requests.log no_ip buffer=64k;
+
+		location /api/skipSegments {
+			include				/etc/nginx/cors.conf;
+			#return 200 "[]";
+			proxy_pass		http://backend_$request_method;
+			#proxy_cache	CACHEZONE;
+			#proxy_cache_valid	10s;
+			#limit_req zone=mylimit;
+
+			#access_log /etc/nginx/logs/download.log no_ip;
+			gzip on;
+			if ($request_method = POST) {
+				access_log /etc/nginx/logs/submissions.log user_agent buffer=64k;
+			}
+
+			#proxy_read_timeout 6s;
+			#proxy_next_upstream error timeout http_500 http_502;
+		}
+
+		location /api/getTopUsers {
+			include				/etc/nginx/cors.conf;
+			proxy_pass		http://backend_GET;
+			proxy_cache		CACHEZONE;
+			proxy_cache_valid	20m;
+		}
+
+		location /api/getTotalStats {
+			include		 /etc/nginx/cors.conf;
+			proxy_pass http://backend_POST;
+			proxy_cache CACHEZONE;
+			proxy_cache_valid 20m;
+			#return 204;
+		}
+
+		location /api/getTopCategoryUsers {
+			include				/etc/nginx/cors.conf;
+			proxy_pass		http://backend_POST;
+			proxy_cache		CACHEZONE;
+			proxy_cache_valid	20m;
+		}
+
+		location /api/getVideoSponsorTimes {
+			include		/etc/nginx/cors.conf;
+			proxy_pass http://backend_GET;
+		}
+		
+		location /api/isUserVIP {
+			include		/etc/nginx/cors.conf;
+			proxy_pass http://backend_GET;
+		}
+
+		location /download/ {
+			#access_log /etc/nginx/logs/download.log no_ip buffer=64k;
+			gzip on;
+			proxy_max_temp_file_size 0;
+			#proxy_cache CACHEZONE;
+			#proxy_cache_valid 20m;
+			#proxy_http_version 1.0;
+			#gzip_types	text/csv;
+			#gzip_comp_level 1;
+			#proxy_buffering off;		
+	
+
+			proxy_pass http://backend_db;
+      #alias		/home/sbadmin/sponsor/docker/database-export/;
+			#return 307 https://rsync.sponsor.ajay.app$request_uri;
+		}
+
+		location /database {
+			proxy_pass  http://backend_db;
+			#return 200 "Disabled for load reasons";
+		}
+
+		location = /database.db {
+			return	404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
+			#alias	/home/sbadmin/sponsor/databases/sponsorTimes.db;
+			#alias	/home/sbadmin/test-db/database.db;
+		}
+
+		#location = /database/sponsorTimes.csv {
+		#	alias /home/sbadmin/sponsorTimes.csv;
+		#}
+
+		#location /api/voteOnSponsorTime {
+		#	return 200 "Success";
+		#}
+	
+		#location /api/viewedVideoSponsorTime {
+		#	return 200 "Success";
+		#}
+
+		location /api {
+			include			/etc/nginx/cors.conf;
+			proxy_pass	http://backend_POST;
+		}
+		
+		location / {
+			root	/home/sbadmin/SponsorBlockSite/public-prod;
+			error_page 404 /404.html;
+		}
+		
+		listen [::]:443 default_server ssl http2 ipv6only=on backlog=323999;
+		listen 443 default_server ssl http2 reuseport backlog=3000999; # managed by Certbot
+		listen 4443 default_server ssl http2 proxy_protocol reuseport backlog=3000999;
+		#listen 443 http3 reuseport;
+		#ssl_protocols TLSv1.2 TLSv1.3;
+		listen 8081 proxy_protocol;
+		port_in_redirect off;
+    		ssl_certificate /home/sbadmin/certs/cert.pem;
+   		ssl_certificate_key /home/sbadmin/certs/key.pem;
+		#ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+		#ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+		include				/etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+		ssl_dhparam			/etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+	}
+
+	server {
+		server_name cdnsponsor.ajay.app;
+		error_page 404 /404.html;
+
+		#location /database/ {
+		#	alias /home/sbadmin/sponsor/docker/database-export/;
+		#}
+
+		#location /download/ {
+		#	alias /home/sbadmin/sponsor/docker/database-export/;
+		#}
+
+		location / {
+			root /home/sbadmin/SponsorBlockSite/public-prod;
+		}
+
+
+		listen 443 ssl; # managed by Certbot
+		ssl_certificate /home/sbadmin/certs/cert.pem;
+                ssl_certificate_key /home/sbadmin/certs/key.pem;
+    		#ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+    		#ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+		include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+		ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+	}
+
+	server {
+		access_log off;
+
+		return 301 https://$host$request_uri;
+
+		listen [::]:80 ipv6only=on;
+		listen 8080 proxy_protocol;
+		listen 80;
+		server_name sponsor.ajay.app api.sponsor.ajay.app, cdnsponsor.ajay.app, wiki.sponsor.ajay.app;
+		return 404; # managed by Certbot
+	}
+
+	server {
+		server_name wiki.sponsor.ajay.app; # managed by Certbot
+		
+		location /.well-known/ {
+			 root    /home/sbadmin/SponsorBlockSite/public-prod;
+		}
+
+		location ~* ^/index.php/(?<pagename>.*)$ {
+			return 301 /w/$pagename;
+		}
+
+		location / {
+			proxy_pass http://10.0.0.3:8080;
+		}
+		
+		port_in_redirect off;
+		listen [::]:443 ssl http2; 
+		listen 443  ssl http2; # managed by Certbot
+		listen 8081 proxy_protocol;
+		#listen 443 http3 reuseport;
+		#ssl_protocols TLSv1.2 TLSv1.3;
+		#listen 80;
+		ssl_certificate /home/sbadmin/certs/cert.pem;
+   		ssl_certificate_key /home/sbadmin/certs/key.pem;
+    		#ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+    		#ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+		include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+		ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+	}
+}

nginx/proxy.conf

@@ -0,0 +1,12 @@
+proxy_redirect            off;
+proxy_set_header          Host            $host;
+proxy_set_header          X-Real-IP       $remote_addr;
+proxy_set_header          X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header          Connection      "";
+client_max_body_size      10m;
+client_body_buffer_size   128k;
+proxy_connect_timeout     5s;
+#proxy_send_timeout        10;
+proxy_read_timeout        30s;
+proxy_buffers             32 4k;
+proxy_http_version        1.1;

nodemon.json

@@ -0,0 +1,5 @@
+{
+    "watch": ["src"],
+    "ext": "ts,json",
+    "exec": "(npm test || echo test failed) && npm start"
+}

package.json

@@ -2,17 +2,52 @@
   "name": "sponsor_block_server",
   "version": "0.1.0",
   "description": "Server that holds the SponsorBlock database",
-  "main": "index.js",
+  "main": "src/index.ts",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "node index.js"
+    "test": "npm run tsc && ts-node test/test.ts",
+    "dev": "nodemon",
+    "dev:bash": "nodemon -x 'npm test ; npm start'",
+    "postgres:docker": "docker run --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:alpine",
+    "redis:docker": "docker run --rm -p 6379:6379 redis:alpine",
+    "start": "ts-node src/index.ts",
+    "tsc": "tsc -p tsconfig.json",
+    "lint": "eslint src test",
+    "lint:fix": "eslint src test --fix"
   },
   "author": "Ajay Ramachandran",
   "license": "MIT",
   "dependencies": {
+    "axios": "^0.24.0",
+    "better-sqlite3": "^7.4.5",
+    "cron": "^1.8.2",
     "express": "^4.17.1",
-    "http": "0.0.0",
-    "sqlite3": "^4.0.9",
-    "uuid": "^3.3.2"
+    "express-promise-router": "^4.1.1",
+    "express-rate-limit": "^6.3.0",
+    "lodash": "^4.17.21",
+    "pg": "^8.7.1",
+    "rate-limit-redis": "^3.0.1",
+    "redis": "^4.0.6",
+    "sync-mysql": "^3.0.1"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.4.1",
+    "@types/cron": "^1.7.3",
+    "@types/express": "^4.17.13",
+    "@types/lodash": "^4.14.178",
+    "@types/mocha": "^9.0.0",
+    "@types/node": "^16.11.11",
+    "@types/pg": "^8.6.1",
+    "@typescript-eslint/eslint-plugin": "^5.5.0",
+    "@typescript-eslint/parser": "^5.5.0",
+    "eslint": "^8.3.0",
+    "mocha": "^9.1.3",
+    "nodemon": "^2.0.15",
+    "sinon": "^12.0.1",
+    "ts-mock-imports": "^1.3.8",
+    "ts-node": "^10.4.0",
+    "typescript": "^4.5.2"
+  },
+  "engines": {
+    "node": ">=10"
   }
 }

src/app.ts

@@ -0,0 +1,215 @@
+import express, { Request, RequestHandler, Response, Router } from "express";
+import { config } from "./config";
+import { oldSubmitSponsorTimes } from "./routes/oldSubmitSponsorTimes";
+import { oldGetVideoSponsorTimes } from "./routes/oldGetVideoSponsorTimes";
+import { postSegmentShift } from "./routes/postSegmentShift";
+import { postWarning } from "./routes/postWarning";
+import { getIsUserVIP } from "./routes/getIsUserVIP";
+import { deleteLockCategoriesEndpoint } from "./routes/deleteLockCategories";
+import { postLockCategories } from "./routes/postLockCategories";
+import { endpoint as getUserInfo } from "./routes/getUserInfo";
+import { getDaysSavedFormatted } from "./routes/getDaysSavedFormatted";
+import { getTotalStats } from "./routes/getTotalStats";
+import { getTopUsers } from "./routes/getTopUsers";
+import { getViewsForUser } from "./routes/getViewsForUser";
+import { getSavedTimeForUser } from "./routes/getSavedTimeForUser";
+import { addUserAsVIP } from "./routes/addUserAsVIP";
+import { shadowBanUser } from "./routes/shadowBanUser";
+import { getUsername } from "./routes/getUsername";
+import { setUsername } from "./routes/setUsername";
+import { viewedVideoSponsorTime } from "./routes/viewedVideoSponsorTime";
+import { voteOnSponsorTime, getUserID as voteGetUserID } from "./routes/voteOnSponsorTime";
+import { getSkipSegmentsByHash } from "./routes/getSkipSegmentsByHash";
+import { postSkipSegments } from "./routes/postSkipSegments";
+import { endpoint as getSkipSegments } from "./routes/getSkipSegments";
+import { userCounter } from "./middleware/userCounter";
+import { loggerMiddleware } from "./middleware/logger";
+import { corsMiddleware } from "./middleware/cors";
+import { apiCspMiddleware } from "./middleware/apiCsp";
+import { rateLimitMiddleware } from "./middleware/requestRateLimit";
+import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
+import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
+import { postClearCache } from "./routes/postClearCache";
+import { addUnlistedVideo } from "./routes/addUnlistedVideo";
+import { postPurgeAllSegments } from "./routes/postPurgeAllSegments";
+import { getUserID } from "./routes/getUserID";
+import { getLockCategories } from "./routes/getLockCategories";
+import { getLockCategoriesByHash } from "./routes/getLockCategoriesByHash";
+import { endpoint as getSearchSegments } from "./routes/getSearchSegments";
+import { getStatus } from "./routes/getStatus";
+import { getLockReason } from "./routes/getLockReason";
+import { getUserStats } from "./routes/getUserStats";
+import ExpressPromiseRouter from "express-promise-router";
+import { Server } from "http";
+import { youtubeApiProxy } from "./routes/youtubeApiProxy";
+import { getChapterNames } from "./routes/getChapterNames";
+import { postRating } from "./routes/ratings/postRating";
+import { getRating } from "./routes/ratings/getRating";
+import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache";
+import { getTopCategoryUsers } from "./routes/getTopCategoryUsers";
+import { addUserAsTempVIP } from "./routes/addUserAsTempVIP";
+
+export function createServer(callback: () => void): Server {
+    // Create a service (the app object is just a callback).
+    const app = express();
+
+    const router = ExpressPromiseRouter();
+    app.use(router);
+
+    //setup CORS correctly
+    router.use(corsMiddleware);
+    router.use(loggerMiddleware);
+    router.use("/api/", apiCspMiddleware);
+    router.use(express.json());
+
+    if (config.userCounterURL) router.use(userCounter);
+
+    // Setup pretty JSON
+    if (config.mode === "development") app.set("json spaces", 2);
+
+    // Set production mode
+    app.set("env", config.mode || "production");
+
+    setupRoutes(router);
+
+    return app.listen(config.port, callback);
+}
+
+function setupRoutes(router: Router) {
+    // Rate limit endpoint lists
+    const voteEndpoints: RequestHandler[] = [voteOnSponsorTime];
+    const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime];
+    const postRateEndpoints: RequestHandler[] = [postRating];
+    if (config.rateLimit) {
+        if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID));
+        if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
+        if (config.rateLimit.rate) postRateEndpoints.unshift(rateLimitMiddleware(config.rateLimit.rate));
+    }
+
+    //add the get function
+    router.get("/api/getVideoSponsorTimes", oldGetVideoSponsorTimes);
+
+    //add the oldpost function
+    router.get("/api/postVideoSponsorTimes", oldSubmitSponsorTimes);
+    router.post("/api/postVideoSponsorTimes", oldSubmitSponsorTimes);
+
+    //add the skip segments functions
+    router.get("/api/skipSegments", getSkipSegments);
+    router.post("/api/skipSegments", postSkipSegments);
+
+    // add the privacy protecting skip segments functions
+    router.get("/api/skipSegments/:prefix", getSkipSegmentsByHash);
+
+    //voting endpoint
+    router.get("/api/voteOnSponsorTime", ...voteEndpoints);
+    router.post("/api/voteOnSponsorTime", ...voteEndpoints);
+
+    //Endpoint when a submission is skipped
+    router.get("/api/viewedVideoSponsorTime", ...viewEndpoints);
+    router.post("/api/viewedVideoSponsorTime", ...viewEndpoints);
+
+    //To set your username for the stats view
+    router.post("/api/setUsername", setUsername);
+
+    //get what username this user has
+    router.get("/api/getUsername", getUsername);
+
+    //Endpoint used to hide a certain user's data
+    router.post("/api/shadowBanUser", shadowBanUser);
+
+    //Endpoint used to make a user a VIP user with special privileges
+    router.post("/api/addUserAsVIP", addUserAsVIP);
+    //Endpoint to add a user as a temporary VIP
+    router.post("/api/addUserAsTempVIP", addUserAsTempVIP);
+
+    //Gets all the views added up for one userID
+    //Useful to see how much one user has contributed
+    router.get("/api/getViewsForUser", getViewsForUser);
+
+    //Gets all the saved time added up (views * sponsor length) for one userID
+    //Useful to see how much one user has contributed
+    //In minutes
+    router.get("/api/getSavedTimeForUser", getSavedTimeForUser);
+
+    router.get("/api/getTopUsers", getTopUsers);
+    router.get("/api/getTopCategoryUsers", getTopCategoryUsers);
+
+    //send out totals
+    //send the total submissions, total views and total minutes saved
+    router.get("/api/getTotalStats", getTotalStats);
+
+    router.get("/api/getUserInfo", getUserInfo);
+    router.get("/api/userInfo", getUserInfo);
+
+    //send out a formatted time saved total
+    router.get("/api/getDaysSavedFormatted", getDaysSavedFormatted);
+
+    //submit video to lock categories
+    router.post("/api/noSegments", postLockCategories);
+    router.post("/api/lockCategories", postLockCategories);
+
+    router.delete("/api/noSegments", deleteLockCategoriesEndpoint);
+    router.delete("/api/lockCategories", deleteLockCategoriesEndpoint);
+
+    //get if user is a vip
+    router.get("/api/isUserVIP", getIsUserVIP);
+
+    //sent user a warning
+    router.post("/api/warnUser", postWarning);
+
+    //get if user is a vip
+    router.post("/api/segmentShift", postSegmentShift);
+
+    //get segment info
+    router.get("/api/segmentInfo", getSegmentInfo);
+
+    //clear cache as VIP
+    router.post("/api/clearCache", postClearCache);
+
+    //purge all segments for VIP
+    router.post("/api/purgeAllSegments", postPurgeAllSegments);
+
+    router.post("/api/unlistedVideo", addUnlistedVideo);
+
+    // get userID from username
+    router.get("/api/userID", getUserID);
+
+    // get lock categores from userID
+    router.get("/api/lockCategories", getLockCategories);
+
+    // get privacy protecting lock categories functions
+    router.get("/api/lockCategories/:prefix", getLockCategoriesByHash);
+
+    // get all segments that match a search
+    router.get("/api/searchSegments", getSearchSegments);
+
+    // autocomplete chapter names
+    router.get("/api/chapterNames", getChapterNames);
+
+    // get status
+    router.get("/api/status/:value", getStatus);
+    router.get("/api/status", getStatus);
+
+    router.get("/api/youtubeApiProxy", youtubeApiProxy);
+    // get user category stats
+    router.get("/api/userStats", getUserStats);
+
+    router.get("/api/lockReason", getLockReason);
+
+    // ratings
+    router.get("/api/ratings/rate/:prefix", getRating);
+    router.get("/api/ratings/rate", getRating);
+    router.post("/api/ratings/rate", postRateEndpoints);
+    router.post("/api/ratings/clearCache", ratingPostClearCache);
+
+    if (config.postgres?.enabled) {
+        router.get("/database", (req, res) => dumpDatabase(req, res, true));
+        router.get("/database.json", (req, res) => dumpDatabase(req, res, false));
+        router.get("/database/*", downloadFile);
+        router.use("/download", express.static(appExportPath));
+    } else {
+        router.get("/database.db", function (req: Request, res: Response) {
+            res.sendFile("./databases/sponsorTimes.db", { root: "./" });
+        });
+    }
+}

src/config.ts

@@ -0,0 +1,185 @@
+import fs from "fs";
+import { SBSConfig } from "./types/config.model";
+import packageJson from "../package.json";
+import { isBoolean, isNumber } from "lodash";
+
+const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test;
+const configFile = process.env.TEST_POSTGRES ? "ci.json"
+    : isTestMode ? "test.json"
+        : "config.json";
+export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8"));
+
+addDefaults(config, {
+    port: 8080,
+    behindProxy: "X-Forwarded-For",
+    db: "./databases/sponsorTimes.db",
+    privateDB: "./databases/private.db",
+    createDatabaseIfNotExist: true,
+    schemaFolder: "./databases",
+    dbSchema: "./databases/_sponsorTimes.db.sql",
+    privateDBSchema: "./databases/_private.db.sql",
+    readOnly: false,
+    webhooks: [],
+    categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"],
+    categorySupport: {
+        sponsor: ["skip", "mute", "full"],
+        selfpromo: ["skip", "mute", "full"],
+        exclusive_access: ["full"],
+        interaction: ["skip", "mute"],
+        intro: ["skip", "mute"],
+        outro: ["skip", "mute"],
+        preview: ["skip", "mute"],
+        filler: ["skip", "mute"],
+        music_offtopic: ["skip"],
+        poi_highlight: ["poi"],
+        chapter: ["chapter"]
+    },
+    maxNumberOfActiveWarnings: 1,
+    hoursAfterWarningExpires: 16300000,
+    adminUserID: "",
+    discordCompletelyIncorrectReportWebhookURL: null,
+    discordFirstTimeSubmissionsWebhookURL: null,
+    discordNeuralBlockRejectWebhookURL: null,
+    discordFailedReportChannelWebhookURL: null,
+    discordReportChannelWebhookURL: null,
+    getTopUsersCacheTimeMinutes: 240,
+    globalSalt: null,
+    mode: "",
+    neuralBlockURL: null,
+    proxySubmission: null,
+    rateLimit: {
+        vote: {
+            windowMs: 900000,
+            max: 15,
+            message: "OK",
+            statusCode: 200,
+        },
+        view: {
+            windowMs: 900000,
+            max: 10,
+            statusCode: 200,
+            message: "OK",
+        },
+        rate: {
+            windowMs: 900000,
+            max: 20,
+            statusCode: 200,
+            message: "Success",
+        }
+    },
+    userCounterURL: null,
+    newLeafURLs: null,
+    maxRewardTimePerSegmentInSeconds: 600,
+    poiMinimumStartTime: 2,
+    postgres: {
+        enabled: false,
+        user: "",
+        host: "",
+        password: "",
+        port: 5432,
+        max: 150,
+        min: 10
+    },
+    dumpDatabase: {
+        enabled: false,
+        minTimeBetweenMs: 180000,
+        appExportPath: "./docker/database-export",
+        tables: [{
+            name: "sponsorTimes",
+            order: "timeSubmitted"
+        },
+        {
+            name: "userNames"
+        },
+        {
+            name: "categoryVotes"
+        },
+        {
+            name: "lockCategories",
+        },
+        {
+            name: "warnings",
+            order: "issueTime"
+        },
+        {
+            name: "vipUsers"
+        },
+        {
+            name: "unlistedVideos"
+        },
+        {
+            name: "videoInfo"
+        },
+        {
+            name: "ratings"
+        }]
+    },
+    diskCacheURL: null,
+    crons: null,
+    redis: {
+        enabled: false,
+        socket: {
+            host: "",
+            port: 0
+        },
+        disableOfflineQueue: true
+    }
+});
+loadFromEnv(config);
+migrate(config);
+
+// Add defaults
+function addDefaults(config: SBSConfig, defaults: SBSConfig) {
+    for (const key in defaults) {
+        if (!Object.prototype.hasOwnProperty.call(config, key)) {
+            config[key] = defaults[key];
+        }
+    }
+}
+
+function migrate(config: SBSConfig) {
+    // Redis change
+    if (config.redis) {
+        const redisConfig = config.redis as any;
+        if (redisConfig.host || redisConfig.port) {
+            config.redis.socket = {
+                host: redisConfig.host,
+                port: redisConfig.port
+            };
+        }
+
+        if (redisConfig.enable_offline_queue !== undefined) {
+            config.disableOfflineQueue = !redisConfig.enable_offline_queue;
+        }
+
+        if (redisConfig.socket?.host && redisConfig.enabled === undefined) {
+            redisConfig.enabled = true;
+        }
+    }
+
+    if (config.postgres && config.postgres.user && config.postgres.enabled === undefined) {
+        config.postgres.enabled = true;
+    }
+}
+
+function loadFromEnv(config: SBSConfig, prefix = "") {
+    for (const key in config) {
+        const fullKey = (prefix ? `${prefix}_` : "") + key;
+        const data = config[key];
+
+        if (data && typeof data === "object" && !Array.isArray(data)) {
+            loadFromEnv(data, fullKey);
+        } else if (process.env[fullKey]) {
+            const value = process.env[fullKey];
+            if (isNumber(value)) {
+                config[key] = parseInt(value, 10);
+            } else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") {
+                config[key] = value === "true";
+            } else if (key === "newLeafURLs") {
+                config[key] = [value];
+            } else {
+                config[key] = value;
+            }
+        }
+    }
+}

src/cronjob/downvoteSegmentArchiveJob.ts

@@ -0,0 +1,67 @@
+import { CronJob } from "cron";
+
+import { config as serverConfig } from "../config";
+import { Logger } from "../utils/logger";
+import { db } from "../databases/databases";
+import { DBSegment } from "../types/segments.model";
+
+const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
+
+export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
+    const timeNow = runTime || new Date().getTime();
+    const threshold = dayLimit * 86400000;
+
+    Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
+    try {
+    // insert into archive sponsorTime
+    await db.prepare(
+        "run",
+        `INSERT INTO "archivedSponsorTimes" 
+        SELECT * 
+        FROM "sponsorTimes" 
+        WHERE "votes" < ? AND (? - "timeSubmitted") > ?`,
+        [
+            voteLimit,
+            timeNow,
+            threshold
+        ]
+    ) as DBSegment[];
+
+    } catch (err) {
+        Logger.error("Execption when insert segment in archivedSponsorTimes");
+        Logger.error(err as string);
+        return 1;
+    }
+
+    // remove from sponsorTime
+    try {
+    await db.prepare(
+        "run",
+        'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?',
+        [
+            voteLimit,
+            timeNow,
+            threshold
+        ]
+    ) as DBSegment[];
+
+    } catch (err) {
+        Logger.error("Execption when deleting segment in sponsorTimes");
+        Logger.error(err as string);
+        return 1;
+    }
+
+    Logger.info("DownvoteSegmentArchiveJob finished");
+    return 0;
+};
+
+const DownvoteSegmentArchiveJob = new CronJob(
+    jobConfig?.schedule || "0 0 * * * 0",
+    () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
+);
+
+if (serverConfig?.crons?.enabled && jobConfig && !jobConfig.schedule) {
+    Logger.error("Invalid cron schedule for downvoteSegmentArchive");
+}
+
+export default DownvoteSegmentArchiveJob;

src/cronjob/index.ts

@@ -0,0 +1,13 @@
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
+
+export function startAllCrons (): void {
+    if (config?.crons?.enabled) {
+        Logger.info("Crons started");
+
+        DownvoteSegmentArchiveJob.start();
+    } else {
+        Logger.info("Crons dissabled");
+    }
+}

src/databases/IDatabase.ts

@@ -0,0 +1,7 @@
+export interface IDatabase {
+    init(): Promise<void>;
+
+    prepare(type: QueryType, query: string, params?: any[]): Promise<any | any[] | void>;
+}
+
+export type QueryType = "get" | "all" | "run";

src/databases/Mysql.ts

@@ -0,0 +1,35 @@
+import { Logger } from "../utils/logger";
+import { IDatabase, QueryType } from "./IDatabase";
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import MysqlInterface from "sync-mysql";
+
+export class Mysql implements IDatabase {
+    private connection: any;
+
+    constructor(private config: unknown) {
+    }
+
+    // eslint-disable-next-line require-await
+    async init(): Promise<void> {
+        this.connection = new MysqlInterface(this.config);
+    }
+
+    prepare(type: QueryType, query: string, params?: any[]): Promise<any[]> {
+        Logger.debug(`prepare (mysql): type: ${type}, query: ${query}, params: ${params}`);
+        const queryResult = this.connection.query(query, params);
+
+        switch (type) {
+            case "get": {
+                return queryResult[0];
+            }
+            case "all": {
+                return queryResult;
+            }
+            case "run": {
+                break;
+            }
+        }
+    }
+
+}

src/databases/Postgres.ts

@@ -0,0 +1,137 @@
+import { Logger } from "../utils/logger";
+import { IDatabase, QueryType } from "./IDatabase";
+import { Client, Pool, PoolClient, types } from "pg";
+
+import fs from "fs";
+
+// return numeric (pg_type oid=1700) as float
+types.setTypeParser(1700, function(val) {
+    return parseFloat(val);
+});
+
+// return int8 (pg_type oid=20) as int
+types.setTypeParser(20, function(val) {
+    return parseInt(val, 10);
+});
+
+export class Postgres implements IDatabase {
+    private pool: Pool;
+
+    constructor(private config: Record<string, any>) {}
+
+    async init(): Promise<void> {
+        this.pool = new Pool(this.config.postgres);
+
+        if (!this.config.readOnly) {
+            if (this.config.createDbIfNotExists) {
+                await this.createDB();
+            }
+
+            if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) {
+                await this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
+            }
+
+            // Upgrade database if required
+            await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+
+            try {
+                await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+            } catch (e) {
+                Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
+                Logger.warn(e as string);
+            }
+        }
+    }
+
+    async prepare(type: QueryType, query: string, params?: any[]): Promise<any[]> {
+        // Convert query to use numbered parameters
+        let count = 1;
+        for (let char = 0; char < query.length; char++) {
+            if (query.charAt(char) === "?") {
+                query = `${query.slice(0, char)}$${count}${query.slice(char + 1)}`;
+                count++;
+            }
+        }
+
+        Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
+
+        let client: PoolClient;
+        try {
+            client = await this.pool.connect();
+            const queryResult = await client.query({ text: query, values: params });
+
+            switch (type) {
+                case "get": {
+                    const value = queryResult.rows[0];
+                    Logger.debug(`result (postgres): ${JSON.stringify(value)}`);
+                    return value;
+                }
+                case "all": {
+                    const values = queryResult.rows;
+                    Logger.debug(`result (postgres): ${JSON.stringify(values)}`);
+                    return values;
+                }
+                case "run": {
+                    break;
+                }
+            }
+        } catch (err) {
+            Logger.error(`prepare (postgres): ${err}`);
+        } finally {
+            client?.release();
+        }
+    }
+
+    private async createDB() {
+        const client = new Client({
+            ...this.config.postgres,
+            database: "postgres"
+        });
+
+        await client.connect();
+
+        if ((await client.query(`SELECT * FROM pg_database WHERE datname = '${this.config.postgres.database}'`)).rowCount == 0) {
+            await client.query(`CREATE DATABASE "${this.config.postgres.database}"
+                                WITH 
+                                OWNER = ${this.config.postgres.user}
+                                CONNECTION LIMIT = -1;`
+            );
+        }
+
+        client.end();
+    }
+
+    private async upgradeDB(fileNamePrefix: string, schemaFolder: string) {
+        const versionCodeInfo = await this.pool.query("SELECT value FROM config WHERE key = 'version'");
+        let versionCode = versionCodeInfo.rows[0] ? versionCodeInfo.rows[0].value : 0;
+
+        let path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+        Logger.debug(`db update: trying ${path}`);
+        while (fs.existsSync(path)) {
+            Logger.debug(`db update: updating ${path}`);
+            await this.pool.query(this.processUpgradeQuery(fs.readFileSync(path).toString()));
+
+            versionCode = (await this.pool.query("SELECT value FROM config WHERE key = 'version'"))?.rows[0]?.value;
+            path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+            Logger.debug(`db update: trying ${path}`);
+        }
+        Logger.debug(`db update: no file ${path}`);
+    }
+
+    private async applyIndexes(fileNamePrefix: string, schemaFolder: string) {
+        const path = `${schemaFolder}/_${fileNamePrefix}_indexes.sql`;
+        if (fs.existsSync(path)) {
+            await this.pool.query(fs.readFileSync(path).toString());
+        } else {
+            Logger.debug(`failed to apply indexes to ${fileNamePrefix}`);
+        }
+    }
+
+    private processUpgradeQuery(query: string): string {
+        let result = query;
+        result = result.replace(/sha256\((.*?)\)/gm, "encode(digest($1, 'sha256'), 'hex')");
+        result = result.replace(/integer/gmi, "NUMERIC");
+
+        return result;
+    }
+}

src/databases/Sqlite.ts

@@ -0,0 +1,108 @@
+import { IDatabase, QueryType } from "./IDatabase";
+import Sqlite3, { Database, Database as SQLiteDatabase } from "better-sqlite3";
+import fs from "fs";
+import path from "path";
+import { getHash } from "../utils/getHash";
+import { Logger } from "../utils/logger";
+
+export class Sqlite implements IDatabase {
+    private db: SQLiteDatabase;
+
+    constructor(private config: SqliteConfig)
+    {
+    }
+
+    // eslint-disable-next-line require-await
+    async prepare(type: QueryType, query: string, params: any[] = []): Promise<any[]> {
+        // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`);
+        const preparedQuery = this.db.prepare(Sqlite.processQuery(query));
+
+        switch (type) {
+            case "get": {
+                return preparedQuery.get(...params);
+            }
+            case "all": {
+                return preparedQuery.all(...params);
+            }
+            case "run": {
+                preparedQuery.run(...params);
+                break;
+            }
+        }
+    }
+
+    // eslint-disable-next-line require-await
+    async init(): Promise<void> {
+        // Make dirs if required
+        if (!fs.existsSync(path.join(this.config.dbPath, "../"))) {
+            fs.mkdirSync(path.join(this.config.dbPath, "../"));
+        }
+
+        this.db = new Sqlite3(this.config.dbPath, { readonly: this.config.readOnly, fileMustExist: !this.config.createDbIfNotExists });
+
+        if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) {
+            this.db.exec(Sqlite.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
+        }
+
+        if (!this.config.readOnly) {
+            this.db.function("sha256", (str: string) => {
+                return getHash(str, 1);
+            });
+
+            // Upgrade database if required
+            Sqlite.upgradeDB(this.db, this.config.fileNamePrefix, this.config.dbSchemaFolder);
+        }
+
+        this.db.function("regexp", { deterministic: true }, (regex: string, str: string) => {
+            return str.match(regex) ? 1 : 0;
+        });
+
+        // Enable WAL mode checkpoint number
+        if (this.config.enableWalCheckpointNumber) {
+            this.db.exec("PRAGMA journal_mode=WAL;");
+            this.db.exec("PRAGMA wal_autocheckpoint=1;");
+        }
+
+        // Enable Memory-Mapped IO
+        this.db.exec("pragma mmap_size= 500000000;");
+    }
+
+    attachDatabase(database: string, attachAs: string): void {
+        this.db.prepare(`ATTACH ? as ${attachAs}`).run(database);
+    }
+
+    private static processQuery(query: string): string {
+        return query.replace(/ ~\* /g, " REGEXP ");
+    }
+
+    private static upgradeDB(db: Database, fileNamePrefix: string, schemaFolder: string) {
+        const versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
+        let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
+
+        let path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+        Logger.debug(`db update: trying ${path}`);
+        while (fs.existsSync(path)) {
+            Logger.debug(`db update: updating ${path}`);
+            db.exec(this.processUpgradeQuery(fs.readFileSync(path).toString()));
+
+            versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
+            path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+            Logger.debug(`db update: trying ${path}`);
+        }
+        Logger.debug(`db update: no file ${path}`);
+    }
+
+    private static processUpgradeQuery(query: string): string {
+        return query.replace(/^.*--!sqlite-ignore/gm, "");
+    }
+}
+
+export interface SqliteConfig {
+    dbPath: string;
+    dbSchemaFileName: string;
+    dbSchemaFolder: string;
+    fileNamePrefix: string;
+    readOnly: boolean;
+    createDbIfNotExists: boolean;
+    enableWalCheckpointNumber: boolean
+}

src/databases/databases.ts

@@ -0,0 +1,89 @@
+import { config } from "../config";
+import { Sqlite } from "./Sqlite";
+import { Mysql } from "./Mysql";
+import { Postgres } from "./Postgres";
+import { IDatabase } from "./IDatabase";
+
+let db: IDatabase;
+let privateDB: IDatabase;
+if (config.mysql) {
+    db = new Mysql(config.mysql);
+    privateDB = new Mysql(config.privateMysql);
+} else if (config.postgres?.enabled) {
+    db = new Postgres({
+        dbSchemaFileName: config.dbSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "sponsorTimes",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        postgres: {
+            user: config.postgres?.user,
+            host: config.postgres?.host,
+            database: "sponsorTimes",
+            password: config.postgres?.password,
+            port: config.postgres?.port,
+            max: config.postgres?.max,
+            min: config.postgres?.min,
+        }
+    });
+
+    privateDB = new Postgres({
+        dbSchemaFileName: config.privateDBSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "private",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        postgres: {
+            user: config.postgres?.user,
+            host: config.postgres?.host,
+            database: "privateDB",
+            password: config.postgres?.password,
+            port: config.postgres?.port,
+            max: config.postgres?.max,
+            min: config.postgres?.min,
+        }
+    });
+} else {
+    db = new Sqlite({
+        dbPath: config.db,
+        dbSchemaFileName: config.dbSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "sponsorTimes",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        enableWalCheckpointNumber: !config.readOnly && config.mode === "production"
+    });
+    privateDB = new Sqlite({
+        dbPath: config.privateDB,
+        dbSchemaFileName: config.privateDBSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "private",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        enableWalCheckpointNumber: false
+    });
+}
+async function initDb(): Promise<void> {
+    await db.init();
+    await privateDB.init();
+
+    if (db instanceof Sqlite) {
+        // Attach private db to main db
+        (db as Sqlite).attachDatabase(config.privateDB, "privateDB");
+    }
+
+    if (config.mode === "mirror" && db instanceof Postgres) {
+        const tables = config?.dumpDatabase?.tables ?? [];
+        const tableNames = tables.map(table => table.name);
+        for (const table of tableNames) {
+            const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`;
+            await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
+}
+
+export {
+    db,
+    privateDB,
+    initDb,
+};

src/index.ts

@@ -0,0 +1,30 @@
+import { config } from "./config";
+import { initDb } from "./databases/databases";
+import { createServer } from "./app";
+import { Logger } from "./utils/logger";
+import { startAllCrons } from "./cronjob";
+import { getCommit } from "./utils/getCommit";
+
+async function init() {
+    process.on("unhandledRejection", (error: any) => {
+        // eslint-disable-next-line no-console
+        console.dir(error?.stack);
+        process.exit(1);
+    });
+
+    await initDb();
+    // edge case clause for creating compatible .db files, do not enable
+    if (config.mode === "init-db-and-exit") process.exit(0);
+    // do not enable init-db-only mode for usage.
+    (global as any).HEADCOMMIT = config.mode === "development" ? "development"
+        : config.mode === "test" ? "test"
+            : getCommit() as string;
+    createServer(() => {
+        Logger.info(`Server started on port ${config.port}.`);
+
+        // ignite cron job after server created
+        startAllCrons();
+    }).setTimeout(15000);
+}
+
+init();

src/middleware/apiCsp.ts

@@ -0,0 +1,6 @@
+import { NextFunction, Request, Response } from "express";
+
+export function apiCspMiddleware(req: Request, res: Response, next: NextFunction): void {
+    res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
+    next();
+}

src/middleware/cors.ts

@@ -0,0 +1,8 @@
+import { NextFunction, Request, Response } from "express";
+
+export function corsMiddleware(req: Request, res: Response, next: NextFunction): void {
+    res.header("Access-Control-Allow-Origin", "*");
+    res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE");
+    res.header("Access-Control-Allow-Headers", "Content-Type");
+    next();
+}

src/middleware/logger.ts

@@ -0,0 +1,7 @@
+import { Logger } from "../utils/logger";
+import { NextFunction, Request, Response } from "express";
+
+export function loggerMiddleware(req: Request, res: Response, next: NextFunction): void {
+    Logger.info(`Request received: ${req.method} ${req.url}`);
+    next();
+}

src/middleware/requestRateLimit.ts

@@ -0,0 +1,36 @@
+import { getIP } from "../utils/getIP";
+import { getHash } from "../utils/getHash";
+import { getHashCache } from "../utils/getHashCache";
+import rateLimit, { RateLimitRequestHandler } from "express-rate-limit";
+import { RateLimitConfig } from "../types/config.model";
+import { Request } from "express";
+import { isUserVIP } from "../utils/isUserVIP";
+import { UserID } from "../types/user.model";
+import RedisStore, { RedisReply } from "rate-limit-redis";
+import redis from "../utils/redis";
+import { config } from "../config";
+import { Logger } from "../utils/logger";
+
+export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): RateLimitRequestHandler {
+    return rateLimit({
+        windowMs: limitConfig.windowMs,
+        max: limitConfig.max,
+        message: limitConfig.message,
+        statusCode: limitConfig.statusCode,
+        legacyHeaders: false,
+        standardHeaders: false,
+        keyGenerator: (req) => {
+            return getHash(getIP(req), 1);
+        },
+        handler: async (req, res, next) => {
+            if (getUserID === undefined || !await isUserVIP(await getHashCache(getUserID(req)))) {
+                return res.status(limitConfig.statusCode).send(limitConfig.message);
+            } else {
+                return next();
+            }
+        },
+        store: config.redis?.enabled ? new RedisStore({
+            sendCommand: (...args: string[]) => redis.sendCommand(args).catch((err) => Logger.error(err)) as Promise<RedisReply>,
+        }) : null,
+    });
+}

src/middleware/userCounter.ts

@@ -0,0 +1,15 @@
+import axios from "axios";
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import { getIP } from "../utils/getIP";
+import { getHash } from "../utils/getHash";
+import { NextFunction, Request, Response } from "express";
+
+export function userCounter(req: Request, res: Response, next: NextFunction): void {
+    if (req.method !== "OPTIONS") {
+        axios.post(`${config.userCounterURL}/api/v1/addIP?hashedIP=${getHash(getIP(req), 1)}`)
+            .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
+    }
+
+    next();
+}

src/routes/addUnlistedVideo.ts

@@ -0,0 +1,42 @@
+import { Request, Response } from "express";
+import { db } from "../databases/databases";
+import { getService } from "../utils/getService";
+import { Logger } from "../utils/logger";
+
+/**
+ * Optional API method that will be used temporarily to help collect
+ * unlisted videos created before 2017
+ *
+ * https://support.google.com/youtube/answer/9230970
+ */
+
+export async function addUnlistedVideo(req: Request, res: Response): Promise<Response> {
+    const {
+        body: {
+            videoID = null,
+            year = 0,
+            views = 0,
+            channelID = "Unknown",
+            service
+        }
+    } = req;
+
+    if (typeof(videoID) !== "string" || videoID.length !== 11) {
+        return res.status(400).send("Invalid parameters");
+    }
+
+    try {
+        const timeSubmitted = Date.now();
+        await db.prepare(
+            "run",
+            `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted", "service") values (?, ?, ?, ?, ?, ?)`,
+            [videoID, year, views, channelID, timeSubmitted, getService(service)]
+        );
+
+        return res.sendStatus(200);
+    } catch (err) {
+        Logger.error(err as string);
+    }
+
+    return res.sendStatus(500);
+}