Add config for min

add additional poi_highlight tests

.editorconfig

@@ -0,0 +1,17 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{js,json,ts,tsx}]
+charset = utf-8
+indent_style = space
+indent_size = 4
+
+[package.json]
+indent_size = 2

.eslintrc.js

@@ -0,0 +1,32 @@
+module.exports = {
+    env: {
+        browser: false,
+        es2021: true,
+        node: true,
+    },
+    extends: [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended",
+    ],
+    parser: "@typescript-eslint/parser",
+    parserOptions: {
+        ecmaVersion: 12,
+        sourceType: "module",
+    },
+    plugins: ["@typescript-eslint"],
+    rules: {
+        // TODO: Remove warn rules when not needed anymore
+        "@typescript-eslint/no-empty-interface": "off",
+        "@typescript-eslint/no-explicit-any": "off",
+        "indent": ["warn", 4, { "SwitchCase": 1 }],
+        "no-multiple-empty-lines": ["error", { max: 2, maxEOF: 0 }],
+        "no-self-assign": "off",
+        "no-trailing-spaces": "warn",
+        "object-curly-spacing": ["warn", "always"],
+        "prefer-template": "warn",
+        "quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
+        "require-await": "warn",
+        "semi": "warn",
+        "no-console": "warn"
+    },
+};

.github/workflows/ci.yml

@@ -1,18 +1,21 @@
-name: CI
+name: SQLite CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 jobs:
-
-  build:
-    name: Run Tests
+  test:
+    name: Run Tests with SQLite
     runs-on: ubuntu-latest
 
     steps:
       # Initialization
-      - uses: actions/checkout@v1
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
       - run: npm install
-
       - name: Run Tests
+        timeout-minutes: 5
         run: npm test

.github/workflows/db-backup.yml

@@ -0,0 +1,18 @@
+name: Docker image builds
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - containers/backup-db/**
+  workflow_dispatch:
+
+jobs:
+  backup-db:
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      name: "db-backup"
+      username: "ajayyy"
+      folder: "./containers/backup-db"
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docker-build.yml

@@ -0,0 +1,46 @@
+# Based on https://github.com/ajayyy/sb-mirror/blob/main/.github/workflows/docker-build.yml
+name: multi-build-docker
+on: 
+  workflow_call:
+    inputs:
+      name:
+        required: true
+        type: string
+      username:
+        required: true
+        type: string
+      folder:
+        required: true
+        type: string
+    secrets:
+      GH_TOKEN:
+        required: true
+
+jobs:
+  build_container:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/${{ inputs.username }}/${{ inputs.name }}
+          tags: |
+            type-raw,value=alpine
+          flavor: |
+            latest=true
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GH_TOKEN }}
+      - name: push
+        uses: docker/build-push-action@v3
+        with:
+          context: ${{ inputs.folder }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/eslint.yml

@@ -0,0 +1,21 @@
+name: Linting
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  lint:
+    name: Lint with ESLint
+    runs-on: ubuntu-latest
+
+    steps:
+      # Initialization
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Run Tests
+        timeout-minutes: 5
+        run: npm run lint

.github/workflows/generate-sqlite-base.yml

@@ -0,0 +1,28 @@
+name: create-sqlite-base
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - databases/**
+
+jobs:
+  make-base-db:
+    name: Generate SQLite base .db
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Set config
+        run: |
+          echo '{"mode": "init-db-and-exit"}' > config.json
+      - name: Run Server
+        timeout-minutes: 10
+        run: npm start
+      - uses: actions/upload-artifact@v2
+        with:
+          name: SponsorTimesDB.db
+          path: databases/sponsorTimes.db

.github/workflows/postgres-redis-ci.yml

@@ -0,0 +1,29 @@
+name: PostgreSQL + Redis CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  test:
+    name: Run Tests with PostgreSQL and Redis
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the docker-compose stack
+        env:
+          PG_USER: ci_db_user
+          PG_PASS: ci_db_pass
+        run: docker-compose -f docker/docker-compose-ci.yml up -d
+      - name: Check running containers
+        run: docker ps
+      - uses: actions/setup-node@v2
+      - run: npm install
+      - name: Run Tests
+        env:
+          TEST_POSTGRES: true
+        timeout-minutes: 5
+        run: npm test

.github/workflows/sb-server.yml

@@ -0,0 +1,25 @@
+name: Docker image builds
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  sb-server:
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      name: "sb-server"
+      username: "ajayyy"
+      folder: "."
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  rsync-host:
+    needs: sb-server
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      name: "rsync-host"
+      username: "ajayyy"
+      folder: "./containers/rsync"
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/take-action.yml

@@ -0,0 +1,12 @@
+name: Assign issue to contributor
+on: [issue_comment]
+
+jobs:
+  assign:
+    name: Take an issue
+    runs-on: ubuntu-latest
+    steps:
+      - name: take the issue
+        uses: bdougie/take-action@main
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.gitignore

@@ -2,47 +2,9 @@
 logs
 *.log
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
-
-# Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
 
 # Dependency directories
 node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
 
 # TypeScript cache
 *.tsbuildinfo
@@ -53,53 +15,33 @@ typings/
 # Optional eslint cache
 .eslintcache
 
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
 # dotenv environment variables file
 .env
 .env.test
 
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# next.js build output
-.next
-
-# nuxt.js build output
-.nuxt
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-# DynamoDB Local files
-.dynamodb/
-
 # Databases
 databases/sponsorTimes.db
 databases/sponsorTimes.db-shm
 databases/sponsorTimes.db-wal
 databases/private.db
+databases/private.db-shm
+databases/private.db-wal
 databases/sponsorTimesReal.db
 test/databases/sponsorTimes.db
 test/databases/sponsorTimes.db-shm
 test/databases/sponsorTimes.db-wal
 test/databases/private.db
+databases/cache
+docker/database-export
 
 # Config files
 config.json
+docker/database.env
+
+# Other
+working
 
 # Mac files
-.DS_Store
+.DS_Store
+/.idea/
+/dist/

DatabaseSchema.md

@@ -0,0 +1,283 @@
+# SponsorTimesDB
+
+[vipUsers](#vipUsers)  
+[sponsorTimes](#sponsorTimes)  
+[userNames](#userNames)  
+[categoryVotes](#categoryVotes)  
+[lockCategories](#lockCategories)  
+[warnings](#warnings)  
+[shadowBannedUsers](#shadowBannedUsers)  
+[unlistedVideos](#unlistedVideos)  
+[config](#config)  
+[archivedSponsorTimes](#archivedSponsorTimes)
+
+### vipUsers
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| vipUsers_index | userID |
+
+### sponsorTimes  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| startTime | REAL | not null |
+| endTime | REAL | not null |
+| votes | INTEGER | not null |
+| locked | INTEGER | not null, default '0' |
+| incorrectVotes | INTEGER | not null, default 1 |
+| UUID | TEXT | not null, unique |
+| userID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| views | INTEGER | not null |
+| category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
+| service | TEXT | not null, default 'YouTube' |
+| videoDuration | INTEGER | not null, default '0' |
+| hidden | INTEGER | not null, default '0' |
+| reputation | REAL | not null, default '0' |
+| shadowHidden | INTEGER | not null |
+| hashedVideoID | TEXT | not null, default '', sha256 |
+| userAgent | TEXT | not null, default '' |
+| description | TEXT | not null, default '' |
+
+| index | field |
+| -- | :--: |
+| sponsorTime_timeSubmitted | timeSubmitted |
+| sponsorTime_userID | userID |
+| sponsorTimes_UUID | UUID |
+| sponsorTimes_hashedVideoID | hashedVideoID, category |
+| sponsorTimes_videoID | videoID, service, category, timeSubmitted |
+
+### userNames
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+| userName | TEXT | not null |
+| locked | INTEGER | not nul, default '0' |
+
+| index | field |
+| -- | :--: |
+| userNames_userID | userID |
+
+### categoryVotes
+
+| Name | Type | |
+| -- | :--: | -- |
+| UUID | TEXT | not null |
+| category | TEXT | not null |
+| votes | INTEGER | not null, default 0 |
+
+| index | field |
+| -- | :--: |
+| categoryVotes_UUID_public | UUID, category |
+
+### lockCategories  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| userID | TEXT | not null |
+| actionType | TEXT | not null, default 'skip' |
+| category | TEXT | not null |
+| hashedVideoID | TEXT | not null, default '' |
+| reason | TEXT | not null, default '' |
+| service | TEXT | not null, default 'YouTube' |
+
+| index | field |
+| -- | :--: |
+| lockCategories_videoID | videoID, service, category |
+
+### warnings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+| issueTime | INTEGER | not null |
+| issuerUserID | TEXT | not null |
+| enabled | INTEGER | not null |
+| reason | TEXT | not null, default '' |
+
+| index | field |
+| -- | :--: |
+| warnings_index | userID |
+| warnings_issueTime | issueTime |
+
+### shadowBannedUsers  
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| shadowBannedUsers_index | userID |
+
+### videoInfo  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| channelID | TEXT | not null |
+| title | TEXT | not null |
+| published | REAL | not null |
+| genreUrl | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| videoInfo_videoID | timeSubmitted |
+| videoInfo_channelID | userID |
+
+### unlistedVideos  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| year | TEXT | not null |
+| views | TEXT | not null |
+| channelID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| service | TEXT | not null, default 'YouTube' |
+
+### config
+
+| Name | Type | |
+| -- | :--: | -- |
+| key | TEXT | not null, unique |
+| value | TEXT | not null |
+
+### archivedSponsorTimes
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| startTime | REAL | not null |
+| endTime | REAL | not null |
+| votes | INTEGER | not null |
+| locked | INTEGER | not null, default '0' |
+| incorrectVotes | INTEGER | not null, default 1 |
+| UUID | TEXT | not null, unique |
+| userID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| views | INTEGER | not null |
+| category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
+| service | TEXT | not null, default 'YouTube' |
+| videoDuration | INTEGER | not null, default '0' |
+| hidden | INTEGER | not null, default '0' |
+| reputation | REAL | not null, default '0' |
+| shadowHidden | INTEGER | not null |
+| hashedVideoID | TEXT | not null, default '', sha256 |
+| userAgent | TEXT | not null, default '' |
+
+### ratings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| service | TEXT | not null, default 'YouTube' |
+| type | INTEGER | not null |
+| count | INTEGER | not null |
+| hashedVideoID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| ratings_hashedVideoID_gin | hashedVideoID |
+| ratings_hashedVideoID | hashedVideoID, service |
+| ratings_videoID | videoID, service |
+
+# Private 
+
+[votes](#votes)  
+[categoryVotes](#categoryVotes)  
+[sponsorTimes](#sponsorTimes)  
+[config](#config)  
+[ratings](#ratings)  
+[tempVipLog](#tempVipLog)  
+[userNameLogs](#userNameLogs)  
+
+### votes
+
+| Name | Type | |
+| -- | :--: | -- |
+| UUID | TEXT | not null |
+| userID | TEXT | not null |
+| hashedIP | TEXT | not null |
+| type | INTEGER | not null |
+
+| index | field |
+| -- | :--: |
+| votes_userID | UUID |
+
+### categoryVotes
+
+| Name | Type | |
+| -- | :--: | -- |
+| UUID | TEXT | not null |
+| userID | TEXT | not null |
+| hashedIP | TEXT | not null |
+| category | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+
+| index | field |
+| -- | :--: |
+| categoryVotes_UUID | UUID, userID, hasedIP, category |
+
+### sponsorTimes  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| hashedIP | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| service | TEXT | not null, default 'YouTube' |
+
+| index | field |
+| -- | :--: |
+| sponsorTimes_hashedIP | hashedIP |
+| privateDB_sponsorTimes_videoID_v2 | videoID, service |
+
+### config  
+
+| Name | Type | |
+| -- | :--: | -- |
+| key | TEXT | not null |
+| value | TEXT | not null |
+
+### ratings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| service | TEXT | not null, default 'YouTube' |
+| userID | TEXT | not null |
+| type | INTEGER | not null |
+| timeSubmitted | INTEGER | not null |
+| hashedIP | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| ratings_videoID | videoID, service, userID, timeSubmitted |
+
+### tempVipLog
+| Name | Type | |
+| -- | :--: | -- |
+| issuerUserID | TEXT | not null |
+| targetUserID | TEXT | not null |
+| enabled | BOOLEAN | not null |
+| updatedAt | INTEGER | not null |
+
+### userNameLogs
+
+| Name | Type | |
+| -- | :--: | -- |
+| userID | TEXT | not null |
+| newUserName | TEXT | not null |
+| oldUserName | TEXT | not null |
+| updatedByAdmin | BOOLEAN | not null |
+| updatedAt | INTEGER | not null |

Dockerfile

@@ -1,11 +1,16 @@
-FROM node:12
-WORKDIR /usr/src/app
-COPY package.json .
-RUN npm install
-COPY index.js .
+FROM node:16-alpine as builder
+RUN apk add --no-cache --virtual .build-deps python3 make g++
+COPY package.json package-lock.json tsconfig.json entrypoint.sh ./
 COPY src src
-RUN mkdir databases
-COPY databases/*.sql databases/
+RUN npm ci && npm run tsc
+
+FROM node:16-alpine as app
+WORKDIR /usr/src/app
+RUN apk add git postgresql-client
+COPY --from=builder ./node_modules ./node_modules
+COPY --from=builder ./dist ./dist
+COPY ./.git ./.git
 COPY entrypoint.sh .
+COPY databases/*.sql databases/
 EXPOSE 8080
 CMD ./entrypoint.sh

README.MD

@@ -6,9 +6,9 @@ This is the server backend for it
 
 # Server
 
-This is a simple Sqlite database that will hold all the timing data.
+This uses a Postgres or Sqlite database to hold all the timing data.
 
-To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
+To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
 
 Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.
 
@@ -34,4 +34,4 @@ If you want to make changes, run `npm run dev` to automatically reload the serve
 
 # API Docs
 
-Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs)
+Available [here](https://wiki.sponsor.ajay.app/index.php/API_Docs)

ci.json

@@ -0,0 +1,71 @@
+{
+    "port": 8080,
+    "mockPort": 8081,
+    "globalSalt": "testSalt",
+    "adminUserID": "4bdfdc9cddf2c7d07a8a87b57bf6d25389fb75d1399674ee0e0938a6a60f4c3b",
+    "newLeafURLs": ["placeholder"],
+    "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook",
+    "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook",
+    "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook",
+    "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
+    "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
+    "behindProxy": true,
+    "postgres": {
+        "user": "ci_db_user",
+        "password": "ci_db_pass",
+        "host": "localhost",
+        "port": 5432
+    },
+    "redis": {
+        "socket": {
+            "host": "localhost",
+            "port": 6379
+        }
+    },
+    "createDatabaseIfNotExist": true,
+    "schemaFolder": "./databases",
+    "dbSchema": "./databases/_sponsorTimes.db.sql",
+    "privateDBSchema": "./databases/_private.db.sql",
+    "categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
+    "mode": "test",
+    "readOnly": false,
+    "webhooks": [
+        {
+            "url": "http://127.0.0.1:8081/CustomWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8081/FailedWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8099/WrongPort",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }
+    ],
+    "maxNumberOfActiveWarnings": 3,
+    "hoursAfterWarningExpires": 24,
+    "rateLimit": {
+        "vote": {
+          "windowMs": 900000,
+          "max": 20,
+          "message": "Too many votes, please try again later",
+          "statusCode": 429
+        },
+        "view": {
+          "windowMs": 900000,
+          "max": 20,
+          "statusCode": 200
+        }
+    }
+}

config.json.example

@@ -4,7 +4,7 @@
     "port": 80,
     "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]",
     "adminUserID": "[the hashed id of the user who can perform admin actions]",
-    "youtubeAPIKey": null, //get this from Google Cloud Platform [optional]
+    "newLeafURLs": ["http://localhost:3241"],
     "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional]
     "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional]
     "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
@@ -19,10 +19,11 @@
     "schemaFolder": "./databases",
     "dbSchema": "./databases/_sponsorTimes.db.sql",
     "privateDBSchema": "./databases/_private.db.sql",
+    // when using redis, add `"enable_offline_queue": false` to force commands to fail instead of timing out
     "mode": "development",
     "readOnly": false,
     "webhooks": [],
-    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected
+    "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "preview", "music_offtopic", "poi_highlight"], // List of supported categories any other category will be rejected
     "getTopUsersCacheTimeMinutes": 5, // cacheTime for getTopUsers result in minutes
     "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire
     "hoursAfterWarningExpire": 24,
@@ -38,5 +39,32 @@
         "max": 20, // 20 requests in 15min time window
         "statusCode": 200
       }
+    },
+    "maxRewardTimePerSegmentInSeconds": 86400, // maximum time a user get rewarded in the leaderboard for a single segment
+    "dumpDatabase": {
+        "enabled": true,
+        "minTimeBetweenMs": 60000, // 1 minute between dumps
+        "appExportPath": "./docker/database-export",
+        "postgresExportPath": "/opt/exports",
+        "tables": [{
+            "name": "sponsorTimes",
+            "order": "timeSubmitted"
+        },
+        {
+            "name": "userNames"
+        },
+        {
+            "name": "categoryVotes"
+        },
+        {
+            "name": "lockCategories"
+        },
+        {
+            "name": "warnings",
+            "order": "issueTime"
+        },
+        {
+            "name": "vipUsers"
+        }]
     }
 }

containers/backup-db/Dockerfile

@@ -0,0 +1,13 @@
+FROM alpine
+RUN apk add postgresql-client
+RUN apk add restic --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community/
+
+COPY ./backup.sh /usr/src/app/backup.sh
+RUN chmod +x /usr/src/app/backup.sh
+COPY ./backup.sh /usr/src/app/forget.sh
+RUN chmod +x /usr/src/app/forget.sh
+
+RUN echo '30  *  *  *  *    /usr/src/app/backup.sh' >> /etc/crontabs/root
+RUN echo '10  0  *  *  1    /usr/src/app/forget.sh' >> /etc/crontabs/root
+
+CMD crond -l 2 -f

containers/backup-db/backup.sh

@@ -0,0 +1,6 @@
+mkdir ./dump
+
+pg_dump -f ./dump/sponsorTimes.dump sponsorTimes
+pg_dump -f ./dump/privateDB.dump privateDB
+
+restic backup ./dump

containers/backup-db/forget.sh

@@ -0,0 +1 @@
+restic forget --prune --keep-last 48 --keep-daily 7 --keep-weekly 8

containers/rsync/Dockerfile

@@ -0,0 +1,6 @@
+FROM ghcr.io/ajayyy/sb-server:latest
+EXPOSE 873/tcp
+RUN apk add rsync>3.2.4-r0
+RUN mkdir /usr/src/app/database-export
+
+CMD rsync --no-detach --daemon & ./entrypoint.sh

databases/_private.db.sql

@@ -1,8 +1,5 @@
 BEGIN TRANSACTION;
 
-CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
-	"userID"	TEXT NOT NULL
-);
 CREATE TABLE IF NOT EXISTS "votes" (
 	"UUID"	TEXT NOT NULL,
 	"userID"	TEXT NOT NULL,
@@ -29,7 +26,4 @@ CREATE TABLE IF NOT EXISTS "config" (
 	"value" TEXT NOT NULL
 );
 
-CREATE INDEX IF NOT EXISTS sponsorTimes_hashedIP on sponsorTimes(hashedIP);
-CREATE INDEX IF NOT EXISTS votes_userID on votes(UUID);
-
 COMMIT;

databases/_private_indexes.sql

@@ -0,0 +1,26 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_v4"
+    ON public."sponsorTimes" USING btree
+    ("videoID" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST);
+
+-- votes
+
+CREATE INDEX IF NOT EXISTS "votes_userID"
+    ON public.votes USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
+    ON public."categoryVotes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- ratings
+
+CREATE INDEX IF NOT EXISTS "ratings_videoID"
+    ON public."ratings" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;

databases/_sponsorTimes.db.sql

@@ -3,6 +3,11 @@ BEGIN TRANSACTION;
 CREATE TABLE IF NOT EXISTS "vipUsers" (
 	"userID"	TEXT NOT NULL
 );
+
+COMMIT;
+
+BEGIN TRANSACTION;
+
 CREATE TABLE IF NOT EXISTS "sponsorTimes" (
 	"videoID"	TEXT NOT NULL,
 	"startTime"	REAL NOT NULL,
@@ -12,25 +17,27 @@ CREATE TABLE IF NOT EXISTS "sponsorTimes" (
 	"userID"	TEXT NOT NULL,
 	"timeSubmitted"	INTEGER NOT NULL,
 	"views"	INTEGER NOT NULL,
-    "category" TEXT NOT NULL,
+	"category" TEXT NOT NULL,
 	"shadowHidden"	INTEGER NOT NULL
 );
+
 CREATE TABLE IF NOT EXISTS "userNames" (
 	"userID"	TEXT NOT NULL,
 	"userName"	TEXT NOT NULL
 );
+
 CREATE TABLE IF NOT EXISTS "categoryVotes" (
 	"UUID"	TEXT NOT NULL,
 	"category"	TEXT NOT NULL,
-	"votes"	INTEGER NOT NULL default '0'
+	"votes"	INTEGER NOT NULL default 0
 );
 
 CREATE TABLE IF NOT EXISTS "config" (
-    "key" TEXT NOT NULL UNIQUE,
+	"key" TEXT NOT NULL UNIQUE,
 	"value" TEXT NOT NULL
 );
 
-CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID);
-CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID);
+CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
+CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore
 
 COMMIT;

databases/_sponsorTimes_indexes.sql

@@ -0,0 +1,106 @@
+-- sponsorTimes
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted"
+    ON public."sponsorTimes" USING btree
+    ("timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTime_userID"
+    ON public."sponsorTimes" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
+    ON public."sponsorTimes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID"
+    ON public."sponsorTimes" USING btree
+    (service COLLATE pg_catalog."default" ASC NULLS LAST, "hashedVideoID" text_pattern_ops ASC NULLS LAST, "startTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"
+    ON public."sponsorTimes" USING btree
+    (service COLLATE pg_catalog."default" ASC NULLS LAST, "videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "startTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "sponsorTimes_description_gin"
+    ON public."sponsorTimes" USING gin
+    ("description" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
+    TABLESPACE pg_default;
+
+-- userNames
+
+CREATE INDEX IF NOT EXISTS "userNames_userID"
+    ON public."userNames" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- vipUsers
+
+CREATE INDEX IF NOT EXISTS "vipUsers_index"
+    ON public."vipUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- warnings
+
+CREATE INDEX IF NOT EXISTS "warnings_index"
+    ON public.warnings USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "warnings_issueTime"
+    ON public.warnings USING btree
+    ("issueTime" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- lockCategories
+
+CREATE INDEX IF NOT EXISTS "lockCategories_videoID"
+    ON public."lockCategories" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- categoryVotes
+
+CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
+    ON public."categoryVotes" USING btree
+    ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- shadowBannedUsers
+
+CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
+    ON public."shadowBannedUsers" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- videoInfo
+CREATE INDEX IF NOT EXISTS "videoInfo_videoID"
+    ON public."videoInfo" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "videoInfo_channelID"
+    ON public."videoInfo" USING btree
+    ("channelID" COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- ratings
+
+CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID_gin"
+    ON public."ratings" USING gin
+    ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID"
+    ON public."ratings" USING btree
+    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "ratings_videoID"
+    ON public."ratings" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;

databases/_upgrade_private_1.sql

@@ -3,6 +3,6 @@ BEGIN TRANSACTION;
 /* for testing the db upgrade, don't remove because it looks empty */
 
 /* Add version to config */
-INSERT INTO config (key, value) VALUES("version", 1);
+INSERT INTO config (key, value) VALUES('version', 1);
 
 COMMIT;

databases/_upgrade_private_2.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "userNameLogs" (
+	"userID"	TEXT NOT NULL,
+	"newUserName"	TEXT NOT NULL,
+	"oldUserName"	TEXT NOT NULL,
+	"updatedByAdmin"	BOOLEAN NOT NULL,
+	"updatedAt"	INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 2 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_3.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "sponsorTimes" ADD "service" TEXT NOT NULL default 'YouTube';
+-- UPDATE "sponsorTimes" SET "service" = "YouTube";
+
+DROP INDEX IF EXISTS "privateDB_sponsorTimes_videoID";
+
+UPDATE "config" SET value = 3 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_4.sql

@@ -0,0 +1,14 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "ratings" (
+	"videoID"		TEXT NOT NULL,
+	"service"		TEXT NOT NULL default 'YouTube',
+	"type"			INTEGER NOT NULL,
+	"userID"		TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"hashedIP"		TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 4 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_5.sql

@@ -0,0 +1,12 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "tempVipLog" (
+	"issuerUserID"	TEXT NOT NULL,
+	"targetUserID"	TEXT NOT NULL,
+	"enabled"				BOOLEAN NOT NULL,
+	"updatedAt"			INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 5 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_6.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+DROP INDEX IF EXISTS "sponsorTimes_hashedIP", "privateDB_sponsorTimes_videoID_v2"; --!sqlite-ignore
+
+UPDATE "config" SET value = 6 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_7.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "votes" ADD "normalUserID" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_8.sql

@@ -0,0 +1,15 @@
+BEGIN TRANSACTION;
+
+-- Add primary keys
+
+ALTER TABLE "userNameLogs" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "categoryVotes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "sponsorTimes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "config" ADD PRIMARY KEY ("key"); --!sqlite-ignore
+ALTER TABLE "ratings" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "tempVipLog" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "votes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_9.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+-- Add primary keys
+
+DROP INDEX IF EXISTS "privateDB_sponsorTimes_v3"; --!sqlite-ignore
+
+UPDATE "config" SET value = 9 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_1.sql

@@ -6,20 +6,20 @@ CREATE TABLE "sqlb_temp_table_1" (
 	"startTime"	REAL NOT NULL,
 	"endTime"	REAL NOT NULL,
 	"votes"	INTEGER NOT NULL,
-    "incorrectVotes" INTEGER NOT NULL default '1',
+	"incorrectVotes" INTEGER NOT NULL default 1,
 	"UUID"	TEXT NOT NULL UNIQUE,
 	"userID"	TEXT NOT NULL,
 	"timeSubmitted"	INTEGER NOT NULL,
 	"views"	INTEGER NOT NULL,
-	"category"	TEXT NOT NULL DEFAULT "sponsor",
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
 	"shadowHidden"	INTEGER NOT NULL
 );
-INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
+INSERT INTO sqlb_temp_table_1 SELECT "videoID","startTime","endTime","votes",'1',"UUID","userID","timeSubmitted","views","category","shadowHidden" FROM "sponsorTimes";
 
-DROP TABLE sponsorTimes;
+DROP TABLE "sponsorTimes";
 ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
 
 /* Add version to config */
-INSERT INTO config (key, value) VALUES("version", 1);
+INSERT INTO config (key, value) VALUES('version', 1);
 
 COMMIT;

databases/_upgrade_sponsorTimes_10.sql

@@ -0,0 +1,30 @@
+BEGIN TRANSACTION;
+
+/* Add Hidden field */
+CREATE TABLE "sqlb_temp_table_10" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"hidden"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 10 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_11.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* Rename table: noSegments to lockCategories */
+ALTER TABLE "noSegments" RENAME TO "lockCategories";
+
+UPDATE "config" SET value = 11 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_12.sql

@@ -0,0 +1,31 @@
+BEGIN TRANSACTION;
+
+/* Add reputation field */
+CREATE TABLE "sqlb_temp_table_12" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"hidden"	INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT 0,
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 12 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_13.sql

@@ -0,0 +1,17 @@
+BEGIN TRANSACTION;
+
+/* Add locked field */
+CREATE TABLE "sqlb_temp_table_13" (
+	"userID"	TEXT NOT NULL,
+	"userName"	TEXT NOT NULL,
+	"locked" INTEGER NOT NULL default '0'
+);
+
+INSERT INTO sqlb_temp_table_13 SELECT "userID", "userName", 0 FROM "userNames";
+
+DROP TABLE "userNames";
+ALTER TABLE sqlb_temp_table_13 RENAME TO "userNames";
+
+UPDATE "config" SET value = 13 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_14.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "shadowBannedUsers" (
+	"userID"	TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 14 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_15.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "unlistedVideos" (
+	"videoID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 15 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_16.sql

@@ -0,0 +1,15 @@
+BEGIN TRANSACTION;
+
+DROP TABLE "unlistedVideos";
+
+CREATE TABLE IF NOT EXISTS "unlistedVideos" (
+	"videoID"	TEXT NOT NULL,
+	"year"	TEXT NOT NULL,
+	"views"	TEXT NOT NULL,
+	"channelID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL
+);
+
+UPDATE "config" SET value = 16 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_17.sql

@@ -0,0 +1,19 @@
+BEGIN TRANSACTION;
+
+/* Add reason field */
+CREATE TABLE "sqlb_temp_table_17" (
+	"userID" TEXT NOT NULL,
+	"issueTime" INTEGER NOT NULL,
+	"issuerUserID" TEXT NOT NULL,
+	enabled INTEGER NOT NULL,
+	"reason" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_17 SELECT "userID","issueTime","issuerUserID","enabled", '' FROM "warnings";
+
+DROP TABLE warnings;
+ALTER TABLE sqlb_temp_table_17 RENAME TO "warnings";
+
+UPDATE "config" SET value = 17 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_18.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "lockCategories" ADD "hashedVideoID" TEXT NOT NULL default '';
+UPDATE "lockCategories" SET "hashedVideoID" = sha256("videoID");
+
+UPDATE "config" SET value = 18 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_19.sql

@@ -0,0 +1,32 @@
+BEGIN TRANSACTION;
+
+/* Add actionType field */
+CREATE TABLE "sqlb_temp_table_19" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"actionType"	TEXT NOT NULL DEFAULT 'skip',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"hidden"	INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT 0,
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_19 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'skip',"service","videoDuration","hidden","reputation","shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_19 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 19 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_2.sql

@@ -8,6 +8,6 @@ CREATE TABLE "noSegments" (
 );
 
 /* Add version to config */
-UPDATE config SET value = 2 WHERE key = 'version';
+UPDATE "config" SET value = 2 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_20.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "lockCategories" ADD "reason" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 20 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_21.sql

@@ -0,0 +1,26 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" (
+	"videoID" TEXT NOT NULL,
+	"startTime" REAL NOT NULL,
+	"endTime" REAL NOT NULL,
+	"votes" INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL DEFAULT '0',
+	"incorrectVotes" INTEGER NOT NULL DEFAULT 1,
+	"UUID" TEXT NOT NULL UNIQUE,
+	"userID" TEXT NOT NULL,
+	"timeSubmitted" INTEGER NOT NULL,
+	"views" INTEGER NOT NULL,
+	"category" TEXT NOT NULL DEFAULT 'sponsor',
+	"service" TEXT NOT NULL DEFAULT 'Youtube',
+	"actionType" TEXT NOT NULL DEFAULT 'skip',
+	"videoDuration" INTEGER NOT NULL DEFAULT '0',
+	"hidden" INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT '0',
+	"shadowHidden" INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL DEFAULT ''
+);
+
+UPDATE "config" SET value = 21 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_22.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "sponsorTimes" ADD "userAgent" TEXT NOT NULL default '';
+
+ALTER TABLE "archivedSponsorTimes" ADD "userAgent" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 22 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_23.sql

@@ -0,0 +1,12 @@
+BEGIN TRANSACTION;
+
+DELETE FROM "userNames" WHERE ctid NOT IN ( --!sqlite-ignore
+    SELECT MIN(ctid) FROM "userNames"       --!sqlite-ignore
+    GROUP BY "userID"                       --!sqlite-ignore
+);                                          --!sqlite-ignore
+
+ALTER TABLE "userNames" ADD UNIQUE("userID");   --!sqlite-ignore
+
+UPDATE "config" SET value = 23 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_24.sql

@@ -0,0 +1,21 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "lockCategories" ADD "service" TEXT NOT NULL default 'YouTube';
+
+UPDATE "lockCategories" 
+SET "service" = "sponsorTimes"."service" 
+FROM "sponsorTimes" 
+WHERE "lockCategories"."videoID" = "sponsorTimes"."videoID";
+
+ALTER TABLE "unlistedVideos" ADD "service" TEXT NOT NULL default 'YouTube';
+
+UPDATE "unlistedVideos" 
+SET "service" = "sponsorTimes"."service" 
+FROM "sponsorTimes"
+WHERE "unlistedVideos"."videoID" = "sponsorTimes"."videoID";
+
+DROP INDEX IF EXISTS "noSegments_videoID";
+
+UPDATE "config" SET value = 24 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_25.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "videoInfo" (
+	"videoID"	TEXT PRIMARY KEY NOT NULL,
+	"channelID"	TEXT NOT NULL,
+	"title"	    TEXT NOT NULL,
+	"published"	REAL NOT NULL,
+	"genreUrl"	REAL NOT NULL
+);
+
+UPDATE "config" SET value = 25 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_26.sql

@@ -0,0 +1,18 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "sqlb_temp_table_26" (
+	"videoID"	TEXT PRIMARY KEY NOT NULL,
+	"channelID"	TEXT NOT NULL,
+	"title"	    TEXT NOT NULL,
+	"published"	REAL NOT NULL,
+	"genreUrl"	TEXT NOT NULL
+);
+
+INSERT INTO sqlb_temp_table_26 SELECT "videoID", "channelID", "title", "published", '' FROM "videoInfo";
+
+DROP TABLE "videoInfo";
+ALTER TABLE sqlb_temp_table_26 RENAME TO "videoInfo";
+
+UPDATE "config" SET value = 26 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_27.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "sponsorTimes" ADD "description" TEXT NOT NULL default '';
+ALTER TABLE "archivedSponsorTimes" ADD "description" TEXT NOT NULL default '';
+
+UPDATE "config" SET value = 27 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_28.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "ratings" (
+	"videoID"	    TEXT NOT NULL,
+	"service"	    TEXT NOT NULL default 'YouTube',
+	"type"	        INTEGER NOT NULL,
+	"count"	        INTEGER NOT NULL,
+	"hashedVideoID"	TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 28 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_29.sql

@@ -0,0 +1,21 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "sqlb_temp_table_29" (
+	"videoID"		TEXT NOT NULL,
+	"userID"		TEXT NOT NULL,
+	"actionType"	TEXT NOT NULL DEFAULT 'skip',
+	"category"		TEXT NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default '',
+	"reason" 		TEXT NOT NULL default '',
+	"service"		TEXT NOT NULL default 'YouTube'
+);
+
+INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'skip',"category","hashedVideoID","reason","service" FROM "lockCategories";
+INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'mute',"category","hashedVideoID","reason","service" FROM "lockCategories";
+
+DROP TABLE "lockCategories";
+ALTER TABLE sqlb_temp_table_29 RENAME TO "lockCategories";
+
+UPDATE "config" SET value = 29 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_3.sql

@@ -1,13 +1,11 @@
 BEGIN TRANSACTION;
 
-/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' 
+/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */
 /* Add hash field */
-ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default "";
-UPDATE sponsorTimes SET hashedVideoID = sha256(videoID);
-
-CREATE INDEX IF NOT EXISTS sponsorTimes_hashedVideoID on sponsorTimes(hashedVideoID);
+ALTER TABLE "sponsorTimes" ADD "hashedVideoID" TEXT NOT NULL default '';
+UPDATE "sponsorTimes" SET "hashedVideoID" = sha256("videoID");
 
 /* Bump version in config */
-UPDATE config SET value = 3 WHERE key = "version";
+UPDATE "config" SET value = 3 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_30.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+UPDATE "sponsorTimes" SET "actionType" = 'poi' WHERE "category" = 'poi_highlight';
+
+UPDATE "config" SET value = 30 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_31.sql

@@ -0,0 +1,25 @@
+BEGIN TRANSACTION;
+
+/* START lockCategory migrations
+no sponsor migrations
+no selfpromo migrations */
+
+/* exclusive_access migrations */
+DELETE FROM "lockCategories" WHERE "category" = 'exclusive_access' AND "actionType" != 'full';
+/* delete all full locks on categories without full */
+DELETE FROM "lockCategories" WHERE "actionType" = 'full' AND "category" in ('interaction', 'intro', 'outro', 'preview', 'filler', 'music_offtopic', 'poi_highlight');
+/* delete all non-skip music_offtopic locks */
+DELETE FROM "lockCategories" WHERE "category" = 'music_offtopic' AND "actionType" != 'skip';
+/* convert all poi_highlight to actionType poi */
+UPDATE "lockCategories" SET "actionType" = 'poi' WHERE "category" = 'poi_highlight' AND "actionType" = 'skip';
+/* delete all non-skip poi_highlight locks */
+DELETE FROM "lockCategories" WHERE "category" = 'poi_highlight' AND "actionType" != 'poi';
+
+/* END lockCategory migrations */
+
+/* delete all redundant userName entries */
+DELETE FROM "userNames" WHERE "userName" = "userID" AND "locked" = 0;
+
+UPDATE "config" SET value = 31 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_32.sql

@@ -0,0 +1,19 @@
+BEGIN TRANSACTION;
+
+-- Add primary keys
+
+ALTER TABLE "sponsorTimes" ADD PRIMARY KEY ("UUID"); --!sqlite-ignore
+ALTER TABLE "vipUsers" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
+ALTER TABLE "userNames" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
+ALTER TABLE "categoryVotes" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "lockCategories" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "warnings" ADD PRIMARY KEY ("userID", "issueTime"); --!sqlite-ignore
+ALTER TABLE "shadowBannedUsers" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
+ALTER TABLE "unlistedVideos" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+ALTER TABLE "config" ADD PRIMARY KEY ("key"); --!sqlite-ignore
+ALTER TABLE "archivedSponsorTimes" ADD PRIMARY KEY ("UUID"); --!sqlite-ignore
+ALTER TABLE "ratings" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
+
+UPDATE "config" SET value = 32 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_4.sql

@@ -2,11 +2,11 @@ BEGIN TRANSACTION;
 
 /* Create warnings table */
 CREATE TABLE "warnings" (
-    userID TEXT NOT NULL,
-    issueTime INTEGER NOT NULL,
-    issuerUserID TEXT NOT NULL
+    "userID" TEXT NOT NULL,
+    "issueTime" INTEGER NOT NULL,
+    "issuerUserID" TEXT NOT NULL
 );
 
-UPDATE config SET value = 4 WHERE key = "version";
+UPDATE "config" SET value = 4 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_5.sql

@@ -0,0 +1,17 @@
+BEGIN TRANSACTION;
+
+/* Add enabled field */
+CREATE TABLE "sqlb_temp_table_5" (
+	"userID" TEXT NOT NULL,
+    "issueTime" INTEGER NOT NULL,
+    "issuerUserID" TEXT NOT NULL,
+    enabled INTEGER NOT NULL
+);
+INSERT INTO sqlb_temp_table_5 SELECT "userID","issueTime","issuerUserID",1 FROM "warnings";
+
+DROP TABLE warnings;
+ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";;
+
+UPDATE "config" SET value = 5 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_6.sql

@@ -0,0 +1,27 @@
+BEGIN TRANSACTION;
+
+/* Add new voting field */
+CREATE TABLE "sqlb_temp_table_6" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_6 SELECT "videoID","startTime","endTime","votes",'0',"incorrectVotes","UUID","userID","timeSubmitted","views","category","shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_6 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 6 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_7.sql

@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_7" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_8.sql

@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_8" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_9.sql

@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_9" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL default '0',
+	"incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+	"service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"videoDuration"	REAL NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 9 WHERE key = 'version';
+
+COMMIT;

docker/database.env.example

@@ -0,0 +1,3 @@
+POSTGRES_USER=sponsorblock
+POSTGRES_PASSWORD=<choose-a-password>
+POSTGRES_DB=sponsorTimes

docker/docker-compose-ci.yml

@@ -0,0 +1,13 @@
+version: '3'
+services:
+  postgres:
+    image: postgres:alpine
+    environment:
+      - POSTGRES_USER=${PG_USER}
+      - POSTGRES_PASSWORD=${PG_PASS}
+    ports:
+      - 5432:5432
+  redis:
+    image: redis:alpine
+    ports:
+      - 6379:6379

docker/docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3'
+services:
+  database:
+    container_name: database
+    image: postgres:13
+    env_file: 
+      - database.env
+    volumes:
+      - database-data:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+    restart: always
+  redis:
+    container_name: redis
+    image: redis:6.0
+    command: /usr/local/etc/redis/redis.conf
+    volumes:
+      - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
+    ports:
+      - 32773:6379
+    restart: always
+  newleaf:
+    image: abeltramo/newleaf:latest
+    container_name: newleaf
+    restart: always
+    ports:
+      - 3241:3000
+    volumes:
+      - ./newleaf/configuration.py:/workdir/configuration.py
+  rsync:
+    image: mchangrh/rsync:latest
+    container_name: rsync
+    restart: always
+    ports:
+      - 873:873
+    volumes:
+      - ./rsync/rsyncd.conf:/etc/rsyncd.conf
+      - ./database-export/:/mirror
+
+volumes:
+  database-data:

docker/migrate/database.pgload.txt

@@ -0,0 +1,5 @@
+load database
+     from ./database.db
+     into postgresql://sponsorblock:pw@127.0.0.1:5432/sponsorTimes
+
+with quote identifiers;

docker/newleaf/configuration.py

@@ -0,0 +1,17 @@
+# ==============================
+#  You MUST set these settings.
+# ==============================
+
+# A URL that this site can be accessed on. Do not include a trailing slash.
+website_origin = "http://newleaf:3000"
+
+
+# ==============================
+#  These settings are optional.
+# ==============================
+
+# The address of the interface to bind to.
+#bind_host = "0.0.0.0"
+
+# The port to bind to.
+#bind_port = 3000

docker/redis/redis.conf

@@ -0,0 +1,4 @@
+maxmemory-policy allkeys-lru
+maxmemory 6500mb
+
+appendonly no

docker/rsync/rsyncd.conf

@@ -0,0 +1,15 @@
+pid file = /var/run/rsyncd.pid
+lock file = /var/run/rsync.lock
+log file = /var/log/rsync.log
+# replace with user accessing the files
+
+[sponsorblock]
+use chroot = no
+max connections = 10
+# path to mirrored files
+path = /mirror
+comment = sponsorblock-database
+read only = true
+refuse options = c delete zl
+# disallow checksumming and compression level to reduce CPU/IO load
+# disallow deleting files clientside

entrypoint.sh

@@ -1,26 +1,12 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 echo 'Entrypoint script'
 cd /usr/src/app
-cp /etc/sponsorblock/config.json . || cat <<EOF > config.json
+
+# blank config, use defaults
+cat <<EOF > config.json
 {
-    "port": 8080,
-    "globalSalt": "[CHANGE THIS]",
-    "adminUserID": "[CHANGE THIS]",
-    "youtubeAPIKey": null,
-    "discordReportChannelWebhookURL": null, 
-    "discordFirstTimeSubmissionsWebhookURL": null, 
-    "discordAutoModWebhookURL": null,
-    "proxySubmission": null,
-    "behindProxy": "X-Forwarded-For",
-    "db": "./databases/sponsorTimes.db",
-    "privateDB": "./databases/private.db",
-    "createDatabaseIfNotExist": true,
-    "schemaFolder": "./databases",
-    "dbSchema": "./databases/_sponsorTimes.db.sql",
-    "privateDBSchema": "./databases/_private.db.sql",
-    "mode": "development",
-    "readOnly": false
 }
 EOF
-node index.js
+
+node dist/src/index.js

index.js

@@ -1,6 +0,0 @@
-var config = require('./src/config.js');
-var createServer = require('./src/app.js');
-const logger = require('./src/utils/logger.js');
-var server = createServer(() => {
-  logger.info("Server started on port " + config.port + ".");
-});

nginx/cors.conf

@@ -0,0 +1,11 @@
+if ($request_method = 'OPTIONS') {
+  add_header 'Access-Control-Allow-Origin' '*';
+  add_header 'Access-Control-Allow-Methods' 'GET, POST, DELETE';
+  add_header 'Access-Control-Allow-Headers' 'Content-Type';
+  # cache CORS for 24 hours
+  add_header 'Access-Control-Max-Age' 86400;
+  # return empty response for preflight
+  add_header 'Content-Type' 'text/plain; charset=UTF-8';
+  add_header 'Content-Length' 0;
+  return 204;
+}

nginx/error.conf

@@ -0,0 +1,7 @@
+error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html;
+
+location = /error.html {
+  ssi on;
+  internal;
+  root /etc/nginx/error;
+}

nginx/error/error.html

@@ -0,0 +1 @@
+<!--# echo var="status"--> <!--# echo var="status_text"--> https://status.sponsor.ajay.app

nginx/error_map.conf

@@ -0,0 +1,15 @@
+map $status $status_text {
+  400 'Bad Request';
+  401 'Unauthorized';
+  403 'Forbidden';
+  404 'Not Found';
+  405 'Method Not Allowed';
+  408 'Request Timeout';
+  409 'Conflict';
+  429 'Too Many Requests';
+  500 'Internal Server Error';
+  502 'Bad Gateway';
+  503 'Service Unavailable';
+  504 'Gateway Timeout';
+  505 'HTTP Version Not Supported';
+}

nginx/nginx.conf

@@ -0,0 +1,317 @@
+worker_processes 2;
+worker_rlimit_nofile 500000;
+worker_shutdown_timeout 10;
+
+events {
+	worker_connections 100000; # Default: 1024
+	#use epoll;
+	#multi_accept on;
+}
+
+http {
+	log_format no_ip    '$remote_user [$time_local] '
+                        '"$request" $status $body_bytes_sent '
+                        '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+    log_format user_agent '[$time_local] '
+                           '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+	#limit_req_zone $binary_remote_addr zone=mylimit:10m rate=10r/s;
+	limit_req_log_level warn;
+	
+	include		/etc/nginx/mime.types;
+	include		/etc/nginx/proxy.conf;
+	
+	# error_map has to be at http level
+	include	  /etc/nginx/error_map.conf;
+	# Custom MIME definition
+	types {
+		text/csv	csv;
+	}
+	# keepalive settings
+	#keepalive_requests		10;
+	keepalive_timeout		10s;
+	http2_idle_timeout		20s; # replaced by keepalive_timeout in 1.19.7	
+
+	access_log off;
+	#error_log /etc/nginx/logs/error.log warn;
+	error_log /dev/null crit;
+
+	upstream backend_GET {
+		least_conn;
+		
+		#keepalive 	5;
+		#server		localhost:4441;
+		#server		localhost:4442;
+		#server		localhost:4443;
+		#server		localhost:4444;
+		#server		localhost:4445;
+		#server		localhost:4446;
+		#server		localhost:4447;
+		#server		localhost:4448;
+		#server 		10.0.0.4:4441 max_fails=25 fail_timeout=20s;
+
+		#server		10.0.0.3:4441 max_fails=25 fail_timeout=20s;
+		#server		10.0.0.3:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.5:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.5:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.6:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.6:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.9:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.9:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.12:4441 max_fails=25 fail_timeout=20s;
+		server		10.0.0.12:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.10:4441 max_fails=25 fail_timeout=20s;
+        server		10.0.0.10:4442 max_fails=25 fail_timeout=20s;
+
+		server		10.0.0.13:4441 max_fails=25 fail_timeout=20s;
+        server		10.0.0.13:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.14:4441 max_fails=25 fail_timeout=20s;
+        server		10.0.0.14:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.11:4441 max_fails=25 fail_timeout=20s;
+        	server		10.0.0.11:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.16:4441 max_fails=25 fail_timeout=20s;
+        	server		10.0.0.16:4442 max_fails=25 fail_timeout=20s;
+		
+		server		10.0.0.17:4441 max_fails=25 fail_timeout=20s;
+        	server		10.0.0.17:4442 max_fails=25 fail_timeout=20s;
+
+		#server		134.209.69.251:80	backup;
+
+		#server		116.203.32.253:80	backup;
+		#server		116.203.32.253:80;
+	}
+	upstream backend_POST {
+		#server		localhost:4441;
+		#server		localhost:4442;
+		server		10.0.0.3:4441 max_fails=25 fail_timeout=15s;
+		server		10.0.0.4:4441 max_fails=25 fail_timeout=15s;
+		#server		10.0.0.3:4442;
+	}
+	upstream backend_db {
+		server		10.0.0.4:4441 max_fails=1 fail_timeout=3s;
+		#server		10.0.0.3:4441;
+		#server		10.0.0.4; 
+	}
+	upstream backend_db_dl {
+		server		10.0.0.4;
+	}
+
+	proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
+	proxy_cache_key "$scheme$request_method$host$request_uri";
+	add_header X-Cache $upstream_cache_status;
+
+	server {
+		server_name sponsor.ajay.app api.sponsor.ajay.app;
+
+		include		  /etc/nginx/error.conf;
+		set_real_ip_from 10.0.0.0/24;
+		real_ip_header proxy_protocol;
+
+		location /news {
+			return	301		https://blog.ajay.app/sponsorblock;
+		}
+
+		location /viewer {
+			return	301		https://sb.ltn.fi;
+		}
+
+		location /test/ {
+			# return			404 "";
+			proxy_pass		http://10.0.0.4:4445/;
+			#proxy_pass		https://sbtest.etcinit.com/;
+		}
+
+		#access_log /etc/nginx/logs/requests.log no_ip buffer=64k;
+
+		location /api/skipSegments {
+			include				/etc/nginx/cors.conf;
+			#return 200 "[]";
+			proxy_pass		http://backend_$request_method;
+			#proxy_cache	CACHEZONE;
+			#proxy_cache_valid	10s;
+			#limit_req zone=mylimit;
+
+			#access_log /etc/nginx/logs/download.log no_ip;
+			gzip on;
+			if ($request_method = POST) {
+				access_log /etc/nginx/logs/submissions.log user_agent buffer=64k;
+			}
+
+			#proxy_read_timeout 6s;
+			#proxy_next_upstream error timeout http_500 http_502;
+		}
+
+		location /api/getTopUsers {
+			include				/etc/nginx/cors.conf;
+			proxy_pass		http://backend_GET;
+			proxy_cache		CACHEZONE;
+			proxy_cache_valid	20m;
+		}
+
+		location /api/getTotalStats {
+			include		 /etc/nginx/cors.conf;
+			proxy_pass http://backend_POST;
+			proxy_cache CACHEZONE;
+			proxy_cache_valid 20m;
+			#return 204;
+		}
+
+		location /api/getTopCategoryUsers {
+			include				/etc/nginx/cors.conf;
+			proxy_pass		http://backend_POST;
+			proxy_cache		CACHEZONE;
+			proxy_cache_valid	20m;
+		}
+
+		location /api/getVideoSponsorTimes {
+			include		/etc/nginx/cors.conf;
+			proxy_pass http://backend_GET;
+		}
+		
+		location /api/isUserVIP {
+			include		/etc/nginx/cors.conf;
+			proxy_pass http://backend_GET;
+		}
+
+		location /download/ {
+			#access_log /etc/nginx/logs/download.log no_ip buffer=64k;
+			gzip on;
+			proxy_max_temp_file_size 0;
+			#proxy_cache CACHEZONE;
+			#proxy_cache_valid 20m;
+			#proxy_http_version 1.0;
+			#gzip_types	text/csv;
+			#gzip_comp_level 1;
+			#proxy_buffering off;		
+	
+
+			proxy_pass http://backend_db;
+      #alias		/home/sbadmin/sponsor/docker/database-export/;
+			#return 307 https://rsync.sponsor.ajay.app$request_uri;
+		}
+
+		location /database {
+			proxy_pass  http://backend_db;
+			#return 200 "Disabled for load reasons";
+		}
+
+		location = /database.db {
+			return	404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
+			#alias	/home/sbadmin/sponsor/databases/sponsorTimes.db;
+			#alias	/home/sbadmin/test-db/database.db;
+		}
+
+		#location = /database/sponsorTimes.csv {
+		#	alias /home/sbadmin/sponsorTimes.csv;
+		#}
+
+		#location /api/voteOnSponsorTime {
+		#	return 200 "Success";
+		#}
+	
+		#location /api/viewedVideoSponsorTime {
+		#	return 200 "Success";
+		#}
+
+		location /api {
+			include			/etc/nginx/cors.conf;
+			proxy_pass	http://backend_POST;
+		}
+		
+		location / {
+			root	/home/sbadmin/SponsorBlockSite/public-prod;
+			error_page 404 /404.html;
+		}
+		
+		listen [::]:443 default_server ssl http2 ipv6only=on backlog=323999;
+		listen 443 default_server ssl http2 reuseport backlog=3000999; # managed by Certbot
+		listen 4443 default_server ssl http2 proxy_protocol reuseport backlog=3000999;
+		#listen 443 http3 reuseport;
+		#ssl_protocols TLSv1.2 TLSv1.3;
+		listen 8081 proxy_protocol;
+		port_in_redirect off;
+    		ssl_certificate /home/sbadmin/certs/cert.pem;
+   		ssl_certificate_key /home/sbadmin/certs/key.pem;
+		#ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+		#ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+		include				/etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+		ssl_dhparam			/etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+	}
+
+	server {
+		server_name cdnsponsor.ajay.app;
+		error_page 404 /404.html;
+
+		#location /database/ {
+		#	alias /home/sbadmin/sponsor/docker/database-export/;
+		#}
+
+		#location /download/ {
+		#	alias /home/sbadmin/sponsor/docker/database-export/;
+		#}
+
+		location / {
+			root /home/sbadmin/SponsorBlockSite/public-prod;
+		}
+
+
+		listen 443 ssl; # managed by Certbot
+		ssl_certificate /home/sbadmin/certs/cert.pem;
+                ssl_certificate_key /home/sbadmin/certs/key.pem;
+    		#ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+    		#ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+		include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+		ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+	}
+
+	server {
+		access_log off;
+
+		return 301 https://$host$request_uri;
+
+		listen [::]:80 ipv6only=on;
+		listen 8080 proxy_protocol;
+		listen 80;
+		server_name sponsor.ajay.app api.sponsor.ajay.app, cdnsponsor.ajay.app, wiki.sponsor.ajay.app;
+		return 404; # managed by Certbot
+	}
+
+	server {
+		server_name wiki.sponsor.ajay.app; # managed by Certbot
+		
+		location /.well-known/ {
+			 root    /home/sbadmin/SponsorBlockSite/public-prod;
+		}
+
+		location ~* ^/index.php/(?<pagename>.*)$ {
+			return 301 /w/$pagename;
+		}
+
+		location / {
+			proxy_pass http://10.0.0.3:8080;
+		}
+		
+		port_in_redirect off;
+		listen [::]:443 ssl http2; 
+		listen 443  ssl http2; # managed by Certbot
+		listen 8081 proxy_protocol;
+		#listen 443 http3 reuseport;
+		#ssl_protocols TLSv1.2 TLSv1.3;
+		#listen 80;
+		ssl_certificate /home/sbadmin/certs/cert.pem;
+   		ssl_certificate_key /home/sbadmin/certs/key.pem;
+    		#ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
+    		#ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
+		include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+		ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+	}
+}

nginx/proxy.conf

@@ -0,0 +1,12 @@
+proxy_redirect            off;
+proxy_set_header          Host            $host;
+proxy_set_header          X-Real-IP       $remote_addr;
+proxy_set_header          X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header          Connection      "";
+client_max_body_size      10m;
+client_body_buffer_size   128k;
+proxy_connect_timeout     5s;
+#proxy_send_timeout        10;
+proxy_read_timeout        30s;
+proxy_buffers             32 4k;
+proxy_http_version        1.1;

nodemon.json

@@ -0,0 +1,5 @@
+{
+    "watch": ["src"],
+    "ext": "ts,json",
+    "exec": "(npm test || echo test failed) && npm start"
+}

package.json

@@ -2,29 +2,52 @@
   "name": "sponsor_block_server",
   "version": "0.1.0",
   "description": "Server that holds the SponsorBlock database",
-  "main": "index.js",
+  "main": "src/index.ts",
   "scripts": {
-    "test": "node test.js",
-    "dev": "nodemon -x \"(npm test || echo test failed) && npm start\"",
+    "test": "npm run tsc && ts-node test/test.ts",
+    "dev": "nodemon",
     "dev:bash": "nodemon -x 'npm test ; npm start'",
-    "start": "node index.js"
+    "postgres:docker": "docker run --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:alpine",
+    "redis:docker": "docker run --rm -p 6379:6379 redis:alpine",
+    "start": "ts-node src/index.ts",
+    "tsc": "tsc -p tsconfig.json",
+    "lint": "eslint src test",
+    "lint:fix": "eslint src test --fix"
   },
   "author": "Ajay Ramachandran",
   "license": "MIT",
   "dependencies": {
-    "better-sqlite3": "^5.4.3",
+    "axios": "^0.24.0",
+    "better-sqlite3": "^7.4.5",
+    "cron": "^1.8.2",
     "express": "^4.17.1",
-    "express-rate-limit": "^5.1.3",
-    "http": "0.0.0",
-    "iso8601-duration": "^1.2.0",
-    "node-fetch": "^2.6.0",
-    "redis": "^3.0.2",
-    "sync-mysql": "^3.0.1",
-    "uuid": "^3.3.2",
-    "youtube-api": "^2.0.10"
+    "express-promise-router": "^4.1.1",
+    "express-rate-limit": "^6.3.0",
+    "lodash": "^4.17.21",
+    "pg": "^8.7.1",
+    "rate-limit-redis": "^3.0.1",
+    "redis": "^4.0.6",
+    "sync-mysql": "^3.0.1"
   },
   "devDependencies": {
-    "mocha": "^7.1.1",
-    "nodemon": "^2.0.2"
+    "@types/better-sqlite3": "^7.4.1",
+    "@types/cron": "^1.7.3",
+    "@types/express": "^4.17.13",
+    "@types/lodash": "^4.14.178",
+    "@types/mocha": "^9.0.0",
+    "@types/node": "^16.11.11",
+    "@types/pg": "^8.6.1",
+    "@typescript-eslint/eslint-plugin": "^5.5.0",
+    "@typescript-eslint/parser": "^5.5.0",
+    "eslint": "^8.3.0",
+    "mocha": "^9.1.3",
+    "nodemon": "^2.0.15",
+    "sinon": "^12.0.1",
+    "ts-mock-imports": "^1.3.8",
+    "ts-node": "^10.4.0",
+    "typescript": "^4.5.2"
+  },
+  "engines": {
+    "node": ">=10"
   }
 }

src/app.js

@@ -1,137 +0,0 @@
-var express = require('express');
-// Create a service (the app object is just a callback).
-var app = express();
-var config = require('./config.js');
-var redis = require('./utils/redis.js');
-const getIP = require('./utils/getIP.js');
-const getHash = require('./utils/getHash.js');
-
-// Middleware 
-const rateLimitMiddleware = require('./middleware/requestRateLimit.js');
-var corsMiddleware = require('./middleware/cors.js');
-var loggerMiddleware = require('./middleware/logger.js');
-const userCounter = require('./middleware/userCounter.js');
-
-// Routes
-var getSkipSegments = require('./routes/getSkipSegments.js').endpoint;
-var postSkipSegments = require('./routes/postSkipSegments.js');
-var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js');
-var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js');
-var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js');
-var setUsername = require('./routes/setUsername.js');
-var getUsername = require('./routes/getUsername.js');
-var shadowBanUser = require('./routes/shadowBanUser.js');
-var addUserAsVIP = require('./routes/addUserAsVIP.js');
-var getSavedTimeForUser = require('./routes/getSavedTimeForUser.js');
-var getViewsForUser = require('./routes/getViewsForUser.js');
-var getTopUsers = require('./routes/getTopUsers.js');
-var getTotalStats = require('./routes/getTotalStats.js');
-var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js');
-var getUserInfo = require('./routes/getUserInfo.js');
-var postNoSegments = require('./routes/postNoSegments.js');
-var deleteNoSegments = require('./routes/deleteNoSegments.js');
-var getIsUserVIP = require('./routes/getIsUserVIP.js');
-var warnUser = require('./routes/postWarning.js');
-var postSegmentShift = require('./routes/postSegmentShift.js');
-
-// Old Routes
-var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js');
-var oldSubmitSponsorTimes = require('./routes/oldSubmitSponsorTimes.js');
-
-// Rate limit endpoint lists
-let voteEndpoints = [voteOnSponsorTime.endpoint];
-let viewEndpoints = [viewedVideoSponsorTime];
-if (config.rateLimit) {
-    if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote));
-    if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
-}
-
-//setup CORS correctly
-app.use(corsMiddleware);
-app.use(loggerMiddleware);
-app.use(express.json())
-
-if (config.userCounterURL) app.use(userCounter);
-
-// Setup pretty JSON
-if (config.mode === "development") app.set('json spaces', 2);
-
-// Set production mode
-app.set('env', config.mode || 'production');
-
-//add the get function
-app.get('/api/getVideoSponsorTimes', oldGetVideoSponsorTimes);
-
-//add the oldpost function
-app.get('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
-app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes);
-
-//add the skip segments functions
-app.get('/api/skipSegments', getSkipSegments);
-app.post('/api/skipSegments', postSkipSegments);
-
-// add the privacy protecting skip segments functions
-app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash);
-
-//voting endpoint
-app.get('/api/voteOnSponsorTime', ...voteEndpoints);
-app.post('/api/voteOnSponsorTime', ...voteEndpoints);
-
-//Endpoint when a submission is skipped
-app.get('/api/viewedVideoSponsorTime', ...viewEndpoints);
-app.post('/api/viewedVideoSponsorTime', ...viewEndpoints);
-
-//To set your username for the stats view
-app.post('/api/setUsername', setUsername);
-
-//get what username this user has
-app.get('/api/getUsername', getUsername);
-
-//Endpoint used to hide a certain user's data
-app.post('/api/shadowBanUser', shadowBanUser);
-
-//Endpoint used to make a user a VIP user with special privileges
-app.post('/api/addUserAsVIP', addUserAsVIP);
-
-//Gets all the views added up for one userID
-//Useful to see how much one user has contributed
-app.get('/api/getViewsForUser', getViewsForUser);
-
-//Gets all the saved time added up (views * sponsor length) for one userID
-//Useful to see how much one user has contributed
-//In minutes
-app.get('/api/getSavedTimeForUser', getSavedTimeForUser);
-
-app.get('/api/getTopUsers', getTopUsers);
-
-//send out totals
-//send the total submissions, total views and total minutes saved
-app.get('/api/getTotalStats', getTotalStats);
-
-app.get('/api/getUserInfo', getUserInfo);
-
-//send out a formatted time saved total
-app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
-
-//submit video containing no segments
-app.post('/api/noSegments', postNoSegments);
-
-app.delete('/api/noSegments', deleteNoSegments);
-
-//get if user is a vip
-app.get('/api/isUserVIP', getIsUserVIP);
-
-//sent user a warning
-app.post('/api/warnUser', warnUser);
-
-//get if user is a vip
-app.post('/api/segmentShift', postSegmentShift);
-
-app.get('/database.db', function (req, res) {
-    res.sendFile("./databases/sponsorTimes.db", { root: "./" });
-});
-
-// Create an HTTP service.
-module.exports = function createServer (callback) {
-    return app.listen(config.port, callback);
-}

src/app.ts

@@ -0,0 +1,215 @@
+import express, { Request, RequestHandler, Response, Router } from "express";
+import { config } from "./config";
+import { oldSubmitSponsorTimes } from "./routes/oldSubmitSponsorTimes";
+import { oldGetVideoSponsorTimes } from "./routes/oldGetVideoSponsorTimes";
+import { postSegmentShift } from "./routes/postSegmentShift";
+import { postWarning } from "./routes/postWarning";
+import { getIsUserVIP } from "./routes/getIsUserVIP";
+import { deleteLockCategoriesEndpoint } from "./routes/deleteLockCategories";
+import { postLockCategories } from "./routes/postLockCategories";
+import { endpoint as getUserInfo } from "./routes/getUserInfo";
+import { getDaysSavedFormatted } from "./routes/getDaysSavedFormatted";
+import { getTotalStats } from "./routes/getTotalStats";
+import { getTopUsers } from "./routes/getTopUsers";
+import { getViewsForUser } from "./routes/getViewsForUser";
+import { getSavedTimeForUser } from "./routes/getSavedTimeForUser";
+import { addUserAsVIP } from "./routes/addUserAsVIP";
+import { shadowBanUser } from "./routes/shadowBanUser";
+import { getUsername } from "./routes/getUsername";
+import { setUsername } from "./routes/setUsername";
+import { viewedVideoSponsorTime } from "./routes/viewedVideoSponsorTime";
+import { voteOnSponsorTime, getUserID as voteGetUserID } from "./routes/voteOnSponsorTime";
+import { getSkipSegmentsByHash } from "./routes/getSkipSegmentsByHash";
+import { postSkipSegments } from "./routes/postSkipSegments";
+import { endpoint as getSkipSegments } from "./routes/getSkipSegments";
+import { userCounter } from "./middleware/userCounter";
+import { loggerMiddleware } from "./middleware/logger";
+import { corsMiddleware } from "./middleware/cors";
+import { apiCspMiddleware } from "./middleware/apiCsp";
+import { rateLimitMiddleware } from "./middleware/requestRateLimit";
+import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
+import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
+import { postClearCache } from "./routes/postClearCache";
+import { addUnlistedVideo } from "./routes/addUnlistedVideo";
+import { postPurgeAllSegments } from "./routes/postPurgeAllSegments";
+import { getUserID } from "./routes/getUserID";
+import { getLockCategories } from "./routes/getLockCategories";
+import { getLockCategoriesByHash } from "./routes/getLockCategoriesByHash";
+import { endpoint as getSearchSegments } from "./routes/getSearchSegments";
+import { getStatus } from "./routes/getStatus";
+import { getLockReason } from "./routes/getLockReason";
+import { getUserStats } from "./routes/getUserStats";
+import ExpressPromiseRouter from "express-promise-router";
+import { Server } from "http";
+import { youtubeApiProxy } from "./routes/youtubeApiProxy";
+import { getChapterNames } from "./routes/getChapterNames";
+import { postRating } from "./routes/ratings/postRating";
+import { getRating } from "./routes/ratings/getRating";
+import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache";
+import { getTopCategoryUsers } from "./routes/getTopCategoryUsers";
+import { addUserAsTempVIP } from "./routes/addUserAsTempVIP";
+
+export function createServer(callback: () => void): Server {
+    // Create a service (the app object is just a callback).
+    const app = express();
+
+    const router = ExpressPromiseRouter();
+    app.use(router);
+
+    //setup CORS correctly
+    router.use(corsMiddleware);
+    router.use(loggerMiddleware);
+    router.use("/api/", apiCspMiddleware);
+    router.use(express.json());
+
+    if (config.userCounterURL) router.use(userCounter);
+
+    // Setup pretty JSON
+    if (config.mode === "development") app.set("json spaces", 2);
+
+    // Set production mode
+    app.set("env", config.mode || "production");
+
+    setupRoutes(router);
+
+    return app.listen(config.port, callback);
+}
+
+function setupRoutes(router: Router) {
+    // Rate limit endpoint lists
+    const voteEndpoints: RequestHandler[] = [voteOnSponsorTime];
+    const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime];
+    const postRateEndpoints: RequestHandler[] = [postRating];
+    if (config.rateLimit) {
+        if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID));
+        if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
+        if (config.rateLimit.rate) postRateEndpoints.unshift(rateLimitMiddleware(config.rateLimit.rate));
+    }
+
+    //add the get function
+    router.get("/api/getVideoSponsorTimes", oldGetVideoSponsorTimes);
+
+    //add the oldpost function
+    router.get("/api/postVideoSponsorTimes", oldSubmitSponsorTimes);
+    router.post("/api/postVideoSponsorTimes", oldSubmitSponsorTimes);
+
+    //add the skip segments functions
+    router.get("/api/skipSegments", getSkipSegments);
+    router.post("/api/skipSegments", postSkipSegments);
+
+    // add the privacy protecting skip segments functions
+    router.get("/api/skipSegments/:prefix", getSkipSegmentsByHash);
+
+    //voting endpoint
+    router.get("/api/voteOnSponsorTime", ...voteEndpoints);
+    router.post("/api/voteOnSponsorTime", ...voteEndpoints);
+
+    //Endpoint when a submission is skipped
+    router.get("/api/viewedVideoSponsorTime", ...viewEndpoints);
+    router.post("/api/viewedVideoSponsorTime", ...viewEndpoints);
+
+    //To set your username for the stats view
+    router.post("/api/setUsername", setUsername);
+
+    //get what username this user has
+    router.get("/api/getUsername", getUsername);
+
+    //Endpoint used to hide a certain user's data
+    router.post("/api/shadowBanUser", shadowBanUser);
+
+    //Endpoint used to make a user a VIP user with special privileges
+    router.post("/api/addUserAsVIP", addUserAsVIP);
+    //Endpoint to add a user as a temporary VIP
+    router.post("/api/addUserAsTempVIP", addUserAsTempVIP);
+
+    //Gets all the views added up for one userID
+    //Useful to see how much one user has contributed
+    router.get("/api/getViewsForUser", getViewsForUser);
+
+    //Gets all the saved time added up (views * sponsor length) for one userID
+    //Useful to see how much one user has contributed
+    //In minutes
+    router.get("/api/getSavedTimeForUser", getSavedTimeForUser);
+
+    router.get("/api/getTopUsers", getTopUsers);
+    router.get("/api/getTopCategoryUsers", getTopCategoryUsers);
+
+    //send out totals
+    //send the total submissions, total views and total minutes saved
+    router.get("/api/getTotalStats", getTotalStats);
+
+    router.get("/api/getUserInfo", getUserInfo);
+    router.get("/api/userInfo", getUserInfo);
+
+    //send out a formatted time saved total
+    router.get("/api/getDaysSavedFormatted", getDaysSavedFormatted);
+
+    //submit video to lock categories
+    router.post("/api/noSegments", postLockCategories);
+    router.post("/api/lockCategories", postLockCategories);
+
+    router.delete("/api/noSegments", deleteLockCategoriesEndpoint);
+    router.delete("/api/lockCategories", deleteLockCategoriesEndpoint);
+
+    //get if user is a vip
+    router.get("/api/isUserVIP", getIsUserVIP);
+
+    //sent user a warning
+    router.post("/api/warnUser", postWarning);
+
+    //get if user is a vip
+    router.post("/api/segmentShift", postSegmentShift);
+
+    //get segment info
+    router.get("/api/segmentInfo", getSegmentInfo);
+
+    //clear cache as VIP
+    router.post("/api/clearCache", postClearCache);
+
+    //purge all segments for VIP
+    router.post("/api/purgeAllSegments", postPurgeAllSegments);
+
+    router.post("/api/unlistedVideo", addUnlistedVideo);
+
+    // get userID from username
+    router.get("/api/userID", getUserID);
+
+    // get lock categores from userID
+    router.get("/api/lockCategories", getLockCategories);
+
+    // get privacy protecting lock categories functions
+    router.get("/api/lockCategories/:prefix", getLockCategoriesByHash);
+
+    // get all segments that match a search
+    router.get("/api/searchSegments", getSearchSegments);
+
+    // autocomplete chapter names
+    router.get("/api/chapterNames", getChapterNames);
+
+    // get status
+    router.get("/api/status/:value", getStatus);
+    router.get("/api/status", getStatus);
+
+    router.get("/api/youtubeApiProxy", youtubeApiProxy);
+    // get user category stats
+    router.get("/api/userStats", getUserStats);
+
+    router.get("/api/lockReason", getLockReason);
+
+    // ratings
+    router.get("/api/ratings/rate/:prefix", getRating);
+    router.get("/api/ratings/rate", getRating);
+    router.post("/api/ratings/rate", postRateEndpoints);
+    router.post("/api/ratings/clearCache", ratingPostClearCache);
+
+    if (config.postgres?.enabled) {
+        router.get("/database", (req, res) => dumpDatabase(req, res, true));
+        router.get("/database.json", (req, res) => dumpDatabase(req, res, false));
+        router.get("/database/*", downloadFile);
+        router.use("/download", express.static(appExportPath));
+    } else {
+        router.get("/database.db", function (req: Request, res: Response) {
+            res.sendFile("./databases/sponsorTimes.db", { root: "./" });
+        });
+    }
+}

src/config.js

@@ -1,37 +0,0 @@
-
-const fs = require('fs');
-let config = {};
-
-// Check to see if launched in test mode
-if (process.env.npm_lifecycle_script === 'node test.js') {
-  config = JSON.parse(fs.readFileSync('test.json'));
-} else {
-  config = JSON.parse(fs.readFileSync('config.json'));
-}
-
-addDefaults(config, {
-  "port": 80,
-  "behindProxy": "X-Forwarded-For",
-  "db": "./databases/sponsorTimes.db",
-  "privateDB": "./databases/private.db",
-  "createDatabaseIfNotExist": true,
-  "schemaFolder": "./databases",
-  "dbSchema": "./databases/_sponsorTimes.db.sql",
-  "privateDBSchema": "./databases/_private.db.sql",
-  "readOnly": false,
-  "webhooks": [],
-  "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
-  "maxNumberOfActiveWarnings": 3,
-  "hoursAfterWarningExpires": 24
-})
-
-module.exports = config;
-
-// Add defaults
-function addDefaults(config, defaults) {
-  for (const key in defaults) {
-      if(!config.hasOwnProperty(key)) {
-        config[key] = defaults[key];
-      }
-  }
-};

src/config.ts

@@ -0,0 +1,185 @@
+import fs from "fs";
+import { SBSConfig } from "./types/config.model";
+import packageJson from "../package.json";
+import { isBoolean, isNumber } from "lodash";
+
+const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test;
+const configFile = process.env.TEST_POSTGRES ? "ci.json"
+    : isTestMode ? "test.json"
+        : "config.json";
+export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8"));
+
+addDefaults(config, {
+    port: 8080,
+    behindProxy: "X-Forwarded-For",
+    db: "./databases/sponsorTimes.db",
+    privateDB: "./databases/private.db",
+    createDatabaseIfNotExist: true,
+    schemaFolder: "./databases",
+    dbSchema: "./databases/_sponsorTimes.db.sql",
+    privateDBSchema: "./databases/_private.db.sql",
+    readOnly: false,
+    webhooks: [],
+    categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"],
+    categorySupport: {
+        sponsor: ["skip", "mute", "full"],
+        selfpromo: ["skip", "mute", "full"],
+        exclusive_access: ["full"],
+        interaction: ["skip", "mute"],
+        intro: ["skip", "mute"],
+        outro: ["skip", "mute"],
+        preview: ["skip", "mute"],
+        filler: ["skip", "mute"],
+        music_offtopic: ["skip"],
+        poi_highlight: ["poi"],
+        chapter: ["chapter"]
+    },
+    maxNumberOfActiveWarnings: 1,
+    hoursAfterWarningExpires: 16300000,
+    adminUserID: "",
+    discordCompletelyIncorrectReportWebhookURL: null,
+    discordFirstTimeSubmissionsWebhookURL: null,
+    discordNeuralBlockRejectWebhookURL: null,
+    discordFailedReportChannelWebhookURL: null,
+    discordReportChannelWebhookURL: null,
+    getTopUsersCacheTimeMinutes: 240,
+    globalSalt: null,
+    mode: "",
+    neuralBlockURL: null,
+    proxySubmission: null,
+    rateLimit: {
+        vote: {
+            windowMs: 900000,
+            max: 15,
+            message: "OK",
+            statusCode: 200,
+        },
+        view: {
+            windowMs: 900000,
+            max: 10,
+            statusCode: 200,
+            message: "OK",
+        },
+        rate: {
+            windowMs: 900000,
+            max: 20,
+            statusCode: 200,
+            message: "Success",
+        }
+    },
+    userCounterURL: null,
+    newLeafURLs: null,
+    maxRewardTimePerSegmentInSeconds: 600,
+    poiMinimumStartTime: 2,
+    postgres: {
+        enabled: false,
+        user: "",
+        host: "",
+        password: "",
+        port: 5432,
+        max: 150,
+        min: 10
+    },
+    dumpDatabase: {
+        enabled: false,
+        minTimeBetweenMs: 180000,
+        appExportPath: "./docker/database-export",
+        tables: [{
+            name: "sponsorTimes",
+            order: "timeSubmitted"
+        },
+        {
+            name: "userNames"
+        },
+        {
+            name: "categoryVotes"
+        },
+        {
+            name: "lockCategories",
+        },
+        {
+            name: "warnings",
+            order: "issueTime"
+        },
+        {
+            name: "vipUsers"
+        },
+        {
+            name: "unlistedVideos"
+        },
+        {
+            name: "videoInfo"
+        },
+        {
+            name: "ratings"
+        }]
+    },
+    diskCacheURL: null,
+    crons: null,
+    redis: {
+        enabled: false,
+        socket: {
+            host: "",
+            port: 0
+        },
+        disableOfflineQueue: true
+    }
+});
+loadFromEnv(config);
+migrate(config);
+
+// Add defaults
+function addDefaults(config: SBSConfig, defaults: SBSConfig) {
+    for (const key in defaults) {
+        if (!Object.prototype.hasOwnProperty.call(config, key)) {
+            config[key] = defaults[key];
+        }
+    }
+}
+
+function migrate(config: SBSConfig) {
+    // Redis change
+    if (config.redis) {
+        const redisConfig = config.redis as any;
+        if (redisConfig.host || redisConfig.port) {
+            config.redis.socket = {
+                host: redisConfig.host,
+                port: redisConfig.port
+            };
+        }
+
+        if (redisConfig.enable_offline_queue !== undefined) {
+            config.disableOfflineQueue = !redisConfig.enable_offline_queue;
+        }
+
+        if (redisConfig.socket?.host && redisConfig.enabled === undefined) {
+            redisConfig.enabled = true;
+        }
+    }
+
+    if (config.postgres && config.postgres.user && config.postgres.enabled === undefined) {
+        config.postgres.enabled = true;
+    }
+}
+
+function loadFromEnv(config: SBSConfig, prefix = "") {
+    for (const key in config) {
+        const fullKey = (prefix ? `${prefix}_` : "") + key;
+        const data = config[key];
+
+        if (data && typeof data === "object" && !Array.isArray(data)) {
+            loadFromEnv(data, fullKey);
+        } else if (process.env[fullKey]) {
+            const value = process.env[fullKey];
+            if (isNumber(value)) {
+                config[key] = parseInt(value, 10);
+            } else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") {
+                config[key] = value === "true";
+            } else if (key === "newLeafURLs") {
+                config[key] = [value];
+            } else {
+                config[key] = value;
+            }
+        }
+    }
+}

src/cronjob/downvoteSegmentArchiveJob.ts

@@ -0,0 +1,67 @@
+import { CronJob } from "cron";
+
+import { config as serverConfig } from "../config";
+import { Logger } from "../utils/logger";
+import { db } from "../databases/databases";
+import { DBSegment } from "../types/segments.model";
+
+const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
+
+export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
+    const timeNow = runTime || new Date().getTime();
+    const threshold = dayLimit * 86400000;
+
+    Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
+    try {
+    // insert into archive sponsorTime
+    await db.prepare(
+        "run",
+        `INSERT INTO "archivedSponsorTimes" 
+        SELECT * 
+        FROM "sponsorTimes" 
+        WHERE "votes" < ? AND (? - "timeSubmitted") > ?`,
+        [
+            voteLimit,
+            timeNow,
+            threshold
+        ]
+    ) as DBSegment[];
+
+    } catch (err) {
+        Logger.error("Execption when insert segment in archivedSponsorTimes");
+        Logger.error(err as string);
+        return 1;
+    }
+
+    // remove from sponsorTime
+    try {
+    await db.prepare(
+        "run",
+        'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?',
+        [
+            voteLimit,
+            timeNow,
+            threshold
+        ]
+    ) as DBSegment[];
+
+    } catch (err) {
+        Logger.error("Execption when deleting segment in sponsorTimes");
+        Logger.error(err as string);
+        return 1;
+    }
+
+    Logger.info("DownvoteSegmentArchiveJob finished");
+    return 0;
+};
+
+const DownvoteSegmentArchiveJob = new CronJob(
+    jobConfig?.schedule || "0 0 * * * 0",
+    () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
+);
+
+if (serverConfig?.crons?.enabled && jobConfig && !jobConfig.schedule) {
+    Logger.error("Invalid cron schedule for downvoteSegmentArchive");
+}
+
+export default DownvoteSegmentArchiveJob;

src/cronjob/index.ts

@@ -0,0 +1,13 @@
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
+
+export function startAllCrons (): void {
+    if (config?.crons?.enabled) {
+        Logger.info("Crons started");
+
+        DownvoteSegmentArchiveJob.start();
+    } else {
+        Logger.info("Crons dissabled");
+    }
+}

src/databases/IDatabase.ts

@@ -0,0 +1,7 @@
+export interface IDatabase {
+    init(): Promise<void>;
+
+    prepare(type: QueryType, query: string, params?: any[]): Promise<any | any[] | void>;
+}
+
+export type QueryType = "get" | "all" | "run";

src/databases/Mysql.js

@@ -1,28 +0,0 @@
-var MysqlInterface = require('sync-mysql');
-const logger = require('../utils/logger.js');
-
-class Mysql {
-  constructor(msConfig) {
-    this.connection = new MysqlInterface(msConfig);
-  }
-
-  exec(query) {
-    this.prepare('run', query, []);
-  }
-
-  prepare (type, query, params) {
-    logger.debug("prepare (mysql): type: " + type + ", query: " + query + ", params: " + params);
-    if (type === 'get') {
-      return this.connection.query(query, params)[0];
-    } else if (type === 'run') {
-      this.connection.query(query, params);
-    } else if (type === 'all') {
-      return this.connection.query(query, params);
-    } else {
-      logger.warn('returning undefined...');
-      return undefined;
-    }
-  }
-}
-
-module.exports = Mysql;

src/databases/Mysql.ts

@@ -0,0 +1,35 @@
+import { Logger } from "../utils/logger";
+import { IDatabase, QueryType } from "./IDatabase";
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import MysqlInterface from "sync-mysql";
+
+export class Mysql implements IDatabase {
+    private connection: any;
+
+    constructor(private config: unknown) {
+    }
+
+    // eslint-disable-next-line require-await
+    async init(): Promise<void> {
+        this.connection = new MysqlInterface(this.config);
+    }
+
+    prepare(type: QueryType, query: string, params?: any[]): Promise<any[]> {
+        Logger.debug(`prepare (mysql): type: ${type}, query: ${query}, params: ${params}`);
+        const queryResult = this.connection.query(query, params);
+
+        switch (type) {
+            case "get": {
+                return queryResult[0];
+            }
+            case "all": {
+                return queryResult;
+            }
+            case "run": {
+                break;
+            }
+        }
+    }
+
+}

src/databases/Postgres.ts

@@ -0,0 +1,137 @@
+import { Logger } from "../utils/logger";
+import { IDatabase, QueryType } from "./IDatabase";
+import { Client, Pool, PoolClient, types } from "pg";
+
+import fs from "fs";
+
+// return numeric (pg_type oid=1700) as float
+types.setTypeParser(1700, function(val) {
+    return parseFloat(val);
+});
+
+// return int8 (pg_type oid=20) as int
+types.setTypeParser(20, function(val) {
+    return parseInt(val, 10);
+});
+
+export class Postgres implements IDatabase {
+    private pool: Pool;
+
+    constructor(private config: Record<string, any>) {}
+
+    async init(): Promise<void> {
+        this.pool = new Pool(this.config.postgres);
+
+        if (!this.config.readOnly) {
+            if (this.config.createDbIfNotExists) {
+                await this.createDB();
+            }
+
+            if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) {
+                await this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
+            }
+
+            // Upgrade database if required
+            await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+
+            try {
+                await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
+            } catch (e) {
+                Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
+                Logger.warn(e as string);
+            }
+        }
+    }
+
+    async prepare(type: QueryType, query: string, params?: any[]): Promise<any[]> {
+        // Convert query to use numbered parameters
+        let count = 1;
+        for (let char = 0; char < query.length; char++) {
+            if (query.charAt(char) === "?") {
+                query = `${query.slice(0, char)}$${count}${query.slice(char + 1)}`;
+                count++;
+            }
+        }
+
+        Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
+
+        let client: PoolClient;
+        try {
+            client = await this.pool.connect();
+            const queryResult = await client.query({ text: query, values: params });
+
+            switch (type) {
+                case "get": {
+                    const value = queryResult.rows[0];
+                    Logger.debug(`result (postgres): ${JSON.stringify(value)}`);
+                    return value;
+                }
+                case "all": {
+                    const values = queryResult.rows;
+                    Logger.debug(`result (postgres): ${JSON.stringify(values)}`);
+                    return values;
+                }
+                case "run": {
+                    break;
+                }
+            }
+        } catch (err) {
+            Logger.error(`prepare (postgres): ${err}`);
+        } finally {
+            client?.release();
+        }
+    }
+
+    private async createDB() {
+        const client = new Client({
+            ...this.config.postgres,
+            database: "postgres"
+        });
+
+        await client.connect();
+
+        if ((await client.query(`SELECT * FROM pg_database WHERE datname = '${this.config.postgres.database}'`)).rowCount == 0) {
+            await client.query(`CREATE DATABASE "${this.config.postgres.database}"
+                                WITH 
+                                OWNER = ${this.config.postgres.user}
+                                CONNECTION LIMIT = -1;`
+            );
+        }
+
+        client.end();
+    }
+
+    private async upgradeDB(fileNamePrefix: string, schemaFolder: string) {
+        const versionCodeInfo = await this.pool.query("SELECT value FROM config WHERE key = 'version'");
+        let versionCode = versionCodeInfo.rows[0] ? versionCodeInfo.rows[0].value : 0;
+
+        let path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+        Logger.debug(`db update: trying ${path}`);
+        while (fs.existsSync(path)) {
+            Logger.debug(`db update: updating ${path}`);
+            await this.pool.query(this.processUpgradeQuery(fs.readFileSync(path).toString()));
+
+            versionCode = (await this.pool.query("SELECT value FROM config WHERE key = 'version'"))?.rows[0]?.value;
+            path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+            Logger.debug(`db update: trying ${path}`);
+        }
+        Logger.debug(`db update: no file ${path}`);
+    }
+
+    private async applyIndexes(fileNamePrefix: string, schemaFolder: string) {
+        const path = `${schemaFolder}/_${fileNamePrefix}_indexes.sql`;
+        if (fs.existsSync(path)) {
+            await this.pool.query(fs.readFileSync(path).toString());
+        } else {
+            Logger.debug(`failed to apply indexes to ${fileNamePrefix}`);
+        }
+    }
+
+    private processUpgradeQuery(query: string): string {
+        let result = query;
+        result = result.replace(/sha256\((.*?)\)/gm, "encode(digest($1, 'sha256'), 'hex')");
+        result = result.replace(/integer/gmi, "NUMERIC");
+
+        return result;
+    }
+}

src/databases/Sqlite.js

@@ -1,33 +0,0 @@
-const { db } = require("./databases");
-var config = require('../config.js');
-const logger = require('../utils/logger.js');
-
-class Sqlite {
-  constructor(connection) {
-    this.connection = connection;
-  }
-
-  getConnection() {
-    return this.connection;
-  }
-
-  prepare(type, query, params) {
-    if (type === 'get') {
-      return this.connection.prepare(query).get(...params);
-    } else if (type === 'run') {
-      this.connection.prepare(query).run(...params);
-    } else if (type === 'all') {
-      return this.connection.prepare(query).all(...params); 
-    } else {
-      logger.debug('sqlite query: returning undefined')
-      logger.debug("prepare:        type: " + type + ", query: " + query + ", params: " + params);
-      return undefined;
-    }
-  }
-
-  exec(query) {
-    return this.connection.exec(query);
-  }
-}
-
-module.exports = Sqlite;

src/databases/Sqlite.ts

@@ -0,0 +1,108 @@
+import { IDatabase, QueryType } from "./IDatabase";
+import Sqlite3, { Database, Database as SQLiteDatabase } from "better-sqlite3";
+import fs from "fs";
+import path from "path";
+import { getHash } from "../utils/getHash";
+import { Logger } from "../utils/logger";
+
+export class Sqlite implements IDatabase {
+    private db: SQLiteDatabase;
+
+    constructor(private config: SqliteConfig)
+    {
+    }
+
+    // eslint-disable-next-line require-await
+    async prepare(type: QueryType, query: string, params: any[] = []): Promise<any[]> {
+        // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`);
+        const preparedQuery = this.db.prepare(Sqlite.processQuery(query));
+
+        switch (type) {
+            case "get": {
+                return preparedQuery.get(...params);
+            }
+            case "all": {
+                return preparedQuery.all(...params);
+            }
+            case "run": {
+                preparedQuery.run(...params);
+                break;
+            }
+        }
+    }
+
+    // eslint-disable-next-line require-await
+    async init(): Promise<void> {
+        // Make dirs if required
+        if (!fs.existsSync(path.join(this.config.dbPath, "../"))) {
+            fs.mkdirSync(path.join(this.config.dbPath, "../"));
+        }
+
+        this.db = new Sqlite3(this.config.dbPath, { readonly: this.config.readOnly, fileMustExist: !this.config.createDbIfNotExists });
+
+        if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) {
+            this.db.exec(Sqlite.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
+        }
+
+        if (!this.config.readOnly) {
+            this.db.function("sha256", (str: string) => {
+                return getHash(str, 1);
+            });
+
+            // Upgrade database if required
+            Sqlite.upgradeDB(this.db, this.config.fileNamePrefix, this.config.dbSchemaFolder);
+        }
+
+        this.db.function("regexp", { deterministic: true }, (regex: string, str: string) => {
+            return str.match(regex) ? 1 : 0;
+        });
+
+        // Enable WAL mode checkpoint number
+        if (this.config.enableWalCheckpointNumber) {
+            this.db.exec("PRAGMA journal_mode=WAL;");
+            this.db.exec("PRAGMA wal_autocheckpoint=1;");
+        }
+
+        // Enable Memory-Mapped IO
+        this.db.exec("pragma mmap_size= 500000000;");
+    }
+
+    attachDatabase(database: string, attachAs: string): void {
+        this.db.prepare(`ATTACH ? as ${attachAs}`).run(database);
+    }
+
+    private static processQuery(query: string): string {
+        return query.replace(/ ~\* /g, " REGEXP ");
+    }
+
+    private static upgradeDB(db: Database, fileNamePrefix: string, schemaFolder: string) {
+        const versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
+        let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
+
+        let path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+        Logger.debug(`db update: trying ${path}`);
+        while (fs.existsSync(path)) {
+            Logger.debug(`db update: updating ${path}`);
+            db.exec(this.processUpgradeQuery(fs.readFileSync(path).toString()));
+
+            versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
+            path = `${schemaFolder}/_upgrade_${fileNamePrefix}_${(parseInt(versionCode) + 1)}.sql`;
+            Logger.debug(`db update: trying ${path}`);
+        }
+        Logger.debug(`db update: no file ${path}`);
+    }
+
+    private static processUpgradeQuery(query: string): string {
+        return query.replace(/^.*--!sqlite-ignore/gm, "");
+    }
+}
+
+export interface SqliteConfig {
+    dbPath: string;
+    dbSchemaFileName: string;
+    dbSchemaFolder: string;
+    fileNamePrefix: string;
+    readOnly: boolean;
+    createDbIfNotExists: boolean;
+    enableWalCheckpointNumber: boolean
+}

src/databases/databases.js

@@ -1,81 +0,0 @@
-var config = require('../config.js');
-var Sqlite3 = require('better-sqlite3');
-var fs = require('fs');
-var path = require('path');
-var Sqlite = require('./Sqlite.js')
-var Mysql = require('./Mysql.js');
-const logger = require('../utils/logger.js');
-const getHash = require('../utils/getHash.js');
-
-let options = {
-  readonly: config.readOnly,
-  fileMustExist: !config.createDatabaseIfNotExist
-};
-
-if (config.mysql) {
-  module.exports = {
-    db: new Mysql(config.mysql),
-    privateDB: new Mysql(config.privateMysql)
-  };
-} else {
-  // Make dirs if required
-  if (!fs.existsSync(path.join(config.db, "../"))) {
-    fs.mkdirSync(path.join(config.db, "../"));
-  }
-  if (!fs.existsSync(path.join(config.privateDB, "../"))) {
-    fs.mkdirSync(path.join(config.privateDB, "../"));
-  }
-
-  var db = new Sqlite3(config.db, options);
-  var privateDB = new Sqlite3(config.privateDB, options);
-
-  if (config.createDatabaseIfNotExist && !config.readOnly) {
-    if (fs.existsSync(config.dbSchema)) db.exec(fs.readFileSync(config.dbSchema).toString());
-    if (fs.existsSync(config.privateDBSchema)) privateDB.exec(fs.readFileSync(config.privateDBSchema).toString());
-  }
-
-  if (!config.readOnly) {
-    db.function("sha256", (string) => {
-      return getHash(string, 1);
-    });
-    
-    // Upgrade database if required
-    ugradeDB(db, "sponsorTimes");
-    ugradeDB(privateDB, "private")
-  }
-
-  // Attach private db to main db
-  db.prepare("ATTACH ? as privateDB").run(config.privateDB);
-
-  // Enable WAL mode checkpoint number
-  if (!config.readOnly && config.mode === "production") {
-    db.exec("PRAGMA journal_mode=WAL;");
-    db.exec("PRAGMA wal_autocheckpoint=1;");
-  }
-
-  // Enable Memory-Mapped IO
-  db.exec("pragma mmap_size= 500000000;");
-  privateDB.exec("pragma mmap_size= 500000000;");
-
-  module.exports = {
-    db: new Sqlite(db),
-    privateDB: new Sqlite(privateDB)
-  };
-
-  function ugradeDB(db, prefix) {
-    let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
-    let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;
-
-    let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
-    logger.debug('db update: trying ' + path);
-    while (fs.existsSync(path)) {
-      logger.debug('db update: updating ' + path);
-      db.exec(fs.readFileSync(path).toString());
-
-      versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value;
-      path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql";
-      logger.debug('db update: trying ' + path);
-    }
-    logger.debug('db update: no file ' + path);
-  }
-}

src/databases/databases.ts

@@ -0,0 +1,89 @@
+import { config } from "../config";
+import { Sqlite } from "./Sqlite";
+import { Mysql } from "./Mysql";
+import { Postgres } from "./Postgres";
+import { IDatabase } from "./IDatabase";
+
+let db: IDatabase;
+let privateDB: IDatabase;
+if (config.mysql) {
+    db = new Mysql(config.mysql);
+    privateDB = new Mysql(config.privateMysql);
+} else if (config.postgres?.enabled) {
+    db = new Postgres({
+        dbSchemaFileName: config.dbSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "sponsorTimes",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        postgres: {
+            user: config.postgres?.user,
+            host: config.postgres?.host,
+            database: "sponsorTimes",
+            password: config.postgres?.password,
+            port: config.postgres?.port,
+            max: config.postgres?.max,
+            min: config.postgres?.min,
+        }
+    });
+
+    privateDB = new Postgres({
+        dbSchemaFileName: config.privateDBSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "private",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        postgres: {
+            user: config.postgres?.user,
+            host: config.postgres?.host,
+            database: "privateDB",
+            password: config.postgres?.password,
+            port: config.postgres?.port,
+            max: config.postgres?.max,
+            min: config.postgres?.min,
+        }
+    });
+} else {
+    db = new Sqlite({
+        dbPath: config.db,
+        dbSchemaFileName: config.dbSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "sponsorTimes",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        enableWalCheckpointNumber: !config.readOnly && config.mode === "production"
+    });
+    privateDB = new Sqlite({
+        dbPath: config.privateDB,
+        dbSchemaFileName: config.privateDBSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: "private",
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        enableWalCheckpointNumber: false
+    });
+}
+async function initDb(): Promise<void> {
+    await db.init();
+    await privateDB.init();
+
+    if (db instanceof Sqlite) {
+        // Attach private db to main db
+        (db as Sqlite).attachDatabase(config.privateDB, "privateDB");
+    }
+
+    if (config.mode === "mirror" && db instanceof Postgres) {
+        const tables = config?.dumpDatabase?.tables ?? [];
+        const tableNames = tables.map(table => table.name);
+        for (const table of tableNames) {
+            const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`;
+            await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
+}
+
+export {
+    db,
+    privateDB,
+    initDb,
+};

src/index.ts

@@ -0,0 +1,30 @@
+import { config } from "./config";
+import { initDb } from "./databases/databases";
+import { createServer } from "./app";
+import { Logger } from "./utils/logger";
+import { startAllCrons } from "./cronjob";
+import { getCommit } from "./utils/getCommit";
+
+async function init() {
+    process.on("unhandledRejection", (error: any) => {
+        // eslint-disable-next-line no-console
+        console.dir(error?.stack);
+        process.exit(1);
+    });
+
+    await initDb();
+    // edge case clause for creating compatible .db files, do not enable
+    if (config.mode === "init-db-and-exit") process.exit(0);
+    // do not enable init-db-only mode for usage.
+    (global as any).HEADCOMMIT = config.mode === "development" ? "development"
+        : config.mode === "test" ? "test"
+            : getCommit() as string;
+    createServer(() => {
+        Logger.info(`Server started on port ${config.port}.`);
+
+        // ignite cron job after server created
+        startAllCrons();
+    }).setTimeout(15000);
+}
+
+init();

src/middleware/apiCsp.ts

@@ -0,0 +1,6 @@
+import { NextFunction, Request, Response } from "express";
+
+export function apiCspMiddleware(req: Request, res: Response, next: NextFunction): void {
+    res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
+    next();
+}

src/middleware/cors.js

@@ -1,5 +0,0 @@
-module.exports = function corsMiddleware(req, res, next) {
-  res.header("Access-Control-Allow-Origin", "*");
-  res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-  next();
-}