Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 19:47:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into fullVideoLabels

Browse Source
This commit is contained in:
Michael C
2022-05-09 02:42:18 -04:00
parent 65a661ca4d 60a3c017e5
commit 6a826b63c9
33 changed files with 653 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
.github/workflows/docker-build.yml vendored Normal file
View File

@@ -0,0 +1,46 @@
# Based on https://github.com/ajayyy/sb-mirror/blob/main/.github/workflows/docker-build.yml
name: multi-build-docker
on:
workflow_call:
inputs:
name:
required: true
type: string
username:
required: true
type: string
folder:
required: true
type: string
secrets:
GH_TOKEN:
required: true
jobs:
build_container:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
ghcr.io/${{ inputs.username }}/${{ inputs.name }}
tags: |
type-raw,value=alpine
flavor: |
latest=true
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GH_TOKEN }}
- name: push
uses: docker/build-push-action@v3
with:
context: ${{ inputs.folder }}
push: true
tags: ${{ steps.meta.outputs.tags }}

25
.github/workflows/sb-server.yml vendored Normal file
View File

@@ -0,0 +1,25 @@
name: Docker image builds
on:
push:
branches:
- master
workflow_dispatch:
jobs:
sb-server:
uses: ./.github/workflows/docker-build.yml
with:
name: "sb-server"
username: "ajayyy"
folder: "."
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
rsync-host:
needs: sb-server
uses: ./.github/workflows/docker-build.yml
with:
name: "rsync-host"
username: "ajayyy"
folder: "./rsync"
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

14
Dockerfile
View File

@@ -1,14 +1,16 @@
FROM node:14-alpine as builder FROM node:16-alpine as builder
RUN apk add --no-cache --virtual .build-deps python make g++ RUN apk add --no-cache --virtual .build-deps python3 make g++
COPY package.json package-lock.json tsconfig.json entrypoint.sh ./ COPY package.json package-lock.json tsconfig.json entrypoint.sh ./
COPY src src COPY src src
RUN npm ci && npm run tsc RUN npm ci && npm run tsc
FROM node:14-alpine as app FROM node:16-alpine as app
WORKDIR /usr/src/app WORKDIR /usr/src/app
COPY --from=builder node_modules . RUN apk add git postgresql-client
COPY --from=builder dist ./dist COPY --from=builder ./node_modules ./node_modules
COPY --from=builder ./dist ./dist
COPY ./.git ./.git
COPY entrypoint.sh . COPY entrypoint.sh .
COPY databases/*.sql databases/ COPY databases/*.sql databases/
EXPOSE 8080 EXPOSE 8080
CMD ./entrypoint.sh CMD ./entrypoint.sh

7
ci.json
View File

@@ -17,13 +17,16 @@
"port": 5432 "port": 5432
}, },
"redis": { "redis": {
"host": "localhost", "socket": {
"port": 6379 "host": "localhost",
"port": 6379
}
}, },
"createDatabaseIfNotExist": true, "createDatabaseIfNotExist": true,
"schemaFolder": "./databases", "schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql", "dbSchema": "./databases/_sponsorTimes.db.sql",
"privateDBSchema": "./databases/_private.db.sql", "privateDBSchema": "./databases/_private.db.sql",
"categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
"mode": "test", "mode": "test",
"readOnly": false, "readOnly": false,
"webhooks": [ "webhooks": [

1
docker/docker-compose.yml
View File

@@ -7,7 +7,6 @@ services:
- database.env - database.env
volumes: volumes:
- database-data:/var/lib/postgresql/data - database-data:/var/lib/postgresql/data
- ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
ports: ports:
- 5432:5432 - 5432:5432
restart: always restart: always

24
entrypoint.sh
View File

@@ -2,25 +2,11 @@
set -e set -e
echo 'Entrypoint script' echo 'Entrypoint script'
cd /usr/src/app cd /usr/src/app
cp /etc/sponsorblock/config.json . || cat <<EOF > config.json
# blank config, use defaults
cat <<EOF > config.json
{ {
"port": 8080,
"globalSalt": "[CHANGE THIS]",
"adminUserID": "[CHANGE THIS]",
"youtubeAPIKey": null,
"discordReportChannelWebhookURL": null,
"discordFirstTimeSubmissionsWebhookURL": null,
"discordAutoModWebhookURL": null,
"proxySubmission": null,
"behindProxy": "X-Forwarded-For",
"db": "./databases/sponsorTimes.db",
"privateDB": "./databases/private.db",
"createDatabaseIfNotExist": true,
"schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql",
"privateDBSchema": "./databases/_private.db.sql",
"mode": "development",
"readOnly": false
} }
EOF EOF
node dist/index.js
node dist/src/index.js

12
nginx/nginx.conf
View File

@@ -49,7 +49,7 @@ http {
#server localhost:4446; #server localhost:4446;
#server localhost:4447; #server localhost:4447;
#server localhost:4448; #server localhost:4448;
server 10.0.0.4:4441 max_fails=25 fail_timeout=20s; #server 10.0.0.4:4441 max_fails=25 fail_timeout=20s;
#server 10.0.0.3:4441 max_fails=25 fail_timeout=20s; #server 10.0.0.3:4441 max_fails=25 fail_timeout=20s;
#server 10.0.0.3:4442 max_fails=25 fail_timeout=20s; #server 10.0.0.3:4442 max_fails=25 fail_timeout=20s;
@@ -72,8 +72,17 @@ http {
server 10.0.0.13:4441 max_fails=25 fail_timeout=20s; server 10.0.0.13:4441 max_fails=25 fail_timeout=20s;
server 10.0.0.13:4442 max_fails=25 fail_timeout=20s; server 10.0.0.13:4442 max_fails=25 fail_timeout=20s;
server 10.0.0.14:4441 max_fails=25 fail_timeout=20s;
server 10.0.0.14:4442 max_fails=25 fail_timeout=20s;
server 10.0.0.11:4441 max_fails=25 fail_timeout=20s; server 10.0.0.11:4441 max_fails=25 fail_timeout=20s;
server 10.0.0.11:4442 max_fails=25 fail_timeout=20s; server 10.0.0.11:4442 max_fails=25 fail_timeout=20s;
server 10.0.0.16:4441 max_fails=25 fail_timeout=20s;
server 10.0.0.16:4442 max_fails=25 fail_timeout=20s;
server 10.0.0.17:4441 max_fails=25 fail_timeout=20s;
server 10.0.0.17:4442 max_fails=25 fail_timeout=20s;
#server 134.209.69.251:80 backup; #server 134.209.69.251:80 backup;
@@ -84,6 +93,7 @@ http {
#server localhost:4441; #server localhost:4441;
#server localhost:4442; #server localhost:4442;
server 10.0.0.3:4441 max_fails=25 fail_timeout=15s; server 10.0.0.3:4441 max_fails=25 fail_timeout=15s;
server 10.0.0.4:4441 max_fails=25 fail_timeout=15s;
#server 10.0.0.3:4442; #server 10.0.0.3:4442;
} }
upstream backend_db { upstream backend_db {

286
package-lock.json generated
View File

@@ -15,22 +15,21 @@
"cron": "^1.8.2", "cron": "^1.8.2",
"express": "^4.17.1", "express": "^4.17.1",
"express-promise-router": "^4.1.1", "express-promise-router": "^4.1.1",
"express-rate-limit": "^5.5.1", "express-rate-limit": "^6.3.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"pg": "^8.7.1", "pg": "^8.7.1",
"redis": "^3.1.2", "rate-limit-redis": "^3.0.1",
"redis": "^4.0.6",
"sync-mysql": "^3.0.1" "sync-mysql": "^3.0.1"
}, },
"devDependencies": { "devDependencies": {
"@types/better-sqlite3": "^7.4.1", "@types/better-sqlite3": "^7.4.1",
"@types/cron": "^1.7.3", "@types/cron": "^1.7.3",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/express-rate-limit": "^5.1.3",
"@types/lodash": "^4.14.178", "@types/lodash": "^4.14.178",
"@types/mocha": "^9.0.0", "@types/mocha": "^9.0.0",
"@types/node": "^16.11.11", "@types/node": "^16.11.11",
"@types/pg": "^8.6.1", "@types/pg": "^8.6.1",
"@types/redis": "^2.8.32",
"@typescript-eslint/eslint-plugin": "^5.5.0", "@typescript-eslint/eslint-plugin": "^5.5.0",
"@typescript-eslint/parser": "^5.5.0", "@typescript-eslint/parser": "^5.5.0",
"eslint": "^8.3.0", "eslint": "^8.3.0",
@@ -128,6 +127,65 @@
"integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
"dev": true "dev": true
}, },
"node_modules/@node-redis/bloom": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz",
"integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==",
"peerDependencies": {
"@node-redis/client": "^1.0.0"
}
},
"node_modules/@node-redis/client": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz",
"integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==",
"dependencies": {
"cluster-key-slot": "1.1.0",
"generic-pool": "3.8.2",
"redis-parser": "3.0.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@node-redis/client/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/@node-redis/graph": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz",
"integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==",
"peerDependencies": {
"@node-redis/client": "^1.0.0"
}
},
"node_modules/@node-redis/json": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz",
"integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==",
"peerDependencies": {
"@node-redis/client": "^1.0.0"
}
},
"node_modules/@node-redis/search": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz",
"integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==",
"peerDependencies": {
"@node-redis/client": "^1.0.0"
}
},
"node_modules/@node-redis/time-series": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz",
"integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==",
"peerDependencies": {
"@node-redis/client": "^1.0.0"
}
},
"node_modules/@nodelib/fs.scandir": { "node_modules/@nodelib/fs.scandir": {
"version": "2.1.5", "version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -293,15 +351,6 @@
"@types/serve-static": "*" "@types/serve-static": "*"
} }
}, },
"node_modules/@types/express-rate-limit": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/@types/express-rate-limit/-/express-rate-limit-5.1.3.tgz",
"integrity": "sha512-H+TYy3K53uPU2TqPGFYaiWc2xJV6+bIFkDd/Ma2/h67Pa6ARk9kWE0p/K9OH1Okm0et9Sfm66fmXoAxsH2PHXg==",
"dev": true,
"dependencies": {
"@types/express": "*"
}
},
"node_modules/@types/express-serve-static-core": { "node_modules/@types/express-serve-static-core": {
"version": "4.17.26", "version": "4.17.26",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz",
@@ -366,15 +415,6 @@
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"devOptional": true "devOptional": true
}, },
"node_modules/@types/redis": {
"version": "2.8.32",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
"integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/serve-static": { "node_modules/@types/serve-static": {
"version": "1.13.10", "version": "1.13.10",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz",
@@ -1214,6 +1254,14 @@
"node": ">=4" "node": ">=4"
} }
}, },
"node_modules/cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/code-point-at": { "node_modules/code-point-at": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
@@ -1444,14 +1492,6 @@
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
}, },
"node_modules/denque": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": { "node_modules/depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -1889,9 +1929,15 @@
} }
}, },
"node_modules/express-rate-limit": { "node_modules/express-rate-limit": {
"version": "5.5.1", "version": "6.3.0",
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz", "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.3.0.tgz",
"integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg==" "integrity": "sha512-932Io1VGKjM3ppi7xW9sb1J5nVkEJSUiOtHw2oE+JyHks1e+AXuOBSXbJKM0mcXwEnW1TibJibQ455Ow1YFjfg==",
"engines": {
"node": ">= 12.9.0"
},
"peerDependencies": {
"express": "^4"
}
}, },
"node_modules/express/node_modules/debug": { "node_modules/express/node_modules/debug": {
"version": "2.6.9", "version": "2.6.9",
@@ -2205,6 +2251,14 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/generic-pool": {
"version": "3.8.2",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz",
"integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==",
"engines": {
"node": ">= 4"
}
},
"node_modules/get-caller-file": { "node_modules/get-caller-file": {
"version": "2.0.5", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -3140,9 +3194,9 @@
} }
}, },
"node_modules/moment": { "node_modules/moment": {
"version": "2.29.1", "version": "2.29.2",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz",
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==",
"engines": { "engines": {
"node": "*" "node": "*"
} }
@@ -3876,6 +3930,17 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/rate-limit-redis": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz",
"integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==",
"engines": {
"node": ">= 14.5.0"
},
"peerDependencies": {
"express-rate-limit": "^6"
}
},
"node_modules/raw-body": { "node_modules/raw-body": {
"version": "2.4.0", "version": "2.4.0",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
@@ -3931,28 +3996,18 @@
} }
}, },
"node_modules/redis": { "node_modules/redis": {
"version": "3.1.2", "version": "4.0.6",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", "resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz",
"integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", "integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==",
"dependencies": { "dependencies": {
"denque": "^1.5.0", "@node-redis/bloom": "1.0.1",
"redis-commands": "^1.7.0", "@node-redis/client": "1.0.5",
"redis-errors": "^1.2.0", "@node-redis/graph": "1.0.0",
"redis-parser": "^3.0.0" "@node-redis/json": "1.0.2",
}, "@node-redis/search": "1.0.5",
"engines": { "@node-redis/time-series": "1.0.2"
"node": ">=10"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-redis"
} }
}, },
"node_modules/redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"node_modules/redis-errors": { "node_modules/redis-errors": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
@@ -5169,6 +5224,54 @@
"integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
"dev": true "dev": true
}, },
"@node-redis/bloom": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz",
"integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==",
"requires": {}
},
"@node-redis/client": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz",
"integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==",
"requires": {
"cluster-key-slot": "1.1.0",
"generic-pool": "3.8.2",
"redis-parser": "3.0.0",
"yallist": "4.0.0"
},
"dependencies": {
"yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
}
}
},
"@node-redis/graph": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz",
"integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==",
"requires": {}
},
"@node-redis/json": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz",
"integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==",
"requires": {}
},
"@node-redis/search": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz",
"integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==",
"requires": {}
},
"@node-redis/time-series": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz",
"integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==",
"requires": {}
},
"@nodelib/fs.scandir": { "@nodelib/fs.scandir": {
"version": "2.1.5", "version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -5319,15 +5422,6 @@
"@types/serve-static": "*" "@types/serve-static": "*"
} }
}, },
"@types/express-rate-limit": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/@types/express-rate-limit/-/express-rate-limit-5.1.3.tgz",
"integrity": "sha512-H+TYy3K53uPU2TqPGFYaiWc2xJV6+bIFkDd/Ma2/h67Pa6ARk9kWE0p/K9OH1Okm0et9Sfm66fmXoAxsH2PHXg==",
"dev": true,
"requires": {
"@types/express": "*"
}
},
"@types/express-serve-static-core": { "@types/express-serve-static-core": {
"version": "4.17.26", "version": "4.17.26",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz",
@@ -5392,15 +5486,6 @@
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"devOptional": true "devOptional": true
}, },
"@types/redis": {
"version": "2.8.32",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
"integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/serve-static": { "@types/serve-static": {
"version": "1.13.10", "version": "1.13.10",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz",
@@ -6029,6 +6114,11 @@
} }
} }
}, },
"cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw=="
},
"code-point-at": { "code-point-at": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
@@ -6209,11 +6299,6 @@
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
}, },
"denque": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw=="
},
"depd": { "depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -6557,9 +6642,10 @@
} }
}, },
"express-rate-limit": { "express-rate-limit": {
"version": "5.5.1", "version": "6.3.0",
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz", "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.3.0.tgz",
"integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg==" "integrity": "sha512-932Io1VGKjM3ppi7xW9sb1J5nVkEJSUiOtHw2oE+JyHks1e+AXuOBSXbJKM0mcXwEnW1TibJibQ455Ow1YFjfg==",
"requires": {}
}, },
"fast-deep-equal": { "fast-deep-equal": {
"version": "3.1.3", "version": "3.1.3",
@@ -6802,6 +6888,11 @@
} }
} }
}, },
"generic-pool": {
"version": "3.8.2",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz",
"integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg=="
},
"get-caller-file": { "get-caller-file": {
"version": "2.0.5", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -7502,9 +7593,9 @@
} }
}, },
"moment": { "moment": {
"version": "2.29.1", "version": "2.29.2",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz",
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg=="
}, },
"moment-timezone": { "moment-timezone": {
"version": "0.5.34", "version": "0.5.34",
@@ -8051,6 +8142,12 @@
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
}, },
"rate-limit-redis": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz",
"integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==",
"requires": {}
},
"raw-body": { "raw-body": {
"version": "2.4.0", "version": "2.4.0",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
@@ -8094,21 +8191,18 @@
} }
}, },
"redis": { "redis": {
"version": "3.1.2", "version": "4.0.6",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", "resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz",
"integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", "integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==",
"requires": { "requires": {
"denque": "^1.5.0", "@node-redis/bloom": "1.0.1",
"redis-commands": "^1.7.0", "@node-redis/client": "1.0.5",
"redis-errors": "^1.2.0", "@node-redis/graph": "1.0.0",
"redis-parser": "^3.0.0" "@node-redis/json": "1.0.2",
"@node-redis/search": "1.0.5",
"@node-redis/time-series": "1.0.2"
} }
}, },
"redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"redis-errors": { "redis-errors": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",

7
package.json
View File

@@ -23,22 +23,21 @@
"cron": "^1.8.2", "cron": "^1.8.2",
"express": "^4.17.1", "express": "^4.17.1",
"express-promise-router": "^4.1.1", "express-promise-router": "^4.1.1",
"express-rate-limit": "^5.5.1", "express-rate-limit": "^6.3.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"pg": "^8.7.1", "pg": "^8.7.1",
"redis": "^3.1.2", "rate-limit-redis": "^3.0.1",
"redis": "^4.0.6",
"sync-mysql": "^3.0.1" "sync-mysql": "^3.0.1"
}, },
"devDependencies": { "devDependencies": {
"@types/better-sqlite3": "^7.4.1", "@types/better-sqlite3": "^7.4.1",
"@types/cron": "^1.7.3", "@types/cron": "^1.7.3",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/express-rate-limit": "^5.1.3",
"@types/lodash": "^4.14.178", "@types/lodash": "^4.14.178",
"@types/mocha": "^9.0.0", "@types/mocha": "^9.0.0",
"@types/node": "^16.11.11", "@types/node": "^16.11.11",
"@types/pg": "^8.6.1", "@types/pg": "^8.6.1",
"@types/redis": "^2.8.32",
"@typescript-eslint/eslint-plugin": "^5.5.0", "@typescript-eslint/eslint-plugin": "^5.5.0",
"@typescript-eslint/parser": "^5.5.0", "@typescript-eslint/parser": "^5.5.0",
"eslint": "^8.3.0", "eslint": "^8.3.0",

6
rsync/Dockerfile Normal file
View File

@@ -0,0 +1,6 @@
FROM ghcr.io/ajayyy/sb-server:latest
EXPOSE 873/tcp
RUN apk add rsync>3.2.4-r0
RUN mkdir /usr/src/app/database-export
CMD rsync --no-detach --daemon & ./entrypoint.sh

8
src/app.ts
View File

@@ -27,7 +27,7 @@ import { loggerMiddleware } from "./middleware/logger";
import { corsMiddleware } from "./middleware/cors"; import { corsMiddleware } from "./middleware/cors";
import { apiCspMiddleware } from "./middleware/apiCsp"; import { apiCspMiddleware } from "./middleware/apiCsp";
import { rateLimitMiddleware } from "./middleware/requestRateLimit"; import { rateLimitMiddleware } from "./middleware/requestRateLimit";
import dumpDatabase, { appExportPath, redirectLink } from "./routes/dumpDatabase"; import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo"; import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
import { postClearCache } from "./routes/postClearCache"; import { postClearCache } from "./routes/postClearCache";
import { addUnlistedVideo } from "./routes/addUnlistedVideo"; import { addUnlistedVideo } from "./routes/addUnlistedVideo";
@@ -208,14 +208,14 @@ function setupRoutes(router: Router) {
router.get("/api/videoLabels", getVideoLabels); router.get("/api/videoLabels", getVideoLabels);
router.get("/api/videoLabels/:prefix", getVideoLabelsByHash); router.get("/api/videoLabels/:prefix", getVideoLabelsByHash);
if (config.postgres) { if (config.postgres?.enabled) {
router.get("/database", (req, res) => dumpDatabase(req, res, true)); router.get("/database", (req, res) => dumpDatabase(req, res, true));
router.get("/database.json", (req, res) => dumpDatabase(req, res, false)); router.get("/database.json", (req, res) => dumpDatabase(req, res, false));
router.get("/database/*", redirectLink); router.get("/database/*", downloadFile);
router.use("/download", express.static(appExportPath)); router.use("/download", express.static(appExportPath));
} else { } else {
router.get("/database.db", function (req: Request, res: Response) { router.get("/database.db", function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", { root: "./" }); res.sendFile("./databases/sponsorTimes.db", { root: "./" });
}); });
} }
} }

102
src/config.ts
View File

@@ -1,6 +1,7 @@
import fs from "fs"; import fs from "fs";
import { SBSConfig } from "./types/config.model"; import { SBSConfig } from "./types/config.model";
import packageJson from "../package.json"; import packageJson from "../package.json";
import { isBoolean, isNumber } from "lodash";
const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test; const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test;
const configFile = process.env.TEST_POSTGRES ? "ci.json" const configFile = process.env.TEST_POSTGRES ? "ci.json"
@@ -9,7 +10,7 @@ const configFile = process.env.TEST_POSTGRES ? "ci.json"
export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8")); export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8"));
addDefaults(config, { addDefaults(config, {
port: 80, port: 8080,
behindProxy: "X-Forwarded-For", behindProxy: "X-Forwarded-For",
db: "./databases/sponsorTimes.db", db: "./databases/sponsorTimes.db",
privateDB: "./databases/private.db", privateDB: "./databases/private.db",
@@ -19,7 +20,7 @@ addDefaults(config, {
privateDBSchema: "./databases/_private.db.sql", privateDBSchema: "./databases/_private.db.sql",
readOnly: false, readOnly: false,
webhooks: [], webhooks: [],
categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"],
categorySupport: { categorySupport: {
sponsor: ["skip", "mute", "full"], sponsor: ["skip", "mute", "full"],
selfpromo: ["skip", "mute", "full"], selfpromo: ["skip", "mute", "full"],
@@ -34,14 +35,14 @@ addDefaults(config, {
chapter: ["chapter"] chapter: ["chapter"]
}, },
maxNumberOfActiveWarnings: 1, maxNumberOfActiveWarnings: 1,
hoursAfterWarningExpires: 24, hoursAfterWarningExpires: 16300000,
adminUserID: "", adminUserID: "",
discordCompletelyIncorrectReportWebhookURL: null, discordCompletelyIncorrectReportWebhookURL: null,
discordFirstTimeSubmissionsWebhookURL: null, discordFirstTimeSubmissionsWebhookURL: null,
discordNeuralBlockRejectWebhookURL: null, discordNeuralBlockRejectWebhookURL: null,
discordFailedReportChannelWebhookURL: null, discordFailedReportChannelWebhookURL: null,
discordReportChannelWebhookURL: null, discordReportChannelWebhookURL: null,
getTopUsersCacheTimeMinutes: 0, getTopUsersCacheTimeMinutes: 240,
globalSalt: null, globalSalt: null,
mode: "", mode: "",
neuralBlockURL: null, neuralBlockURL: null,
@@ -49,15 +50,15 @@ addDefaults(config, {
rateLimit: { rateLimit: {
vote: { vote: {
windowMs: 900000, windowMs: 900000,
max: 20, max: 15,
message: "Too many votes, please try again later", message: "OK",
statusCode: 429, statusCode: 200,
}, },
view: { view: {
windowMs: 900000, windowMs: 900000,
max: 20, max: 10,
statusCode: 200, statusCode: 200,
message: "Too many views, please try again later", message: "OK",
}, },
rate: { rate: {
windowMs: 900000, windowMs: 900000,
@@ -70,12 +71,17 @@ addDefaults(config, {
newLeafURLs: null, newLeafURLs: null,
maxRewardTimePerSegmentInSeconds: 600, maxRewardTimePerSegmentInSeconds: 600,
poiMinimumStartTime: 2, poiMinimumStartTime: 2,
postgres: null, postgres: {
enabled: false,
user: "",
host: "",
password: "",
port: 5432
},
dumpDatabase: { dumpDatabase: {
enabled: false, enabled: false,
minTimeBetweenMs: 60000, minTimeBetweenMs: 180000,
appExportPath: "./docker/database-export", appExportPath: "./docker/database-export",
postgresExportPath: "/opt/exports",
tables: [{ tables: [{
name: "sponsorTimes", name: "sponsorTimes",
order: "timeSubmitted" order: "timeSubmitted"
@@ -95,11 +101,32 @@ addDefaults(config, {
}, },
{ {
name: "vipUsers" name: "vipUsers"
},
{
name: "unlistedVideos"
},
{
name: "videoInfo"
},
{
name: "ratings"
}] }]
}, },
diskCache: null, diskCache: {
crons: null max: 10737418240
},
crons: null,
redis: {
enabled: false,
socket: {
host: "",
port: 0
},
disableOfflineQueue: true
}
}); });
loadFromEnv(config);
migrate(config);
// Add defaults // Add defaults
function addDefaults(config: SBSConfig, defaults: SBSConfig) { function addDefaults(config: SBSConfig, defaults: SBSConfig) {
@@ -109,3 +136,50 @@ function addDefaults(config: SBSConfig, defaults: SBSConfig) {
} }
} }
} }
function migrate(config: SBSConfig) {
// Redis change
if (config.redis) {
const redisConfig = config.redis as any;
if (redisConfig.host || redisConfig.port) {
config.redis.socket = {
host: redisConfig.host,
port: redisConfig.port
};
}
if (redisConfig.enable_offline_queue !== undefined) {
config.disableOfflineQueue = !redisConfig.enable_offline_queue;
}
if (redisConfig.socket?.host && redisConfig.enabled === undefined) {
redisConfig.enabled = true;
}
}
if (config.postgres && config.postgres.user && config.postgres.enabled === undefined) {
config.postgres.enabled = true;
}
}
function loadFromEnv(config: SBSConfig, prefix = "") {
for (const key in config) {
const fullKey = (prefix ? `${prefix}_` : "") + key;
const data = config[key];
if (typeof data === "object" && !Array.isArray(data)) {
loadFromEnv(data, fullKey);
} else if (process.env[fullKey]) {
const value = process.env[fullKey];
if (isNumber(value)) {
config[key] = parseInt(value, 10);
} else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") {
config[key] = value === "true";
} else if (key === "newLeafURLs") {
config[key] = [value];
} else {
config[key] = value;
}
}
}
}

4
src/databases/Postgres.ts
View File

@@ -94,10 +94,6 @@ export class Postgres implements IDatabase {
await client.query(`CREATE DATABASE "${this.config.postgres.database}" await client.query(`CREATE DATABASE "${this.config.postgres.database}"
WITH WITH
OWNER = ${this.config.postgres.user} OWNER = ${this.config.postgres.user}
ENCODING = 'UTF8'
LC_COLLATE = 'en_US.utf8'
LC_CTYPE = 'en_US.utf8'
TABLESPACE = pg_default
CONNECTION LIMIT = -1;` CONNECTION LIMIT = -1;`
); );
} }

4
src/databases/databases.ts
View File

@@ -9,7 +9,7 @@ let privateDB: IDatabase;
if (config.mysql) { if (config.mysql) {
db = new Mysql(config.mysql); db = new Mysql(config.mysql);
privateDB = new Mysql(config.privateMysql); privateDB = new Mysql(config.privateMysql);
} else if (config.postgres) { } else if (config.postgres?.enabled) {
db = new Postgres({ db = new Postgres({
dbSchemaFileName: config.dbSchema, dbSchemaFileName: config.dbSchema,
dbSchemaFolder: config.schemaFolder, dbSchemaFolder: config.schemaFolder,
@@ -72,7 +72,7 @@ async function initDb(): Promise<void> {
const tables = config?.dumpDatabase?.tables ?? []; const tables = config?.dumpDatabase?.tables ?? [];
const tableNames = tables.map(table => table.name); const tableNames = tables.map(table => table.name);
for (const table of tableNames) { for (const table of tableNames) {
const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`; const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`;
await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
} }
} }

15
src/middleware/requestRateLimit.ts
View File

@@ -1,19 +1,23 @@
import { getIP } from "../utils/getIP"; import { getIP } from "../utils/getIP";
import { getHash } from "../utils/getHash"; import { getHash } from "../utils/getHash";
import { getHashCache } from "../utils/getHashCache"; import { getHashCache } from "../utils/getHashCache";
import rateLimit from "express-rate-limit"; import rateLimit, { RateLimitRequestHandler } from "express-rate-limit";
import { RateLimitConfig } from "../types/config.model"; import { RateLimitConfig } from "../types/config.model";
import { Request } from "express"; import { Request } from "express";
import { isUserVIP } from "../utils/isUserVIP"; import { isUserVIP } from "../utils/isUserVIP";
import { UserID } from "../types/user.model"; import { UserID } from "../types/user.model";
import RedisStore from "rate-limit-redis";
import redis from "../utils/redis";
import { config } from "../config";
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): rateLimit.RateLimit { export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): RateLimitRequestHandler {
return rateLimit({ return rateLimit({
windowMs: limitConfig.windowMs, windowMs: limitConfig.windowMs,
max: limitConfig.max, max: limitConfig.max,
message: limitConfig.message, message: limitConfig.message,
statusCode: limitConfig.statusCode, statusCode: limitConfig.statusCode,
headers: false, legacyHeaders: false,
standardHeaders: false,
keyGenerator: (req) => { keyGenerator: (req) => {
return getHash(getIP(req), 1); return getHash(getIP(req), 1);
}, },
@@ -23,6 +27,9 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r
} else { } else {
return next(); return next();
} }
} },
store: config.redis?.enabled ? new RedisStore({
sendCommand: (...args: string[]) => redis.sendCommand(args),
}) : null,
}); });
} }

25
src/routes/addUserAsTempVIP.ts
View File

@@ -9,6 +9,7 @@ import { isUserVIP } from "../utils/isUserVIP";
import { HashedUserID } from "../types/user.model"; import { HashedUserID } from "../types/user.model";
import redis from "../utils/redis"; import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys"; import { tempVIPKey } from "../utils/redisKeys";
import { Logger } from "../utils/logger";
interface AddUserAsTempVIPRequest extends Request { interface AddUserAsTempVIPRequest extends Request {
query: { query: {
@@ -65,12 +66,22 @@ export async function addUserAsTempVIP(req: AddUserAsTempVIPRequest, res: Respon
if (!channelInfo?.id) { if (!channelInfo?.id) {
return res.status(404).send(`No channel found for videoID ${channelVideoID}`); return res.status(404).send(`No channel found for videoID ${channelVideoID}`);
} }
await redis.setAsyncEx(tempVIPKey(userID), channelInfo?.id, dayInSeconds);
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
}
await redis.delAsync(tempVIPKey(userID)); try {
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]); await redis.setEx(tempVIPKey(userID), dayInSeconds, channelInfo?.id);
return res.status(200).send(`Temp VIP removed`); await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
} catch (e) {
Logger.error(e as string);
return res.status(500).send();
}
}
try {
await redis.del(tempVIPKey(userID));
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP removed`);
} catch (e) {
Logger.error(e as string);
return res.status(500).send();
}
} }

37
src/routes/dumpDatabase.ts
View File

@@ -5,6 +5,7 @@ import { config } from "../config";
import util from "util"; import util from "util";
import fs from "fs"; import fs from "fs";
import path from "path"; import path from "path";
import { ChildProcess, exec, ExecOptions, spawn } from "child_process";
const unlink = util.promisify(fs.unlink); const unlink = util.promisify(fs.unlink);
const ONE_MINUTE = 1000 * 60; const ONE_MINUTE = 1000 * 60;
@@ -32,9 +33,19 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
const tables = config?.dumpDatabase?.tables ?? []; const tables = config?.dumpDatabase?.tables ?? [];
const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE; const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
export const appExportPath = config?.dumpDatabase?.appExportPath ?? "./docker/database-export"; export const appExportPath = config?.dumpDatabase?.appExportPath ?? "./docker/database-export";
const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? "/opt/exports";
const tableNames = tables.map(table => table.name); const tableNames = tables.map(table => table.name);
const credentials: ExecOptions = {
env: {
...process.env,
PGHOST: config.postgres.host,
PGPORT: String(config.postgres.port),
PGUSER: config.postgres.user,
PGPASSWORD: String(config.postgres.password),
PGDATABASE: "sponsorTimes",
}
}
interface TableDumpList { interface TableDumpList {
fileName: string; fileName: string;
tableName: string; tableName: string;
@@ -100,7 +111,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
res.status(404).send("Database dump is disabled"); res.status(404).send("Database dump is disabled");
return; return;
} }
if (!config.postgres) { if (!config.postgres?.enabled) {
res.status(404).send("Not supported on this instance"); res.status(404).send("Not supported on this instance");
return; return;
} }
@@ -170,12 +181,12 @@ async function getDbVersion(): Promise<number> {
return row.value; return row.value;
} }
export async function redirectLink(req: Request, res: Response): Promise<void> { export async function downloadFile(req: Request, res: Response): Promise<void> {
if (!config?.dumpDatabase?.enabled) { if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled"); res.status(404).send("Database dump is disabled");
return; return;
} }
if (!config.postgres) { if (!config.postgres?.enabled) {
res.status(404).send("Not supported on this instance"); res.status(404).send("Not supported on this instance");
return; return;
} }
@@ -183,7 +194,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path); const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
if (file) { if (file) {
res.redirect(`/download/${file.fileName}`); res.sendFile(file.fileName, { root: appExportPath });
} else { } else {
res.sendStatus(404); res.sendStatus(404);
} }
@@ -210,9 +221,19 @@ async function queueDump(): Promise<void> {
for (const table of tables) { for (const table of tables) {
const fileName = `${table.name}_${startTime}.csv`; const fileName = `${table.name}_${startTime}.csv`;
const file = `${postgresExportPath}/${fileName}`; const file = `${appExportPath}/${fileName}`;
await db.prepare("run", `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '${file}' WITH (FORMAT CSV, HEADER true);`); await new Promise<string>((resolve) => {
exec(`psql -c "\\copy (SELECT * FROM \\"${table.name}\\"${table.order ? ` ORDER BY \\"${table.order}\\"` : ``})`
+ ` TO '${file}' WITH (FORMAT CSV, HEADER true);"`, credentials, (error, stdout, stderr) => {
if (error) {
Logger.error(`[dumpDatabase] Failed to dump ${table.name} to ${file} due to ${stderr}`);
}
resolve(error ? stderr : stdout);
});
})
dumpFiles.push({ dumpFiles.push({
fileName, fileName,
tableName: table.name, tableName: table.name,

3
src/routes/getSkipSegments.ts
View File

@@ -223,7 +223,8 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//The 3 makes -2 the minimum votes before being ignored completely //The 3 makes -2 the minimum votes before being ignored completely
//this can be changed if this system increases in popularity. //this can be changed if this system increases in popularity.
const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost); const repFactor = choice.votes > 0 ? Math.max(1, choice.reputation + 1) : 1;
const weight = Math.exp(choice.votes * repFactor + 3 + boost);
totalWeight += Math.max(weight, 0); totalWeight += Math.max(weight, 0);
return { ...choice, weight }; return { ...choice, weight };

8
src/routes/getStatus.ts
View File

@@ -10,8 +10,12 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
value = Array.isArray(value) ? value[0] : value; value = Array.isArray(value) ? value[0] : value;
try { try {
const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value; const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value;
const numberRequests = await redis.increment("statusRequest"); let statusRequests: unknown = 0;
const statusRequests = numberRequests?.replies?.[0]; try {
const numberRequests = await redis.increment("statusRequest");
statusRequests = numberRequests?.[0];
} catch (error) { } // eslint-disable-line no-empty
const statusValues: Record<string, any> = { const statusValues: Record<string, any> = {
uptime: process.uptime(), uptime: process.uptime(),
commit: (global as any).HEADCOMMIT || "unknown", commit: (global as any).HEADCOMMIT || "unknown",

13
src/routes/getTopCategoryUsers.ts
View File

@@ -28,13 +28,16 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]); ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
for (const row of rows) { if (rows) {
userNames.push(row.userName); for (const row of rows) {
viewCounts.push(row.viewCount); userNames.push(row.userName);
totalSubmissions.push(row.totalSubmissions); viewCounts.push(row.viewCount);
minutesSaved.push(row.minutesSaved); totalSubmissions.push(row.totalSubmissions);
minutesSaved.push(row.minutesSaved);
}
} }
return { return {
userNames, userNames,
viewCounts, viewCounts,

238
src/routes/postSkipSegments.ts
View File

@@ -16,6 +16,7 @@ import { getReputation } from "../utils/reputation";
import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model"; import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
import { HashedUserID, UserID } from "../types/user.model"; import { HashedUserID, UserID } from "../types/user.model";
import { isUserVIP } from "../utils/isUserVIP"; import { isUserVIP } from "../utils/isUserVIP";
import { isUserTempVIP } from "../utils/isUserTempVIP";
import { parseUserAgent } from "../utils/userAgent"; import { parseUserAgent } from "../utils/userAgent";
import { getService } from "../utils/getService"; import { getService } from "../utils/getService";
import axios from "axios"; import axios from "axios";
@@ -81,19 +82,19 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
axios.post(config.discordFirstTimeSubmissionsWebhookURL, { axios.post(config.discordFirstTimeSubmissionsWebhookURL, {
"embeds": [{ embeds: [{
"title": data?.title, title: data?.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`, url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
"description": `Submission ID: ${UUID}\ description: `Submission ID: ${UUID}\
\n\nTimestamp: \ \n\nTimestamp: \
${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n\nCategory: ${segmentInfo.category}`, \n\nCategory: ${segmentInfo.category}`,
"color": 10813440, color: 10813440,
"author": { author: {
"name": userID, name: userID,
}, },
"thumbnail": { thumbnail: {
"url": getMaxResThumbnail(data) || "", url: getMaxResThumbnail(data) || "",
}, },
}], }],
}) })
@@ -112,55 +113,6 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
} }
} }
async function sendWebhooksNB(userID: string, videoID: string, UUID: string, startTime: number, endTime: number, category: string, probability: number, ytData: any) {
const submissionInfoRow = await db.prepare("get", `SELECT
(select count(1) from "sponsorTimes" where "userID" = ?) count,
(select count(1) from "sponsorTimes" where "userID" = ? and "votes" <= -2) disregarded,
coalesce((select "userName" FROM "userNames" WHERE "userID" = ?), ?) "userName"`,
[userID, userID, userID, userID]);
let submittedBy: string;
// If a userName was created then show both
if (submissionInfoRow.userName !== userID) {
submittedBy = `${submissionInfoRow.userName}\n${userID}`;
} else {
submittedBy = userID;
}
// Send discord message
if (config.discordNeuralBlockRejectWebhookURL === null) return;
axios.post(config.discordNeuralBlockRejectWebhookURL, {
"embeds": [{
"title": ytData.items[0].snippet.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`,
"description": `**Submission ID:** ${UUID}\
\n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n**Predicted Probability:** ${probability}\
\n**Category:** ${category}\
\n**Submitted by:** ${submittedBy}\
\n**Total User Submissions:** ${submissionInfoRow.count}\
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`,
"color": 10813440,
"thumbnail": {
"url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
},
}]
})
.then(res => {
if (res.status >= 400) {
Logger.error("Error sending NeuralBlock Discord hook");
Logger.error(JSON.stringify(res));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send NeuralBlock Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
// callback: function(reject: "String containing reason the submission was rejected") // callback: function(reject: "String containing reason the submission was rejected")
// returns: string when an error, false otherwise // returns: string when an error, false otherwise
@@ -168,98 +120,46 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// false for a pass - it was confusing and lead to this bug - any use of this function in // false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem. // the future could have the same problem.
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service }) { submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
if (apiVideoInfo) {
const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => {
if (!apiVideoInfo) return undefined;
const { err, data } = apiVideoInfo; const { err, data } = apiVideoInfo;
if (err) return false; // return undefined if API error
if (err) return undefined;
return data?.lengthSeconds;
};
// get duration from API
const apiDuration = apiVideoDuration(apiVideoInfo);
// if API fail or returns 0, get duration from client
const duration = apiDuration || submission.videoDuration;
// return false on undefined or 0
if (!duration) return false;
const duration = apiVideoInfo?.data?.lengthSeconds; const segments = submission.segments;
const segments = submission.segments; // map all times to float array
let nbString = ""; const allSegmentTimes = segments.map(segment => [parseFloat(segment.segment[0]), parseFloat(segment.segment[1])]);
for (let i = 0; i < segments.length; i++) {
if (duration == 0) {
// Allow submission if the duration is 0 (bug in youtube api)
return false;
} else {
if (segments[i].category === "sponsor") {
//Prepare timestamps to send to NB all at once
nbString = `${nbString}${segments[i].segment[0]},${segments[i].segment[1]};`;
}
}
}
// Get all submissions for this user // add previous submissions by this user
const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]); const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? AND "videoID" = ? AND "votes" > -1 AND "hidden" = 0`, [submission.userID, submission.videoID]);
const allSegmentTimes = [];
if (allSubmittedByUser !== undefined) {
//add segments the user has previously submitted
for (const segmentInfo of allSubmittedByUser) {
allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
}
}
//add segments they are trying to add in this submission if (allSubmittedByUser) {
for (let i = 0; i < segments.length; i++) { //add segments the user has previously submitted
const startTime = parseFloat(segments[i].segment[0]); const allSubmittedTimes = allSubmittedByUser.map((segment: { startTime: string, endTime: string }) => [parseFloat(segment.startTime), parseFloat(segment.endTime)]);
const endTime = parseFloat(segments[i].segment[1]); allSegmentTimes.push(...allSubmittedTimes);
allSegmentTimes.push([startTime, endTime]);
}
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
return a[0] - b[0] || a[1] - b[1];
}));
const videoDuration = data?.lengthSeconds;
if (videoDuration != 0) {
let allSegmentDuration = 0;
//sum all segment times together
allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
if (allSegmentDuration > (videoDuration / 100) * 80) {
// Reject submission if all segments combine are over 80% of the video
return "Total length of your submitted segments are over 80% of the video.";
}
}
// Check NeuralBlock
const neuralBlockURL = config.neuralBlockURL;
if (!neuralBlockURL) return false;
const response = await axios.get(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID}
&segments=${nbString.substring(0, nbString.length - 1)}`, { validateStatus: () => true });
if (response.status !== 200) return false;
const nbPredictions = response.data;
let nbDecision = false;
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
for (let i = 0; i < segments.length; i++) {
if (segments[i].category === "sponsor") {
if (nbPredictions.probabilities[predictionIdx] < 0.70) {
nbDecision = true; // At least one bad entry
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime, submission.service);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
}
predictionIdx++;
}
}
if (nbDecision) {
return "Rejected based on NeuralBlock predictions.";
} else {
return false;
}
} else {
Logger.debug("Skipped YouTube API");
// Can't moderate the submission without calling the youtube API
// so allow by default.
return false;
} }
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort((a, b) => a[0] - b[0] || a[1] - b[1]));
//sum all segment times together
const allSegmentDuration = allSegmentsSorted.reduce((acc, curr) => acc + (curr[1] - curr[0]), 0);
if (allSegmentDuration > (duration / 100) * 80) {
// Reject submission if all segments combine are over 80% of the video
return "Total length of your submitted segments are over 80% of the video.";
}
return false;
} }
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> { function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
@@ -310,7 +210,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
invalidFields.push("userID"); invalidFields.push("userID");
if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`); if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`);
} }
if (!Array.isArray(segments) || segments.length < 1) { if (!Array.isArray(segments) || segments.length == 0) {
invalidFields.push("segments"); invalidFields.push("segments");
} }
// validate start and end times (no : marks) // validate start and end times (no : marks)
@@ -323,7 +223,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
} }
if (typeof segmentPair.description !== "string" if (typeof segmentPair.description !== "string"
|| (segmentPair.description.length > 60 && segmentPair.actionType === ActionType.Chapter) || (segmentPair.actionType === ActionType.Chapter && segmentPair.description.length > 60 )
|| (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) { || (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) {
invalidFields.push("segment description"); invalidFields.push("segment description");
} }
@@ -402,7 +302,7 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
} }
if (!isVIP && segments[i].category === "sponsor" if (!isVIP && segments[i].category === "sponsor"
&& segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) { && segments[i].actionType !== ActionType.Full && (endTime - startTime) < 1) {
// Too short // Too short
return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 }; return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 };
} }
@@ -425,32 +325,19 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
return CHECK_PASS; return CHECK_PASS;
} }
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise<CheckResult & { decreaseVotes: number; } > { async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise<CheckResult> {
// Auto moderator check // Auto moderator check
if (!isVIP && service == Service.YouTube) { if (service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service });//startTime, endTime, category: segments[i].category}); const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration });
if (autoModerateResult) {
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected.
// However, this behavior is consistent with other automod functions
// already in place.
//decreaseVotes = -2; //Disable for now
} else if (autoModerateResult) {
//Normal automod behavior
return { return {
pass: false, pass: false,
errorCode: 403, errorCode: 403,
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`, errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
decreaseVotes
}; };
} }
} }
return CHECK_PASS;
return {
...CHECK_PASS,
decreaseVotes
};
} }
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) { async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
@@ -601,11 +488,12 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
const userWarningCheckResult = await checkUserActiveWarning(userID); const userWarningCheckResult = await checkUserActiveWarning(userID);
if (!userWarningCheckResult.pass) { if (!userWarningCheckResult.pass) {
Logger.warn(`Caught a submission for for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`); Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);
return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage); return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage);
} }
const isVIP = await isUserVIP(userID); const isVIP = await isUserVIP(userID);
const isTempVIP = await isUserTempVIP(userID, videoID);
const rawIP = getIP(req); const rawIP = getIP(req);
const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam); const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam);
@@ -618,13 +506,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage); return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage);
} }
let decreaseVotes = 0; if (!isVIP && !isTempVIP) {
// Auto check by NB const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam);
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, isVIP, service, apiVideoInfo, decreaseVotes); if (!autoModerateCheckResult.pass) {
if (!autoModerateCheckResult.pass) { return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage);
return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage); }
} else {
decreaseVotes = autoModerateCheckResult.decreaseVotes;
} }
// Will be filled when submitting // Will be filled when submitting
@@ -645,7 +531,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//check to see if this user is shadowbanned //check to see if this user is shadowbanned
const shadowBanRow = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID]); const shadowBanRow = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID]);
const startingVotes = 0 + decreaseVotes; const startingVotes = 0;
const reputation = await getReputation(userID); const reputation = await getReputation(userID);
for (const segmentInfo of segments) { for (const segmentInfo of segments) {

26
src/routes/voteOnSponsorTime.ts
View File

@@ -1,6 +1,7 @@
import { Request, Response } from "express"; import { Request, Response } from "express";
import { Logger } from "../utils/logger"; import { Logger } from "../utils/logger";
import { isUserVIP } from "../utils/isUserVIP"; import { isUserVIP } from "../utils/isUserVIP";
import { isUserTempVIP } from "../utils/isUserTempVIP";
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi"; import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model"; import { APIVideoInfo } from "../types/youtubeApi.model";
import { db, privateDB } from "../databases/databases"; import { db, privateDB } from "../databases/databases";
@@ -9,12 +10,10 @@ import { getFormattedTime } from "../utils/getFormattedTime";
import { getIP } from "../utils/getIP"; import { getIP } from "../utils/getIP";
import { getHashCache } from "../utils/getHashCache"; import { getHashCache } from "../utils/getHashCache";
import { config } from "../config"; import { config } from "../config";
import { HashedUserID, UserID } from "../types/user.model"; import { UserID } from "../types/user.model";
import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType } from "../types/segments.model"; import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType } from "../types/segments.model";
import { QueryCacher } from "../utils/queryCacher"; import { QueryCacher } from "../utils/queryCacher";
import axios from "axios"; import axios from "axios";
import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys";
const voteTypes = { const voteTypes = {
normal: 0, normal: 0,
@@ -44,6 +43,7 @@ interface VoteData {
row: { row: {
votes: number; votes: number;
views: number; views: number;
locked: boolean;
}; };
category: string; category: string;
incrementAmount: number; incrementAmount: number;
@@ -55,14 +55,6 @@ function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<API
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null; return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
} }
const isUserTempVIP = async (nonAnonUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
const channelID = apiVideoInfo?.data?.authorId;
const { err, reply } = await redis.getAsync(tempVIPKey(nonAnonUserID));
return err || !reply ? false : (reply == channelID);
};
const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2); const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2);
async function updateSegmentVideoDuration(UUID: SegmentUUID) { async function updateSegmentVideoDuration(UUID: SegmentUUID) {
@@ -178,7 +170,7 @@ async function sendWebhooks(voteData: VoteData) {
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}s#requiredSegment=${voteData.UUID}`, "url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}s#requiredSegment=${voteData.UUID}`,
"description": `**${voteData.row.votes} Votes Prior | \ "description": `**${voteData.row.votes} Votes Prior | \
${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \ ${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \
Views**\n\n**Submission ID:** ${voteData.UUID}\ Views**\n\n**Locked**: ${voteData.row.locked}\n\n**Submission ID:** ${voteData.UUID}\
\n**Category:** ${submissionInfoRow.category}\ \n**Category:** ${submissionInfoRow.category}\
\n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\ \n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\
\n\n**Total User Submissions:** ${submissionInfoRow.count}\ \n\n**Total User Submissions:** ${submissionInfoRow.count}\
@@ -189,7 +181,7 @@ async function sendWebhooks(voteData: VoteData) {
"author": { "author": {
"name": voteData.finalResponse?.webhookMessage ?? "name": voteData.finalResponse?.webhookMessage ??
voteData.finalResponse?.finalMessage ?? voteData.finalResponse?.finalMessage ??
getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission), `${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`,
}, },
"thumbnail": { "thumbnail": {
"url": getMaxResThumbnail(data) || "", "url": getMaxResThumbnail(data) || "",
@@ -476,11 +468,11 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
// Only change the database if they have made a submission before and haven't voted recently // Only change the database if they have made a submission before and haven't voted recently
const userAbleToVote = (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0) const userAbleToVote = (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0)
&& !finalResponse.blockVote
&& finalResponse.finalStatus === 200
&& (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined) && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined);
&& !finalResponse.blockVote
&& finalResponse.finalStatus === 200;
const ableToVote = isVIP || isTempVIP || userAbleToVote; const ableToVote = isVIP || isTempVIP || userAbleToVote;
@@ -534,4 +526,4 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
Logger.error(err as string); Logger.error(err as string);
return { status: 500, message: finalResponse.finalMessage ?? undefined, json: { error: "Internal error creating segment vote" } }; return { status: 500, message: finalResponse.finalMessage ?? undefined, json: { error: "Internal error creating segment vote" } };
} }
} }

13
src/types/config.model.ts
View File

@@ -2,6 +2,14 @@ import { PoolConfig } from "pg";
import * as redis from "redis"; import * as redis from "redis";
import { CacheOptions } from "@ajayyy/lru-diskcache"; import { CacheOptions } from "@ajayyy/lru-diskcache";
interface RedisConfig extends redis.RedisClientOptions {
enabled: boolean;
}
interface CustomPostgresConfig extends PoolConfig {
enabled: boolean;
}
export interface SBSConfig { export interface SBSConfig {
[index: string]: any [index: string]: any
port: number; port: number;
@@ -41,9 +49,9 @@ export interface SBSConfig {
privateMysql?: any; privateMysql?: any;
minimumPrefix?: string; minimumPrefix?: string;
maximumPrefix?: string; maximumPrefix?: string;
redis?: redis.ClientOpts; redis?: RedisConfig;
maxRewardTimePerSegmentInSeconds?: number; maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig; postgres?: CustomPostgresConfig;
dumpDatabase?: DumpDatabase; dumpDatabase?: DumpDatabase;
diskCache: CacheOptions; diskCache: CacheOptions;
crons: CronJobOptions; crons: CronJobOptions;
@@ -76,7 +84,6 @@ export interface DumpDatabase {
enabled: boolean; enabled: boolean;
minTimeBetweenMs: number; minTimeBetweenMs: number;
appExportPath: string; appExportPath: string;
postgresExportPath: string;
tables: DumpDatabaseTable[]; tables: DumpDatabaseTable[];
} }

17
src/utils/getHashCache.ts
View File

@@ -18,18 +18,19 @@ export async function getHashCache<T extends string>(value: T, times = defaulted
async function getFromRedis<T extends string>(key: HashedValue): Promise<T & HashedValue> { async function getFromRedis<T extends string>(key: HashedValue): Promise<T & HashedValue> {
const redisKey = shaHashKey(key); const redisKey = shaHashKey(key);
const { err, reply } = await redis.getAsync(redisKey);
if (!err && reply) { try {
try { const reply = await redis.get(redisKey);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`); Logger.debug(`Got data from redis: ${reply}`);
return reply as T & HashedValue; return reply as T & HashedValue;
} catch (e) {
// If all else, continue on hashing
} }
} } catch (e) {} // eslint-disable-line no-empty
const data = getHash(key, cachedHashTimes);
// Otherwise, calculate it
const data = getHash(key, cachedHashTimes);
redis.set(key, data);
redis.setAsync(key, data);
return data as T & HashedValue; return data as T & HashedValue;
} }

24
src/utils/isUserTempVIP.ts Normal file
View File

@@ -0,0 +1,24 @@
import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys";
import { HashedUserID } from "../types/user.model";
import { YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model";
import { VideoID } from "../types/segments.model";
import { config } from "../config";
import { Logger } from "./logger";
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
}
export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
const channelID = apiVideoInfo?.data?.authorId;
try {
const reply = await redis.get(tempVIPKey(hashedUserID));
return reply && reply == channelID;
} catch (e) {
Logger.error(e as string);
return false;
}
};

36
src/utils/queryCacher.ts
View File

@@ -5,21 +5,18 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { UserID } from "../types/user.model"; import { UserID } from "../types/user.model";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> { async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const { err, reply } = await redis.getAsync(key); try {
const reply = await redis.get(key);
if (!err && reply) { if (reply) {
try {
Logger.debug(`Got data from redis: ${reply}`); Logger.debug(`Got data from redis: ${reply}`);
return JSON.parse(reply); return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
} }
} } catch (e) { } //eslint-disable-line no-empty
const data = await fetchFromDB(); const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data)); redis.set(key, JSON.stringify(data));
return data; return data;
} }
@@ -30,18 +27,17 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Promise<Array<T>>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise<Array<T>> { async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Promise<Array<T>>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise<Array<T>> {
const cachedValues = await Promise.all(values.map(async (value) => { const cachedValues = await Promise.all(values.map(async (value) => {
const key = keyGenerator(value); const key = keyGenerator(value);
const { err, reply } = await redis.getAsync(key); try {
const reply = await redis.get(key);
if (!err && reply) { if (reply) {
try {
Logger.debug(`Got data from redis: ${reply}`); Logger.debug(`Got data from redis: ${reply}`);
return { return {
value, value,
result: JSON.parse(reply) result: JSON.parse(reply)
}; };
} catch (e) { } //eslint-disable-line no-empty }
} } catch (e) { } //eslint-disable-line no-empty
return { return {
value, value,
@@ -71,7 +67,7 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
} }
for (const key in newResults) { for (const key in newResults) {
redis.setAsync(key, JSON.stringify(newResults[key])); redis.set(key, JSON.stringify(newResults[key]));
} }
}); });
} }
@@ -81,16 +77,16 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void { function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void {
if (videoInfo) { if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)); redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID)); if (videoInfo.userID) redis.del(reputationKey(videoInfo.userID));
} }
} }
function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void { function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
if (videoInfo) { if (videoInfo) {
redis.delAsync(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)); redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
} }
} }

71
src/utils/redis.ts
View File

@@ -1,59 +1,56 @@
import { config } from "../config"; import { config } from "../config";
import { Logger } from "./logger"; import { Logger } from "./logger";
import redis, { Callback } from "redis"; import { createClient } from "redis";
import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@node-redis/client/dist/lib/commands";
import { ClientCommandOptions } from "@node-redis/client/dist/lib/client";
import { RedisReply } from "rate-limit-redis";
interface RedisSB { interface RedisSB {
get(key: string, callback?: Callback<string | null>): void; get(key: RedisCommandArgument): Promise<string>;
getAsync?(key: string): Promise<{err: Error | null, reply: string | null}>; set(key: RedisCommandArgument, value: RedisCommandArgument): Promise<string>;
set(key: string, value: string, callback?: Callback<string | null>): void; setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
setAsync?(key: string, value: string): Promise<{err: Error | null, reply: string | null}>; del(...keys: [RedisCommandArgument]): Promise<number>;
setAsyncEx?(key: string, value: string, seconds: number): Promise<{err: Error | null, reply: string | null}>; increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
delAsync?(...keys: [string]): Promise<Error | null>; sendCommand(args: RedisCommandArguments, options?: ClientCommandOptions): Promise<RedisReply>;
close?(flush?: boolean): void; quit(): Promise<void>;
increment?(key: string): Promise<{err: Error| null, replies: any[] | null}>;
} }
let exportObject: RedisSB = { let exportClient: RedisSB = {
get: (key, callback?) => callback(null, undefined), get: () => new Promise((resolve) => resolve(null)),
getAsync: () => set: () => new Promise((resolve) => resolve(null)),
new Promise((resolve) => resolve({ err: null, reply: undefined })), setEx: () => new Promise((resolve) => resolve(null)),
set: (key, value, callback) => callback(null, undefined), del: () => new Promise((resolve) => resolve(null)),
setAsync: () => increment: () => new Promise((resolve) => resolve(null)),
new Promise((resolve) => resolve({ err: null, reply: undefined })), sendCommand: () => new Promise((resolve) => resolve(null)),
setAsyncEx: () => quit: () => new Promise((resolve) => resolve(null)),
new Promise((resolve) => resolve({ err: null, reply: undefined })),
delAsync: () =>
new Promise((resolve) => resolve(null)),
increment: () =>
new Promise((resolve) => resolve({ err: null, replies: undefined })),
}; };
if (config.redis) { if (config.redis?.enabled) {
Logger.info("Connected to redis"); Logger.info("Connected to redis");
const client = redis.createClient(config.redis); const client = createClient(config.redis);
exportObject = client; client.connect();
exportClient = client;
const timeoutDuration = 200; const timeoutDuration = 200;
exportObject.getAsync = (key) => new Promise((resolve) => { const get = client.get.bind(client);
const timeout = setTimeout(() => resolve({ err: null, reply: undefined }), timeoutDuration); exportClient.get = (key) => new Promise((resolve, reject) => {
client.get(key, (err, reply) => { const timeout = setTimeout(() => reject(), timeoutDuration);
get(key).then((reply) => {
clearTimeout(timeout); clearTimeout(timeout);
resolve({ err, reply }); resolve(reply);
}); }).catch((err) => reject(err));
}); });
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({ err, reply }))); exportClient.increment = (key) => new Promise((resolve, reject) =>
exportObject.setAsyncEx = (key, value, seconds) => new Promise((resolve) => client.setex(key, seconds, value, (err, reply) => resolve({ err, reply })));
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
exportObject.close = (flush) => client.end(flush);
exportObject.increment = (key) => new Promise((resolve) =>
client.multi() client.multi()
.incr(key) .incr(key)
.expire(key, 60) .expire(key, 60)
.exec((err, replies) => resolve({ err, replies })) .exec()
.then((reply) => resolve(reply))
.catch((err) => reject(err))
); );
client.on("error", function(error) { client.on("error", function(error) {
Logger.error(error); Logger.error(error);
}); });
} }
export default exportObject; export default exportClient;

1
test.json
View File

@@ -16,6 +16,7 @@
"schemaFolder": "./databases", "schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql", "dbSchema": "./databases/_sponsorTimes.db.sql",
"privateDBSchema": "./databases/_private.db.sql", "privateDBSchema": "./databases/_private.db.sql",
"categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
"mode": "test", "mode": "test",
"readOnly": false, "readOnly": false,
"webhooks": [ "webhooks": [

4
test/cases/getStatus.ts
View File

@@ -89,7 +89,7 @@ describe("getStatus", () => {
}); });
it("Should be able to get statusRequests only", function (done) { it("Should be able to get statusRequests only", function (done) {
if (!config.redis) this.skip(); if (!config.redis?.enabled) this.skip();
client.get(`${endpoint}/statusRequests`) client.get(`${endpoint}/statusRequests`)
.then(res => { .then(res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);
@@ -100,7 +100,7 @@ describe("getStatus", () => {
}); });
it("Should be able to get status with statusRequests", function (done) { it("Should be able to get status with statusRequests", function (done) {
if (!config.redis) this.skip(); if (!config.redis?.enabled) this.skip();
client.get(endpoint) client.get(endpoint)
.then(res => { .then(res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);

16
test/cases/postSkipSegments.ts
View File

@@ -734,22 +734,6 @@ describe("postSkipSegments", () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it("Should be rejected if NB's predicted probability is <70%.", (done) => {
const videoID = "LevkAjUE6d4";
postSkipSegmentParam({
videoID,
startTime: 40,
endTime: 60,
userID: submitUserTwo,
category: "sponsor"
})
.then(res => {
assert.strictEqual(res.status, 200);
done();
})
.catch(err => done(err));
});
it("Should be rejected with custom message if user has to many active warnings", (done) => { it("Should be rejected with custom message if user has to many active warnings", (done) => {
postSkipSegmentJSON({ postSkipSegmentJSON({
userID: warnUser01, userID: warnUser01,

17
test/cases/redisTest.ts
View File

@@ -11,22 +11,21 @@ const randKey2 = genRandom(16);
describe("redis test", function() { describe("redis test", function() {
before(async function() { before(async function() {
if (!config.redis) this.skip(); if (!config.redis?.enabled) this.skip();
await redis.setAsync(randKey1, randValue1); await redis.set(randKey1, randValue1);
}); });
it("Should get stored value", (done) => { it("Should get stored value", (done) => {
redis.getAsync(randKey1) redis.get(randKey1)
.then(res => { .then(res => {
if (res.err) assert.fail(res.err); assert.strictEqual(res, randValue1);
assert.strictEqual(res.reply, randValue1);
done(); done();
}); }).catch(err => assert.fail(err));
}); });
it("Should not be able to get not stored value", (done) => { it("Should not be able to get not stored value", (done) => {
redis.getAsync(randKey2) redis.get(randKey2)
.then(res => { .then(res => {
if (res.reply || res.err ) assert.fail("Value should not be found"); if (res) assert.fail("Value should not be found");
done(); done();
}); }).catch(err => assert.fail(err));
}); });
}); });

14
test/cases/tempVip.ts
View File

@@ -6,6 +6,7 @@ import { client } from "../utils/httpClient";
import { db, privateDB } from "../../src/databases/databases"; import { db, privateDB } from "../../src/databases/databases";
import redis from "../../src/utils/redis"; import redis from "../../src/utils/redis";
import assert from "assert"; import assert from "assert";
import { Logger } from "../../src/utils/logger";
// helpers // helpers
const getSegment = (UUID: string) => db.prepare("get", `SELECT "votes", "locked", "category" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const getSegment = (UUID: string) => db.prepare("get", `SELECT "votes", "locked", "category" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
@@ -51,13 +52,18 @@ const postVoteCategory = (userID: string, UUID: string, category: string) => cli
} }
}); });
const checkUserVIP = async (publicID: HashedUserID) => { const checkUserVIP = async (publicID: HashedUserID) => {
const { reply } = await redis.getAsync(tempVIPKey(publicID)); try {
return reply; const reply = await redis.get(tempVIPKey(publicID));
return reply;
} catch (e) {
Logger.error(e as string);
return false;
}
}; };
describe("tempVIP test", function() { describe("tempVIP test", function() {
before(async function() { before(async function() {
if (!config.redis) this.skip(); if (!config.redis?.enabled) this.skip();
const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
await db.prepare("run", insertSponsorTimeQuery, ["channelid-convert", 0, 1, 0, 0, UUID0, "testman", 0, 50, "sponsor", 0]); await db.prepare("run", insertSponsorTimeQuery, ["channelid-convert", 0, 1, 0, 0, UUID0, "testman", 0, 50, "sponsor", 0]);
@@ -67,7 +73,7 @@ describe("tempVIP test", function() {
await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP1]); await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP1]);
await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP2]); await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP2]);
// clear redis if running consecutive tests // clear redis if running consecutive tests
await redis.delAsync(tempVIPKey(publicTempVIPOne)); await redis.del(tempVIPKey(publicTempVIPOne));
}); });
it("Should update db version when starting the application", () => { it("Should update db version when starting the application", () => {

2
test/test.ts
View File

@@ -57,7 +57,7 @@ async function init() {
mocha.run((failures) => { mocha.run((failures) => {
mockServer.close(); mockServer.close();
server.close(); server.close();
redis.close(true); redis.quit();
process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures
}); });
}); });