mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2025-12-06 19:47:00 +03:00
Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into fullVideoLabels
This commit is contained in:
Michael C
46
.github/workflows/docker-build.yml
vendored
Normal file
46
.github/workflows/docker-build.yml
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
# Based on https://github.com/ajayyy/sb-mirror/blob/main/.github/workflows/docker-build.yml
|
||||
name: multi-build-docker
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
name:
|
||||
required: true
|
||||
type: string
|
||||
username:
|
||||
required: true
|
||||
type: string
|
||||
folder:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
GH_TOKEN:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
build_container:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ inputs.username }}/${{ inputs.name }}
|
||||
tags: |
|
||||
type-raw,value=alpine
|
||||
flavor: |
|
||||
latest=true
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GH_TOKEN }}
|
||||
- name: push
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
context: ${{ inputs.folder }}
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
25
.github/workflows/sb-server.yml
vendored
Normal file
25
.github/workflows/sb-server.yml
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
name: Docker image builds
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
sb-server:
|
||||
uses: ./.github/workflows/docker-build.yml
|
||||
with:
|
||||
name: "sb-server"
|
||||
username: "ajayyy"
|
||||
folder: "."
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
rsync-host:
|
||||
needs: sb-server
|
||||
uses: ./.github/workflows/docker-build.yml
|
||||
with:
|
||||
name: "rsync-host"
|
||||
username: "ajayyy"
|
||||
folder: "./rsync"
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
12
Dockerfile
12
Dockerfile
@@ -1,13 +1,15 @@
|
||||
FROM node:14-alpine as builder
|
||||
RUN apk add --no-cache --virtual .build-deps python make g++
|
||||
FROM node:16-alpine as builder
|
||||
RUN apk add --no-cache --virtual .build-deps python3 make g++
|
||||
COPY package.json package-lock.json tsconfig.json entrypoint.sh ./
|
||||
COPY src src
|
||||
RUN npm ci && npm run tsc
|
||||
|
||||
FROM node:14-alpine as app
|
||||
FROM node:16-alpine as app
|
||||
WORKDIR /usr/src/app
|
||||
COPY --from=builder node_modules .
|
||||
COPY --from=builder dist ./dist
|
||||
RUN apk add git postgresql-client
|
||||
COPY --from=builder ./node_modules ./node_modules
|
||||
COPY --from=builder ./dist ./dist
|
||||
COPY ./.git ./.git
|
||||
COPY entrypoint.sh .
|
||||
COPY databases/*.sql databases/
|
||||
EXPOSE 8080
|
||||
|
||||
3
ci.json
3
ci.json
@@ -17,13 +17,16 @@
|
||||
"port": 5432
|
||||
},
|
||||
"redis": {
|
||||
"socket": {
|
||||
"host": "localhost",
|
||||
"port": 6379
|
||||
}
|
||||
},
|
||||
"createDatabaseIfNotExist": true,
|
||||
"schemaFolder": "./databases",
|
||||
"dbSchema": "./databases/_sponsorTimes.db.sql",
|
||||
"privateDBSchema": "./databases/_private.db.sql",
|
||||
"categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
|
||||
"mode": "test",
|
||||
"readOnly": false,
|
||||
"webhooks": [
|
||||
|
||||
@@ -7,7 +7,6 @@ services:
|
||||
- database.env
|
||||
volumes:
|
||||
- database-data:/var/lib/postgresql/data
|
||||
- ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
|
||||
ports:
|
||||
- 5432:5432
|
||||
restart: always
|
||||
|
||||
@@ -2,25 +2,11 @@
|
||||
set -e
|
||||
echo 'Entrypoint script'
|
||||
cd /usr/src/app
|
||||
cp /etc/sponsorblock/config.json . || cat <<EOF > config.json
|
||||
|
||||
# blank config, use defaults
|
||||
cat <<EOF > config.json
|
||||
{
|
||||
"port": 8080,
|
||||
"globalSalt": "[CHANGE THIS]",
|
||||
"adminUserID": "[CHANGE THIS]",
|
||||
"youtubeAPIKey": null,
|
||||
"discordReportChannelWebhookURL": null,
|
||||
"discordFirstTimeSubmissionsWebhookURL": null,
|
||||
"discordAutoModWebhookURL": null,
|
||||
"proxySubmission": null,
|
||||
"behindProxy": "X-Forwarded-For",
|
||||
"db": "./databases/sponsorTimes.db",
|
||||
"privateDB": "./databases/private.db",
|
||||
"createDatabaseIfNotExist": true,
|
||||
"schemaFolder": "./databases",
|
||||
"dbSchema": "./databases/_sponsorTimes.db.sql",
|
||||
"privateDBSchema": "./databases/_private.db.sql",
|
||||
"mode": "development",
|
||||
"readOnly": false
|
||||
}
|
||||
EOF
|
||||
node dist/index.js
|
||||
|
||||
node dist/src/index.js
|
||||
@@ -49,7 +49,7 @@ http {
|
||||
#server localhost:4446;
|
||||
#server localhost:4447;
|
||||
#server localhost:4448;
|
||||
server 10.0.0.4:4441 max_fails=25 fail_timeout=20s;
|
||||
#server 10.0.0.4:4441 max_fails=25 fail_timeout=20s;
|
||||
|
||||
#server 10.0.0.3:4441 max_fails=25 fail_timeout=20s;
|
||||
#server 10.0.0.3:4442 max_fails=25 fail_timeout=20s;
|
||||
@@ -72,9 +72,18 @@ http {
|
||||
server 10.0.0.13:4441 max_fails=25 fail_timeout=20s;
|
||||
server 10.0.0.13:4442 max_fails=25 fail_timeout=20s;
|
||||
|
||||
server 10.0.0.14:4441 max_fails=25 fail_timeout=20s;
|
||||
server 10.0.0.14:4442 max_fails=25 fail_timeout=20s;
|
||||
|
||||
server 10.0.0.11:4441 max_fails=25 fail_timeout=20s;
|
||||
server 10.0.0.11:4442 max_fails=25 fail_timeout=20s;
|
||||
|
||||
server 10.0.0.16:4441 max_fails=25 fail_timeout=20s;
|
||||
server 10.0.0.16:4442 max_fails=25 fail_timeout=20s;
|
||||
|
||||
server 10.0.0.17:4441 max_fails=25 fail_timeout=20s;
|
||||
server 10.0.0.17:4442 max_fails=25 fail_timeout=20s;
|
||||
|
||||
#server 134.209.69.251:80 backup;
|
||||
|
||||
#server 116.203.32.253:80 backup;
|
||||
@@ -84,6 +93,7 @@ http {
|
||||
#server localhost:4441;
|
||||
#server localhost:4442;
|
||||
server 10.0.0.3:4441 max_fails=25 fail_timeout=15s;
|
||||
server 10.0.0.4:4441 max_fails=25 fail_timeout=15s;
|
||||
#server 10.0.0.3:4442;
|
||||
}
|
||||
upstream backend_db {
|
||||
|
||||
286
package-lock.json
generated
286
package-lock.json
generated
@@ -15,22 +15,21 @@
|
||||
"cron": "^1.8.2",
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^4.1.1",
|
||||
"express-rate-limit": "^5.5.1",
|
||||
"express-rate-limit": "^6.3.0",
|
||||
"lodash": "^4.17.21",
|
||||
"pg": "^8.7.1",
|
||||
"redis": "^3.1.2",
|
||||
"rate-limit-redis": "^3.0.1",
|
||||
"redis": "^4.0.6",
|
||||
"sync-mysql": "^3.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.4.1",
|
||||
"@types/cron": "^1.7.3",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/express-rate-limit": "^5.1.3",
|
||||
"@types/lodash": "^4.14.178",
|
||||
"@types/mocha": "^9.0.0",
|
||||
"@types/node": "^16.11.11",
|
||||
"@types/pg": "^8.6.1",
|
||||
"@types/redis": "^2.8.32",
|
||||
"@typescript-eslint/eslint-plugin": "^5.5.0",
|
||||
"@typescript-eslint/parser": "^5.5.0",
|
||||
"eslint": "^8.3.0",
|
||||
@@ -128,6 +127,65 @@
|
||||
"integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@node-redis/bloom": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz",
|
||||
"integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==",
|
||||
"peerDependencies": {
|
||||
"@node-redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-redis/client": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz",
|
||||
"integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==",
|
||||
"dependencies": {
|
||||
"cluster-key-slot": "1.1.0",
|
||||
"generic-pool": "3.8.2",
|
||||
"redis-parser": "3.0.0",
|
||||
"yallist": "4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-redis/client/node_modules/yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
|
||||
},
|
||||
"node_modules/@node-redis/graph": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz",
|
||||
"integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==",
|
||||
"peerDependencies": {
|
||||
"@node-redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-redis/json": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz",
|
||||
"integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==",
|
||||
"peerDependencies": {
|
||||
"@node-redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-redis/search": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz",
|
||||
"integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==",
|
||||
"peerDependencies": {
|
||||
"@node-redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-redis/time-series": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz",
|
||||
"integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==",
|
||||
"peerDependencies": {
|
||||
"@node-redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
@@ -293,15 +351,6 @@
|
||||
"@types/serve-static": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/express-rate-limit": {
|
||||
"version": "5.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/express-rate-limit/-/express-rate-limit-5.1.3.tgz",
|
||||
"integrity": "sha512-H+TYy3K53uPU2TqPGFYaiWc2xJV6+bIFkDd/Ma2/h67Pa6ARk9kWE0p/K9OH1Okm0et9Sfm66fmXoAxsH2PHXg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/express": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/express-serve-static-core": {
|
||||
"version": "4.17.26",
|
||||
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz",
|
||||
@@ -366,15 +415,6 @@
|
||||
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
|
||||
"devOptional": true
|
||||
},
|
||||
"node_modules/@types/redis": {
|
||||
"version": "2.8.32",
|
||||
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
|
||||
"integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/serve-static": {
|
||||
"version": "1.13.10",
|
||||
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz",
|
||||
@@ -1214,6 +1254,14 @@
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/cluster-key-slot": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
|
||||
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
|
||||
@@ -1444,14 +1492,6 @@
|
||||
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
|
||||
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
|
||||
},
|
||||
"node_modules/denque": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
|
||||
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/depd": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
|
||||
@@ -1889,9 +1929,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/express-rate-limit": {
|
||||
"version": "5.5.1",
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz",
|
||||
"integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg=="
|
||||
"version": "6.3.0",
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.3.0.tgz",
|
||||
"integrity": "sha512-932Io1VGKjM3ppi7xW9sb1J5nVkEJSUiOtHw2oE+JyHks1e+AXuOBSXbJKM0mcXwEnW1TibJibQ455Ow1YFjfg==",
|
||||
"engines": {
|
||||
"node": ">= 12.9.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"express": "^4"
|
||||
}
|
||||
},
|
||||
"node_modules/express/node_modules/debug": {
|
||||
"version": "2.6.9",
|
||||
@@ -2205,6 +2251,14 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/generic-pool": {
|
||||
"version": "3.8.2",
|
||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz",
|
||||
"integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==",
|
||||
"engines": {
|
||||
"node": ">= 4"
|
||||
}
|
||||
},
|
||||
"node_modules/get-caller-file": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||
@@ -3140,9 +3194,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/moment": {
|
||||
"version": "2.29.1",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
|
||||
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==",
|
||||
"version": "2.29.2",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz",
|
||||
"integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
@@ -3876,6 +3930,17 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/rate-limit-redis": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz",
|
||||
"integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==",
|
||||
"engines": {
|
||||
"node": ">= 14.5.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"express-rate-limit": "^6"
|
||||
}
|
||||
},
|
||||
"node_modules/raw-body": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
|
||||
@@ -3931,28 +3996,18 @@
|
||||
}
|
||||
},
|
||||
"node_modules/redis": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz",
|
||||
"integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==",
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz",
|
||||
"integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==",
|
||||
"dependencies": {
|
||||
"denque": "^1.5.0",
|
||||
"redis-commands": "^1.7.0",
|
||||
"redis-errors": "^1.2.0",
|
||||
"redis-parser": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-redis"
|
||||
"@node-redis/bloom": "1.0.1",
|
||||
"@node-redis/client": "1.0.5",
|
||||
"@node-redis/graph": "1.0.0",
|
||||
"@node-redis/json": "1.0.2",
|
||||
"@node-redis/search": "1.0.5",
|
||||
"@node-redis/time-series": "1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-commands": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
|
||||
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
|
||||
},
|
||||
"node_modules/redis-errors": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
|
||||
@@ -5169,6 +5224,54 @@
|
||||
"integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
|
||||
"dev": true
|
||||
},
|
||||
"@node-redis/bloom": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz",
|
||||
"integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==",
|
||||
"requires": {}
|
||||
},
|
||||
"@node-redis/client": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz",
|
||||
"integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==",
|
||||
"requires": {
|
||||
"cluster-key-slot": "1.1.0",
|
||||
"generic-pool": "3.8.2",
|
||||
"redis-parser": "3.0.0",
|
||||
"yallist": "4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"@node-redis/graph": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz",
|
||||
"integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==",
|
||||
"requires": {}
|
||||
},
|
||||
"@node-redis/json": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz",
|
||||
"integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==",
|
||||
"requires": {}
|
||||
},
|
||||
"@node-redis/search": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz",
|
||||
"integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==",
|
||||
"requires": {}
|
||||
},
|
||||
"@node-redis/time-series": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz",
|
||||
"integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==",
|
||||
"requires": {}
|
||||
},
|
||||
"@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
@@ -5319,15 +5422,6 @@
|
||||
"@types/serve-static": "*"
|
||||
}
|
||||
},
|
||||
"@types/express-rate-limit": {
|
||||
"version": "5.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/express-rate-limit/-/express-rate-limit-5.1.3.tgz",
|
||||
"integrity": "sha512-H+TYy3K53uPU2TqPGFYaiWc2xJV6+bIFkDd/Ma2/h67Pa6ARk9kWE0p/K9OH1Okm0et9Sfm66fmXoAxsH2PHXg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/express": "*"
|
||||
}
|
||||
},
|
||||
"@types/express-serve-static-core": {
|
||||
"version": "4.17.26",
|
||||
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz",
|
||||
@@ -5392,15 +5486,6 @@
|
||||
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
|
||||
"devOptional": true
|
||||
},
|
||||
"@types/redis": {
|
||||
"version": "2.8.32",
|
||||
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
|
||||
"integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/serve-static": {
|
||||
"version": "1.13.10",
|
||||
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz",
|
||||
@@ -6029,6 +6114,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"cluster-key-slot": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
|
||||
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw=="
|
||||
},
|
||||
"code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
|
||||
@@ -6209,11 +6299,6 @@
|
||||
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
|
||||
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
|
||||
},
|
||||
"denque": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
|
||||
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw=="
|
||||
},
|
||||
"depd": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
|
||||
@@ -6557,9 +6642,10 @@
|
||||
}
|
||||
},
|
||||
"express-rate-limit": {
|
||||
"version": "5.5.1",
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz",
|
||||
"integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg=="
|
||||
"version": "6.3.0",
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.3.0.tgz",
|
||||
"integrity": "sha512-932Io1VGKjM3ppi7xW9sb1J5nVkEJSUiOtHw2oE+JyHks1e+AXuOBSXbJKM0mcXwEnW1TibJibQ455Ow1YFjfg==",
|
||||
"requires": {}
|
||||
},
|
||||
"fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
@@ -6802,6 +6888,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"generic-pool": {
|
||||
"version": "3.8.2",
|
||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz",
|
||||
"integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg=="
|
||||
},
|
||||
"get-caller-file": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||
@@ -7502,9 +7593,9 @@
|
||||
}
|
||||
},
|
||||
"moment": {
|
||||
"version": "2.29.1",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
|
||||
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
|
||||
"version": "2.29.2",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz",
|
||||
"integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg=="
|
||||
},
|
||||
"moment-timezone": {
|
||||
"version": "0.5.34",
|
||||
@@ -8051,6 +8142,12 @@
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
|
||||
},
|
||||
"rate-limit-redis": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz",
|
||||
"integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==",
|
||||
"requires": {}
|
||||
},
|
||||
"raw-body": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
|
||||
@@ -8094,21 +8191,18 @@
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz",
|
||||
"integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==",
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz",
|
||||
"integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==",
|
||||
"requires": {
|
||||
"denque": "^1.5.0",
|
||||
"redis-commands": "^1.7.0",
|
||||
"redis-errors": "^1.2.0",
|
||||
"redis-parser": "^3.0.0"
|
||||
"@node-redis/bloom": "1.0.1",
|
||||
"@node-redis/client": "1.0.5",
|
||||
"@node-redis/graph": "1.0.0",
|
||||
"@node-redis/json": "1.0.2",
|
||||
"@node-redis/search": "1.0.5",
|
||||
"@node-redis/time-series": "1.0.2"
|
||||
}
|
||||
},
|
||||
"redis-commands": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
|
||||
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
|
||||
},
|
||||
"redis-errors": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
|
||||
|
||||
@@ -23,22 +23,21 @@
|
||||
"cron": "^1.8.2",
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^4.1.1",
|
||||
"express-rate-limit": "^5.5.1",
|
||||
"express-rate-limit": "^6.3.0",
|
||||
"lodash": "^4.17.21",
|
||||
"pg": "^8.7.1",
|
||||
"redis": "^3.1.2",
|
||||
"rate-limit-redis": "^3.0.1",
|
||||
"redis": "^4.0.6",
|
||||
"sync-mysql": "^3.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.4.1",
|
||||
"@types/cron": "^1.7.3",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/express-rate-limit": "^5.1.3",
|
||||
"@types/lodash": "^4.14.178",
|
||||
"@types/mocha": "^9.0.0",
|
||||
"@types/node": "^16.11.11",
|
||||
"@types/pg": "^8.6.1",
|
||||
"@types/redis": "^2.8.32",
|
||||
"@typescript-eslint/eslint-plugin": "^5.5.0",
|
||||
"@typescript-eslint/parser": "^5.5.0",
|
||||
"eslint": "^8.3.0",
|
||||
|
||||
6
rsync/Dockerfile
Normal file
6
rsync/Dockerfile
Normal file
@@ -0,0 +1,6 @@
|
||||
FROM ghcr.io/ajayyy/sb-server:latest
|
||||
EXPOSE 873/tcp
|
||||
RUN apk add rsync>3.2.4-r0
|
||||
RUN mkdir /usr/src/app/database-export
|
||||
|
||||
CMD rsync --no-detach --daemon & ./entrypoint.sh
|
||||
@@ -27,7 +27,7 @@ import { loggerMiddleware } from "./middleware/logger";
|
||||
import { corsMiddleware } from "./middleware/cors";
|
||||
import { apiCspMiddleware } from "./middleware/apiCsp";
|
||||
import { rateLimitMiddleware } from "./middleware/requestRateLimit";
|
||||
import dumpDatabase, { appExportPath, redirectLink } from "./routes/dumpDatabase";
|
||||
import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
|
||||
import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
|
||||
import { postClearCache } from "./routes/postClearCache";
|
||||
import { addUnlistedVideo } from "./routes/addUnlistedVideo";
|
||||
@@ -208,10 +208,10 @@ function setupRoutes(router: Router) {
|
||||
router.get("/api/videoLabels", getVideoLabels);
|
||||
router.get("/api/videoLabels/:prefix", getVideoLabelsByHash);
|
||||
|
||||
if (config.postgres) {
|
||||
if (config.postgres?.enabled) {
|
||||
router.get("/database", (req, res) => dumpDatabase(req, res, true));
|
||||
router.get("/database.json", (req, res) => dumpDatabase(req, res, false));
|
||||
router.get("/database/*", redirectLink);
|
||||
router.get("/database/*", downloadFile);
|
||||
router.use("/download", express.static(appExportPath));
|
||||
} else {
|
||||
router.get("/database.db", function (req: Request, res: Response) {
|
||||
|
||||
102
src/config.ts
102
src/config.ts
@@ -1,6 +1,7 @@
|
||||
import fs from "fs";
|
||||
import { SBSConfig } from "./types/config.model";
|
||||
import packageJson from "../package.json";
|
||||
import { isBoolean, isNumber } from "lodash";
|
||||
|
||||
const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test;
|
||||
const configFile = process.env.TEST_POSTGRES ? "ci.json"
|
||||
@@ -9,7 +10,7 @@ const configFile = process.env.TEST_POSTGRES ? "ci.json"
|
||||
export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8"));
|
||||
|
||||
addDefaults(config, {
|
||||
port: 80,
|
||||
port: 8080,
|
||||
behindProxy: "X-Forwarded-For",
|
||||
db: "./databases/sponsorTimes.db",
|
||||
privateDB: "./databases/private.db",
|
||||
@@ -19,7 +20,7 @@ addDefaults(config, {
|
||||
privateDBSchema: "./databases/_private.db.sql",
|
||||
readOnly: false,
|
||||
webhooks: [],
|
||||
categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
|
||||
categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"],
|
||||
categorySupport: {
|
||||
sponsor: ["skip", "mute", "full"],
|
||||
selfpromo: ["skip", "mute", "full"],
|
||||
@@ -34,14 +35,14 @@ addDefaults(config, {
|
||||
chapter: ["chapter"]
|
||||
},
|
||||
maxNumberOfActiveWarnings: 1,
|
||||
hoursAfterWarningExpires: 24,
|
||||
hoursAfterWarningExpires: 16300000,
|
||||
adminUserID: "",
|
||||
discordCompletelyIncorrectReportWebhookURL: null,
|
||||
discordFirstTimeSubmissionsWebhookURL: null,
|
||||
discordNeuralBlockRejectWebhookURL: null,
|
||||
discordFailedReportChannelWebhookURL: null,
|
||||
discordReportChannelWebhookURL: null,
|
||||
getTopUsersCacheTimeMinutes: 0,
|
||||
getTopUsersCacheTimeMinutes: 240,
|
||||
globalSalt: null,
|
||||
mode: "",
|
||||
neuralBlockURL: null,
|
||||
@@ -49,15 +50,15 @@ addDefaults(config, {
|
||||
rateLimit: {
|
||||
vote: {
|
||||
windowMs: 900000,
|
||||
max: 20,
|
||||
message: "Too many votes, please try again later",
|
||||
statusCode: 429,
|
||||
max: 15,
|
||||
message: "OK",
|
||||
statusCode: 200,
|
||||
},
|
||||
view: {
|
||||
windowMs: 900000,
|
||||
max: 20,
|
||||
max: 10,
|
||||
statusCode: 200,
|
||||
message: "Too many views, please try again later",
|
||||
message: "OK",
|
||||
},
|
||||
rate: {
|
||||
windowMs: 900000,
|
||||
@@ -70,12 +71,17 @@ addDefaults(config, {
|
||||
newLeafURLs: null,
|
||||
maxRewardTimePerSegmentInSeconds: 600,
|
||||
poiMinimumStartTime: 2,
|
||||
postgres: null,
|
||||
postgres: {
|
||||
enabled: false,
|
||||
user: "",
|
||||
host: "",
|
||||
password: "",
|
||||
port: 5432
|
||||
},
|
||||
dumpDatabase: {
|
||||
enabled: false,
|
||||
minTimeBetweenMs: 60000,
|
||||
minTimeBetweenMs: 180000,
|
||||
appExportPath: "./docker/database-export",
|
||||
postgresExportPath: "/opt/exports",
|
||||
tables: [{
|
||||
name: "sponsorTimes",
|
||||
order: "timeSubmitted"
|
||||
@@ -95,11 +101,32 @@ addDefaults(config, {
|
||||
},
|
||||
{
|
||||
name: "vipUsers"
|
||||
},
|
||||
{
|
||||
name: "unlistedVideos"
|
||||
},
|
||||
{
|
||||
name: "videoInfo"
|
||||
},
|
||||
{
|
||||
name: "ratings"
|
||||
}]
|
||||
},
|
||||
diskCache: null,
|
||||
crons: null
|
||||
diskCache: {
|
||||
max: 10737418240
|
||||
},
|
||||
crons: null,
|
||||
redis: {
|
||||
enabled: false,
|
||||
socket: {
|
||||
host: "",
|
||||
port: 0
|
||||
},
|
||||
disableOfflineQueue: true
|
||||
}
|
||||
});
|
||||
loadFromEnv(config);
|
||||
migrate(config);
|
||||
|
||||
// Add defaults
|
||||
function addDefaults(config: SBSConfig, defaults: SBSConfig) {
|
||||
@@ -109,3 +136,50 @@ function addDefaults(config: SBSConfig, defaults: SBSConfig) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function migrate(config: SBSConfig) {
|
||||
// Redis change
|
||||
if (config.redis) {
|
||||
const redisConfig = config.redis as any;
|
||||
if (redisConfig.host || redisConfig.port) {
|
||||
config.redis.socket = {
|
||||
host: redisConfig.host,
|
||||
port: redisConfig.port
|
||||
};
|
||||
}
|
||||
|
||||
if (redisConfig.enable_offline_queue !== undefined) {
|
||||
config.disableOfflineQueue = !redisConfig.enable_offline_queue;
|
||||
}
|
||||
|
||||
if (redisConfig.socket?.host && redisConfig.enabled === undefined) {
|
||||
redisConfig.enabled = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (config.postgres && config.postgres.user && config.postgres.enabled === undefined) {
|
||||
config.postgres.enabled = true;
|
||||
}
|
||||
}
|
||||
|
||||
function loadFromEnv(config: SBSConfig, prefix = "") {
|
||||
for (const key in config) {
|
||||
const fullKey = (prefix ? `${prefix}_` : "") + key;
|
||||
const data = config[key];
|
||||
|
||||
if (typeof data === "object" && !Array.isArray(data)) {
|
||||
loadFromEnv(data, fullKey);
|
||||
} else if (process.env[fullKey]) {
|
||||
const value = process.env[fullKey];
|
||||
if (isNumber(value)) {
|
||||
config[key] = parseInt(value, 10);
|
||||
} else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") {
|
||||
config[key] = value === "true";
|
||||
} else if (key === "newLeafURLs") {
|
||||
config[key] = [value];
|
||||
} else {
|
||||
config[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -94,10 +94,6 @@ export class Postgres implements IDatabase {
|
||||
await client.query(`CREATE DATABASE "${this.config.postgres.database}"
|
||||
WITH
|
||||
OWNER = ${this.config.postgres.user}
|
||||
ENCODING = 'UTF8'
|
||||
LC_COLLATE = 'en_US.utf8'
|
||||
LC_CTYPE = 'en_US.utf8'
|
||||
TABLESPACE = pg_default
|
||||
CONNECTION LIMIT = -1;`
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ let privateDB: IDatabase;
|
||||
if (config.mysql) {
|
||||
db = new Mysql(config.mysql);
|
||||
privateDB = new Mysql(config.privateMysql);
|
||||
} else if (config.postgres) {
|
||||
} else if (config.postgres?.enabled) {
|
||||
db = new Postgres({
|
||||
dbSchemaFileName: config.dbSchema,
|
||||
dbSchemaFolder: config.schemaFolder,
|
||||
@@ -72,7 +72,7 @@ async function initDb(): Promise<void> {
|
||||
const tables = config?.dumpDatabase?.tables ?? [];
|
||||
const tableNames = tables.map(table => table.name);
|
||||
for (const table of tableNames) {
|
||||
const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`;
|
||||
const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`;
|
||||
await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,23 @@
|
||||
import { getIP } from "../utils/getIP";
|
||||
import { getHash } from "../utils/getHash";
|
||||
import { getHashCache } from "../utils/getHashCache";
|
||||
import rateLimit from "express-rate-limit";
|
||||
import rateLimit, { RateLimitRequestHandler } from "express-rate-limit";
|
||||
import { RateLimitConfig } from "../types/config.model";
|
||||
import { Request } from "express";
|
||||
import { isUserVIP } from "../utils/isUserVIP";
|
||||
import { UserID } from "../types/user.model";
|
||||
import RedisStore from "rate-limit-redis";
|
||||
import redis from "../utils/redis";
|
||||
import { config } from "../config";
|
||||
|
||||
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): rateLimit.RateLimit {
|
||||
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): RateLimitRequestHandler {
|
||||
return rateLimit({
|
||||
windowMs: limitConfig.windowMs,
|
||||
max: limitConfig.max,
|
||||
message: limitConfig.message,
|
||||
statusCode: limitConfig.statusCode,
|
||||
headers: false,
|
||||
legacyHeaders: false,
|
||||
standardHeaders: false,
|
||||
keyGenerator: (req) => {
|
||||
return getHash(getIP(req), 1);
|
||||
},
|
||||
@@ -23,6 +27,9 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r
|
||||
} else {
|
||||
return next();
|
||||
}
|
||||
}
|
||||
},
|
||||
store: config.redis?.enabled ? new RedisStore({
|
||||
sendCommand: (...args: string[]) => redis.sendCommand(args),
|
||||
}) : null,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import { isUserVIP } from "../utils/isUserVIP";
|
||||
import { HashedUserID } from "../types/user.model";
|
||||
import redis from "../utils/redis";
|
||||
import { tempVIPKey } from "../utils/redisKeys";
|
||||
import { Logger } from "../utils/logger";
|
||||
|
||||
interface AddUserAsTempVIPRequest extends Request {
|
||||
query: {
|
||||
@@ -65,12 +66,22 @@ export async function addUserAsTempVIP(req: AddUserAsTempVIPRequest, res: Respon
|
||||
if (!channelInfo?.id) {
|
||||
return res.status(404).send(`No channel found for videoID ${channelVideoID}`);
|
||||
}
|
||||
await redis.setAsyncEx(tempVIPKey(userID), channelInfo?.id, dayInSeconds);
|
||||
|
||||
try {
|
||||
await redis.setEx(tempVIPKey(userID), dayInSeconds, channelInfo?.id);
|
||||
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
|
||||
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
|
||||
} catch (e) {
|
||||
Logger.error(e as string);
|
||||
return res.status(500).send();
|
||||
}
|
||||
|
||||
await redis.delAsync(tempVIPKey(userID));
|
||||
}
|
||||
try {
|
||||
await redis.del(tempVIPKey(userID));
|
||||
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
|
||||
return res.status(200).send(`Temp VIP removed`);
|
||||
} catch (e) {
|
||||
Logger.error(e as string);
|
||||
return res.status(500).send();
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { config } from "../config";
|
||||
import util from "util";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { ChildProcess, exec, ExecOptions, spawn } from "child_process";
|
||||
const unlink = util.promisify(fs.unlink);
|
||||
|
||||
const ONE_MINUTE = 1000 * 60;
|
||||
@@ -32,9 +33,19 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
|
||||
const tables = config?.dumpDatabase?.tables ?? [];
|
||||
const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
|
||||
export const appExportPath = config?.dumpDatabase?.appExportPath ?? "./docker/database-export";
|
||||
const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? "/opt/exports";
|
||||
const tableNames = tables.map(table => table.name);
|
||||
|
||||
const credentials: ExecOptions = {
|
||||
env: {
|
||||
...process.env,
|
||||
PGHOST: config.postgres.host,
|
||||
PGPORT: String(config.postgres.port),
|
||||
PGUSER: config.postgres.user,
|
||||
PGPASSWORD: String(config.postgres.password),
|
||||
PGDATABASE: "sponsorTimes",
|
||||
}
|
||||
}
|
||||
|
||||
interface TableDumpList {
|
||||
fileName: string;
|
||||
tableName: string;
|
||||
@@ -100,7 +111,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
|
||||
res.status(404).send("Database dump is disabled");
|
||||
return;
|
||||
}
|
||||
if (!config.postgres) {
|
||||
if (!config.postgres?.enabled) {
|
||||
res.status(404).send("Not supported on this instance");
|
||||
return;
|
||||
}
|
||||
@@ -170,12 +181,12 @@ async function getDbVersion(): Promise<number> {
|
||||
return row.value;
|
||||
}
|
||||
|
||||
export async function redirectLink(req: Request, res: Response): Promise<void> {
|
||||
export async function downloadFile(req: Request, res: Response): Promise<void> {
|
||||
if (!config?.dumpDatabase?.enabled) {
|
||||
res.status(404).send("Database dump is disabled");
|
||||
return;
|
||||
}
|
||||
if (!config.postgres) {
|
||||
if (!config.postgres?.enabled) {
|
||||
res.status(404).send("Not supported on this instance");
|
||||
return;
|
||||
}
|
||||
@@ -183,7 +194,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
|
||||
const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
|
||||
|
||||
if (file) {
|
||||
res.redirect(`/download/${file.fileName}`);
|
||||
res.sendFile(file.fileName, { root: appExportPath });
|
||||
} else {
|
||||
res.sendStatus(404);
|
||||
}
|
||||
@@ -210,9 +221,19 @@ async function queueDump(): Promise<void> {
|
||||
|
||||
for (const table of tables) {
|
||||
const fileName = `${table.name}_${startTime}.csv`;
|
||||
const file = `${postgresExportPath}/${fileName}`;
|
||||
await db.prepare("run", `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
|
||||
TO '${file}' WITH (FORMAT CSV, HEADER true);`);
|
||||
const file = `${appExportPath}/${fileName}`;
|
||||
|
||||
await new Promise<string>((resolve) => {
|
||||
exec(`psql -c "\\copy (SELECT * FROM \\"${table.name}\\"${table.order ? ` ORDER BY \\"${table.order}\\"` : ``})`
|
||||
+ ` TO '${file}' WITH (FORMAT CSV, HEADER true);"`, credentials, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
Logger.error(`[dumpDatabase] Failed to dump ${table.name} to ${file} due to ${stderr}`);
|
||||
}
|
||||
|
||||
resolve(error ? stderr : stdout);
|
||||
});
|
||||
})
|
||||
|
||||
dumpFiles.push({
|
||||
fileName,
|
||||
tableName: table.name,
|
||||
|
||||
@@ -223,7 +223,8 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
|
||||
|
||||
//The 3 makes -2 the minimum votes before being ignored completely
|
||||
//this can be changed if this system increases in popularity.
|
||||
const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost);
|
||||
const repFactor = choice.votes > 0 ? Math.max(1, choice.reputation + 1) : 1;
|
||||
const weight = Math.exp(choice.votes * repFactor + 3 + boost);
|
||||
totalWeight += Math.max(weight, 0);
|
||||
|
||||
return { ...choice, weight };
|
||||
|
||||
@@ -10,8 +10,12 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
|
||||
value = Array.isArray(value) ? value[0] : value;
|
||||
try {
|
||||
const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value;
|
||||
let statusRequests: unknown = 0;
|
||||
try {
|
||||
const numberRequests = await redis.increment("statusRequest");
|
||||
const statusRequests = numberRequests?.replies?.[0];
|
||||
statusRequests = numberRequests?.[0];
|
||||
} catch (error) { } // eslint-disable-line no-empty
|
||||
|
||||
const statusValues: Record<string, any> = {
|
||||
uptime: process.uptime(),
|
||||
commit: (global as any).HEADCOMMIT || "unknown",
|
||||
|
||||
@@ -28,12 +28,15 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
|
||||
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
|
||||
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
|
||||
|
||||
if (rows) {
|
||||
for (const row of rows) {
|
||||
userNames.push(row.userName);
|
||||
viewCounts.push(row.viewCount);
|
||||
totalSubmissions.push(row.totalSubmissions);
|
||||
minutesSaved.push(row.minutesSaved);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
userNames,
|
||||
|
||||
@@ -16,6 +16,7 @@ import { getReputation } from "../utils/reputation";
|
||||
import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
|
||||
import { HashedUserID, UserID } from "../types/user.model";
|
||||
import { isUserVIP } from "../utils/isUserVIP";
|
||||
import { isUserTempVIP } from "../utils/isUserTempVIP";
|
||||
import { parseUserAgent } from "../utils/userAgent";
|
||||
import { getService } from "../utils/getService";
|
||||
import axios from "axios";
|
||||
@@ -81,19 +82,19 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
|
||||
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
|
||||
|
||||
axios.post(config.discordFirstTimeSubmissionsWebhookURL, {
|
||||
"embeds": [{
|
||||
"title": data?.title,
|
||||
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
|
||||
"description": `Submission ID: ${UUID}\
|
||||
embeds: [{
|
||||
title: data?.title,
|
||||
url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
|
||||
description: `Submission ID: ${UUID}\
|
||||
\n\nTimestamp: \
|
||||
${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
|
||||
\n\nCategory: ${segmentInfo.category}`,
|
||||
"color": 10813440,
|
||||
"author": {
|
||||
"name": userID,
|
||||
color: 10813440,
|
||||
author: {
|
||||
name: userID,
|
||||
},
|
||||
"thumbnail": {
|
||||
"url": getMaxResThumbnail(data) || "",
|
||||
thumbnail: {
|
||||
url: getMaxResThumbnail(data) || "",
|
||||
},
|
||||
}],
|
||||
})
|
||||
@@ -112,55 +113,6 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
|
||||
}
|
||||
}
|
||||
|
||||
async function sendWebhooksNB(userID: string, videoID: string, UUID: string, startTime: number, endTime: number, category: string, probability: number, ytData: any) {
|
||||
const submissionInfoRow = await db.prepare("get", `SELECT
|
||||
(select count(1) from "sponsorTimes" where "userID" = ?) count,
|
||||
(select count(1) from "sponsorTimes" where "userID" = ? and "votes" <= -2) disregarded,
|
||||
coalesce((select "userName" FROM "userNames" WHERE "userID" = ?), ?) "userName"`,
|
||||
[userID, userID, userID, userID]);
|
||||
|
||||
let submittedBy: string;
|
||||
// If a userName was created then show both
|
||||
if (submissionInfoRow.userName !== userID) {
|
||||
submittedBy = `${submissionInfoRow.userName}\n${userID}`;
|
||||
} else {
|
||||
submittedBy = userID;
|
||||
}
|
||||
|
||||
// Send discord message
|
||||
if (config.discordNeuralBlockRejectWebhookURL === null) return;
|
||||
|
||||
axios.post(config.discordNeuralBlockRejectWebhookURL, {
|
||||
"embeds": [{
|
||||
"title": ytData.items[0].snippet.title,
|
||||
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`,
|
||||
"description": `**Submission ID:** ${UUID}\
|
||||
\n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
|
||||
\n**Predicted Probability:** ${probability}\
|
||||
\n**Category:** ${category}\
|
||||
\n**Submitted by:** ${submittedBy}\
|
||||
\n**Total User Submissions:** ${submissionInfoRow.count}\
|
||||
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`,
|
||||
"color": 10813440,
|
||||
"thumbnail": {
|
||||
"url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
|
||||
},
|
||||
}]
|
||||
})
|
||||
.then(res => {
|
||||
if (res.status >= 400) {
|
||||
Logger.error("Error sending NeuralBlock Discord hook");
|
||||
Logger.error(JSON.stringify(res));
|
||||
Logger.error("\n");
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
Logger.error("Failed to send NeuralBlock Discord hook.");
|
||||
Logger.error(JSON.stringify(err));
|
||||
Logger.error("\n");
|
||||
});
|
||||
}
|
||||
|
||||
// callback: function(reject: "String containing reason the submission was rejected")
|
||||
// returns: string when an error, false otherwise
|
||||
|
||||
@@ -168,99 +120,47 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
|
||||
// false for a pass - it was confusing and lead to this bug - any use of this function in
|
||||
// the future could have the same problem.
|
||||
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
|
||||
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service }) {
|
||||
if (apiVideoInfo) {
|
||||
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
|
||||
|
||||
const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => {
|
||||
if (!apiVideoInfo) return undefined;
|
||||
const { err, data } = apiVideoInfo;
|
||||
if (err) return false;
|
||||
// return undefined if API error
|
||||
if (err) return undefined;
|
||||
return data?.lengthSeconds;
|
||||
};
|
||||
// get duration from API
|
||||
const apiDuration = apiVideoDuration(apiVideoInfo);
|
||||
// if API fail or returns 0, get duration from client
|
||||
const duration = apiDuration || submission.videoDuration;
|
||||
// return false on undefined or 0
|
||||
if (!duration) return false;
|
||||
|
||||
const duration = apiVideoInfo?.data?.lengthSeconds;
|
||||
const segments = submission.segments;
|
||||
let nbString = "";
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
if (duration == 0) {
|
||||
// Allow submission if the duration is 0 (bug in youtube api)
|
||||
return false;
|
||||
} else {
|
||||
if (segments[i].category === "sponsor") {
|
||||
//Prepare timestamps to send to NB all at once
|
||||
nbString = `${nbString}${segments[i].segment[0]},${segments[i].segment[1]};`;
|
||||
}
|
||||
}
|
||||
}
|
||||
// map all times to float array
|
||||
const allSegmentTimes = segments.map(segment => [parseFloat(segment.segment[0]), parseFloat(segment.segment[1])]);
|
||||
|
||||
// Get all submissions for this user
|
||||
const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]);
|
||||
const allSegmentTimes = [];
|
||||
if (allSubmittedByUser !== undefined) {
|
||||
// add previous submissions by this user
|
||||
const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? AND "videoID" = ? AND "votes" > -1 AND "hidden" = 0`, [submission.userID, submission.videoID]);
|
||||
|
||||
if (allSubmittedByUser) {
|
||||
//add segments the user has previously submitted
|
||||
for (const segmentInfo of allSubmittedByUser) {
|
||||
allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
|
||||
}
|
||||
}
|
||||
|
||||
//add segments they are trying to add in this submission
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
const startTime = parseFloat(segments[i].segment[0]);
|
||||
const endTime = parseFloat(segments[i].segment[1]);
|
||||
allSegmentTimes.push([startTime, endTime]);
|
||||
const allSubmittedTimes = allSubmittedByUser.map((segment: { startTime: string, endTime: string }) => [parseFloat(segment.startTime), parseFloat(segment.endTime)]);
|
||||
allSegmentTimes.push(...allSubmittedTimes);
|
||||
}
|
||||
|
||||
//merge all the times into non-overlapping arrays
|
||||
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
|
||||
return a[0] - b[0] || a[1] - b[1];
|
||||
}));
|
||||
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort((a, b) => a[0] - b[0] || a[1] - b[1]));
|
||||
|
||||
const videoDuration = data?.lengthSeconds;
|
||||
if (videoDuration != 0) {
|
||||
let allSegmentDuration = 0;
|
||||
//sum all segment times together
|
||||
allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
|
||||
if (allSegmentDuration > (videoDuration / 100) * 80) {
|
||||
const allSegmentDuration = allSegmentsSorted.reduce((acc, curr) => acc + (curr[1] - curr[0]), 0);
|
||||
|
||||
if (allSegmentDuration > (duration / 100) * 80) {
|
||||
// Reject submission if all segments combine are over 80% of the video
|
||||
return "Total length of your submitted segments are over 80% of the video.";
|
||||
}
|
||||
}
|
||||
|
||||
// Check NeuralBlock
|
||||
const neuralBlockURL = config.neuralBlockURL;
|
||||
if (!neuralBlockURL) return false;
|
||||
const response = await axios.get(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID}
|
||||
&segments=${nbString.substring(0, nbString.length - 1)}`, { validateStatus: () => true });
|
||||
if (response.status !== 200) return false;
|
||||
|
||||
const nbPredictions = response.data;
|
||||
let nbDecision = false;
|
||||
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
if (segments[i].category === "sponsor") {
|
||||
if (nbPredictions.probabilities[predictionIdx] < 0.70) {
|
||||
nbDecision = true; // At least one bad entry
|
||||
const startTime = parseFloat(segments[i].segment[0]);
|
||||
const endTime = parseFloat(segments[i].segment[1]);
|
||||
|
||||
const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime, submission.service);
|
||||
// Send to Discord
|
||||
// Note, if this is too spammy. Consider sending all the segments as one Webhook
|
||||
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
|
||||
}
|
||||
predictionIdx++;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (nbDecision) {
|
||||
return "Rejected based on NeuralBlock predictions.";
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
Logger.debug("Skipped YouTube API");
|
||||
|
||||
// Can't moderate the submission without calling the youtube API
|
||||
// so allow by default.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
|
||||
if (config.newLeafURLs !== null) {
|
||||
@@ -310,7 +210,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
|
||||
invalidFields.push("userID");
|
||||
if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`);
|
||||
}
|
||||
if (!Array.isArray(segments) || segments.length < 1) {
|
||||
if (!Array.isArray(segments) || segments.length == 0) {
|
||||
invalidFields.push("segments");
|
||||
}
|
||||
// validate start and end times (no : marks)
|
||||
@@ -323,7 +223,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
|
||||
}
|
||||
|
||||
if (typeof segmentPair.description !== "string"
|
||||
|| (segmentPair.description.length > 60 && segmentPair.actionType === ActionType.Chapter)
|
||||
|| (segmentPair.actionType === ActionType.Chapter && segmentPair.description.length > 60 )
|
||||
|| (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) {
|
||||
invalidFields.push("segment description");
|
||||
}
|
||||
@@ -402,7 +302,7 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
|
||||
}
|
||||
|
||||
if (!isVIP && segments[i].category === "sponsor"
|
||||
&& segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) {
|
||||
&& segments[i].actionType !== ActionType.Full && (endTime - startTime) < 1) {
|
||||
// Too short
|
||||
return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 };
|
||||
}
|
||||
@@ -425,32 +325,19 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
|
||||
return CHECK_PASS;
|
||||
}
|
||||
|
||||
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise<CheckResult & { decreaseVotes: number; } > {
|
||||
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise<CheckResult> {
|
||||
// Auto moderator check
|
||||
if (!isVIP && service == Service.YouTube) {
|
||||
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service });//startTime, endTime, category: segments[i].category});
|
||||
|
||||
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
|
||||
// If NB automod rejects, the submission will start with -2 votes.
|
||||
// Note, if one submission is bad all submissions will be affected.
|
||||
// However, this behavior is consistent with other automod functions
|
||||
// already in place.
|
||||
//decreaseVotes = -2; //Disable for now
|
||||
} else if (autoModerateResult) {
|
||||
//Normal automod behavior
|
||||
if (service == Service.YouTube) {
|
||||
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration });
|
||||
if (autoModerateResult) {
|
||||
return {
|
||||
pass: false,
|
||||
errorCode: 403,
|
||||
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`,
|
||||
decreaseVotes
|
||||
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...CHECK_PASS,
|
||||
decreaseVotes
|
||||
};
|
||||
return CHECK_PASS;
|
||||
}
|
||||
|
||||
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
|
||||
@@ -601,11 +488,12 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
|
||||
|
||||
const userWarningCheckResult = await checkUserActiveWarning(userID);
|
||||
if (!userWarningCheckResult.pass) {
|
||||
Logger.warn(`Caught a submission for for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);
|
||||
Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);
|
||||
return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage);
|
||||
}
|
||||
|
||||
const isVIP = await isUserVIP(userID);
|
||||
const isTempVIP = await isUserTempVIP(userID, videoID);
|
||||
const rawIP = getIP(req);
|
||||
|
||||
const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam);
|
||||
@@ -618,13 +506,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
|
||||
return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage);
|
||||
}
|
||||
|
||||
let decreaseVotes = 0;
|
||||
// Auto check by NB
|
||||
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, isVIP, service, apiVideoInfo, decreaseVotes);
|
||||
if (!isVIP && !isTempVIP) {
|
||||
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam);
|
||||
if (!autoModerateCheckResult.pass) {
|
||||
return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage);
|
||||
} else {
|
||||
decreaseVotes = autoModerateCheckResult.decreaseVotes;
|
||||
}
|
||||
}
|
||||
|
||||
// Will be filled when submitting
|
||||
@@ -645,7 +531,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
|
||||
|
||||
//check to see if this user is shadowbanned
|
||||
const shadowBanRow = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID]);
|
||||
const startingVotes = 0 + decreaseVotes;
|
||||
const startingVotes = 0;
|
||||
const reputation = await getReputation(userID);
|
||||
|
||||
for (const segmentInfo of segments) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Request, Response } from "express";
|
||||
import { Logger } from "../utils/logger";
|
||||
import { isUserVIP } from "../utils/isUserVIP";
|
||||
import { isUserTempVIP } from "../utils/isUserTempVIP";
|
||||
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
|
||||
import { APIVideoInfo } from "../types/youtubeApi.model";
|
||||
import { db, privateDB } from "../databases/databases";
|
||||
@@ -9,12 +10,10 @@ import { getFormattedTime } from "../utils/getFormattedTime";
|
||||
import { getIP } from "../utils/getIP";
|
||||
import { getHashCache } from "../utils/getHashCache";
|
||||
import { config } from "../config";
|
||||
import { HashedUserID, UserID } from "../types/user.model";
|
||||
import { UserID } from "../types/user.model";
|
||||
import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType } from "../types/segments.model";
|
||||
import { QueryCacher } from "../utils/queryCacher";
|
||||
import axios from "axios";
|
||||
import redis from "../utils/redis";
|
||||
import { tempVIPKey } from "../utils/redisKeys";
|
||||
|
||||
const voteTypes = {
|
||||
normal: 0,
|
||||
@@ -44,6 +43,7 @@ interface VoteData {
|
||||
row: {
|
||||
votes: number;
|
||||
views: number;
|
||||
locked: boolean;
|
||||
};
|
||||
category: string;
|
||||
incrementAmount: number;
|
||||
@@ -55,14 +55,6 @@ function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<API
|
||||
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
|
||||
}
|
||||
|
||||
const isUserTempVIP = async (nonAnonUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
|
||||
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
|
||||
const channelID = apiVideoInfo?.data?.authorId;
|
||||
const { err, reply } = await redis.getAsync(tempVIPKey(nonAnonUserID));
|
||||
|
||||
return err || !reply ? false : (reply == channelID);
|
||||
};
|
||||
|
||||
const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2);
|
||||
|
||||
async function updateSegmentVideoDuration(UUID: SegmentUUID) {
|
||||
@@ -178,7 +170,7 @@ async function sendWebhooks(voteData: VoteData) {
|
||||
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}s#requiredSegment=${voteData.UUID}`,
|
||||
"description": `**${voteData.row.votes} Votes Prior | \
|
||||
${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \
|
||||
Views**\n\n**Submission ID:** ${voteData.UUID}\
|
||||
Views**\n\n**Locked**: ${voteData.row.locked}\n\n**Submission ID:** ${voteData.UUID}\
|
||||
\n**Category:** ${submissionInfoRow.category}\
|
||||
\n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\
|
||||
\n\n**Total User Submissions:** ${submissionInfoRow.count}\
|
||||
@@ -189,7 +181,7 @@ async function sendWebhooks(voteData: VoteData) {
|
||||
"author": {
|
||||
"name": voteData.finalResponse?.webhookMessage ??
|
||||
voteData.finalResponse?.finalMessage ??
|
||||
getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission),
|
||||
`${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`,
|
||||
},
|
||||
"thumbnail": {
|
||||
"url": getMaxResThumbnail(data) || "",
|
||||
@@ -476,11 +468,11 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
|
||||
|
||||
// Only change the database if they have made a submission before and haven't voted recently
|
||||
const userAbleToVote = (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0)
|
||||
&& !finalResponse.blockVote
|
||||
&& finalResponse.finalStatus === 200
|
||||
&& (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
|
||||
&& (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
|
||||
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
|
||||
&& !finalResponse.blockVote
|
||||
&& finalResponse.finalStatus === 200;
|
||||
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined);
|
||||
|
||||
|
||||
const ableToVote = isVIP || isTempVIP || userAbleToVote;
|
||||
|
||||
@@ -2,6 +2,14 @@ import { PoolConfig } from "pg";
|
||||
import * as redis from "redis";
|
||||
import { CacheOptions } from "@ajayyy/lru-diskcache";
|
||||
|
||||
interface RedisConfig extends redis.RedisClientOptions {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
interface CustomPostgresConfig extends PoolConfig {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface SBSConfig {
|
||||
[index: string]: any
|
||||
port: number;
|
||||
@@ -41,9 +49,9 @@ export interface SBSConfig {
|
||||
privateMysql?: any;
|
||||
minimumPrefix?: string;
|
||||
maximumPrefix?: string;
|
||||
redis?: redis.ClientOpts;
|
||||
redis?: RedisConfig;
|
||||
maxRewardTimePerSegmentInSeconds?: number;
|
||||
postgres?: PoolConfig;
|
||||
postgres?: CustomPostgresConfig;
|
||||
dumpDatabase?: DumpDatabase;
|
||||
diskCache: CacheOptions;
|
||||
crons: CronJobOptions;
|
||||
@@ -76,7 +84,6 @@ export interface DumpDatabase {
|
||||
enabled: boolean;
|
||||
minTimeBetweenMs: number;
|
||||
appExportPath: string;
|
||||
postgresExportPath: string;
|
||||
tables: DumpDatabaseTable[];
|
||||
}
|
||||
|
||||
|
||||
@@ -18,18 +18,19 @@ export async function getHashCache<T extends string>(value: T, times = defaulted
|
||||
|
||||
async function getFromRedis<T extends string>(key: HashedValue): Promise<T & HashedValue> {
|
||||
const redisKey = shaHashKey(key);
|
||||
const { err, reply } = await redis.getAsync(redisKey);
|
||||
|
||||
if (!err && reply) {
|
||||
try {
|
||||
const reply = await redis.get(redisKey);
|
||||
|
||||
if (reply) {
|
||||
Logger.debug(`Got data from redis: ${reply}`);
|
||||
return reply as T & HashedValue;
|
||||
} catch (e) {
|
||||
// If all else, continue on hashing
|
||||
}
|
||||
}
|
||||
const data = getHash(key, cachedHashTimes);
|
||||
} catch (e) {} // eslint-disable-line no-empty
|
||||
|
||||
// Otherwise, calculate it
|
||||
const data = getHash(key, cachedHashTimes);
|
||||
redis.set(key, data);
|
||||
|
||||
redis.setAsync(key, data);
|
||||
return data as T & HashedValue;
|
||||
}
|
||||
24
src/utils/isUserTempVIP.ts
Normal file
24
src/utils/isUserTempVIP.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import redis from "../utils/redis";
|
||||
import { tempVIPKey } from "../utils/redisKeys";
|
||||
import { HashedUserID } from "../types/user.model";
|
||||
import { YouTubeAPI } from "../utils/youtubeApi";
|
||||
import { APIVideoInfo } from "../types/youtubeApi.model";
|
||||
import { VideoID } from "../types/segments.model";
|
||||
import { config } from "../config";
|
||||
import { Logger } from "./logger";
|
||||
|
||||
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
|
||||
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
|
||||
}
|
||||
|
||||
export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
|
||||
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
|
||||
const channelID = apiVideoInfo?.data?.authorId;
|
||||
try {
|
||||
const reply = await redis.get(tempVIPKey(hashedUserID));
|
||||
return reply && reply == channelID;
|
||||
} catch (e) {
|
||||
Logger.error(e as string);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
@@ -5,21 +5,18 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
|
||||
import { UserID } from "../types/user.model";
|
||||
|
||||
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
|
||||
const { err, reply } = await redis.getAsync(key);
|
||||
|
||||
if (!err && reply) {
|
||||
try {
|
||||
const reply = await redis.get(key);
|
||||
if (reply) {
|
||||
Logger.debug(`Got data from redis: ${reply}`);
|
||||
|
||||
return JSON.parse(reply);
|
||||
} catch (e) {
|
||||
// If all else, continue on to fetching from the database
|
||||
}
|
||||
}
|
||||
} catch (e) { } //eslint-disable-line no-empty
|
||||
|
||||
const data = await fetchFromDB();
|
||||
|
||||
redis.setAsync(key, JSON.stringify(data));
|
||||
redis.set(key, JSON.stringify(data));
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -30,18 +27,17 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
|
||||
async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Promise<Array<T>>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise<Array<T>> {
|
||||
const cachedValues = await Promise.all(values.map(async (value) => {
|
||||
const key = keyGenerator(value);
|
||||
const { err, reply } = await redis.getAsync(key);
|
||||
|
||||
if (!err && reply) {
|
||||
try {
|
||||
const reply = await redis.get(key);
|
||||
if (reply) {
|
||||
Logger.debug(`Got data from redis: ${reply}`);
|
||||
|
||||
return {
|
||||
value,
|
||||
result: JSON.parse(reply)
|
||||
};
|
||||
} catch (e) { } //eslint-disable-line no-empty
|
||||
}
|
||||
} catch (e) { } //eslint-disable-line no-empty
|
||||
|
||||
return {
|
||||
value,
|
||||
@@ -71,7 +67,7 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
|
||||
}
|
||||
|
||||
for (const key in newResults) {
|
||||
redis.setAsync(key, JSON.stringify(newResults[key]));
|
||||
redis.set(key, JSON.stringify(newResults[key]));
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -81,16 +77,16 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
|
||||
|
||||
function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void {
|
||||
if (videoInfo) {
|
||||
redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
|
||||
redis.delAsync(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
|
||||
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
|
||||
if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID));
|
||||
redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
|
||||
redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
|
||||
redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
|
||||
if (videoInfo.userID) redis.del(reputationKey(videoInfo.userID));
|
||||
}
|
||||
}
|
||||
|
||||
function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
|
||||
if (videoInfo) {
|
||||
redis.delAsync(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
|
||||
redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,59 +1,56 @@
|
||||
import { config } from "../config";
|
||||
import { Logger } from "./logger";
|
||||
import redis, { Callback } from "redis";
|
||||
import { createClient } from "redis";
|
||||
import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@node-redis/client/dist/lib/commands";
|
||||
import { ClientCommandOptions } from "@node-redis/client/dist/lib/client";
|
||||
import { RedisReply } from "rate-limit-redis";
|
||||
|
||||
interface RedisSB {
|
||||
get(key: string, callback?: Callback<string | null>): void;
|
||||
getAsync?(key: string): Promise<{err: Error | null, reply: string | null}>;
|
||||
set(key: string, value: string, callback?: Callback<string | null>): void;
|
||||
setAsync?(key: string, value: string): Promise<{err: Error | null, reply: string | null}>;
|
||||
setAsyncEx?(key: string, value: string, seconds: number): Promise<{err: Error | null, reply: string | null}>;
|
||||
delAsync?(...keys: [string]): Promise<Error | null>;
|
||||
close?(flush?: boolean): void;
|
||||
increment?(key: string): Promise<{err: Error| null, replies: any[] | null}>;
|
||||
get(key: RedisCommandArgument): Promise<string>;
|
||||
set(key: RedisCommandArgument, value: RedisCommandArgument): Promise<string>;
|
||||
setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
|
||||
del(...keys: [RedisCommandArgument]): Promise<number>;
|
||||
increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
|
||||
sendCommand(args: RedisCommandArguments, options?: ClientCommandOptions): Promise<RedisReply>;
|
||||
quit(): Promise<void>;
|
||||
}
|
||||
|
||||
let exportObject: RedisSB = {
|
||||
get: (key, callback?) => callback(null, undefined),
|
||||
getAsync: () =>
|
||||
new Promise((resolve) => resolve({ err: null, reply: undefined })),
|
||||
set: (key, value, callback) => callback(null, undefined),
|
||||
setAsync: () =>
|
||||
new Promise((resolve) => resolve({ err: null, reply: undefined })),
|
||||
setAsyncEx: () =>
|
||||
new Promise((resolve) => resolve({ err: null, reply: undefined })),
|
||||
delAsync: () =>
|
||||
new Promise((resolve) => resolve(null)),
|
||||
increment: () =>
|
||||
new Promise((resolve) => resolve({ err: null, replies: undefined })),
|
||||
let exportClient: RedisSB = {
|
||||
get: () => new Promise((resolve) => resolve(null)),
|
||||
set: () => new Promise((resolve) => resolve(null)),
|
||||
setEx: () => new Promise((resolve) => resolve(null)),
|
||||
del: () => new Promise((resolve) => resolve(null)),
|
||||
increment: () => new Promise((resolve) => resolve(null)),
|
||||
sendCommand: () => new Promise((resolve) => resolve(null)),
|
||||
quit: () => new Promise((resolve) => resolve(null)),
|
||||
};
|
||||
|
||||
if (config.redis) {
|
||||
if (config.redis?.enabled) {
|
||||
Logger.info("Connected to redis");
|
||||
const client = redis.createClient(config.redis);
|
||||
exportObject = client;
|
||||
const client = createClient(config.redis);
|
||||
client.connect();
|
||||
exportClient = client;
|
||||
|
||||
const timeoutDuration = 200;
|
||||
exportObject.getAsync = (key) => new Promise((resolve) => {
|
||||
const timeout = setTimeout(() => resolve({ err: null, reply: undefined }), timeoutDuration);
|
||||
client.get(key, (err, reply) => {
|
||||
const get = client.get.bind(client);
|
||||
exportClient.get = (key) => new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => reject(), timeoutDuration);
|
||||
get(key).then((reply) => {
|
||||
clearTimeout(timeout);
|
||||
resolve({ err, reply });
|
||||
resolve(reply);
|
||||
}).catch((err) => reject(err));
|
||||
});
|
||||
});
|
||||
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({ err, reply })));
|
||||
exportObject.setAsyncEx = (key, value, seconds) => new Promise((resolve) => client.setex(key, seconds, value, (err, reply) => resolve({ err, reply })));
|
||||
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
|
||||
exportObject.close = (flush) => client.end(flush);
|
||||
exportObject.increment = (key) => new Promise((resolve) =>
|
||||
exportClient.increment = (key) => new Promise((resolve, reject) =>
|
||||
client.multi()
|
||||
.incr(key)
|
||||
.expire(key, 60)
|
||||
.exec((err, replies) => resolve({ err, replies }))
|
||||
.exec()
|
||||
.then((reply) => resolve(reply))
|
||||
.catch((err) => reject(err))
|
||||
);
|
||||
client.on("error", function(error) {
|
||||
Logger.error(error);
|
||||
});
|
||||
}
|
||||
|
||||
export default exportObject;
|
||||
export default exportClient;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"schemaFolder": "./databases",
|
||||
"dbSchema": "./databases/_sponsorTimes.db.sql",
|
||||
"privateDBSchema": "./databases/_private.db.sql",
|
||||
"categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
|
||||
"mode": "test",
|
||||
"readOnly": false,
|
||||
"webhooks": [
|
||||
|
||||
@@ -89,7 +89,7 @@ describe("getStatus", () => {
|
||||
});
|
||||
|
||||
it("Should be able to get statusRequests only", function (done) {
|
||||
if (!config.redis) this.skip();
|
||||
if (!config.redis?.enabled) this.skip();
|
||||
client.get(`${endpoint}/statusRequests`)
|
||||
.then(res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
@@ -100,7 +100,7 @@ describe("getStatus", () => {
|
||||
});
|
||||
|
||||
it("Should be able to get status with statusRequests", function (done) {
|
||||
if (!config.redis) this.skip();
|
||||
if (!config.redis?.enabled) this.skip();
|
||||
client.get(endpoint)
|
||||
.then(res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
|
||||
@@ -734,22 +734,6 @@ describe("postSkipSegments", () => {
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be rejected if NB's predicted probability is <70%.", (done) => {
|
||||
const videoID = "LevkAjUE6d4";
|
||||
postSkipSegmentParam({
|
||||
videoID,
|
||||
startTime: 40,
|
||||
endTime: 60,
|
||||
userID: submitUserTwo,
|
||||
category: "sponsor"
|
||||
})
|
||||
.then(res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
done();
|
||||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be rejected with custom message if user has to many active warnings", (done) => {
|
||||
postSkipSegmentJSON({
|
||||
userID: warnUser01,
|
||||
|
||||
@@ -11,22 +11,21 @@ const randKey2 = genRandom(16);
|
||||
|
||||
describe("redis test", function() {
|
||||
before(async function() {
|
||||
if (!config.redis) this.skip();
|
||||
await redis.setAsync(randKey1, randValue1);
|
||||
if (!config.redis?.enabled) this.skip();
|
||||
await redis.set(randKey1, randValue1);
|
||||
});
|
||||
it("Should get stored value", (done) => {
|
||||
redis.getAsync(randKey1)
|
||||
redis.get(randKey1)
|
||||
.then(res => {
|
||||
if (res.err) assert.fail(res.err);
|
||||
assert.strictEqual(res.reply, randValue1);
|
||||
assert.strictEqual(res, randValue1);
|
||||
done();
|
||||
});
|
||||
}).catch(err => assert.fail(err));
|
||||
});
|
||||
it("Should not be able to get not stored value", (done) => {
|
||||
redis.getAsync(randKey2)
|
||||
redis.get(randKey2)
|
||||
.then(res => {
|
||||
if (res.reply || res.err ) assert.fail("Value should not be found");
|
||||
if (res) assert.fail("Value should not be found");
|
||||
done();
|
||||
});
|
||||
}).catch(err => assert.fail(err));
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import { client } from "../utils/httpClient";
|
||||
import { db, privateDB } from "../../src/databases/databases";
|
||||
import redis from "../../src/utils/redis";
|
||||
import assert from "assert";
|
||||
import { Logger } from "../../src/utils/logger";
|
||||
|
||||
// helpers
|
||||
const getSegment = (UUID: string) => db.prepare("get", `SELECT "votes", "locked", "category" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
|
||||
@@ -51,13 +52,18 @@ const postVoteCategory = (userID: string, UUID: string, category: string) => cli
|
||||
}
|
||||
});
|
||||
const checkUserVIP = async (publicID: HashedUserID) => {
|
||||
const { reply } = await redis.getAsync(tempVIPKey(publicID));
|
||||
try {
|
||||
const reply = await redis.get(tempVIPKey(publicID));
|
||||
return reply;
|
||||
} catch (e) {
|
||||
Logger.error(e as string);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
describe("tempVIP test", function() {
|
||||
before(async function() {
|
||||
if (!config.redis) this.skip();
|
||||
if (!config.redis?.enabled) this.skip();
|
||||
|
||||
const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
|
||||
await db.prepare("run", insertSponsorTimeQuery, ["channelid-convert", 0, 1, 0, 0, UUID0, "testman", 0, 50, "sponsor", 0]);
|
||||
@@ -67,7 +73,7 @@ describe("tempVIP test", function() {
|
||||
await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP1]);
|
||||
await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP2]);
|
||||
// clear redis if running consecutive tests
|
||||
await redis.delAsync(tempVIPKey(publicTempVIPOne));
|
||||
await redis.del(tempVIPKey(publicTempVIPOne));
|
||||
});
|
||||
|
||||
it("Should update db version when starting the application", () => {
|
||||
|
||||
@@ -57,7 +57,7 @@ async function init() {
|
||||
mocha.run((failures) => {
|
||||
mockServer.close();
|
||||
server.close();
|
||||
redis.close(true);
|
||||
redis.quit();
|
||||
process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user