diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..024df87 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,46 @@ +# Based on https://github.com/ajayyy/sb-mirror/blob/main/.github/workflows/docker-build.yml +name: multi-build-docker +on: + workflow_call: + inputs: + name: + required: true + type: string + username: + required: true + type: string + folder: + required: true + type: string + secrets: + GH_TOKEN: + required: true + +jobs: + build_container: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ inputs.username }}/${{ inputs.name }} + tags: | + type-raw,value=alpine + flavor: | + latest=true + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GH_TOKEN }} + - name: push + uses: docker/build-push-action@v3 + with: + context: ${{ inputs.folder }} + push: true + tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/.github/workflows/sb-server.yml b/.github/workflows/sb-server.yml new file mode 100644 index 0000000..1e46555 --- /dev/null +++ b/.github/workflows/sb-server.yml @@ -0,0 +1,25 @@ +name: Docker image builds +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + sb-server: + uses: ./.github/workflows/docker-build.yml + with: + name: "sb-server" + username: "ajayyy" + folder: "." + secrets: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + rsync-host: + needs: sb-server + uses: ./.github/workflows/docker-build.yml + with: + name: "rsync-host" + username: "ajayyy" + folder: "./rsync" + secrets: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c39d92e..26b77cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,16 @@ -FROM node:14-alpine as builder -RUN apk add --no-cache --virtual .build-deps python make g++ +FROM node:16-alpine as builder +RUN apk add --no-cache --virtual .build-deps python3 make g++ COPY package.json package-lock.json tsconfig.json entrypoint.sh ./ COPY src src RUN npm ci && npm run tsc -FROM node:14-alpine as app +FROM node:16-alpine as app WORKDIR /usr/src/app -COPY --from=builder node_modules . -COPY --from=builder dist ./dist +RUN apk add git postgresql-client +COPY --from=builder ./node_modules ./node_modules +COPY --from=builder ./dist ./dist +COPY ./.git ./.git COPY entrypoint.sh . COPY databases/*.sql databases/ EXPOSE 8080 -CMD ./entrypoint.sh +CMD ./entrypoint.sh \ No newline at end of file diff --git a/ci.json b/ci.json index 68262cb..18fb97c 100644 --- a/ci.json +++ b/ci.json @@ -17,13 +17,16 @@ "port": 5432 }, "redis": { - "host": "localhost", - "port": 6379 + "socket": { + "host": "localhost", + "port": 6379 + } }, "createDatabaseIfNotExist": true, "schemaFolder": "./databases", "dbSchema": "./databases/_sponsorTimes.db.sql", "privateDBSchema": "./databases/_private.db.sql", + "categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], "mode": "test", "readOnly": false, "webhooks": [ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e5e10e0..7b6e1b5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,7 +7,6 @@ services: - database.env volumes: - database-data:/var/lib/postgresql/data - - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports ports: - 5432:5432 restart: always diff --git a/entrypoint.sh b/entrypoint.sh index 9055d2c..9b73c6b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,25 +2,11 @@ set -e echo 'Entrypoint script' cd /usr/src/app -cp /etc/sponsorblock/config.json . || cat < config.json + +# blank config, use defaults +cat < config.json { - "port": 8080, - "globalSalt": "[CHANGE THIS]", - "adminUserID": "[CHANGE THIS]", - "youtubeAPIKey": null, - "discordReportChannelWebhookURL": null, - "discordFirstTimeSubmissionsWebhookURL": null, - "discordAutoModWebhookURL": null, - "proxySubmission": null, - "behindProxy": "X-Forwarded-For", - "db": "./databases/sponsorTimes.db", - "privateDB": "./databases/private.db", - "createDatabaseIfNotExist": true, - "schemaFolder": "./databases", - "dbSchema": "./databases/_sponsorTimes.db.sql", - "privateDBSchema": "./databases/_private.db.sql", - "mode": "development", - "readOnly": false } EOF -node dist/index.js + +node dist/src/index.js \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 69526ec..38447d6 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -49,7 +49,7 @@ http { #server localhost:4446; #server localhost:4447; #server localhost:4448; - server 10.0.0.4:4441 max_fails=25 fail_timeout=20s; + #server 10.0.0.4:4441 max_fails=25 fail_timeout=20s; #server 10.0.0.3:4441 max_fails=25 fail_timeout=20s; #server 10.0.0.3:4442 max_fails=25 fail_timeout=20s; @@ -72,8 +72,17 @@ http { server 10.0.0.13:4441 max_fails=25 fail_timeout=20s; server 10.0.0.13:4442 max_fails=25 fail_timeout=20s; + server 10.0.0.14:4441 max_fails=25 fail_timeout=20s; + server 10.0.0.14:4442 max_fails=25 fail_timeout=20s; + server 10.0.0.11:4441 max_fails=25 fail_timeout=20s; server 10.0.0.11:4442 max_fails=25 fail_timeout=20s; + + server 10.0.0.16:4441 max_fails=25 fail_timeout=20s; + server 10.0.0.16:4442 max_fails=25 fail_timeout=20s; + + server 10.0.0.17:4441 max_fails=25 fail_timeout=20s; + server 10.0.0.17:4442 max_fails=25 fail_timeout=20s; #server 134.209.69.251:80 backup; @@ -84,6 +93,7 @@ http { #server localhost:4441; #server localhost:4442; server 10.0.0.3:4441 max_fails=25 fail_timeout=15s; + server 10.0.0.4:4441 max_fails=25 fail_timeout=15s; #server 10.0.0.3:4442; } upstream backend_db { diff --git a/package-lock.json b/package-lock.json index e48a32b..ebf1447 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,22 +15,21 @@ "cron": "^1.8.2", "express": "^4.17.1", "express-promise-router": "^4.1.1", - "express-rate-limit": "^5.5.1", + "express-rate-limit": "^6.3.0", "lodash": "^4.17.21", "pg": "^8.7.1", - "redis": "^3.1.2", + "rate-limit-redis": "^3.0.1", + "redis": "^4.0.6", "sync-mysql": "^3.0.1" }, "devDependencies": { "@types/better-sqlite3": "^7.4.1", "@types/cron": "^1.7.3", "@types/express": "^4.17.13", - "@types/express-rate-limit": "^5.1.3", "@types/lodash": "^4.14.178", "@types/mocha": "^9.0.0", "@types/node": "^16.11.11", "@types/pg": "^8.6.1", - "@types/redis": "^2.8.32", "@typescript-eslint/eslint-plugin": "^5.5.0", "@typescript-eslint/parser": "^5.5.0", "eslint": "^8.3.0", @@ -128,6 +127,65 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@node-redis/bloom": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz", + "integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==", + "peerDependencies": { + "@node-redis/client": "^1.0.0" + } + }, + "node_modules/@node-redis/client": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz", + "integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==", + "dependencies": { + "cluster-key-slot": "1.1.0", + "generic-pool": "3.8.2", + "redis-parser": "3.0.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@node-redis/client/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/@node-redis/graph": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz", + "integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==", + "peerDependencies": { + "@node-redis/client": "^1.0.0" + } + }, + "node_modules/@node-redis/json": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz", + "integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==", + "peerDependencies": { + "@node-redis/client": "^1.0.0" + } + }, + "node_modules/@node-redis/search": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz", + "integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==", + "peerDependencies": { + "@node-redis/client": "^1.0.0" + } + }, + "node_modules/@node-redis/time-series": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz", + "integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==", + "peerDependencies": { + "@node-redis/client": "^1.0.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -293,15 +351,6 @@ "@types/serve-static": "*" } }, - "node_modules/@types/express-rate-limit": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/@types/express-rate-limit/-/express-rate-limit-5.1.3.tgz", - "integrity": "sha512-H+TYy3K53uPU2TqPGFYaiWc2xJV6+bIFkDd/Ma2/h67Pa6ARk9kWE0p/K9OH1Okm0et9Sfm66fmXoAxsH2PHXg==", - "dev": true, - "dependencies": { - "@types/express": "*" - } - }, "node_modules/@types/express-serve-static-core": { "version": "4.17.26", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz", @@ -366,15 +415,6 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "devOptional": true }, - "node_modules/@types/redis": { - "version": "2.8.32", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz", - "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/serve-static": { "version": "1.13.10", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", @@ -1214,6 +1254,14 @@ "node": ">=4" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -1444,14 +1492,6 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, - "node_modules/denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", - "engines": { - "node": ">=0.10" - } - }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1889,9 +1929,15 @@ } }, "node_modules/express-rate-limit": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz", - "integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg==" + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.3.0.tgz", + "integrity": "sha512-932Io1VGKjM3ppi7xW9sb1J5nVkEJSUiOtHw2oE+JyHks1e+AXuOBSXbJKM0mcXwEnW1TibJibQ455Ow1YFjfg==", + "engines": { + "node": ">= 12.9.0" + }, + "peerDependencies": { + "express": "^4" + } }, "node_modules/express/node_modules/debug": { "version": "2.6.9", @@ -2205,6 +2251,14 @@ "node": ">=0.10.0" } }, + "node_modules/generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -3140,9 +3194,9 @@ } }, "node_modules/moment": { - "version": "2.29.1", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", - "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", + "version": "2.29.2", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", + "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==", "engines": { "node": "*" } @@ -3876,6 +3930,17 @@ "node": ">= 0.6" } }, + "node_modules/rate-limit-redis": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz", + "integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==", + "engines": { + "node": ">= 14.5.0" + }, + "peerDependencies": { + "express-rate-limit": "^6" + } + }, "node_modules/raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", @@ -3931,28 +3996,18 @@ } }, "node_modules/redis": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", - "integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz", + "integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==", "dependencies": { - "denque": "^1.5.0", - "redis-commands": "^1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-redis" + "@node-redis/bloom": "1.0.1", + "@node-redis/client": "1.0.5", + "@node-redis/graph": "1.0.0", + "@node-redis/json": "1.0.2", + "@node-redis/search": "1.0.5", + "@node-redis/time-series": "1.0.2" } }, - "node_modules/redis-commands": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", - "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" - }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -5169,6 +5224,54 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@node-redis/bloom": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz", + "integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==", + "requires": {} + }, + "@node-redis/client": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz", + "integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==", + "requires": { + "cluster-key-slot": "1.1.0", + "generic-pool": "3.8.2", + "redis-parser": "3.0.0", + "yallist": "4.0.0" + }, + "dependencies": { + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "@node-redis/graph": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz", + "integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==", + "requires": {} + }, + "@node-redis/json": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz", + "integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==", + "requires": {} + }, + "@node-redis/search": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz", + "integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==", + "requires": {} + }, + "@node-redis/time-series": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz", + "integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==", + "requires": {} + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5319,15 +5422,6 @@ "@types/serve-static": "*" } }, - "@types/express-rate-limit": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/@types/express-rate-limit/-/express-rate-limit-5.1.3.tgz", - "integrity": "sha512-H+TYy3K53uPU2TqPGFYaiWc2xJV6+bIFkDd/Ma2/h67Pa6ARk9kWE0p/K9OH1Okm0et9Sfm66fmXoAxsH2PHXg==", - "dev": true, - "requires": { - "@types/express": "*" - } - }, "@types/express-serve-static-core": { "version": "4.17.26", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz", @@ -5392,15 +5486,6 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "devOptional": true }, - "@types/redis": { - "version": "2.8.32", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz", - "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/serve-static": { "version": "1.13.10", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", @@ -6029,6 +6114,11 @@ } } }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -6209,11 +6299,6 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, - "denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -6557,9 +6642,10 @@ } }, "express-rate-limit": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz", - "integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg==" + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.3.0.tgz", + "integrity": "sha512-932Io1VGKjM3ppi7xW9sb1J5nVkEJSUiOtHw2oE+JyHks1e+AXuOBSXbJKM0mcXwEnW1TibJibQ455Ow1YFjfg==", + "requires": {} }, "fast-deep-equal": { "version": "3.1.3", @@ -6802,6 +6888,11 @@ } } }, + "generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -7502,9 +7593,9 @@ } }, "moment": { - "version": "2.29.1", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", - "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + "version": "2.29.2", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", + "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==" }, "moment-timezone": { "version": "0.5.34", @@ -8051,6 +8142,12 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limit-redis": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-3.0.1.tgz", + "integrity": "sha512-L6yhOUBrAZ8VEMX9DwlM3X6hfm8yq+gBO4LoOW7+JgmNq59zE7QmLz4v5VnwYPvLeSh/e7PDcrzUI3UumJw1iw==", + "requires": {} + }, "raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", @@ -8094,21 +8191,18 @@ } }, "redis": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", - "integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz", + "integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==", "requires": { - "denque": "^1.5.0", - "redis-commands": "^1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0" + "@node-redis/bloom": "1.0.1", + "@node-redis/client": "1.0.5", + "@node-redis/graph": "1.0.0", + "@node-redis/json": "1.0.2", + "@node-redis/search": "1.0.5", + "@node-redis/time-series": "1.0.2" } }, - "redis-commands": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", - "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" - }, "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", diff --git a/package.json b/package.json index efee6ac..4ddd8dc 100644 --- a/package.json +++ b/package.json @@ -23,22 +23,21 @@ "cron": "^1.8.2", "express": "^4.17.1", "express-promise-router": "^4.1.1", - "express-rate-limit": "^5.5.1", + "express-rate-limit": "^6.3.0", "lodash": "^4.17.21", "pg": "^8.7.1", - "redis": "^3.1.2", + "rate-limit-redis": "^3.0.1", + "redis": "^4.0.6", "sync-mysql": "^3.0.1" }, "devDependencies": { "@types/better-sqlite3": "^7.4.1", "@types/cron": "^1.7.3", "@types/express": "^4.17.13", - "@types/express-rate-limit": "^5.1.3", "@types/lodash": "^4.14.178", "@types/mocha": "^9.0.0", "@types/node": "^16.11.11", "@types/pg": "^8.6.1", - "@types/redis": "^2.8.32", "@typescript-eslint/eslint-plugin": "^5.5.0", "@typescript-eslint/parser": "^5.5.0", "eslint": "^8.3.0", diff --git a/rsync/Dockerfile b/rsync/Dockerfile new file mode 100644 index 0000000..c5093da --- /dev/null +++ b/rsync/Dockerfile @@ -0,0 +1,6 @@ +FROM ghcr.io/ajayyy/sb-server:latest +EXPOSE 873/tcp +RUN apk add rsync>3.2.4-r0 +RUN mkdir /usr/src/app/database-export + +CMD rsync --no-detach --daemon & ./entrypoint.sh \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index c922cb4..2f529c1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -27,7 +27,7 @@ import { loggerMiddleware } from "./middleware/logger"; import { corsMiddleware } from "./middleware/cors"; import { apiCspMiddleware } from "./middleware/apiCsp"; import { rateLimitMiddleware } from "./middleware/requestRateLimit"; -import dumpDatabase, { appExportPath, redirectLink } from "./routes/dumpDatabase"; +import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase"; import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo"; import { postClearCache } from "./routes/postClearCache"; import { addUnlistedVideo } from "./routes/addUnlistedVideo"; @@ -208,14 +208,14 @@ function setupRoutes(router: Router) { router.get("/api/videoLabels", getVideoLabels); router.get("/api/videoLabels/:prefix", getVideoLabelsByHash); - if (config.postgres) { + if (config.postgres?.enabled) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); router.get("/database.json", (req, res) => dumpDatabase(req, res, false)); - router.get("/database/*", redirectLink); + router.get("/database/*", downloadFile); router.use("/download", express.static(appExportPath)); } else { router.get("/database.db", function (req: Request, res: Response) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); }); } -} +} \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index f4e6175..acd473e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,7 @@ import fs from "fs"; import { SBSConfig } from "./types/config.model"; import packageJson from "../package.json"; +import { isBoolean, isNumber } from "lodash"; const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test; const configFile = process.env.TEST_POSTGRES ? "ci.json" @@ -9,7 +10,7 @@ const configFile = process.env.TEST_POSTGRES ? "ci.json" export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8")); addDefaults(config, { - port: 80, + port: 8080, behindProxy: "X-Forwarded-For", db: "./databases/sponsorTimes.db", privateDB: "./databases/private.db", @@ -19,7 +20,7 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], + categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"], categorySupport: { sponsor: ["skip", "mute", "full"], selfpromo: ["skip", "mute", "full"], @@ -34,14 +35,14 @@ addDefaults(config, { chapter: ["chapter"] }, maxNumberOfActiveWarnings: 1, - hoursAfterWarningExpires: 24, + hoursAfterWarningExpires: 16300000, adminUserID: "", discordCompletelyIncorrectReportWebhookURL: null, discordFirstTimeSubmissionsWebhookURL: null, discordNeuralBlockRejectWebhookURL: null, discordFailedReportChannelWebhookURL: null, discordReportChannelWebhookURL: null, - getTopUsersCacheTimeMinutes: 0, + getTopUsersCacheTimeMinutes: 240, globalSalt: null, mode: "", neuralBlockURL: null, @@ -49,15 +50,15 @@ addDefaults(config, { rateLimit: { vote: { windowMs: 900000, - max: 20, - message: "Too many votes, please try again later", - statusCode: 429, + max: 15, + message: "OK", + statusCode: 200, }, view: { windowMs: 900000, - max: 20, + max: 10, statusCode: 200, - message: "Too many views, please try again later", + message: "OK", }, rate: { windowMs: 900000, @@ -70,12 +71,17 @@ addDefaults(config, { newLeafURLs: null, maxRewardTimePerSegmentInSeconds: 600, poiMinimumStartTime: 2, - postgres: null, + postgres: { + enabled: false, + user: "", + host: "", + password: "", + port: 5432 + }, dumpDatabase: { enabled: false, - minTimeBetweenMs: 60000, + minTimeBetweenMs: 180000, appExportPath: "./docker/database-export", - postgresExportPath: "/opt/exports", tables: [{ name: "sponsorTimes", order: "timeSubmitted" @@ -95,11 +101,32 @@ addDefaults(config, { }, { name: "vipUsers" + }, + { + name: "unlistedVideos" + }, + { + name: "videoInfo" + }, + { + name: "ratings" }] }, - diskCache: null, - crons: null + diskCache: { + max: 10737418240 + }, + crons: null, + redis: { + enabled: false, + socket: { + host: "", + port: 0 + }, + disableOfflineQueue: true + } }); +loadFromEnv(config); +migrate(config); // Add defaults function addDefaults(config: SBSConfig, defaults: SBSConfig) { @@ -109,3 +136,50 @@ function addDefaults(config: SBSConfig, defaults: SBSConfig) { } } } + +function migrate(config: SBSConfig) { + // Redis change + if (config.redis) { + const redisConfig = config.redis as any; + if (redisConfig.host || redisConfig.port) { + config.redis.socket = { + host: redisConfig.host, + port: redisConfig.port + }; + } + + if (redisConfig.enable_offline_queue !== undefined) { + config.disableOfflineQueue = !redisConfig.enable_offline_queue; + } + + if (redisConfig.socket?.host && redisConfig.enabled === undefined) { + redisConfig.enabled = true; + } + } + + if (config.postgres && config.postgres.user && config.postgres.enabled === undefined) { + config.postgres.enabled = true; + } +} + +function loadFromEnv(config: SBSConfig, prefix = "") { + for (const key in config) { + const fullKey = (prefix ? `${prefix}_` : "") + key; + const data = config[key]; + + if (typeof data === "object" && !Array.isArray(data)) { + loadFromEnv(data, fullKey); + } else if (process.env[fullKey]) { + const value = process.env[fullKey]; + if (isNumber(value)) { + config[key] = parseInt(value, 10); + } else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") { + config[key] = value === "true"; + } else if (key === "newLeafURLs") { + config[key] = [value]; + } else { + config[key] = value; + } + } + } +} \ No newline at end of file diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 79d2adb..a6e855e 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -94,10 +94,6 @@ export class Postgres implements IDatabase { await client.query(`CREATE DATABASE "${this.config.postgres.database}" WITH OWNER = ${this.config.postgres.user} - ENCODING = 'UTF8' - LC_COLLATE = 'en_US.utf8' - LC_CTYPE = 'en_US.utf8' - TABLESPACE = pg_default CONNECTION LIMIT = -1;` ); } diff --git a/src/databases/databases.ts b/src/databases/databases.ts index 5e2ae0e..52f357c 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -9,7 +9,7 @@ let privateDB: IDatabase; if (config.mysql) { db = new Mysql(config.mysql); privateDB = new Mysql(config.privateMysql); -} else if (config.postgres) { +} else if (config.postgres?.enabled) { db = new Postgres({ dbSchemaFileName: config.dbSchema, dbSchemaFolder: config.schemaFolder, @@ -72,7 +72,7 @@ async function initDb(): Promise { const tables = config?.dumpDatabase?.tables ?? []; const tableNames = tables.map(table => table.name); for (const table of tableNames) { - const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`; + const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`; await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); } } diff --git a/src/middleware/requestRateLimit.ts b/src/middleware/requestRateLimit.ts index 71f8bd9..06fde41 100644 --- a/src/middleware/requestRateLimit.ts +++ b/src/middleware/requestRateLimit.ts @@ -1,19 +1,23 @@ import { getIP } from "../utils/getIP"; import { getHash } from "../utils/getHash"; import { getHashCache } from "../utils/getHashCache"; -import rateLimit from "express-rate-limit"; +import rateLimit, { RateLimitRequestHandler } from "express-rate-limit"; import { RateLimitConfig } from "../types/config.model"; import { Request } from "express"; import { isUserVIP } from "../utils/isUserVIP"; import { UserID } from "../types/user.model"; +import RedisStore from "rate-limit-redis"; +import redis from "../utils/redis"; +import { config } from "../config"; -export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): rateLimit.RateLimit { +export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): RateLimitRequestHandler { return rateLimit({ windowMs: limitConfig.windowMs, max: limitConfig.max, message: limitConfig.message, statusCode: limitConfig.statusCode, - headers: false, + legacyHeaders: false, + standardHeaders: false, keyGenerator: (req) => { return getHash(getIP(req), 1); }, @@ -23,6 +27,9 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r } else { return next(); } - } + }, + store: config.redis?.enabled ? new RedisStore({ + sendCommand: (...args: string[]) => redis.sendCommand(args), + }) : null, }); } diff --git a/src/routes/addUserAsTempVIP.ts b/src/routes/addUserAsTempVIP.ts index fb8d482..3001a0e 100644 --- a/src/routes/addUserAsTempVIP.ts +++ b/src/routes/addUserAsTempVIP.ts @@ -9,6 +9,7 @@ import { isUserVIP } from "../utils/isUserVIP"; import { HashedUserID } from "../types/user.model"; import redis from "../utils/redis"; import { tempVIPKey } from "../utils/redisKeys"; +import { Logger } from "../utils/logger"; interface AddUserAsTempVIPRequest extends Request { query: { @@ -65,12 +66,22 @@ export async function addUserAsTempVIP(req: AddUserAsTempVIPRequest, res: Respon if (!channelInfo?.id) { return res.status(404).send(`No channel found for videoID ${channelVideoID}`); } - await redis.setAsyncEx(tempVIPKey(userID), channelInfo?.id, dayInSeconds); - await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]); - return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`); - } - await redis.delAsync(tempVIPKey(userID)); - await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]); - return res.status(200).send(`Temp VIP removed`); + try { + await redis.setEx(tempVIPKey(userID), dayInSeconds, channelInfo?.id); + await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]); + return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`); + } catch (e) { + Logger.error(e as string); + return res.status(500).send(); + } + } + try { + await redis.del(tempVIPKey(userID)); + await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]); + return res.status(200).send(`Temp VIP removed`); + } catch (e) { + Logger.error(e as string); + return res.status(500).send(); + } } \ No newline at end of file diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index 3fb82e3..73557ad 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -5,6 +5,7 @@ import { config } from "../config"; import util from "util"; import fs from "fs"; import path from "path"; +import { ChildProcess, exec, ExecOptions, spawn } from "child_process"; const unlink = util.promisify(fs.unlink); const ONE_MINUTE = 1000 * 60; @@ -32,9 +33,19 @@ const licenseHeader = `

The API and database follow table.name); +const credentials: ExecOptions = { + env: { + ...process.env, + PGHOST: config.postgres.host, + PGPORT: String(config.postgres.port), + PGUSER: config.postgres.user, + PGPASSWORD: String(config.postgres.password), + PGDATABASE: "sponsorTimes", + } +} + interface TableDumpList { fileName: string; tableName: string; @@ -100,7 +111,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage res.status(404).send("Database dump is disabled"); return; } - if (!config.postgres) { + if (!config.postgres?.enabled) { res.status(404).send("Not supported on this instance"); return; } @@ -170,12 +181,12 @@ async function getDbVersion(): Promise { return row.value; } -export async function redirectLink(req: Request, res: Response): Promise { +export async function downloadFile(req: Request, res: Response): Promise { if (!config?.dumpDatabase?.enabled) { res.status(404).send("Database dump is disabled"); return; } - if (!config.postgres) { + if (!config.postgres?.enabled) { res.status(404).send("Not supported on this instance"); return; } @@ -183,7 +194,7 @@ export async function redirectLink(req: Request, res: Response): Promise { const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path); if (file) { - res.redirect(`/download/${file.fileName}`); + res.sendFile(file.fileName, { root: appExportPath }); } else { res.sendStatus(404); } @@ -210,9 +221,19 @@ async function queueDump(): Promise { for (const table of tables) { const fileName = `${table.name}_${startTime}.csv`; - const file = `${postgresExportPath}/${fileName}`; - await db.prepare("run", `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) - TO '${file}' WITH (FORMAT CSV, HEADER true);`); + const file = `${appExportPath}/${fileName}`; + + await new Promise((resolve) => { + exec(`psql -c "\\copy (SELECT * FROM \\"${table.name}\\"${table.order ? ` ORDER BY \\"${table.order}\\"` : ``})` + + ` TO '${file}' WITH (FORMAT CSV, HEADER true);"`, credentials, (error, stdout, stderr) => { + if (error) { + Logger.error(`[dumpDatabase] Failed to dump ${table.name} to ${file} due to ${stderr}`); + } + + resolve(error ? stderr : stdout); + }); + }) + dumpFiles.push({ fileName, tableName: table.name, diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 237a3be..a9500f8 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -223,7 +223,8 @@ function getWeightedRandomChoice(choices: T[], amountOf //The 3 makes -2 the minimum votes before being ignored completely //this can be changed if this system increases in popularity. - const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost); + const repFactor = choice.votes > 0 ? Math.max(1, choice.reputation + 1) : 1; + const weight = Math.exp(choice.votes * repFactor + 3 + boost); totalWeight += Math.max(weight, 0); return { ...choice, weight }; diff --git a/src/routes/getStatus.ts b/src/routes/getStatus.ts index bc855c2..770ca25 100644 --- a/src/routes/getStatus.ts +++ b/src/routes/getStatus.ts @@ -10,8 +10,12 @@ export async function getStatus(req: Request, res: Response): Promise value = Array.isArray(value) ? value[0] : value; try { const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value; - const numberRequests = await redis.increment("statusRequest"); - const statusRequests = numberRequests?.replies?.[0]; + let statusRequests: unknown = 0; + try { + const numberRequests = await redis.increment("statusRequest"); + statusRequests = numberRequests?.[0]; + } catch (error) { } // eslint-disable-line no-empty + const statusValues: Record = { uptime: process.uptime(), commit: (global as any).HEADCOMMIT || "unknown", diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts index f22fbc1..b9558dd 100644 --- a/src/routes/getTopCategoryUsers.ts +++ b/src/routes/getTopCategoryUsers.ts @@ -28,13 +28,16 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) { GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]); - for (const row of rows) { - userNames.push(row.userName); - viewCounts.push(row.viewCount); - totalSubmissions.push(row.totalSubmissions); - minutesSaved.push(row.minutesSaved); + if (rows) { + for (const row of rows) { + userNames.push(row.userName); + viewCounts.push(row.viewCount); + totalSubmissions.push(row.totalSubmissions); + minutesSaved.push(row.minutesSaved); + } } + return { userNames, viewCounts, diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index d704c85..8ccee51 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -16,6 +16,7 @@ import { getReputation } from "../utils/reputation"; import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model"; import { HashedUserID, UserID } from "../types/user.model"; import { isUserVIP } from "../utils/isUserVIP"; +import { isUserTempVIP } from "../utils/isUserTempVIP"; import { parseUserAgent } from "../utils/userAgent"; import { getService } from "../utils/getService"; import axios from "axios"; @@ -81,19 +82,19 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; axios.post(config.discordFirstTimeSubmissionsWebhookURL, { - "embeds": [{ - "title": data?.title, - "url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`, - "description": `Submission ID: ${UUID}\ + embeds: [{ + title: data?.title, + url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`, + description: `Submission ID: ${UUID}\ \n\nTimestamp: \ ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ \n\nCategory: ${segmentInfo.category}`, - "color": 10813440, - "author": { - "name": userID, + color: 10813440, + author: { + name: userID, }, - "thumbnail": { - "url": getMaxResThumbnail(data) || "", + thumbnail: { + url: getMaxResThumbnail(data) || "", }, }], }) @@ -112,55 +113,6 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: } } -async function sendWebhooksNB(userID: string, videoID: string, UUID: string, startTime: number, endTime: number, category: string, probability: number, ytData: any) { - const submissionInfoRow = await db.prepare("get", `SELECT - (select count(1) from "sponsorTimes" where "userID" = ?) count, - (select count(1) from "sponsorTimes" where "userID" = ? and "votes" <= -2) disregarded, - coalesce((select "userName" FROM "userNames" WHERE "userID" = ?), ?) "userName"`, - [userID, userID, userID, userID]); - - let submittedBy: string; - // If a userName was created then show both - if (submissionInfoRow.userName !== userID) { - submittedBy = `${submissionInfoRow.userName}\n${userID}`; - } else { - submittedBy = userID; - } - - // Send discord message - if (config.discordNeuralBlockRejectWebhookURL === null) return; - - axios.post(config.discordNeuralBlockRejectWebhookURL, { - "embeds": [{ - "title": ytData.items[0].snippet.title, - "url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`, - "description": `**Submission ID:** ${UUID}\ - \n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ - \n**Predicted Probability:** ${probability}\ - \n**Category:** ${category}\ - \n**Submitted by:** ${submittedBy}\ - \n**Total User Submissions:** ${submissionInfoRow.count}\ - \n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`, - "color": 10813440, - "thumbnail": { - "url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "", - }, - }] - }) - .then(res => { - if (res.status >= 400) { - Logger.error("Error sending NeuralBlock Discord hook"); - Logger.error(JSON.stringify(res)); - Logger.error("\n"); - } - }) - .catch(err => { - Logger.error("Failed to send NeuralBlock Discord hook."); - Logger.error(JSON.stringify(err)); - Logger.error("\n"); - }); -} - // callback: function(reject: "String containing reason the submission was rejected") // returns: string when an error, false otherwise @@ -168,98 +120,46 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, - submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service }) { - if (apiVideoInfo) { + submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) { + + const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => { + if (!apiVideoInfo) return undefined; const { err, data } = apiVideoInfo; - if (err) return false; + // return undefined if API error + if (err) return undefined; + return data?.lengthSeconds; + }; + // get duration from API + const apiDuration = apiVideoDuration(apiVideoInfo); + // if API fail or returns 0, get duration from client + const duration = apiDuration || submission.videoDuration; + // return false on undefined or 0 + if (!duration) return false; - const duration = apiVideoInfo?.data?.lengthSeconds; - const segments = submission.segments; - let nbString = ""; - for (let i = 0; i < segments.length; i++) { - if (duration == 0) { - // Allow submission if the duration is 0 (bug in youtube api) - return false; - } else { - if (segments[i].category === "sponsor") { - //Prepare timestamps to send to NB all at once - nbString = `${nbString}${segments[i].segment[0]},${segments[i].segment[1]};`; - } - } - } + const segments = submission.segments; + // map all times to float array + const allSegmentTimes = segments.map(segment => [parseFloat(segment.segment[0]), parseFloat(segment.segment[1])]); - // Get all submissions for this user - const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]); - const allSegmentTimes = []; - if (allSubmittedByUser !== undefined) { - //add segments the user has previously submitted - for (const segmentInfo of allSubmittedByUser) { - allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]); - } - } + // add previous submissions by this user + const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? AND "videoID" = ? AND "votes" > -1 AND "hidden" = 0`, [submission.userID, submission.videoID]); - //add segments they are trying to add in this submission - for (let i = 0; i < segments.length; i++) { - const startTime = parseFloat(segments[i].segment[0]); - const endTime = parseFloat(segments[i].segment[1]); - allSegmentTimes.push([startTime, endTime]); - } - - //merge all the times into non-overlapping arrays - const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) { - return a[0] - b[0] || a[1] - b[1]; - })); - - const videoDuration = data?.lengthSeconds; - if (videoDuration != 0) { - let allSegmentDuration = 0; - //sum all segment times together - allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]); - if (allSegmentDuration > (videoDuration / 100) * 80) { - // Reject submission if all segments combine are over 80% of the video - return "Total length of your submitted segments are over 80% of the video."; - } - } - - // Check NeuralBlock - const neuralBlockURL = config.neuralBlockURL; - if (!neuralBlockURL) return false; - const response = await axios.get(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID} - &segments=${nbString.substring(0, nbString.length - 1)}`, { validateStatus: () => true }); - if (response.status !== 200) return false; - - const nbPredictions = response.data; - let nbDecision = false; - let predictionIdx = 0; //Keep track because only sponsor categories were submitted - for (let i = 0; i < segments.length; i++) { - if (segments[i].category === "sponsor") { - if (nbPredictions.probabilities[predictionIdx] < 0.70) { - nbDecision = true; // At least one bad entry - const startTime = parseFloat(segments[i].segment[0]); - const endTime = parseFloat(segments[i].segment[1]); - - const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime, submission.service); - // Send to Discord - // Note, if this is too spammy. Consider sending all the segments as one Webhook - sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); - } - predictionIdx++; - } - - } - - if (nbDecision) { - return "Rejected based on NeuralBlock predictions."; - } else { - return false; - } - } else { - Logger.debug("Skipped YouTube API"); - - // Can't moderate the submission without calling the youtube API - // so allow by default. - return false; + if (allSubmittedByUser) { + //add segments the user has previously submitted + const allSubmittedTimes = allSubmittedByUser.map((segment: { startTime: string, endTime: string }) => [parseFloat(segment.startTime), parseFloat(segment.endTime)]); + allSegmentTimes.push(...allSubmittedTimes); } + + //merge all the times into non-overlapping arrays + const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort((a, b) => a[0] - b[0] || a[1] - b[1])); + + //sum all segment times together + const allSegmentDuration = allSegmentsSorted.reduce((acc, curr) => acc + (curr[1] - curr[0]), 0); + + if (allSegmentDuration > (duration / 100) * 80) { + // Reject submission if all segments combine are over 80% of the video + return "Total length of your submitted segments are over 80% of the video."; + } + return false; } function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { @@ -310,7 +210,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming invalidFields.push("userID"); if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`); } - if (!Array.isArray(segments) || segments.length < 1) { + if (!Array.isArray(segments) || segments.length == 0) { invalidFields.push("segments"); } // validate start and end times (no : marks) @@ -323,7 +223,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming } if (typeof segmentPair.description !== "string" - || (segmentPair.description.length > 60 && segmentPair.actionType === ActionType.Chapter) + || (segmentPair.actionType === ActionType.Chapter && segmentPair.description.length > 60 ) || (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) { invalidFields.push("segment description"); } @@ -402,7 +302,7 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user } if (!isVIP && segments[i].category === "sponsor" - && segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) { + && segments[i].actionType !== ActionType.Full && (endTime - startTime) < 1) { // Too short return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 }; } @@ -425,32 +325,19 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user return CHECK_PASS; } -async function checkByAutoModerator(videoID: any, userID: any, segments: Array, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise { +async function checkByAutoModerator(videoID: any, userID: any, segments: Array, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise { // Auto moderator check - if (!isVIP && service == Service.YouTube) { - const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service });//startTime, endTime, category: segments[i].category}); - - if (autoModerateResult == "Rejected based on NeuralBlock predictions.") { - // If NB automod rejects, the submission will start with -2 votes. - // Note, if one submission is bad all submissions will be affected. - // However, this behavior is consistent with other automod functions - // already in place. - //decreaseVotes = -2; //Disable for now - } else if (autoModerateResult) { - //Normal automod behavior + if (service == Service.YouTube) { + const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration }); + if (autoModerateResult) { return { pass: false, errorCode: 403, - errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`, - decreaseVotes + errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.` }; } } - - return { - ...CHECK_PASS, - decreaseVotes - }; + return CHECK_PASS; } async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) { @@ -601,11 +488,12 @@ export async function postSkipSegments(req: Request, res: Response): Promise((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce((prev, val) => `${prev} ${val.segment}`, "")}`); + Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce((prev, val) => `${prev} ${val.segment}`, "")}`); return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage); } const isVIP = await isUserVIP(userID); + const isTempVIP = await isUserTempVIP(userID, videoID); const rawIP = getIP(req); const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam); @@ -618,13 +506,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise => { - const apiVideoInfo = await getYouTubeVideoInfo(videoID); - const channelID = apiVideoInfo?.data?.authorId; - const { err, reply } = await redis.getAsync(tempVIPKey(nonAnonUserID)); - - return err || !reply ? false : (reply == channelID); -}; - const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2); async function updateSegmentVideoDuration(UUID: SegmentUUID) { @@ -178,7 +170,7 @@ async function sendWebhooks(voteData: VoteData) { "url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}s#requiredSegment=${voteData.UUID}`, "description": `**${voteData.row.votes} Votes Prior | \ ${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \ - Views**\n\n**Submission ID:** ${voteData.UUID}\ + Views**\n\n**Locked**: ${voteData.row.locked}\n\n**Submission ID:** ${voteData.UUID}\ \n**Category:** ${submissionInfoRow.category}\ \n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\ \n\n**Total User Submissions:** ${submissionInfoRow.count}\ @@ -189,7 +181,7 @@ async function sendWebhooks(voteData: VoteData) { "author": { "name": voteData.finalResponse?.webhookMessage ?? voteData.finalResponse?.finalMessage ?? - getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission), + `${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`, }, "thumbnail": { "url": getMaxResThumbnail(data) || "", @@ -476,11 +468,11 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID // Only change the database if they have made a submission before and haven't voted recently const userAbleToVote = (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0) + && !finalResponse.blockVote + && finalResponse.finalStatus === 200 && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined - && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined) - && !finalResponse.blockVote - && finalResponse.finalStatus === 200; + && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined); const ableToVote = isVIP || isTempVIP || userAbleToVote; @@ -534,4 +526,4 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID Logger.error(err as string); return { status: 500, message: finalResponse.finalMessage ?? undefined, json: { error: "Internal error creating segment vote" } }; } -} \ No newline at end of file +} diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 1a5a22f..2fe08f6 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -2,6 +2,14 @@ import { PoolConfig } from "pg"; import * as redis from "redis"; import { CacheOptions } from "@ajayyy/lru-diskcache"; +interface RedisConfig extends redis.RedisClientOptions { + enabled: boolean; +} + +interface CustomPostgresConfig extends PoolConfig { + enabled: boolean; +} + export interface SBSConfig { [index: string]: any port: number; @@ -41,9 +49,9 @@ export interface SBSConfig { privateMysql?: any; minimumPrefix?: string; maximumPrefix?: string; - redis?: redis.ClientOpts; + redis?: RedisConfig; maxRewardTimePerSegmentInSeconds?: number; - postgres?: PoolConfig; + postgres?: CustomPostgresConfig; dumpDatabase?: DumpDatabase; diskCache: CacheOptions; crons: CronJobOptions; @@ -76,7 +84,6 @@ export interface DumpDatabase { enabled: boolean; minTimeBetweenMs: number; appExportPath: string; - postgresExportPath: string; tables: DumpDatabaseTable[]; } diff --git a/src/utils/getHashCache.ts b/src/utils/getHashCache.ts index 14d30c6..f801545 100644 --- a/src/utils/getHashCache.ts +++ b/src/utils/getHashCache.ts @@ -18,18 +18,19 @@ export async function getHashCache(value: T, times = defaulted async function getFromRedis(key: HashedValue): Promise { const redisKey = shaHashKey(key); - const { err, reply } = await redis.getAsync(redisKey); - if (!err && reply) { - try { + try { + const reply = await redis.get(redisKey); + + if (reply) { Logger.debug(`Got data from redis: ${reply}`); return reply as T & HashedValue; - } catch (e) { - // If all else, continue on hashing } - } - const data = getHash(key, cachedHashTimes); + } catch (e) {} // eslint-disable-line no-empty + + // Otherwise, calculate it + const data = getHash(key, cachedHashTimes); + redis.set(key, data); - redis.setAsync(key, data); return data as T & HashedValue; } \ No newline at end of file diff --git a/src/utils/isUserTempVIP.ts b/src/utils/isUserTempVIP.ts new file mode 100644 index 0000000..a2758bc --- /dev/null +++ b/src/utils/isUserTempVIP.ts @@ -0,0 +1,24 @@ +import redis from "../utils/redis"; +import { tempVIPKey } from "../utils/redisKeys"; +import { HashedUserID } from "../types/user.model"; +import { YouTubeAPI } from "../utils/youtubeApi"; +import { APIVideoInfo } from "../types/youtubeApi.model"; +import { VideoID } from "../types/segments.model"; +import { config } from "../config"; +import { Logger } from "./logger"; + +function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { + return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null; +} + +export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID): Promise => { + const apiVideoInfo = await getYouTubeVideoInfo(videoID); + const channelID = apiVideoInfo?.data?.authorId; + try { + const reply = await redis.get(tempVIPKey(hashedUserID)); + return reply && reply == channelID; + } catch (e) { + Logger.error(e as string); + return false; + } +}; \ No newline at end of file diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index 57af472..028c4e7 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -5,21 +5,18 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { UserID } from "../types/user.model"; async function get(fetchFromDB: () => Promise, key: string): Promise { - const { err, reply } = await redis.getAsync(key); - - if (!err && reply) { - try { + try { + const reply = await redis.get(key); + if (reply) { Logger.debug(`Got data from redis: ${reply}`); return JSON.parse(reply); - } catch (e) { - // If all else, continue on to fetching from the database } - } + } catch (e) { } //eslint-disable-line no-empty const data = await fetchFromDB(); - redis.setAsync(key, JSON.stringify(data)); + redis.set(key, JSON.stringify(data)); return data; } @@ -30,18 +27,17 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { async function getAndSplit(fetchFromDB: (values: U[]) => Promise>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise> { const cachedValues = await Promise.all(values.map(async (value) => { const key = keyGenerator(value); - const { err, reply } = await redis.getAsync(key); - - if (!err && reply) { - try { + try { + const reply = await redis.get(key); + if (reply) { Logger.debug(`Got data from redis: ${reply}`); return { value, result: JSON.parse(reply) }; - } catch (e) { } //eslint-disable-line no-empty - } + } + } catch (e) { } //eslint-disable-line no-empty return { value, @@ -71,7 +67,7 @@ async function getAndSplit(fetchFromDB: (values: U[]) => Pr } for (const key in newResults) { - redis.setAsync(key, JSON.stringify(newResults[key])); + redis.set(key, JSON.stringify(newResults[key])); } }); } @@ -81,16 +77,16 @@ async function getAndSplit(fetchFromDB: (values: U[]) => Pr function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void { if (videoInfo) { - redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); - redis.delAsync(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)); - redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); - if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID)); + redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); + redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)); + redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); + if (videoInfo.userID) redis.del(reputationKey(videoInfo.userID)); } } function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void { if (videoInfo) { - redis.delAsync(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)); + redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)); } } diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 7faefe6..e1bc2e9 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -1,59 +1,56 @@ import { config } from "../config"; import { Logger } from "./logger"; -import redis, { Callback } from "redis"; +import { createClient } from "redis"; +import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@node-redis/client/dist/lib/commands"; +import { ClientCommandOptions } from "@node-redis/client/dist/lib/client"; +import { RedisReply } from "rate-limit-redis"; interface RedisSB { - get(key: string, callback?: Callback): void; - getAsync?(key: string): Promise<{err: Error | null, reply: string | null}>; - set(key: string, value: string, callback?: Callback): void; - setAsync?(key: string, value: string): Promise<{err: Error | null, reply: string | null}>; - setAsyncEx?(key: string, value: string, seconds: number): Promise<{err: Error | null, reply: string | null}>; - delAsync?(...keys: [string]): Promise; - close?(flush?: boolean): void; - increment?(key: string): Promise<{err: Error| null, replies: any[] | null}>; + get(key: RedisCommandArgument): Promise; + set(key: RedisCommandArgument, value: RedisCommandArgument): Promise; + setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; + del(...keys: [RedisCommandArgument]): Promise; + increment?(key: RedisCommandArgument): Promise; + sendCommand(args: RedisCommandArguments, options?: ClientCommandOptions): Promise; + quit(): Promise; } -let exportObject: RedisSB = { - get: (key, callback?) => callback(null, undefined), - getAsync: () => - new Promise((resolve) => resolve({ err: null, reply: undefined })), - set: (key, value, callback) => callback(null, undefined), - setAsync: () => - new Promise((resolve) => resolve({ err: null, reply: undefined })), - setAsyncEx: () => - new Promise((resolve) => resolve({ err: null, reply: undefined })), - delAsync: () => - new Promise((resolve) => resolve(null)), - increment: () => - new Promise((resolve) => resolve({ err: null, replies: undefined })), +let exportClient: RedisSB = { + get: () => new Promise((resolve) => resolve(null)), + set: () => new Promise((resolve) => resolve(null)), + setEx: () => new Promise((resolve) => resolve(null)), + del: () => new Promise((resolve) => resolve(null)), + increment: () => new Promise((resolve) => resolve(null)), + sendCommand: () => new Promise((resolve) => resolve(null)), + quit: () => new Promise((resolve) => resolve(null)), }; -if (config.redis) { +if (config.redis?.enabled) { Logger.info("Connected to redis"); - const client = redis.createClient(config.redis); - exportObject = client; + const client = createClient(config.redis); + client.connect(); + exportClient = client; const timeoutDuration = 200; - exportObject.getAsync = (key) => new Promise((resolve) => { - const timeout = setTimeout(() => resolve({ err: null, reply: undefined }), timeoutDuration); - client.get(key, (err, reply) => { + const get = client.get.bind(client); + exportClient.get = (key) => new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(), timeoutDuration); + get(key).then((reply) => { clearTimeout(timeout); - resolve({ err, reply }); - }); + resolve(reply); + }).catch((err) => reject(err)); }); - exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({ err, reply }))); - exportObject.setAsyncEx = (key, value, seconds) => new Promise((resolve) => client.setex(key, seconds, value, (err, reply) => resolve({ err, reply }))); - exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err))); - exportObject.close = (flush) => client.end(flush); - exportObject.increment = (key) => new Promise((resolve) => + exportClient.increment = (key) => new Promise((resolve, reject) => client.multi() .incr(key) .expire(key, 60) - .exec((err, replies) => resolve({ err, replies })) + .exec() + .then((reply) => resolve(reply)) + .catch((err) => reject(err)) ); client.on("error", function(error) { Logger.error(error); }); } -export default exportObject; +export default exportClient; diff --git a/test.json b/test.json index e10ff44..95d2d79 100644 --- a/test.json +++ b/test.json @@ -16,6 +16,7 @@ "schemaFolder": "./databases", "dbSchema": "./databases/_sponsorTimes.db.sql", "privateDBSchema": "./databases/_private.db.sql", + "categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], "mode": "test", "readOnly": false, "webhooks": [ diff --git a/test/cases/getStatus.ts b/test/cases/getStatus.ts index 970ebed..abf22b3 100644 --- a/test/cases/getStatus.ts +++ b/test/cases/getStatus.ts @@ -89,7 +89,7 @@ describe("getStatus", () => { }); it("Should be able to get statusRequests only", function (done) { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); client.get(`${endpoint}/statusRequests`) .then(res => { assert.strictEqual(res.status, 200); @@ -100,7 +100,7 @@ describe("getStatus", () => { }); it("Should be able to get status with statusRequests", function (done) { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); client.get(endpoint) .then(res => { assert.strictEqual(res.status, 200); diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 498d703..4e5ff5d 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -734,22 +734,6 @@ describe("postSkipSegments", () => { .catch(err => done(err)); }); - it("Should be rejected if NB's predicted probability is <70%.", (done) => { - const videoID = "LevkAjUE6d4"; - postSkipSegmentParam({ - videoID, - startTime: 40, - endTime: 60, - userID: submitUserTwo, - category: "sponsor" - }) - .then(res => { - assert.strictEqual(res.status, 200); - done(); - }) - .catch(err => done(err)); - }); - it("Should be rejected with custom message if user has to many active warnings", (done) => { postSkipSegmentJSON({ userID: warnUser01, diff --git a/test/cases/redisTest.ts b/test/cases/redisTest.ts index 3ab238d..a6d21bb 100644 --- a/test/cases/redisTest.ts +++ b/test/cases/redisTest.ts @@ -11,22 +11,21 @@ const randKey2 = genRandom(16); describe("redis test", function() { before(async function() { - if (!config.redis) this.skip(); - await redis.setAsync(randKey1, randValue1); + if (!config.redis?.enabled) this.skip(); + await redis.set(randKey1, randValue1); }); it("Should get stored value", (done) => { - redis.getAsync(randKey1) + redis.get(randKey1) .then(res => { - if (res.err) assert.fail(res.err); - assert.strictEqual(res.reply, randValue1); + assert.strictEqual(res, randValue1); done(); - }); + }).catch(err => assert.fail(err)); }); it("Should not be able to get not stored value", (done) => { - redis.getAsync(randKey2) + redis.get(randKey2) .then(res => { - if (res.reply || res.err ) assert.fail("Value should not be found"); + if (res) assert.fail("Value should not be found"); done(); - }); + }).catch(err => assert.fail(err)); }); }); \ No newline at end of file diff --git a/test/cases/tempVip.ts b/test/cases/tempVip.ts index c2fbc10..6f23913 100644 --- a/test/cases/tempVip.ts +++ b/test/cases/tempVip.ts @@ -6,6 +6,7 @@ import { client } from "../utils/httpClient"; import { db, privateDB } from "../../src/databases/databases"; import redis from "../../src/utils/redis"; import assert from "assert"; +import { Logger } from "../../src/utils/logger"; // helpers const getSegment = (UUID: string) => db.prepare("get", `SELECT "votes", "locked", "category" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); @@ -51,13 +52,18 @@ const postVoteCategory = (userID: string, UUID: string, category: string) => cli } }); const checkUserVIP = async (publicID: HashedUserID) => { - const { reply } = await redis.getAsync(tempVIPKey(publicID)); - return reply; + try { + const reply = await redis.get(tempVIPKey(publicID)); + return reply; + } catch (e) { + Logger.error(e as string); + return false; + } }; describe("tempVIP test", function() { before(async function() { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; await db.prepare("run", insertSponsorTimeQuery, ["channelid-convert", 0, 1, 0, 0, UUID0, "testman", 0, 50, "sponsor", 0]); @@ -67,7 +73,7 @@ describe("tempVIP test", function() { await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP1]); await db.prepare("run", 'INSERT INTO "vipUsers" ("userID") VALUES (?)', [publicPermVIP2]); // clear redis if running consecutive tests - await redis.delAsync(tempVIPKey(publicTempVIPOne)); + await redis.del(tempVIPKey(publicTempVIPOne)); }); it("Should update db version when starting the application", () => { diff --git a/test/test.ts b/test/test.ts index 3d72c34..e13f4ba 100644 --- a/test/test.ts +++ b/test/test.ts @@ -57,7 +57,7 @@ async function init() { mocha.run((failures) => { mockServer.close(); server.close(); - redis.close(true); + redis.quit(); process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures }); });