From 5b177a3e53e061f1fad4b9bc0925fd0af29edec4 Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 3 May 2022 22:08:44 -0400 Subject: [PATCH] Prepare dockerfile for use, allow configuring via env vars --- Dockerfile | 6 +-- entrypoint.sh | 22 ++------- src/app.ts | 2 +- src/config.ts | 78 +++++++++++++++++++++++++----- src/databases/databases.ts | 2 +- src/middleware/requestRateLimit.ts | 2 +- src/routes/dumpDatabase.ts | 4 +- src/types/config.model.ts | 12 ++++- src/utils/redis.ts | 2 +- test.json | 1 + test/cases/getStatus.ts | 4 +- test/cases/redisTest.ts | 2 +- test/cases/tempVip.ts | 2 +- 13 files changed, 93 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index c39d92e..357fced 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ -FROM node:14-alpine as builder +FROM node:16-alpine as builder RUN apk add --no-cache --virtual .build-deps python make g++ COPY package.json package-lock.json tsconfig.json entrypoint.sh ./ COPY src src RUN npm ci && npm run tsc -FROM node:14-alpine as app +FROM node:16-alpine as app WORKDIR /usr/src/app COPY --from=builder node_modules . COPY --from=builder dist ./dist COPY entrypoint.sh . COPY databases/*.sql databases/ EXPOSE 8080 -CMD ./entrypoint.sh +CMD ./entrypoint.sh \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 9055d2c..09b2035 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,25 +2,11 @@ set -e echo 'Entrypoint script' cd /usr/src/app + +# blank config, use defaults cp /etc/sponsorblock/config.json . || cat < config.json { - "port": 8080, - "globalSalt": "[CHANGE THIS]", - "adminUserID": "[CHANGE THIS]", - "youtubeAPIKey": null, - "discordReportChannelWebhookURL": null, - "discordFirstTimeSubmissionsWebhookURL": null, - "discordAutoModWebhookURL": null, - "proxySubmission": null, - "behindProxy": "X-Forwarded-For", - "db": "./databases/sponsorTimes.db", - "privateDB": "./databases/private.db", - "createDatabaseIfNotExist": true, - "schemaFolder": "./databases", - "dbSchema": "./databases/_sponsorTimes.db.sql", - "privateDBSchema": "./databases/_private.db.sql", - "mode": "development", - "readOnly": false } EOF -node dist/index.js + +node dist/index.js \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index 267e44d..9522c87 100644 --- a/src/app.ts +++ b/src/app.ts @@ -202,7 +202,7 @@ function setupRoutes(router: Router) { router.post("/api/ratings/rate", postRateEndpoints); router.post("/api/ratings/clearCache", ratingPostClearCache); - if (config.postgres) { + if (config.postgres?.enabled) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); router.get("/database.json", (req, res) => dumpDatabase(req, res, false)); router.get("/database/*", redirectLink); diff --git a/src/config.ts b/src/config.ts index d6f59ca..d42520c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,7 @@ import fs from "fs"; import { SBSConfig } from "./types/config.model"; import packageJson from "../package.json"; +import { isBoolean, isNumber } from "lodash"; const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test; const configFile = process.env.TEST_POSTGRES ? "ci.json" @@ -8,9 +9,10 @@ const configFile = process.env.TEST_POSTGRES ? "ci.json" : "config.json"; export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8")); +loadFromEnv(config); migrate(config); addDefaults(config, { - port: 80, + port: 8080, behindProxy: "X-Forwarded-For", db: "./databases/sponsorTimes.db", privateDB: "./databases/private.db", @@ -20,7 +22,7 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], + categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"], categorySupport: { sponsor: ["skip", "mute", "full"], selfpromo: ["skip", "mute", "full"], @@ -35,14 +37,14 @@ addDefaults(config, { chapter: ["chapter"] }, maxNumberOfActiveWarnings: 1, - hoursAfterWarningExpires: 24, + hoursAfterWarningExpires: 16300000, adminUserID: "", discordCompletelyIncorrectReportWebhookURL: null, discordFirstTimeSubmissionsWebhookURL: null, discordNeuralBlockRejectWebhookURL: null, discordFailedReportChannelWebhookURL: null, discordReportChannelWebhookURL: null, - getTopUsersCacheTimeMinutes: 0, + getTopUsersCacheTimeMinutes: 240, globalSalt: null, mode: "", neuralBlockURL: null, @@ -50,15 +52,15 @@ addDefaults(config, { rateLimit: { vote: { windowMs: 900000, - max: 20, - message: "Too many votes, please try again later", - statusCode: 429, + max: 15, + message: "OK", + statusCode: 200, }, view: { windowMs: 900000, - max: 20, + max: 10, statusCode: 200, - message: "Too many views, please try again later", + message: "OK", }, rate: { windowMs: 900000, @@ -71,10 +73,16 @@ addDefaults(config, { newLeafURLs: null, maxRewardTimePerSegmentInSeconds: 600, poiMinimumStartTime: 2, - postgres: null, + postgres: { + enabled: null, + user: "", + host: "", + password: "", + port: 5432 + }, dumpDatabase: { enabled: false, - minTimeBetweenMs: 60000, + minTimeBetweenMs: 180000, appExportPath: "./docker/database-export", postgresExportPath: "/opt/exports", tables: [{ @@ -96,10 +104,29 @@ addDefaults(config, { }, { name: "vipUsers" + }, + { + name: "unlistedVideos" + }, + { + name: "videoInfo" + }, + { + name: "ratings" }] }, - diskCache: null, - crons: null + diskCache: { + max: 10737418240 + }, + crons: null, + redis: { + enabled: null, + socket: { + host: "", + port: 0 + }, + disableOfflineQueue: true + } }); // Add defaults @@ -125,5 +152,30 @@ function migrate(config: SBSConfig) { if (redisConfig.enable_offline_queue !== undefined) { config.disableOfflineQueue = !redisConfig.enable_offline_queue; } + + if (redisConfig.socket.host && redisConfig.enabled === null) { + redisConfig.enabled = true; + } + } + + if (config.postgres && config.postgres.user && config.postgres.enabled === null) { + config.postgres.enabled = true; + } +} + +function loadFromEnv(config: SBSConfig, prefix = "") { + for (const key in config) { + if (typeof config[key] === "object") { + loadFromEnv(config[key], (prefix ? `${prefix}.` : "") + key); + } else if (process.env[key]) { + const value = process.env[key]; + if (isNumber(value)) { + config[key] = parseInt(value, 10); + } else if (isBoolean(value)) { + config[key] = value === "true"; + } else { + config[key] = value; + } + } } } \ No newline at end of file diff --git a/src/databases/databases.ts b/src/databases/databases.ts index 5e2ae0e..9ff8742 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -9,7 +9,7 @@ let privateDB: IDatabase; if (config.mysql) { db = new Mysql(config.mysql); privateDB = new Mysql(config.privateMysql); -} else if (config.postgres) { +} else if (config.postgres?.enabled) { db = new Postgres({ dbSchemaFileName: config.dbSchema, dbSchemaFolder: config.schemaFolder, diff --git a/src/middleware/requestRateLimit.ts b/src/middleware/requestRateLimit.ts index a188011..06fde41 100644 --- a/src/middleware/requestRateLimit.ts +++ b/src/middleware/requestRateLimit.ts @@ -28,7 +28,7 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r return next(); } }, - store: config.redis ? new RedisStore({ + store: config.redis?.enabled ? new RedisStore({ sendCommand: (...args: string[]) => redis.sendCommand(args), }) : null, }); diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index 3fb82e3..3c39857 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -100,7 +100,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage res.status(404).send("Database dump is disabled"); return; } - if (!config.postgres) { + if (!config.postgres?.enabled) { res.status(404).send("Not supported on this instance"); return; } @@ -175,7 +175,7 @@ export async function redirectLink(req: Request, res: Response): Promise { res.status(404).send("Database dump is disabled"); return; } - if (!config.postgres) { + if (!config.postgres?.enabled) { res.status(404).send("Not supported on this instance"); return; } diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 52ba2f2..0b22508 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -2,6 +2,14 @@ import { PoolConfig } from "pg"; import * as redis from "redis"; import { CacheOptions } from "@ajayyy/lru-diskcache"; +interface RedisConfig extends redis.RedisClientOptions { + enabled: boolean; +} + +interface CustomPostgresConfig extends PoolConfig { + enabled: boolean; +} + export interface SBSConfig { [index: string]: any port: number; @@ -41,9 +49,9 @@ export interface SBSConfig { privateMysql?: any; minimumPrefix?: string; maximumPrefix?: string; - redis?: redis.RedisClientOptions; + redis?: RedisConfig; maxRewardTimePerSegmentInSeconds?: number; - postgres?: PoolConfig; + postgres?: CustomPostgresConfig; dumpDatabase?: DumpDatabase; diskCache: CacheOptions; crons: CronJobOptions; diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 3c3123e..5f70e35 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -25,7 +25,7 @@ let exportClient: RedisSB = { quit: () => new Promise((resolve, reject) => reject()), }; -if (config.redis) { +if (config.redis?.enabled) { Logger.info("Connected to redis"); const client = createClient(config.redis); client.connect(); diff --git a/test.json b/test.json index e10ff44..95d2d79 100644 --- a/test.json +++ b/test.json @@ -16,6 +16,7 @@ "schemaFolder": "./databases", "dbSchema": "./databases/_sponsorTimes.db.sql", "privateDBSchema": "./databases/_private.db.sql", + "categoryList": ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], "mode": "test", "readOnly": false, "webhooks": [ diff --git a/test/cases/getStatus.ts b/test/cases/getStatus.ts index 970ebed..abf22b3 100644 --- a/test/cases/getStatus.ts +++ b/test/cases/getStatus.ts @@ -89,7 +89,7 @@ describe("getStatus", () => { }); it("Should be able to get statusRequests only", function (done) { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); client.get(`${endpoint}/statusRequests`) .then(res => { assert.strictEqual(res.status, 200); @@ -100,7 +100,7 @@ describe("getStatus", () => { }); it("Should be able to get status with statusRequests", function (done) { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); client.get(endpoint) .then(res => { assert.strictEqual(res.status, 200); diff --git a/test/cases/redisTest.ts b/test/cases/redisTest.ts index e86f123..a6d21bb 100644 --- a/test/cases/redisTest.ts +++ b/test/cases/redisTest.ts @@ -11,7 +11,7 @@ const randKey2 = genRandom(16); describe("redis test", function() { before(async function() { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); await redis.set(randKey1, randValue1); }); it("Should get stored value", (done) => { diff --git a/test/cases/tempVip.ts b/test/cases/tempVip.ts index 3051e0c..6f23913 100644 --- a/test/cases/tempVip.ts +++ b/test/cases/tempVip.ts @@ -63,7 +63,7 @@ const checkUserVIP = async (publicID: HashedUserID) => { describe("tempVIP test", function() { before(async function() { - if (!config.redis) this.skip(); + if (!config.redis?.enabled) this.skip(); const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; await db.prepare("run", insertSponsorTimeQuery, ["channelid-convert", 0, 1, 0, 0, UUID0, "testman", 0, 50, "sponsor", 0]);