From d6ba5684e0b62c81fff41d80f96019656f564dbd Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Fri, 26 Feb 2021 01:57:45 +0100 Subject: [PATCH 01/44] Limit reward time per segment --- config.json.example | 3 ++- src/config.ts | 1 + src/routes/getTopUsers.ts | 3 ++- src/types/config.model.ts | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index f8548c1..075872a 100644 --- a/config.json.example +++ b/config.json.example @@ -38,5 +38,6 @@ "max": 20, // 20 requests in 15min time window "statusCode": 200 } - } + }, + "maxRewardTimePerSegmentInSeconds": 86400 // maximum time a user get rewarded in the leaderboard for a single segment } diff --git a/src/config.ts b/src/config.ts index 7a172cc..a37fb6c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -45,6 +45,7 @@ addDefaults(config, { }, userCounterURL: null, youtubeAPIKey: null, + maxRewardTimePerSegmentInSeconds: 86400, }); // Add defaults diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index cf069a1..c7a6141 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -5,6 +5,7 @@ import {Request, Response} from 'express'; const MILLISECONDS_IN_MINUTE = 60000; const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); +const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400; function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) { return new Promise((resolve) => { @@ -25,7 +26,7 @@ function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = f } const rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + - "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " + + "SUM(((CASE WHEN endTime - startTime > " + maxRewardTimePerSegmentInSeconds + " THEN " + maxRewardTimePerSegmentInSeconds + " ELSE endTime - startTime END) / 60) * views) as minutesSaved," + "SUM(votes) as userVotes, " + additionalFields + "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " + diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 88af3d2..110b210 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -36,6 +36,7 @@ export interface SBSConfig { minimumPrefix?: string; maximumPrefix?: string; redis?: redis.ClientOpts; + maxRewardTimePerSegmentInSeconds?: number; } export interface WebhookConfig { From ac945254d6169138680196d4173a60d84f8ee75f Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 2 Mar 2021 01:22:02 +0100 Subject: [PATCH 02/44] Limit reward time for getSavedTimeForUser endpoint --- src/routes/getSavedTimeForUser.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/routes/getSavedTimeForUser.ts b/src/routes/getSavedTimeForUser.ts index fbf28dc..9bcd420 100644 --- a/src/routes/getSavedTimeForUser.ts +++ b/src/routes/getSavedTimeForUser.ts @@ -1,6 +1,9 @@ import {db} from '../databases/databases'; import {Request, Response} from 'express'; import {getHash} from '../utils/getHash'; +import {config} from '../config'; + +const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400; export function getSavedTimeForUser(req: Request, res: Response) { let userID = req.query.userID as string; @@ -15,7 +18,7 @@ export function getSavedTimeForUser(req: Request, res: Response) { userID = getHash(userID); try { - let row = db.prepare("get", "SELECT SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE userID = ? AND votes > -1 AND shadowHidden != 1 ", [userID]); + let row = db.prepare("get", "SELECT SUM(((CASE WHEN endTime - startTime > " + maxRewardTimePerSegmentInSeconds + " THEN " + maxRewardTimePerSegmentInSeconds + " ELSE endTime - startTime END) / 60) * views) as minutesSaved FROM sponsorTimes WHERE userID = ? AND votes > -1 AND shadowHidden != 1 ", [userID]); if (row.minutesSaved != null) { res.send({ From e883f76e54ffe2149864e41d9df9b11bf277e5ff Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sat, 20 Mar 2021 01:17:50 +0100 Subject: [PATCH 03/44] Fix quote inconsistency --- src/routes/getTopUsers.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index 1303682..6f66e7e 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -16,12 +16,12 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole let additionalFields = ''; if (categoryStatsEnabled) { - additionalFields += `SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as "categorySponsor", - SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as "categorySumIntro", - SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as "categorySumOutro", - SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as "categorySumInteraction", - SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as "categorySelfpromo", - SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic", `; + additionalFields += `SUM(CASE WHEN category = "sponsor" THEN 1 ELSE 0 END) as "categorySponsor", + SUM(CASE WHEN category = "intro" THEN 1 ELSE 0 END) as "categorySumIntro", + SUM(CASE WHEN category = "outro" THEN 1 ELSE 0 END) as "categorySumOutro", + SUM(CASE WHEN category = "interaction" THEN 1 ELSE 0 END) as "categorySumInteraction", + SUM(CASE WHEN category = "selfpromo" THEN 1 ELSE 0 END) as "categorySelfpromo", + SUM(CASE WHEN category = "music_offtopic" THEN 1 ELSE 0 END) as "categoryMusicOfftopic", `; } const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", @@ -32,7 +32,7 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID" WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20 - ORDER BY "` + sortBy + `" DESC LIMIT 100`, []); + ORDER BY "${sortBy}" DESC LIMIT 100`, []); for (let i = 0; i < rows.length; i++) { userNames[i] = rows[i].userName; From 84b86bb6a1fd19a3a5ac75664df16190294ddfc5 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 21 Mar 2021 22:40:57 +0100 Subject: [PATCH 04/44] Make dumpDatabase configurable --- config.json.example | 25 +++++++++++++++++++++++++ src/config.ts | 27 ++++++++++++++++++++++++++- src/routes/dumpDatabase.ts | 37 ++++++++++++++----------------------- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/config.json.example b/config.json.example index f8548c1..d859e4d 100644 --- a/config.json.example +++ b/config.json.example @@ -38,5 +38,30 @@ "max": 20, // 20 requests in 15min time window "statusCode": 200 } + }, + "dumpDatabase": { + "enabled": true, + "minTimeBetweenMs": 60000, // 1 minute between dumps + "exportPath": "/opt/exports", + "tables": [{ + "name": "sponsorTimes", + "order": "timeSubmitted" + }, + { + "name": "userNames" + }, + { + "name": "categoryVotes" + }, + { + "name": "noSegments", + }, + { + "name": "warnings", + "order": "issueTime" + }, + { + "name": "vipUsers" + }] } } diff --git a/src/config.ts b/src/config.ts index 93c6a43..477aa72 100644 --- a/src/config.ts +++ b/src/config.ts @@ -45,7 +45,32 @@ addDefaults(config, { }, userCounterURL: null, youtubeAPIKey: null, - postgres: null + postgres: null, + dumpDatabase: { + enabled: true, + minTimeBetweenMs: 60000, + exportPath: '/opt/exports', + tables: [{ + name: "sponsorTimes", + order: "timeSubmitted" + }, + { + name: "userNames" + }, + { + name: "categoryVotes" + }, + { + name: "noSegments", + }, + { + name: "warnings", + order: "issueTime" + }, + { + name: "vipUsers" + }] + } }); // Add defaults diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index d687f07..ec3e801 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -11,26 +11,13 @@ const licenseHeader = `

The API and database follow Attribution Template

If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.

`; -const tables = [{ - name: "sponsorTimes", - order: "timeSubmitted" -}, -{ - name: "userNames" -}, -{ - name: "categoryVotes" -}, -{ - name: "noSegments", -}, -{ - name: "warnings", - order: "issueTime" -}, -{ - name: "vipUsers" -}]; +const tables = config?.dumpDatabase?.tables ?? []; +const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE; +const exportPath = config?.dumpDatabase?.exportPath ?? '/opt/exports'; + +if (tables.length === 0) { + Logger.warn('[dumpDatabase] No tables configured'); +} const links: string[] = tables.map((table) => `/database/${table.name}.csv`); @@ -40,13 +27,17 @@ const linksHTML: string = tables.map((table) => `

ONE_MINUTE; + const updateQueued = now - lastUpdate > MILLISECONDS_BETWEEN_DUMPS; res.status(200) @@ -72,7 +63,7 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool for (const table of tables) { db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) - TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); + TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); } } -} \ No newline at end of file +} From 514ea03655f043fa8a65b1f1666b3ffea91247b8 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 21 Mar 2021 22:59:16 +0100 Subject: [PATCH 05/44] Add TS for dumpDatabase config --- src/types/config.model.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/types/config.model.ts b/src/types/config.model.ts index b27cfaa..c0611b7 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -38,6 +38,7 @@ export interface SBSConfig { maximumPrefix?: string; redis?: redis.ClientOpts; postgres?: PoolConfig; + dumpDatabase?: DumpDatabase; } export interface WebhookConfig { @@ -61,4 +62,16 @@ export interface PostgresConfig { createDbIfNotExists: boolean; enableWalCheckpointNumber: boolean; postgres: PoolConfig; -} \ No newline at end of file +} + +export interface DumpDatabase { + enabled: boolean; + minTimeBetweenMs: number; + exportPath: string; + tables: DumpDatabaseTable[]; +} + +export interface DumpDatabaseTable { + name: string; + order?: string; +} From 8219b0398e43230bdbefaeaf22302943f508dfbb Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 22 Mar 2021 01:35:45 +0100 Subject: [PATCH 06/44] Fix invalid json in example config --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index d859e4d..65f9b42 100644 --- a/config.json.example +++ b/config.json.example @@ -54,7 +54,7 @@ "name": "categoryVotes" }, { - "name": "noSegments", + "name": "noSegments" }, { "name": "warnings", From 2c3dde0d2ed1beb80197bfb60f5f55b6f46c69c5 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 22 Mar 2021 22:18:23 +0100 Subject: [PATCH 07/44] Timestamp based dump filenames and garbage collection --- config.json.example | 3 +- src/config.ts | 3 +- src/routes/dumpDatabase.ts | 120 ++++++++++++++++++++++++++++++++++--- src/types/config.model.ts | 3 +- 4 files changed, 119 insertions(+), 10 deletions(-) diff --git a/config.json.example b/config.json.example index 65f9b42..580b8fb 100644 --- a/config.json.example +++ b/config.json.example @@ -42,7 +42,8 @@ "dumpDatabase": { "enabled": true, "minTimeBetweenMs": 60000, // 1 minute between dumps - "exportPath": "/opt/exports", + "appExportPath": "/opt/exports", + "postgresExportPath": "/opt/exports", "tables": [{ "name": "sponsorTimes", "order": "timeSubmitted" diff --git a/src/config.ts b/src/config.ts index 477aa72..0e985dc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,7 +49,8 @@ addDefaults(config, { dumpDatabase: { enabled: true, minTimeBetweenMs: 60000, - exportPath: '/opt/exports', + appExportPath: '/opt/exports', + postgresExportPath: '/opt/exports', tables: [{ name: "sponsorTimes", order: "timeSubmitted" diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index ec3e801..b7f2726 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -2,10 +2,29 @@ import {db} from '../databases/databases'; import {Logger} from '../utils/logger'; import {Request, Response} from 'express'; import { config } from '../config'; +const util = require('util'); +const fs = require('fs'); +const path = require('path'); +const unlink = util.promisify(fs.unlink); +const fstat = util.promisify(fs.fstat); const ONE_MINUTE = 1000 * 60; -const styleHeader = `` +const styleHeader = `` const licenseHeader = `

The API and database follow CC BY-NC-SA 4.0 unless you have explicit permission.

Attribution Template

@@ -13,7 +32,15 @@ const licenseHeader = `

The API and database follow { + return new Promise((resolve, reject) => { + // Get list of table names + // Create array for each table + const tableFiles = tableNames.reduce((obj: any, tableName) => { + obj[tableName] = []; + return obj; + }, {}); + // read files in export directory + fs.readdir(exportPath, (err: any, files: string[]) => { + if (err) Logger.error(err); + if (err) return resolve(); + files.forEach(file => { + // we only care about files that start with "_" and ends with .csv + tableNames.forEach(tableName => { + if (file.startsWith(`${tableName}_`) && file.endsWith('.csv')) { + // extract the timestamp from the filename + // we could also use the fs.stat mtime + const timestamp = Number(file.split('_')[1].replace('.csv', '')); + tableFiles[tableName].push({ + file: path.join(exportPath, file), + timestamp, + }); + } + }); + }); + const outdatedTime = Math.floor(Date.now() - (MILLISECONDS_BETWEEN_DUMPS * 1.5)); + for (let tableName in tableFiles) { + const files = tableFiles[tableName]; + files.forEach(async (item: any) => { + if (item.timestamp < outdatedTime) { + // remove old file + await unlink(item.file).catch((error: any) => { + Logger.error(`[dumpDatabase] Garbage collection failed ${error}`); + }); + } + }); + } + resolve(); + }); + }); +} + +export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) { if (config?.dumpDatabase?.enabled === false) { res.status(404).send("Database dump is disabled"); return; @@ -48,22 +118,58 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool Send a request to https://sponsor.ajay.app/database.json, or visit this page to trigger the database dump to run. Then, you can download the csv files below, or use the links returned from the JSON request.

Links

- ${linksHTML}
+ + + + + + + + + ${latestDumpFiles.map((item:any) => { + return ` + + + + + `; + }).join('')} + ${latestDumpFiles.length === 0 ? '' : ''} + +
TableCSV
${item.tableName}${item.fileName}
Please wait: Generating files
+
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); } else { res.send({ lastUpdated: lastUpdate, updateQueued, - links + links: latestDumpFiles.map((item:any) => { + return { + table: item.tableName, + url: `/download/${item.fileName}`, + size: item.fileSize, + }; + }), }) } if (updateQueued) { lastUpdate = Date.now(); + + await removeOutdatedDumps(appExportPath); + + const dumpFiles = []; for (const table of tables) { - db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) - TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); + const fileName = `${table.name}_${lastUpdate}.csv`; + const file = `${postgresExportPath}/${fileName}`; + await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) + TO '${file}' WITH (FORMAT CSV, HEADER true);`); + dumpFiles.push({ + fileName, + tableName: table.name, + }); } + latestDumpFiles = [...dumpFiles]; } } diff --git a/src/types/config.model.ts b/src/types/config.model.ts index c0611b7..f46cc17 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -67,7 +67,8 @@ export interface PostgresConfig { export interface DumpDatabase { enabled: boolean; minTimeBetweenMs: number; - exportPath: string; + appExportPath: string; + postgresExportPath: string; tables: DumpDatabaseTable[]; } From 5c827baa1adc2d2d9f568a740c5167bdfce9055f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 25 Mar 2021 18:48:44 -0400 Subject: [PATCH 08/44] Update db link --- README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.MD b/README.MD index 8f61c57..e309e54 100644 --- a/README.MD +++ b/README.MD @@ -8,7 +8,7 @@ This is the server backend for it This uses a Postgres or Sqlite database to hold all the timing data. -To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. +To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea. From 1eca55d96c4a327c3dbd89af6d4cc74ad24afc22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 21:52:03 +0000 Subject: [PATCH 09/44] Bump y18n from 4.0.0 to 4.0.1 Bumps [y18n](https://github.com/yargs/y18n) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/yargs/y18n/releases) - [Changelog](https://github.com/yargs/y18n/blob/master/CHANGELOG.md) - [Commits](https://github.com/yargs/y18n/commits) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index dd69f2e..f17b109 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2989,9 +2989,9 @@ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, "y18n": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", - "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", + "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", "dev": true }, "yallist": { From 77a4c2fe34d67002b2427d921c8599cd224ab03d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 13 Apr 2021 03:04:02 +0200 Subject: [PATCH 10/44] Update nginx config --- nginx/nginx.conf | 87 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index daee81f..76a77d3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -2,7 +2,7 @@ worker_processes 8; worker_rlimit_nofile 8192; events { - worker_connections 32768; ## Default: 1024 + worker_connections 132768; ## Default: 1024 } http { @@ -12,19 +12,35 @@ http { upstream backend_GET { least_conn; - server localhost:4442; - server localhost:4443; - server localhost:4444; - server localhost:4445; - server localhost:4446; + server localhost:4441; + server localhost:4442; + #server localhost:4443; + #server localhost:4444; + #server localhost:4445; + #server localhost:4446; #server localhost:4447; #server localhost:4448; + + server 10.0.0.3:4441; + server 10.0.0.3:4442; + + #server 134.209.69.251:80 backup; + + server 116.203.32.253:80 backup; + #server 116.203.32.253:80; } upstream backend_POST { - server localhost:4441; + #server localhost:4441; + #server localhost:4442; + server 10.0.0.3:4441; + #server 10.0.0.3:4442; + } + upstream backend_db { + #server localhost:4441; + server 10.0.0.3:4441; } - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m; + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m; proxy_cache_key "$scheme$request_method$host$request_uri"; add_header X-Cache $upstream_cache_status; @@ -43,14 +59,16 @@ http { # internal; #} + #proxy_send_timeout 120s; + location @myerrordirective_500 { - return 502 "Internal Server Error"; + return 400 "Internal Server Error"; } location @myerrordirective_502 { - return 502 "Bad Gateway"; + return 400 "Bad Gateway"; } location @myerrordirective_504 { - return 502 "Gateway Timeout"; + return 400 "Gateway Timeout"; } @@ -62,17 +80,16 @@ http { return 301 https://sb.ltn.fi; } - location /invidious/ { - proxy_pass https://invidious.fdn.fr/; - } - location /test/ { proxy_pass http://localhost:4440/; #proxy_pass https://sbtest.etcinit.com/; } location /api/skipSegments { - proxy_pass http://backend_$request_method; + #return 200 "[]"; + proxy_pass http://backend_$request_method; + #proxy_cache CACHEZONE; + #proxy_cache_valid 2m; } location /api/getTopUsers { @@ -83,24 +100,43 @@ http { location /api/getTotalStats { proxy_pass http://backend_GET; - } + #return 200 ""; + } location /api/getVideoSponsorTimes { proxy_pass http://backend_GET; } - - location = /database.db { - alias /home/sbadmin/sponsor/databases/sponsorTimes.db; + + location /database { + proxy_pass http://backend_db; } - + + location = /database.db { + #return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals."; + #alias /home/sbadmin/sponsor/databases/sponsorTimes.db; + alias /home/sbadmin/test-db/database.db; + } + + location = /database/sponsorTimes.csv { + alias /home/sbadmin/sponsorTimes.csv; + } + + #location /api/voteOnSponsorTime { + # return 200 "Success"; + #} + + #location /api/viewedVideoSponsorTime { + # return 200 "Success"; + #} + location /api { proxy_pass http://backend_POST; } location / { - root /home/sbadmin/caddy/SponsorBlockSite/public-prod; - + root /home/sbadmin/SponsorBlockSite/public-prod; + ### CORS if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; @@ -132,12 +168,15 @@ http { } - listen 443 ssl; # managed by Certbot + listen 443 default_server ssl; # managed by Certbot + #listen 80; ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + + } From 389885893c6d1f1493f738831e76cb63057d3e0d Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Thu, 15 Apr 2021 01:22:01 +0200 Subject: [PATCH 11/44] Only get enabled warnings --- src/routes/getUserInfo.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index ca1066c..eb96c48 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -52,8 +52,8 @@ async function dbGetViewsForUser(userID: string) { async function dbGetWarningsForUser(userID: string): Promise { try { - let rows = await db.prepare('all', `SELECT * FROM "warnings" WHERE "userID" = ?`, [userID]); - return rows.length; + let row = await db.prepare('get', `SELECT COUNT(1) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]); + return row?.total ?? 0; } catch (err) { Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0'); return 0; From 9b0ba9031ea819cf83ee0f39860dfccc83f0b0b0 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Thu, 15 Apr 2021 01:22:25 +0200 Subject: [PATCH 12/44] Optimize code --- src/routes/getUserInfo.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index eb96c48..9960ece 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -5,7 +5,7 @@ import {Logger} from '../utils/logger' async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSaved: number, segmentCount: number }> { try { - let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(*) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); + let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(1) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); if (row.minutesSaved != null) { return { minutesSaved: row.minutesSaved, @@ -39,12 +39,7 @@ async function dbGetUsername(userID: string) { async function dbGetViewsForUser(userID: string) { try { let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); - //increase the view count by one - if (row.viewCount != null) { - return row.viewCount; - } else { - return 0; - } + return row?.viewCount ?? 0; } catch (err) { return false; } From 112b232f9ea1f61f564bfb0a2df9a738ee0e8d6a Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Thu, 15 Apr 2021 23:03:50 +0200 Subject: [PATCH 13/44] Fix UserID types --- src/routes/getUserInfo.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index 9960ece..15f7e29 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -1,9 +1,10 @@ import {db} from '../databases/databases'; import {getHash} from '../utils/getHash'; import {Request, Response} from 'express'; -import {Logger} from '../utils/logger' +import {Logger} from '../utils/logger'; +import { HashedUserID, UserID } from '../types/user.model'; -async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSaved: number, segmentCount: number }> { +async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> { try { let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(1) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); if (row.minutesSaved != null) { @@ -22,7 +23,7 @@ async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSa } } -async function dbGetUsername(userID: string) { +async function dbGetUsername(userID: HashedUserID) { try { let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); if (row !== undefined) { @@ -36,7 +37,7 @@ async function dbGetUsername(userID: string) { } } -async function dbGetViewsForUser(userID: string) { +async function dbGetViewsForUser(userID: HashedUserID) { try { let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); return row?.viewCount ?? 0; @@ -45,7 +46,7 @@ async function dbGetViewsForUser(userID: string) { } } -async function dbGetWarningsForUser(userID: string): Promise { +async function dbGetWarningsForUser(userID: HashedUserID): Promise { try { let row = await db.prepare('get', `SELECT COUNT(1) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]); return row?.total ?? 0; @@ -56,7 +57,7 @@ async function dbGetWarningsForUser(userID: string): Promise { } export async function getUserInfo(req: Request, res: Response) { - let userID = req.query.userID as string; + let userID = req.query.userID as UserID; if (userID == undefined) { //invalid request From cb4ecea830a582dbb1b326a8a1587cbacaf9e4c2 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Thu, 15 Apr 2021 23:05:18 +0200 Subject: [PATCH 14/44] Add vip info to getUserInfo --- src/routes/getUserInfo.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index 15f7e29..5f0c905 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -1,5 +1,6 @@ import {db} from '../databases/databases'; import {getHash} from '../utils/getHash'; +import {isUserVIP} from '../utils/isUserVIP'; import {Request, Response} from 'express'; import {Logger} from '../utils/logger'; import { HashedUserID, UserID } from '../types/user.model'; @@ -66,17 +67,18 @@ export async function getUserInfo(req: Request, res: Response) { } //hash the userID - userID = getHash(userID); + const hashedUserID: HashedUserID = getHash(userID); - const segmentsSummary = await dbGetSubmittedSegmentSummary(userID); + const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID); if (segmentsSummary) { res.send({ - userID, - userName: await dbGetUsername(userID), + userID: hashedUserID, + userName: await dbGetUsername(hashedUserID), minutesSaved: segmentsSummary.minutesSaved, segmentCount: segmentsSummary.segmentCount, - viewCount: await dbGetViewsForUser(userID), - warnings: await dbGetWarningsForUser(userID), + viewCount: await dbGetViewsForUser(hashedUserID), + warnings: await dbGetWarningsForUser(hashedUserID), + vip: await isUserVIP(hashedUserID), }); } else { res.status(400).send(); From 5b2f05741ed8e35d114123fab4cc2717b0c97bb4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 17 Apr 2021 16:23:45 -0400 Subject: [PATCH 15/44] Fix no segments votes being allowed --- src/routes/voteOnSponsorTime.ts | 76 +++++++++++++++++---------------- test/cases/voteOnSponsorTime.ts | 21 +++++---- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 29f0e66..a1e81e7 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -175,51 +175,54 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS const timeSubmitted = Date.now(); const voteAmount = isVIP ? 500 : 1; + const ableToVote = isVIP || finalResponse.finalStatus === 200 || true; - // Add the vote - if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) { - // Update the already existing db entry - await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]); - } else { - // Add a db entry - await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]); - } + if (ableToVote) { + // Add the vote + if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) { + // Update the already existing db entry + await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]); + } else { + // Add a db entry + await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]); + } - // Add the info into the private db - if (usersLastVoteInfo?.votes > 0) { - // Reverse the previous vote - await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]); + // Add the info into the private db + if (usersLastVoteInfo?.votes > 0) { + // Reverse the previous vote + await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]); - await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]); - } else { - await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]); - } + await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]); + } else { + await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]); + } - // See if the submissions category is ready to change - const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]); + // See if the submissions category is ready to change + const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]); - const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); - const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); - const startingVotes = isSubmissionVIP ? 10000 : 1; + const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); + const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); + const startingVotes = isSubmissionVIP ? 10000 : 1; - // Change this value from 1 in the future to make it harder to change categories - // Done this way without ORs incase the value is zero - const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes; + // Change this value from 1 in the future to make it harder to change categories + // Done this way without ORs incase the value is zero + const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes; - // Add submission as vote - if (!currentCategoryInfo && submissionInfo) { - await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]); + // Add submission as vote + if (!currentCategoryInfo && submissionInfo) { + await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]); - await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]); - } + await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]); + } - const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; + const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; - //TODO: In the future, raise this number from zero to make it harder to change categories - // VIPs change it every time - if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) { - // Replace the category - await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); + //TODO: In the future, raise this number from zero to make it harder to change categories + // VIPs change it every time + if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) { + // Replace the category + await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); + } } res.sendStatus(finalResponse.finalStatus); @@ -371,7 +374,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) { const ableToVote = isVIP || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined - && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined); + && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined) + && finalResponse.finalStatus === 200; if (ableToVote) { //update the votes table diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index 47a1f19..2aae65f 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -410,12 +410,13 @@ describe('voteOnSponsorTime', () => { it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => { fetch(getbaseURL() - + "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=0") + + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0") .then(async res => { - if (res.status === 403) { + let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]); + if (res.status === 403 && row.votes === 2) { done(); } else { - done("Status code was " + res.status + " instead of 403"); + done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row)); } }) .catch(err => done(err)); @@ -423,12 +424,13 @@ describe('voteOnSponsorTime', () => { it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => { fetch(getbaseURL() - + "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=1") + + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=1") .then(async res => { - if (res.status === 200) { + let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]); + if (res.status === 200 && row.votes === 3) { done(); } else { - done("Status code was " + res.status + " instead of 200"); + done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row)); } }) .catch(err => done(err)); @@ -436,12 +438,13 @@ describe('voteOnSponsorTime', () => { it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => { fetch(getbaseURL() - + "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&category=outro") + + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro") .then(async res => { - if (res.status === 403) { + let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]); + if (res.status === 403 && row.category === "sponsor") { done(); } else { - done("Status code was " + res.status + " instead of 403"); + done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row)); } }) .catch(err => done(err)); From 9d06bda4f8d93942e39ec6aa1c7eb2f6eeedfdcd Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 17 Apr 2021 16:37:39 -0400 Subject: [PATCH 16/44] Don't allow downvoting dead submissions --- src/routes/voteOnSponsorTime.ts | 12 +++++++++--- test/cases/voteOnSponsorTime.ts | 19 +++++++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 240baf9..4cb7690 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -286,13 +286,19 @@ export async function voteOnSponsorTime(req: Request, res: Response) { return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res); } - if (type == 1 && !isVIP && !isOwnSubmission) { + if (type !== undefined && !isVIP && !isOwnSubmission) { // Check if upvoting hidden segment const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); if (voteInfo && voteInfo.votes <= -2) { - res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP."); - return; + if (type == 1) { + res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP."); + return; + } else if (type == 0) { + // Already downvoted enough, ignore + res.status(200).send(); + return; + } } } diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index 2aae65f..86c3233 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -368,10 +368,25 @@ describe('voteOnSponsorTime', () => { fetch(getbaseURL() + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1") .then(async res => { - if (res.status === 403) { + let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]); + if (res.status === 403 && row.votes === -3) { done(); } else { - done("Status code was " + res.status + " instead of 403"); + done("Status code was " + res.status + ", row is " + JSON.stringify(row)); + } + }) + .catch(err => done(err)); + }); + + it('Non-VIP should not be able to downvote "dead" submission', (done: Done) => { + fetch(getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=0") + .then(async res => { + let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]); + if (res.status === 200 && row.votes === -3) { + done(); + } else { + done("Status code was " + res.status + ", row is " + JSON.stringify(row)); } }) .catch(err => done(err)); From 058c05a1f72f77276c21c53f07c4adc8d43f0cf3 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 18 Apr 2021 04:49:05 +0200 Subject: [PATCH 17/44] Fix permission issues --- docker/docker-compose.yml | 8 ++++---- nginx/nginx.conf | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4ee69ee..f7408d3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,9 +7,9 @@ services: - database.env volumes: - database-data:/var/lib/postgresql/data - - ./database-export/:/opt/exports + - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports ports: - - 127.0.0.1:5432:5432 + - 5432:5432 redis: container_name: redis image: redis @@ -17,7 +17,7 @@ services: volumes: - ./redis/redis.conf:/usr/local/etc/redis/redis.conf ports: - - 127.0.0.1:32773:6379 + - 32773:6379 volumes: - database-data: \ No newline at end of file + database-data: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 76a77d3..6302691 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -108,6 +108,9 @@ http { proxy_pass http://backend_GET; } + location /database/ { + alias /home/sbadmin/sponsor/docker/database-export/; + } location /database { proxy_pass http://backend_db; } @@ -118,9 +121,10 @@ http { alias /home/sbadmin/test-db/database.db; } - location = /database/sponsorTimes.csv { - alias /home/sbadmin/sponsorTimes.csv; - } + #location = /database/sponsorTimes.csv { + # alias /home/sbadmin/sponsorTimes.csv; + #} + #location /api/voteOnSponsorTime { # return 200 "Success"; From a06ab724adab93c082cb7a972c859768304b8c24 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 17 Apr 2021 23:06:39 -0400 Subject: [PATCH 18/44] Fix file locations + formatting --- src/config.ts | 2 +- src/routes/dumpDatabase.ts | 28 +++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/config.ts b/src/config.ts index 0e985dc..6139a6e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,7 +49,7 @@ addDefaults(config, { dumpDatabase: { enabled: true, minTimeBetweenMs: 60000, - appExportPath: '/opt/exports', + appExportPath: './docker/database-export', postgresExportPath: '/opt/exports', tables: [{ name: "sponsorTimes", diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index b7f2726..8e33440 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -2,11 +2,10 @@ import {db} from '../databases/databases'; import {Logger} from '../utils/logger'; import {Request, Response} from 'express'; import { config } from '../config'; -const util = require('util'); -const fs = require('fs'); -const path = require('path'); +import util from 'util'; +import fs from 'fs'; +import path from 'path'; const unlink = util.promisify(fs.unlink); -const fstat = util.promisify(fs.fstat); const ONE_MINUTE = 1000 * 60; @@ -16,7 +15,7 @@ const styleHeader = `