From 84b86bb6a1fd19a3a5ac75664df16190294ddfc5 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 21 Mar 2021 22:40:57 +0100 Subject: [PATCH 1/5] Make dumpDatabase configurable --- config.json.example | 25 +++++++++++++++++++++++++ src/config.ts | 27 ++++++++++++++++++++++++++- src/routes/dumpDatabase.ts | 37 ++++++++++++++----------------------- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/config.json.example b/config.json.example index f8548c1..d859e4d 100644 --- a/config.json.example +++ b/config.json.example @@ -38,5 +38,30 @@ "max": 20, // 20 requests in 15min time window "statusCode": 200 } + }, + "dumpDatabase": { + "enabled": true, + "minTimeBetweenMs": 60000, // 1 minute between dumps + "exportPath": "/opt/exports", + "tables": [{ + "name": "sponsorTimes", + "order": "timeSubmitted" + }, + { + "name": "userNames" + }, + { + "name": "categoryVotes" + }, + { + "name": "noSegments", + }, + { + "name": "warnings", + "order": "issueTime" + }, + { + "name": "vipUsers" + }] } } diff --git a/src/config.ts b/src/config.ts index 93c6a43..477aa72 100644 --- a/src/config.ts +++ b/src/config.ts @@ -45,7 +45,32 @@ addDefaults(config, { }, userCounterURL: null, youtubeAPIKey: null, - postgres: null + postgres: null, + dumpDatabase: { + enabled: true, + minTimeBetweenMs: 60000, + exportPath: '/opt/exports', + tables: [{ + name: "sponsorTimes", + order: "timeSubmitted" + }, + { + name: "userNames" + }, + { + name: "categoryVotes" + }, + { + name: "noSegments", + }, + { + name: "warnings", + order: "issueTime" + }, + { + name: "vipUsers" + }] + } }); // Add defaults diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index d687f07..ec3e801 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -11,26 +11,13 @@ const licenseHeader = `

The API and database follow Attribution Template

If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.

`; -const tables = [{ - name: "sponsorTimes", - order: "timeSubmitted" -}, -{ - name: "userNames" -}, -{ - name: "categoryVotes" -}, -{ - name: "noSegments", -}, -{ - name: "warnings", - order: "issueTime" -}, -{ - name: "vipUsers" -}]; +const tables = config?.dumpDatabase?.tables ?? []; +const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE; +const exportPath = config?.dumpDatabase?.exportPath ?? '/opt/exports'; + +if (tables.length === 0) { + Logger.warn('[dumpDatabase] No tables configured'); +} const links: string[] = tables.map((table) => `/database/${table.name}.csv`); @@ -40,13 +27,17 @@ const linksHTML: string = tables.map((table) => `

ONE_MINUTE; + const updateQueued = now - lastUpdate > MILLISECONDS_BETWEEN_DUMPS; res.status(200) @@ -72,7 +63,7 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool for (const table of tables) { db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) - TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); + TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); } } -} \ No newline at end of file +} From 514ea03655f043fa8a65b1f1666b3ffea91247b8 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Sun, 21 Mar 2021 22:59:16 +0100 Subject: [PATCH 2/5] Add TS for dumpDatabase config --- src/types/config.model.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/types/config.model.ts b/src/types/config.model.ts index b27cfaa..c0611b7 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -38,6 +38,7 @@ export interface SBSConfig { maximumPrefix?: string; redis?: redis.ClientOpts; postgres?: PoolConfig; + dumpDatabase?: DumpDatabase; } export interface WebhookConfig { @@ -61,4 +62,16 @@ export interface PostgresConfig { createDbIfNotExists: boolean; enableWalCheckpointNumber: boolean; postgres: PoolConfig; -} \ No newline at end of file +} + +export interface DumpDatabase { + enabled: boolean; + minTimeBetweenMs: number; + exportPath: string; + tables: DumpDatabaseTable[]; +} + +export interface DumpDatabaseTable { + name: string; + order?: string; +} From 8219b0398e43230bdbefaeaf22302943f508dfbb Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 22 Mar 2021 01:35:45 +0100 Subject: [PATCH 3/5] Fix invalid json in example config --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index d859e4d..65f9b42 100644 --- a/config.json.example +++ b/config.json.example @@ -54,7 +54,7 @@ "name": "categoryVotes" }, { - "name": "noSegments", + "name": "noSegments" }, { "name": "warnings", From 2c3dde0d2ed1beb80197bfb60f5f55b6f46c69c5 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 22 Mar 2021 22:18:23 +0100 Subject: [PATCH 4/5] Timestamp based dump filenames and garbage collection --- config.json.example | 3 +- src/config.ts | 3 +- src/routes/dumpDatabase.ts | 120 ++++++++++++++++++++++++++++++++++--- src/types/config.model.ts | 3 +- 4 files changed, 119 insertions(+), 10 deletions(-) diff --git a/config.json.example b/config.json.example index 65f9b42..580b8fb 100644 --- a/config.json.example +++ b/config.json.example @@ -42,7 +42,8 @@ "dumpDatabase": { "enabled": true, "minTimeBetweenMs": 60000, // 1 minute between dumps - "exportPath": "/opt/exports", + "appExportPath": "/opt/exports", + "postgresExportPath": "/opt/exports", "tables": [{ "name": "sponsorTimes", "order": "timeSubmitted" diff --git a/src/config.ts b/src/config.ts index 477aa72..0e985dc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,7 +49,8 @@ addDefaults(config, { dumpDatabase: { enabled: true, minTimeBetweenMs: 60000, - exportPath: '/opt/exports', + appExportPath: '/opt/exports', + postgresExportPath: '/opt/exports', tables: [{ name: "sponsorTimes", order: "timeSubmitted" diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index ec3e801..b7f2726 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -2,10 +2,29 @@ import {db} from '../databases/databases'; import {Logger} from '../utils/logger'; import {Request, Response} from 'express'; import { config } from '../config'; +const util = require('util'); +const fs = require('fs'); +const path = require('path'); +const unlink = util.promisify(fs.unlink); +const fstat = util.promisify(fs.fstat); const ONE_MINUTE = 1000 * 60; -const styleHeader = `` +const styleHeader = `` const licenseHeader = `

The API and database follow CC BY-NC-SA 4.0 unless you have explicit permission.

Attribution Template

@@ -13,7 +32,15 @@ const licenseHeader = `

The API and database follow { + return new Promise((resolve, reject) => { + // Get list of table names + // Create array for each table + const tableFiles = tableNames.reduce((obj: any, tableName) => { + obj[tableName] = []; + return obj; + }, {}); + // read files in export directory + fs.readdir(exportPath, (err: any, files: string[]) => { + if (err) Logger.error(err); + if (err) return resolve(); + files.forEach(file => { + // we only care about files that start with "_" and ends with .csv + tableNames.forEach(tableName => { + if (file.startsWith(`${tableName}_`) && file.endsWith('.csv')) { + // extract the timestamp from the filename + // we could also use the fs.stat mtime + const timestamp = Number(file.split('_')[1].replace('.csv', '')); + tableFiles[tableName].push({ + file: path.join(exportPath, file), + timestamp, + }); + } + }); + }); + const outdatedTime = Math.floor(Date.now() - (MILLISECONDS_BETWEEN_DUMPS * 1.5)); + for (let tableName in tableFiles) { + const files = tableFiles[tableName]; + files.forEach(async (item: any) => { + if (item.timestamp < outdatedTime) { + // remove old file + await unlink(item.file).catch((error: any) => { + Logger.error(`[dumpDatabase] Garbage collection failed ${error}`); + }); + } + }); + } + resolve(); + }); + }); +} + +export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) { if (config?.dumpDatabase?.enabled === false) { res.status(404).send("Database dump is disabled"); return; @@ -48,22 +118,58 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool Send a request to https://sponsor.ajay.app/database.json, or visit this page to trigger the database dump to run. Then, you can download the csv files below, or use the links returned from the JSON request.

Links

- ${linksHTML}
+ + + + + + + + + ${latestDumpFiles.map((item:any) => { + return ` + + + + + `; + }).join('')} + ${latestDumpFiles.length === 0 ? '' : ''} + +
TableCSV
${item.tableName}${item.fileName}
Please wait: Generating files
+
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); } else { res.send({ lastUpdated: lastUpdate, updateQueued, - links + links: latestDumpFiles.map((item:any) => { + return { + table: item.tableName, + url: `/download/${item.fileName}`, + size: item.fileSize, + }; + }), }) } if (updateQueued) { lastUpdate = Date.now(); + + await removeOutdatedDumps(appExportPath); + + const dumpFiles = []; for (const table of tables) { - db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) - TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); + const fileName = `${table.name}_${lastUpdate}.csv`; + const file = `${postgresExportPath}/${fileName}`; + await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) + TO '${file}' WITH (FORMAT CSV, HEADER true);`); + dumpFiles.push({ + fileName, + tableName: table.name, + }); } + latestDumpFiles = [...dumpFiles]; } } diff --git a/src/types/config.model.ts b/src/types/config.model.ts index c0611b7..f46cc17 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -67,7 +67,8 @@ export interface PostgresConfig { export interface DumpDatabase { enabled: boolean; minTimeBetweenMs: number; - exportPath: string; + appExportPath: string; + postgresExportPath: string; tables: DumpDatabaseTable[]; } From a06ab724adab93c082cb7a972c859768304b8c24 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 17 Apr 2021 23:06:39 -0400 Subject: [PATCH 5/5] Fix file locations + formatting --- src/config.ts | 2 +- src/routes/dumpDatabase.ts | 28 +++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/config.ts b/src/config.ts index 0e985dc..6139a6e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,7 +49,7 @@ addDefaults(config, { dumpDatabase: { enabled: true, minTimeBetweenMs: 60000, - appExportPath: '/opt/exports', + appExportPath: './docker/database-export', postgresExportPath: '/opt/exports', tables: [{ name: "sponsorTimes", diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index b7f2726..8e33440 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -2,11 +2,10 @@ import {db} from '../databases/databases'; import {Logger} from '../utils/logger'; import {Request, Response} from 'express'; import { config } from '../config'; -const util = require('util'); -const fs = require('fs'); -const path = require('path'); +import util from 'util'; +import fs from 'fs'; +import path from 'path'; const unlink = util.promisify(fs.unlink); -const fstat = util.promisify(fs.fstat); const ONE_MINUTE = 1000 * 60; @@ -16,7 +15,7 @@ const styleHeader = `