Make dumpDatabase configurable

This commit is contained in:
Nanobyte
2021-03-21 22:40:57 +01:00
parent fcd7be632c
commit 84b86bb6a1
3 changed files with 65 additions and 24 deletions

View File

@@ -38,5 +38,30 @@
"max": 20, // 20 requests in 15min time window
"statusCode": 200
}
},
"dumpDatabase": {
"enabled": true,
"minTimeBetweenMs": 60000, // 1 minute between dumps
"exportPath": "/opt/exports",
"tables": [{
"name": "sponsorTimes",
"order": "timeSubmitted"
},
{
"name": "userNames"
},
{
"name": "categoryVotes"
},
{
"name": "noSegments",
},
{
"name": "warnings",
"order": "issueTime"
},
{
"name": "vipUsers"
}]
}
}

View File

@@ -45,7 +45,32 @@ addDefaults(config, {
},
userCounterURL: null,
youtubeAPIKey: null,
postgres: null
postgres: null,
dumpDatabase: {
enabled: true,
minTimeBetweenMs: 60000,
exportPath: '/opt/exports',
tables: [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "noSegments",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}]
}
});
// Add defaults

View File

@@ -11,26 +11,13 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
const tables = [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "noSegments",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}];
const tables = config?.dumpDatabase?.tables ?? [];
const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
const exportPath = config?.dumpDatabase?.exportPath ?? '/opt/exports';
if (tables.length === 0) {
Logger.warn('[dumpDatabase] No tables configured');
}
const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
@@ -40,13 +27,17 @@ const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.n
let lastUpdate = 0;
export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
if (config?.dumpDatabase?.enabled === false) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
res.status(404).send("Not supported on this instance");
return;
}
const now = Date.now();
const updateQueued = now - lastUpdate > ONE_MINUTE;
const updateQueued = now - lastUpdate > MILLISECONDS_BETWEEN_DUMPS;
res.status(200)
@@ -72,7 +63,7 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
for (const table of tables) {
db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
TO '${exportPath}/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
}
}
}
}