diff --git a/Dockerfile b/Dockerfile index 2fc664d..26b77cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN npm ci && npm run tsc FROM node:16-alpine as app WORKDIR /usr/src/app -RUN apk add git +RUN apk add git postgresql-client COPY --from=builder ./node_modules ./node_modules COPY --from=builder ./dist ./dist COPY ./.git ./.git diff --git a/src/app.ts b/src/app.ts index 9522c87..77f1c2b 100644 --- a/src/app.ts +++ b/src/app.ts @@ -27,7 +27,7 @@ import { loggerMiddleware } from "./middleware/logger"; import { corsMiddleware } from "./middleware/cors"; import { apiCspMiddleware } from "./middleware/apiCsp"; import { rateLimitMiddleware } from "./middleware/requestRateLimit"; -import dumpDatabase, { appExportPath, redirectLink } from "./routes/dumpDatabase"; +import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase"; import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo"; import { postClearCache } from "./routes/postClearCache"; import { addUnlistedVideo } from "./routes/addUnlistedVideo"; @@ -205,7 +205,7 @@ function setupRoutes(router: Router) { if (config.postgres?.enabled) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); router.get("/database.json", (req, res) => dumpDatabase(req, res, false)); - router.get("/database/*", redirectLink); + router.get("/database/*", downloadFile); router.use("/download", express.static(appExportPath)); } else { router.get("/database.db", function (req: Request, res: Response) { diff --git a/src/config.ts b/src/config.ts index cbfbf6b..5e51e81 100644 --- a/src/config.ts +++ b/src/config.ts @@ -82,7 +82,6 @@ addDefaults(config, { enabled: false, minTimeBetweenMs: 180000, appExportPath: "./docker/database-export", - postgresExportPath: "/opt/exports", tables: [{ name: "sponsorTimes", order: "timeSubmitted" diff --git a/src/databases/databases.ts b/src/databases/databases.ts index 9ff8742..52f357c 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -72,7 +72,7 @@ async function initDb(): Promise { const tables = config?.dumpDatabase?.tables ?? []; const tableNames = tables.map(table => table.name); for (const table of tableNames) { - const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`; + const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`; await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); } } diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index 3c39857..73557ad 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -5,6 +5,7 @@ import { config } from "../config"; import util from "util"; import fs from "fs"; import path from "path"; +import { ChildProcess, exec, ExecOptions, spawn } from "child_process"; const unlink = util.promisify(fs.unlink); const ONE_MINUTE = 1000 * 60; @@ -32,9 +33,19 @@ const licenseHeader = `

The API and database follow table.name); +const credentials: ExecOptions = { + env: { + ...process.env, + PGHOST: config.postgres.host, + PGPORT: String(config.postgres.port), + PGUSER: config.postgres.user, + PGPASSWORD: String(config.postgres.password), + PGDATABASE: "sponsorTimes", + } +} + interface TableDumpList { fileName: string; tableName: string; @@ -170,7 +181,7 @@ async function getDbVersion(): Promise { return row.value; } -export async function redirectLink(req: Request, res: Response): Promise { +export async function downloadFile(req: Request, res: Response): Promise { if (!config?.dumpDatabase?.enabled) { res.status(404).send("Database dump is disabled"); return; @@ -183,7 +194,7 @@ export async function redirectLink(req: Request, res: Response): Promise { const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path); if (file) { - res.redirect(`/download/${file.fileName}`); + res.sendFile(file.fileName, { root: appExportPath }); } else { res.sendStatus(404); } @@ -210,9 +221,19 @@ async function queueDump(): Promise { for (const table of tables) { const fileName = `${table.name}_${startTime}.csv`; - const file = `${postgresExportPath}/${fileName}`; - await db.prepare("run", `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) - TO '${file}' WITH (FORMAT CSV, HEADER true);`); + const file = `${appExportPath}/${fileName}`; + + await new Promise((resolve) => { + exec(`psql -c "\\copy (SELECT * FROM \\"${table.name}\\"${table.order ? ` ORDER BY \\"${table.order}\\"` : ``})` + + ` TO '${file}' WITH (FORMAT CSV, HEADER true);"`, credentials, (error, stdout, stderr) => { + if (error) { + Logger.error(`[dumpDatabase] Failed to dump ${table.name} to ${file} due to ${stderr}`); + } + + resolve(error ? stderr : stdout); + }); + }) + dumpFiles.push({ fileName, tableName: table.name, diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 0b22508..2fe08f6 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -84,7 +84,6 @@ export interface DumpDatabase { enabled: boolean; minTimeBetweenMs: number; appExportPath: string; - postgresExportPath: string; tables: DumpDatabaseTable[]; }