diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 9856d28..4ea56af 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -1,39 +1,43 @@ BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "vipUsers" ( - "userID" TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "category" TEXT NOT NULL, - "shadowHidden" INTEGER NOT NULL -); -CREATE TABLE IF NOT EXISTS "userNames" ( - "userID" TEXT NOT NULL, - "userName" TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS "categoryVotes" ( - "UUID" TEXT NOT NULL, - "category" TEXT NOT NULL, - "votes" INTEGER NOT NULL default '0' +CREATE TABLE IF NOT EXISTS vipUsers ( + userID TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "config" ( - "key" TEXT NOT NULL UNIQUE, - "value" TEXT NOT NULL +COMMIT; + +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS sponsorTimes ( + videoID TEXT NOT NULL, + startTime REAL NOT NULL, + endTime REAL NOT NULL, + votes INTEGER NOT NULL, + UUID TEXT NOT NULL UNIQUE, + userID TEXT NOT NULL, + timeSubmitted INTEGER NOT NULL, + views INTEGER NOT NULL, + category TEXT NOT NULL, + shadowHidden INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS userNames ( + userID TEXT NOT NULL, + userName TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS categoryVotes ( + UUID TEXT NOT NULL, + category TEXT NOT NULL, + votes INTEGER NOT NULL default 0 +); + +CREATE TABLE IF NOT EXISTS config ( + key TEXT NOT NULL UNIQUE, + value TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID); CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID); - CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_1.sql b/databases/_upgrade_sponsorTimes_1.sql index 5629cc8..c950d08 100644 --- a/databases/_upgrade_sponsorTimes_1.sql +++ b/databases/_upgrade_sponsorTimes_1.sql @@ -6,20 +6,20 @@ CREATE TABLE "sqlb_temp_table_1" ( "startTime" REAL NOT NULL, "endTime" REAL NOT NULL, "votes" INTEGER NOT NULL, - "incorrectVotes" INTEGER NOT NULL default '1', + "incorrectVotes" INTEGER NOT NULL default 1, "UUID" TEXT NOT NULL UNIQUE, "userID" TEXT NOT NULL, "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, - "category" TEXT NOT NULL DEFAULT "sponsor", + "category" TEXT NOT NULL DEFAULT 'sponsor', "shadowHidden" INTEGER NOT NULL ); -INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes; +INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,'1',UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes; DROP TABLE sponsorTimes; ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; /* Add version to config */ -INSERT INTO config (key, value) VALUES("version", 1); +INSERT INTO config (key, value) VALUES('version', 1); COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_3.sql b/databases/_upgrade_sponsorTimes_3.sql index 5480ec2..ae3ac7e 100644 --- a/databases/_upgrade_sponsorTimes_3.sql +++ b/databases/_upgrade_sponsorTimes_3.sql @@ -1,13 +1,13 @@ BEGIN TRANSACTION; -/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' +/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */ /* Add hash field */ -ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default ""; +ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default ''; UPDATE sponsorTimes SET hashedVideoID = sha256(videoID); CREATE INDEX IF NOT EXISTS sponsorTimes_hashedVideoID on sponsorTimes(hashedVideoID); /* Bump version in config */ -UPDATE config SET value = 3 WHERE key = "version"; +UPDATE config SET value = 3 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_4.sql b/databases/_upgrade_sponsorTimes_4.sql index 91e51e2..09948a5 100644 --- a/databases/_upgrade_sponsorTimes_4.sql +++ b/databases/_upgrade_sponsorTimes_4.sql @@ -7,6 +7,6 @@ CREATE TABLE "warnings" ( issuerUserID TEXT NOT NULL ); -UPDATE config SET value = 4 WHERE key = "version"; +UPDATE config SET value = 4 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_5.sql b/databases/_upgrade_sponsorTimes_5.sql index 1132af2..cc85949 100644 --- a/databases/_upgrade_sponsorTimes_5.sql +++ b/databases/_upgrade_sponsorTimes_5.sql @@ -10,8 +10,8 @@ CREATE TABLE "sqlb_temp_table_5" ( INSERT INTO sqlb_temp_table_5 SELECT userID,issueTime,issuerUserID,1 FROM warnings; DROP TABLE warnings; -ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings"; +ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";; -UPDATE config SET value = 5 WHERE key = "version"; +UPDATE config SET value = 5 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_6.sql b/databases/_upgrade_sponsorTimes_6.sql index 3901602..1a1afbb 100644 --- a/databases/_upgrade_sponsorTimes_6.sql +++ b/databases/_upgrade_sponsorTimes_6.sql @@ -12,16 +12,16 @@ CREATE TABLE "sqlb_temp_table_6" ( "userID" TEXT NOT NULL, "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, - "category" TEXT NOT NULL DEFAULT "sponsor", + "category" TEXT NOT NULL DEFAULT 'sponsor', "shadowHidden" INTEGER NOT NULL, - "hashedVideoID" TEXT NOT NULL default "" + "hashedVideoID" TEXT NOT NULL default '' ); -INSERT INTO sqlb_temp_table_6 SELECT videoID,startTime,endTime,votes,"0",incorrectVotes,UUID,userID,timeSubmitted,views,category,shadowHidden,hashedVideoID FROM sponsorTimes; +INSERT INTO sqlb_temp_table_6 SELECT videoID,startTime,endTime,votes,'0',incorrectVotes,UUID,userID,timeSubmitted,views,category,shadowHidden,hashedVideoID FROM sponsorTimes; DROP TABLE sponsorTimes; ALTER TABLE sqlb_temp_table_6 RENAME TO "sponsorTimes"; -UPDATE config SET value = 6 WHERE key = "version"; +UPDATE config SET value = 6 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 3f790d1..dd69f2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -754,6 +754,11 @@ "is-obj": "^1.0.0" } }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", diff --git a/package.json b/package.json index 1cd2362..af27bfc 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "license": "MIT", "dependencies": { "better-sqlite3": "^5.4.3", + "dotenv": "^8.2.0", "express": "^4.17.1", "express-rate-limit": "^5.1.3", "http": "0.0.0", diff --git a/src/config.ts b/src/config.ts index 7a172cc..93c6a43 100644 --- a/src/config.ts +++ b/src/config.ts @@ -45,6 +45,7 @@ addDefaults(config, { }, userCounterURL: null, youtubeAPIKey: null, + postgres: null }); // Add defaults diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 1bb4e08..5e76cda 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -4,27 +4,25 @@ import { Pool } from 'pg'; import fs from "fs"; -export class Mysql implements IDatabase { +export class Postgres implements IDatabase { private pool: Pool; constructor(private config: any) {} - init(): void { - this.pool = new Pool(); + async init(): Promise { + this.pool = new Pool(this.config.postgres); if (!this.config.readOnly) { - // Upgrade database if required - this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder); - if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) { - this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString())); + await this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString())); } + + // Upgrade database if required + await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder); } } async prepare(type: QueryType, query: string, params?: any[]) { - Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); - // Convert query to use numbered parameters let count = 1; for (let char = 0; char < query.length; char++) { @@ -34,7 +32,9 @@ export class Mysql implements IDatabase { } } - const queryResult = await this.pool.query(query, params); + Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); + + const queryResult = await this.pool.query({text: query, values: params}); switch (type) { case 'get': { @@ -51,7 +51,7 @@ export class Mysql implements IDatabase { private async upgradeDB(fileNamePrefix: string, schemaFolder: string) { const versionCodeInfo = await this.pool.query("SELECT value FROM config WHERE key = 'version'"); - let versionCode = versionCodeInfo ? versionCodeInfo.rows[0].value : 0; + let versionCode = versionCodeInfo.rows[0] ? versionCodeInfo.rows[0].value : 0; let path = schemaFolder + "/_upgrade_" + fileNamePrefix + "_" + (parseInt(versionCode) + 1) + ".sql"; Logger.debug('db update: trying ' + path); @@ -67,7 +67,9 @@ export class Mysql implements IDatabase { } private processUpgradeQuery(query: string): string { - let result = query.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')"); + let result = query.toLocaleLowerCase(); + result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')"); + result = result.replace(/integer/gm, "numeric"); return result; } diff --git a/src/databases/databases.ts b/src/databases/databases.ts index ef75d0a..41c6b9e 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -1,6 +1,7 @@ import {config} from '../config'; import {Sqlite} from './Sqlite'; import {Mysql} from './Mysql'; +import {Postgres} from './Postgres'; import {IDatabase} from './IDatabase'; @@ -9,6 +10,26 @@ let privateDB: IDatabase; if (config.mysql) { db = new Mysql(config.mysql); privateDB = new Mysql(config.privateMysql); +} else if (config.postgres) { + db = new Postgres({ + dbSchemaFileName: config.dbSchema, + dbSchemaFolder: config.schemaFolder, + fileNamePrefix: 'sponsorTimes', + readOnly: config.readOnly, + createDbIfNotExists: config.createDatabaseIfNotExist, + enableWalCheckpointNumber: !config.readOnly && config.mode === "production", + postgres: config.postgres + }); + + privateDB = new Sqlite({ + dbPath: config.privateDB, + dbSchemaFileName: config.privateDBSchema, + dbSchemaFolder: config.schemaFolder, + fileNamePrefix: 'private', + readOnly: config.readOnly, + createDbIfNotExists: config.createDatabaseIfNotExist, + enableWalCheckpointNumber: false + }); } else { db = new Sqlite({ dbPath: config.db, diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 98d7178..f3e9665 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -52,11 +52,14 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C const segments: Segment[] = []; try { + categories.filter((category) => !/[^a-z|_|-]/.test(category)); + const segmentsByCategory: SBRecord = (await db .prepare( 'all', - `SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes WHERE videoID = ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`, - [videoID, categories] + `SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes + WHERE videoID = ? AND category IN (${categories.map((c) => "'" + c + "'")}) ORDER BY startTime`, + [videoID] )).reduce((acc: SBRecord, segment: DBSegment) => { acc[segment.category] = acc[segment.category] || []; acc[segment.category].push(segment); @@ -64,10 +67,15 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C return acc; }, {}); + console.log(segmentsByCategory) + + for (const [category, categorySegments] of Object.entries(segmentsByCategory)) { segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache))); } + console.log(segments) + return segments; } catch (err) { if (err) { @@ -84,11 +92,14 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, try { type SegmentWithHashPerVideoID = SBRecord}>; + categories.filter((category) => !/[^a-z|_|-]/.test(category)); + const segmentPerVideoID: SegmentWithHashPerVideoID = (await db .prepare( 'all', - `SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes WHERE hashedVideoID LIKE ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`, - [hashedVideoIDPrefix + '%', categories] + `SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes + WHERE hashedVideoID LIKE ? AND category IN (${categories.map((c) => "'" + c + "'")}) ORDER BY startTime`, + [hashedVideoIDPrefix + '%'] )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { hash: segment.hashedVideoID, diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 88af3d2..b27cfaa 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -1,3 +1,4 @@ +import { PoolConfig } from 'pg'; import * as redis from 'redis'; export interface SBSConfig { @@ -36,6 +37,7 @@ export interface SBSConfig { minimumPrefix?: string; maximumPrefix?: string; redis?: redis.ClientOpts; + postgres?: PoolConfig; } export interface WebhookConfig { @@ -50,3 +52,13 @@ export interface RateLimitConfig { message: string; statusCode: number; } + +export interface PostgresConfig { + dbSchemaFileName: string; + dbSchemaFolder: string; + fileNamePrefix: string; + readOnly: boolean; + createDbIfNotExists: boolean; + enableWalCheckpointNumber: boolean; + postgres: PoolConfig; +} \ No newline at end of file