From eb2d76a4b735a678836b61f619db0fc59d292362 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Tue, 20 May 2025 16:39:47 +0530 Subject: [PATCH] refactor: migrate database handling to Bun's SQLite and ensure data directory exists --- scripts/manage-db.ts | 397 ++++++++++++++++++++----------------------- src/lib/db/index.ts | 19 ++- 2 files changed, 198 insertions(+), 218 deletions(-) diff --git a/scripts/manage-db.ts b/scripts/manage-db.ts index 4827773..33f24e7 100644 --- a/scripts/manage-db.ts +++ b/scripts/manage-db.ts @@ -1,7 +1,6 @@ import fs from "fs"; import path from "path"; -import { client, db } from "../src/lib/db"; -import { configs } from "../src/lib/db"; +import { Database } from "bun:sqlite"; import { v4 as uuidv4 } from "uuid"; // Command line arguments @@ -21,13 +20,15 @@ const dataDbFile = path.join(dataDir, "gitea-mirror.db"); const dataDevDbFile = path.join(dataDir, "gitea-mirror-dev.db"); // Database path - ensure we use absolute path -const dbPath = - process.env.DATABASE_URL || `file:${path.join(dataDir, "gitea-mirror.db")}`; +const dbPath = path.join(dataDir, "gitea-mirror.db"); /** * Ensure all required tables exist */ async function ensureTablesExist() { + // Create or open the database + const db = new Database(dbPath); + const requiredTables = [ "users", "configs", @@ -38,44 +39,46 @@ async function ensureTablesExist() { for (const table of requiredTables) { try { - await client.execute(`SELECT 1 FROM ${table} LIMIT 1`); - } catch (error) { - if (error instanceof Error && error.message.includes("SQLITE_ERROR")) { + // Check if table exists + const result = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='${table}'`).get(); + + if (!result) { console.warn(`⚠️ Table '${table}' is missing. Creating it now...`); + switch (table) { case "users": - await client.execute( - `CREATE TABLE users ( + db.exec(` + CREATE TABLE users ( id TEXT PRIMARY KEY, username TEXT NOT NULL, password TEXT NOT NULL, email TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL - )` - ); + ) + `); break; case "configs": - await client.execute( - `CREATE TABLE configs ( + db.exec(` + CREATE TABLE configs ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, name TEXT NOT NULL, is_active INTEGER NOT NULL DEFAULT 1, github_config TEXT NOT NULL, gitea_config TEXT NOT NULL, - include TEXT NOT NULL DEFAULT '[]', + include TEXT NOT NULL DEFAULT '["*"]', exclude TEXT NOT NULL DEFAULT '[]', schedule_config TEXT NOT NULL, created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), FOREIGN KEY (user_id) REFERENCES users(id) - )` - ); + ) + `); break; case "repositories": - await client.execute( - `CREATE TABLE repositories ( + db.exec(` + CREATE TABLE repositories ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, config_id TEXT NOT NULL, @@ -104,12 +107,12 @@ async function ensureTablesExist() { updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), FOREIGN KEY (user_id) REFERENCES users(id), FOREIGN KEY (config_id) REFERENCES configs(id) - )` - ); + ) + `); break; case "organizations": - await client.execute( - `CREATE TABLE organizations ( + db.exec(` + CREATE TABLE organizations ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, config_id TEXT NOT NULL, @@ -125,12 +128,12 @@ async function ensureTablesExist() { updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), FOREIGN KEY (user_id) REFERENCES users(id), FOREIGN KEY (config_id) REFERENCES configs(id) - )` - ); + ) + `); break; case "mirror_jobs": - await client.execute( - `CREATE TABLE mirror_jobs ( + db.exec(` + CREATE TABLE mirror_jobs ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, repository_id TEXT, @@ -142,15 +145,15 @@ async function ensureTablesExist() { message TEXT NOT NULL, timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (user_id) REFERENCES users(id) - )` - ); + ) + `); break; } console.log(`✅ Table '${table}' created successfully.`); - } else { - console.error(`❌ Error checking table '${table}':`, error); - process.exit(1); } + } catch (error) { + console.error(`❌ Error checking table '${table}':`, error); + process.exit(1); } } } @@ -180,10 +183,11 @@ async function checkDatabase() { // Check for users try { - const userCountResult = await client.execute( - `SELECT COUNT(*) as count FROM users` - ); - const userCount = userCountResult.rows[0].count; + const db = new Database(dbPath); + + // Check for users + const userCountResult = db.query(`SELECT COUNT(*) as count FROM users`).get(); + const userCount = userCountResult?.count || 0; if (userCount === 0) { console.log("ℹ️ No users found in the database."); @@ -197,10 +201,8 @@ async function checkDatabase() { } // Check for configurations - const configCountResult = await client.execute( - `SELECT COUNT(*) as count FROM configs` - ); - const configCount = configCountResult.rows[0].count; + const configCountResult = db.query(`SELECT COUNT(*) as count FROM configs`).get(); + const configCount = configCountResult?.count || 0; if (configCount === 0) { console.log("ℹ️ No configurations found in the database."); @@ -243,7 +245,8 @@ async function initializeDatabase() { // Check if we can connect to it try { - await client.execute(`SELECT COUNT(*) as count FROM users`); + const db = new Database(dbPath); + db.query(`SELECT COUNT(*) as count FROM users`).get(); console.log("✅ Database is valid and accessible."); return; } catch (error) { @@ -257,135 +260,118 @@ async function initializeDatabase() { console.log(`Initializing database at ${dbPath}...`); try { + const db = new Database(dbPath); + // Create tables if they don't exist - await client.execute( - `CREATE TABLE IF NOT EXISTS users ( + db.exec(` + CREATE TABLE IF NOT EXISTS users ( id TEXT PRIMARY KEY, username TEXT NOT NULL, password TEXT NOT NULL, email TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL - )` - ); + ) + `); // NOTE: We no longer create a default admin user - user will create one via signup page - await client.execute( - `CREATE TABLE IF NOT EXISTS configs ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, - name TEXT NOT NULL, - is_active INTEGER NOT NULL DEFAULT 1, - github_config TEXT NOT NULL, - gitea_config TEXT NOT NULL, - include TEXT NOT NULL DEFAULT '["*"]', - exclude TEXT NOT NULL DEFAULT '[]', - schedule_config TEXT NOT NULL, - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - FOREIGN KEY (user_id) REFERENCES users(id) -); -` - ); + db.exec(` + CREATE TABLE IF NOT EXISTS configs ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + github_config TEXT NOT NULL, + gitea_config TEXT NOT NULL, + include TEXT NOT NULL DEFAULT '["*"]', + exclude TEXT NOT NULL DEFAULT '[]', + schedule_config TEXT NOT NULL, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); - await client.execute( - `CREATE TABLE IF NOT EXISTS repositories ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, - config_id TEXT NOT NULL, - name TEXT NOT NULL, - full_name TEXT NOT NULL, - url TEXT NOT NULL, - clone_url TEXT NOT NULL, - owner TEXT NOT NULL, - organization TEXT, - mirrored_location TEXT DEFAULT '', + db.exec(` + CREATE TABLE IF NOT EXISTS repositories ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + config_id TEXT NOT NULL, + name TEXT NOT NULL, + full_name TEXT NOT NULL, + url TEXT NOT NULL, + clone_url TEXT NOT NULL, + owner TEXT NOT NULL, + organization TEXT, + mirrored_location TEXT DEFAULT '', + is_private INTEGER NOT NULL DEFAULT 0, + is_fork INTEGER NOT NULL DEFAULT 0, + forked_from TEXT, + has_issues INTEGER NOT NULL DEFAULT 0, + is_starred INTEGER NOT NULL DEFAULT 0, + is_archived INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL DEFAULT 0, + has_lfs INTEGER NOT NULL DEFAULT 0, + has_submodules INTEGER NOT NULL DEFAULT 0, + default_branch TEXT NOT NULL, + visibility TEXT NOT NULL DEFAULT 'public', + status TEXT NOT NULL DEFAULT 'imported', + last_mirrored INTEGER, + error_message TEXT, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + FOREIGN KEY (user_id) REFERENCES users(id), + FOREIGN KEY (config_id) REFERENCES configs(id) + ) + `); - is_private INTEGER NOT NULL DEFAULT 0, - is_fork INTEGER NOT NULL DEFAULT 0, - forked_from TEXT, + db.exec(` + CREATE TABLE IF NOT EXISTS organizations ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + config_id TEXT NOT NULL, + name TEXT NOT NULL, + avatar_url TEXT NOT NULL, + membership_role TEXT NOT NULL DEFAULT 'member', + is_included INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'imported', + last_mirrored INTEGER, + error_message TEXT, + repository_count INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + FOREIGN KEY (user_id) REFERENCES users(id), + FOREIGN KEY (config_id) REFERENCES configs(id) + ) + `); - has_issues INTEGER NOT NULL DEFAULT 0, - is_starred INTEGER NOT NULL DEFAULT 0, - is_archived INTEGER NOT NULL DEFAULT 0, - - size INTEGER NOT NULL DEFAULT 0, - has_lfs INTEGER NOT NULL DEFAULT 0, - has_submodules INTEGER NOT NULL DEFAULT 0, - - default_branch TEXT NOT NULL, - visibility TEXT NOT NULL DEFAULT 'public', - - status TEXT NOT NULL DEFAULT 'imported', - last_mirrored INTEGER, - error_message TEXT, - - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - - FOREIGN KEY (user_id) REFERENCES users(id), - FOREIGN KEY (config_id) REFERENCES configs(id) -); -` - ); - - await client.execute( - `CREATE TABLE IF NOT EXISTS organizations ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, - config_id TEXT NOT NULL, - name TEXT NOT NULL, - - avatar_url TEXT NOT NULL, - membership_role TEXT NOT NULL DEFAULT 'member', - - is_included INTEGER NOT NULL DEFAULT 1, - - status TEXT NOT NULL DEFAULT 'imported', - last_mirrored INTEGER, - error_message TEXT, - - repository_count INTEGER NOT NULL DEFAULT 0, - - created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), - - FOREIGN KEY (user_id) REFERENCES users(id), - FOREIGN KEY (config_id) REFERENCES configs(id) -); -` - ); - - await client.execute( - `CREATE TABLE IF NOT EXISTS mirror_jobs ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, - repository_id TEXT, - repository_name TEXT, - organization_id TEXT, - organization_name TEXT, - details TEXT, - status TEXT NOT NULL DEFAULT 'imported', - message TEXT NOT NULL, - timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (user_id) REFERENCES users(id) -); -` - ); + db.exec(` + CREATE TABLE IF NOT EXISTS mirror_jobs ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + repository_id TEXT, + repository_name TEXT, + organization_id TEXT, + organization_name TEXT, + details TEXT, + status TEXT NOT NULL DEFAULT 'imported', + message TEXT NOT NULL, + timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); // Insert default config if none exists - const configCountResult = await client.execute( - `SELECT COUNT(*) as count FROM configs` - ); - const configCount = configCountResult.rows[0].count; + const configCountResult = db.query(`SELECT COUNT(*) as count FROM configs`).get(); + const configCount = configCountResult?.count || 0; + if (configCount === 0) { // Get the first user - const firstUserResult = await client.execute( - `SELECT id FROM users LIMIT 1` - ); - if (firstUserResult.rows.length > 0) { - const userId = firstUserResult.rows[0].id; + const firstUserResult = db.query(`SELECT id FROM users LIMIT 1`).get(); + + if (firstUserResult) { + const userId = firstUserResult.id; const configId = uuidv4(); const githubConfig = JSON.stringify({ username: process.env.GITHUB_USERNAME || "", @@ -415,24 +401,23 @@ async function initializeDatabase() { nextRun: null, }); - await client.execute( - ` + const stmt = db.prepare(` INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `, - [ - configId, - userId, - "Default Configuration", - 1, - githubConfig, - giteaConfig, - include, - exclude, - scheduleConfig, - Date.now(), - Date.now(), - ] + `); + + stmt.run( + configId, + userId, + "Default Configuration", + 1, + githubConfig, + giteaConfig, + include, + exclude, + scheduleConfig, + Date.now(), + Date.now() ); } } @@ -452,8 +437,7 @@ async function resetUsers() { try { // Check if the database exists - const dbFilePath = dbPath.replace("file:", ""); - const doesDbExist = fs.existsSync(dbFilePath); + const doesDbExist = fs.existsSync(dbPath); if (!doesDbExist) { console.log( @@ -462,11 +446,11 @@ async function resetUsers() { return; } + const db = new Database(dbPath); + // Count existing users - const userCountResult = await client.execute( - `SELECT COUNT(*) as count FROM users` - ); - const userCount = userCountResult.rows[0].count; + const userCountResult = db.query(`SELECT COUNT(*) as count FROM users`).get(); + const userCount = userCountResult?.count || 0; if (userCount === 0) { console.log("ℹ️ No users found in the database. Nothing to reset."); @@ -474,63 +458,43 @@ async function resetUsers() { } // Delete all users - await client.execute(`DELETE FROM users`); + db.exec(`DELETE FROM users`); console.log(`✅ Deleted ${userCount} users from the database.`); // Check dependent configurations that need to be removed - const configCount = await client.execute( - `SELECT COUNT(*) as count FROM configs` - ); + const configCountResult = db.query(`SELECT COUNT(*) as count FROM configs`).get(); + const configCount = configCountResult?.count || 0; - if ( - configCount.rows && - configCount.rows[0] && - Number(configCount.rows[0].count) > 0 - ) { - await client.execute(`DELETE FROM configs`); - console.log(`✅ Deleted ${configCount.rows[0].count} configurations.`); + if (configCount > 0) { + db.exec(`DELETE FROM configs`); + console.log(`✅ Deleted ${configCount} configurations.`); } // Check for dependent repositories - const repoCount = await client.execute( - `SELECT COUNT(*) as count FROM repositories` - ); + const repoCountResult = db.query(`SELECT COUNT(*) as count FROM repositories`).get(); + const repoCount = repoCountResult?.count || 0; - if ( - repoCount.rows && - repoCount.rows[0] && - Number(repoCount.rows[0].count) > 0 - ) { - await client.execute(`DELETE FROM repositories`); - console.log(`✅ Deleted ${repoCount.rows[0].count} repositories.`); + if (repoCount > 0) { + db.exec(`DELETE FROM repositories`); + console.log(`✅ Deleted ${repoCount} repositories.`); } // Check for dependent organizations - const orgCount = await client.execute( - `SELECT COUNT(*) as count FROM organizations` - ); + const orgCountResult = db.query(`SELECT COUNT(*) as count FROM organizations`).get(); + const orgCount = orgCountResult?.count || 0; - if ( - orgCount.rows && - orgCount.rows[0] && - Number(orgCount.rows[0].count) > 0 - ) { - await client.execute(`DELETE FROM organizations`); - console.log(`✅ Deleted ${orgCount.rows[0].count} organizations.`); + if (orgCount > 0) { + db.exec(`DELETE FROM organizations`); + console.log(`✅ Deleted ${orgCount} organizations.`); } // Check for dependent mirror jobs - const jobCount = await client.execute( - `SELECT COUNT(*) as count FROM mirror_jobs` - ); + const jobCountResult = db.query(`SELECT COUNT(*) as count FROM mirror_jobs`).get(); + const jobCount = jobCountResult?.count || 0; - if ( - jobCount.rows && - jobCount.rows[0] && - Number(jobCount.rows[0].count) > 0 - ) { - await client.execute(`DELETE FROM mirror_jobs`); - console.log(`✅ Deleted ${jobCount.rows[0].count} mirror jobs.`); + if (jobCount > 0) { + db.exec(`DELETE FROM mirror_jobs`); + console.log(`✅ Deleted ${jobCount} mirror jobs.`); } console.log( @@ -636,7 +600,8 @@ async function fixDatabaseIssues() { // Check if we can connect to the database try { // Try to query the database - await db.select().from(configs).limit(1); + const db = new Database(dbPath); + db.query(`SELECT 1 FROM sqlite_master LIMIT 1`).get(); console.log(`✅ Successfully connected to the database.`); } catch (error) { console.error("❌ Error connecting to the database:", error); diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index bb9307f..3151549 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -2,17 +2,32 @@ import { z } from "zod"; import { Database } from "bun:sqlite"; import { drizzle } from "drizzle-orm/bun-sqlite"; import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core"; - +import fs from "fs"; import path from "path"; import { configSchema } from "./schema"; // Define the database URL - for development we'll use a local SQLite file const dataDir = path.join(process.cwd(), "data"); +// Ensure data directory exists +if (!fs.existsSync(dataDir)) { + fs.mkdirSync(dataDir, { recursive: true }); +} + const dbUrl = process.env.DATABASE_URL || `file:${path.join(dataDir, "gitea-mirror.db")}`; // Create a SQLite database instance using Bun's native driver -export const sqlite = new Database(dbUrl); +let sqlite: Database; +try { + // Create an empty database file if it doesn't exist + if (!fs.existsSync(path.join(dataDir, "gitea-mirror.db"))) { + fs.writeFileSync(path.join(dataDir, "gitea-mirror.db"), ""); + } + sqlite = new Database(dbUrl); +} catch (error) { + console.error("Error opening database:", error); + throw error; +} // Simple async wrapper around Bun's SQLite API for compatibility export const client = {