Mirror/gitea-mirror
1
0
Fork 0
mirror of https://github.com/RayLabsHQ/gitea-mirror.git synced 2025-12-08 20:46:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor: migrate database handling to Bun's SQLite and ensure data directory exists

Browse Source
This commit is contained in:
Arunavo Ray
2025-05-20 16:39:47 +05:30
parent 145bee8d96
commit eb2d76a4b7
2 changed files with 198 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

397
scripts/manage-db.ts
View File

@@ -1,7 +1,6 @@
import fs from "fs"; import fs from "fs";
import path from "path"; import path from "path";
import { client, db } from "../src/lib/db"; import { Database } from "bun:sqlite";
import { configs } from "../src/lib/db";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
// Command line arguments // Command line arguments
@@ -21,13 +20,15 @@ const dataDbFile = path.join(dataDir, "gitea-mirror.db");
const dataDevDbFile = path.join(dataDir, "gitea-mirror-dev.db"); const dataDevDbFile = path.join(dataDir, "gitea-mirror-dev.db");
// Database path - ensure we use absolute path // Database path - ensure we use absolute path
const dbPath = const dbPath = path.join(dataDir, "gitea-mirror.db");
process.env.DATABASE_URL || `file:${path.join(dataDir, "gitea-mirror.db")}`;
/** /**
* Ensure all required tables exist * Ensure all required tables exist
*/ */
async function ensureTablesExist() { async function ensureTablesExist() {
// Create or open the database
const db = new Database(dbPath);
const requiredTables = [ const requiredTables = [
"users", "users",
"configs", "configs",
@@ -38,44 +39,46 @@ async function ensureTablesExist() {
for (const table of requiredTables) { for (const table of requiredTables) {
try { try {
await client.execute(`SELECT 1 FROM ${table} LIMIT 1`); // Check if table exists
} catch (error) { const result = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='${table}'`).get();
if (error instanceof Error && error.message.includes("SQLITE_ERROR")) {
if (!result) {
console.warn(`⚠️ Table '${table}' is missing. Creating it now...`); console.warn(`⚠️ Table '${table}' is missing. Creating it now...`);
switch (table) { switch (table) {
case "users": case "users":
await client.execute( db.exec(`
`CREATE TABLE users ( CREATE TABLE users (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
username TEXT NOT NULL, username TEXT NOT NULL,
password TEXT NOT NULL, password TEXT NOT NULL,
email TEXT NOT NULL, email TEXT NOT NULL,
created_at INTEGER NOT NULL, created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL updated_at INTEGER NOT NULL
)` )
); `);
break; break;
case "configs": case "configs":
await client.execute( db.exec(`
`CREATE TABLE configs ( CREATE TABLE configs (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
is_active INTEGER NOT NULL DEFAULT 1, is_active INTEGER NOT NULL DEFAULT 1,
github_config TEXT NOT NULL, github_config TEXT NOT NULL,
gitea_config TEXT NOT NULL, gitea_config TEXT NOT NULL,
include TEXT NOT NULL DEFAULT '[]', include TEXT NOT NULL DEFAULT '["*"]',
exclude TEXT NOT NULL DEFAULT '[]', exclude TEXT NOT NULL DEFAULT '[]',
schedule_config TEXT NOT NULL, schedule_config TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id) FOREIGN KEY (user_id) REFERENCES users(id)
)` )
); `);
break; break;
case "repositories": case "repositories":
await client.execute( db.exec(`
`CREATE TABLE repositories ( CREATE TABLE repositories (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
config_id TEXT NOT NULL, config_id TEXT NOT NULL,
@@ -104,12 +107,12 @@ async function ensureTablesExist() {
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id), FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (config_id) REFERENCES configs(id) FOREIGN KEY (config_id) REFERENCES configs(id)
)` )
); `);
break; break;
case "organizations": case "organizations":
await client.execute( db.exec(`
`CREATE TABLE organizations ( CREATE TABLE organizations (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
config_id TEXT NOT NULL, config_id TEXT NOT NULL,
@@ -125,12 +128,12 @@ async function ensureTablesExist() {
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id), FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (config_id) REFERENCES configs(id) FOREIGN KEY (config_id) REFERENCES configs(id)
)` )
); `);
break; break;
case "mirror_jobs": case "mirror_jobs":
await client.execute( db.exec(`
`CREATE TABLE mirror_jobs ( CREATE TABLE mirror_jobs (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
repository_id TEXT, repository_id TEXT,
@@ -142,15 +145,15 @@ async function ensureTablesExist() {
message TEXT NOT NULL, message TEXT NOT NULL,
timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id) FOREIGN KEY (user_id) REFERENCES users(id)
)` )
); `);
break; break;
} }
console.log(`✅ Table '${table}' created successfully.`); console.log(`✅ Table '${table}' created successfully.`);
} else {
console.error(`❌ Error checking table '${table}':`, error);
process.exit(1);
} }
} catch (error) {
console.error(`❌ Error checking table '${table}':`, error);
process.exit(1);
} }
} }
} }
@@ -180,10 +183,11 @@ async function checkDatabase() {
// Check for users // Check for users
try { try {
const userCountResult = await client.execute( const db = new Database(dbPath);
`SELECT COUNT(*) as count FROM users`
); // Check for users
const userCount = userCountResult.rows[0].count; const userCountResult = db.query(`SELECT COUNT(*) as count FROM users`).get();
const userCount = userCountResult?.count || 0;
if (userCount === 0) { if (userCount === 0) {
console.log("ℹ️ No users found in the database."); console.log("ℹ️ No users found in the database.");
@@ -197,10 +201,8 @@ async function checkDatabase() {
} }
// Check for configurations // Check for configurations
const configCountResult = await client.execute( const configCountResult = db.query(`SELECT COUNT(*) as count FROM configs`).get();
`SELECT COUNT(*) as count FROM configs` const configCount = configCountResult?.count || 0;
);
const configCount = configCountResult.rows[0].count;
if (configCount === 0) { if (configCount === 0) {
console.log("ℹ️ No configurations found in the database."); console.log("ℹ️ No configurations found in the database.");
@@ -243,7 +245,8 @@ async function initializeDatabase() {
// Check if we can connect to it // Check if we can connect to it
try { try {
await client.execute(`SELECT COUNT(*) as count FROM users`); const db = new Database(dbPath);
db.query(`SELECT COUNT(*) as count FROM users`).get();
console.log("✅ Database is valid and accessible."); console.log("✅ Database is valid and accessible.");
return; return;
} catch (error) { } catch (error) {
@@ -257,135 +260,118 @@ async function initializeDatabase() {
console.log(`Initializing database at ${dbPath}...`); console.log(`Initializing database at ${dbPath}...`);
try { try {
const db = new Database(dbPath);
// Create tables if they don't exist // Create tables if they don't exist
await client.execute( db.exec(`
`CREATE TABLE IF NOT EXISTS users ( CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
username TEXT NOT NULL, username TEXT NOT NULL,
password TEXT NOT NULL, password TEXT NOT NULL,
email TEXT NOT NULL, email TEXT NOT NULL,
created_at INTEGER NOT NULL, created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL updated_at INTEGER NOT NULL
)` )
); `);
// NOTE: We no longer create a default admin user - user will create one via signup page // NOTE: We no longer create a default admin user - user will create one via signup page
await client.execute( db.exec(`
`CREATE TABLE IF NOT EXISTS configs ( CREATE TABLE IF NOT EXISTS configs (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
is_active INTEGER NOT NULL DEFAULT 1, is_active INTEGER NOT NULL DEFAULT 1,
github_config TEXT NOT NULL, github_config TEXT NOT NULL,
gitea_config TEXT NOT NULL, gitea_config TEXT NOT NULL,
include TEXT NOT NULL DEFAULT '["*"]', include TEXT NOT NULL DEFAULT '["*"]',
exclude TEXT NOT NULL DEFAULT '[]', exclude TEXT NOT NULL DEFAULT '[]',
schedule_config TEXT NOT NULL, schedule_config TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id) FOREIGN KEY (user_id) REFERENCES users(id)
); )
` `);
);
await client.execute( db.exec(`
`CREATE TABLE IF NOT EXISTS repositories ( CREATE TABLE IF NOT EXISTS repositories (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
config_id TEXT NOT NULL, config_id TEXT NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
full_name TEXT NOT NULL, full_name TEXT NOT NULL,
url TEXT NOT NULL, url TEXT NOT NULL,
clone_url TEXT NOT NULL, clone_url TEXT NOT NULL,
owner TEXT NOT NULL, owner TEXT NOT NULL,
organization TEXT, organization TEXT,
mirrored_location TEXT DEFAULT '', mirrored_location TEXT DEFAULT '',
is_private INTEGER NOT NULL DEFAULT 0,
is_fork INTEGER NOT NULL DEFAULT 0,
forked_from TEXT,
has_issues INTEGER NOT NULL DEFAULT 0,
is_starred INTEGER NOT NULL DEFAULT 0,
is_archived INTEGER NOT NULL DEFAULT 0,
size INTEGER NOT NULL DEFAULT 0,
has_lfs INTEGER NOT NULL DEFAULT 0,
has_submodules INTEGER NOT NULL DEFAULT 0,
default_branch TEXT NOT NULL,
visibility TEXT NOT NULL DEFAULT 'public',
status TEXT NOT NULL DEFAULT 'imported',
last_mirrored INTEGER,
error_message TEXT,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (config_id) REFERENCES configs(id)
)
`);
is_private INTEGER NOT NULL DEFAULT 0, db.exec(`
is_fork INTEGER NOT NULL DEFAULT 0, CREATE TABLE IF NOT EXISTS organizations (
forked_from TEXT, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
config_id TEXT NOT NULL,
name TEXT NOT NULL,
avatar_url TEXT NOT NULL,
membership_role TEXT NOT NULL DEFAULT 'member',
is_included INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'imported',
last_mirrored INTEGER,
error_message TEXT,
repository_count INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (config_id) REFERENCES configs(id)
)
`);
has_issues INTEGER NOT NULL DEFAULT 0, db.exec(`
is_starred INTEGER NOT NULL DEFAULT 0, CREATE TABLE IF NOT EXISTS mirror_jobs (
is_archived INTEGER NOT NULL DEFAULT 0, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
size INTEGER NOT NULL DEFAULT 0, repository_id TEXT,
has_lfs INTEGER NOT NULL DEFAULT 0, repository_name TEXT,
has_submodules INTEGER NOT NULL DEFAULT 0, organization_id TEXT,
organization_name TEXT,
default_branch TEXT NOT NULL, details TEXT,
visibility TEXT NOT NULL DEFAULT 'public', status TEXT NOT NULL DEFAULT 'imported',
message TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'imported', timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_mirrored INTEGER, FOREIGN KEY (user_id) REFERENCES users(id)
error_message TEXT, )
`);
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (config_id) REFERENCES configs(id)
);
`
);
await client.execute(
`CREATE TABLE IF NOT EXISTS organizations (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
config_id TEXT NOT NULL,
name TEXT NOT NULL,
avatar_url TEXT NOT NULL,
membership_role TEXT NOT NULL DEFAULT 'member',
is_included INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'imported',
last_mirrored INTEGER,
error_message TEXT,
repository_count INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (config_id) REFERENCES configs(id)
);
`
);
await client.execute(
`CREATE TABLE IF NOT EXISTS mirror_jobs (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
repository_id TEXT,
repository_name TEXT,
organization_id TEXT,
organization_name TEXT,
details TEXT,
status TEXT NOT NULL DEFAULT 'imported',
message TEXT NOT NULL,
timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
);
`
);
// Insert default config if none exists // Insert default config if none exists
const configCountResult = await client.execute( const configCountResult = db.query(`SELECT COUNT(*) as count FROM configs`).get();
`SELECT COUNT(*) as count FROM configs` const configCount = configCountResult?.count || 0;
);
const configCount = configCountResult.rows[0].count;
if (configCount === 0) { if (configCount === 0) {
// Get the first user // Get the first user
const firstUserResult = await client.execute( const firstUserResult = db.query(`SELECT id FROM users LIMIT 1`).get();
`SELECT id FROM users LIMIT 1`
); if (firstUserResult) {
if (firstUserResult.rows.length > 0) { const userId = firstUserResult.id;
const userId = firstUserResult.rows[0].id;
const configId = uuidv4(); const configId = uuidv4();
const githubConfig = JSON.stringify({ const githubConfig = JSON.stringify({
username: process.env.GITHUB_USERNAME || "", username: process.env.GITHUB_USERNAME || "",
@@ -415,24 +401,23 @@ async function initializeDatabase() {
nextRun: null, nextRun: null,
}); });
await client.execute( const stmt = db.prepare(`
`
INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, created_at, updated_at) INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, `);
[
configId, stmt.run(
userId, configId,
"Default Configuration", userId,
1, "Default Configuration",
githubConfig, 1,
giteaConfig, githubConfig,
include, giteaConfig,
exclude, include,
scheduleConfig, exclude,
Date.now(), scheduleConfig,
Date.now(), Date.now(),
] Date.now()
); );
} }
} }
@@ -452,8 +437,7 @@ async function resetUsers() {
try { try {
// Check if the database exists // Check if the database exists
const dbFilePath = dbPath.replace("file:", ""); const doesDbExist = fs.existsSync(dbPath);
const doesDbExist = fs.existsSync(dbFilePath);
if (!doesDbExist) { if (!doesDbExist) {
console.log( console.log(
@@ -462,11 +446,11 @@ async function resetUsers() {
return; return;
} }
const db = new Database(dbPath);
// Count existing users // Count existing users
const userCountResult = await client.execute( const userCountResult = db.query(`SELECT COUNT(*) as count FROM users`).get();
`SELECT COUNT(*) as count FROM users` const userCount = userCountResult?.count || 0;
);
const userCount = userCountResult.rows[0].count;
if (userCount === 0) { if (userCount === 0) {
console.log("ℹ️ No users found in the database. Nothing to reset."); console.log("ℹ️ No users found in the database. Nothing to reset.");
@@ -474,63 +458,43 @@ async function resetUsers() {
} }
// Delete all users // Delete all users
await client.execute(`DELETE FROM users`); db.exec(`DELETE FROM users`);
console.log(`✅ Deleted ${userCount} users from the database.`); console.log(`✅ Deleted ${userCount} users from the database.`);
// Check dependent configurations that need to be removed // Check dependent configurations that need to be removed
const configCount = await client.execute( const configCountResult = db.query(`SELECT COUNT(*) as count FROM configs`).get();
`SELECT COUNT(*) as count FROM configs` const configCount = configCountResult?.count || 0;
);
if ( if (configCount > 0) {
configCount.rows && db.exec(`DELETE FROM configs`);
configCount.rows[0] && console.log(`✅ Deleted ${configCount} configurations.`);
Number(configCount.rows[0].count) > 0
) {
await client.execute(`DELETE FROM configs`);
console.log(`✅ Deleted ${configCount.rows[0].count} configurations.`);
} }
// Check for dependent repositories // Check for dependent repositories
const repoCount = await client.execute( const repoCountResult = db.query(`SELECT COUNT(*) as count FROM repositories`).get();
`SELECT COUNT(*) as count FROM repositories` const repoCount = repoCountResult?.count || 0;
);
if ( if (repoCount > 0) {
repoCount.rows && db.exec(`DELETE FROM repositories`);
repoCount.rows[0] && console.log(`✅ Deleted ${repoCount} repositories.`);
Number(repoCount.rows[0].count) > 0
) {
await client.execute(`DELETE FROM repositories`);
console.log(`✅ Deleted ${repoCount.rows[0].count} repositories.`);
} }
// Check for dependent organizations // Check for dependent organizations
const orgCount = await client.execute( const orgCountResult = db.query(`SELECT COUNT(*) as count FROM organizations`).get();
`SELECT COUNT(*) as count FROM organizations` const orgCount = orgCountResult?.count || 0;
);
if ( if (orgCount > 0) {
orgCount.rows && db.exec(`DELETE FROM organizations`);
orgCount.rows[0] && console.log(`✅ Deleted ${orgCount} organizations.`);
Number(orgCount.rows[0].count) > 0
) {
await client.execute(`DELETE FROM organizations`);
console.log(`✅ Deleted ${orgCount.rows[0].count} organizations.`);
} }
// Check for dependent mirror jobs // Check for dependent mirror jobs
const jobCount = await client.execute( const jobCountResult = db.query(`SELECT COUNT(*) as count FROM mirror_jobs`).get();
`SELECT COUNT(*) as count FROM mirror_jobs` const jobCount = jobCountResult?.count || 0;
);
if ( if (jobCount > 0) {
jobCount.rows && db.exec(`DELETE FROM mirror_jobs`);
jobCount.rows[0] && console.log(`✅ Deleted ${jobCount} mirror jobs.`);
Number(jobCount.rows[0].count) > 0
) {
await client.execute(`DELETE FROM mirror_jobs`);
console.log(`✅ Deleted ${jobCount.rows[0].count} mirror jobs.`);
} }
console.log( console.log(
@@ -636,7 +600,8 @@ async function fixDatabaseIssues() {
// Check if we can connect to the database // Check if we can connect to the database
try { try {
// Try to query the database // Try to query the database
await db.select().from(configs).limit(1); const db = new Database(dbPath);
db.query(`SELECT 1 FROM sqlite_master LIMIT 1`).get();
console.log(`✅ Successfully connected to the database.`); console.log(`✅ Successfully connected to the database.`);
} catch (error) { } catch (error) {
console.error("❌ Error connecting to the database:", error); console.error("❌ Error connecting to the database:", error);

19
src/lib/db/index.ts
View File

@@ -2,17 +2,32 @@ import { z } from "zod";
import { Database } from "bun:sqlite"; import { Database } from "bun:sqlite";
import { drizzle } from "drizzle-orm/bun-sqlite"; import { drizzle } from "drizzle-orm/bun-sqlite";
import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core"; import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core";
import fs from "fs";
import path from "path"; import path from "path";
import { configSchema } from "./schema"; import { configSchema } from "./schema";
// Define the database URL - for development we'll use a local SQLite file // Define the database URL - for development we'll use a local SQLite file
const dataDir = path.join(process.cwd(), "data"); const dataDir = path.join(process.cwd(), "data");
// Ensure data directory exists
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
}
const dbUrl = const dbUrl =
process.env.DATABASE_URL || `file:${path.join(dataDir, "gitea-mirror.db")}`; process.env.DATABASE_URL || `file:${path.join(dataDir, "gitea-mirror.db")}`;
// Create a SQLite database instance using Bun's native driver // Create a SQLite database instance using Bun's native driver
export const sqlite = new Database(dbUrl); let sqlite: Database;
try {
// Create an empty database file if it doesn't exist
if (!fs.existsSync(path.join(dataDir, "gitea-mirror.db"))) {
fs.writeFileSync(path.join(dataDir, "gitea-mirror.db"), "");
}
sqlite = new Database(dbUrl);
} catch (error) {
console.error("Error opening database:", error);
throw error;
}
// Simple async wrapper around Bun's SQLite API for compatibility // Simple async wrapper around Bun's SQLite API for compatibility
export const client = { export const client = {