Compare commits

...

5 Commits

Author SHA1 Message Date
Arunavo Ray
fe6bcc5288 chore: bump version to 3.13.2 2026-03-15 14:11:22 +05:30
ARUNAVO RAY
e26ed3aa9c fix: rewrite migration 0009 for SQLite compatibility and add migration validation (#230)
SQLite rejects ALTER TABLE ADD COLUMN with expression defaults like
DEFAULT (unixepoch()), which Drizzle-kit generated for the imported_at
column. This broke upgrades from v3.12.x to v3.13.0 (#228, #229).

Changes:
- Rewrite migration 0009 using table-recreation pattern (CREATE, INSERT
  SELECT, DROP, RENAME) instead of ALTER TABLE
- Add migration validation script with SQLite-specific lint rules that
  catch known invalid patterns before they ship
- Add upgrade-path testing with seeded data and verification fixtures
- Add runtime repair for users whose migration record may be stale
- Add explicit migration validation step to CI workflow

Fixes #228
Fixes #229
2026-03-15 14:10:06 +05:30
Arunavo Ray
efb96b6e60 chore: bump version to 3.13.1 2026-03-15 09:54:44 +05:30
Arunavo Ray
342cafed0e fix: force Go 1.25.8 toolchain and update x/crypto for git-lfs build
The git-lfs go.mod contains a `toolchain go1.25.3` directive which
causes Go to auto-download and use Go 1.25.3 instead of our installed
1.25.8. Set GOTOOLCHAIN=local to force using the installed version.

Also update golang.org/x/crypto to latest before building to resolve
CVE-2025-47913 (needs >= 0.43.0, was pinned at 0.36.0).
2026-03-15 09:35:50 +05:30
Arunavo Ray
fc7c6b59d7 docs: update README to reference Gitea/Forgejo as supported targets 2026-03-15 09:33:41 +05:30
8 changed files with 469 additions and 51 deletions

View File

@@ -48,6 +48,12 @@ jobs:
- name: Run tests
run: bun test --coverage
- name: Check Drizzle migrations
run: bun run db:check
- name: Validate migrations (SQLite lint + upgrade path)
run: bun test:migrations
- name: Build Astro project
run: bunx --bun astro build

View File

@@ -37,9 +37,13 @@ RUN ARCH="$(dpkg --print-architecture)" \
&& wget -qO /tmp/go.tar.gz "https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz" \
&& tar -C /usr/local -xzf /tmp/go.tar.gz \
&& rm /tmp/go.tar.gz
ENV PATH="/usr/local/go/bin:${PATH}"
ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}"
# Force using our installed Go (not the version in go.mod toolchain directive)
ENV GOTOOLCHAIN=local
RUN git clone --branch "v${GIT_LFS_VERSION}" --depth 1 https://github.com/git-lfs/git-lfs.git /tmp/git-lfs \
&& cd /tmp/git-lfs \
&& go get golang.org/x/crypto@latest \
&& go mod tidy \
&& make \
&& install -m 755 /tmp/git-lfs/bin/git-lfs /usr/local/bin/git-lfs

View File

@@ -1,7 +1,7 @@
<p align="center">
<img src=".github/assets/logo.png" alt="Gitea Mirror Logo" width="120" />
<h1>Gitea Mirror</h1>
<p><i>Automatically mirror repositories from GitHub to your self-hosted Gitea instance.</i></p>
<p><i>Automatically mirror repositories from GitHub to your self-hosted Gitea/Forgejo instance.</i></p>
<p align="center">
<a href="https://github.com/RayLabsHQ/gitea-mirror/releases/latest"><img src="https://img.shields.io/github/v/tag/RayLabsHQ/gitea-mirror?label=release" alt="release"/></a>
<a href="https://github.com/RayLabsHQ/gitea-mirror/actions/workflows/astro-build-test.yml"><img src="https://img.shields.io/github/actions/workflow/status/RayLabsHQ/gitea-mirror/astro-build-test.yml?branch=main" alt="build"/></a>
@@ -19,7 +19,7 @@ docker compose -f docker-compose.alt.yml up -d
# Access at http://localhost:4321
```
First user signup becomes admin. Configure GitHub and Gitea through the web interface!
First user signup becomes admin. Configure GitHub and Gitea/Forgejo through the web interface!
<p align="center">
<img src=".github/assets/dashboard.png" alt="Dashboard" width="600" />
@@ -28,7 +28,7 @@ First user signup becomes admin. Configure GitHub and Gitea through the web inte
## ✨ Features
- 🔁 Mirror public, private, and starred GitHub repos to Gitea
- 🔁 Mirror public, private, and starred GitHub repos to Gitea/Forgejo
- 🏢 Mirror entire organizations with flexible strategies
- 🎯 Custom destination control for repos and organizations
- 📦 **Git LFS support** - Mirror large files with Git LFS
@@ -199,12 +199,12 @@ bun run dev
1. **First Time Setup**
- Navigate to http://localhost:4321
- Create admin account (first user signup)
- Configure GitHub and Gitea connections
- Configure GitHub and Gitea/Forgejo connections
2. **Mirror Strategies**
- **Preserve Structure**: Maintains GitHub organization structure
- **Single Organization**: All repos go to one Gitea organization
- **Flat User**: All repos under your Gitea user account
- **Single Organization**: All repos go to one Gitea/Forgejo organization
- **Flat User**: All repos under your Gitea/Forgejo user account
- **Mixed Mode**: Personal repos in one org, organization repos preserve structure
3. **Customization**
@@ -217,13 +217,13 @@ bun run dev
### Git LFS (Large File Storage)
Mirror Git LFS objects along with your repositories:
- Enable "Mirror LFS" option in Settings → Mirror Options
- Requires Gitea server with LFS enabled (`LFS_START_SERVER = true`)
- Requires Gitea/Forgejo server with LFS enabled (`LFS_START_SERVER = true`)
- Requires Git v2.1.2+ on the server
### Metadata Mirroring
Transfer complete repository metadata from GitHub to Gitea:
Transfer complete repository metadata from GitHub to Gitea/Forgejo:
- **Issues** - Mirror all issues with comments and labels
- **Pull Requests** - Transfer PR discussions to Gitea
- **Pull Requests** - Transfer PR discussions to Gitea/Forgejo
- **Labels** - Preserve repository labels
- **Milestones** - Keep project milestones
- **Wiki** - Mirror wiki content
@@ -243,7 +243,7 @@ Gitea Mirror provides powerful automatic synchronization features:
#### Features (v3.4.0+)
- **Auto-discovery**: Automatically discovers and imports new GitHub repositories
- **Repository cleanup**: Removes repositories that no longer exist in GitHub
- **Proper intervals**: Mirrors respect your configured sync intervals (not Gitea's default 24h)
- **Proper intervals**: Mirrors respect your configured sync intervals (not Gitea/Forgejo's default 24h)
- **Smart scheduling**: Only syncs repositories that need updating
- **Auto-start on boot** (v3.5.3+): Automatically imports and mirrors all repositories when `SCHEDULE_ENABLED=true` or `GITEA_MIRROR_INTERVAL` is set - no manual clicks required!
@@ -254,7 +254,7 @@ Navigate to the Configuration page and enable "Automatic Syncing" with your pref
**🚀 Set it and forget it!** With these environment variables, Gitea Mirror will automatically:
1. **Import** all your GitHub repositories on startup (no manual import needed!)
2. **Mirror** them to Gitea immediately
2. **Mirror** them to Gitea/Forgejo immediately
3. **Keep them synchronized** based on your interval
4. **Auto-discover** new repos you create/star on GitHub
5. **Clean up** repos you delete from GitHub
@@ -284,16 +284,16 @@ CLEANUP_DRY_RUN=false # Set to true to test without changes
- **Auto-Start**: When `SCHEDULE_ENABLED=true` or `GITEA_MIRROR_INTERVAL` is set, the service automatically imports all GitHub repositories and mirrors them on startup. No manual "Import" or "Mirror" button clicks required!
- The scheduler checks every minute for tasks to run. The `GITEA_MIRROR_INTERVAL` determines how often each repository is actually synced. For example, with `8h`, each repo syncs every 8 hours from its last successful sync.
- **Large repo bootstrap**: For first-time mirroring of large repositories (especially with metadata/LFS), avoid very short intervals (for example `5m`). Start with a longer interval (`1h` to `8h`) or temporarily disable scheduling during the initial import/mirror run, then enable your regular interval after the first pass completes.
- **Why this matters**: If your Gitea instance takes a long time to complete migrations/imports, aggressive schedules can cause repeated retries and duplicate-looking mirror attempts.
- **Why this matters**: If your Gitea/Forgejo instance takes a long time to complete migrations/imports, aggressive schedules can cause repeated retries and duplicate-looking mirror attempts.
**🛡️ Backup Protection Features**:
- **No Accidental Deletions**: Repository cleanup is automatically skipped if GitHub is inaccessible (account deleted, banned, or API errors)
- **Archive Never Deletes Data**: The `archive` action preserves all repository data:
- Regular repositories: Made read-only using Gitea's archive feature
- Mirror repositories: Renamed with `archived-` prefix (Gitea API limitation prevents archiving mirrors)
- Regular repositories: Made read-only using Gitea/Forgejo's archive feature
- Mirror repositories: Renamed with `archived-` prefix (Gitea/Forgejo API limitation prevents archiving mirrors)
- Failed operations: Repository remains fully accessible even if marking as archived fails
- **Manual Sync on Demand**: Archived mirrors stay in Gitea with automatic syncs disabled; trigger `Manual Sync` from the Repositories page whenever you need fresh data.
- **The Whole Point of Backups**: Your Gitea mirrors are preserved even when GitHub sources disappear - that's why you have backups!
- **Manual Sync on Demand**: Archived mirrors stay in Gitea/Forgejo with automatic syncs disabled; trigger `Manual Sync` from the Repositories page whenever you need fresh data.
- **The Whole Point of Backups**: Your Gitea/Forgejo mirrors are preserved even when GitHub sources disappear - that's why you have backups!
- **Strongly Recommended**: Always use `CLEANUP_ORPHANED_REPO_ACTION=archive` (default) instead of `delete`
## Troubleshooting
@@ -309,7 +309,7 @@ For existing pull-mirror repositories, changing the GitHub token in Gitea Mirror
If sync logs show authentication failures (for example `terminal prompts disabled`), do one of the following:
1. In Gitea/Forgejo, open repository **Settings → Mirror Settings** and update the mirror authorization password/token.
2. Or delete and re-mirror the repository from Gitea Mirror so it is recreated with current credentials.
2. Or delete and re-mirror the repository so it is recreated with current credentials.
### Re-sync Metadata After Changing Mirror Options
@@ -334,7 +334,7 @@ If your Gitea/Forgejo server has `mirror.MIN_INTERVAL` set to a higher value (fo
To avoid this:
1. Set Gitea Mirror interval to a value greater than or equal to your server `MIN_INTERVAL`.
2. Do not rely on manual per-repository mirror interval edits in Gitea/Forgejo, because Gitea Mirror will overwrite them on sync.
2. Do not rely on manual per-repository mirror interval edits in Gitea/Forgejo, as they will be overwritten on sync.
## Development
@@ -356,13 +356,13 @@ bun run build
- **Frontend**: Astro, React, Shadcn UI, Tailwind CSS v4
- **Backend**: Bun runtime, SQLite, Drizzle ORM
- **APIs**: GitHub (Octokit), Gitea REST API
- **APIs**: GitHub (Octokit), Gitea/Forgejo REST API
- **Auth**: Better Auth with session-based authentication
## Security
### Token Encryption
- All GitHub and Gitea API tokens are encrypted at rest using AES-256-GCM
- All GitHub and Gitea/Forgejo API tokens are encrypted at rest using AES-256-GCM
- Encryption is automatic and transparent to users
- Set `ENCRYPTION_SECRET` environment variable for production deployments
- Falls back to `BETTER_AUTH_SECRET` if not set
@@ -456,13 +456,13 @@ Gitea Mirror can also act as an OIDC provider for other applications. Register O
## Known Limitations
### Pull Request Mirroring Implementation
Pull requests **cannot be created as actual PRs** in Gitea due to API limitations. Instead, they are mirrored as **enriched issues** with comprehensive metadata.
Pull requests **cannot be created as actual PRs** in Gitea/Forgejo due to API limitations. Instead, they are mirrored as **enriched issues** with comprehensive metadata.
**Why real PR mirroring isn't possible:**
- Gitea's API doesn't support creating pull requests from external sources
- Gitea/Forgejo's API doesn't support creating pull requests from external sources
- Real PRs require actual Git branches with commits to exist in the repository
- Would require complex branch synchronization and commit replication
- The mirror relationship is one-way (GitHub → Gitea) for repository content
- The mirror relationship is one-way (GitHub → Gitea/Forgejo) for repository content
**How we handle Pull Requests:**
PRs are mirrored as issues with rich metadata including:
@@ -476,7 +476,7 @@ PRs are mirrored as issues with rich metadata including:
- 🔀 Base and head branch information
- ✅ Merge status tracking
This approach preserves all important PR information while working within Gitea's API constraints. The PRs appear in Gitea's issue tracker with clear visual distinction and comprehensive details.
This approach preserves all important PR information while working within Gitea/Forgejo's API constraints. The PRs appear in the issue tracker with clear visual distinction and comprehensive details.
## Contributing

View File

@@ -1,24 +1,149 @@
ALTER TABLE `repositories` ADD `imported_at` integer DEFAULT (unixepoch()) NOT NULL;--> statement-breakpoint
UPDATE `repositories`
SET `imported_at` = COALESCE(
(
SELECT MIN(`mj`.`timestamp`)
FROM `mirror_jobs` `mj`
WHERE `mj`.`user_id` = `repositories`.`user_id`
AND `mj`.`status` = 'imported'
AND (
(`mj`.`repository_id` IS NOT NULL AND `mj`.`repository_id` = `repositories`.`id`)
OR (
`mj`.`repository_id` IS NULL
AND `mj`.`repository_name` IS NOT NULL
AND (
lower(trim(`mj`.`repository_name`)) = `repositories`.`normalized_full_name`
OR lower(trim(`mj`.`repository_name`)) = lower(trim(`repositories`.`name`))
)
)
)
),
`repositories`.`created_at`,
`imported_at`
);--> statement-breakpoint
CREATE TABLE `__new_repositories` (
`id` text PRIMARY KEY NOT NULL,
`user_id` text NOT NULL,
`config_id` text NOT NULL,
`name` text NOT NULL,
`full_name` text NOT NULL,
`normalized_full_name` text NOT NULL,
`url` text NOT NULL,
`clone_url` text NOT NULL,
`owner` text NOT NULL,
`organization` text,
`mirrored_location` text DEFAULT '',
`is_private` integer DEFAULT false NOT NULL,
`is_fork` integer DEFAULT false NOT NULL,
`forked_from` text,
`has_issues` integer DEFAULT false NOT NULL,
`is_starred` integer DEFAULT false NOT NULL,
`is_archived` integer DEFAULT false NOT NULL,
`size` integer DEFAULT 0 NOT NULL,
`has_lfs` integer DEFAULT false NOT NULL,
`has_submodules` integer DEFAULT false NOT NULL,
`language` text,
`description` text,
`default_branch` text NOT NULL,
`visibility` text DEFAULT 'public' NOT NULL,
`status` text DEFAULT 'imported' NOT NULL,
`last_mirrored` integer,
`error_message` text,
`destination_org` text,
`metadata` text,
`imported_at` integer DEFAULT (unixepoch()) NOT NULL,
`created_at` integer DEFAULT (unixepoch()) NOT NULL,
`updated_at` integer DEFAULT (unixepoch()) NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action,
FOREIGN KEY (`config_id`) REFERENCES `configs`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
INSERT INTO `__new_repositories` (
`id`,
`user_id`,
`config_id`,
`name`,
`full_name`,
`normalized_full_name`,
`url`,
`clone_url`,
`owner`,
`organization`,
`mirrored_location`,
`is_private`,
`is_fork`,
`forked_from`,
`has_issues`,
`is_starred`,
`is_archived`,
`size`,
`has_lfs`,
`has_submodules`,
`language`,
`description`,
`default_branch`,
`visibility`,
`status`,
`last_mirrored`,
`error_message`,
`destination_org`,
`metadata`,
`imported_at`,
`created_at`,
`updated_at`
)
SELECT
`repositories`.`id`,
`repositories`.`user_id`,
`repositories`.`config_id`,
`repositories`.`name`,
`repositories`.`full_name`,
`repositories`.`normalized_full_name`,
`repositories`.`url`,
`repositories`.`clone_url`,
`repositories`.`owner`,
`repositories`.`organization`,
`repositories`.`mirrored_location`,
`repositories`.`is_private`,
`repositories`.`is_fork`,
`repositories`.`forked_from`,
`repositories`.`has_issues`,
`repositories`.`is_starred`,
`repositories`.`is_archived`,
`repositories`.`size`,
`repositories`.`has_lfs`,
`repositories`.`has_submodules`,
`repositories`.`language`,
`repositories`.`description`,
`repositories`.`default_branch`,
`repositories`.`visibility`,
`repositories`.`status`,
`repositories`.`last_mirrored`,
`repositories`.`error_message`,
`repositories`.`destination_org`,
`repositories`.`metadata`,
COALESCE(
(
SELECT MIN(`mj`.`timestamp`)
FROM `mirror_jobs` `mj`
WHERE `mj`.`user_id` = `repositories`.`user_id`
AND `mj`.`status` = 'imported'
AND (
(`mj`.`repository_id` IS NOT NULL AND `mj`.`repository_id` = `repositories`.`id`)
OR (
`mj`.`repository_id` IS NULL
AND `mj`.`repository_name` IS NOT NULL
AND (
lower(trim(`mj`.`repository_name`)) = `repositories`.`normalized_full_name`
OR lower(trim(`mj`.`repository_name`)) = lower(trim(`repositories`.`name`))
)
)
)
),
`repositories`.`created_at`,
unixepoch()
) AS `imported_at`,
`repositories`.`created_at`,
`repositories`.`updated_at`
FROM `repositories`;
--> statement-breakpoint
DROP TABLE `repositories`;
--> statement-breakpoint
ALTER TABLE `__new_repositories` RENAME TO `repositories`;
--> statement-breakpoint
CREATE INDEX `idx_repositories_user_id` ON `repositories` (`user_id`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_config_id` ON `repositories` (`config_id`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_status` ON `repositories` (`status`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_owner` ON `repositories` (`owner`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_organization` ON `repositories` (`organization`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_is_fork` ON `repositories` (`is_fork`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_is_starred` ON `repositories` (`is_starred`);
--> statement-breakpoint
CREATE INDEX `idx_repositories_user_imported_at` ON `repositories` (`user_id`,`imported_at`);
--> statement-breakpoint
CREATE UNIQUE INDEX `uniq_repositories_user_full_name` ON `repositories` (`user_id`,`full_name`);
--> statement-breakpoint
CREATE UNIQUE INDEX `uniq_repositories_user_normalized_full_name` ON `repositories` (`user_id`,`normalized_full_name`);

View File

@@ -1,7 +1,7 @@
{
"name": "gitea-mirror",
"type": "module",
"version": "3.13.0",
"version": "3.13.2",
"engines": {
"bun": ">=1.2.9"
},
@@ -34,6 +34,7 @@
"start": "bun dist/server/entry.mjs",
"start:fresh": "bun run cleanup-db && bun run manage-db init && bun dist/server/entry.mjs",
"test": "bun test",
"test:migrations": "bun scripts/validate-migrations.ts",
"test:watch": "bun test --watch",
"test:coverage": "bun test --coverage",
"test:e2e": "bash tests/e2e/run-e2e.sh",

View File

@@ -0,0 +1,212 @@
#!/usr/bin/env bun
import { Database } from "bun:sqlite";
import { readFileSync } from "fs";
import path from "path";
type JournalEntry = {
idx: number;
tag: string;
when: number;
breakpoints: boolean;
};
type Migration = {
entry: JournalEntry;
statements: string[];
};
type UpgradeFixture = {
seed: (db: Database) => void;
verify: (db: Database) => void;
};
type TableInfoRow = {
cid: number;
name: string;
type: string;
notnull: number;
dflt_value: string | null;
pk: number;
};
const migrationsFolder = path.join(process.cwd(), "drizzle");
const migrations = loadMigrations();
const latestMigration = migrations.at(-1);
/**
* Known SQLite limitations that Drizzle-kit's auto-generated migrations
* can violate. Each rule is checked against every SQL statement.
*/
const SQLITE_LINT_RULES: { pattern: RegExp; message: string }[] = [
{
pattern: /ALTER\s+TABLE\s+\S+\s+ADD\s+(?:COLUMN\s+)?\S+[^;]*DEFAULT\s*\(/i,
message:
"ALTER TABLE ADD COLUMN with an expression default (e.g. DEFAULT (unixepoch())) " +
"is not allowed in SQLite. Use the table-recreation pattern instead " +
"(CREATE new table, INSERT SELECT, DROP old, RENAME).",
},
{
pattern: /ALTER\s+TABLE\s+\S+\s+ADD\s+(?:COLUMN\s+)?\S+[^;]*DEFAULT\s+CURRENT_(TIME|DATE|TIMESTAMP)\b/i,
message:
"ALTER TABLE ADD COLUMN with DEFAULT CURRENT_TIME/CURRENT_DATE/CURRENT_TIMESTAMP " +
"is not allowed in SQLite. Use the table-recreation pattern instead.",
},
];
function loadMigrations(): Migration[] {
const journalPath = path.join(migrationsFolder, "meta", "_journal.json");
const journal = JSON.parse(readFileSync(journalPath, "utf8")) as {
entries: JournalEntry[];
};
return journal.entries.map((entry) => {
const migrationPath = path.join(migrationsFolder, `${entry.tag}.sql`);
const statements = readFileSync(migrationPath, "utf8")
.split("--> statement-breakpoint")
.map((statement) => statement.trim())
.filter(Boolean);
return { entry, statements };
});
}
function assert(condition: unknown, message: string): asserts condition {
if (!condition) {
throw new Error(message);
}
}
function runMigration(db: Database, migration: Migration) {
db.run("BEGIN");
try {
for (const statement of migration.statements) {
db.run(statement);
}
db.run("COMMIT");
} catch (error) {
try {
db.run("ROLLBACK");
} catch {
// Ignore rollback errors so the original failure is preserved.
}
throw error;
}
}
function runMigrations(db: Database, selectedMigrations: Migration[]) {
for (const migration of selectedMigrations) {
runMigration(db, migration);
}
}
function seedPre0009Database(db: Database) {
// Seed every existing table so ALTER TABLE paths run against non-empty data.
db.run("INSERT INTO users (id, email, username, name) VALUES ('u1', 'u1@example.com', 'user1', 'User One')");
db.run("INSERT INTO configs (id, user_id, name, github_config, gitea_config, schedule_config, cleanup_config) VALUES ('c1', 'u1', 'Default', '{}', '{}', '{}', '{}')");
db.run("INSERT INTO accounts (id, account_id, user_id, provider_id, access_token, refresh_token, id_token, access_token_expires_at, refresh_token_expires_at, scope) VALUES ('acct1', 'acct-1', 'u1', 'github', 'access-token', 'refresh-token', 'id-token', 2000, 3000, 'repo')");
db.run("INSERT INTO events (id, user_id, channel, payload) VALUES ('evt1', 'u1', 'sync', '{\"status\":\"queued\"}')");
db.run("INSERT INTO mirror_jobs (id, user_id, repository_id, repository_name, status, message, timestamp) VALUES ('job1', 'u1', 'r1', 'owner/repo', 'imported', 'Imported repository', 900)");
db.run("INSERT INTO organizations (id, user_id, config_id, name, avatar_url, public_repository_count, private_repository_count, fork_repository_count) VALUES ('org1', 'u1', 'c1', 'Example Org', 'https://example.com/org.png', 1, 0, 0)");
db.run("INSERT INTO repositories (id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url, owner, organization, default_branch, created_at, updated_at, metadata) VALUES ('r1', 'u1', 'c1', 'repo', 'owner/repo', 'owner/repo', 'https://example.com/repo', 'https://example.com/repo.git', 'owner', 'Example Org', 'main', 1000, 1100, '{\"issues\":true}')");
db.run("INSERT INTO sessions (id, token, user_id, expires_at) VALUES ('sess1', 'session-token', 'u1', 4000)");
db.run("INSERT INTO verification_tokens (id, token, identifier, type, expires_at) VALUES ('vt1', 'verify-token', 'u1@example.com', 'email', 5000)");
db.run("INSERT INTO verifications (id, identifier, value, expires_at) VALUES ('ver1', 'u1@example.com', '123456', 6000)");
db.run("INSERT INTO oauth_applications (id, client_id, client_secret, name, redirect_urls, type, user_id) VALUES ('app1', 'client-1', 'secret-1', 'Example App', '[\"https://example.com/callback\"]', 'confidential', 'u1')");
db.run("INSERT INTO oauth_access_tokens (id, access_token, refresh_token, access_token_expires_at, refresh_token_expires_at, client_id, user_id, scopes) VALUES ('oat1', 'oauth-access-token', 'oauth-refresh-token', 7000, 8000, 'client-1', 'u1', '[\"repo\"]')");
db.run("INSERT INTO oauth_consent (id, user_id, client_id, scopes, consent_given) VALUES ('consent1', 'u1', 'client-1', '[\"repo\"]', true)");
db.run("INSERT INTO sso_providers (id, issuer, domain, oidc_config, user_id, provider_id) VALUES ('sso1', 'https://issuer.example.com', 'example.com', '{}', 'u1', 'provider-1')");
db.run("INSERT INTO rate_limits (id, user_id, provider, `limit`, remaining, used, reset, retry_after, status, last_checked) VALUES ('rl1', 'u1', 'github', 5000, 4999, 1, 9000, NULL, 'ok', 8500)");
}
function verify0009Migration(db: Database) {
const repositoryColumns = db.query("PRAGMA table_info(repositories)").all() as TableInfoRow[];
const importedAtColumn = repositoryColumns.find((column) => column.name === "imported_at");
assert(importedAtColumn, "Expected repositories.imported_at column to exist after migration");
assert(importedAtColumn.notnull === 1, "Expected repositories.imported_at to be NOT NULL");
assert(importedAtColumn.dflt_value === "unixepoch()", `Expected repositories.imported_at default to be unixepoch(), got ${importedAtColumn.dflt_value ?? "null"}`);
const existingRepo = db.query("SELECT imported_at FROM repositories WHERE id = 'r1'").get() as { imported_at: number } | null;
assert(existingRepo?.imported_at === 900, `Expected existing repository imported_at to backfill from mirror_jobs timestamp 900, got ${existingRepo?.imported_at ?? "null"}`);
db.run("INSERT INTO repositories (id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url, owner, default_branch) VALUES ('r2', 'u1', 'c1', 'repo-two', 'owner/repo-two', 'owner/repo-two', 'https://example.com/repo-two', 'https://example.com/repo-two.git', 'owner', 'main')");
const newRepo = db.query("SELECT imported_at FROM repositories WHERE id = 'r2'").get() as { imported_at: number } | null;
assert(typeof newRepo?.imported_at === "number" && newRepo.imported_at > 0, "Expected new repository insert to receive imported_at from the column default");
const importedAtIndex = db
.query("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'repositories' AND name = 'idx_repositories_user_imported_at'")
.get() as { name: string } | null;
assert(importedAtIndex?.name === "idx_repositories_user_imported_at", "Expected repositories imported_at index to exist after migration");
}
const latestUpgradeFixtures: Record<string, UpgradeFixture> = {
"0009_nervous_tyger_tiger": {
seed: seedPre0009Database,
verify: verify0009Migration,
},
};
function lintMigrations(selectedMigrations: Migration[]) {
const violations: string[] = [];
for (const migration of selectedMigrations) {
for (const statement of migration.statements) {
for (const rule of SQLITE_LINT_RULES) {
if (rule.pattern.test(statement)) {
violations.push(`[${migration.entry.tag}] ${rule.message}\n Statement: ${statement.slice(0, 120)}...`);
}
}
}
}
assert(
violations.length === 0,
`SQLite lint found ${violations.length} violation(s):\n\n${violations.join("\n\n")}`,
);
}
function validateMigrations() {
assert(latestMigration, "No migrations found in drizzle/meta/_journal.json");
// Lint all migrations for known SQLite pitfalls before running anything.
lintMigrations(migrations);
const emptyDb = new Database(":memory:");
try {
runMigrations(emptyDb, migrations);
} finally {
emptyDb.close();
}
const upgradeFixture = latestUpgradeFixtures[latestMigration.entry.tag];
assert(
upgradeFixture,
`Missing upgrade fixture for latest migration ${latestMigration.entry.tag}. Add one in scripts/validate-migrations.ts.`,
);
const upgradeDb = new Database(":memory:");
try {
runMigrations(upgradeDb, migrations.slice(0, -1));
upgradeFixture.seed(upgradeDb);
runMigration(upgradeDb, latestMigration);
upgradeFixture.verify(upgradeDb);
} finally {
upgradeDb.close();
}
console.log(
`Validated ${migrations.length} migrations from scratch and upgrade path for ${latestMigration.entry.tag}.`,
);
}
try {
validateMigrations();
} catch (error) {
console.error("Migration validation failed:");
console.error(error instanceof Error ? error.stack ?? error.message : String(error));
process.exit(1);
}

View File

@@ -35,13 +35,54 @@ if (process.env.NODE_ENV !== "test") {
// Create drizzle instance with the SQLite client
db = drizzle({ client: sqlite });
/**
* Fix migration records that were marked as applied but whose DDL actually
* failed (e.g. the v3.13.0 release where ALTER TABLE with expression default
* was rejected by SQLite). Without this, Drizzle skips the migration on
* retry because it thinks it already ran.
*
* Drizzle tracks migrations by `created_at` (= journal timestamp) and only
* looks at the most recent record. If the last recorded timestamp is >= the
* failed migration's timestamp but the expected column is missing, we delete
* stale records so the migration re-runs.
*/
function repairFailedMigrations() {
try {
const migrationsTableExists = sqlite
.query("SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'")
.get();
if (!migrationsTableExists) return;
// Migration 0009 journal timestamp (from drizzle/meta/_journal.json)
const MIGRATION_0009_TIMESTAMP = 1773542995732;
const lastMigration = sqlite
.query("SELECT id, created_at FROM __drizzle_migrations ORDER BY created_at DESC LIMIT 1")
.get() as { id: number; created_at: number } | null;
if (!lastMigration || Number(lastMigration.created_at) < MIGRATION_0009_TIMESTAMP) return;
// Migration 0009 is recorded as applied — verify the column actually exists
const columns = sqlite.query("PRAGMA table_info(repositories)").all() as { name: string }[];
const hasImportedAt = columns.some((c) => c.name === "imported_at");
if (!hasImportedAt) {
console.log("🔧 Detected failed migration 0009 (imported_at column missing). Removing stale record so it can re-run...");
sqlite.prepare("DELETE FROM __drizzle_migrations WHERE created_at >= ?").run(MIGRATION_0009_TIMESTAMP);
}
} catch (error) {
console.warn("⚠️ Migration repair check failed (non-fatal):", error);
}
}
/**
* Run Drizzle migrations
*/
function runDrizzleMigrations() {
try {
console.log("🔄 Checking for pending migrations...");
// Check if migrations table exists
const migrationsTableExists = sqlite
.query("SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'")
@@ -51,9 +92,12 @@ if (process.env.NODE_ENV !== "test") {
console.log("📦 First time setup - running initial migrations...");
}
// Fix any migrations that were recorded but actually failed (e.g. v3.13.0 bug)
repairFailedMigrations();
// Run migrations using Drizzle migrate function
migrate(db, { migrationsFolder: "./drizzle" });
console.log("✅ Database migrations completed successfully");
} catch (error) {
console.error("❌ Error running migrations:", error);

View File

@@ -0,0 +1,26 @@
import { expect, test } from "bun:test";
function decodeOutput(output: ArrayBufferLike | Uint8Array | null | undefined) {
if (!output) {
return "";
}
return Buffer.from(output as ArrayBufferLike).toString("utf8");
}
test("migration validation script passes", () => {
const result = Bun.spawnSync({
cmd: ["bun", "scripts/validate-migrations.ts"],
cwd: process.cwd(),
stdout: "pipe",
stderr: "pipe",
});
const stdout = decodeOutput(result.stdout);
const stderr = decodeOutput(result.stderr);
expect(
result.exitCode,
`Migration validation script failed.\nstdout:\n${stdout}\nstderr:\n${stderr}`,
).toBe(0);
});