From fb27ddfee525804ac379b6d20c9045288ab6ba94 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Thu, 23 Oct 2025 23:08:32 +0530 Subject: [PATCH 01/11] fix: preserve chronological issue mirroring --- src/lib/db/schema.ts | 2 ++ src/lib/env-config-loader.ts | 11 +++++++++++ src/lib/gitea.ts | 20 ++++++++++++++++++-- src/lib/utils/config-defaults.ts | 2 ++ src/lib/utils/config-mapper.ts | 4 ++++ src/types/config.ts | 2 ++ 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 442e65a..1137050 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -54,6 +54,8 @@ export const giteaConfigSchema = z.object({ .enum(["skip", "reference", "full-copy"]) .default("reference"), // Mirror options + issueConcurrency: z.number().int().min(1).default(3), + pullRequestConcurrency: z.number().int().min(1).default(5), mirrorReleases: z.boolean().default(false), releaseLimit: z.number().default(10), mirrorMetadata: z.boolean().default(false), diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index b03cb62..635b3ab 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -49,6 +49,9 @@ interface EnvConfig { mirrorLabels?: boolean; mirrorMilestones?: boolean; mirrorMetadata?: boolean; + releaseLimit?: number; + issueConcurrency?: number; + pullRequestConcurrency?: number; }; schedule: { enabled?: boolean; @@ -136,6 +139,8 @@ function parseEnvConfig(): EnvConfig { mirrorMilestones: process.env.MIRROR_MILESTONES === 'true', mirrorMetadata: process.env.MIRROR_METADATA === 'true', releaseLimit: process.env.RELEASE_LIMIT ? parseInt(process.env.RELEASE_LIMIT, 10) : undefined, + issueConcurrency: process.env.MIRROR_ISSUE_CONCURRENCY ? parseInt(process.env.MIRROR_ISSUE_CONCURRENCY, 10) : undefined, + pullRequestConcurrency: process.env.MIRROR_PULL_REQUEST_CONCURRENCY ? parseInt(process.env.MIRROR_PULL_REQUEST_CONCURRENCY, 10) : undefined, }, schedule: { enabled: process.env.SCHEDULE_ENABLED === 'true' || @@ -277,6 +282,12 @@ export async function initializeConfigFromEnv(): Promise { // Mirror metadata options mirrorReleases: envConfig.mirror.mirrorReleases ?? existingConfig?.[0]?.giteaConfig?.mirrorReleases ?? false, releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, + issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0 + ? envConfig.mirror.issueConcurrency + : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3, + pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0 + ? envConfig.mirror.pullRequestConcurrency + : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index 908b6a2..0dec27e 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -1558,6 +1558,8 @@ export const mirrorGitRepoIssuesToGitea = async ({ repo, state: "all", per_page: 100, + sort: "created", + direction: "asc", }, (res) => res.data ); @@ -1590,6 +1592,12 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); + const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3; + const issueConcurrencyLimit = + Number.isFinite(rawIssueConcurrency) + ? Math.max(1, Math.floor(rawIssueConcurrency)) + : 3; + // Process issues in parallel with concurrency control await processWithRetry( filteredIssues, @@ -1694,7 +1702,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ return issue; }, { - concurrencyLimit: 3, // Process 3 issues at a time + concurrencyLimit: issueConcurrencyLimit, maxRetries: 2, retryDelay: 2000, onProgress: (completed, total, result) => { @@ -1966,6 +1974,8 @@ export async function mirrorGitRepoPullRequestsToGitea({ repo, state: "all", per_page: 100, + sort: "created", + direction: "asc", }, (res) => res.data ); @@ -2022,6 +2032,12 @@ export async function mirrorGitRepoPullRequestsToGitea({ const { processWithRetry } = await import("@/lib/utils/concurrency"); + const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5; + const pullRequestConcurrencyLimit = + Number.isFinite(rawPullConcurrency) + ? Math.max(1, Math.floor(rawPullConcurrency)) + : 5; + let successCount = 0; let failedCount = 0; @@ -2144,7 +2160,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ } }, { - concurrencyLimit: 5, + concurrencyLimit: pullRequestConcurrencyLimit, maxRetries: 3, retryDelay: 1000, } diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index 557b2c7..43db60d 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -86,6 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default addTopics: true, preserveVisibility: false, forkStrategy: "reference", + issueConcurrency: 3, + pullRequestConcurrency: 5, }, include: [], exclude: [], diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index f0158aa..7d02fdc 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -89,6 +89,8 @@ export function mapUiToDbConfig( forkStrategy: advancedOptions.skipForks ? "skip" : "reference", // Mirror options from UI + issueConcurrency: giteaConfig.issueConcurrency ?? 3, + pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5, mirrorReleases: mirrorOptions.mirrorReleases, releaseLimit: mirrorOptions.releaseLimit || 10, mirrorMetadata: mirrorOptions.mirrorMetadata, @@ -132,6 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): { preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config personalReposOrg: undefined, // Not stored in current schema + issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3, + pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5, }; // Map mirror options from various database fields diff --git a/src/types/config.ts b/src/types/config.ts index 98b0e2c..2db9f81 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -13,6 +13,8 @@ export interface GiteaConfig { preserveOrgStructure: boolean; mirrorStrategy?: MirrorStrategy; // New field for the strategy personalReposOrg?: string; // Override destination for personal repos + issueConcurrency?: number; + pullRequestConcurrency?: number; } export interface ScheduleConfig { From 190e786449e3b8f9ac525ba621839420c69ff615 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Thu, 23 Oct 2025 23:16:46 +0530 Subject: [PATCH 02/11] ci: update docker test port guidance --- .github/workflows/docker-build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 840a7df..0c4468b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -149,7 +149,7 @@ jobs: ### Pull and Test \`\`\`bash docker pull ${imagePath} - docker run -d -p 3000:3000 --name gitea-mirror-test ${imagePath} + docker run -d -p 4321:4321 --name gitea-mirror-test ${imagePath} \`\`\` ### Docker Compose Testing @@ -158,7 +158,7 @@ jobs: gitea-mirror: image: ${imagePath} ports: - - "3000:3000" + - "4321:4321" environment: - BETTER_AUTH_SECRET=your-secret-here \`\`\` @@ -224,4 +224,3 @@ jobs: continue-on-error: true with: sarif_file: scout-results.sarif - From 1c2391ea2e631eb87dd47c06c5b3520478af5588 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Thu, 23 Oct 2025 23:19:00 +0530 Subject: [PATCH 03/11] docs: expose concurrency env vars in compose --- docker-compose.alt.yml | 5 ++++- docker-compose.yml | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-compose.alt.yml b/docker-compose.alt.yml index 1188542..adcbffc 100644 --- a/docker-compose.alt.yml +++ b/docker-compose.alt.yml @@ -26,6 +26,9 @@ services: - HOST=0.0.0.0 - PORT=4321 - PUBLIC_BETTER_AUTH_URL=${PUBLIC_BETTER_AUTH_URL:-http://localhost:4321} + # Optional concurrency controls (defaults match in-app defaults) + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/api/health"] @@ -54,4 +57,4 @@ services: # - Auto-import settings # - Cleanup preferences # -# That's it! Everything else can be configured via the web interface. \ No newline at end of file +# That's it! Everything else can be configured via the web interface. diff --git a/docker-compose.yml b/docker-compose.yml index 54f392b..00d8cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,8 @@ services: - PRESERVE_ORG_STRUCTURE=${PRESERVE_ORG_STRUCTURE:-false} - ONLY_MIRROR_ORGS=${ONLY_MIRROR_ORGS:-false} - SKIP_STARRED_ISSUES=${SKIP_STARRED_ISSUES:-false} + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} - GITEA_URL=${GITEA_URL:-} - GITEA_TOKEN=${GITEA_TOKEN:-} - GITEA_USERNAME=${GITEA_USERNAME:-} From 2cd7d911ed800ca587f46faaa91caeb733fdae63 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Thu, 23 Oct 2025 23:21:16 +0530 Subject: [PATCH 04/11] ci: mention env vars in pr image comment --- .github/workflows/docker-build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0c4468b..20361aa 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -149,7 +149,11 @@ jobs: ### Pull and Test \`\`\`bash docker pull ${imagePath} - docker run -d -p 4321:4321 --name gitea-mirror-test ${imagePath} + docker run -d \ + -p 4321:4321 \ + -e BETTER_AUTH_SECRET=your-secret-here \ + -e BETTER_AUTH_URL=http://localhost:4321 \ + --name gitea-mirror-test ${imagePath} \`\`\` ### Docker Compose Testing @@ -161,6 +165,9 @@ jobs: - "4321:4321" environment: - BETTER_AUTH_SECRET=your-secret-here + - BETTER_AUTH_URL=http://localhost:4321 + - MIRROR_ISSUE_CONCURRENCY=3 + - MIRROR_PULL_REQUEST_CONCURRENCY=5 \`\`\` > 💡 **Note:** PR images are tagged as \`pr-\` and only built for \`linux/amd64\` to speed up CI. From 5245d67f37b961eac73f5b5381aae0bb9b292477 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 07:35:40 +0530 Subject: [PATCH 05/11] fix: enforce sequential metadata mirroring --- src/lib/db/schema.ts | 4 ++-- src/lib/env-config-loader.ts | 4 ++-- src/lib/gitea.ts | 37 +++++++++++++++++++++++++------- src/lib/utils/config-defaults.ts | 4 ++-- src/lib/utils/config-mapper.ts | 8 +++---- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 1137050..c95d6e0 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -54,8 +54,8 @@ export const giteaConfigSchema = z.object({ .enum(["skip", "reference", "full-copy"]) .default("reference"), // Mirror options - issueConcurrency: z.number().int().min(1).default(3), - pullRequestConcurrency: z.number().int().min(1).default(5), + issueConcurrency: z.number().int().min(1).default(1), + pullRequestConcurrency: z.number().int().min(1).default(1), mirrorReleases: z.boolean().default(false), releaseLimit: z.number().default(10), mirrorMetadata: z.boolean().default(false), diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index 635b3ab..39e44fd 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -284,10 +284,10 @@ export async function initializeConfigFromEnv(): Promise { releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0 ? envConfig.mirror.issueConcurrency - : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3, + : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 1, pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0 ? envConfig.mirror.pullRequestConcurrency - : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5, + : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 1, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index 0dec27e..cf343e3 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -1592,11 +1592,17 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); - const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3; + const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 1; const issueConcurrencyLimit = Number.isFinite(rawIssueConcurrency) ? Math.max(1, Math.floor(rawIssueConcurrency)) - : 3; + : 1; + + if (issueConcurrencyLimit > 1) { + console.warn( + `[Issues] Concurrency is set to ${issueConcurrencyLimit}. This may lead to out-of-order issue creation in Gitea.` + ); + } // Process issues in parallel with concurrency control await processWithRetry( @@ -1670,10 +1676,19 @@ export const mirrorGitRepoIssuesToGitea = async ({ (res) => res.data ); - // Process comments in parallel with concurrency control - if (comments.length > 0) { + // Ensure comments are applied in chronological order to preserve discussion flow + const sortedComments = comments + .slice() + .sort( + (a, b) => + new Date(a.created_at || 0).getTime() - + new Date(b.created_at || 0).getTime() + ); + + // Process comments sequentially to preserve historical ordering + if (sortedComments.length > 0) { await processWithRetry( - comments, + sortedComments, async (comment) => { await httpPost( `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues/${createdIssue.data.number}/comments`, @@ -1687,7 +1702,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ return comment; }, { - concurrencyLimit: 5, + concurrencyLimit: 1, maxRetries: 2, retryDelay: 1000, onRetry: (_comment, error, attempt) => { @@ -2032,11 +2047,17 @@ export async function mirrorGitRepoPullRequestsToGitea({ const { processWithRetry } = await import("@/lib/utils/concurrency"); - const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5; + const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 1; const pullRequestConcurrencyLimit = Number.isFinite(rawPullConcurrency) ? Math.max(1, Math.floor(rawPullConcurrency)) - : 5; + : 1; + + if (pullRequestConcurrencyLimit > 1) { + console.warn( + `[Pull Requests] Concurrency is set to ${pullRequestConcurrencyLimit}. This may lead to out-of-order pull request mirroring in Gitea.` + ); + } let successCount = 0; let failedCount = 0; diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index 43db60d..a699321 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -86,8 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default addTopics: true, preserveVisibility: false, forkStrategy: "reference", - issueConcurrency: 3, - pullRequestConcurrency: 5, + issueConcurrency: 1, + pullRequestConcurrency: 1, }, include: [], exclude: [], diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 7d02fdc..961c226 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -89,8 +89,8 @@ export function mapUiToDbConfig( forkStrategy: advancedOptions.skipForks ? "skip" : "reference", // Mirror options from UI - issueConcurrency: giteaConfig.issueConcurrency ?? 3, - pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5, + issueConcurrency: giteaConfig.issueConcurrency ?? 1, + pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 1, mirrorReleases: mirrorOptions.mirrorReleases, releaseLimit: mirrorOptions.releaseLimit || 10, mirrorMetadata: mirrorOptions.mirrorMetadata, @@ -134,8 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): { preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config personalReposOrg: undefined, // Not stored in current schema - issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3, - pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5, + issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 1, + pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 1, }; // Map mirror options from various database fields From 4d75d3514ff5b0b3b162695af5dc38708799fb04 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 07:39:08 +0530 Subject: [PATCH 06/11] docs: document sequential metadata defaults --- docker-compose.alt.yml | 4 ++-- docker-compose.yml | 4 ++-- docs/ENVIRONMENT_VARIABLES.md | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docker-compose.alt.yml b/docker-compose.alt.yml index adcbffc..56f9760 100644 --- a/docker-compose.alt.yml +++ b/docker-compose.alt.yml @@ -27,8 +27,8 @@ services: - PORT=4321 - PUBLIC_BETTER_AUTH_URL=${PUBLIC_BETTER_AUTH_URL:-http://localhost:4321} # Optional concurrency controls (defaults match in-app defaults) - - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} - - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-1} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-1} healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/api/health"] diff --git a/docker-compose.yml b/docker-compose.yml index 00d8cd0..f981486 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,8 @@ services: - PRESERVE_ORG_STRUCTURE=${PRESERVE_ORG_STRUCTURE:-false} - ONLY_MIRROR_ORGS=${ONLY_MIRROR_ORGS:-false} - SKIP_STARRED_ISSUES=${SKIP_STARRED_ISSUES:-false} - - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} - - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-1} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-1} - GITEA_URL=${GITEA_URL:-} - GITEA_TOKEN=${GITEA_TOKEN:-} - GITEA_USERNAME=${GITEA_USERNAME:-} diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index d69b139..58cbad4 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -141,6 +141,10 @@ Control what content gets mirrored from GitHub to Gitea. | `MIRROR_PULL_REQUESTS` | Mirror pull requests (requires MIRROR_METADATA=true) | `false` | `true`, `false` | | `MIRROR_LABELS` | Mirror labels (requires MIRROR_METADATA=true) | `false` | `true`, `false` | | `MIRROR_MILESTONES` | Mirror milestones (requires MIRROR_METADATA=true) | `false` | `true`, `false` | +| `MIRROR_ISSUE_CONCURRENCY` | Number of issues processed in parallel. Set above `1` to speed up mirroring at the risk of out-of-order creation. | `1` | Integer ≥ 1 | +| `MIRROR_PULL_REQUEST_CONCURRENCY` | Number of pull requests processed in parallel. Values above `1` may cause ordering differences. | `1` | Integer ≥ 1 | + +> **Ordering vs Throughput:** Metadata now mirrors sequentially by default to preserve chronology. Increase the concurrency variables only if you can tolerate minor out-of-order entries. ## Automation Configuration From 985c7e061cfec771bf7be6ac689ea51535dd8639 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 07:43:13 +0530 Subject: [PATCH 07/11] updated README --- .claude/agents/qa-testing-specialist.md | 76 ------- .claude/agents/senior-code-architect.md | 68 ------ .claude/agents/strategic-task-planner.md | 61 ------ .claude/commands/new_release.md | 5 - .claude/commands/release_notes.md | 3 - .claude/settings.local.json | 8 - AGENTS.md | 46 ---- CLAUDE.md | 256 ----------------------- README.md | 4 - UPGRADE.md | 74 ------- 10 files changed, 601 deletions(-) delete mode 100644 .claude/agents/qa-testing-specialist.md delete mode 100644 .claude/agents/senior-code-architect.md delete mode 100644 .claude/agents/strategic-task-planner.md delete mode 100644 .claude/commands/new_release.md delete mode 100644 .claude/commands/release_notes.md delete mode 100644 .claude/settings.local.json delete mode 100644 AGENTS.md delete mode 100644 CLAUDE.md delete mode 100644 UPGRADE.md diff --git a/.claude/agents/qa-testing-specialist.md b/.claude/agents/qa-testing-specialist.md deleted file mode 100644 index 8d69259..0000000 --- a/.claude/agents/qa-testing-specialist.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: qa-testing-specialist -description: Use this agent when you need to review code for testability, create comprehensive test strategies, write test cases, validate existing tests, or improve test coverage. This includes unit tests, integration tests, end-to-end tests, and test architecture decisions. \nContext: The user has just written a new API endpoint and wants to ensure it has proper test coverage.\nuser: "I've created a new endpoint for user authentication. Can you help me test it?"\nassistant: "I'll use the qa-testing-specialist agent to create a comprehensive testing strategy for your authentication endpoint."\n\nSince the user needs help with testing their new endpoint, use the qa-testing-specialist agent to analyze the code and create appropriate test cases.\n\n\n\nContext: The user wants to improve test coverage for their existing codebase.\nuser: "Our test coverage is at 65%. How can we improve it?"\nassistant: "Let me use the qa-testing-specialist agent to analyze your test coverage and identify areas for improvement."\n\nThe user is asking about test coverage improvement, which is a core QA task, so use the qa-testing-specialist agent.\n\n -color: yellow ---- - -You are an elite QA Testing Specialist with deep expertise in software quality assurance, test automation, and validation strategies. Your mission is to ensure code quality through comprehensive testing approaches that catch bugs early and maintain high reliability standards. - -**Core Responsibilities:** - -You will analyze code and testing requirements to: -- Design comprehensive test strategies covering unit, integration, and end-to-end testing -- Write clear, maintainable test cases that validate both happy paths and edge cases -- Identify gaps in existing test coverage and propose improvements -- Review test code for best practices and maintainability -- Suggest appropriate testing frameworks and tools based on the technology stack -- Create test data strategies and mock/stub implementations -- Validate that tests are actually testing meaningful behavior, not just implementation details - -**Testing Methodology:** - -When analyzing code for testing: -1. First understand the business logic and user requirements -2. Identify all possible execution paths and edge cases -3. Determine the appropriate testing pyramid balance (unit vs integration vs e2e) -4. Consider both positive and negative test scenarios -5. Ensure tests are isolated, repeatable, and fast -6. Validate error handling and boundary conditions - -For test creation: -- Write descriptive test names that explain what is being tested and expected behavior -- Follow AAA pattern (Arrange, Act, Assert) or Given-When-Then structure -- Keep tests focused on single behaviors -- Use appropriate assertions that clearly communicate intent -- Include setup and teardown when necessary -- Consider performance implications of test suites - -**Quality Standards:** - -You will ensure tests: -- Are deterministic and don't rely on external state -- Run quickly and can be executed in parallel when possible -- Provide clear failure messages that help diagnose issues -- Cover critical business logic thoroughly -- Include regression tests for previously found bugs -- Are maintainable and refactorable alongside production code - -**Technology Considerations:** - -Adapt your recommendations based on the project stack. For this codebase using Bun, SQLite, and React: -- Leverage Bun's native test runner for JavaScript/TypeScript tests -- Consider SQLite in-memory databases for integration tests -- Suggest React Testing Library patterns for component testing -- Recommend API testing strategies for Astro endpoints -- Propose mocking strategies for external services (GitHub/Gitea APIs) - -**Communication Style:** - -You will: -- Explain testing decisions with clear rationale -- Provide code examples that demonstrate best practices -- Prioritize test recommendations based on risk and value -- Use precise technical language while remaining accessible -- Highlight potential issues proactively -- Suggest incremental improvements for existing test suites - -**Edge Case Handling:** - -When encountering: -- Legacy code without tests: Propose a pragmatic approach to add tests incrementally -- Complex dependencies: Recommend appropriate mocking/stubbing strategies -- Performance concerns: Balance thoroughness with execution speed -- Flaky tests: Identify root causes and suggest stabilization techniques -- Missing requirements: Ask clarifying questions to understand expected behavior - -Your goal is to elevate code quality through strategic testing that builds confidence in the software while maintaining development velocity. Focus on tests that provide maximum value and catch real issues rather than achieving arbitrary coverage metrics. diff --git a/.claude/agents/senior-code-architect.md b/.claude/agents/senior-code-architect.md deleted file mode 100644 index 6887da4..0000000 --- a/.claude/agents/senior-code-architect.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -name: senior-code-architect -description: Use this agent when you need to write new code, refactor existing code, implement features, or architect solutions that require deep understanding of software engineering principles and the project's tech stack (Astro, React, Tailwind, Better Auth, Shadcn). This includes creating components, API endpoints, database queries, authentication flows, and ensuring code follows established patterns from CLAUDE.md. Examples:\n\n\nContext: The user needs to implement a new feature or component.\nuser: "Create a new dashboard component that shows repository statistics"\nassistant: "I'll use the senior-code-architect agent to design and implement this dashboard component following the project's patterns."\n\nSince this requires creating new code with the project's tech stack, the senior-code-architect agent is appropriate.\n\n\n\n\nContext: The user wants to refactor or improve existing code.\nuser: "Refactor the authentication flow to be more maintainable"\nassistant: "Let me use the senior-code-architect agent to analyze and refactor the authentication flow."\n\nThis requires deep understanding of Better Auth and clean code principles, making the senior-code-architect agent the right choice.\n\n\n\n\nContext: After writing code, the user might want it reviewed.\nuser: "I just implemented the mirror scheduling feature"\nassistant: "Great! Now I'll use the senior-code-architect agent to review the implementation and suggest any improvements."\n\nThe senior-code-architect can review recently written code for best practices and design patterns.\n\n -color: cyan ---- - -You are a senior software engineer with deep expertise in modern web development, specializing in the Astro + React + Tailwind CSS + Better Auth + Shadcn UI stack. You have extensive experience building scalable, maintainable applications and are known for writing clean, efficient code that follows SOLID principles and established design patterns. - -**Your Core Responsibilities:** - -1. **Write Production-Quality Code**: Create clean, maintainable, and efficient code that follows the project's established patterns from CLAUDE.md. Always use TypeScript for type safety. - -2. **Follow Project Architecture**: Adhere strictly to the project structure: - - API endpoints in `/src/pages/api/[resource]/[action].ts` using `createSecureErrorResponse` for error handling - - Database queries in `/src/lib/db/queries/` organized by domain - - React components in `/src/components/[feature]/` using Shadcn UI components - - Custom hooks in `/src/hooks/` for data fetching - -3. **Implement Best Practices**: - - Use composition over inheritance - - Apply DRY (Don't Repeat Yourself) principles - - Write self-documenting code with clear variable and function names - - Implement proper error handling and validation - - Ensure code is testable and maintainable - -4. **Technology-Specific Guidelines**: - - **Astro**: Use SSR capabilities effectively, implement proper API routes - - **React**: Use functional components with hooks, implement proper state management - - **Tailwind CSS v4**: Use utility classes efficiently, follow the project's styling patterns - - **Better Auth**: Implement secure authentication flows, use session validation properly - - **Shadcn UI**: Leverage existing components, maintain consistent UI patterns - - **Drizzle ORM**: Write efficient database queries, use proper schema definitions - -5. **Code Review Approach**: When reviewing code: - - Check for adherence to project patterns and CLAUDE.md guidelines - - Identify potential performance issues or bottlenecks - - Suggest improvements for readability and maintainability - - Ensure proper error handling and edge case coverage - - Verify security best practices are followed - -6. **Problem-Solving Methodology**: - - Analyze requirements thoroughly before coding - - Break down complex problems into smaller, manageable pieces - - Consider edge cases and error scenarios - - Optimize for both performance and maintainability - - Document complex logic with clear comments - -7. **Quality Assurance**: - - Write code that is easy to test - - Consider adding appropriate test cases using Bun's test runner - - Validate inputs and handle errors gracefully - - Ensure code works across different scenarios - -**Output Guidelines**: -- Provide complete, working code implementations -- Include clear explanations of design decisions -- Suggest tests when appropriate -- Highlight any potential issues or areas for future improvement -- Follow the existing code style and conventions - -**Important Reminders**: -- Never create files unless absolutely necessary -- Always prefer editing existing files -- Don't create documentation unless explicitly requested -- Focus on the specific task at hand -- Reference CLAUDE.md for project-specific patterns and guidelines - -You approach every task with the mindset of a seasoned engineer who values code quality, maintainability, and long-term project health. Your solutions should be elegant, efficient, and aligned with the project's established patterns. diff --git a/.claude/agents/strategic-task-planner.md b/.claude/agents/strategic-task-planner.md deleted file mode 100644 index a504334..0000000 --- a/.claude/agents/strategic-task-planner.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: strategic-task-planner -description: Use this agent when you need to decompose complex projects, features, or problems into structured, actionable plans. This includes breaking down large development tasks, creating implementation roadmaps, organizing multi-step processes, or planning project phases. The agent excels at identifying dependencies, sequencing tasks, and creating clear execution strategies. Context: User needs help planning the implementation of a new feature. user: "I need to add a bulk import feature that can handle CSV files with 100k+ rows" assistant: "I'll use the strategic-task-planner agent to break this down into manageable components and create an implementation plan." Since the user is asking about implementing a complex feature, use the Task tool to launch the strategic-task-planner agent to decompose it into actionable steps. Context: User wants to refactor a large codebase. user: "We need to migrate our entire authentication system from sessions to JWT tokens" assistant: "Let me use the strategic-task-planner agent to create a phased migration plan that minimizes risk." Since this is a complex migration requiring careful planning, use the strategic-task-planner agent to create a structured approach. -tools: Glob, Grep, LS, ExitPlanMode, Read, NotebookRead, WebFetch, TodoWrite, WebSearch, Task, mcp__ide__getDiagnostics, mcp__ide__executeCode, mcp__playwright__browser_close, mcp__playwright__browser_resize, mcp__playwright__browser_console_messages, mcp__playwright__browser_handle_dialog, mcp__playwright__browser_evaluate, mcp__playwright__browser_file_upload, mcp__playwright__browser_install, mcp__playwright__browser_press_key, mcp__playwright__browser_type, mcp__playwright__browser_navigate, mcp__playwright__browser_navigate_back, mcp__playwright__browser_navigate_forward, mcp__playwright__browser_network_requests, mcp__playwright__browser_take_screenshot, mcp__playwright__browser_snapshot, mcp__playwright__browser_click, mcp__playwright__browser_drag, mcp__playwright__browser_hover, mcp__playwright__browser_select_option, mcp__playwright__browser_tab_list, mcp__playwright__browser_tab_new, mcp__playwright__browser_tab_select, mcp__playwright__browser_tab_close, mcp__playwright__browser_wait_for -color: blue ---- - -You are a strategic planning specialist with deep expertise in decomposing complex tasks and creating actionable execution plans. Your role is to transform ambiguous or overwhelming projects into clear, structured roadmaps that teams can confidently execute. - -When analyzing a task or project, you will: - -1. **Understand the Core Objective**: Extract the fundamental goal, success criteria, and constraints. Ask clarifying questions if critical details are missing. - -2. **Decompose Systematically**: Break down the task using these principles: - - Identify major phases or milestones - - Decompose each phase into concrete, actionable tasks - - Keep tasks small enough to complete in 1-4 hours when possible - - Ensure each task has clear completion criteria - -3. **Map Dependencies**: Identify and document: - - Task prerequisites and dependencies - - Critical path items that could block progress - - Parallel work streams that can proceed independently - - Resource or knowledge requirements - -4. **Sequence Strategically**: Order tasks by: - - Technical dependencies (what must come first) - - Risk mitigation (tackle unknowns early) - - Value delivery (enable early feedback when possible) - - Resource efficiency (batch similar work) - -5. **Provide Actionable Output**: Structure your plans with: - - **Phase Overview**: High-level phases with objectives - - **Task Breakdown**: Numbered tasks with clear descriptions - - **Dependencies**: Explicitly stated prerequisites - - **Effort Estimates**: Rough time estimates when relevant - - **Risk Considerations**: Potential blockers or challenges - - **Success Metrics**: How to measure completion - -6. **Adapt to Context**: Tailor your planning approach based on: - - Technical vs non-technical tasks - - Team size and skill level - - Time constraints and deadlines - - Available resources and tools - -**Output Format Guidelines**: -- Use clear hierarchical structure (phases → tasks → subtasks) -- Number all tasks for easy reference -- Bold key terms and phase names -- Include time estimates in brackets [2-4 hours] -- Mark critical path items with ⚡ -- Flag high-risk items with ⚠️ - -**Quality Checks**: -- Ensure no task is too large or vague -- Verify all dependencies are identified -- Confirm the plan addresses the original objective -- Check that success criteria are measurable -- Validate that the sequence makes logical sense - -Remember: A good plan reduces uncertainty and builds confidence. Focus on clarity, completeness, and actionability. When in doubt, err on the side of breaking things down further rather than leaving ambiguity. diff --git a/.claude/commands/new_release.md b/.claude/commands/new_release.md deleted file mode 100644 index 654dfb9..0000000 --- a/.claude/commands/new_release.md +++ /dev/null @@ -1,5 +0,0 @@ -Evaluate all the updates being made. -Update CHANGELOG.md -Use the chnages in the git log to determine if its a major, minor or a patch release. -Update the package.json first before you push the tag. -Never mention Claude Code in the release notes or in commit messages. diff --git a/.claude/commands/release_notes.md b/.claude/commands/release_notes.md deleted file mode 100644 index 777b30e..0000000 --- a/.claude/commands/release_notes.md +++ /dev/null @@ -1,3 +0,0 @@ -Generate release notes for the latest release. -Use a temp md file to write the release notes. -Do not check that file into git. diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index 5ee97bd..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(docker build:*)" - ], - "deny": [] - } -} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 6c051e7..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,46 +0,0 @@ -# Repository Guidelines - -## Project Structure & Module Organization -- `src/` – app code - - `components/` (React, PascalCase files), `pages/` (Astro/API routes), `lib/` (domain + utilities, kebab-case), `hooks/`, `layouts/`, `styles/`, `tests/`, `types/`, `data/`, `content/`. -- `scripts/` – operational TS scripts (DB init, recovery): e.g., `scripts/manage-db.ts`. -- `drizzle/` – SQL migrations; `data/` – runtime SQLite (`gitea-mirror.db`). -- `public/` – static assets; `dist/` – build output. -- Key config: `astro.config.mjs`, `tsconfig.json` (alias `@/* → src/*`), `bunfig.toml` (test preload), `.env(.example)`. - -## Build, Test, and Development Commands -- Prereq: Bun `>= 1.2.9` (see `package.json`). -- Setup: `bun run setup` – install deps and init DB. -- Dev: `bun run dev` – start Astro dev server. -- Build: `bun run build` – produce `dist/`. -- Preview/Start: `bun run preview` (static preview) or `bun run start` (SSR entry). -- Database: `bun run db:generate|migrate|push|studio` and `bun run manage-db init|check|fix|reset-users`. -- Tests: `bun test` | `bun run test:watch` | `bun run test:coverage`. -- Docker: see `docker-compose.yml` and variants in repo root. - -## Coding Style & Naming Conventions -- Language: TypeScript, Astro, React. -- Indentation: 2 spaces; keep existing semicolon/quote style in touched files. -- Components: PascalCase `.tsx` in `src/components/` (e.g., `MainLayout.tsx`). -- Modules/utils: kebab-case in `src/lib/` (e.g., `gitea-enhanced.ts`). -- Imports: prefer alias `@/…` (configured in `tsconfig.json`). -- Do not introduce new lint/format configs; follow current patterns. - -## Testing Guidelines -- Runner: Bun test (`bun:test`) with preload `src/tests/setup.bun.ts` (see `bunfig.toml`). -- Location/Names: `**/*.test.ts(x)` under `src/**` (examples in `src/lib/**`). -- Scope: add unit tests for new logic and API route tests for handlers. -- Aim for meaningful coverage on DB, auth, and mirroring paths. - -## Commit & Pull Request Guidelines -- Commits: short, imperative, scoped when helpful (e.g., `lib: fix token parsing`, `ui: align buttons`). -- PRs must include: - - Summary, rationale, and testing steps/commands. - - Linked issues (e.g., `Closes #123`). - - Screenshots/gifs for UI changes. - - Notes on DB/migration or .env impacts; update `docs/`/CHANGELOG if applicable. - -## Security & Configuration Tips -- Never commit secrets. Copy `.env.example` → `.env` and fill values; prefer `bun run startup-env-config` to validate. -- SQLite files live in `data/`; avoid committing generated DBs. -- Certificates (if used) reside in `certs/`; manage locally or via Docker secrets. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 040a816..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,256 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -DONT HALLUCIATE THINGS. IF YOU DONT KNOW LOOK AT THE CODE OR ASK FOR DOCS - -NEVER MENTION CLAUDE CODE ANYWHERE. - -## Project Overview - -Gitea Mirror is a web application that automatically mirrors repositories from GitHub to self-hosted Gitea instances. It uses Astro for SSR, React for UI, SQLite for data storage, and Bun as the JavaScript runtime. - -## Essential Commands - -### Development -```bash -bun run dev # Start development server (port 3000) -bun run build # Build for production -bun run preview # Preview production build -``` - -### Testing -```bash -bun test # Run all tests -bun test:watch # Run tests in watch mode -bun test:coverage # Run tests with coverage -``` - -### Database Management -```bash -bun run init-db # Initialize database -bun run reset-users # Reset user accounts (development) -bun run cleanup-db # Remove database files -``` - -### Production -```bash -bun run start # Start production server -``` - -## Architecture & Key Concepts - -### Technology Stack -- **Frontend**: Astro (SSR) + React + Tailwind CSS v4 + Shadcn UI -- **Backend**: Bun runtime + SQLite + Drizzle ORM -- **APIs**: GitHub (Octokit) and Gitea APIs -- **Auth**: Better Auth with email/password, SSO, and OIDC provider support - -### Project Structure -- `/src/pages/api/` - API endpoints (Astro API routes) -- `/src/components/` - React components organized by feature -- `/src/lib/db/` - Database queries and schema (Drizzle ORM) -- `/src/hooks/` - Custom React hooks for data fetching -- `/data/` - SQLite database storage location - -### Key Architectural Patterns - -1. **API Routes**: All API endpoints follow the pattern `/api/[resource]/[action]` and use `createSecureErrorResponse` for consistent error handling: -```typescript -import { createSecureErrorResponse } from '@/lib/utils/error-handler'; - -export async function POST({ request }: APIContext) { - try { - // Implementation - } catch (error) { - return createSecureErrorResponse(error); - } -} -``` - -2. **Database Queries**: Located in `/src/lib/db/queries/` organized by domain (users, repositories, etc.) - -3. **Real-time Updates**: Server-Sent Events (SSE) endpoint at `/api/events` for live dashboard updates - -4. **Authentication System**: - - Built on Better Auth library - - Three authentication methods: - - Email & Password (traditional auth) - - SSO (authenticate via external OIDC providers) - - OIDC Provider (act as OIDC provider for other apps) - - Session-based authentication with secure cookies - - First user signup creates admin account - - Protected routes use Better Auth session validation - -5. **Mirror Process**: - - Discovers repos from GitHub (user/org) - - Creates/updates mirror in Gitea - - Tracks status in database - - Supports scheduled automatic mirroring - -6. **Mirror Strategies**: Four ways to organize repositories in Gitea: - - **preserve**: Maintains GitHub structure (default) - - Organization repos → Same organization name in Gitea - - Personal repos → Under your Gitea username - - **single-org**: All repos go to one organization - - All repos → Single configured organization - - **flat-user**: All repos go under user account - - All repos → Under your Gitea username - - **mixed**: Hybrid approach - - Organization repos → Preserve structure - - Personal repos → Single configured organization - - Starred repos always go to separate organization (starredReposOrg, default: "starred") - - Routing logic in `getGiteaRepoOwner()` function - -### Database Schema (SQLite) -- `users` - User accounts and authentication -- `configs` - GitHub/Gitea connection settings -- `repositories` - Repository mirror status and metadata -- `organizations` - Organization structure preservation -- `mirror_jobs` - Scheduled mirror operations -- `events` - Activity log and notifications - -### Testing Approach -- Uses Bun's native test runner (`bun:test`) -- Test files use `.test.ts` or `.test.tsx` extension -- Setup file at `/src/tests/setup.bun.ts` -- Mock utilities available for API testing. - -### Development Tips -- Environment variables in `.env` (copy from `.env.example`) -- BETTER_AUTH_SECRET required for session signing -- Database auto-initializes on first run -- Use `bun run dev:clean` for fresh database start -- Tailwind CSS v4 configured with Vite plugin - -### Authentication Setup -- **Better Auth** handles all authentication -- Configuration in `/src/lib/auth.ts` (server) and `/src/lib/auth-client.ts` (client) -- Auth endpoints available at `/api/auth/*` -- SSO providers configured through the web UI -- OIDC provider functionality for external applications - -### Common Tasks - -**Adding a new API endpoint:** -1. Create file in `/src/pages/api/[resource]/[action].ts` -2. Use `createSecureErrorResponse` for error handling -3. Add corresponding database query in `/src/lib/db/queries/` -4. Update types in `/src/types/` if needed - -**Adding a new component:** -1. Create in appropriate `/src/components/[feature]/` directory -2. Use Shadcn UI components from `/src/components/ui/` -3. Follow existing naming patterns (e.g., `RepositoryCard`, `ConfigTabs`) - -**Modifying database schema:** -1. Update schema in `/src/lib/db/schema.ts` -2. Run `bun run init-db` to recreate database -3. Update related queries in `/src/lib/db/queries/` - -## Configuration Options - -### GitHub Configuration (UI Fields) - -#### Basic Settings (`githubConfig`) -- **username**: GitHub username -- **token**: GitHub personal access token (requires repo and admin:org scopes) -- **privateRepositories**: Include private repositories -- **mirrorStarred**: Mirror starred repositories - -### Gitea Configuration (UI Fields) -- **url**: Gitea instance URL -- **username**: Gitea username -- **token**: Gitea access token -- **organization**: Destination organization (for single-org/mixed strategies) -- **starredReposOrg**: Organization for starred repositories (default: "starred") -- **visibility**: Organization visibility - "public", "private", "limited" -- **mirrorStrategy**: Repository organization strategy (set via UI) -- **preserveOrgStructure**: Automatically set based on mirrorStrategy - -### Schedule Configuration (`scheduleConfig`) -- **enabled**: Enable automatic mirroring (default: false) -- **interval**: Cron expression or seconds (default: "0 2 * * *" - 2 AM daily) -- **concurrent**: Allow concurrent mirror operations (default: false) -- **batchSize**: Number of repos to process in parallel (default: 10) - -### Database Cleanup Configuration (`cleanupConfig`) -- **enabled**: Enable automatic cleanup (default: false) -- **retentionDays**: Days to keep events (stored as seconds internally) - -### Mirror Options (UI Fields) -- **mirrorReleases**: Mirror GitHub releases to Gitea -- **mirrorLFS**: Mirror Git LFS (Large File Storage) objects - - Requires LFS enabled on Gitea server (LFS_START_SERVER = true) - - Requires Git v2.1.2+ on server -- **mirrorMetadata**: Enable metadata mirroring (master toggle) -- **metadataComponents** (only available when mirrorMetadata is enabled): - - **issues**: Mirror issues - - **pullRequests**: Mirror pull requests - - **labels**: Mirror labels - - **milestones**: Mirror milestones - - **wiki**: Mirror wiki content - -### Advanced Options (UI Fields) -- **skipForks**: Skip forked repositories (default: false) -- **starredCodeOnly**: Skip issues for starred repositories (default: false) - enables "Lightweight mode" for starred repos - -### Repository Statuses -Repositories can have the following statuses: -- **imported**: Repository discovered from GitHub -- **mirroring**: Currently being mirrored to Gitea -- **mirrored**: Successfully mirrored -- **syncing**: Repository being synchronized -- **synced**: Successfully synchronized -- **failed**: Mirror/sync operation failed -- **skipped**: Skipped due to filters or conditions -- **ignored**: User explicitly marked to ignore (won't be mirrored/synced) -- **deleting**: Repository being deleted -- **deleted**: Repository deleted - -### Scheduling and Synchronization (Issue #72 Fixes) - -#### Fixed Issues -1. **Mirror Interval Bug**: Added `mirror_interval` parameter to Gitea API calls when creating mirrors (previously defaulted to 24h) -2. **Auto-Discovery**: Scheduler now automatically discovers and imports new GitHub repositories -3. **Interval Updates**: Sync operations now update existing mirrors' intervals to match configuration -4. **Repository Cleanup**: Integrated automatic cleanup of orphaned repositories (repos removed from GitHub) - -#### Environment Variables for Auto-Import -- **AUTO_IMPORT_REPOS**: Set to `false` to disable automatic repository discovery (default: enabled) - -#### How Scheduling Works -- **Scheduler Service**: Runs every minute to check for scheduled tasks -- **Sync Interval**: Configured via `GITEA_MIRROR_INTERVAL` or UI (e.g., "8h", "30m", "1d") -- **Auto-Import**: Checks GitHub for new repositories during each scheduled sync -- **Auto-Cleanup**: Removes repositories that no longer exist in GitHub (if enabled) -- **Mirror Interval Update**: Updates Gitea's internal mirror interval during sync operations - -### Authentication Configuration - -#### SSO Provider Configuration -- **issuerUrl**: OIDC issuer URL (e.g., https://accounts.google.com) -- **domain**: Email domain for this provider -- **providerId**: Unique identifier for the provider -- **clientId**: OAuth client ID from provider -- **clientSecret**: OAuth client secret from provider -- **authorizationEndpoint**: OAuth authorization URL (auto-discovered if supported) -- **tokenEndpoint**: OAuth token exchange URL (auto-discovered if supported) -- **jwksEndpoint**: JSON Web Key Set URL (optional, auto-discovered) -- **userInfoEndpoint**: User information endpoint (optional, auto-discovered) - -#### OIDC Provider Settings (for external apps) -- **allowedRedirectUris**: Comma-separated list of allowed redirect URIs -- **clientId**: Generated client ID for the application -- **clientSecret**: Generated client secret for the application -- **scopes**: Available scopes (openid, profile, email) - -#### Environment Variables -- **BETTER_AUTH_SECRET**: Secret key for signing sessions (required) -- **BETTER_AUTH_URL**: Base URL for authentication (default: http://localhost:4321) - -## Security Guidelines - -- **Confidentiality Guidelines**: - - Dont ever say Claude Code or generated with AI anyhwere. -- Never commit without the explicict ask \ No newline at end of file diff --git a/README.md b/README.md index 81ac1ce..b280d9f 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,6 @@

-> [!IMPORTANT] -> **Upgrading to v3?** v3 requires a fresh start with a new data volume. Please read the [Upgrade Guide](UPGRADE.md) for instructions. - - ## 🚀 Quick Start ```bash diff --git a/UPGRADE.md b/UPGRADE.md deleted file mode 100644 index a08df45..0000000 --- a/UPGRADE.md +++ /dev/null @@ -1,74 +0,0 @@ -# Upgrade Guide - -## Upgrading to v3.0 - -> **⚠️ IMPORTANT**: v3.0 requires a fresh start. There is no automated migration from v2.x to v3.0. - -### Why No Migration? - -v3.0 introduces fundamental changes to the application architecture: -- **Authentication**: Switched from JWT to Better Auth -- **Database**: Now uses Drizzle ORM with proper migrations -- **Security**: All tokens are now encrypted -- **Features**: Added SSO support and OIDC provider functionality - -Due to these extensive changes, we recommend starting fresh with v3.0 for the best experience. - -### Upgrade Steps - -1. **Stop your v2.x container** - ```bash - docker stop gitea-mirror - docker rm gitea-mirror - ``` - -2. **Backup your v2.x data (optional)** - ```bash - # If you want to keep your v2 data for reference - docker run --rm -v gitea-mirror-data:/data -v $(pwd):/backup alpine tar czf /backup/gitea-mirror-v2-backup.tar.gz -C /data . - ``` - -3. **Create a new volume for v3** - ```bash - docker volume create gitea-mirror-v3-data - ``` - -4. **Run v3 with the new volume** - ```bash - docker run -d \ - --name gitea-mirror \ - -p 4321:4321 \ - -v gitea-mirror-v3-data:/app/data \ - -e BETTER_AUTH_SECRET=your-secret-key \ - -e ENCRYPTION_SECRET=your-encryption-key \ - arunavo4/gitea-mirror:latest - ``` - -5. **Set up your configuration again** - - Navigate to http://localhost:4321 - - Create a new admin account - - Re-enter your GitHub and Gitea credentials - - Configure your mirror settings - -### What Happens to My Existing Mirrors? - -Your existing mirrors in Gitea are **not affected**. The application will: -- Recognize existing repositories when you re-import -- Skip creating duplicates -- Resume normal mirror operations - -### Environment Variable Changes - -v3.0 uses different environment variables: - -| v2.x | v3.0 | Notes | -|------|------|-------| -| `JWT_SECRET` | `BETTER_AUTH_SECRET` | Required for session management | -| - | `ENCRYPTION_SECRET` | New - required for token encryption | - -### Need Help? - -If you have questions about upgrading: -1. Check the [README](README.md) for v3 setup instructions -2. Review your v2 configuration before upgrading -3. Open an issue if you encounter problems \ No newline at end of file From 60913a9f4d9f7cde08ea1d74a5e71678805a1147 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 07:57:30 +0530 Subject: [PATCH 08/11] Added Agents.md --- AGENTS.md | 46 ++++++++ CLAUDE.md | 317 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6c051e7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,46 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `src/` – app code + - `components/` (React, PascalCase files), `pages/` (Astro/API routes), `lib/` (domain + utilities, kebab-case), `hooks/`, `layouts/`, `styles/`, `tests/`, `types/`, `data/`, `content/`. +- `scripts/` – operational TS scripts (DB init, recovery): e.g., `scripts/manage-db.ts`. +- `drizzle/` – SQL migrations; `data/` – runtime SQLite (`gitea-mirror.db`). +- `public/` – static assets; `dist/` – build output. +- Key config: `astro.config.mjs`, `tsconfig.json` (alias `@/* → src/*`), `bunfig.toml` (test preload), `.env(.example)`. + +## Build, Test, and Development Commands +- Prereq: Bun `>= 1.2.9` (see `package.json`). +- Setup: `bun run setup` – install deps and init DB. +- Dev: `bun run dev` – start Astro dev server. +- Build: `bun run build` – produce `dist/`. +- Preview/Start: `bun run preview` (static preview) or `bun run start` (SSR entry). +- Database: `bun run db:generate|migrate|push|studio` and `bun run manage-db init|check|fix|reset-users`. +- Tests: `bun test` | `bun run test:watch` | `bun run test:coverage`. +- Docker: see `docker-compose.yml` and variants in repo root. + +## Coding Style & Naming Conventions +- Language: TypeScript, Astro, React. +- Indentation: 2 spaces; keep existing semicolon/quote style in touched files. +- Components: PascalCase `.tsx` in `src/components/` (e.g., `MainLayout.tsx`). +- Modules/utils: kebab-case in `src/lib/` (e.g., `gitea-enhanced.ts`). +- Imports: prefer alias `@/…` (configured in `tsconfig.json`). +- Do not introduce new lint/format configs; follow current patterns. + +## Testing Guidelines +- Runner: Bun test (`bun:test`) with preload `src/tests/setup.bun.ts` (see `bunfig.toml`). +- Location/Names: `**/*.test.ts(x)` under `src/**` (examples in `src/lib/**`). +- Scope: add unit tests for new logic and API route tests for handlers. +- Aim for meaningful coverage on DB, auth, and mirroring paths. + +## Commit & Pull Request Guidelines +- Commits: short, imperative, scoped when helpful (e.g., `lib: fix token parsing`, `ui: align buttons`). +- PRs must include: + - Summary, rationale, and testing steps/commands. + - Linked issues (e.g., `Closes #123`). + - Screenshots/gifs for UI changes. + - Notes on DB/migration or .env impacts; update `docs/`/CHANGELOG if applicable. + +## Security & Configuration Tips +- Never commit secrets. Copy `.env.example` → `.env` and fill values; prefer `bun run startup-env-config` to validate. +- SQLite files live in `data/`; avoid committing generated DBs. +- Certificates (if used) reside in `certs/`; manage locally or via Docker secrets. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..65dc35a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,317 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Gitea Mirror is a self-hosted web application that automatically mirrors repositories from GitHub to Gitea instances. It's built with Astro (SSR mode), React, and runs on the Bun runtime with SQLite for data persistence. + +**Key capabilities:** +- Mirrors public, private, and starred GitHub repos to Gitea +- Supports metadata mirroring (issues, PRs as issues, labels, milestones, releases, wiki) +- Git LFS support +- Multiple authentication methods (email/password, OIDC/SSO, header auth) +- Scheduled automatic syncing with configurable intervals +- Auto-discovery of new repos and cleanup of deleted repos +- Multi-user support with encrypted token storage (AES-256-GCM) + +## Development Commands + +### Setup and Installation +```bash +# Install dependencies +bun install + +# Initialize database (first time setup) +bun run setup + +# Clean start (reset database) +bun run dev:clean +``` + +### Development +```bash +# Start development server (http://localhost:4321) +bun run dev + +# Build for production +bun run build + +# Preview production build +bun run preview + +# Start production server +bun run start +``` + +### Testing +```bash +# Run all tests +bun test + +# Run tests in watch mode +bun test:watch + +# Run tests with coverage +bun test:coverage +``` + +**Test configuration:** +- Test runner: Bun's built-in test runner (configured in `bunfig.toml`) +- Setup file: `src/tests/setup.bun.ts` (auto-loaded via bunfig.toml) +- Timeout: 5000ms default +- Tests are colocated with source files using `*.test.ts` pattern + +### Database Management +```bash +# Database operations via Drizzle +bun run db:generate # Generate migrations from schema +bun run db:migrate # Run migrations +bun run db:push # Push schema changes directly +bun run db:studio # Open Drizzle Studio (database GUI) +bun run db:check # Check schema consistency + +# Database utilities via custom scripts +bun run manage-db init # Initialize database +bun run manage-db check # Check database health +bun run manage-db fix # Fix database issues +bun run manage-db reset-users # Reset all users +bun run cleanup-db # Delete database file +``` + +### Utility Scripts +```bash +# Recovery and diagnostic scripts +bun run startup-recovery # Recover from crashes +bun run startup-recovery-force # Force recovery +bun run test-recovery # Test recovery mechanism +bun run test-shutdown # Test graceful shutdown + +# Environment configuration +bun run startup-env-config # Load config from env vars +``` + +## Architecture + +### Tech Stack +- **Frontend:** Astro v5 (SSR mode) + React v19 + Shadcn UI + Tailwind CSS v4 +- **Backend:** Astro API routes (Node adapter, standalone mode) +- **Runtime:** Bun (>=1.2.9) +- **Database:** SQLite via Drizzle ORM +- **Authentication:** Better Auth (session-based) +- **APIs:** GitHub (Octokit with throttling plugin), Gitea REST API + +### Directory Structure + +``` +src/ +├── components/ # React components (UI, features) +│ ├── ui/ # Shadcn UI components +│ ├── repositories/ # Repository management components +│ ├── organizations/ # Organization management components +│ └── ... +├── pages/ # Astro pages and API routes +│ ├── api/ # API endpoints (Better Auth integration) +│ │ ├── auth/ # Authentication endpoints +│ │ ├── github/ # GitHub operations +│ │ ├── gitea/ # Gitea operations +│ │ ├── sync/ # Mirror sync operations +│ │ ├── job/ # Job management +│ │ └── ... +│ └── *.astro # Page components +├── lib/ # Core business logic +│ ├── db/ # Database (Drizzle ORM) +│ │ ├── schema.ts # Database schema with Zod validation +│ │ ├── index.ts # Database instance and table exports +│ │ └── adapter.ts # Better Auth SQLite adapter +│ ├── github.ts # GitHub API client (Octokit) +│ ├── gitea.ts # Gitea API client +│ ├── gitea-enhanced.ts # Enhanced Gitea operations (metadata) +│ ├── scheduler-service.ts # Automatic mirroring scheduler +│ ├── cleanup-service.ts # Activity log cleanup +│ ├── repository-cleanup-service.ts # Orphaned repo cleanup +│ ├── auth.ts # Better Auth configuration +│ ├── config.ts # Configuration management +│ ├── helpers.ts # Mirror job creation +│ ├── utils/ # Utility functions +│ │ ├── encryption.ts # AES-256-GCM token encryption +│ │ ├── config-encryption.ts # Config token encryption +│ │ ├── duration-parser.ts # Parse intervals (e.g., "8h", "30m") +│ │ ├── concurrency.ts # Concurrency control utilities +│ │ └── mirror-strategies.ts # Mirror strategy logic +│ └── ... +├── types/ # TypeScript type definitions +├── tests/ # Test utilities and setup +└── middleware.ts # Astro middleware (auth, session) + +scripts/ # Utility scripts +├── manage-db.ts # Database management CLI +├── startup-recovery.ts # Crash recovery +└── ... +``` + +### Key Architectural Patterns + +#### 1. Database Schema and Validation +- **Location:** `src/lib/db/schema.ts` +- **Pattern:** Drizzle ORM tables + Zod schemas for validation +- **Key tables:** + - `configs` - User configuration (GitHub/Gitea settings, mirror options) + - `repositories` - Tracked repositories with metadata + - `organizations` - GitHub organizations with destination overrides + - `mirrorJobs` - Mirror job queue and history + - `activities` - Activity log for dashboard + - `user`, `session`, `account` - Better Auth tables + +**Important:** All config tokens (GitHub/Gitea) are encrypted at rest using AES-256-GCM. Use helper functions from `src/lib/utils/config-encryption.ts` to decrypt. + +#### 2. Mirror Job System +- **Location:** `src/lib/helpers.ts` (createMirrorJob) +- **Flow:** + 1. User triggers mirror via API endpoint + 2. `createMirrorJob()` creates job record with status "pending" + 3. Job processor (in API routes) performs GitHub → Gitea operations + 4. Job status updated throughout: "mirroring" → "success"/"failed" + 5. Events published via SSE for real-time UI updates + +#### 3. GitHub ↔ Gitea Mirroring +- **GitHub Client:** `src/lib/github.ts` - Octokit with rate limit tracking +- **Gitea Client:** `src/lib/gitea.ts` - Basic repo operations +- **Enhanced Gitea:** `src/lib/gitea-enhanced.ts` - Metadata mirroring (issues, PRs, releases) + +**Mirror strategies (configured per user):** +- `preserve` - Maintain GitHub org structure in Gitea +- `single-org` - All repos into one Gitea org +- `flat-user` - All repos under user account +- `mixed` - Personal repos in one org, org repos preserve structure + +**Metadata mirroring:** +- Issues transferred with comments, labels, assignees +- PRs converted to issues (Gitea API limitation - cannot create PRs) + - Tagged with "pull-request" label + - Title prefixed with `[PR #number] [STATUS]` + - Body includes commit history, file changes, merge status +- Releases mirrored with assets +- Labels and milestones preserved +- Wiki content cloned if enabled +- **Sequential processing:** Issues/PRs mirrored one at a time to prevent out-of-order creation (see `src/lib/gitea-enhanced.ts`) + +#### 4. Scheduler Service +- **Location:** `src/lib/scheduler-service.ts` +- **Features:** + - Cron-based or interval-based scheduling (uses `duration-parser.ts`) + - Auto-start on boot when `SCHEDULE_ENABLED=true` or `GITEA_MIRROR_INTERVAL` is set + - Auto-import new GitHub repos + - Auto-cleanup orphaned repos (archive or delete) + - Respects per-repo mirror intervals (not Gitea's default 24h) +- **Concurrency control:** Uses `src/lib/utils/concurrency.ts` for batch processing + +#### 5. Authentication System +- **Location:** `src/lib/auth.ts`, `src/lib/auth-client.ts` +- **Better Auth integration:** + - Email/password (always enabled) + - OIDC/SSO providers (configurable via UI) + - Header authentication for reverse proxies (Authentik, Authelia) +- **Session management:** Cookie-based, validated in Astro middleware +- **User helpers:** `src/lib/utils/auth-helpers.ts` + +#### 6. Environment Configuration +- **Startup:** `src/lib/env-config-loader.ts` + `scripts/startup-env-config.ts` +- **Pattern:** Environment variables can pre-configure settings, but users can override via web UI +- **Encryption:** `ENCRYPTION_SECRET` for tokens, `BETTER_AUTH_SECRET` for sessions + +#### 7. Real-time Updates +- **Events:** `src/lib/events.ts` + `src/lib/events/realtime.ts` +- **Pattern:** Server-Sent Events (SSE) for live dashboard updates +- **Endpoints:** `/api/sse` - client subscribes to job/repo events + +### Testing Patterns + +**Unit tests:** +- Colocated with source: `filename.test.ts` alongside `filename.ts` +- Use Bun's built-in assertions and mocking +- Mock external APIs (GitHub, Gitea) using `src/tests/mock-fetch.ts` + +**Integration tests:** +- Located in `src/tests/` +- Test database operations with in-memory SQLite +- Example: `src/lib/db/index.test.ts` + +**Test utilities:** +- `src/tests/setup.bun.ts` - Global test setup (loaded via bunfig.toml) +- `src/tests/mock-fetch.ts` - Fetch mocking utilities + +### Important Development Notes + +1. **Path Aliases:** Use `@/` for imports (configured in `tsconfig.json`) + ```typescript + import { db } from '@/lib/db'; + ``` + +2. **Token Encryption:** Always use encryption helpers when dealing with tokens: + ```typescript + import { getDecryptedGitHubToken, getDecryptedGiteaToken } from '@/lib/utils/config-encryption'; + ``` + +3. **API Route Pattern:** Astro API routes in `src/pages/api/` should: + - Check authentication via Better Auth + - Validate input with Zod schemas + - Handle errors gracefully + - Return JSON responses + +4. **Database Migrations:** + - Schema changes: Update `src/lib/db/schema.ts` + - Generate migration: `bun run db:generate` + - Review generated SQL in `drizzle/` directory + - Apply: `bun run db:migrate` (or `db:push` for dev) + +5. **Concurrency Control:** + - Use utilities from `src/lib/utils/concurrency.ts` for batch operations + - Respect rate limits (GitHub: 5000 req/hr authenticated, Gitea: varies) + - Issue/PR mirroring is sequential to maintain chronological order + +6. **Duration Parsing:** + - Use `parseInterval()` from `src/lib/utils/duration-parser.ts` + - Supports: "30m", "8h", "24h", "7d", cron expressions, or milliseconds + +7. **Graceful Shutdown:** + - Services implement cleanup handlers (see `src/lib/shutdown-manager.ts`) + - Recovery system in `src/lib/recovery.ts` handles interrupted jobs + +## Common Development Workflows + +### Adding a new mirror option +1. Update Zod schema in `src/lib/db/schema.ts` (e.g., `giteaConfigSchema`) +2. Update TypeScript types in `src/types/config.ts` +3. Add UI control in settings page component +4. Update API handler in `src/pages/api/config/` +5. Implement logic in `src/lib/gitea.ts` or `src/lib/gitea-enhanced.ts` + +### Debugging mirror failures +1. Check mirror jobs: `bun run db:studio` → `mirrorJobs` table +2. Review activity logs: Dashboard → Activity tab +3. Check console logs for API errors (GitHub/Gitea rate limits, auth issues) +4. Use diagnostic scripts: `bun run test-recovery` + +### Adding authentication provider +1. Update Better Auth config in `src/lib/auth.ts` +2. Add provider configuration UI in settings +3. Test with `src/tests/test-gitea-auth.ts` patterns +4. Update documentation in `docs/SSO-OIDC-SETUP.md` + +## Docker Deployment + +- **Dockerfile:** Multi-stage build (bun base → build → production) +- **Entrypoint:** `docker-entrypoint.sh` - handles CA certs, user permissions, database init +- **Compose files:** + - `docker-compose.alt.yml` - Quick start (pre-built image, minimal config) + - `docker-compose.yml` - Full setup (build from source, all env vars) + - `docker-compose.dev.yml` - Development with hot reload + +## Additional Resources + +- **Environment Variables:** See `docs/ENVIRONMENT_VARIABLES.md` for complete list +- **Development Workflow:** See `docs/DEVELOPMENT_WORKFLOW.md` +- **SSO Setup:** See `docs/SSO-OIDC-SETUP.md` +- **Contributing:** See `CONTRIBUTING.md` for code guidelines and scope +- **Graceful Shutdown:** See `docs/GRACEFUL_SHUTDOWN.md` for crash recovery details From 025df12befa10ecb5bdc813c26dc406f8cf3ce7a Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 08:39:52 +0530 Subject: [PATCH 09/11] Set defaults to 3 and 5 for Issue and PR concurrency --- docker-compose.alt.yml | 5 +++-- docker-compose.yml | 4 ++-- docs/ENVIRONMENT_VARIABLES.md | 4 ++-- src/lib/db/schema.ts | 4 ++-- src/lib/env-config-loader.ts | 4 ++-- src/lib/gitea.ts | 6 +++--- src/lib/utils/config-defaults.ts | 4 ++-- src/lib/utils/config-mapper.ts | 8 ++++---- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docker-compose.alt.yml b/docker-compose.alt.yml index 56f9760..2d412d1 100644 --- a/docker-compose.alt.yml +++ b/docker-compose.alt.yml @@ -27,8 +27,9 @@ services: - PORT=4321 - PUBLIC_BETTER_AUTH_URL=${PUBLIC_BETTER_AUTH_URL:-http://localhost:4321} # Optional concurrency controls (defaults match in-app defaults) - - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-1} - - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-1} + # If you want perfect ordering of issues and PRs, set these at 1 + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/api/health"] diff --git a/docker-compose.yml b/docker-compose.yml index f981486..00d8cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,8 @@ services: - PRESERVE_ORG_STRUCTURE=${PRESERVE_ORG_STRUCTURE:-false} - ONLY_MIRROR_ORGS=${ONLY_MIRROR_ORGS:-false} - SKIP_STARRED_ISSUES=${SKIP_STARRED_ISSUES:-false} - - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-1} - - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-1} + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} - GITEA_URL=${GITEA_URL:-} - GITEA_TOKEN=${GITEA_TOKEN:-} - GITEA_USERNAME=${GITEA_USERNAME:-} diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 58cbad4..3e0b314 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -141,8 +141,8 @@ Control what content gets mirrored from GitHub to Gitea. | `MIRROR_PULL_REQUESTS` | Mirror pull requests (requires MIRROR_METADATA=true) | `false` | `true`, `false` | | `MIRROR_LABELS` | Mirror labels (requires MIRROR_METADATA=true) | `false` | `true`, `false` | | `MIRROR_MILESTONES` | Mirror milestones (requires MIRROR_METADATA=true) | `false` | `true`, `false` | -| `MIRROR_ISSUE_CONCURRENCY` | Number of issues processed in parallel. Set above `1` to speed up mirroring at the risk of out-of-order creation. | `1` | Integer ≥ 1 | -| `MIRROR_PULL_REQUEST_CONCURRENCY` | Number of pull requests processed in parallel. Values above `1` may cause ordering differences. | `1` | Integer ≥ 1 | +| `MIRROR_ISSUE_CONCURRENCY` | Number of issues processed in parallel. Set above `1` to speed up mirroring at the risk of out-of-order creation. | `3` | Integer ≥ 1 | +| `MIRROR_PULL_REQUEST_CONCURRENCY` | Number of pull requests processed in parallel. Values above `1` may cause ordering differences. | `5` | Integer ≥ 1 | > **Ordering vs Throughput:** Metadata now mirrors sequentially by default to preserve chronology. Increase the concurrency variables only if you can tolerate minor out-of-order entries. diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index c95d6e0..1137050 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -54,8 +54,8 @@ export const giteaConfigSchema = z.object({ .enum(["skip", "reference", "full-copy"]) .default("reference"), // Mirror options - issueConcurrency: z.number().int().min(1).default(1), - pullRequestConcurrency: z.number().int().min(1).default(1), + issueConcurrency: z.number().int().min(1).default(3), + pullRequestConcurrency: z.number().int().min(1).default(5), mirrorReleases: z.boolean().default(false), releaseLimit: z.number().default(10), mirrorMetadata: z.boolean().default(false), diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index 39e44fd..635b3ab 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -284,10 +284,10 @@ export async function initializeConfigFromEnv(): Promise { releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0 ? envConfig.mirror.issueConcurrency - : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 1, + : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3, pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0 ? envConfig.mirror.pullRequestConcurrency - : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 1, + : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index cf343e3..a7e64e3 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -1592,7 +1592,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); - const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 1; + const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3; const issueConcurrencyLimit = Number.isFinite(rawIssueConcurrency) ? Math.max(1, Math.floor(rawIssueConcurrency)) @@ -1600,7 +1600,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ if (issueConcurrencyLimit > 1) { console.warn( - `[Issues] Concurrency is set to ${issueConcurrencyLimit}. This may lead to out-of-order issue creation in Gitea.` + `[Issues] Concurrency is set to ${issueConcurrencyLimit}. This may lead to out-of-order issue creation in Gitea but is faster.` ); } @@ -2047,7 +2047,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ const { processWithRetry } = await import("@/lib/utils/concurrency"); - const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 1; + const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5; const pullRequestConcurrencyLimit = Number.isFinite(rawPullConcurrency) ? Math.max(1, Math.floor(rawPullConcurrency)) diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index a699321..43db60d 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -86,8 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default addTopics: true, preserveVisibility: false, forkStrategy: "reference", - issueConcurrency: 1, - pullRequestConcurrency: 1, + issueConcurrency: 3, + pullRequestConcurrency: 5, }, include: [], exclude: [], diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 961c226..7d02fdc 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -89,8 +89,8 @@ export function mapUiToDbConfig( forkStrategy: advancedOptions.skipForks ? "skip" : "reference", // Mirror options from UI - issueConcurrency: giteaConfig.issueConcurrency ?? 1, - pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 1, + issueConcurrency: giteaConfig.issueConcurrency ?? 3, + pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5, mirrorReleases: mirrorOptions.mirrorReleases, releaseLimit: mirrorOptions.releaseLimit || 10, mirrorMetadata: mirrorOptions.mirrorMetadata, @@ -134,8 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): { preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config personalReposOrg: undefined, // Not stored in current schema - issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 1, - pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 1, + issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3, + pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5, }; // Map mirror options from various database fields From e7a102ee45ab9bd75a2a884f9c85d9c7c376a4d0 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 08:42:14 +0530 Subject: [PATCH 10/11] mirror: show github timestamps in metadata --- src/lib/gitea.ts | 19 +++++++++++++++---- src/lib/utils.test.ts | 14 +++++++++++++- src/lib/utils.ts | 9 +++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index a7e64e3..e246c1c 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -12,6 +12,7 @@ import { createMirrorJob } from "./helpers"; import { db, organizations, repositories } from "./db"; import { eq, and } from "drizzle-orm"; import { decryptConfigTokens } from "./utils/config-encryption"; +import { formatDateShort } from "./utils"; /** * Helper function to get organization configuration including destination override @@ -1646,11 +1647,15 @@ export const mirrorGitRepoIssuesToGitea = async ({ .join(", ")} on GitHub.` : ""; + const issueAuthor = issue.user?.login ?? "unknown"; + const issueCreatedOn = formatDateShort(issue.created_at); + const issueOriginHeader = `Originally created by @${issueAuthor} on GitHub${ + issueCreatedOn ? ` (${issueCreatedOn})` : "" + }.`; + const issuePayload: any = { title: issue.title, - body: `Originally created by @${ - issue.user?.login - } on GitHub.${originalAssignees}\n\n${issue.body || ""}`, + body: `${issueOriginHeader}${originalAssignees}\n\n${issue.body ?? ""}`, closed: issue.state === "closed", labels: giteaLabelIds, }; @@ -1690,10 +1695,16 @@ export const mirrorGitRepoIssuesToGitea = async ({ await processWithRetry( sortedComments, async (comment) => { + const commenter = comment.user?.login ?? "unknown"; + const commentDate = formatDateShort(comment.created_at); + const commentHeader = `@${commenter} commented on GitHub${ + commentDate ? ` (${commentDate})` : "" + }:`; + await httpPost( `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues/${createdIssue.data.number}/comments`, { - body: `@${comment.user?.login} commented on GitHub:\n\n${comment.body}`, + body: `${commentHeader}\n\n${comment.body ?? ""}`, }, { Authorization: `token ${decryptedConfig.giteaConfig!.token}`, diff --git a/src/lib/utils.test.ts b/src/lib/utils.test.ts index a71ed85..7d7ac5a 100644 --- a/src/lib/utils.test.ts +++ b/src/lib/utils.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test"; -import { jsonResponse, formatDate, truncate, safeParse, parseErrorMessage, showErrorToast } from "./utils"; +import { jsonResponse, formatDate, formatDateShort, truncate, safeParse, parseErrorMessage, showErrorToast } from "./utils"; describe("jsonResponse", () => { test("creates a Response with JSON content", () => { @@ -65,6 +65,18 @@ describe("formatDate", () => { }); }); +describe("formatDateShort", () => { + test("returns formatted date when input is provided", () => { + const formatted = formatDateShort("2014-10-20T15:32:10Z"); + expect(formatted).toBe("Oct 20, 2014"); + }); + + test("returns undefined when date is missing", () => { + expect(formatDateShort(null)).toBeUndefined(); + expect(formatDateShort(undefined)).toBeUndefined(); + }); +}); + describe("truncate", () => { test("truncates a string that exceeds the length", () => { const str = "This is a long string that needs truncation"; diff --git a/src/lib/utils.ts b/src/lib/utils.ts index e674517..810edee 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -29,6 +29,15 @@ export function formatDate(date?: Date | string | null): string { }).format(new Date(date)); } +export function formatDateShort(date?: Date | string | null): string | undefined { + if (!date) return undefined; + return new Intl.DateTimeFormat("en-US", { + year: "numeric", + month: "short", + day: "numeric", + }).format(new Date(date)); +} + export function formatLastSyncTime(date: Date | string | null): string { if (!date) return "Never"; From 921ab948a1dd78bbef13e20ae4f50ceab57fe6df Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Fri, 24 Oct 2025 08:49:08 +0530 Subject: [PATCH 11/11] updated env vars for ci comment --- .github/workflows/docker-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 20361aa..74929bc 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -166,8 +166,7 @@ jobs: environment: - BETTER_AUTH_SECRET=your-secret-here - BETTER_AUTH_URL=http://localhost:4321 - - MIRROR_ISSUE_CONCURRENCY=3 - - MIRROR_PULL_REQUEST_CONCURRENCY=5 + - BETTER_AUTH_TRUSTED_ORIGINS=http://localhost:4321 \`\`\` > 💡 **Note:** PR images are tagged as \`pr-\` and only built for \`linux/amd64\` to speed up CI.