diff --git a/docker-compose.alt.yml b/docker-compose.alt.yml index 56f9760..2d412d1 100644 --- a/docker-compose.alt.yml +++ b/docker-compose.alt.yml @@ -27,8 +27,9 @@ services: - PORT=4321 - PUBLIC_BETTER_AUTH_URL=${PUBLIC_BETTER_AUTH_URL:-http://localhost:4321} # Optional concurrency controls (defaults match in-app defaults) - - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-1} - - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-1} + # If you want perfect ordering of issues and PRs, set these at 1 + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/api/health"] diff --git a/docker-compose.yml b/docker-compose.yml index f981486..00d8cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,8 @@ services: - PRESERVE_ORG_STRUCTURE=${PRESERVE_ORG_STRUCTURE:-false} - ONLY_MIRROR_ORGS=${ONLY_MIRROR_ORGS:-false} - SKIP_STARRED_ISSUES=${SKIP_STARRED_ISSUES:-false} - - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-1} - - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-1} + - MIRROR_ISSUE_CONCURRENCY=${MIRROR_ISSUE_CONCURRENCY:-3} + - MIRROR_PULL_REQUEST_CONCURRENCY=${MIRROR_PULL_REQUEST_CONCURRENCY:-5} - GITEA_URL=${GITEA_URL:-} - GITEA_TOKEN=${GITEA_TOKEN:-} - GITEA_USERNAME=${GITEA_USERNAME:-} diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 58cbad4..3e0b314 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -141,8 +141,8 @@ Control what content gets mirrored from GitHub to Gitea. | `MIRROR_PULL_REQUESTS` | Mirror pull requests (requires MIRROR_METADATA=true) | `false` | `true`, `false` | | `MIRROR_LABELS` | Mirror labels (requires MIRROR_METADATA=true) | `false` | `true`, `false` | | `MIRROR_MILESTONES` | Mirror milestones (requires MIRROR_METADATA=true) | `false` | `true`, `false` | -| `MIRROR_ISSUE_CONCURRENCY` | Number of issues processed in parallel. Set above `1` to speed up mirroring at the risk of out-of-order creation. | `1` | Integer ≥ 1 | -| `MIRROR_PULL_REQUEST_CONCURRENCY` | Number of pull requests processed in parallel. Values above `1` may cause ordering differences. | `1` | Integer ≥ 1 | +| `MIRROR_ISSUE_CONCURRENCY` | Number of issues processed in parallel. Set above `1` to speed up mirroring at the risk of out-of-order creation. | `3` | Integer ≥ 1 | +| `MIRROR_PULL_REQUEST_CONCURRENCY` | Number of pull requests processed in parallel. Values above `1` may cause ordering differences. | `5` | Integer ≥ 1 | > **Ordering vs Throughput:** Metadata now mirrors sequentially by default to preserve chronology. Increase the concurrency variables only if you can tolerate minor out-of-order entries. diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index c95d6e0..1137050 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -54,8 +54,8 @@ export const giteaConfigSchema = z.object({ .enum(["skip", "reference", "full-copy"]) .default("reference"), // Mirror options - issueConcurrency: z.number().int().min(1).default(1), - pullRequestConcurrency: z.number().int().min(1).default(1), + issueConcurrency: z.number().int().min(1).default(3), + pullRequestConcurrency: z.number().int().min(1).default(5), mirrorReleases: z.boolean().default(false), releaseLimit: z.number().default(10), mirrorMetadata: z.boolean().default(false), diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index 39e44fd..635b3ab 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -284,10 +284,10 @@ export async function initializeConfigFromEnv(): Promise { releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0 ? envConfig.mirror.issueConcurrency - : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 1, + : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3, pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0 ? envConfig.mirror.pullRequestConcurrency - : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 1, + : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index cf343e3..a7e64e3 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -1592,7 +1592,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); - const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 1; + const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3; const issueConcurrencyLimit = Number.isFinite(rawIssueConcurrency) ? Math.max(1, Math.floor(rawIssueConcurrency)) @@ -1600,7 +1600,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ if (issueConcurrencyLimit > 1) { console.warn( - `[Issues] Concurrency is set to ${issueConcurrencyLimit}. This may lead to out-of-order issue creation in Gitea.` + `[Issues] Concurrency is set to ${issueConcurrencyLimit}. This may lead to out-of-order issue creation in Gitea but is faster.` ); } @@ -2047,7 +2047,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ const { processWithRetry } = await import("@/lib/utils/concurrency"); - const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 1; + const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5; const pullRequestConcurrencyLimit = Number.isFinite(rawPullConcurrency) ? Math.max(1, Math.floor(rawPullConcurrency)) diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index a699321..43db60d 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -86,8 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default addTopics: true, preserveVisibility: false, forkStrategy: "reference", - issueConcurrency: 1, - pullRequestConcurrency: 1, + issueConcurrency: 3, + pullRequestConcurrency: 5, }, include: [], exclude: [], diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 961c226..7d02fdc 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -89,8 +89,8 @@ export function mapUiToDbConfig( forkStrategy: advancedOptions.skipForks ? "skip" : "reference", // Mirror options from UI - issueConcurrency: giteaConfig.issueConcurrency ?? 1, - pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 1, + issueConcurrency: giteaConfig.issueConcurrency ?? 3, + pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5, mirrorReleases: mirrorOptions.mirrorReleases, releaseLimit: mirrorOptions.releaseLimit || 10, mirrorMetadata: mirrorOptions.mirrorMetadata, @@ -134,8 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): { preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config personalReposOrg: undefined, // Not stored in current schema - issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 1, - pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 1, + issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3, + pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5, }; // Map mirror options from various database fields