From fb27ddfee525804ac379b6d20c9045288ab6ba94 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Thu, 23 Oct 2025 23:08:32 +0530 Subject: [PATCH] fix: preserve chronological issue mirroring --- src/lib/db/schema.ts | 2 ++ src/lib/env-config-loader.ts | 11 +++++++++++ src/lib/gitea.ts | 20 ++++++++++++++++++-- src/lib/utils/config-defaults.ts | 2 ++ src/lib/utils/config-mapper.ts | 4 ++++ src/types/config.ts | 2 ++ 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 442e65a..1137050 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -54,6 +54,8 @@ export const giteaConfigSchema = z.object({ .enum(["skip", "reference", "full-copy"]) .default("reference"), // Mirror options + issueConcurrency: z.number().int().min(1).default(3), + pullRequestConcurrency: z.number().int().min(1).default(5), mirrorReleases: z.boolean().default(false), releaseLimit: z.number().default(10), mirrorMetadata: z.boolean().default(false), diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index b03cb62..635b3ab 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -49,6 +49,9 @@ interface EnvConfig { mirrorLabels?: boolean; mirrorMilestones?: boolean; mirrorMetadata?: boolean; + releaseLimit?: number; + issueConcurrency?: number; + pullRequestConcurrency?: number; }; schedule: { enabled?: boolean; @@ -136,6 +139,8 @@ function parseEnvConfig(): EnvConfig { mirrorMilestones: process.env.MIRROR_MILESTONES === 'true', mirrorMetadata: process.env.MIRROR_METADATA === 'true', releaseLimit: process.env.RELEASE_LIMIT ? parseInt(process.env.RELEASE_LIMIT, 10) : undefined, + issueConcurrency: process.env.MIRROR_ISSUE_CONCURRENCY ? parseInt(process.env.MIRROR_ISSUE_CONCURRENCY, 10) : undefined, + pullRequestConcurrency: process.env.MIRROR_PULL_REQUEST_CONCURRENCY ? parseInt(process.env.MIRROR_PULL_REQUEST_CONCURRENCY, 10) : undefined, }, schedule: { enabled: process.env.SCHEDULE_ENABLED === 'true' || @@ -277,6 +282,12 @@ export async function initializeConfigFromEnv(): Promise { // Mirror metadata options mirrorReleases: envConfig.mirror.mirrorReleases ?? existingConfig?.[0]?.giteaConfig?.mirrorReleases ?? false, releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, + issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0 + ? envConfig.mirror.issueConcurrency + : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3, + pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0 + ? envConfig.mirror.pullRequestConcurrency + : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index 908b6a2..0dec27e 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -1558,6 +1558,8 @@ export const mirrorGitRepoIssuesToGitea = async ({ repo, state: "all", per_page: 100, + sort: "created", + direction: "asc", }, (res) => res.data ); @@ -1590,6 +1592,12 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); + const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3; + const issueConcurrencyLimit = + Number.isFinite(rawIssueConcurrency) + ? Math.max(1, Math.floor(rawIssueConcurrency)) + : 3; + // Process issues in parallel with concurrency control await processWithRetry( filteredIssues, @@ -1694,7 +1702,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ return issue; }, { - concurrencyLimit: 3, // Process 3 issues at a time + concurrencyLimit: issueConcurrencyLimit, maxRetries: 2, retryDelay: 2000, onProgress: (completed, total, result) => { @@ -1966,6 +1974,8 @@ export async function mirrorGitRepoPullRequestsToGitea({ repo, state: "all", per_page: 100, + sort: "created", + direction: "asc", }, (res) => res.data ); @@ -2022,6 +2032,12 @@ export async function mirrorGitRepoPullRequestsToGitea({ const { processWithRetry } = await import("@/lib/utils/concurrency"); + const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5; + const pullRequestConcurrencyLimit = + Number.isFinite(rawPullConcurrency) + ? Math.max(1, Math.floor(rawPullConcurrency)) + : 5; + let successCount = 0; let failedCount = 0; @@ -2144,7 +2160,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ } }, { - concurrencyLimit: 5, + concurrencyLimit: pullRequestConcurrencyLimit, maxRetries: 3, retryDelay: 1000, } diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index 557b2c7..43db60d 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -86,6 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default addTopics: true, preserveVisibility: false, forkStrategy: "reference", + issueConcurrency: 3, + pullRequestConcurrency: 5, }, include: [], exclude: [], diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index f0158aa..7d02fdc 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -89,6 +89,8 @@ export function mapUiToDbConfig( forkStrategy: advancedOptions.skipForks ? "skip" : "reference", // Mirror options from UI + issueConcurrency: giteaConfig.issueConcurrency ?? 3, + pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5, mirrorReleases: mirrorOptions.mirrorReleases, releaseLimit: mirrorOptions.releaseLimit || 10, mirrorMetadata: mirrorOptions.mirrorMetadata, @@ -132,6 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): { preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config personalReposOrg: undefined, // Not stored in current schema + issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3, + pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5, }; // Map mirror options from various database fields diff --git a/src/types/config.ts b/src/types/config.ts index 98b0e2c..2db9f81 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -13,6 +13,8 @@ export interface GiteaConfig { preserveOrgStructure: boolean; mirrorStrategy?: MirrorStrategy; // New field for the strategy personalReposOrg?: string; // Override destination for personal repos + issueConcurrency?: number; + pullRequestConcurrency?: number; } export interface ScheduleConfig {