fix: preserve chronological issue mirroring

This commit is contained in:
Arunavo Ray
2025-10-23 23:08:32 +05:30
parent fd5e68c1d4
commit fb27ddfee5
6 changed files with 39 additions and 2 deletions

View File

@@ -54,6 +54,8 @@ export const giteaConfigSchema = z.object({
.enum(["skip", "reference", "full-copy"]) .enum(["skip", "reference", "full-copy"])
.default("reference"), .default("reference"),
// Mirror options // Mirror options
issueConcurrency: z.number().int().min(1).default(3),
pullRequestConcurrency: z.number().int().min(1).default(5),
mirrorReleases: z.boolean().default(false), mirrorReleases: z.boolean().default(false),
releaseLimit: z.number().default(10), releaseLimit: z.number().default(10),
mirrorMetadata: z.boolean().default(false), mirrorMetadata: z.boolean().default(false),

View File

@@ -49,6 +49,9 @@ interface EnvConfig {
mirrorLabels?: boolean; mirrorLabels?: boolean;
mirrorMilestones?: boolean; mirrorMilestones?: boolean;
mirrorMetadata?: boolean; mirrorMetadata?: boolean;
releaseLimit?: number;
issueConcurrency?: number;
pullRequestConcurrency?: number;
}; };
schedule: { schedule: {
enabled?: boolean; enabled?: boolean;
@@ -136,6 +139,8 @@ function parseEnvConfig(): EnvConfig {
mirrorMilestones: process.env.MIRROR_MILESTONES === 'true', mirrorMilestones: process.env.MIRROR_MILESTONES === 'true',
mirrorMetadata: process.env.MIRROR_METADATA === 'true', mirrorMetadata: process.env.MIRROR_METADATA === 'true',
releaseLimit: process.env.RELEASE_LIMIT ? parseInt(process.env.RELEASE_LIMIT, 10) : undefined, releaseLimit: process.env.RELEASE_LIMIT ? parseInt(process.env.RELEASE_LIMIT, 10) : undefined,
issueConcurrency: process.env.MIRROR_ISSUE_CONCURRENCY ? parseInt(process.env.MIRROR_ISSUE_CONCURRENCY, 10) : undefined,
pullRequestConcurrency: process.env.MIRROR_PULL_REQUEST_CONCURRENCY ? parseInt(process.env.MIRROR_PULL_REQUEST_CONCURRENCY, 10) : undefined,
}, },
schedule: { schedule: {
enabled: process.env.SCHEDULE_ENABLED === 'true' || enabled: process.env.SCHEDULE_ENABLED === 'true' ||
@@ -277,6 +282,12 @@ export async function initializeConfigFromEnv(): Promise<void> {
// Mirror metadata options // Mirror metadata options
mirrorReleases: envConfig.mirror.mirrorReleases ?? existingConfig?.[0]?.giteaConfig?.mirrorReleases ?? false, mirrorReleases: envConfig.mirror.mirrorReleases ?? existingConfig?.[0]?.giteaConfig?.mirrorReleases ?? false,
releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10,
issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0
? envConfig.mirror.issueConcurrency
: existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3,
pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0
? envConfig.mirror.pullRequestConcurrency
: existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5,
mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false,
mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false,
mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false,

View File

@@ -1558,6 +1558,8 @@ export const mirrorGitRepoIssuesToGitea = async ({
repo, repo,
state: "all", state: "all",
per_page: 100, per_page: 100,
sort: "created",
direction: "asc",
}, },
(res) => res.data (res) => res.data
); );
@@ -1590,6 +1592,12 @@ export const mirrorGitRepoIssuesToGitea = async ({
// Import the processWithRetry function // Import the processWithRetry function
const { processWithRetry } = await import("@/lib/utils/concurrency"); const { processWithRetry } = await import("@/lib/utils/concurrency");
const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3;
const issueConcurrencyLimit =
Number.isFinite(rawIssueConcurrency)
? Math.max(1, Math.floor(rawIssueConcurrency))
: 3;
// Process issues in parallel with concurrency control // Process issues in parallel with concurrency control
await processWithRetry( await processWithRetry(
filteredIssues, filteredIssues,
@@ -1694,7 +1702,7 @@ export const mirrorGitRepoIssuesToGitea = async ({
return issue; return issue;
}, },
{ {
concurrencyLimit: 3, // Process 3 issues at a time concurrencyLimit: issueConcurrencyLimit,
maxRetries: 2, maxRetries: 2,
retryDelay: 2000, retryDelay: 2000,
onProgress: (completed, total, result) => { onProgress: (completed, total, result) => {
@@ -1966,6 +1974,8 @@ export async function mirrorGitRepoPullRequestsToGitea({
repo, repo,
state: "all", state: "all",
per_page: 100, per_page: 100,
sort: "created",
direction: "asc",
}, },
(res) => res.data (res) => res.data
); );
@@ -2022,6 +2032,12 @@ export async function mirrorGitRepoPullRequestsToGitea({
const { processWithRetry } = await import("@/lib/utils/concurrency"); const { processWithRetry } = await import("@/lib/utils/concurrency");
const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5;
const pullRequestConcurrencyLimit =
Number.isFinite(rawPullConcurrency)
? Math.max(1, Math.floor(rawPullConcurrency))
: 5;
let successCount = 0; let successCount = 0;
let failedCount = 0; let failedCount = 0;
@@ -2144,7 +2160,7 @@ export async function mirrorGitRepoPullRequestsToGitea({
} }
}, },
{ {
concurrencyLimit: 5, concurrencyLimit: pullRequestConcurrencyLimit,
maxRetries: 3, maxRetries: 3,
retryDelay: 1000, retryDelay: 1000,
} }

View File

@@ -86,6 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
addTopics: true, addTopics: true,
preserveVisibility: false, preserveVisibility: false,
forkStrategy: "reference", forkStrategy: "reference",
issueConcurrency: 3,
pullRequestConcurrency: 5,
}, },
include: [], include: [],
exclude: [], exclude: [],

View File

@@ -89,6 +89,8 @@ export function mapUiToDbConfig(
forkStrategy: advancedOptions.skipForks ? "skip" : "reference", forkStrategy: advancedOptions.skipForks ? "skip" : "reference",
// Mirror options from UI // Mirror options from UI
issueConcurrency: giteaConfig.issueConcurrency ?? 3,
pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5,
mirrorReleases: mirrorOptions.mirrorReleases, mirrorReleases: mirrorOptions.mirrorReleases,
releaseLimit: mirrorOptions.releaseLimit || 10, releaseLimit: mirrorOptions.releaseLimit || 10,
mirrorMetadata: mirrorOptions.mirrorMetadata, mirrorMetadata: mirrorOptions.mirrorMetadata,
@@ -132,6 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): {
preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility
mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config
personalReposOrg: undefined, // Not stored in current schema personalReposOrg: undefined, // Not stored in current schema
issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3,
pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5,
}; };
// Map mirror options from various database fields // Map mirror options from various database fields

View File

@@ -13,6 +13,8 @@ export interface GiteaConfig {
preserveOrgStructure: boolean; preserveOrgStructure: boolean;
mirrorStrategy?: MirrorStrategy; // New field for the strategy mirrorStrategy?: MirrorStrategy; // New field for the strategy
personalReposOrg?: string; // Override destination for personal repos personalReposOrg?: string; // Override destination for personal repos
issueConcurrency?: number;
pullRequestConcurrency?: number;
} }
export interface ScheduleConfig { export interface ScheduleConfig {