fix: enforce sequential metadata mirroring

This commit is contained in:
Arunavo Ray
2025-10-24 07:35:40 +05:30
parent 2cd7d911ed
commit 5245d67f37
5 changed files with 39 additions and 18 deletions

View File

@@ -54,8 +54,8 @@ export const giteaConfigSchema = z.object({
.enum(["skip", "reference", "full-copy"]) .enum(["skip", "reference", "full-copy"])
.default("reference"), .default("reference"),
// Mirror options // Mirror options
issueConcurrency: z.number().int().min(1).default(3), issueConcurrency: z.number().int().min(1).default(1),
pullRequestConcurrency: z.number().int().min(1).default(5), pullRequestConcurrency: z.number().int().min(1).default(1),
mirrorReleases: z.boolean().default(false), mirrorReleases: z.boolean().default(false),
releaseLimit: z.number().default(10), releaseLimit: z.number().default(10),
mirrorMetadata: z.boolean().default(false), mirrorMetadata: z.boolean().default(false),

View File

@@ -284,10 +284,10 @@ export async function initializeConfigFromEnv(): Promise<void> {
releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10, releaseLimit: envConfig.mirror.releaseLimit ?? existingConfig?.[0]?.giteaConfig?.releaseLimit ?? 10,
issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0 issueConcurrency: envConfig.mirror.issueConcurrency && envConfig.mirror.issueConcurrency > 0
? envConfig.mirror.issueConcurrency ? envConfig.mirror.issueConcurrency
: existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 3, : existingConfig?.[0]?.giteaConfig?.issueConcurrency ?? 1,
pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0 pullRequestConcurrency: envConfig.mirror.pullRequestConcurrency && envConfig.mirror.pullRequestConcurrency > 0
? envConfig.mirror.pullRequestConcurrency ? envConfig.mirror.pullRequestConcurrency
: existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 5, : existingConfig?.[0]?.giteaConfig?.pullRequestConcurrency ?? 1,
mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false, mirrorMetadata: envConfig.mirror.mirrorMetadata ?? (envConfig.mirror.mirrorIssues || envConfig.mirror.mirrorPullRequests || envConfig.mirror.mirrorLabels || envConfig.mirror.mirrorMilestones) ?? existingConfig?.[0]?.giteaConfig?.mirrorMetadata ?? false,
mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false, mirrorIssues: envConfig.mirror.mirrorIssues ?? existingConfig?.[0]?.giteaConfig?.mirrorIssues ?? false,
mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false, mirrorPullRequests: envConfig.mirror.mirrorPullRequests ?? existingConfig?.[0]?.giteaConfig?.mirrorPullRequests ?? false,

View File

@@ -1592,11 +1592,17 @@ export const mirrorGitRepoIssuesToGitea = async ({
// Import the processWithRetry function // Import the processWithRetry function
const { processWithRetry } = await import("@/lib/utils/concurrency"); const { processWithRetry } = await import("@/lib/utils/concurrency");
const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 3; const rawIssueConcurrency = config.giteaConfig?.issueConcurrency ?? 1;
const issueConcurrencyLimit = const issueConcurrencyLimit =
Number.isFinite(rawIssueConcurrency) Number.isFinite(rawIssueConcurrency)
? Math.max(1, Math.floor(rawIssueConcurrency)) ? Math.max(1, Math.floor(rawIssueConcurrency))
: 3; : 1;
if (issueConcurrencyLimit > 1) {
console.warn(
`[Issues] Concurrency is set to ${issueConcurrencyLimit}. This may lead to out-of-order issue creation in Gitea.`
);
}
// Process issues in parallel with concurrency control // Process issues in parallel with concurrency control
await processWithRetry( await processWithRetry(
@@ -1670,10 +1676,19 @@ export const mirrorGitRepoIssuesToGitea = async ({
(res) => res.data (res) => res.data
); );
// Process comments in parallel with concurrency control // Ensure comments are applied in chronological order to preserve discussion flow
if (comments.length > 0) { const sortedComments = comments
.slice()
.sort(
(a, b) =>
new Date(a.created_at || 0).getTime() -
new Date(b.created_at || 0).getTime()
);
// Process comments sequentially to preserve historical ordering
if (sortedComments.length > 0) {
await processWithRetry( await processWithRetry(
comments, sortedComments,
async (comment) => { async (comment) => {
await httpPost( await httpPost(
`${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues/${createdIssue.data.number}/comments`, `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues/${createdIssue.data.number}/comments`,
@@ -1687,7 +1702,7 @@ export const mirrorGitRepoIssuesToGitea = async ({
return comment; return comment;
}, },
{ {
concurrencyLimit: 5, concurrencyLimit: 1,
maxRetries: 2, maxRetries: 2,
retryDelay: 1000, retryDelay: 1000,
onRetry: (_comment, error, attempt) => { onRetry: (_comment, error, attempt) => {
@@ -2032,11 +2047,17 @@ export async function mirrorGitRepoPullRequestsToGitea({
const { processWithRetry } = await import("@/lib/utils/concurrency"); const { processWithRetry } = await import("@/lib/utils/concurrency");
const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 5; const rawPullConcurrency = config.giteaConfig?.pullRequestConcurrency ?? 1;
const pullRequestConcurrencyLimit = const pullRequestConcurrencyLimit =
Number.isFinite(rawPullConcurrency) Number.isFinite(rawPullConcurrency)
? Math.max(1, Math.floor(rawPullConcurrency)) ? Math.max(1, Math.floor(rawPullConcurrency))
: 5; : 1;
if (pullRequestConcurrencyLimit > 1) {
console.warn(
`[Pull Requests] Concurrency is set to ${pullRequestConcurrencyLimit}. This may lead to out-of-order pull request mirroring in Gitea.`
);
}
let successCount = 0; let successCount = 0;
let failedCount = 0; let failedCount = 0;

View File

@@ -86,8 +86,8 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
addTopics: true, addTopics: true,
preserveVisibility: false, preserveVisibility: false,
forkStrategy: "reference", forkStrategy: "reference",
issueConcurrency: 3, issueConcurrency: 1,
pullRequestConcurrency: 5, pullRequestConcurrency: 1,
}, },
include: [], include: [],
exclude: [], exclude: [],

View File

@@ -89,8 +89,8 @@ export function mapUiToDbConfig(
forkStrategy: advancedOptions.skipForks ? "skip" : "reference", forkStrategy: advancedOptions.skipForks ? "skip" : "reference",
// Mirror options from UI // Mirror options from UI
issueConcurrency: giteaConfig.issueConcurrency ?? 3, issueConcurrency: giteaConfig.issueConcurrency ?? 1,
pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 5, pullRequestConcurrency: giteaConfig.pullRequestConcurrency ?? 1,
mirrorReleases: mirrorOptions.mirrorReleases, mirrorReleases: mirrorOptions.mirrorReleases,
releaseLimit: mirrorOptions.releaseLimit || 10, releaseLimit: mirrorOptions.releaseLimit || 10,
mirrorMetadata: mirrorOptions.mirrorMetadata, mirrorMetadata: mirrorOptions.mirrorMetadata,
@@ -134,8 +134,8 @@ export function mapDbToUiConfig(dbConfig: any): {
preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility preserveOrgStructure: dbConfig.giteaConfig?.preserveVisibility || false, // Map preserveVisibility
mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config mirrorStrategy: dbConfig.githubConfig?.mirrorStrategy || "preserve", // Get from GitHub config
personalReposOrg: undefined, // Not stored in current schema personalReposOrg: undefined, // Not stored in current schema
issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3, issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 1,
pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5, pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 1,
}; };
// Map mirror options from various database fields // Map mirror options from various database fields