fix: implement proper mirror strategies for starred and org repos

This commit is contained in:
Arunavo Ray
2025-06-17 09:26:26 +05:30
parent c7e310b340
commit 7af1f6da17
8 changed files with 242 additions and 25 deletions

View File

@@ -6,6 +6,7 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import {
mirrorGithubRepoToGitea,
mirrorGitHubOrgRepoToGiteaOrg,
getGiteaRepoOwner,
} from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github";
import { processWithResilience } from "@/lib/utils/concurrency";
@@ -96,12 +97,20 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name}`);
// Mirror the repository based on whether it's in an organization
if (repo.organization && config.githubConfig.preserveOrgStructure) {
// Determine where the repository should be mirrored
const owner = getGiteaRepoOwner({
config,
repository: repoData,
});
console.log(`Repository ${repo.name} will be mirrored to owner: ${owner}`);
// Check if owner is different from the user (means it's going to an org)
if (owner !== config.giteaConfig?.username) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
octokit,
orgName: repo.organization,
orgName: owner,
repository: repoData,
});
} else {

View File

@@ -93,6 +93,7 @@ export const POST: APIRoute = async ({ request }) => {
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
mirroredLocation: repo.mirroredLocation || "",
};
// Log the start of retry operation
@@ -134,13 +135,14 @@ export const POST: APIRoute = async ({ request }) => {
throw new Error("Octokit client is not initialized.");
}
console.log(`Importing repo: ${repo.name} ${owner}`);
console.log(`Importing repo: ${repo.name} to owner: ${owner}`);
if (repo.organization && config.githubConfig.preserveOrgStructure) {
// Check if owner is different from the user (means it's going to an org)
if (owner !== config.giteaConfig?.username) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
octokit,
orgName: repo.organization,
orgName: owner,
repository: {
...repoData,
status: repoStatusEnum.parse("imported"),

View File

@@ -112,6 +112,7 @@ export const POST: APIRoute = async ({ request }) => {
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
mirroredLocation: repo.mirroredLocation || "",
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
},
@@ -133,6 +134,7 @@ export const POST: APIRoute = async ({ request }) => {
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
})),
};

View File

@@ -6,7 +6,6 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import { syncGiteaRepo } from "@/lib/gitea";
import type { SyncRepoResponse } from "@/types/sync";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
import { createSecureErrorResponse } from "@/lib/utils";
export const POST: APIRoute = async ({ request }) => {
@@ -68,9 +67,6 @@ export const POST: APIRoute = async ({ request }) => {
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 5;
// Generate a batch ID to group related repositories
const batchId = uuidv4();
// Process repositories in parallel with resilience to container restarts
await processWithResilience(
repos,
@@ -84,6 +80,7 @@ export const POST: APIRoute = async ({ request }) => {
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
};
// Log the start of syncing
@@ -100,7 +97,6 @@ export const POST: APIRoute = async ({ request }) => {
{
userId: config.userId || "",
jobType: "sync",
batchId,
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
concurrencyLimit: CONCURRENCY_LIMIT,
@@ -135,6 +131,7 @@ export const POST: APIRoute = async ({ request }) => {
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
})),
};

View File

@@ -97,6 +97,7 @@ export const POST: APIRoute = async ({ request }) => {
status: "imported" as Repository["status"],
lastMirrored: undefined,
errorMessage: undefined,
mirroredLocation: "",
createdAt: repoData.created_at
? new Date(repoData.created_at)
: new Date(),