fix: implement proper mirror strategies for starred and org repos

This commit is contained in:
Arunavo Ray
2025-06-17 09:26:26 +05:30
parent c7e310b340
commit 7af1f6da17
8 changed files with 242 additions and 25 deletions

View File

@@ -26,13 +26,39 @@ export const getGiteaRepoOwner = ({
throw new Error("Gitea username is required.");
}
// if the config has preserveOrgStructure set to true, then use the org name as the owner
if (config.githubConfig.preserveOrgStructure && repository.organization) {
return repository.organization;
// Check if repository is starred - starred repos always go to starredReposOrg
if (repository.isStarred && config.giteaConfig.starredReposOrg) {
return config.giteaConfig.starredReposOrg;
}
// if the config has preserveOrgStructure set to false, then use the gitea username as the owner
return config.giteaConfig.username;
// Get the mirror strategy - use preserveOrgStructure for backward compatibility
const mirrorStrategy = config.giteaConfig.mirrorStrategy ||
(config.githubConfig.preserveOrgStructure ? "preserve" : "flat-user");
switch (mirrorStrategy) {
case "preserve":
// Keep GitHub structure - org repos go to same org, personal repos to user
if (repository.organization) {
return repository.organization;
}
return config.giteaConfig.username;
case "single-org":
// All non-starred repos go to the destination organization
if (config.giteaConfig.organization) {
return config.giteaConfig.organization;
}
// Fallback to username if no organization specified
return config.giteaConfig.username;
case "flat-user":
// All non-starred repos go under the user account
return config.giteaConfig.username;
default:
// Default fallback
return config.giteaConfig.username;
}
};
export const isRepoPresentInGitea = async ({

View File

@@ -6,6 +6,7 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import {
mirrorGithubRepoToGitea,
mirrorGitHubOrgRepoToGiteaOrg,
getGiteaRepoOwner,
} from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github";
import { processWithResilience } from "@/lib/utils/concurrency";
@@ -96,12 +97,20 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name}`);
// Mirror the repository based on whether it's in an organization
if (repo.organization && config.githubConfig.preserveOrgStructure) {
// Determine where the repository should be mirrored
const owner = getGiteaRepoOwner({
config,
repository: repoData,
});
console.log(`Repository ${repo.name} will be mirrored to owner: ${owner}`);
// Check if owner is different from the user (means it's going to an org)
if (owner !== config.giteaConfig?.username) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
octokit,
orgName: repo.organization,
orgName: owner,
repository: repoData,
});
} else {

View File

@@ -93,6 +93,7 @@ export const POST: APIRoute = async ({ request }) => {
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
mirroredLocation: repo.mirroredLocation || "",
};
// Log the start of retry operation
@@ -134,13 +135,14 @@ export const POST: APIRoute = async ({ request }) => {
throw new Error("Octokit client is not initialized.");
}
console.log(`Importing repo: ${repo.name} ${owner}`);
console.log(`Importing repo: ${repo.name} to owner: ${owner}`);
if (repo.organization && config.githubConfig.preserveOrgStructure) {
// Check if owner is different from the user (means it's going to an org)
if (owner !== config.giteaConfig?.username) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
octokit,
orgName: repo.organization,
orgName: owner,
repository: {
...repoData,
status: repoStatusEnum.parse("imported"),

View File

@@ -112,6 +112,7 @@ export const POST: APIRoute = async ({ request }) => {
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
mirroredLocation: repo.mirroredLocation || "",
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
},
@@ -133,6 +134,7 @@ export const POST: APIRoute = async ({ request }) => {
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
})),
};

View File

@@ -6,7 +6,6 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import { syncGiteaRepo } from "@/lib/gitea";
import type { SyncRepoResponse } from "@/types/sync";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
import { createSecureErrorResponse } from "@/lib/utils";
export const POST: APIRoute = async ({ request }) => {
@@ -68,9 +67,6 @@ export const POST: APIRoute = async ({ request }) => {
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 5;
// Generate a batch ID to group related repositories
const batchId = uuidv4();
// Process repositories in parallel with resilience to container restarts
await processWithResilience(
repos,
@@ -84,6 +80,7 @@ export const POST: APIRoute = async ({ request }) => {
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
};
// Log the start of syncing
@@ -100,7 +97,6 @@ export const POST: APIRoute = async ({ request }) => {
{
userId: config.userId || "",
jobType: "sync",
batchId,
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
concurrencyLimit: CONCURRENCY_LIMIT,
@@ -135,6 +131,7 @@ export const POST: APIRoute = async ({ request }) => {
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
})),
};

View File

@@ -97,6 +97,7 @@ export const POST: APIRoute = async ({ request }) => {
status: "imported" as Repository["status"],
lastMirrored: undefined,
errorMessage: undefined,
mirroredLocation: "",
createdAt: repoData.created_at
? new Date(repoData.created_at)
: new Date(),