import { repoStatusEnum, type RepositoryVisibility, type RepoStatus, } from "@/types/Repository"; import { Octokit } from "@octokit/rest"; import type { Config } from "@/types/config"; import type { Organization, Repository } from "./db/schema"; import { httpPost, httpGet } from "./http-client"; import { createMirrorJob } from "./helpers"; import { db, organizations, repositories } from "./db"; import { eq, and } from "drizzle-orm"; /** * Helper function to get organization configuration including destination override */ export const getOrganizationConfig = async ({ orgName, userId, }: { orgName: string; userId: string; }): Promise => { try { const [orgConfig] = await db .select() .from(organizations) .where(and(eq(organizations.name, orgName), eq(organizations.userId, userId))) .limit(1); return orgConfig || null; } catch (error) { console.error(`Error fetching organization config for ${orgName}:`, error); return null; } }; /** * Enhanced async version of getGiteaRepoOwner that supports organization overrides */ export const getGiteaRepoOwnerAsync = async ({ config, repository, }: { config: Partial; repository: Repository; }): Promise => { if (!config.githubConfig || !config.giteaConfig) { throw new Error("GitHub or Gitea config is required."); } if (!config.giteaConfig.username) { throw new Error("Gitea username is required."); } if (!config.userId) { throw new Error("User ID is required for organization overrides."); } // Check if repository is starred - starred repos always go to starredReposOrg (highest priority) if (repository.isStarred && config.giteaConfig.starredReposOrg) { return config.giteaConfig.starredReposOrg; } // Check for repository-specific override (second highest priority) if (repository.destinationOrg) { console.log(`Using repository override: ${repository.fullName} -> ${repository.destinationOrg}`); return repository.destinationOrg; } // Check for organization-specific override if (repository.organization) { const orgConfig = await getOrganizationConfig({ orgName: repository.organization, userId: config.userId, }); if (orgConfig?.destinationOrg) { console.log(`Using organization override: ${repository.organization} -> ${orgConfig.destinationOrg}`); return orgConfig.destinationOrg; } } // Check for personal repos override (when it's user's repo, not an organization) if (!repository.organization && config.giteaConfig.personalReposOrg) { console.log(`Using personal repos override: ${config.giteaConfig.personalReposOrg}`); return config.giteaConfig.personalReposOrg; } // Fall back to existing strategy logic return getGiteaRepoOwner({ config, repository }); }; export const getGiteaRepoOwner = ({ config, repository, }: { config: Partial; repository: Repository; }): string => { if (!config.githubConfig || !config.giteaConfig) { throw new Error("GitHub or Gitea config is required."); } if (!config.giteaConfig.username) { throw new Error("Gitea username is required."); } // Check if repository is starred - starred repos always go to starredReposOrg if (repository.isStarred && config.giteaConfig.starredReposOrg) { return config.giteaConfig.starredReposOrg; } // Get the mirror strategy - use preserveOrgStructure for backward compatibility const mirrorStrategy = config.giteaConfig.mirrorStrategy || (config.githubConfig.preserveOrgStructure ? "preserve" : "flat-user"); switch (mirrorStrategy) { case "preserve": // Keep GitHub structure - org repos go to same org, personal repos to user (or override) if (repository.organization) { return repository.organization; } // Use personal repos override if configured, otherwise use username return config.giteaConfig.personalReposOrg || config.giteaConfig.username; case "single-org": // All non-starred repos go to the destination organization if (config.giteaConfig.organization) { return config.giteaConfig.organization; } // Fallback to username if no organization specified return config.giteaConfig.username; case "flat-user": // All non-starred repos go under the user account return config.giteaConfig.username; case "mixed": // Mixed mode: personal repos to single org, organization repos preserve structure if (repository.organization) { // Organization repos preserve their structure return repository.organization; } // Personal repos go to configured organization (same as single-org) if (config.giteaConfig.organization) { return config.giteaConfig.organization; } // Fallback to username if no organization specified return config.giteaConfig.username; default: // Default fallback return config.giteaConfig.username; } }; export const isRepoPresentInGitea = async ({ config, owner, repoName, }: { config: Partial; owner: string; repoName: string; }): Promise => { try { if (!config.giteaConfig?.url || !config.giteaConfig?.token) { throw new Error("Gitea config is required."); } // Check if the repository exists at the specified owner location const response = await fetch( `${config.giteaConfig.url}/api/v1/repos/${owner}/${repoName}`, { headers: { Authorization: `token ${config.giteaConfig.token}`, }, } ); return response.ok; } catch (error) { console.error("Error checking if repo exists in Gitea:", error); return false; } }; /** * Helper function to check if a repository exists in Gitea. * First checks the recorded mirroredLocation, then falls back to the expected location. */ export const checkRepoLocation = async ({ config, repository, expectedOwner, }: { config: Partial; repository: Repository; expectedOwner: string; }): Promise<{ present: boolean; actualOwner: string }> => { // First check if we have a recorded mirroredLocation and if the repo exists there if ( repository.mirroredLocation && repository.mirroredLocation.trim() !== "" ) { const [mirroredOwner] = repository.mirroredLocation.split("/"); if (mirroredOwner) { const mirroredPresent = await isRepoPresentInGitea({ config, owner: mirroredOwner, repoName: repository.name, }); if (mirroredPresent) { console.log( `Repository found at recorded mirrored location: ${repository.mirroredLocation}` ); return { present: true, actualOwner: mirroredOwner }; } } } // If not found at the recorded location, check the expected location const present = await isRepoPresentInGitea({ config, owner: expectedOwner, repoName: repository.name, }); if (present) { return { present: true, actualOwner: expectedOwner }; } // Repository not found at any location return { present: false, actualOwner: expectedOwner }; }; export const mirrorGithubRepoToGitea = async ({ octokit, repository, config, }: { octokit: Octokit; repository: Repository; config: Partial; }): Promise => { try { if (!config.userId || !config.githubConfig || !config.giteaConfig) { throw new Error("github config and gitea config are required."); } if (!config.giteaConfig.username) { throw new Error("Gitea username is required."); } // Get the correct owner based on the strategy (with organization overrides) const repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); const isExisting = await isRepoPresentInGitea({ config, owner: repoOwner, repoName: repository.name, }); if (isExisting) { console.log( `Repository ${repository.name} already exists in Gitea under ${repoOwner}. Updating database status.` ); // Update database to reflect that the repository is already mirrored await db .update(repositories) .set({ status: repoStatusEnum.parse("mirrored"), updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${repoOwner}/${repository.name}`, }) .where(eq(repositories.id, repository.id!)); // Append log for "mirrored" status await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Repository ${repository.name} already exists in Gitea`, details: `Repository ${repository.name} was found to already exist in Gitea under ${repoOwner} and database status was updated.`, status: "mirrored", }); console.log( `Repository ${repository.name} database status updated to mirrored` ); return; } console.log(`Mirroring repository ${repository.name}`); // Mark repos as "mirroring" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("mirroring"), updatedAt: new Date(), }) .where(eq(repositories.id, repository.id!)); // Append log for "mirroring" status await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Started mirroring repository: ${repository.name}`, details: `Repository ${repository.name} is now in the mirroring state.`, status: "mirroring", }); let cloneAddress = repository.cloneUrl; // If the repository is private, inject the GitHub token into the clone URL if (repository.isPrivate) { if (!config.githubConfig.token) { throw new Error( "GitHub token is required to mirror private repositories." ); } cloneAddress = repository.cloneUrl.replace( "https://", `https://${config.githubConfig.token}@` ); } const apiUrl = `${config.giteaConfig.url}/api/v1/repos/migrate`; // Handle organization creation if needed for single-org or preserve strategies if (repoOwner !== config.giteaConfig.username && !repository.isStarred) { // Need to create the organization if it doesn't exist await getOrCreateGiteaOrg({ orgName: repoOwner, config, }); } const response = await httpPost( apiUrl, { clone_addr: cloneAddress, repo_name: repository.name, mirror: true, wiki: config.githubConfig.mirrorWiki || false, // will mirror wiki if it exists private: repository.isPrivate, repo_owner: repoOwner, description: "", service: "git", }, { Authorization: `token ${config.giteaConfig.token}`, } ); //mirror releases await mirrorGitHubReleasesToGitea({ config, octokit, repository, }); // clone issues // Skip issues for starred repos if skipStarredIssues is enabled const shouldMirrorIssues = config.githubConfig.mirrorIssues && !(repository.isStarred && config.githubConfig.skipStarredIssues); if (shouldMirrorIssues) { await mirrorGitRepoIssuesToGitea({ config, octokit, repository, isRepoInOrg: false, }); } console.log(`Repository ${repository.name} mirrored successfully`); // Mark repos as "mirrored" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("mirrored"), updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${repoOwner}/${repository.name}`, }) .where(eq(repositories.id, repository.id!)); // Append log for "mirrored" status await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Successfully mirrored repository: ${repository.name}`, details: `Repository ${repository.name} was mirrored to Gitea.`, status: "mirrored", }); return response.data; } catch (error) { console.error( `Error while mirroring repository ${repository.name}: ${ error instanceof Error ? error.message : String(error) }` ); // Mark repos as "failed" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("failed"), updatedAt: new Date(), errorMessage: error instanceof Error ? error.message : "Unknown error", }) .where(eq(repositories.id, repository.id!)); // Append log for failure await createMirrorJob({ userId: config.userId ?? "", // userId is going to be there anyways repositoryId: repository.id, repositoryName: repository.name, message: `Failed to mirror repository: ${repository.name}`, details: `Repository ${repository.name} failed to mirror. Error: ${ error instanceof Error ? error.message : "Unknown error" }`, status: "failed", }); if (error instanceof Error) { throw new Error(`Failed to mirror repository: ${error.message}`); } throw new Error("Failed to mirror repository: An unknown error occurred."); } }; export async function getOrCreateGiteaOrg({ orgName, orgId, config, }: { orgId?: string; //db id orgName: string; config: Partial; }): Promise { if ( !config.giteaConfig?.url || !config.giteaConfig?.token || !config.userId ) { throw new Error("Gitea config is required."); } try { console.log(`Attempting to get or create Gitea organization: ${orgName}`); const orgRes = await fetch( `${config.giteaConfig.url}/api/v1/orgs/${orgName}`, { headers: { Authorization: `token ${config.giteaConfig.token}`, "Content-Type": "application/json", }, } ); console.log( `Get org response status: ${orgRes.status} for org: ${orgName}` ); if (orgRes.ok) { // Check if response is actually JSON const contentType = orgRes.headers.get("content-type"); if (!contentType || !contentType.includes("application/json")) { console.warn( `Expected JSON response but got content-type: ${contentType}` ); const responseText = await orgRes.text(); console.warn(`Response body: ${responseText}`); throw new Error( `Invalid response format from Gitea API. Expected JSON but got: ${contentType}` ); } // Clone the response to handle potential JSON parsing errors const orgResClone = orgRes.clone(); try { const org = await orgRes.json(); console.log( `Successfully retrieved existing org: ${orgName} with ID: ${org.id}` ); // Note: Organization events are handled by the main mirroring process // to avoid duplicate events return org.id; } catch (jsonError) { const responseText = await orgResClone.text(); console.error( `Failed to parse JSON response for existing org: ${responseText}` ); throw new Error( `Failed to parse JSON response from Gitea API: ${ jsonError instanceof Error ? jsonError.message : String(jsonError) }` ); } } console.log(`Organization ${orgName} not found, attempting to create it`); const createRes = await fetch(`${config.giteaConfig.url}/api/v1/orgs`, { method: "POST", headers: { Authorization: `token ${config.giteaConfig.token}`, "Content-Type": "application/json", }, body: JSON.stringify({ username: orgName, full_name: `${orgName} Org`, description: `Mirrored organization from GitHub ${orgName}`, visibility: config.giteaConfig?.visibility || "public", }), }); console.log( `Create org response status: ${createRes.status} for org: ${orgName}` ); if (!createRes.ok) { const errorText = await createRes.text(); console.error( `Failed to create org ${orgName}. Status: ${createRes.status}, Response: ${errorText}` ); throw new Error(`Failed to create Gitea org: ${errorText}`); } // Check if response is actually JSON const createContentType = createRes.headers.get("content-type"); if (!createContentType || !createContentType.includes("application/json")) { console.warn( `Expected JSON response but got content-type: ${createContentType}` ); const responseText = await createRes.text(); console.warn(`Response body: ${responseText}`); throw new Error( `Invalid response format from Gitea API. Expected JSON but got: ${createContentType}` ); } // Note: Organization creation events are handled by the main mirroring process // to avoid duplicate events // Clone the response to handle potential JSON parsing errors const createResClone = createRes.clone(); try { const newOrg = await createRes.json(); console.log( `Successfully created new org: ${orgName} with ID: ${newOrg.id}` ); return newOrg.id; } catch (jsonError) { const responseText = await createResClone.text(); console.error( `Failed to parse JSON response for new org: ${responseText}` ); throw new Error( `Failed to parse JSON response from Gitea API: ${ jsonError instanceof Error ? jsonError.message : String(jsonError) }` ); } } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error occurred in getOrCreateGiteaOrg."; console.error( `Error in getOrCreateGiteaOrg for ${orgName}: ${errorMessage}` ); await createMirrorJob({ userId: config.userId, organizationId: orgId, organizationName: orgName, message: `Failed to create or fetch Gitea organization: ${orgName}`, status: "failed", details: `Error: ${errorMessage}`, }); throw new Error(`Error in getOrCreateGiteaOrg: ${errorMessage}`); } } export async function mirrorGitHubRepoToGiteaOrg({ octokit, config, repository, giteaOrgId, orgName, }: { octokit: Octokit; config: Partial; repository: Repository; giteaOrgId: number; orgName: string; }) { try { if ( !config.giteaConfig?.url || !config.giteaConfig?.token || !config.userId ) { throw new Error("Gitea config is required."); } const isExisting = await isRepoPresentInGitea({ config, owner: orgName, repoName: repository.name, }); if (isExisting) { console.log( `Repository ${repository.name} already exists in Gitea organization ${orgName}. Updating database status.` ); // Update database to reflect that the repository is already mirrored await db .update(repositories) .set({ status: repoStatusEnum.parse("mirrored"), updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${orgName}/${repository.name}`, }) .where(eq(repositories.id, repository.id!)); // Create a mirror job log entry await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Repository ${repository.name} already exists in Gitea organization ${orgName}`, details: `Repository ${repository.name} was found to already exist in Gitea organization ${orgName} and database status was updated.`, status: "mirrored", }); console.log( `Repository ${repository.name} database status updated to mirrored in organization ${orgName}` ); return; } console.log( `Mirroring repository ${repository.name} to organization ${orgName}` ); let cloneAddress = repository.cloneUrl; if (repository.isPrivate) { if (!config.githubConfig?.token) { throw new Error( "GitHub token is required to mirror private repositories." ); } cloneAddress = repository.cloneUrl.replace( "https://", `https://${config.githubConfig.token}@` ); } // Mark repos as "mirroring" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("mirroring"), updatedAt: new Date(), }) .where(eq(repositories.id, repository.id!)); // Note: "mirroring" status events are handled by the concurrency system // to avoid duplicate events during batch operations const apiUrl = `${config.giteaConfig.url}/api/v1/repos/migrate`; const migrateRes = await httpPost( apiUrl, { clone_addr: cloneAddress, uid: giteaOrgId, repo_name: repository.name, mirror: true, wiki: config.githubConfig?.mirrorWiki || false, // will mirror wiki if it exists private: repository.isPrivate, }, { Authorization: `token ${config.giteaConfig.token}`, } ); //mirror releases await mirrorGitHubReleasesToGitea({ config, octokit, repository, }); // Clone issues // Skip issues for starred repos if skipStarredIssues is enabled const shouldMirrorIssues = config.githubConfig?.mirrorIssues && !(repository.isStarred && config.githubConfig?.skipStarredIssues); if (shouldMirrorIssues) { await mirrorGitRepoIssuesToGitea({ config, octokit, repository, isRepoInOrg: true, }); } console.log( `Repository ${repository.name} mirrored successfully to organization ${orgName}` ); // Mark repos as "mirrored" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("mirrored"), updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${orgName}/${repository.name}`, }) .where(eq(repositories.id, repository.id!)); //create a mirror job await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Repository ${repository.name} mirrored successfully`, details: `Repository ${repository.name} was mirrored to Gitea`, status: "mirrored", }); return migrateRes.data; } catch (error) { console.error( `Error while mirroring repository ${repository.name}: ${ error instanceof Error ? error.message : String(error) }` ); // Mark repos as "failed" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("failed"), updatedAt: new Date(), errorMessage: error instanceof Error ? error.message : "Unknown error", }) .where(eq(repositories.id, repository.id!)); // Append log for failure await createMirrorJob({ userId: config.userId || "", // userId is going to be there anyways repositoryId: repository.id, repositoryName: repository.name, message: `Failed to mirror repository: ${repository.name}`, details: `Repository ${repository.name} failed to mirror. Error: ${ error instanceof Error ? error.message : "Unknown error" }`, status: "failed", }); if (error instanceof Error) { throw new Error(`Failed to mirror repository: ${error.message}`); } throw new Error("Failed to mirror repository: An unknown error occurred."); } } export async function mirrorGitHubOrgRepoToGiteaOrg({ config, octokit, repository, orgName, }: { config: Partial; octokit: Octokit; repository: Repository; orgName: string; }) { try { if (!config.giteaConfig?.url || !config.giteaConfig?.token) { throw new Error("Gitea config is required."); } const giteaOrgId = await getOrCreateGiteaOrg({ orgName, config, }); await mirrorGitHubRepoToGiteaOrg({ octokit, config, repository, giteaOrgId, orgName, }); } catch (error) { if (error instanceof Error) { throw new Error(`Failed to mirror repository: ${error.message}`); } throw new Error("Failed to mirror repository: An unknown error occurred."); } } export async function mirrorGitHubOrgToGitea({ organization, octokit, config, }: { organization: Organization; octokit: Octokit; config: Partial; }) { try { if ( !config.userId || !config.id || !config.githubConfig?.token || !config.giteaConfig?.url ) { throw new Error("Config, GitHub token and Gitea URL are required."); } console.log(`Mirroring organization ${organization.name}`); //mark the org as "mirroring" in DB await db .update(organizations) .set({ isIncluded: true, status: repoStatusEnum.parse("mirroring"), updatedAt: new Date(), }) .where(eq(organizations.id, organization.id!)); // Append log for "mirroring" status await createMirrorJob({ userId: config.userId, organizationId: organization.id, organizationName: organization.name, message: `Started mirroring organization: ${organization.name}`, details: `Organization ${organization.name} is now in the mirroring state.`, status: repoStatusEnum.parse("mirroring"), }); // Get the mirror strategy - use preserveOrgStructure for backward compatibility const mirrorStrategy = config.giteaConfig?.mirrorStrategy || (config.githubConfig?.preserveOrgStructure ? "preserve" : "flat-user"); let giteaOrgId: number; let targetOrgName: string; // Determine the target organization based on strategy if (mirrorStrategy === "single-org" && config.giteaConfig?.organization) { // For single-org strategy, use the configured destination organization targetOrgName = config.giteaConfig.organization; giteaOrgId = await getOrCreateGiteaOrg({ orgId: organization.id, orgName: targetOrgName, config, }); console.log(`Using single organization strategy: all repos will go to ${targetOrgName}`); } else if (mirrorStrategy === "preserve") { // For preserve strategy, create/use an org with the same name as GitHub targetOrgName = organization.name; giteaOrgId = await getOrCreateGiteaOrg({ orgId: organization.id, orgName: targetOrgName, config, }); } else { // For flat-user strategy, we shouldn't create organizations at all // Skip organization creation and let individual repos be handled by getGiteaRepoOwner console.log(`Using flat-user strategy: repos will be placed under user account`); targetOrgName = config.giteaConfig?.username || ""; } //query the db with the org name and get the repos const orgRepos = await db .select() .from(repositories) .where(eq(repositories.organization, organization.name)); if (orgRepos.length === 0) { console.log( `No repositories found for organization ${organization.name} - marking as successfully mirrored` ); } else { console.log( `Mirroring ${orgRepos.length} repositories for organization ${organization.name}` ); // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); // Process repositories in parallel with concurrency control await processWithRetry( orgRepos, async (repo) => { // Prepare repository data const repoData = { ...repo, status: repo.status as RepoStatus, visibility: repo.visibility as RepositoryVisibility, lastMirrored: repo.lastMirrored ?? undefined, errorMessage: repo.errorMessage ?? undefined, organization: repo.organization ?? undefined, forkedFrom: repo.forkedFrom ?? undefined, mirroredLocation: repo.mirroredLocation || "", }; // Log the start of mirroring console.log( `Starting mirror for repository: ${repo.name} from GitHub org ${organization.name}` ); // Mirror the repository based on strategy if (mirrorStrategy === "flat-user") { // For flat-user strategy, mirror directly to user account await mirrorGithubRepoToGitea({ octokit, repository: repoData, config, }); } else { // For preserve and single-org strategies, use organization await mirrorGitHubRepoToGiteaOrg({ octokit, config, repository: repoData, giteaOrgId: giteaOrgId!, orgName: targetOrgName, }); } return repo; }, { concurrencyLimit: 3, // Process 3 repositories at a time maxRetries: 2, retryDelay: 2000, onProgress: (completed, total, result) => { const percentComplete = Math.round((completed / total) * 100); if (result) { console.log( `Mirrored repository "${result.name}" in organization ${organization.name} (${completed}/${total}, ${percentComplete}%)` ); } }, onRetry: (repo, error, attempt) => { console.log( `Retrying repository ${repo.name} in organization ${organization.name} (attempt ${attempt}): ${error.message}` ); }, } ); } console.log(`Organization ${organization.name} mirrored successfully`); // Mark org as "mirrored" in DB await db .update(organizations) .set({ status: repoStatusEnum.parse("mirrored"), updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, }) .where(eq(organizations.id, organization.id!)); // Append log for "mirrored" status await createMirrorJob({ userId: config.userId, organizationId: organization.id, organizationName: organization.name, message: `Successfully mirrored organization: ${organization.name}`, details: orgRepos.length === 0 ? `Organization ${organization.name} was processed successfully (no repositories found).` : `Organization ${organization.name} was mirrored to Gitea with ${orgRepos.length} repositories.`, status: repoStatusEnum.parse("mirrored"), }); } catch (error) { console.error( `Error while mirroring organization ${organization.name}: ${ error instanceof Error ? error.message : String(error) }` ); // Mark org as "failed" in DB await db .update(organizations) .set({ status: repoStatusEnum.parse("failed"), updatedAt: new Date(), errorMessage: error instanceof Error ? error.message : "Unknown error", }) .where(eq(organizations.id, organization.id!)); // Append log for failure await createMirrorJob({ userId: config.userId || "", // userId is going to be there anyways organizationId: organization.id, organizationName: organization.name, message: `Failed to mirror organization: ${organization.name}`, details: `Organization ${organization.name} failed to mirror. Error: ${ error instanceof Error ? error.message : "Unknown error" }`, status: repoStatusEnum.parse("failed"), }); if (error instanceof Error) { throw new Error(`Failed to mirror repository: ${error.message}`); } throw new Error("Failed to mirror repository: An unknown error occurred."); } } export const syncGiteaRepo = async ({ config, repository, }: { config: Partial; repository: Repository; }) => { try { if ( !config.userId || !config.giteaConfig?.url || !config.giteaConfig?.token || !config.giteaConfig?.username ) { throw new Error("Gitea config is required."); } console.log(`Syncing repository ${repository.name}`); // Mark repo as "syncing" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("syncing"), updatedAt: new Date(), }) .where(eq(repositories.id, repository.id!)); // Append log for "syncing" status await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Started syncing repository: ${repository.name}`, details: `Repository ${repository.name} is now in the syncing state.`, status: repoStatusEnum.parse("syncing"), }); // Get the expected owner based on current config (with organization overrides) const repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); // Check if repo exists at the expected location or alternate location const { present, actualOwner } = await checkRepoLocation({ config, repository, expectedOwner: repoOwner, }); if (!present) { throw new Error( `Repository ${repository.name} not found in Gitea at any expected location` ); } // Use the actual owner where the repo was found const apiUrl = `${config.giteaConfig.url}/api/v1/repos/${actualOwner}/${repository.name}/mirror-sync`; const response = await httpPost(apiUrl, undefined, { Authorization: `token ${config.giteaConfig.token}`, }); // Mark repo as "synced" in DB await db .update(repositories) .set({ status: repoStatusEnum.parse("synced"), updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${actualOwner}/${repository.name}`, }) .where(eq(repositories.id, repository.id!)); // Append log for "synced" status await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Successfully synced repository: ${repository.name}`, details: `Repository ${repository.name} was synced with Gitea.`, status: repoStatusEnum.parse("synced"), }); console.log(`Repository ${repository.name} synced successfully`); return response.data; } catch (error) { console.error( `Error while syncing repository ${repository.name}: ${ error instanceof Error ? error.message : String(error) }` ); // Optional: update repo with error status await db .update(repositories) .set({ status: repoStatusEnum.parse("failed"), updatedAt: new Date(), errorMessage: (error as Error).message, }) .where(eq(repositories.id, repository.id!)); // Append log for "error" status if (config.userId && repository.id && repository.name) { await createMirrorJob({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, message: `Failed to sync repository: ${repository.name}`, details: (error as Error).message, status: repoStatusEnum.parse("failed"), }); } if (error instanceof Error) { throw new Error(`Failed to sync repository: ${error.message}`); } throw new Error("Failed to sync repository: An unknown error occurred."); } }; export const mirrorGitRepoIssuesToGitea = async ({ config, octokit, repository, isRepoInOrg, }: { config: Partial; octokit: Octokit; repository: Repository; isRepoInOrg: boolean; }) => { //things covered here are- issue, title, body, labels, comments and assignees if ( !config.githubConfig?.token || !config.giteaConfig?.token || !config.giteaConfig?.url || !config.giteaConfig?.username ) { throw new Error("Missing GitHub or Gitea configuration."); } const repoOrigin = isRepoInOrg ? repository.organization : config.githubConfig.username; const [owner, repo] = repository.fullName.split("/"); // Fetch GitHub issues const issues = await octokit.paginate( octokit.rest.issues.listForRepo, { owner, repo, state: "all", per_page: 100, }, (res) => res.data ); // Filter out pull requests const filteredIssues = issues.filter((issue) => !(issue as any).pull_request); console.log( `Mirroring ${filteredIssues.length} issues from ${repository.fullName}` ); if (filteredIssues.length === 0) { console.log(`No issues to mirror for ${repository.fullName}`); return; } // Get existing labels from Gitea const giteaLabelsRes = await httpGet( `${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/labels`, { Authorization: `token ${config.giteaConfig.token}`, } ); const giteaLabels = giteaLabelsRes.data; const labelMap = new Map( giteaLabels.map((label: any) => [label.name, label.id]) ); // Import the processWithRetry function const { processWithRetry } = await import("@/lib/utils/concurrency"); // Process issues in parallel with concurrency control await processWithRetry( filteredIssues, async (issue) => { const githubLabelNames = issue.labels ?.map((l) => (typeof l === "string" ? l : l.name)) .filter((l): l is string => !!l) || []; const giteaLabelIds: number[] = []; // Resolve or create labels in Gitea for (const name of githubLabelNames) { if (labelMap.has(name)) { giteaLabelIds.push(labelMap.get(name)!); } else { try { const created = await httpPost( `${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${ repository.name }/labels`, { name, color: "#ededed" }, // Default color { Authorization: `token ${config.giteaConfig!.token}`, } ); labelMap.set(name, created.data.id); giteaLabelIds.push(created.data.id); } catch (labelErr) { console.error( `Failed to create label "${name}" in Gitea: ${labelErr}` ); } } } const originalAssignees = issue.assignees && issue.assignees.length > 0 ? `\n\nOriginally assigned to: ${issue.assignees .map((a) => `@${a.login}`) .join(", ")} on GitHub.` : ""; const issuePayload: any = { title: issue.title, body: `Originally created by @${ issue.user?.login } on GitHub.${originalAssignees}\n\n${issue.body || ""}`, closed: issue.state === "closed", labels: giteaLabelIds, }; // Create the issue in Gitea const createdIssue = await httpPost( `${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${ repository.name }/issues`, issuePayload, { Authorization: `token ${config.giteaConfig!.token}`, } ); // Clone comments const comments = await octokit.paginate( octokit.rest.issues.listComments, { owner, repo, issue_number: issue.number, per_page: 100, }, (res) => res.data ); // Process comments in parallel with concurrency control if (comments.length > 0) { await processWithRetry( comments, async (comment) => { await httpPost( `${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${ repository.name }/issues/${createdIssue.data.number}/comments`, { body: `@${comment.user?.login} commented on GitHub:\n\n${comment.body}`, }, { Authorization: `token ${config.giteaConfig!.token}`, } ); return comment; }, { concurrencyLimit: 5, maxRetries: 2, retryDelay: 1000, onRetry: (_comment, error, attempt) => { console.log( `Retrying comment (attempt ${attempt}): ${error.message}` ); }, } ); } return issue; }, { concurrencyLimit: 3, // Process 3 issues at a time maxRetries: 2, retryDelay: 2000, onProgress: (completed, total, result) => { const percentComplete = Math.round((completed / total) * 100); if (result) { console.log( `Mirrored issue "${result.title}" (${completed}/${total}, ${percentComplete}%)` ); } }, onRetry: (issue, error, attempt) => { console.log( `Retrying issue "${issue.title}" (attempt ${attempt}): ${error.message}` ); }, } ); console.log( `Completed mirroring ${filteredIssues.length} issues for ${repository.fullName}` ); }; export async function mirrorGitHubReleasesToGitea({ octokit, repository, config, }: { octokit: Octokit; repository: Repository; config: Partial; }) { if ( !config.giteaConfig?.username || !config.giteaConfig?.token || !config.giteaConfig?.url ) { throw new Error("Gitea config is incomplete for mirroring releases."); } const repoOwner = await getGiteaRepoOwnerAsync({ config, repository, }); const { url, token } = config.giteaConfig; const releases = await octokit.rest.repos.listReleases({ owner: repository.owner, repo: repository.name, }); for (const release of releases.data) { await httpPost( `${url}/api/v1/repos/${repoOwner}/${repository.name}/releases`, { tag_name: release.tag_name, target: release.target_commitish, title: release.name || release.tag_name, note: release.body || "", draft: release.draft, prerelease: release.prerelease, }, { Authorization: `token ${token}`, } ); } console.log(`✅ Mirrored ${releases.data.length} GitHub releases to Gitea`); }