import type { APIRoute } from "astro"; import type { MirrorRepoRequest, MirrorRepoResponse } from "@/types/mirror"; import { db, configs, repositories } from "@/lib/db"; import { eq, inArray } from "drizzle-orm"; import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository"; import { mirrorGithubRepoToGitea, mirrorGitHubOrgRepoToGiteaOrg, } from "@/lib/gitea"; import { createGitHubClient } from "@/lib/github"; import { processWithResilience } from "@/lib/utils/concurrency"; import { v4 as uuidv4 } from "uuid"; export const POST: APIRoute = async ({ request }) => { try { const body: MirrorRepoRequest = await request.json(); const { userId, repositoryIds } = body; if (!userId || !repositoryIds || !Array.isArray(repositoryIds)) { return new Response( JSON.stringify({ success: false, message: "userId and repositoryIds are required.", }), { status: 400, headers: { "Content-Type": "application/json" } } ); } if (repositoryIds.length === 0) { return new Response( JSON.stringify({ success: false, message: "No repository IDs provided.", }), { status: 400, headers: { "Content-Type": "application/json" } } ); } // Fetch config const configResult = await db .select() .from(configs) .where(eq(configs.userId, userId)) .limit(1); const config = configResult[0]; if (!config || !config.githubConfig.token) { return new Response( JSON.stringify({ error: "Config missing for the user or token." }), { status: 400, headers: { "Content-Type": "application/json" } } ); } // Fetch repos const repos = await db .select() .from(repositories) .where(inArray(repositories.id, repositoryIds)); if (!repos.length) { return new Response( JSON.stringify({ error: "No repositories found for the given IDs." }), { status: 404, headers: { "Content-Type": "application/json" } } ); } // Start async mirroring in background with parallel processing and resilience setTimeout(async () => { if (!config.githubConfig.token) { throw new Error("GitHub token is missing."); } // Create a single Octokit instance to be reused const octokit = createGitHubClient(config.githubConfig.token); // Define the concurrency limit - adjust based on API rate limits const CONCURRENCY_LIMIT = 3; // Generate a batch ID to group related repositories const batchId = uuidv4(); // Process repositories in parallel with resilience to container restarts await processWithResilience( repos, async (repo) => { // Prepare repository data const repoData = { ...repo, status: repoStatusEnum.parse("imported"), organization: repo.organization ?? undefined, lastMirrored: repo.lastMirrored ?? undefined, errorMessage: repo.errorMessage ?? undefined, forkedFrom: repo.forkedFrom ?? undefined, visibility: repositoryVisibilityEnum.parse(repo.visibility), mirroredLocation: repo.mirroredLocation || "", }; // Log the start of mirroring console.log(`Starting mirror for repository: ${repo.name}`); // Mirror the repository based on whether it's in an organization if (repo.organization && config.githubConfig.preserveOrgStructure) { await mirrorGitHubOrgRepoToGiteaOrg({ config, octokit, orgName: repo.organization, repository: repoData, }); } else { await mirrorGithubRepoToGitea({ octokit, repository: repoData, config, }); } return repo; }, { userId: config.userId || "", jobType: "mirror", batchId, getItemId: (repo) => repo.id, getItemName: (repo) => repo.name, concurrencyLimit: CONCURRENCY_LIMIT, maxRetries: 2, retryDelay: 2000, checkpointInterval: 5, // Checkpoint every 5 repositories to reduce event frequency onProgress: (completed, total, result) => { const percentComplete = Math.round((completed / total) * 100); console.log(`Mirroring progress: ${percentComplete}% (${completed}/${total})`); if (result) { console.log(`Successfully mirrored repository: ${result.name}`); } }, onRetry: (repo, error, attempt) => { console.log(`Retrying repository ${repo.name} (attempt ${attempt}): ${error.message}`); } } ); console.log("All repository mirroring tasks completed"); }, 0); const responsePayload: MirrorRepoResponse = { success: true, message: "Mirror job started.", repositories: repos.map((repo) => ({ ...repo, status: repoStatusEnum.parse(repo.status), organization: repo.organization ?? undefined, lastMirrored: repo.lastMirrored ?? undefined, errorMessage: repo.errorMessage ?? undefined, forkedFrom: repo.forkedFrom ?? undefined, visibility: repositoryVisibilityEnum.parse(repo.visibility), mirroredLocation: repo.mirroredLocation || "", })), }; // Return the updated repo list to the user return new Response(JSON.stringify(responsePayload), { status: 200, headers: { "Content-Type": "application/json" }, }); } catch (error) { console.error("Error mirroring repositories:", error); return new Response( JSON.stringify({ error: error instanceof Error ? error.message : "An unknown error occurred", }), { status: 500, headers: { "Content-Type": "application/json" } } ); } };