gitea-mirror/src/pages/api/job/mirror-repo.ts

import type { APIRoute } from "astro";
import type { MirrorRepoRequest, MirrorRepoResponse } from "@/types/mirror";
import { db, configs, repositories } from "@/lib/db";
import { eq, inArray } from "drizzle-orm";
import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import {
  mirrorGithubRepoToGitea,
  mirrorGitHubOrgRepoToGiteaOrg,
} from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";

export const POST: APIRoute = async ({ request }) => {
  try {
    const body: MirrorRepoRequest = await request.json();
    const { userId, repositoryIds } = body;

    if (!userId || !repositoryIds || !Array.isArray(repositoryIds)) {
      return new Response(
        JSON.stringify({
          success: false,
          message: "userId and repositoryIds are required.",
        }),
        { status: 400, headers: { "Content-Type": "application/json" } }
      );
    }

    if (repositoryIds.length === 0) {
      return new Response(
        JSON.stringify({
          success: false,
          message: "No repository IDs provided.",
        }),
        { status: 400, headers: { "Content-Type": "application/json" } }
      );
    }

    // Fetch config
    const configResult = await db
      .select()
      .from(configs)
      .where(eq(configs.userId, userId))
      .limit(1);

    const config = configResult[0];

    if (!config || !config.githubConfig.token) {
      return new Response(
        JSON.stringify({ error: "Config missing for the user or token." }),
        { status: 400, headers: { "Content-Type": "application/json" } }
      );
    }

    // Fetch repos
    const repos = await db
      .select()
      .from(repositories)
      .where(inArray(repositories.id, repositoryIds));

    if (!repos.length) {
      return new Response(
        JSON.stringify({ error: "No repositories found for the given IDs." }),
        { status: 404, headers: { "Content-Type": "application/json" } }
      );
    }

    // Start async mirroring in background with parallel processing and resilience
    setTimeout(async () => {
      if (!config.githubConfig.token) {
        throw new Error("GitHub token is missing.");
      }

      // Create a single Octokit instance to be reused
      const octokit = createGitHubClient(config.githubConfig.token);

      // Define the concurrency limit - adjust based on API rate limits
      const CONCURRENCY_LIMIT = 3;

      // Generate a batch ID to group related repositories
      const batchId = uuidv4();

      // Process repositories in parallel with resilience to container restarts
      await processWithResilience(
        repos,
        async (repo) => {
          // Prepare repository data
          const repoData = {
            ...repo,
            status: repoStatusEnum.parse("imported"),
            organization: repo.organization ?? undefined,
            lastMirrored: repo.lastMirrored ?? undefined,
            errorMessage: repo.errorMessage ?? undefined,
            forkedFrom: repo.forkedFrom ?? undefined,
            visibility: repositoryVisibilityEnum.parse(repo.visibility),
            mirroredLocation: repo.mirroredLocation || "",
          };

          // Log the start of mirroring
          console.log(`Starting mirror for repository: ${repo.name}`);

          // Mirror the repository based on whether it's in an organization
          if (repo.organization && config.githubConfig.preserveOrgStructure) {
            await mirrorGitHubOrgRepoToGiteaOrg({
              config,
              octokit,
              orgName: repo.organization,
              repository: repoData,
            });
          } else {
            await mirrorGithubRepoToGitea({
              octokit,
              repository: repoData,
              config,
            });
          }

          return repo;
        },
        {
          userId: config.userId || "",
          jobType: "mirror",
          batchId,
          getItemId: (repo) => repo.id,
          getItemName: (repo) => repo.name,
          concurrencyLimit: CONCURRENCY_LIMIT,
          maxRetries: 2,
          retryDelay: 2000,
          checkpointInterval: 5, // Checkpoint every 5 repositories to reduce event frequency
          onProgress: (completed, total, result) => {
            const percentComplete = Math.round((completed / total) * 100);
            console.log(`Mirroring progress: ${percentComplete}% (${completed}/${total})`);

            if (result) {
              console.log(`Successfully mirrored repository: ${result.name}`);
            }
          },
          onRetry: (repo, error, attempt) => {
            console.log(`Retrying repository ${repo.name} (attempt ${attempt}): ${error.message}`);
          }
        }
      );

      console.log("All repository mirroring tasks completed");
    }, 0);

    const responsePayload: MirrorRepoResponse = {
      success: true,
      message: "Mirror job started.",
      repositories: repos.map((repo) => ({
        ...repo,
        status: repoStatusEnum.parse(repo.status),
        organization: repo.organization ?? undefined,
        lastMirrored: repo.lastMirrored ?? undefined,
        errorMessage: repo.errorMessage ?? undefined,
        forkedFrom: repo.forkedFrom ?? undefined,
        visibility: repositoryVisibilityEnum.parse(repo.visibility),
        mirroredLocation: repo.mirroredLocation || "",
      })),
    };

    // Return the updated repo list to the user
    return new Response(JSON.stringify(responsePayload), {
      status: 200,
      headers: { "Content-Type": "application/json" },
    });
  } catch (error) {
    // Enhanced error logging for better debugging
    console.error("=== ERROR MIRRORING REPOSITORIES ===");
    console.error("Error type:", error?.constructor?.name);
    console.error("Error message:", error instanceof Error ? error.message : String(error));

    if (error instanceof Error) {
      console.error("Error stack:", error.stack);
    }

    // Log additional context
    console.error("Request details:");
    console.error("- URL:", request.url);
    console.error("- Method:", request.method);
    console.error("- Headers:", Object.fromEntries(request.headers.entries()));

    // If it's a JSON parsing error, provide more context
    if (error instanceof SyntaxError && error.message.includes('JSON')) {
      console.error("🚨 JSON PARSING ERROR DETECTED:");
      console.error("This suggests the response from Gitea API is not valid JSON");
      console.error("Common causes:");
      console.error("- Gitea server returned HTML error page instead of JSON");
      console.error("- Network connection interrupted");
      console.error("- Gitea server is down or misconfigured");
      console.error("- Authentication token is invalid");
      console.error("Check your Gitea server logs and configuration");
    }

    console.error("=====================================");

    return new Response(
      JSON.stringify({
        error: error instanceof Error ? error.message : "An unknown error occurred",
        errorType: error?.constructor?.name || "Unknown",
        timestamp: new Date().toISOString(),
        troubleshooting: error instanceof SyntaxError && error.message.includes('JSON')
          ? "JSON parsing error detected. Check Gitea server status and logs. Ensure Gitea is returning valid JSON responses."
          : "Check application logs for more details"
      }),
      { status: 500, headers: { "Content-Type": "application/json" } }
    );
  }
};