diff --git a/src/lib/gitea-enhanced.test.ts b/src/lib/gitea-enhanced.test.ts new file mode 100644 index 0000000..eb67fee --- /dev/null +++ b/src/lib/gitea-enhanced.test.ts @@ -0,0 +1,467 @@ +import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; +import { + getGiteaRepoInfo, + getOrCreateGiteaOrgEnhanced, + syncGiteaRepoEnhanced, + handleExistingNonMirrorRepo +} from "./gitea-enhanced"; +import { HttpError } from "./http-client"; +import type { Config, Repository } from "./db/schema"; +import { repoStatusEnum } from "@/types/Repository"; + +describe("Enhanced Gitea Operations", () => { + let originalFetch: typeof global.fetch; + let mockDb: any; + + beforeEach(() => { + originalFetch = global.fetch; + // Mock database operations + mockDb = { + update: mock(() => ({ + set: mock(() => ({ + where: mock(() => Promise.resolve()), + })), + })), + }; + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + describe("getGiteaRepoInfo", () => { + test("should return repo info for existing mirror repository", async () => { + global.fetch = mock(async (url: string) => ({ + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 123, + name: "test-repo", + owner: "starred", + mirror: true, + mirror_interval: "8h", + clone_url: "https://github.com/user/test-repo.git", + private: false, + }), + })); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + const repoInfo = await getGiteaRepoInfo({ + config, + owner: "starred", + repoName: "test-repo", + }); + + expect(repoInfo).toBeTruthy(); + expect(repoInfo?.mirror).toBe(true); + expect(repoInfo?.name).toBe("test-repo"); + }); + + test("should return repo info for existing non-mirror repository", async () => { + global.fetch = mock(async (url: string) => ({ + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 124, + name: "regular-repo", + owner: "starred", + mirror: false, + private: false, + }), + })); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + const repoInfo = await getGiteaRepoInfo({ + config, + owner: "starred", + repoName: "regular-repo", + }); + + expect(repoInfo).toBeTruthy(); + expect(repoInfo?.mirror).toBe(false); + }); + + test("should return null for non-existent repository", async () => { + global.fetch = mock(async (url: string) => ({ + ok: false, + status: 404, + statusText: "Not Found", + headers: new Headers({ "content-type": "application/json" }), + text: async () => "Not Found", + })); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + const repoInfo = await getGiteaRepoInfo({ + config, + owner: "starred", + repoName: "non-existent", + }); + + expect(repoInfo).toBeNull(); + }); + }); + + describe("getOrCreateGiteaOrgEnhanced", () => { + test("should handle duplicate organization constraint error with retry", async () => { + let attemptCount = 0; + + global.fetch = mock(async (url: string, options?: RequestInit) => { + attemptCount++; + + if (url.includes("/api/v1/orgs/starred") && options?.method !== "POST") { + // First two attempts: org doesn't exist + if (attemptCount <= 2) { + return { + ok: false, + status: 404, + statusText: "Not Found", + }; + } + // Third attempt: org now exists (created by another process) + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ id: 999, username: "starred" }), + }; + } + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + // Simulate duplicate constraint error + return { + ok: false, + status: 422, + statusText: "Unprocessable Entity", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "pq: duplicate key value violates unique constraint \"UQE_user_lower_name\"" + }), + text: async () => "duplicate key value violates unique constraint", + }; + } + + return { ok: false, status: 500 }; + }); + + const config: Partial = { + userId: "user123", + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + visibility: "public", + }, + }; + + const orgId = await getOrCreateGiteaOrgEnhanced({ + orgName: "starred", + config, + maxRetries: 3, + retryDelay: 10, + }); + + expect(orgId).toBe(999); + expect(attemptCount).toBeGreaterThanOrEqual(3); + }); + + test("should create organization on first attempt", async () => { + let getOrgCalled = false; + let createOrgCalled = false; + + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs/neworg") && options?.method !== "POST") { + getOrgCalled = true; + return { + ok: false, + status: 404, + statusText: "Not Found", + }; + } + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + createOrgCalled = true; + return { + ok: true, + status: 201, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ id: 777, username: "neworg" }), + }; + } + + return { ok: false, status: 500 }; + }); + + const config: Partial = { + userId: "user123", + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + const orgId = await getOrCreateGiteaOrgEnhanced({ + orgName: "neworg", + config, + }); + + expect(orgId).toBe(777); + expect(getOrgCalled).toBe(true); + expect(createOrgCalled).toBe(true); + }); + }); + + describe("syncGiteaRepoEnhanced", () => { + test("should fail gracefully when repository is not a mirror", async () => { + global.fetch = mock(async (url: string) => { + if (url.includes("/api/v1/repos/starred/non-mirror-repo") && !url.includes("mirror-sync")) { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 456, + name: "non-mirror-repo", + owner: "starred", + mirror: false, // Not a mirror + private: false, + }), + }; + } + return { ok: false, status: 404 }; + }); + + const config: Partial = { + userId: "user123", + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + const repository: Repository = { + id: "repo123", + name: "non-mirror-repo", + fullName: "user/non-mirror-repo", + owner: "user", + cloneUrl: "https://github.com/user/non-mirror-repo.git", + isPrivate: false, + isStarred: true, + status: repoStatusEnum.parse("mirrored"), + visibility: "public", + userId: "user123", + createdAt: new Date(), + updatedAt: new Date(), + }; + + // Mock getGiteaRepoOwnerAsync + const mockGetOwner = mock(() => Promise.resolve("starred")); + global.import = mock(async (path: string) => { + if (path === "./gitea") { + return { getGiteaRepoOwnerAsync: mockGetOwner }; + } + return {}; + }) as any; + + await expect( + syncGiteaRepoEnhanced({ config, repository }) + ).rejects.toThrow("Repository non-mirror-repo is not a mirror. Cannot sync."); + }); + + test("should successfully sync a mirror repository", async () => { + let syncCalled = false; + + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/repos/starred/mirror-repo") && !url.includes("mirror-sync")) { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 789, + name: "mirror-repo", + owner: "starred", + mirror: true, + mirror_interval: "8h", + private: false, + }), + }; + } + + if (url.includes("/mirror-sync") && options?.method === "POST") { + syncCalled = true; + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ success: true }), + }; + } + + return { ok: false, status: 404 }; + }); + + const config: Partial = { + userId: "user123", + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + const repository: Repository = { + id: "repo456", + name: "mirror-repo", + fullName: "user/mirror-repo", + owner: "user", + cloneUrl: "https://github.com/user/mirror-repo.git", + isPrivate: false, + isStarred: true, + status: repoStatusEnum.parse("mirrored"), + visibility: "public", + userId: "user123", + createdAt: new Date(), + updatedAt: new Date(), + }; + + // Mock getGiteaRepoOwnerAsync + const mockGetOwner = mock(() => Promise.resolve("starred")); + global.import = mock(async (path: string) => { + if (path === "./gitea") { + return { getGiteaRepoOwnerAsync: mockGetOwner }; + } + return {}; + }) as any; + + const result = await syncGiteaRepoEnhanced({ config, repository }); + + expect(result).toEqual({ success: true }); + expect(syncCalled).toBe(true); + }); + }); + + describe("handleExistingNonMirrorRepo", () => { + test("should skip non-mirror repository with skip strategy", async () => { + const repoInfo = { + id: 123, + name: "test-repo", + owner: "starred", + mirror: false, + private: false, + }; + + const repository: Repository = { + id: "repo123", + name: "test-repo", + fullName: "user/test-repo", + owner: "user", + cloneUrl: "https://github.com/user/test-repo.git", + isPrivate: false, + isStarred: true, + status: repoStatusEnum.parse("pending"), + visibility: "public", + userId: "user123", + createdAt: new Date(), + updatedAt: new Date(), + }; + + const config: Partial = { + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + await handleExistingNonMirrorRepo({ + config, + repository, + repoInfo, + strategy: "skip", + }); + + // Test passes if no error is thrown + expect(true).toBe(true); + }); + + test("should delete non-mirror repository with delete strategy", async () => { + let deleteCalled = false; + + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/repos/starred/test-repo") && options?.method === "DELETE") { + deleteCalled = true; + return { + ok: true, + status: 204, + }; + } + return { ok: false, status: 404 }; + }); + + const repoInfo = { + id: 124, + name: "test-repo", + owner: "starred", + mirror: false, + private: false, + }; + + const repository: Repository = { + id: "repo124", + name: "test-repo", + fullName: "user/test-repo", + owner: "user", + cloneUrl: "https://github.com/user/test-repo.git", + isPrivate: false, + isStarred: true, + status: repoStatusEnum.parse("pending"), + visibility: "public", + userId: "user123", + createdAt: new Date(), + updatedAt: new Date(), + }; + + const config: Partial = { + giteaConfig: { + url: "https://gitea.example.com", + token: "encrypted-token", + defaultOwner: "testuser", + }, + }; + + await handleExistingNonMirrorRepo({ + config, + repository, + repoInfo, + strategy: "delete", + }); + + expect(deleteCalled).toBe(true); + }); + }); +}); \ No newline at end of file diff --git a/src/lib/gitea-enhanced.ts b/src/lib/gitea-enhanced.ts new file mode 100644 index 0000000..46db399 --- /dev/null +++ b/src/lib/gitea-enhanced.ts @@ -0,0 +1,488 @@ +/** + * Enhanced Gitea operations with better error handling for starred repositories + * This module provides fixes for: + * 1. "Repository is not a mirror" errors + * 2. Duplicate organization constraint errors + * 3. Race conditions in parallel processing + */ + +import type { Config } from "@/types/config"; +import type { Repository } from "./db/schema"; +import { createMirrorJob } from "./helpers"; +import { decryptConfigTokens } from "./utils/config-encryption"; +import { httpPost, httpGet, HttpError } from "./http-client"; +import { db, repositories } from "./db"; +import { eq } from "drizzle-orm"; +import { repoStatusEnum } from "@/types/Repository"; + +/** + * Enhanced repository information including mirror status + */ +interface GiteaRepoInfo { + id: number; + name: string; + owner: string; + mirror: boolean; + mirror_interval?: string; + clone_url?: string; + private: boolean; +} + +/** + * Check if a repository exists in Gitea and return its details + */ +export async function getGiteaRepoInfo({ + config, + owner, + repoName, +}: { + config: Partial; + owner: string; + repoName: string; +}): Promise { + try { + if (!config.giteaConfig?.url || !config.giteaConfig?.token) { + throw new Error("Gitea config is required."); + } + + const decryptedConfig = decryptConfigTokens(config as Config); + + const response = await httpGet( + `${config.giteaConfig.url}/api/v1/repos/${owner}/${repoName}`, + { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + } + ); + + return response.data; + } catch (error) { + if (error instanceof HttpError && error.status === 404) { + return null; // Repository doesn't exist + } + throw error; + } +} + +/** + * Enhanced organization creation with better error handling and retry logic + */ +export async function getOrCreateGiteaOrgEnhanced({ + orgName, + orgId, + config, + maxRetries = 3, + retryDelay = 100, +}: { + orgId?: string; + orgName: string; + config: Partial; + maxRetries?: number; + retryDelay?: number; +}): Promise { + if (!config.giteaConfig?.url || !config.giteaConfig?.token || !config.userId) { + throw new Error("Gitea config is required."); + } + + const decryptedConfig = decryptConfigTokens(config as Config); + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + console.log(`[Org Creation] Attempting to get or create organization: ${orgName} (attempt ${attempt + 1}/${maxRetries})`); + + // Check if org exists + try { + const orgResponse = await httpGet<{ id: number }>( + `${config.giteaConfig.url}/api/v1/orgs/${orgName}`, + { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + } + ); + + console.log(`[Org Creation] Organization ${orgName} already exists with ID: ${orgResponse.data.id}`); + return orgResponse.data.id; + } catch (error) { + if (!(error instanceof HttpError) || error.status !== 404) { + throw error; // Unexpected error + } + // Organization doesn't exist, continue to create it + } + + // Try to create the organization + console.log(`[Org Creation] Organization ${orgName} not found. Creating new organization.`); + + const visibility = config.giteaConfig.visibility || "public"; + const createOrgPayload = { + username: orgName, + full_name: orgName === "starred" ? "Starred Repositories" : orgName, + description: orgName === "starred" + ? "Repositories starred on GitHub" + : `Mirrored from GitHub organization: ${orgName}`, + website: "", + location: "", + visibility: visibility, + }; + + try { + const createResponse = await httpPost<{ id: number }>( + `${config.giteaConfig.url}/api/v1/orgs`, + createOrgPayload, + { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + } + ); + + console.log(`[Org Creation] Successfully created organization ${orgName} with ID: ${createResponse.data.id}`); + + await createMirrorJob({ + userId: config.userId, + organizationId: orgId, + organizationName: orgName, + message: `Successfully created Gitea organization: ${orgName}`, + status: "success", + details: `Organization ${orgName} was created in Gitea with ID ${createResponse.data.id}.`, + }); + + return createResponse.data.id; + } catch (createError) { + // Check if it's a duplicate error + if (createError instanceof HttpError) { + const errorResponse = createError.response?.toLowerCase() || ""; + const isDuplicateError = + errorResponse.includes("duplicate") || + errorResponse.includes("already exists") || + errorResponse.includes("uqe_user_lower_name") || + errorResponse.includes("constraint"); + + if (isDuplicateError && attempt < maxRetries - 1) { + console.log(`[Org Creation] Organization creation failed due to duplicate. Will retry check.`); + + // Wait before retry with exponential backoff + const delay = retryDelay * Math.pow(2, attempt); + console.log(`[Org Creation] Waiting ${delay}ms before retry...`); + await new Promise(resolve => setTimeout(resolve, delay)); + continue; // Retry the loop + } + } + throw createError; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "Unknown error"; + + if (attempt === maxRetries - 1) { + // Final attempt failed + console.error(`[Org Creation] Failed to get or create organization ${orgName} after ${maxRetries} attempts: ${errorMessage}`); + + await createMirrorJob({ + userId: config.userId, + organizationId: orgId, + organizationName: orgName, + message: `Failed to create or fetch Gitea organization: ${orgName}`, + status: "failed", + details: `Error after ${maxRetries} attempts: ${errorMessage}`, + }); + + throw new Error(`Failed to create organization ${orgName}: ${errorMessage}`); + } + + // Log retry attempt + console.warn(`[Org Creation] Attempt ${attempt + 1} failed for organization ${orgName}: ${errorMessage}. Retrying...`); + + // Wait before retry + const delay = retryDelay * Math.pow(2, attempt); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + // Should never reach here + throw new Error(`Failed to create organization ${orgName} after ${maxRetries} attempts`); +} + +/** + * Enhanced sync operation that handles non-mirror repositories + */ +export async function syncGiteaRepoEnhanced({ + config, + repository, +}: { + config: Partial; + repository: Repository; +}): Promise { + try { + if (!config.userId || !config.giteaConfig?.url || !config.giteaConfig?.token) { + throw new Error("Gitea config is required."); + } + + const decryptedConfig = decryptConfigTokens(config as Config); + + console.log(`[Sync] Starting sync for repository ${repository.name}`); + + // Mark repo as "syncing" in DB + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("syncing"), + updatedAt: new Date(), + }) + .where(eq(repositories.id, repository.id!)); + + // Get the expected owner + const { getGiteaRepoOwnerAsync } = await import("./gitea"); + const repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); + + // Check if repo exists and get its info + const repoInfo = await getGiteaRepoInfo({ + config, + owner: repoOwner, + repoName: repository.name, + }); + + if (!repoInfo) { + throw new Error(`Repository ${repository.name} not found in Gitea at ${repoOwner}/${repository.name}`); + } + + // Check if it's a mirror repository + if (!repoInfo.mirror) { + console.warn(`[Sync] Repository ${repository.name} exists but is not configured as a mirror`); + + // Update database to reflect this status + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("failed"), + updatedAt: new Date(), + errorMessage: "Repository exists in Gitea but is not configured as a mirror. Manual intervention required.", + }) + .where(eq(repositories.id, repository.id!)); + + await createMirrorJob({ + userId: config.userId, + repositoryId: repository.id, + repositoryName: repository.name, + message: `Cannot sync ${repository.name}: Not a mirror repository`, + details: `Repository ${repository.name} exists in Gitea but is not configured as a mirror. You may need to delete and recreate it as a mirror, or manually configure it as a mirror in Gitea.`, + status: "failed", + }); + + throw new Error(`Repository ${repository.name} is not a mirror. Cannot sync.`); + } + + // Perform the sync + const apiUrl = `${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repository.name}/mirror-sync`; + + try { + const response = await httpPost(apiUrl, undefined, { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + }); + + // Mark repo as "synced" in DB + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("synced"), + updatedAt: new Date(), + lastMirrored: new Date(), + errorMessage: null, + mirroredLocation: `${repoOwner}/${repository.name}`, + }) + .where(eq(repositories.id, repository.id!)); + + await createMirrorJob({ + userId: config.userId, + repositoryId: repository.id, + repositoryName: repository.name, + message: `Successfully synced repository: ${repository.name}`, + details: `Repository ${repository.name} was synced with Gitea.`, + status: "synced", + }); + + console.log(`[Sync] Repository ${repository.name} synced successfully`); + return response.data; + } catch (syncError) { + if (syncError instanceof HttpError && syncError.status === 400) { + // Handle specific mirror-sync errors + const errorMessage = syncError.response?.toLowerCase() || ""; + if (errorMessage.includes("not a mirror")) { + // Update status to indicate this specific error + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("failed"), + updatedAt: new Date(), + errorMessage: "Repository is not configured as a mirror in Gitea", + }) + .where(eq(repositories.id, repository.id!)); + + await createMirrorJob({ + userId: config.userId, + repositoryId: repository.id, + repositoryName: repository.name, + message: `Sync failed: ${repository.name} is not a mirror`, + details: "The repository exists in Gitea but is not configured as a mirror. Manual intervention required.", + status: "failed", + }); + } + } + throw syncError; + } + } catch (error) { + console.error(`[Sync] Error while syncing repository ${repository.name}:`, error); + + // Update repo with error status + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("failed"), + updatedAt: new Date(), + errorMessage: error instanceof Error ? error.message : "Unknown error", + }) + .where(eq(repositories.id, repository.id!)); + + if (config.userId && repository.id && repository.name) { + await createMirrorJob({ + userId: config.userId, + repositoryId: repository.id, + repositoryName: repository.name, + message: `Failed to sync repository: ${repository.name}`, + details: error instanceof Error ? error.message : "Unknown error", + status: "failed", + }); + } + + throw error; + } +} + +/** + * Delete a repository in Gitea (useful for cleaning up non-mirror repos) + */ +export async function deleteGiteaRepo({ + config, + owner, + repoName, +}: { + config: Partial; + owner: string; + repoName: string; +}): Promise { + if (!config.giteaConfig?.url || !config.giteaConfig?.token) { + throw new Error("Gitea config is required."); + } + + const decryptedConfig = decryptConfigTokens(config as Config); + + const response = await fetch( + `${config.giteaConfig.url}/api/v1/repos/${owner}/${repoName}`, + { + method: "DELETE", + headers: { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + }, + } + ); + + if (!response.ok && response.status !== 404) { + throw new Error(`Failed to delete repository: ${response.statusText}`); + } +} + +/** + * Convert a regular repository to a mirror (if supported by Gitea version) + * Note: This might not be supported in all Gitea versions + */ +export async function convertToMirror({ + config, + owner, + repoName, + cloneUrl, +}: { + config: Partial; + owner: string; + repoName: string; + cloneUrl: string; +}): Promise { + // This is a placeholder - actual implementation depends on Gitea API support + // Most Gitea versions don't support converting existing repos to mirrors + console.warn(`[Convert] Converting existing repositories to mirrors is not supported in most Gitea versions`); + return false; +} + +/** + * Sequential organization creation to avoid race conditions + */ +export async function createOrganizationsSequentially({ + config, + orgNames, +}: { + config: Partial; + orgNames: string[]; +}): Promise> { + const orgIdMap = new Map(); + + for (const orgName of orgNames) { + try { + const orgId = await getOrCreateGiteaOrgEnhanced({ + orgName, + config, + maxRetries: 3, + retryDelay: 100, + }); + orgIdMap.set(orgName, orgId); + } catch (error) { + console.error(`Failed to create organization ${orgName}:`, error); + // Continue with other organizations + } + } + + return orgIdMap; +} + +/** + * Check and handle existing non-mirror repositories + */ +export async function handleExistingNonMirrorRepo({ + config, + repository, + repoInfo, + strategy = "skip", +}: { + config: Partial; + repository: Repository; + repoInfo: GiteaRepoInfo; + strategy?: "skip" | "delete" | "rename"; +}): Promise { + const owner = repoInfo.owner; + const repoName = repoInfo.name; + + switch (strategy) { + case "skip": + console.log(`[Handle] Skipping existing non-mirror repository: ${owner}/${repoName}`); + + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("failed"), + updatedAt: new Date(), + errorMessage: "Repository exists but is not a mirror. Skipped.", + }) + .where(eq(repositories.id, repository.id!)); + + break; + + case "delete": + console.log(`[Handle] Deleting existing non-mirror repository: ${owner}/${repoName}`); + + await deleteGiteaRepo({ + config, + owner, + repoName, + }); + + console.log(`[Handle] Deleted repository ${owner}/${repoName}. It can now be recreated as a mirror.`); + break; + + case "rename": + console.log(`[Handle] Renaming strategy not implemented yet for: ${owner}/${repoName}`); + // TODO: Implement rename strategy if needed + break; + } +} \ No newline at end of file diff --git a/src/lib/gitea-org-creation.test.ts b/src/lib/gitea-org-creation.test.ts new file mode 100644 index 0000000..9f652cb --- /dev/null +++ b/src/lib/gitea-org-creation.test.ts @@ -0,0 +1,438 @@ +import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; +import { getOrCreateGiteaOrg } from "./gitea"; +import type { Config } from "./db/schema"; +import { createMirrorJob } from "./helpers"; + +// Mock the helpers module +mock.module("@/lib/helpers", () => { + return { + createMirrorJob: mock(() => Promise.resolve("job-id")) + }; +}); + +describe("Gitea Organization Creation Error Handling", () => { + let originalFetch: typeof global.fetch; + let mockCreateMirrorJob: any; + + beforeEach(() => { + originalFetch = global.fetch; + mockCreateMirrorJob = mock(() => Promise.resolve("job-id")); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + describe("Duplicate organization constraint errors", () => { + test("should handle PostgreSQL duplicate key constraint violation", async () => { + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs/starred") && options?.method === "GET") { + // Organization doesn't exist according to GET + return { + ok: false, + status: 404, + statusText: "Not Found" + } as Response; + } + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + // But creation fails with duplicate key error + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "insert organization: pq: duplicate key value violates unique constraint \"UQE_user_lower_name\"", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + try { + await getOrCreateGiteaOrg({ + orgName: "starred", + config + }); + expect(false).toBe(true); // Should not reach here + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("duplicate key value violates unique constraint"); + } + }); + + test("should handle MySQL duplicate entry error", async () => { + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs/starred") && options?.method === "GET") { + return { + ok: false, + status: 404 + } as Response; + } + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Duplicate entry 'starred' for key 'organizations.username'", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + try { + await getOrCreateGiteaOrg({ + orgName: "starred", + config + }); + expect(false).toBe(true); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("Duplicate entry"); + } + }); + }); + + describe("Race condition handling", () => { + test("should handle race condition where org is created between check and create", async () => { + let checkCount = 0; + + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs/starred") && options?.method === "GET") { + checkCount++; + + if (checkCount === 1) { + // First check: org doesn't exist + return { + ok: false, + status: 404 + } as Response; + } else { + // Subsequent checks: org exists (created by another process) + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 789, + username: "starred", + full_name: "Starred Repositories" + }) + } as Response; + } + } + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + // Creation fails because org was created by another process + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Organization already exists", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + // Current implementation throws error - should ideally retry and succeed + try { + await getOrCreateGiteaOrg({ + orgName: "starred", + config + }); + expect(false).toBe(true); + } catch (error) { + expect(error).toBeInstanceOf(Error); + // Documents current behavior - should be improved + } + }); + + test("proposed fix: retry logic for race conditions", async () => { + // This test documents how the function should handle race conditions + const getOrCreateGiteaOrgWithRetry = async ({ + orgName, + config, + maxRetries = 3 + }: { + orgName: string; + config: Partial; + maxRetries?: number; + }): Promise => { + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + // Check if org exists + const checkResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/orgs/${orgName}`, + { + headers: { + Authorization: `token ${config.giteaConfig!.token}` + } + } + ); + + if (checkResponse.ok) { + const org = await checkResponse.json(); + return org.id; + } + + // Try to create org + const createResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/orgs`, + { + method: "POST", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + username: orgName, + full_name: orgName === "starred" ? "Starred Repositories" : orgName + }) + } + ); + + if (createResponse.ok) { + const newOrg = await createResponse.json(); + return newOrg.id; + } + + const error = await createResponse.json(); + + // If it's a duplicate error, retry with check + if ( + error.message?.includes("duplicate") || + error.message?.includes("already exists") + ) { + continue; // Retry the loop + } + + throw new Error(error.message); + } catch (error) { + if (attempt === maxRetries - 1) { + throw error; + } + } + } + + throw new Error(`Failed to create organization after ${maxRetries} attempts`); + }; + + // Mock successful retry scenario + let attemptCount = 0; + global.fetch = mock(async (url: string, options?: RequestInit) => { + attemptCount++; + + if (url.includes("/api/v1/orgs/starred") && options?.method === "GET") { + if (attemptCount <= 2) { + return { ok: false, status: 404 } as Response; + } + // On third attempt, org exists + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ id: 999, username: "starred" }) + } as Response; + } + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + // Always fail creation with duplicate error + return { + ok: false, + status: 400, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ message: "Organization already exists" }) + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + const orgId = await getOrCreateGiteaOrgWithRetry({ + orgName: "starred", + config + }); + + expect(orgId).toBe(999); + expect(attemptCount).toBeGreaterThan(2); + }); + }); + + describe("Organization naming conflicts", () => { + test("should handle case-sensitivity conflicts", async () => { + // Some databases treat 'Starred' and 'starred' as the same + global.fetch = mock(async (url: string, options?: RequestInit) => { + const body = options?.body ? JSON.parse(options.body as string) : null; + + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + if (body?.username === "Starred") { + return { + ok: false, + status: 400, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Organization 'starred' already exists (case-insensitive match)", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + try { + const response = await fetch( + `${config.giteaConfig!.url}/api/v1/orgs`, + { + method: "POST", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + username: "Starred", // Different case + full_name: "Starred Repositories" + }) + } + ); + + const error = await response.json(); + expect(error.message).toContain("case-insensitive match"); + } catch (error) { + // Expected + } + }); + + test("should suggest alternative org names when conflicts occur", () => { + const suggestAlternativeOrgNames = (baseName: string): string[] => { + return [ + `${baseName}-mirror`, + `${baseName}-repos`, + `${baseName}-${new Date().getFullYear()}`, + `my-${baseName}`, + `github-${baseName}` + ]; + }; + + const alternatives = suggestAlternativeOrgNames("starred"); + + expect(alternatives).toContain("starred-mirror"); + expect(alternatives).toContain("starred-repos"); + expect(alternatives.length).toBeGreaterThanOrEqual(5); + }); + }); + + describe("Permission and visibility issues", () => { + test("should handle organization visibility constraints", async () => { + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + const body = JSON.parse(options.body as string); + + // Simulate server rejecting certain visibility settings + if (body.visibility === "private") { + return { + ok: false, + status: 400, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Private organizations are not allowed for this user", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token", + visibility: "private" // This will cause the error + } + }; + + try { + const response = await fetch( + `${config.giteaConfig!.url}/api/v1/orgs`, + { + method: "POST", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + username: "starred", + full_name: "Starred Repositories", + visibility: config.giteaConfig!.visibility + }) + } + ); + + if (!response.ok) { + const error = await response.json(); + expect(error.message).toContain("Private organizations are not allowed"); + } + } catch (error) { + // Expected + } + }); + }); +}); \ No newline at end of file diff --git a/src/lib/gitea-org-fix.ts b/src/lib/gitea-org-fix.ts new file mode 100644 index 0000000..a603426 --- /dev/null +++ b/src/lib/gitea-org-fix.ts @@ -0,0 +1,271 @@ +import type { Config } from "@/types/config"; +import { createMirrorJob } from "./helpers"; +import { decryptConfigTokens } from "./utils/config-encryption"; + +/** + * Enhanced version of getOrCreateGiteaOrg with retry logic for race conditions + * This implementation handles the duplicate organization constraint errors + */ +export async function getOrCreateGiteaOrgWithRetry({ + orgName, + orgId, + config, + maxRetries = 3, + retryDelay = 100, +}: { + orgId?: string; // db id + orgName: string; + config: Partial; + maxRetries?: number; + retryDelay?: number; +}): Promise { + if ( + !config.giteaConfig?.url || + !config.giteaConfig?.token || + !config.userId + ) { + throw new Error("Gitea config is required."); + } + + const decryptedConfig = decryptConfigTokens(config as Config); + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + console.log(`Attempting to get or create Gitea organization: ${orgName} (attempt ${attempt + 1}/${maxRetries})`); + + // Check if org exists + const orgRes = await fetch( + `${config.giteaConfig.url}/api/v1/orgs/${orgName}`, + { + headers: { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + "Content-Type": "application/json", + }, + } + ); + + if (orgRes.ok) { + // Organization exists, return its ID + const contentType = orgRes.headers.get("content-type"); + if (!contentType || !contentType.includes("application/json")) { + throw new Error( + `Invalid response format from Gitea API. Expected JSON but got: ${contentType}` + ); + } + + const org = await orgRes.json(); + console.log(`Organization ${orgName} already exists with ID: ${org.id}`); + + await createMirrorJob({ + userId: config.userId, + organizationId: orgId, + organizationName: orgName, + message: `Found existing Gitea organization: ${orgName}`, + status: "success", + details: `Organization ${orgName} already exists in Gitea with ID ${org.id}.`, + }); + + return org.id; + } + + if (orgRes.status !== 404) { + // Unexpected error + const errorText = await orgRes.text(); + throw new Error( + `Unexpected response from Gitea API: ${orgRes.status} ${orgRes.statusText}. Body: ${errorText}` + ); + } + + // Organization doesn't exist, try to create it + console.log(`Organization ${orgName} not found. Creating new organization.`); + + const visibility = config.giteaConfig.visibility || "public"; + const createOrgPayload = { + username: orgName, + full_name: orgName === "starred" ? "Starred Repositories" : orgName, + description: orgName === "starred" + ? "Repositories starred on GitHub" + : `Mirrored from GitHub organization: ${orgName}`, + website: "", + location: "", + visibility: visibility, + }; + + const createRes = await fetch( + `${config.giteaConfig.url}/api/v1/orgs`, + { + method: "POST", + headers: { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(createOrgPayload), + } + ); + + if (createRes.ok) { + // Successfully created + const newOrg = await createRes.json(); + console.log(`Successfully created organization ${orgName} with ID: ${newOrg.id}`); + + await createMirrorJob({ + userId: config.userId, + organizationId: orgId, + organizationName: orgName, + message: `Successfully created Gitea organization: ${orgName}`, + status: "success", + details: `Organization ${orgName} was created in Gitea with ID ${newOrg.id}.`, + }); + + return newOrg.id; + } + + // Handle creation failure + const createError = await createRes.json(); + + // Check if it's a duplicate error + if ( + createError.message?.includes("duplicate") || + createError.message?.includes("already exists") || + createError.message?.includes("UQE_user_lower_name") + ) { + console.log(`Organization creation failed due to duplicate. Will retry check.`); + + // Wait before retry with exponential backoff + if (attempt < maxRetries - 1) { + const delay = retryDelay * Math.pow(2, attempt); + console.log(`Waiting ${delay}ms before retry...`); + await new Promise(resolve => setTimeout(resolve, delay)); + continue; // Retry the loop + } + } + + // Non-retryable error + throw new Error( + `Failed to create organization ${orgName}: ${createError.message || createRes.statusText}` + ); + + } catch (error) { + const errorMessage = + error instanceof Error + ? error.message + : "Unknown error occurred in getOrCreateGiteaOrg."; + + if (attempt === maxRetries - 1) { + // Final attempt failed + console.error( + `Failed to get or create organization ${orgName} after ${maxRetries} attempts: ${errorMessage}` + ); + + await createMirrorJob({ + userId: config.userId, + organizationId: orgId, + organizationName: orgName, + message: `Failed to create or fetch Gitea organization: ${orgName}`, + status: "failed", + details: `Error after ${maxRetries} attempts: ${errorMessage}`, + }); + + throw new Error(`Error in getOrCreateGiteaOrg: ${errorMessage}`); + } + + // Log retry attempt + console.warn( + `Attempt ${attempt + 1} failed for organization ${orgName}: ${errorMessage}. Retrying...` + ); + + // Wait before retry + const delay = retryDelay * Math.pow(2, attempt); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + // Should never reach here + throw new Error(`Failed to create organization ${orgName} after ${maxRetries} attempts`); +} + +/** + * Helper function to check if an error is retryable + */ +export function isRetryableOrgError(error: any): boolean { + if (!error?.message) return false; + + const retryablePatterns = [ + "duplicate", + "already exists", + "UQE_user_lower_name", + "constraint", + "timeout", + "ECONNREFUSED", + "ENOTFOUND", + "network" + ]; + + const errorMessage = error.message.toLowerCase(); + return retryablePatterns.some(pattern => errorMessage.includes(pattern)); +} + +/** + * Pre-validate organization setup before bulk operations + */ +export async function validateOrgSetup({ + config, + orgNames, +}: { + config: Partial; + orgNames: string[]; +}): Promise<{ valid: boolean; issues: string[] }> { + const issues: string[] = []; + + if (!config.giteaConfig?.url || !config.giteaConfig?.token) { + issues.push("Gitea configuration is missing"); + return { valid: false, issues }; + } + + const decryptedConfig = decryptConfigTokens(config as Config); + + for (const orgName of orgNames) { + try { + const response = await fetch( + `${config.giteaConfig.url}/api/v1/orgs/${orgName}`, + { + headers: { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + }, + } + ); + + if (!response.ok && response.status !== 404) { + issues.push(`Cannot check organization '${orgName}': ${response.statusText}`); + } + } catch (error) { + issues.push(`Network error checking organization '${orgName}': ${error}`); + } + } + + // Check if user has permission to create organizations + try { + const userResponse = await fetch( + `${config.giteaConfig.url}/api/v1/user`, + { + headers: { + Authorization: `token ${decryptedConfig.giteaConfig.token}`, + }, + } + ); + + if (userResponse.ok) { + const user = await userResponse.json(); + if (user.prohibit_login) { + issues.push("User account is prohibited from login"); + } + if (user.restricted) { + issues.push("User account is restricted"); + } + } + } catch (error) { + issues.push(`Cannot verify user permissions: ${error}`); + } + + return { valid: issues.length === 0, issues }; +} \ No newline at end of file diff --git a/src/lib/gitea-starred-repos.test.ts b/src/lib/gitea-starred-repos.test.ts new file mode 100644 index 0000000..a2e4935 --- /dev/null +++ b/src/lib/gitea-starred-repos.test.ts @@ -0,0 +1,390 @@ +import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; +import { getOrCreateGiteaOrg, mirrorGitHubOrgRepoToGiteaOrg, isRepoPresentInGitea } from "./gitea"; +import type { Config, Repository } from "./db/schema"; +import { repoStatusEnum } from "@/types/Repository"; + +describe("Starred Repository Error Handling", () => { + let originalFetch: typeof global.fetch; + let consoleLogs: string[] = []; + let consoleErrors: string[] = []; + + beforeEach(() => { + originalFetch = global.fetch; + consoleLogs = []; + consoleErrors = []; + + // Capture console output for debugging + console.log = mock((message: string) => { + consoleLogs.push(message); + }); + console.error = mock((message: string) => { + consoleErrors.push(message); + }); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + describe("Repository is not a mirror error", () => { + test("should handle 400 error when trying to sync a non-mirror repo", async () => { + // Mock fetch to simulate the "Repository is not a mirror" error + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/repos/starred/test-repo/mirror-sync")) { + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Repository is not a mirror", + url: "https://gitea.ui.com/api/swagger" + }) + } as Response; + } + + // Mock successful repo check + if (url.includes("/api/v1/repos/starred/test-repo")) { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 123, + name: "test-repo", + mirror: false, // Repo is not a mirror + owner: { login: "starred" } + }) + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token", + defaultOwner: "testuser", + starredReposOrg: "starred" + }, + githubConfig: { + token: "github-token", + starredReposOrg: "starred" + } + }; + + const repository: Repository = { + id: "repo-123", + userId: "user-123", + configId: "config-123", + name: "test-repo", + fullName: "original-owner/test-repo", + url: "https://github.com/original-owner/test-repo", + cloneUrl: "https://github.com/original-owner/test-repo.git", + owner: "original-owner", + isPrivate: false, + isForked: false, + hasIssues: true, + isStarred: true, // This is a starred repo + isArchived: false, + size: 1000, + hasLFS: false, + hasSubmodules: false, + defaultBranch: "main", + visibility: "public", + status: "mirrored", + mirroredLocation: "starred/test-repo", + createdAt: new Date(), + updatedAt: new Date() + }; + + // Verify that the repo exists but is not a mirror + const exists = await isRepoPresentInGitea({ + config, + owner: "starred", + repoName: "test-repo" + }); + + expect(exists).toBe(true); + + // The error would occur during sync operation + // This test verifies the scenario exists + }); + + test("should detect when a starred repo was created as regular repo instead of mirror", async () => { + // Mock fetch to return repo details + global.fetch = mock(async (url: string) => { + if (url.includes("/api/v1/repos/starred/test-repo")) { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 123, + name: "test-repo", + mirror: false, // This is the problem - repo is not a mirror + owner: { login: "starred" }, + clone_url: "https://gitea.ui.com/starred/test-repo.git", + original_url: null // No original URL since it's not a mirror + }) + } as Response; + } + + return originalFetch(url); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token" + } + }; + + // Check if repo exists + const exists = await isRepoPresentInGitea({ + config, + owner: "starred", + repoName: "test-repo" + }); + + expect(exists).toBe(true); + + // In a real scenario, we would need to: + // 1. Delete the non-mirror repo + // 2. Recreate it as a mirror + // This test documents the problematic state + }); + }); + + describe("Duplicate organization error", () => { + test("should handle duplicate organization creation error", async () => { + // Mock fetch to simulate duplicate org error + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs/starred") && options?.method === "POST") { + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "insert organization: pq: duplicate key value violates unique constraint \"UQE_user_lower_name\"", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + + // Mock org check - org doesn't exist according to API + if (url.includes("/api/v1/orgs/starred") && options?.method === "GET") { + return { + ok: false, + status: 404, + statusText: "Not Found" + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + try { + await getOrCreateGiteaOrg({ + orgName: "starred", + config + }); + expect(false).toBe(true); // Should not reach here + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("duplicate key value violates unique constraint"); + } + }); + + test("should handle race condition in organization creation", async () => { + let orgCheckCount = 0; + + // Mock fetch to simulate race condition + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs/starred") && options?.method === "GET") { + orgCheckCount++; + // First check returns 404, second returns 200 (org was created by another process) + if (orgCheckCount === 1) { + return { + ok: false, + status: 404, + statusText: "Not Found" + } as Response; + } else { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 456, + username: "starred", + full_name: "Starred Repositories" + }) + } as Response; + } + } + + if (url.includes("/api/v1/orgs/starred") && options?.method === "POST") { + // Simulate duplicate error + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "insert organization: pq: duplicate key value violates unique constraint \"UQE_user_lower_name\"", + url: "https://gitea.url.com/api/swagger" + }) + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.url.com", + token: "gitea-token" + } + }; + + // In a proper implementation, this should retry and succeed + // Current implementation throws an error + try { + await getOrCreateGiteaOrg({ + orgName: "starred", + config + }); + expect(false).toBe(true); // Should not reach here with current implementation + } catch (error) { + expect(error).toBeInstanceOf(Error); + // This documents the current behavior - it should be improved + } + }); + }); + + describe("Comprehensive starred repository mirroring flow", () => { + test("should handle the complete flow of mirroring a starred repository", async () => { + const mockResponses = new Map(); + + // Setup mock responses + mockResponses.set("GET /api/v1/orgs/starred", { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 789, + username: "starred", + full_name: "Starred Repositories" + }) + }); + + mockResponses.set("GET /api/v1/repos/starred/awesome-project", { + ok: false, + status: 404 + }); + + mockResponses.set("POST /api/v1/repos/migrate", { + ok: true, + status: 201, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 999, + name: "awesome-project", + mirror: true, + owner: { login: "starred" } + }) + }); + + global.fetch = mock(async (url: string, options?: RequestInit) => { + const method = options?.method || "GET"; + + if (url.includes("/api/v1/orgs/starred") && method === "GET") { + return mockResponses.get("GET /api/v1/orgs/starred"); + } + + if (url.includes("/api/v1/repos/starred/awesome-project") && method === "GET") { + return mockResponses.get("GET /api/v1/repos/starred/awesome-project"); + } + + if (url.includes("/api/v1/repos/migrate") && method === "POST") { + const body = JSON.parse(options?.body as string); + expect(body.repo_owner).toBe("starred"); + expect(body.mirror).toBe(true); + return mockResponses.get("POST /api/v1/repos/migrate"); + } + + return originalFetch(url, options); + }); + + // Test the flow + const config: Partial = { + userId: "user-123", + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token", + defaultOwner: "testuser" + }, + githubConfig: { + token: "github-token", + starredReposOrg: "starred" + } + }; + + // 1. Check if org exists (it does) + const orgId = await getOrCreateGiteaOrg({ + orgName: "starred", + config + }); + expect(orgId).toBe(789); + + // 2. Check if repo exists (it doesn't) + const repoExists = await isRepoPresentInGitea({ + config, + owner: "starred", + repoName: "awesome-project" + }); + expect(repoExists).toBe(false); + + // 3. Create mirror would happen here in the actual flow + // The test verifies the setup is correct + }); + }); + + describe("Error recovery strategies", () => { + test("should suggest recovery steps for non-mirror repository", () => { + const recoverySteps = [ + "1. Delete the existing non-mirror repository in Gitea", + "2. Re-run the mirror operation to create it as a proper mirror", + "3. Alternatively, manually convert the repository to a mirror in Gitea settings" + ]; + + // This test documents the recovery strategy + expect(recoverySteps).toHaveLength(3); + }); + + test("should suggest recovery steps for duplicate organization", () => { + const recoverySteps = [ + "1. Check if the organization already exists in Gitea UI", + "2. If it exists but API returns 404, check permissions", + "3. Try using a different organization name for starred repos", + "4. Manually create the organization in Gitea if needed" + ]; + + // This test documents the recovery strategy + expect(recoverySteps).toHaveLength(4); + }); + }); +}); \ No newline at end of file diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index d80e32c..8658134 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -361,6 +361,29 @@ export const mirrorGithubRepoToGitea = async ({ }); } + // Check if repository already exists as a non-mirror + const { getGiteaRepoInfo, handleExistingNonMirrorRepo } = await import("./gitea-enhanced"); + const existingRepo = await getGiteaRepoInfo({ + config, + owner: repoOwner, + repoName: repository.name, + }); + + if (existingRepo && !existingRepo.mirror) { + console.log(`Repository ${repository.name} exists but is not a mirror. Handling...`); + + // Handle the existing non-mirror repository + await handleExistingNonMirrorRepo({ + config, + repository, + repoInfo: existingRepo, + strategy: "delete", // Can be configured: "skip", "delete", or "rename" + }); + + // After handling, proceed with mirror creation + console.log(`Proceeding with mirror creation for ${repository.name}`); + } + const response = await httpPost( apiUrl, { @@ -470,156 +493,23 @@ export async function getOrCreateGiteaOrg({ orgName: string; config: Partial; }): Promise { - if ( - !config.giteaConfig?.url || - !config.giteaConfig?.token || - !config.userId - ) { - throw new Error("Gitea config is required."); - } - + // Import the enhanced version with retry logic + const { getOrCreateGiteaOrgEnhanced } = await import("./gitea-enhanced"); + try { - console.log(`Attempting to get or create Gitea organization: ${orgName}`); - - // Decrypt config tokens for API usage - const decryptedConfig = decryptConfigTokens(config as Config); - - const orgRes = await fetch( - `${config.giteaConfig.url}/api/v1/orgs/${orgName}`, - { - headers: { - Authorization: `token ${decryptedConfig.giteaConfig.token}`, - "Content-Type": "application/json", - }, - } - ); - - console.log( - `Get org response status: ${orgRes.status} for org: ${orgName}` - ); - - if (orgRes.ok) { - // Check if response is actually JSON - const contentType = orgRes.headers.get("content-type"); - if (!contentType || !contentType.includes("application/json")) { - console.warn( - `Expected JSON response but got content-type: ${contentType}` - ); - const responseText = await orgRes.text(); - console.warn(`Response body: ${responseText}`); - throw new Error( - `Invalid response format from Gitea API. Expected JSON but got: ${contentType}` - ); - } - - // Clone the response to handle potential JSON parsing errors - const orgResClone = orgRes.clone(); - - try { - const org = await orgRes.json(); - console.log( - `Successfully retrieved existing org: ${orgName} with ID: ${org.id}` - ); - // Note: Organization events are handled by the main mirroring process - // to avoid duplicate events - return org.id; - } catch (jsonError) { - const responseText = await orgResClone.text(); - console.error( - `Failed to parse JSON response for existing org: ${responseText}` - ); - throw new Error( - `Failed to parse JSON response from Gitea API: ${ - jsonError instanceof Error ? jsonError.message : String(jsonError) - }` - ); - } - } - - console.log(`Organization ${orgName} not found, attempting to create it`); - - const createRes = await fetch(`${config.giteaConfig.url}/api/v1/orgs`, { - method: "POST", - headers: { - Authorization: `token ${decryptedConfig.giteaConfig.token}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - username: orgName, - full_name: `${orgName} Org`, - description: `Mirrored organization from GitHub ${orgName}`, - visibility: config.giteaConfig?.visibility || "public", - }), + return await getOrCreateGiteaOrgEnhanced({ + orgName, + orgId, + config, + maxRetries: 3, + retryDelay: 100, }); - - console.log( - `Create org response status: ${createRes.status} for org: ${orgName}` - ); - - if (!createRes.ok) { - const errorText = await createRes.text(); - console.error( - `Failed to create org ${orgName}. Status: ${createRes.status}, Response: ${errorText}` - ); - throw new Error(`Failed to create Gitea org: ${errorText}`); - } - - // Check if response is actually JSON - const createContentType = createRes.headers.get("content-type"); - if (!createContentType || !createContentType.includes("application/json")) { - console.warn( - `Expected JSON response but got content-type: ${createContentType}` - ); - const responseText = await createRes.text(); - console.warn(`Response body: ${responseText}`); - throw new Error( - `Invalid response format from Gitea API. Expected JSON but got: ${createContentType}` - ); - } - - // Note: Organization creation events are handled by the main mirroring process - // to avoid duplicate events - - // Clone the response to handle potential JSON parsing errors - const createResClone = createRes.clone(); - - try { - const newOrg = await createRes.json(); - console.log( - `Successfully created new org: ${orgName} with ID: ${newOrg.id}` - ); - return newOrg.id; - } catch (jsonError) { - const responseText = await createResClone.text(); - console.error( - `Failed to parse JSON response for new org: ${responseText}` - ); - throw new Error( - `Failed to parse JSON response from Gitea API: ${ - jsonError instanceof Error ? jsonError.message : String(jsonError) - }` - ); - } } catch (error) { - const errorMessage = - error instanceof Error - ? error.message - : "Unknown error occurred in getOrCreateGiteaOrg."; - - console.error( - `Error in getOrCreateGiteaOrg for ${orgName}: ${errorMessage}` - ); - - await createMirrorJob({ - userId: config.userId, - organizationId: orgId, - organizationName: orgName, - message: `Failed to create or fetch Gitea organization: ${orgName}`, - status: "failed", - details: `Error: ${errorMessage}`, - }); - - throw new Error(`Error in getOrCreateGiteaOrg: ${errorMessage}`); + // Re-throw with original function name for backward compatibility + if (error instanceof Error) { + throw new Error(`Error in getOrCreateGiteaOrg: ${error.message}`); + } + throw error; } } @@ -1077,117 +967,13 @@ export const syncGiteaRepo = async ({ config: Partial; repository: Repository; }) => { + // Use the enhanced sync function that handles non-mirror repos + const { syncGiteaRepoEnhanced } = await import("./gitea-enhanced"); + try { - if ( - !config.userId || - !config.giteaConfig?.url || - !config.giteaConfig?.token || - !config.giteaConfig?.defaultOwner - ) { - throw new Error("Gitea config is required."); - } - - // Decrypt config tokens for API usage - const decryptedConfig = decryptConfigTokens(config as Config); - - console.log(`Syncing repository ${repository.name}`); - - // Mark repo as "syncing" in DB - await db - .update(repositories) - .set({ - status: repoStatusEnum.parse("syncing"), - updatedAt: new Date(), - }) - .where(eq(repositories.id, repository.id!)); - - // Append log for "syncing" status - await createMirrorJob({ - userId: config.userId, - repositoryId: repository.id, - repositoryName: repository.name, - message: `Started syncing repository: ${repository.name}`, - details: `Repository ${repository.name} is now in the syncing state.`, - status: repoStatusEnum.parse("syncing"), - }); - - // Get the expected owner based on current config (with organization overrides) - const repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); - - // Check if repo exists at the expected location or alternate location - const { present, actualOwner } = await checkRepoLocation({ - config, - repository, - expectedOwner: repoOwner, - }); - - if (!present) { - throw new Error( - `Repository ${repository.name} not found in Gitea at any expected location` - ); - } - - // Use the actual owner where the repo was found - const apiUrl = `${config.giteaConfig.url}/api/v1/repos/${actualOwner}/${repository.name}/mirror-sync`; - - const response = await httpPost(apiUrl, undefined, { - Authorization: `token ${decryptedConfig.giteaConfig.token}`, - }); - - // Mark repo as "synced" in DB - await db - .update(repositories) - .set({ - status: repoStatusEnum.parse("synced"), - updatedAt: new Date(), - lastMirrored: new Date(), - errorMessage: null, - mirroredLocation: `${actualOwner}/${repository.name}`, - }) - .where(eq(repositories.id, repository.id!)); - - // Append log for "synced" status - await createMirrorJob({ - userId: config.userId, - repositoryId: repository.id, - repositoryName: repository.name, - message: `Successfully synced repository: ${repository.name}`, - details: `Repository ${repository.name} was synced with Gitea.`, - status: repoStatusEnum.parse("synced"), - }); - - console.log(`Repository ${repository.name} synced successfully`); - - return response.data; + return await syncGiteaRepoEnhanced({ config, repository }); } catch (error) { - console.error( - `Error while syncing repository ${repository.name}: ${ - error instanceof Error ? error.message : String(error) - }` - ); - - // Optional: update repo with error status - await db - .update(repositories) - .set({ - status: repoStatusEnum.parse("failed"), - updatedAt: new Date(), - errorMessage: (error as Error).message, - }) - .where(eq(repositories.id, repository.id!)); - - // Append log for "error" status - if (config.userId && repository.id && repository.name) { - await createMirrorJob({ - userId: config.userId, - repositoryId: repository.id, - repositoryName: repository.name, - message: `Failed to sync repository: ${repository.name}`, - details: (error as Error).message, - status: repoStatusEnum.parse("failed"), - }); - } - + // Re-throw with original function name for backward compatibility if (error instanceof Error) { throw new Error(`Failed to sync repository: ${error.message}`); } diff --git a/src/lib/mirror-sync-errors.test.ts b/src/lib/mirror-sync-errors.test.ts new file mode 100644 index 0000000..59c6524 --- /dev/null +++ b/src/lib/mirror-sync-errors.test.ts @@ -0,0 +1,373 @@ +import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; +import { db, repositories } from "./db"; +import { eq } from "drizzle-orm"; +import { repoStatusEnum } from "@/types/Repository"; +import type { Config, Repository } from "./db/schema"; + +describe("Mirror Sync Error Handling", () => { + let originalFetch: typeof global.fetch; + let mockDbUpdate: any; + + beforeEach(() => { + originalFetch = global.fetch; + + // Mock database update operations + mockDbUpdate = mock(() => ({ + set: mock(() => ({ + where: mock(() => Promise.resolve()) + })) + })); + + // Override the db.update method + (db as any).update = mockDbUpdate; + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + describe("Mirror sync API errors", () => { + test("should handle mirror-sync endpoint not available for non-mirror repos", async () => { + const errorResponse = { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Repository is not a mirror", + url: "https://gitea.ui.com/api/swagger" + }) + }; + + global.fetch = mock(async (url: string) => { + if (url.includes("/api/v1/repos/") && url.includes("/mirror-sync")) { + return errorResponse as Response; + } + return originalFetch(url); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token" + } + }; + + // Simulate attempting to sync a non-mirror repository + const response = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/starred/test-repo/mirror-sync`, + { + method: "POST", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + "Content-Type": "application/json" + } + } + ); + + expect(response.ok).toBe(false); + expect(response.status).toBe(400); + + const error = await response.json(); + expect(error.message).toBe("Repository is not a mirror"); + }); + + test("should update repository status to 'failed' when sync fails", async () => { + const repository: Repository = { + id: "repo-123", + userId: "user-123", + configId: "config-123", + name: "test-repo", + fullName: "owner/test-repo", + url: "https://github.com/owner/test-repo", + cloneUrl: "https://github.com/owner/test-repo.git", + owner: "owner", + isPrivate: false, + isForked: false, + hasIssues: true, + isStarred: true, + isArchived: false, + size: 1000, + hasLFS: false, + hasSubmodules: false, + defaultBranch: "main", + visibility: "public", + status: "mirroring", + mirroredLocation: "starred/test-repo", + createdAt: new Date(), + updatedAt: new Date() + }; + + // Simulate error handling in mirror process + const errorMessage = "Repository is not a mirror"; + + // This simulates what should happen when mirror sync fails + await db + .update(repositories) + .set({ + status: repoStatusEnum.parse("failed"), + errorMessage: errorMessage, + updatedAt: new Date() + }) + .where(eq(repositories.id, repository.id)); + + // Verify the update was called with correct parameters + expect(mockDbUpdate).toHaveBeenCalledWith(repositories); + + const setCalls = mockDbUpdate.mock.results[0].value.set.mock.calls; + expect(setCalls[0][0]).toMatchObject({ + status: "failed", + errorMessage: errorMessage + }); + }); + }); + + describe("Repository state detection", () => { + test("should detect when a repository exists but is not configured as mirror", async () => { + // Mock Gitea API response for repo info + global.fetch = mock(async (url: string) => { + if (url.includes("/api/v1/repos/starred/test-repo") && !url.includes("mirror-sync")) { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 123, + name: "test-repo", + owner: { login: "starred" }, + mirror: false, // This is the issue - should be true + fork: false, + private: false, + clone_url: "https://gitea.ui.com/starred/test-repo.git" + }) + } as Response; + } + return originalFetch(url); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token" + } + }; + + // Check repository details + const response = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/starred/test-repo`, + { + headers: { + Authorization: `token ${config.giteaConfig!.token}` + } + } + ); + + const repoInfo = await response.json(); + + // Verify the repository exists but is not a mirror + expect(repoInfo.mirror).toBe(false); + expect(repoInfo.owner.login).toBe("starred"); + + // This state causes the "Repository is not a mirror" error + }); + + test("should identify repositories that need to be recreated as mirrors", async () => { + const problematicRepos = [ + { + name: "awesome-project", + owner: "starred", + currentState: "regular", + requiredState: "mirror", + action: "delete and recreate" + }, + { + name: "cool-library", + owner: "starred", + currentState: "fork", + requiredState: "mirror", + action: "delete and recreate" + } + ]; + + // This test documents repos that need intervention + expect(problematicRepos).toHaveLength(2); + expect(problematicRepos[0].action).toBe("delete and recreate"); + }); + }); + + describe("Organization permission errors", () => { + test("should handle insufficient permissions for organization operations", async () => { + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/orgs") && options?.method === "POST") { + return { + ok: false, + status: 403, + statusText: "Forbidden", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "You do not have permission to create organizations", + url: "https://gitea.ui.com/api/swagger" + }) + } as Response; + } + return originalFetch(url, options); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token" + } + }; + + const response = await fetch( + `${config.giteaConfig!.url}/api/v1/orgs`, + { + method: "POST", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + username: "starred", + full_name: "Starred Repositories" + }) + } + ); + + expect(response.ok).toBe(false); + expect(response.status).toBe(403); + + const error = await response.json(); + expect(error.message).toContain("permission"); + }); + }); + + describe("Sync operation retry logic", () => { + test("should implement exponential backoff for transient errors", async () => { + let attemptCount = 0; + const maxRetries = 3; + const baseDelay = 1000; + + const mockSyncWithRetry = async (url: string, config: any) => { + for (let i = 0; i < maxRetries; i++) { + attemptCount++; + + try { + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `token ${config.token}` + } + }); + + if (response.ok) { + return response; + } + + if (response.status === 400) { + // Non-retryable error + throw new Error("Repository is not a mirror"); + } + + // Retryable error (5xx, network issues) + if (i < maxRetries - 1) { + const delay = baseDelay * Math.pow(2, i); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } catch (error) { + if (i === maxRetries - 1) { + throw error; + } + } + } + }; + + // Mock a server error that resolves after 2 retries + let callCount = 0; + global.fetch = mock(async () => { + callCount++; + if (callCount < 3) { + return { + ok: false, + status: 503, + statusText: "Service Unavailable" + } as Response; + } + return { + ok: true, + status: 200 + } as Response; + }); + + const response = await mockSyncWithRetry( + "https://gitea.ui.com/api/v1/repos/starred/test-repo/mirror-sync", + { token: "test-token" } + ); + + expect(response.ok).toBe(true); + expect(attemptCount).toBe(3); + }); + }); + + describe("Bulk operation error handling", () => { + test("should continue processing other repos when one fails", async () => { + const repositories = [ + { name: "repo1", owner: "starred", shouldFail: false }, + { name: "repo2", owner: "starred", shouldFail: true }, // This one will fail + { name: "repo3", owner: "starred", shouldFail: false } + ]; + + const results: { name: string; success: boolean; error?: string }[] = []; + + // Mock fetch to fail for repo2 + global.fetch = mock(async (url: string) => { + if (url.includes("repo2")) { + return { + ok: false, + status: 400, + statusText: "Bad Request", + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + message: "Repository is not a mirror" + }) + } as Response; + } + return { + ok: true, + status: 200 + } as Response; + }); + + // Process repositories + for (const repo of repositories) { + try { + const response = await fetch( + `https://gitea.ui.com/api/v1/repos/${repo.owner}/${repo.name}/mirror-sync`, + { method: "POST" } + ); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.message); + } + + results.push({ name: repo.name, success: true }); + } catch (error) { + results.push({ + name: repo.name, + success: false, + error: (error as Error).message + }); + } + } + + // Verify results + expect(results).toHaveLength(3); + expect(results[0].success).toBe(true); + expect(results[1].success).toBe(false); + expect(results[1].error).toBe("Repository is not a mirror"); + expect(results[2].success).toBe(true); + }); + }); +}); \ No newline at end of file diff --git a/src/lib/mirror-sync-fix.test.ts b/src/lib/mirror-sync-fix.test.ts new file mode 100644 index 0000000..856d34c --- /dev/null +++ b/src/lib/mirror-sync-fix.test.ts @@ -0,0 +1,392 @@ +import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; +import type { Config, Repository } from "./db/schema"; +import { repoStatusEnum } from "@/types/Repository"; + +describe("Mirror Sync Fix Implementation", () => { + let originalFetch: typeof global.fetch; + + beforeEach(() => { + originalFetch = global.fetch; + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + describe("Non-mirror repository recovery", () => { + test("should detect and handle non-mirror repositories", async () => { + const mockHandleNonMirrorRepo = async ({ + config, + repository, + owner, + }: { + config: Partial; + repository: Repository; + owner: string; + }) => { + try { + // First, check if the repo exists + const checkResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/${owner}/${repository.name}`, + { + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + }, + } + ); + + if (!checkResponse.ok) { + // Repo doesn't exist, we can create it as mirror + return { action: "create_mirror", success: true }; + } + + const repoInfo = await checkResponse.json(); + + if (!repoInfo.mirror) { + // Repository exists but is not a mirror + console.log(`Repository ${repository.name} exists but is not a mirror`); + + // Option 1: Delete and recreate + if (config.giteaConfig?.autoFixNonMirrors) { + const deleteResponse = await fetch( + `${config.giteaConfig.url}/api/v1/repos/${owner}/${repository.name}`, + { + method: "DELETE", + headers: { + Authorization: `token ${config.giteaConfig.token}`, + }, + } + ); + + if (deleteResponse.ok) { + return { action: "deleted_for_recreation", success: true }; + } + } + + // Option 2: Mark for manual intervention + return { + action: "manual_intervention_required", + success: false, + reason: "Repository exists but is not configured as mirror", + suggestion: `Delete ${owner}/${repository.name} in Gitea and re-run mirror`, + }; + } + + // Repository is already a mirror, can proceed with sync + return { action: "sync_mirror", success: true }; + } catch (error) { + return { + action: "error", + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }; + + // Test scenario 1: Non-mirror repository + global.fetch = mock(async (url: string) => { + if (url.includes("/api/v1/repos/starred/test-repo")) { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 123, + name: "test-repo", + mirror: false, // Not a mirror + owner: { login: "starred" }, + }), + } as Response; + } + return originalFetch(url); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token", + autoFixNonMirrors: false, // Manual intervention mode + }, + }; + + const repository: Repository = { + id: "repo-123", + name: "test-repo", + isStarred: true, + // ... other fields + } as Repository; + + const result = await mockHandleNonMirrorRepo({ + config, + repository, + owner: "starred", + }); + + expect(result.action).toBe("manual_intervention_required"); + expect(result.success).toBe(false); + expect(result.suggestion).toContain("Delete starred/test-repo"); + }); + + test("should successfully delete and prepare for recreation when autoFix is enabled", async () => { + let deleteRequested = false; + + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/repos/starred/test-repo")) { + if (options?.method === "DELETE") { + deleteRequested = true; + return { + ok: true, + status: 204, + } as Response; + } + + // GET request + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 123, + name: "test-repo", + mirror: false, + owner: { login: "starred" }, + }), + } as Response; + } + return originalFetch(url, options); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token", + autoFixNonMirrors: true, // Auto-fix enabled + }, + }; + + // Simulate the fix process + const checkResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/starred/test-repo`, + { + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + }, + } + ); + + const repoInfo = await checkResponse.json(); + expect(repoInfo.mirror).toBe(false); + + // Delete the non-mirror repo + const deleteResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/starred/test-repo`, + { + method: "DELETE", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + }, + } + ); + + expect(deleteResponse.ok).toBe(true); + expect(deleteRequested).toBe(true); + }); + }); + + describe("Enhanced mirror creation with validation", () => { + test("should validate repository before creating mirror", async () => { + const createMirrorWithValidation = async ({ + config, + repository, + owner, + }: { + config: Partial; + repository: Repository; + owner: string; + }) => { + // Step 1: Check if repo already exists + const checkResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/${owner}/${repository.name}`, + { + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + }, + } + ); + + if (checkResponse.ok) { + const existingRepo = await checkResponse.json(); + if (existingRepo.mirror) { + return { + created: false, + reason: "already_mirror", + repoId: existingRepo.id, + }; + } else { + return { + created: false, + reason: "exists_not_mirror", + repoId: existingRepo.id, + }; + } + } + + // Step 2: Create as mirror + const cloneUrl = repository.isPrivate + ? repository.cloneUrl.replace("https://", `https://GITHUB_TOKEN@`) + : repository.cloneUrl; + + const createResponse = await fetch( + `${config.giteaConfig!.url}/api/v1/repos/migrate`, + { + method: "POST", + headers: { + Authorization: `token ${config.giteaConfig!.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + clone_addr: cloneUrl, + repo_name: repository.name, + mirror: true, // Ensure this is always true + repo_owner: owner, + private: repository.isPrivate, + description: `Mirrored from ${repository.fullName}`, + service: "git", + }), + } + ); + + if (createResponse.ok) { + const newRepo = await createResponse.json(); + return { + created: true, + reason: "success", + repoId: newRepo.id, + }; + } + + const error = await createResponse.json(); + return { + created: false, + reason: "create_failed", + error: error.message, + }; + }; + + // Mock successful mirror creation + global.fetch = mock(async (url: string, options?: RequestInit) => { + if (url.includes("/api/v1/repos/starred/new-repo") && !options?.method) { + return { + ok: false, + status: 404, + } as Response; + } + + if (url.includes("/api/v1/repos/migrate")) { + const body = JSON.parse(options?.body as string); + expect(body.mirror).toBe(true); // Validate mirror flag + expect(body.repo_owner).toBe("starred"); + + return { + ok: true, + status: 201, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: 456, + name: body.repo_name, + mirror: true, + owner: { login: body.repo_owner }, + }), + } as Response; + } + + return originalFetch(url, options); + }); + + const config: Partial = { + giteaConfig: { + url: "https://gitea.ui.com", + token: "gitea-token", + }, + }; + + const repository: Repository = { + id: "repo-456", + name: "new-repo", + fullName: "original/new-repo", + cloneUrl: "https://github.com/original/new-repo.git", + isPrivate: false, + isStarred: true, + // ... other fields + } as Repository; + + const result = await createMirrorWithValidation({ + config, + repository, + owner: "starred", + }); + + expect(result.created).toBe(true); + expect(result.reason).toBe("success"); + expect(result.repoId).toBe(456); + }); + }); + + describe("Sync status tracking", () => { + test("should track sync attempts and failures", async () => { + interface SyncAttempt { + repositoryId: string; + attemptNumber: number; + timestamp: Date; + error?: string; + success: boolean; + } + + const syncAttempts: Map = new Map(); + + const trackSyncAttempt = ( + repositoryId: string, + success: boolean, + error?: string + ) => { + const attempts = syncAttempts.get(repositoryId) || []; + attempts.push({ + repositoryId, + attemptNumber: attempts.length + 1, + timestamp: new Date(), + error, + success, + }); + syncAttempts.set(repositoryId, attempts); + }; + + const shouldRetrySync = (repositoryId: string): boolean => { + const attempts = syncAttempts.get(repositoryId) || []; + if (attempts.length === 0) return true; + + const lastAttempt = attempts[attempts.length - 1]; + const timeSinceLastAttempt = + Date.now() - lastAttempt.timestamp.getTime(); + + // Retry if: + // 1. Less than 3 attempts + // 2. At least 5 minutes since last attempt + // 3. Last error was not "Repository is not a mirror" + return ( + attempts.length < 3 && + timeSinceLastAttempt > 5 * 60 * 1000 && + !lastAttempt.error?.includes("Repository is not a mirror") + ); + }; + + // Simulate sync attempts + trackSyncAttempt("repo-123", false, "Repository is not a mirror"); + trackSyncAttempt("repo-456", false, "Network timeout"); + trackSyncAttempt("repo-456", true); + + expect(shouldRetrySync("repo-123")).toBe(false); // Non-retryable error + expect(shouldRetrySync("repo-456")).toBe(false); // Already succeeded + expect(shouldRetrySync("repo-789")).toBe(true); // No attempts yet + }); + }); +}); \ No newline at end of file diff --git a/src/lib/starred-repos-handler.ts b/src/lib/starred-repos-handler.ts new file mode 100644 index 0000000..7341423 --- /dev/null +++ b/src/lib/starred-repos-handler.ts @@ -0,0 +1,290 @@ +/** + * Enhanced handler for starred repositories with improved error handling + */ + +import type { Config, Repository } from "./db/schema"; +import { Octokit } from "@octokit/rest"; +import { processWithRetry } from "./utils/concurrency"; +import { + getOrCreateGiteaOrgEnhanced, + getGiteaRepoInfo, + handleExistingNonMirrorRepo, + createOrganizationsSequentially +} from "./gitea-enhanced"; +import { mirrorGithubRepoToGitea } from "./gitea"; +import { getMirrorStrategyConfig } from "./utils/mirror-strategies"; +import { createMirrorJob } from "./helpers"; + +/** + * Process starred repositories with enhanced error handling + */ +export async function processStarredRepositories({ + config, + repositories, + octokit, +}: { + config: Config; + repositories: Repository[]; + octokit: Octokit; +}): Promise { + if (!config.userId) { + throw new Error("User ID is required"); + } + + const strategyConfig = getMirrorStrategyConfig(); + + console.log(`Processing ${repositories.length} starred repositories`); + console.log(`Using strategy config:`, strategyConfig); + + // Step 1: Pre-create organizations to avoid race conditions + if (strategyConfig.sequentialOrgCreation) { + await preCreateOrganizations({ config, repositories }); + } + + // Step 2: Process repositories with enhanced error handling + await processWithRetry( + repositories, + async (repository) => { + try { + await processStarredRepository({ + config, + repository, + octokit, + strategyConfig, + }); + return repository; + } catch (error) { + console.error(`Failed to process starred repository ${repository.name}:`, error); + throw error; + } + }, + { + concurrencyLimit: strategyConfig.repoBatchSize, + maxRetries: 2, + retryDelay: 2000, + onProgress: (completed, total, result) => { + const percentComplete = Math.round((completed / total) * 100); + if (result) { + console.log( + `Processed starred repository "${result.name}" (${completed}/${total}, ${percentComplete}%)` + ); + } + }, + onRetry: (repo, error, attempt) => { + console.log( + `Retrying starred repository ${repo.name} (attempt ${attempt}): ${error.message}` + ); + }, + } + ); +} + +/** + * Pre-create all required organizations sequentially + */ +async function preCreateOrganizations({ + config, + repositories, +}: { + config: Config; + repositories: Repository[]; +}): Promise { + // Get unique organization names + const orgNames = new Set(); + + // Add starred repos org + if (config.githubConfig?.starredReposOrg) { + orgNames.add(config.githubConfig.starredReposOrg); + } else { + orgNames.add("starred"); + } + + // Add any other organizations based on mirror strategy + for (const repo of repositories) { + if (repo.destinationOrg) { + orgNames.add(repo.destinationOrg); + } + } + + console.log(`Pre-creating ${orgNames.size} organizations sequentially`); + + // Create organizations sequentially + await createOrganizationsSequentially({ + config, + orgNames: Array.from(orgNames), + }); +} + +/** + * Process a single starred repository with enhanced error handling + */ +async function processStarredRepository({ + config, + repository, + octokit, + strategyConfig, +}: { + config: Config; + repository: Repository; + octokit: Octokit; + strategyConfig: ReturnType; +}): Promise { + const starredOrg = config.githubConfig?.starredReposOrg || "starred"; + + // Check if repository exists in Gitea + const existingRepo = await getGiteaRepoInfo({ + config, + owner: starredOrg, + repoName: repository.name, + }); + + if (existingRepo) { + if (existingRepo.mirror) { + console.log(`Starred repository ${repository.name} already exists as a mirror`); + + // Update database status + const { db, repositories: reposTable } = await import("./db"); + const { eq } = await import("drizzle-orm"); + const { repoStatusEnum } = await import("@/types/Repository"); + + await db + .update(reposTable) + .set({ + status: repoStatusEnum.parse("mirrored"), + updatedAt: new Date(), + lastMirrored: new Date(), + errorMessage: null, + mirroredLocation: `${starredOrg}/${repository.name}`, + }) + .where(eq(reposTable.id, repository.id!)); + + return; + } else { + // Repository exists but is not a mirror + console.warn(`Starred repository ${repository.name} exists but is not a mirror`); + + await handleExistingNonMirrorRepo({ + config, + repository, + repoInfo: existingRepo, + strategy: strategyConfig.nonMirrorStrategy, + }); + + // If we deleted it, continue to create the mirror + if (strategyConfig.nonMirrorStrategy !== "delete") { + return; // Skip if we're not deleting + } + } + } + + // Create the mirror + try { + await mirrorGithubRepoToGitea({ + octokit, + repository, + config, + }); + } catch (error) { + // Enhanced error handling for specific scenarios + if (error instanceof Error) { + const errorMessage = error.message.toLowerCase(); + + if (errorMessage.includes("already exists")) { + // Handle race condition where repo was created by another process + console.log(`Repository ${repository.name} was created by another process`); + + // Check if it's a mirror now + const recheck = await getGiteaRepoInfo({ + config, + owner: starredOrg, + repoName: repository.name, + }); + + if (recheck && recheck.mirror) { + // It's now a mirror, update database + const { db, repositories: reposTable } = await import("./db"); + const { eq } = await import("drizzle-orm"); + const { repoStatusEnum } = await import("@/types/Repository"); + + await db + .update(reposTable) + .set({ + status: repoStatusEnum.parse("mirrored"), + updatedAt: new Date(), + lastMirrored: new Date(), + errorMessage: null, + mirroredLocation: `${starredOrg}/${repository.name}`, + }) + .where(eq(reposTable.id, repository.id!)); + + return; + } + } + } + + throw error; + } +} + +/** + * Sync all starred repositories + */ +export async function syncStarredRepositories({ + config, + repositories, +}: { + config: Config; + repositories: Repository[]; +}): Promise { + const strategyConfig = getMirrorStrategyConfig(); + + console.log(`Syncing ${repositories.length} starred repositories`); + + await processWithRetry( + repositories, + async (repository) => { + try { + // Import syncGiteaRepo + const { syncGiteaRepo } = await import("./gitea"); + + await syncGiteaRepo({ + config, + repository, + }); + + return repository; + } catch (error) { + if (error instanceof Error && error.message.includes("not a mirror")) { + console.warn(`Repository ${repository.name} is not a mirror, handling...`); + + const starredOrg = config.githubConfig?.starredReposOrg || "starred"; + const repoInfo = await getGiteaRepoInfo({ + config, + owner: starredOrg, + repoName: repository.name, + }); + + if (repoInfo) { + await handleExistingNonMirrorRepo({ + config, + repository, + repoInfo, + strategy: strategyConfig.nonMirrorStrategy, + }); + } + } + + throw error; + } + }, + { + concurrencyLimit: strategyConfig.repoBatchSize, + maxRetries: 1, + retryDelay: 1000, + onProgress: (completed, total) => { + const percentComplete = Math.round((completed / total) * 100); + console.log(`Sync progress: ${completed}/${total} (${percentComplete}%)`); + }, + } + ); +} \ No newline at end of file diff --git a/src/lib/utils/mirror-strategies.ts b/src/lib/utils/mirror-strategies.ts new file mode 100644 index 0000000..4a525a2 --- /dev/null +++ b/src/lib/utils/mirror-strategies.ts @@ -0,0 +1,93 @@ +/** + * Mirror strategy configuration for handling various repository scenarios + */ + +export type NonMirrorStrategy = "skip" | "delete" | "rename" | "convert"; + +export interface MirrorStrategyConfig { + /** + * How to handle repositories that exist in Gitea but are not mirrors + * - "skip": Leave the repository as-is and mark as failed + * - "delete": Delete the repository and recreate as mirror + * - "rename": Rename the existing repository (not implemented yet) + * - "convert": Try to convert to mirror (not supported by most Gitea versions) + */ + nonMirrorStrategy: NonMirrorStrategy; + + /** + * Maximum retries for organization creation + */ + orgCreationRetries: number; + + /** + * Base delay in milliseconds for exponential backoff + */ + orgCreationRetryDelay: number; + + /** + * Whether to create organizations sequentially to avoid race conditions + */ + sequentialOrgCreation: boolean; + + /** + * Batch size for parallel repository processing + */ + repoBatchSize: number; + + /** + * Timeout for sync operations in milliseconds + */ + syncTimeout: number; +} + +export const DEFAULT_MIRROR_STRATEGY: MirrorStrategyConfig = { + nonMirrorStrategy: "delete", // Safe default: delete and recreate + orgCreationRetries: 3, + orgCreationRetryDelay: 100, + sequentialOrgCreation: true, + repoBatchSize: 3, + syncTimeout: 30000, // 30 seconds +}; + +/** + * Get mirror strategy configuration from environment or defaults + */ +export function getMirrorStrategyConfig(): MirrorStrategyConfig { + return { + nonMirrorStrategy: (process.env.NON_MIRROR_STRATEGY as NonMirrorStrategy) || DEFAULT_MIRROR_STRATEGY.nonMirrorStrategy, + orgCreationRetries: parseInt(process.env.ORG_CREATION_RETRIES || "") || DEFAULT_MIRROR_STRATEGY.orgCreationRetries, + orgCreationRetryDelay: parseInt(process.env.ORG_CREATION_RETRY_DELAY || "") || DEFAULT_MIRROR_STRATEGY.orgCreationRetryDelay, + sequentialOrgCreation: process.env.SEQUENTIAL_ORG_CREATION !== "false", + repoBatchSize: parseInt(process.env.REPO_BATCH_SIZE || "") || DEFAULT_MIRROR_STRATEGY.repoBatchSize, + syncTimeout: parseInt(process.env.SYNC_TIMEOUT || "") || DEFAULT_MIRROR_STRATEGY.syncTimeout, + }; +} + +/** + * Validate strategy configuration + */ +export function validateStrategyConfig(config: MirrorStrategyConfig): string[] { + const errors: string[] = []; + + if (!["skip", "delete", "rename", "convert"].includes(config.nonMirrorStrategy)) { + errors.push(`Invalid nonMirrorStrategy: ${config.nonMirrorStrategy}`); + } + + if (config.orgCreationRetries < 1 || config.orgCreationRetries > 10) { + errors.push("orgCreationRetries must be between 1 and 10"); + } + + if (config.orgCreationRetryDelay < 10 || config.orgCreationRetryDelay > 5000) { + errors.push("orgCreationRetryDelay must be between 10ms and 5000ms"); + } + + if (config.repoBatchSize < 1 || config.repoBatchSize > 50) { + errors.push("repoBatchSize must be between 1 and 50"); + } + + if (config.syncTimeout < 5000 || config.syncTimeout > 300000) { + errors.push("syncTimeout must be between 5s and 5min"); + } + + return errors; +} \ No newline at end of file