Files
gitea-mirror/src/lib/gitea-enhanced.ts
2025-10-22 09:33:13 +05:30

575 lines
19 KiB
TypeScript

/**
* Enhanced Gitea operations with better error handling for starred repositories
* This module provides fixes for:
* 1. "Repository is not a mirror" errors
* 2. Duplicate organization constraint errors
* 3. Race conditions in parallel processing
*/
import type { Config } from "@/types/config";
import type { Repository } from "./db/schema";
import { Octokit } from "@octokit/rest";
import { createMirrorJob } from "./helpers";
import { decryptConfigTokens } from "./utils/config-encryption";
import { httpPost, httpGet, httpPatch, HttpError } from "./http-client";
import { db, repositories } from "./db";
import { eq } from "drizzle-orm";
import { repoStatusEnum } from "@/types/Repository";
type SyncDependencies = {
getGiteaRepoOwnerAsync: typeof import("./gitea")["getGiteaRepoOwnerAsync"];
mirrorGitHubReleasesToGitea: typeof import("./gitea")["mirrorGitHubReleasesToGitea"];
};
/**
* Enhanced repository information including mirror status
*/
interface GiteaRepoInfo {
id: number;
name: string;
owner: { login: string } | string;
mirror: boolean;
mirror_interval?: string;
clone_url?: string;
private: boolean;
}
/**
* Check if a repository exists in Gitea and return its details
*/
export async function getGiteaRepoInfo({
config,
owner,
repoName,
}: {
config: Partial<Config>;
owner: string;
repoName: string;
}): Promise<GiteaRepoInfo | null> {
try {
if (!config.giteaConfig?.url || !config.giteaConfig?.token) {
throw new Error("Gitea config is required.");
}
const decryptedConfig = decryptConfigTokens(config as Config);
const response = await httpGet<GiteaRepoInfo>(
`${config.giteaConfig.url}/api/v1/repos/${owner}/${repoName}`,
{
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
}
);
return response.data;
} catch (error) {
if (error instanceof HttpError && error.status === 404) {
return null; // Repository doesn't exist
}
throw error;
}
}
/**
* Enhanced organization creation with better error handling and retry logic
*/
export async function getOrCreateGiteaOrgEnhanced({
orgName,
orgId,
config,
maxRetries = 3,
retryDelay = 100,
}: {
orgId?: string;
orgName: string;
config: Partial<Config>;
maxRetries?: number;
retryDelay?: number;
}): Promise<number> {
if (!config.giteaConfig?.url || !config.giteaConfig?.token || !config.userId) {
throw new Error("Gitea config is required.");
}
const decryptedConfig = decryptConfigTokens(config as Config);
// First, validate the user's authentication by getting their information
console.log(`[Org Creation] Validating user authentication before organization operations`);
try {
const userResponse = await httpGet(
`${config.giteaConfig.url}/api/v1/user`,
{
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
}
);
console.log(`[Org Creation] Authenticated as user: ${userResponse.data.username || userResponse.data.login} (ID: ${userResponse.data.id})`);
} catch (authError) {
if (authError instanceof HttpError && authError.status === 401) {
console.error(`[Org Creation] Authentication failed: Invalid or expired token`);
throw new Error(`Authentication failed: Please check your Gitea token has the required permissions. The token may be invalid or expired.`);
}
console.error(`[Org Creation] Failed to validate authentication:`, authError);
throw new Error(`Failed to validate Gitea authentication: ${authError instanceof Error ? authError.message : String(authError)}`);
}
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
console.log(`[Org Creation] Attempting to get or create organization: ${orgName} (attempt ${attempt + 1}/${maxRetries})`);
// Check if org exists
try {
const orgResponse = await httpGet<{ id: number }>(
`${config.giteaConfig.url}/api/v1/orgs/${orgName}`,
{
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
}
);
console.log(`[Org Creation] Organization ${orgName} already exists with ID: ${orgResponse.data.id}`);
return orgResponse.data.id;
} catch (error) {
if (!(error instanceof HttpError) || error.status !== 404) {
throw error; // Unexpected error
}
// Organization doesn't exist, continue to create it
}
// Try to create the organization
console.log(`[Org Creation] Organization ${orgName} not found. Creating new organization.`);
const visibility = config.giteaConfig.visibility || "public";
const createOrgPayload = {
username: orgName,
full_name: orgName === "starred" ? "Starred Repositories" : orgName,
description: orgName === "starred"
? "Repositories starred on GitHub"
: `Mirrored from GitHub organization: ${orgName}`,
website: "",
location: "",
visibility: visibility,
};
try {
const createResponse = await httpPost<{ id: number }>(
`${config.giteaConfig.url}/api/v1/orgs`,
createOrgPayload,
{
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
}
);
console.log(`[Org Creation] Successfully created organization ${orgName} with ID: ${createResponse.data.id}`);
await createMirrorJob({
userId: config.userId,
organizationId: orgId,
organizationName: orgName,
message: `Successfully created Gitea organization: ${orgName}`,
status: "synced",
details: `Organization ${orgName} was created in Gitea with ID ${createResponse.data.id}.`,
});
return createResponse.data.id;
} catch (createError) {
// Check if it's a duplicate error
if (createError instanceof HttpError) {
const errorResponse = createError.response?.toLowerCase() || "";
const isDuplicateError =
errorResponse.includes("duplicate") ||
errorResponse.includes("already exists") ||
errorResponse.includes("uqe_user_lower_name") ||
errorResponse.includes("constraint");
if (isDuplicateError && attempt < maxRetries - 1) {
console.log(`[Org Creation] Organization creation failed due to duplicate. Will retry check.`);
// Wait before retry with exponential backoff
const delay = process.env.NODE_ENV === 'test' ? 0 : retryDelay * Math.pow(2, attempt);
console.log(`[Org Creation] Waiting ${delay}ms before retry...`);
if (delay > 0) {
await new Promise(resolve => setTimeout(resolve, delay));
}
continue; // Retry the loop
}
// Check for permission errors
if (createError.status === 403) {
console.error(`[Org Creation] Permission denied: User may not have rights to create organizations`);
throw new Error(`Permission denied: Your Gitea user account does not have permission to create organizations. Please ensure your account has the necessary privileges or contact your Gitea administrator.`);
}
// Check for authentication errors
if (createError.status === 401) {
console.error(`[Org Creation] Authentication failed when creating organization`);
throw new Error(`Authentication failed: The Gitea token does not have sufficient permissions to create organizations. Please ensure your token has 'write:organization' scope.`);
}
}
throw createError;
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : "Unknown error";
if (attempt === maxRetries - 1) {
// Final attempt failed
console.error(`[Org Creation] Failed to get or create organization ${orgName} after ${maxRetries} attempts: ${errorMessage}`);
await createMirrorJob({
userId: config.userId,
organizationId: orgId,
organizationName: orgName,
message: `Failed to create or fetch Gitea organization: ${orgName}`,
status: "failed",
details: `Error after ${maxRetries} attempts: ${errorMessage}`,
});
throw new Error(`Failed to create organization ${orgName}: ${errorMessage}`);
}
// Log retry attempt
console.warn(`[Org Creation] Attempt ${attempt + 1} failed for organization ${orgName}: ${errorMessage}. Retrying...`);
// Wait before retry
const delay = retryDelay * Math.pow(2, attempt);
await new Promise(resolve => setTimeout(resolve, delay));
}
}
// Should never reach here
throw new Error(`Failed to create organization ${orgName} after ${maxRetries} attempts`);
}
/**
* Enhanced sync operation that handles non-mirror repositories
*/
export async function syncGiteaRepoEnhanced({
config,
repository,
}: {
config: Partial<Config>;
repository: Repository;
}, deps?: SyncDependencies): Promise<any> {
try {
if (!config.userId || !config.giteaConfig?.url || !config.giteaConfig?.token) {
throw new Error("Gitea config is required.");
}
const decryptedConfig = decryptConfigTokens(config as Config);
console.log(`[Sync] Starting sync for repository ${repository.name}`);
// Mark repo as "syncing" in DB
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("syncing"),
updatedAt: new Date(),
})
.where(eq(repositories.id, repository.id!));
// Get the expected owner
const dependencies = deps ?? (await import("./gitea"));
const repoOwner = await dependencies.getGiteaRepoOwnerAsync({ config, repository });
// Check if repo exists and get its info
const repoInfo = await getGiteaRepoInfo({
config,
owner: repoOwner,
repoName: repository.name,
});
if (!repoInfo) {
throw new Error(`Repository ${repository.name} not found in Gitea at ${repoOwner}/${repository.name}`);
}
// Check if it's a mirror repository
if (!repoInfo.mirror) {
console.warn(`[Sync] Repository ${repository.name} exists but is not configured as a mirror`);
// Update database to reflect this status
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("failed"),
updatedAt: new Date(),
errorMessage: "Repository exists in Gitea but is not configured as a mirror. Manual intervention required.",
})
.where(eq(repositories.id, repository.id!));
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Cannot sync ${repository.name}: Not a mirror repository`,
details: `Repository ${repository.name} exists in Gitea but is not configured as a mirror. You may need to delete and recreate it as a mirror, or manually configure it as a mirror in Gitea.`,
status: "failed",
});
throw new Error(`Repository ${repository.name} is not a mirror. Cannot sync.`);
}
// Update mirror interval if needed
if (config.giteaConfig?.mirrorInterval) {
try {
console.log(`[Sync] Updating mirror interval for ${repository.name} to ${config.giteaConfig.mirrorInterval}`);
const updateUrl = `${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repository.name}`;
await httpPatch(updateUrl, {
mirror_interval: config.giteaConfig.mirrorInterval,
}, {
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
});
console.log(`[Sync] Successfully updated mirror interval for ${repository.name}`);
} catch (updateError) {
console.warn(`[Sync] Failed to update mirror interval for ${repository.name}:`, updateError);
// Continue with sync even if interval update fails
}
}
// Perform the sync
const apiUrl = `${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repository.name}/mirror-sync`;
try {
const response = await httpPost(apiUrl, undefined, {
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
});
const shouldMirrorReleases =
decryptedConfig.giteaConfig?.mirrorReleases &&
!(repository.isStarred && decryptedConfig.githubConfig?.starredCodeOnly);
if (shouldMirrorReleases) {
if (!decryptedConfig.githubConfig?.token) {
console.warn(
`[Sync] Skipping release mirroring for ${repository.name}: Missing GitHub token`
);
} else {
try {
const octokit = new Octokit({ auth: decryptedConfig.githubConfig.token });
await dependencies.mirrorGitHubReleasesToGitea({
config: decryptedConfig,
octokit,
repository,
giteaOwner: repoOwner,
giteaRepoName: repository.name,
});
console.log(`[Sync] Mirrored releases for ${repository.name} after sync`);
} catch (releaseError) {
console.error(
`[Sync] Failed to mirror releases for ${repository.name}: ${
releaseError instanceof Error ? releaseError.message : String(releaseError)
}`
);
}
}
}
// Mark repo as "synced" in DB
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("synced"),
updatedAt: new Date(),
lastMirrored: new Date(),
errorMessage: null,
mirroredLocation: `${repoOwner}/${repository.name}`,
})
.where(eq(repositories.id, repository.id!));
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Successfully synced repository: ${repository.name}`,
details: `Repository ${repository.name} was synced with Gitea.`,
status: "synced",
});
console.log(`[Sync] Repository ${repository.name} synced successfully`);
return response.data;
} catch (syncError) {
if (syncError instanceof HttpError && syncError.status === 400) {
// Handle specific mirror-sync errors
const errorMessage = syncError.response?.toLowerCase() || "";
if (errorMessage.includes("not a mirror")) {
// Update status to indicate this specific error
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("failed"),
updatedAt: new Date(),
errorMessage: "Repository is not configured as a mirror in Gitea",
})
.where(eq(repositories.id, repository.id!));
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Sync failed: ${repository.name} is not a mirror`,
details: "The repository exists in Gitea but is not configured as a mirror. Manual intervention required.",
status: "failed",
});
}
}
throw syncError;
}
} catch (error) {
console.error(`[Sync] Error while syncing repository ${repository.name}:`, error);
// Update repo with error status
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("failed"),
updatedAt: new Date(),
errorMessage: error instanceof Error ? error.message : "Unknown error",
})
.where(eq(repositories.id, repository.id!));
if (config.userId && repository.id && repository.name) {
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Failed to sync repository: ${repository.name}`,
details: error instanceof Error ? error.message : "Unknown error",
status: "failed",
});
}
throw error;
}
}
/**
* Delete a repository in Gitea (useful for cleaning up non-mirror repos)
*/
export async function deleteGiteaRepo({
config,
owner,
repoName,
}: {
config: Partial<Config>;
owner: string;
repoName: string;
}): Promise<void> {
if (!config.giteaConfig?.url || !config.giteaConfig?.token) {
throw new Error("Gitea config is required.");
}
const decryptedConfig = decryptConfigTokens(config as Config);
const response = await fetch(
`${config.giteaConfig.url}/api/v1/repos/${owner}/${repoName}`,
{
method: "DELETE",
headers: {
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
},
}
);
if (!response.ok && response.status !== 404) {
throw new Error(`Failed to delete repository: ${response.statusText}`);
}
}
/**
* Convert a regular repository to a mirror (if supported by Gitea version)
* Note: This might not be supported in all Gitea versions
*/
export async function convertToMirror({
config,
owner,
repoName,
cloneUrl,
}: {
config: Partial<Config>;
owner: string;
repoName: string;
cloneUrl: string;
}): Promise<boolean> {
// This is a placeholder - actual implementation depends on Gitea API support
// Most Gitea versions don't support converting existing repos to mirrors
console.warn(`[Convert] Converting existing repositories to mirrors is not supported in most Gitea versions`);
return false;
}
/**
* Sequential organization creation to avoid race conditions
*/
export async function createOrganizationsSequentially({
config,
orgNames,
}: {
config: Partial<Config>;
orgNames: string[];
}): Promise<Map<string, number>> {
const orgIdMap = new Map<string, number>();
for (const orgName of orgNames) {
try {
const orgId = await getOrCreateGiteaOrgEnhanced({
orgName,
config,
maxRetries: 3,
retryDelay: 100,
});
orgIdMap.set(orgName, orgId);
} catch (error) {
console.error(`Failed to create organization ${orgName}:`, error);
// Continue with other organizations
}
}
return orgIdMap;
}
/**
* Check and handle existing non-mirror repositories
*/
export async function handleExistingNonMirrorRepo({
config,
repository,
repoInfo,
strategy = "skip",
}: {
config: Partial<Config>;
repository: Repository;
repoInfo: GiteaRepoInfo;
strategy?: "skip" | "delete" | "rename";
}): Promise<void> {
const owner = typeof repoInfo.owner === 'string' ? repoInfo.owner : repoInfo.owner.login;
const repoName = repoInfo.name;
switch (strategy) {
case "skip":
console.log(`[Handle] Skipping existing non-mirror repository: ${owner}/${repoName}`);
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("failed"),
updatedAt: new Date(),
errorMessage: "Repository exists but is not a mirror. Skipped.",
})
.where(eq(repositories.id, repository.id!));
break;
case "delete":
console.log(`[Handle] Deleting existing non-mirror repository: ${owner}/${repoName}`);
await deleteGiteaRepo({
config,
owner,
repoName,
});
console.log(`[Handle] Deleted repository ${owner}/${repoName}. It can now be recreated as a mirror.`);
break;
case "rename":
console.log(`[Handle] Renaming strategy not implemented yet for: ${owner}/${repoName}`);
// TODO: Implement rename strategy if needed
break;
}
}