Compare commits

...

4 Commits

Author SHA1 Message Date
Arunavo Ray
62c97ca04c lib: harden metadata sync for existing repos 2026-03-14 17:57:54 +05:30
Arunavo Ray
dd87ddfa3d lib: sync repo topics and descriptions 2026-03-14 17:49:29 +05:30
ARUNAVO RAY
755647e29c scripts: add startup repair progress logs (#223) 2026-03-14 17:44:52 +05:30
dependabot[bot]
018c9d1a23 build(deps): bump devalue (#220) 2026-03-13 00:17:30 +05:30
3 changed files with 447 additions and 97 deletions

View File

@@ -15,33 +15,40 @@ import { repoStatusEnum } from "@/types/Repository";
const isDryRun = process.argv.includes("--dry-run");
const specificRepo = process.argv.find(arg => arg.startsWith("--repo-name="))?.split("=")[1];
const isStartupMode = process.argv.includes("--startup");
const requestTimeoutMs = parsePositiveInteger(process.env.GITEA_REPAIR_REQUEST_TIMEOUT_MS, 15000);
const progressInterval = parsePositiveInteger(process.env.GITEA_REPAIR_PROGRESS_INTERVAL, 100);
async function checkRepoInGitea(config: any, owner: string, repoName: string): Promise<boolean> {
try {
if (!config.giteaConfig?.url || !config.giteaConfig?.token) {
return false;
}
type GiteaLookupResult = {
exists: boolean;
details: any | null;
timedOut: boolean;
error: string | null;
};
const response = await fetch(
`${config.giteaConfig.url}/api/v1/repos/${owner}/${repoName}`,
{
headers: {
Authorization: `token ${config.giteaConfig.token}`,
},
}
);
return response.ok;
} catch (error) {
console.error(`Error checking repo ${owner}/${repoName} in Gitea:`, error);
return false;
function parsePositiveInteger(value: string | undefined, fallback: number): number {
const parsed = Number.parseInt(value ?? "", 10);
if (!Number.isFinite(parsed) || parsed <= 0) {
return fallback;
}
return parsed;
}
async function getRepoDetailsFromGitea(config: any, owner: string, repoName: string): Promise<any> {
function isTimeoutError(error: unknown): boolean {
if (!(error instanceof Error)) {
return false;
}
return error.name === "TimeoutError" || error.name === "AbortError";
}
async function getRepoDetailsFromGitea(config: any, owner: string, repoName: string): Promise<GiteaLookupResult> {
try {
if (!config.giteaConfig?.url || !config.giteaConfig?.token) {
return null;
return {
exists: false,
details: null,
timedOut: false,
error: "Missing Gitea URL or token in config",
};
}
const response = await fetch(
@@ -50,16 +57,41 @@ async function getRepoDetailsFromGitea(config: any, owner: string, repoName: str
headers: {
Authorization: `token ${config.giteaConfig.token}`,
},
signal: AbortSignal.timeout(requestTimeoutMs),
}
);
if (response.ok) {
return await response.json();
return {
exists: true,
details: await response.json(),
timedOut: false,
error: null,
};
}
return null;
if (response.status === 404) {
return {
exists: false,
details: null,
timedOut: false,
error: null,
};
}
return {
exists: false,
details: null,
timedOut: false,
error: `Gitea API returned HTTP ${response.status}`,
};
} catch (error) {
console.error(`Error getting repo details for ${owner}/${repoName}:`, error);
return null;
return {
exists: false,
details: null,
timedOut: isTimeoutError(error),
error: error instanceof Error ? error.message : String(error),
};
}
}
@@ -99,6 +131,8 @@ async function repairMirroredRepositories() {
.from(repositories)
.where(whereConditions);
const totalRepos = repos.length;
if (repos.length === 0) {
if (!isStartupMode) {
console.log("✅ No repositories found that need repair");
@@ -109,13 +143,25 @@ async function repairMirroredRepositories() {
if (!isStartupMode) {
console.log(`📋 Found ${repos.length} repositories to check:`);
console.log("");
} else {
console.log(`Checking ${totalRepos} repositories for status inconsistencies...`);
console.log(`Request timeout: ${requestTimeoutMs}ms | Progress interval: every ${progressInterval} repositories`);
}
const startedAt = Date.now();
const configCache = new Map<string, any>();
let checkedCount = 0;
let repairedCount = 0;
let skippedCount = 0;
let errorCount = 0;
let timeoutCount = 0;
let giteaErrorCount = 0;
let giteaErrorSamples = 0;
let startupSkipWarningCount = 0;
for (const repo of repos) {
checkedCount++;
if (!isStartupMode) {
console.log(`🔍 Checking repository: ${repo.name}`);
console.log(` Current status: ${repo.status}`);
@@ -124,13 +170,29 @@ async function repairMirroredRepositories() {
try {
// Get user configuration
const config = await db
.select()
.from(configs)
.where(eq(configs.id, repo.configId))
.limit(1);
const configKey = String(repo.configId);
let userConfig = configCache.get(configKey);
if (config.length === 0) {
if (!userConfig) {
const config = await db
.select()
.from(configs)
.where(eq(configs.id, repo.configId))
.limit(1);
if (config.length === 0) {
if (!isStartupMode) {
console.log(` ❌ No configuration found for repository`);
}
errorCount++;
continue;
}
userConfig = config[0];
configCache.set(configKey, userConfig);
}
if (!userConfig) {
if (!isStartupMode) {
console.log(` ❌ No configuration found for repository`);
}
@@ -138,7 +200,6 @@ async function repairMirroredRepositories() {
continue;
}
const userConfig = config[0];
const giteaUsername = userConfig.giteaConfig?.defaultOwner;
if (!giteaUsername) {
@@ -153,25 +214,59 @@ async function repairMirroredRepositories() {
let existsInGitea = false;
let actualOwner = giteaUsername;
let giteaRepoDetails = null;
let repoRequestTimedOut = false;
let repoRequestErrored = false;
// First check user location
existsInGitea = await checkRepoInGitea(userConfig, giteaUsername, repo.name);
if (existsInGitea) {
giteaRepoDetails = await getRepoDetailsFromGitea(userConfig, giteaUsername, repo.name);
const userLookup = await getRepoDetailsFromGitea(userConfig, giteaUsername, repo.name);
existsInGitea = userLookup.exists;
giteaRepoDetails = userLookup.details;
if (userLookup.timedOut) {
timeoutCount++;
repoRequestTimedOut = true;
} else if (userLookup.error) {
giteaErrorCount++;
repoRequestErrored = true;
if (!isStartupMode || giteaErrorSamples < 3) {
console.log(` ⚠️ Gitea lookup issue for ${giteaUsername}/${repo.name}: ${userLookup.error}`);
giteaErrorSamples++;
}
}
// If not found in user location and repo has organization, check organization
if (!existsInGitea && repo.organization) {
existsInGitea = await checkRepoInGitea(userConfig, repo.organization, repo.name);
const orgLookup = await getRepoDetailsFromGitea(userConfig, repo.organization, repo.name);
existsInGitea = orgLookup.exists;
if (existsInGitea) {
actualOwner = repo.organization;
giteaRepoDetails = await getRepoDetailsFromGitea(userConfig, repo.organization, repo.name);
giteaRepoDetails = orgLookup.details;
}
if (orgLookup.timedOut) {
timeoutCount++;
repoRequestTimedOut = true;
} else if (orgLookup.error) {
giteaErrorCount++;
repoRequestErrored = true;
if (!isStartupMode || giteaErrorSamples < 3) {
console.log(` ⚠️ Gitea lookup issue for ${repo.organization}/${repo.name}: ${orgLookup.error}`);
giteaErrorSamples++;
}
}
}
if (!existsInGitea) {
if (!isStartupMode) {
console.log(` ⏭️ Repository not found in Gitea - skipping`);
} else if (repoRequestTimedOut || repoRequestErrored) {
if (startupSkipWarningCount < 3) {
console.log(` ⚠️ Skipping ${repo.name}; Gitea was slow/unreachable during lookup`);
startupSkipWarningCount++;
if (startupSkipWarningCount === 3) {
console.log(` Additional slow/unreachable lookup warnings suppressed; progress logs will continue`);
}
}
}
skippedCount++;
continue;
@@ -241,22 +336,43 @@ async function repairMirroredRepositories() {
if (!isStartupMode) {
console.log("");
} else if (checkedCount % progressInterval === 0 || checkedCount === totalRepos) {
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1000);
console.log(
`Repair progress: ${checkedCount}/${totalRepos} checked | repaired=${repairedCount}, skipped=${skippedCount}, errors=${errorCount}, timeouts=${timeoutCount} | elapsed=${elapsedSeconds}s`
);
}
}
if (isStartupMode) {
// In startup mode, only log if there were repairs or errors
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1000);
console.log(
`Repository repair summary: checked=${checkedCount}, repaired=${repairedCount}, skipped=${skippedCount}, errors=${errorCount}, timeouts=${timeoutCount}, elapsed=${elapsedSeconds}s`
);
if (repairedCount > 0) {
console.log(`Repaired ${repairedCount} repository status inconsistencies`);
}
if (errorCount > 0) {
console.log(`Warning: ${errorCount} repositories had errors during repair`);
}
if (timeoutCount > 0) {
console.log(
`Warning: ${timeoutCount} Gitea API requests timed out. Increase GITEA_REPAIR_REQUEST_TIMEOUT_MS if your Gitea instance is under heavy load.`
);
}
if (giteaErrorCount > 0) {
console.log(`Warning: ${giteaErrorCount} Gitea API requests failed with non-timeout errors.`);
}
} else {
console.log("📊 Repair Summary:");
console.log(` Checked: ${checkedCount}`);
console.log(` Repaired: ${repairedCount}`);
console.log(` Skipped: ${skippedCount}`);
console.log(` Errors: ${errorCount}`);
console.log(` Timeouts: ${timeoutCount}`);
if (giteaErrorCount > 0) {
console.log(` Gitea API Errors: ${giteaErrorCount}`);
}
if (isDryRun && repairedCount > 0) {
console.log("");

View File

@@ -374,6 +374,161 @@ export const checkRepoLocation = async ({
return { present: false, actualOwner: expectedOwner };
};
const sanitizeTopicForGitea = (topic: string): string =>
topic
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]+/g, "-")
.replace(/-+/g, "-")
.replace(/^-+/, "")
.replace(/-+$/, "");
const normalizeTopicsForGitea = (
topics: string[],
topicPrefix?: string
): string[] => {
const normalizedPrefix = topicPrefix ? sanitizeTopicForGitea(topicPrefix) : "";
const transformedTopics = topics
.map((topic) => sanitizeTopicForGitea(topic))
.filter((topic) => topic.length > 0)
.map((topic) => (normalizedPrefix ? `${normalizedPrefix}-${topic}` : topic));
return [...new Set(transformedTopics)];
};
const getSourceRepositoryCoordinates = (repository: Repository) => {
const delimiterIndex = repository.fullName.indexOf("/");
if (
delimiterIndex > 0 &&
delimiterIndex < repository.fullName.length - 1
) {
return {
owner: repository.fullName.slice(0, delimiterIndex),
repo: repository.fullName.slice(delimiterIndex + 1),
};
}
return {
owner: repository.owner,
repo: repository.name,
};
};
const fetchGitHubTopics = async ({
octokit,
repository,
}: {
octokit: Octokit;
repository: Repository;
}): Promise<string[] | null> => {
const { owner, repo } = getSourceRepositoryCoordinates(repository);
try {
const response = await octokit.request("GET /repos/{owner}/{repo}/topics", {
owner,
repo,
headers: {
Accept: "application/vnd.github+json",
},
});
const names = (response.data as { names?: unknown }).names;
if (!Array.isArray(names)) {
console.warn(
`[Metadata] Unexpected topics payload for ${repository.fullName}; skipping topic sync.`
);
return null;
}
return names.filter((topic): topic is string => typeof topic === "string");
} catch (error) {
console.warn(
`[Metadata] Failed to fetch topics from GitHub for ${repository.fullName}: ${
error instanceof Error ? error.message : String(error)
}`
);
return null;
}
};
const syncRepositoryMetadataToGitea = async ({
config,
octokit,
repository,
giteaOwner,
giteaRepoName,
giteaToken,
}: {
config: Partial<Config>;
octokit: Octokit;
repository: Repository;
giteaOwner: string;
giteaRepoName: string;
giteaToken: string;
}): Promise<void> => {
const giteaBaseUrl = config.giteaConfig?.url;
if (!giteaBaseUrl) {
return;
}
const repoApiUrl = `${giteaBaseUrl}/api/v1/repos/${giteaOwner}/${giteaRepoName}`;
const authHeaders = {
Authorization: `token ${giteaToken}`,
};
const description = repository.description?.trim() || "";
try {
await httpPatch(
repoApiUrl,
{ description },
authHeaders
);
console.log(
`[Metadata] Synced description for ${repository.fullName} to ${giteaOwner}/${giteaRepoName}`
);
} catch (error) {
console.warn(
`[Metadata] Failed to sync description for ${repository.fullName} to ${giteaOwner}/${giteaRepoName}: ${
error instanceof Error ? error.message : String(error)
}`
);
}
if (config.giteaConfig?.addTopics === false) {
return;
}
const sourceTopics = await fetchGitHubTopics({ octokit, repository });
if (sourceTopics === null) {
console.warn(
`[Metadata] Skipping topic sync for ${repository.fullName} because GitHub topics could not be fetched.`
);
return;
}
const topics = normalizeTopicsForGitea(
sourceTopics,
config.giteaConfig?.topicPrefix
);
try {
await httpPut(
`${repoApiUrl}/topics`,
{ topics },
authHeaders
);
console.log(
`[Metadata] Synced ${topics.length} topic(s) for ${repository.fullName} to ${giteaOwner}/${giteaRepoName}`
);
} catch (error) {
console.warn(
`[Metadata] Failed to sync topics for ${repository.fullName} to ${giteaOwner}/${giteaRepoName}: ${
error instanceof Error ? error.message : String(error)
}`
);
}
};
export const mirrorGithubRepoToGitea = async ({
octokit,
repository,
@@ -465,36 +620,66 @@ export const mirrorGithubRepoToGitea = async ({
});
if (isExisting) {
console.log(
`Repository ${targetRepoName} already exists in Gitea under ${repoOwner}. Updating database status.`
);
// Update database to reflect that the repository is already mirrored
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("mirrored"),
updatedAt: new Date(),
lastMirrored: new Date(),
errorMessage: null,
mirroredLocation: `${repoOwner}/${targetRepoName}`,
})
.where(eq(repositories.id, repository.id!));
// Append log for "mirrored" status
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Repository ${repository.name} already exists in Gitea`,
details: `Repository ${repository.name} was found to already exist in Gitea under ${repoOwner} and database status was updated.`,
status: "mirrored",
const { getGiteaRepoInfo, handleExistingNonMirrorRepo } = await import("./gitea-enhanced");
const existingRepoInfo = await getGiteaRepoInfo({
config,
owner: repoOwner,
repoName: targetRepoName,
});
console.log(
`Repository ${repository.name} database status updated to mirrored`
);
return;
if (existingRepoInfo && !existingRepoInfo.mirror) {
console.log(`Repository ${targetRepoName} exists but is not a mirror. Handling...`);
await handleExistingNonMirrorRepo({
config,
repository,
repoInfo: existingRepoInfo,
strategy: "delete", // Can be configured: "skip", "delete", or "rename"
});
} else if (existingRepoInfo?.mirror) {
console.log(
`Repository ${targetRepoName} already exists in Gitea under ${repoOwner}. Updating database status.`
);
await syncRepositoryMetadataToGitea({
config,
octokit,
repository,
giteaOwner: repoOwner,
giteaRepoName: targetRepoName,
giteaToken: decryptedConfig.giteaConfig.token,
});
// Update database to reflect that the repository is already mirrored
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("mirrored"),
updatedAt: new Date(),
lastMirrored: new Date(),
errorMessage: null,
mirroredLocation: `${repoOwner}/${targetRepoName}`,
})
.where(eq(repositories.id, repository.id!));
// Append log for "mirrored" status
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Repository ${repository.name} already exists in Gitea`,
details: `Repository ${repository.name} was found to already exist in Gitea under ${repoOwner} and database status was updated.`,
status: "mirrored",
});
console.log(
`Repository ${repository.name} database status updated to mirrored`
);
return;
} else {
console.warn(
`[Mirror] Repository ${repoOwner}/${targetRepoName} exists but mirror status could not be verified. Continuing with mirror creation flow.`
);
}
}
console.log(`Mirroring repository ${repository.name}`);
@@ -648,6 +833,15 @@ export const mirrorGithubRepoToGitea = async ({
}
);
await syncRepositoryMetadataToGitea({
config,
octokit,
repository,
giteaOwner: repoOwner,
giteaRepoName: targetRepoName,
giteaToken: decryptedConfig.giteaConfig.token,
});
const metadataState = parseRepositoryMetadataState(repository.metadata);
let metadataUpdated = false;
const skipMetadataForStarred =
@@ -1094,36 +1288,66 @@ export async function mirrorGitHubRepoToGiteaOrg({
});
if (isExisting) {
console.log(
`Repository ${targetRepoName} already exists in Gitea organization ${orgName}. Updating database status.`
);
// Update database to reflect that the repository is already mirrored
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("mirrored"),
updatedAt: new Date(),
lastMirrored: new Date(),
errorMessage: null,
mirroredLocation: `${orgName}/${targetRepoName}`,
})
.where(eq(repositories.id, repository.id!));
// Create a mirror job log entry
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Repository ${targetRepoName} already exists in Gitea organization ${orgName}`,
details: `Repository ${targetRepoName} was found to already exist in Gitea organization ${orgName} and database status was updated.`,
status: "mirrored",
const { getGiteaRepoInfo, handleExistingNonMirrorRepo } = await import("./gitea-enhanced");
const existingRepoInfo = await getGiteaRepoInfo({
config,
owner: orgName,
repoName: targetRepoName,
});
console.log(
`Repository ${targetRepoName} database status updated to mirrored in organization ${orgName}`
);
return;
if (existingRepoInfo && !existingRepoInfo.mirror) {
console.log(`Repository ${targetRepoName} exists but is not a mirror. Handling...`);
await handleExistingNonMirrorRepo({
config,
repository,
repoInfo: existingRepoInfo,
strategy: "delete", // Can be configured: "skip", "delete", or "rename"
});
} else if (existingRepoInfo?.mirror) {
console.log(
`Repository ${targetRepoName} already exists in Gitea organization ${orgName}. Updating database status.`
);
await syncRepositoryMetadataToGitea({
config,
octokit,
repository,
giteaOwner: orgName,
giteaRepoName: targetRepoName,
giteaToken: decryptedConfig.giteaConfig.token,
});
// Update database to reflect that the repository is already mirrored
await db
.update(repositories)
.set({
status: repoStatusEnum.parse("mirrored"),
updatedAt: new Date(),
lastMirrored: new Date(),
errorMessage: null,
mirroredLocation: `${orgName}/${targetRepoName}`,
})
.where(eq(repositories.id, repository.id!));
// Create a mirror job log entry
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Repository ${targetRepoName} already exists in Gitea organization ${orgName}`,
details: `Repository ${targetRepoName} was found to already exist in Gitea organization ${orgName} and database status was updated.`,
status: "mirrored",
});
console.log(
`Repository ${targetRepoName} database status updated to mirrored in organization ${orgName}`
);
return;
} else {
console.warn(
`[Mirror] Repository ${orgName}/${targetRepoName} exists but mirror status could not be verified. Continuing with mirror creation flow.`
);
}
}
console.log(
@@ -1182,6 +1406,7 @@ export async function mirrorGitHubRepoToGiteaOrg({
wiki: shouldMirrorWiki || false,
lfs: config.giteaConfig?.lfs || false,
private: repository.isPrivate,
description: repository.description?.trim() || "",
};
// Add authentication for private repositories
@@ -1204,6 +1429,15 @@ export async function mirrorGitHubRepoToGiteaOrg({
}
);
await syncRepositoryMetadataToGitea({
config,
octokit,
repository,
giteaOwner: orgName,
giteaRepoName: targetRepoName,
giteaToken: decryptedConfig.giteaConfig.token,
});
const metadataState = parseRepositoryMetadataState(repository.metadata);
let metadataUpdated = false;
const skipMetadataForStarred =

8
www/pnpm-lock.yaml generated
View File

@@ -1216,8 +1216,8 @@ packages:
resolution: {integrity: sha512-KxektNH63SrbfUyDiwXqRb1rLwKt33AmMv+5Nhsw1kqZ13SJBRTgZHtGbE+hH3a1mVW1cz+4pqSWVPAtLVXTzQ==}
engines: {node: '>=18'}
devalue@5.6.3:
resolution: {integrity: sha512-nc7XjUU/2Lb+SvEFVGcWLiKkzfw8+qHI7zn8WYXKkLMgfGSHbgCEaR6bJpev8Cm6Rmrb19Gfd/tZvGqx9is3wg==}
devalue@5.6.4:
resolution: {integrity: sha512-Gp6rDldRsFh/7XuouDbxMH3Mx8GMCcgzIb1pDTvNyn8pZGQ22u+Wa+lGV9dQCltFQ7uVw0MhRyb8XDskNFOReA==}
devlop@1.1.0:
resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==}
@@ -3157,7 +3157,7 @@ snapshots:
cssesc: 3.0.0
debug: 4.4.3
deterministic-object-hash: 2.0.2
devalue: 5.6.3
devalue: 5.6.4
diff: 8.0.3
dlv: 1.1.3
dset: 3.1.4
@@ -3368,7 +3368,7 @@ snapshots:
dependencies:
base-64: 1.0.0
devalue@5.6.3: {}
devalue@5.6.4: {}
devlop@1.1.0:
dependencies: