mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2025-12-06 19:46:44 +03:00
feat: add support for mirroring wiki pages in configuration
This commit is contained in:
Arunavo Ray
@@ -19,6 +19,7 @@ JWT_SECRET=change-this-to-a-secure-random-string-in-production
|
||||
# SKIP_FORKS=false
|
||||
# PRIVATE_REPOSITORIES=false
|
||||
# MIRROR_ISSUES=false
|
||||
# MIRROR_WIKI=false
|
||||
# MIRROR_STARRED=false
|
||||
# MIRROR_ORGANIZATIONS=false
|
||||
# PRESERVE_ORG_STRUCTURE=false
|
||||
|
||||
@@ -63,6 +63,7 @@ services:
|
||||
- SKIP_FORKS=${SKIP_FORKS:-false}
|
||||
- PRIVATE_REPOSITORIES=${PRIVATE_REPOSITORIES:-false}
|
||||
- MIRROR_ISSUES=${MIRROR_ISSUES:-false}
|
||||
- MIRROR_WIKI=${MIRROR_WIKI:-false}
|
||||
- MIRROR_STARRED=${MIRROR_STARRED:-false}
|
||||
- MIRROR_ORGANIZATIONS=${MIRROR_ORGANIZATIONS:-false}
|
||||
- PRESERVE_ORG_STRUCTURE=${PRESERVE_ORG_STRUCTURE:-false}
|
||||
|
||||
@@ -30,6 +30,7 @@ services:
|
||||
- SKIP_FORKS=${SKIP_FORKS:-false}
|
||||
- PRIVATE_REPOSITORIES=${PRIVATE_REPOSITORIES:-false}
|
||||
- MIRROR_ISSUES=${MIRROR_ISSUES:-false}
|
||||
- MIRROR_WIKI=${MIRROR_WIKI:-false}
|
||||
- MIRROR_STARRED=${MIRROR_STARRED:-false}
|
||||
- MIRROR_ORGANIZATIONS=${MIRROR_ORGANIZATIONS:-false}
|
||||
- PRESERVE_ORG_STRUCTURE=${PRESERVE_ORG_STRUCTURE:-false}
|
||||
|
||||
@@ -35,6 +35,7 @@ export function ConfigTabs() {
|
||||
skipForks: false,
|
||||
privateRepositories: false,
|
||||
mirrorIssues: false,
|
||||
mirrorWiki: false,
|
||||
mirrorStarred: false,
|
||||
preserveOrgStructure: false,
|
||||
skipStarredIssues: false,
|
||||
|
||||
@@ -240,6 +240,30 @@ export function GitHubConfigForm({ config, setConfig, onAutoSave, isAutoSaving }
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center">
|
||||
<Checkbox
|
||||
id="mirror-wiki"
|
||||
name="mirrorWiki"
|
||||
checked={config.mirrorWiki}
|
||||
onCheckedChange={(checked) =>
|
||||
handleChange({
|
||||
target: {
|
||||
name: "mirrorWiki",
|
||||
type: "checkbox",
|
||||
checked: Boolean(checked),
|
||||
value: "",
|
||||
},
|
||||
} as React.ChangeEvent<HTMLInputElement>)
|
||||
}
|
||||
/>
|
||||
<label
|
||||
htmlFor="mirror-wiki"
|
||||
className="ml-2 block text-sm select-none"
|
||||
>
|
||||
Mirror Wiki
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center">
|
||||
<Checkbox
|
||||
id="preserve-org-structure"
|
||||
|
||||
@@ -55,6 +55,7 @@ The GitHub configuration section allows you to connect to GitHub and specify whi
|
||||
| Skip Forks | Skip forked repositories | `false` |
|
||||
| Private Repositories | Include private repositories | `false` |
|
||||
| Mirror Issues | Mirror issues from GitHub to Gitea | `false` |
|
||||
| Mirror Wiki | Mirror wiki pages from GitHub to Gitea | `false` |
|
||||
| Mirror Starred | Mirror starred repositories | `false` |
|
||||
| Mirror Organizations | Mirror organization repositories | `false` |
|
||||
| Only Mirror Orgs | Only mirror organization repositories | `false` |
|
||||
|
||||
@@ -26,6 +26,7 @@ export const configSchema = z.object({
|
||||
skipForks: z.boolean().default(false),
|
||||
privateRepositories: z.boolean().default(false),
|
||||
mirrorIssues: z.boolean().default(false),
|
||||
mirrorWiki: z.boolean().default(false),
|
||||
mirrorStarred: z.boolean().default(false),
|
||||
useSpecificUser: z.boolean().default(false),
|
||||
singleRepo: z.string().optional(),
|
||||
|
||||
271
src/lib/gitea.ts
271
src/lib/gitea.ts
@@ -80,8 +80,11 @@ export const checkRepoLocation = async ({
|
||||
expectedOwner: string;
|
||||
}): Promise<{ present: boolean; actualOwner: string }> => {
|
||||
// First check if we have a recorded mirroredLocation and if the repo exists there
|
||||
if (repository.mirroredLocation && repository.mirroredLocation.trim() !== "") {
|
||||
const [mirroredOwner] = repository.mirroredLocation.split('/');
|
||||
if (
|
||||
repository.mirroredLocation &&
|
||||
repository.mirroredLocation.trim() !== ""
|
||||
) {
|
||||
const [mirroredOwner] = repository.mirroredLocation.split("/");
|
||||
if (mirroredOwner) {
|
||||
const mirroredPresent = await isRepoPresentInGitea({
|
||||
config,
|
||||
@@ -90,7 +93,9 @@ export const checkRepoLocation = async ({
|
||||
});
|
||||
|
||||
if (mirroredPresent) {
|
||||
console.log(`Repository found at recorded mirrored location: ${repository.mirroredLocation}`);
|
||||
console.log(
|
||||
`Repository found at recorded mirrored location: ${repository.mirroredLocation}`
|
||||
);
|
||||
return { present: true, actualOwner: mirroredOwner };
|
||||
}
|
||||
}
|
||||
@@ -162,7 +167,9 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
status: "mirrored",
|
||||
});
|
||||
|
||||
console.log(`Repository ${repository.name} database status updated to mirrored`);
|
||||
console.log(
|
||||
`Repository ${repository.name} database status updated to mirrored`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -205,16 +212,28 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
|
||||
const apiUrl = `${config.giteaConfig.url}/api/v1/repos/migrate`;
|
||||
|
||||
const response = await httpPost(apiUrl, {
|
||||
clone_addr: cloneAddress,
|
||||
repo_name: repository.name,
|
||||
mirror: true,
|
||||
private: repository.isPrivate,
|
||||
repo_owner: config.giteaConfig.username,
|
||||
description: "",
|
||||
service: "git",
|
||||
}, {
|
||||
"Authorization": `token ${config.giteaConfig.token}`,
|
||||
const response = await httpPost(
|
||||
apiUrl,
|
||||
{
|
||||
clone_addr: cloneAddress,
|
||||
repo_name: repository.name,
|
||||
mirror: true,
|
||||
wiki: config.githubConfig.mirrorWiki || false, // will mirror wiki if it exists
|
||||
private: repository.isPrivate,
|
||||
repo_owner: config.giteaConfig.username,
|
||||
description: "",
|
||||
service: "git",
|
||||
},
|
||||
{
|
||||
Authorization: `token ${config.giteaConfig.token}`,
|
||||
}
|
||||
);
|
||||
|
||||
//mirror releases
|
||||
await mirrorGitHubReleasesToGitea({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
});
|
||||
|
||||
// clone issues
|
||||
@@ -317,16 +336,22 @@ export async function getOrCreateGiteaOrg({
|
||||
}
|
||||
);
|
||||
|
||||
console.log(`Get org response status: ${orgRes.status} for org: ${orgName}`);
|
||||
console.log(
|
||||
`Get org response status: ${orgRes.status} for org: ${orgName}`
|
||||
);
|
||||
|
||||
if (orgRes.ok) {
|
||||
// Check if response is actually JSON
|
||||
const contentType = orgRes.headers.get("content-type");
|
||||
if (!contentType || !contentType.includes("application/json")) {
|
||||
console.warn(`Expected JSON response but got content-type: ${contentType}`);
|
||||
console.warn(
|
||||
`Expected JSON response but got content-type: ${contentType}`
|
||||
);
|
||||
const responseText = await orgRes.text();
|
||||
console.warn(`Response body: ${responseText}`);
|
||||
throw new Error(`Invalid response format from Gitea API. Expected JSON but got: ${contentType}`);
|
||||
throw new Error(
|
||||
`Invalid response format from Gitea API. Expected JSON but got: ${contentType}`
|
||||
);
|
||||
}
|
||||
|
||||
// Clone the response to handle potential JSON parsing errors
|
||||
@@ -334,14 +359,22 @@ export async function getOrCreateGiteaOrg({
|
||||
|
||||
try {
|
||||
const org = await orgRes.json();
|
||||
console.log(`Successfully retrieved existing org: ${orgName} with ID: ${org.id}`);
|
||||
console.log(
|
||||
`Successfully retrieved existing org: ${orgName} with ID: ${org.id}`
|
||||
);
|
||||
// Note: Organization events are handled by the main mirroring process
|
||||
// to avoid duplicate events
|
||||
return org.id;
|
||||
} catch (jsonError) {
|
||||
const responseText = await orgResClone.text();
|
||||
console.error(`Failed to parse JSON response for existing org: ${responseText}`);
|
||||
throw new Error(`Failed to parse JSON response from Gitea API: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`);
|
||||
console.error(
|
||||
`Failed to parse JSON response for existing org: ${responseText}`
|
||||
);
|
||||
throw new Error(
|
||||
`Failed to parse JSON response from Gitea API: ${
|
||||
jsonError instanceof Error ? jsonError.message : String(jsonError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,21 +394,29 @@ export async function getOrCreateGiteaOrg({
|
||||
}),
|
||||
});
|
||||
|
||||
console.log(`Create org response status: ${createRes.status} for org: ${orgName}`);
|
||||
console.log(
|
||||
`Create org response status: ${createRes.status} for org: ${orgName}`
|
||||
);
|
||||
|
||||
if (!createRes.ok) {
|
||||
const errorText = await createRes.text();
|
||||
console.error(`Failed to create org ${orgName}. Status: ${createRes.status}, Response: ${errorText}`);
|
||||
console.error(
|
||||
`Failed to create org ${orgName}. Status: ${createRes.status}, Response: ${errorText}`
|
||||
);
|
||||
throw new Error(`Failed to create Gitea org: ${errorText}`);
|
||||
}
|
||||
|
||||
// Check if response is actually JSON
|
||||
const createContentType = createRes.headers.get("content-type");
|
||||
if (!createContentType || !createContentType.includes("application/json")) {
|
||||
console.warn(`Expected JSON response but got content-type: ${createContentType}`);
|
||||
console.warn(
|
||||
`Expected JSON response but got content-type: ${createContentType}`
|
||||
);
|
||||
const responseText = await createRes.text();
|
||||
console.warn(`Response body: ${responseText}`);
|
||||
throw new Error(`Invalid response format from Gitea API. Expected JSON but got: ${createContentType}`);
|
||||
throw new Error(
|
||||
`Invalid response format from Gitea API. Expected JSON but got: ${createContentType}`
|
||||
);
|
||||
}
|
||||
|
||||
// Note: Organization creation events are handled by the main mirroring process
|
||||
@@ -386,12 +427,20 @@ export async function getOrCreateGiteaOrg({
|
||||
|
||||
try {
|
||||
const newOrg = await createRes.json();
|
||||
console.log(`Successfully created new org: ${orgName} with ID: ${newOrg.id}`);
|
||||
console.log(
|
||||
`Successfully created new org: ${orgName} with ID: ${newOrg.id}`
|
||||
);
|
||||
return newOrg.id;
|
||||
} catch (jsonError) {
|
||||
const responseText = await createResClone.text();
|
||||
console.error(`Failed to parse JSON response for new org: ${responseText}`);
|
||||
throw new Error(`Failed to parse JSON response from Gitea API: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`);
|
||||
console.error(
|
||||
`Failed to parse JSON response for new org: ${responseText}`
|
||||
);
|
||||
throw new Error(
|
||||
`Failed to parse JSON response from Gitea API: ${
|
||||
jsonError instanceof Error ? jsonError.message : String(jsonError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
@@ -399,7 +448,9 @@ export async function getOrCreateGiteaOrg({
|
||||
? error.message
|
||||
: "Unknown error occurred in getOrCreateGiteaOrg.";
|
||||
|
||||
console.error(`Error in getOrCreateGiteaOrg for ${orgName}: ${errorMessage}`);
|
||||
console.error(
|
||||
`Error in getOrCreateGiteaOrg for ${orgName}: ${errorMessage}`
|
||||
);
|
||||
|
||||
await createMirrorJob({
|
||||
userId: config.userId,
|
||||
@@ -469,7 +520,9 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
status: "mirrored",
|
||||
});
|
||||
|
||||
console.log(`Repository ${repository.name} database status updated to mirrored in organization ${orgName}`);
|
||||
console.log(
|
||||
`Repository ${repository.name} database status updated to mirrored in organization ${orgName}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -506,14 +559,26 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
|
||||
const apiUrl = `${config.giteaConfig.url}/api/v1/repos/migrate`;
|
||||
|
||||
const migrateRes = await httpPost(apiUrl, {
|
||||
clone_addr: cloneAddress,
|
||||
uid: giteaOrgId,
|
||||
repo_name: repository.name,
|
||||
mirror: true,
|
||||
private: repository.isPrivate,
|
||||
}, {
|
||||
"Authorization": `token ${config.giteaConfig.token}`,
|
||||
const migrateRes = await httpPost(
|
||||
apiUrl,
|
||||
{
|
||||
clone_addr: cloneAddress,
|
||||
uid: giteaOrgId,
|
||||
repo_name: repository.name,
|
||||
mirror: true,
|
||||
wiki: config.githubConfig?.mirrorWiki || false, // will mirror wiki if it exists
|
||||
private: repository.isPrivate,
|
||||
},
|
||||
{
|
||||
Authorization: `token ${config.giteaConfig.token}`,
|
||||
}
|
||||
);
|
||||
|
||||
//mirror releases
|
||||
await mirrorGitHubReleasesToGitea({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
});
|
||||
|
||||
// Clone issues
|
||||
@@ -677,9 +742,13 @@ export async function mirrorGitHubOrgToGitea({
|
||||
.where(eq(repositories.organization, organization.name));
|
||||
|
||||
if (orgRepos.length === 0) {
|
||||
console.log(`No repositories found for organization ${organization.name} - marking as successfully mirrored`);
|
||||
console.log(
|
||||
`No repositories found for organization ${organization.name} - marking as successfully mirrored`
|
||||
);
|
||||
} else {
|
||||
console.log(`Mirroring ${orgRepos.length} repositories for organization ${organization.name}`);
|
||||
console.log(
|
||||
`Mirroring ${orgRepos.length} repositories for organization ${organization.name}`
|
||||
);
|
||||
|
||||
// Import the processWithRetry function
|
||||
const { processWithRetry } = await import("@/lib/utils/concurrency");
|
||||
@@ -701,7 +770,9 @@ export async function mirrorGitHubOrgToGitea({
|
||||
};
|
||||
|
||||
// Log the start of mirroring
|
||||
console.log(`Starting mirror for repository: ${repo.name} in organization ${organization.name}`);
|
||||
console.log(
|
||||
`Starting mirror for repository: ${repo.name} in organization ${organization.name}`
|
||||
);
|
||||
|
||||
// Mirror the repository
|
||||
await mirrorGitHubRepoToGiteaOrg({
|
||||
@@ -721,12 +792,16 @@ export async function mirrorGitHubOrgToGitea({
|
||||
onProgress: (completed, total, result) => {
|
||||
const percentComplete = Math.round((completed / total) * 100);
|
||||
if (result) {
|
||||
console.log(`Mirrored repository "${result.name}" in organization ${organization.name} (${completed}/${total}, ${percentComplete}%)`);
|
||||
console.log(
|
||||
`Mirrored repository "${result.name}" in organization ${organization.name} (${completed}/${total}, ${percentComplete}%)`
|
||||
);
|
||||
}
|
||||
},
|
||||
onRetry: (repo, error, attempt) => {
|
||||
console.log(`Retrying repository ${repo.name} in organization ${organization.name} (attempt ${attempt}): ${error.message}`);
|
||||
}
|
||||
console.log(
|
||||
`Retrying repository ${repo.name} in organization ${organization.name} (attempt ${attempt}): ${error.message}`
|
||||
);
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -750,9 +825,10 @@ export async function mirrorGitHubOrgToGitea({
|
||||
organizationId: organization.id,
|
||||
organizationName: organization.name,
|
||||
message: `Successfully mirrored organization: ${organization.name}`,
|
||||
details: orgRepos.length === 0
|
||||
? `Organization ${organization.name} was processed successfully (no repositories found).`
|
||||
: `Organization ${organization.name} was mirrored to Gitea with ${orgRepos.length} repositories.`,
|
||||
details:
|
||||
orgRepos.length === 0
|
||||
? `Organization ${organization.name} was processed successfully (no repositories found).`
|
||||
: `Organization ${organization.name} was mirrored to Gitea with ${orgRepos.length} repositories.`,
|
||||
status: repoStatusEnum.parse("mirrored"),
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -836,18 +912,20 @@ export const syncGiteaRepo = async ({
|
||||
const { present, actualOwner } = await checkRepoLocation({
|
||||
config,
|
||||
repository,
|
||||
expectedOwner: repoOwner
|
||||
expectedOwner: repoOwner,
|
||||
});
|
||||
|
||||
if (!present) {
|
||||
throw new Error(`Repository ${repository.name} not found in Gitea at any expected location`);
|
||||
throw new Error(
|
||||
`Repository ${repository.name} not found in Gitea at any expected location`
|
||||
);
|
||||
}
|
||||
|
||||
// Use the actual owner where the repo was found
|
||||
const apiUrl = `${config.giteaConfig.url}/api/v1/repos/${actualOwner}/${repository.name}/mirror-sync`;
|
||||
|
||||
const response = await httpPost(apiUrl, undefined, {
|
||||
"Authorization": `token ${config.giteaConfig.token}`,
|
||||
Authorization: `token ${config.giteaConfig.token}`,
|
||||
});
|
||||
|
||||
// Mark repo as "synced" in DB
|
||||
@@ -951,9 +1029,11 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
);
|
||||
|
||||
// Filter out pull requests
|
||||
const filteredIssues = issues.filter(issue => !(issue as any).pull_request);
|
||||
const filteredIssues = issues.filter((issue) => !(issue as any).pull_request);
|
||||
|
||||
console.log(`Mirroring ${filteredIssues.length} issues from ${repository.fullName}`);
|
||||
console.log(
|
||||
`Mirroring ${filteredIssues.length} issues from ${repository.fullName}`
|
||||
);
|
||||
|
||||
if (filteredIssues.length === 0) {
|
||||
console.log(`No issues to mirror for ${repository.fullName}`);
|
||||
@@ -964,7 +1044,7 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
const giteaLabelsRes = await httpGet(
|
||||
`${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/labels`,
|
||||
{
|
||||
"Authorization": `token ${config.giteaConfig.token}`,
|
||||
Authorization: `token ${config.giteaConfig.token}`,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -994,10 +1074,12 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
} else {
|
||||
try {
|
||||
const created = await httpPost(
|
||||
`${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${repository.name}/labels`,
|
||||
`${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${
|
||||
repository.name
|
||||
}/labels`,
|
||||
{ name, color: "#ededed" }, // Default color
|
||||
{
|
||||
"Authorization": `token ${config.giteaConfig!.token}`,
|
||||
Authorization: `token ${config.giteaConfig!.token}`,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -1029,10 +1111,12 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
|
||||
// Create the issue in Gitea
|
||||
const createdIssue = await httpPost(
|
||||
`${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${repository.name}/issues`,
|
||||
`${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${
|
||||
repository.name
|
||||
}/issues`,
|
||||
issuePayload,
|
||||
{
|
||||
"Authorization": `token ${config.giteaConfig!.token}`,
|
||||
Authorization: `token ${config.giteaConfig!.token}`,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -1054,12 +1138,14 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
comments,
|
||||
async (comment) => {
|
||||
await httpPost(
|
||||
`${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${repository.name}/issues/${createdIssue.data.number}/comments`,
|
||||
`${config.giteaConfig!.url}/api/v1/repos/${repoOrigin}/${
|
||||
repository.name
|
||||
}/issues/${createdIssue.data.number}/comments`,
|
||||
{
|
||||
body: `@${comment.user?.login} commented on GitHub:\n\n${comment.body}`,
|
||||
},
|
||||
{
|
||||
"Authorization": `token ${config.giteaConfig!.token}`,
|
||||
Authorization: `token ${config.giteaConfig!.token}`,
|
||||
}
|
||||
);
|
||||
return comment;
|
||||
@@ -1069,8 +1155,10 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
maxRetries: 2,
|
||||
retryDelay: 1000,
|
||||
onRetry: (_comment, error, attempt) => {
|
||||
console.log(`Retrying comment (attempt ${attempt}): ${error.message}`);
|
||||
}
|
||||
console.log(
|
||||
`Retrying comment (attempt ${attempt}): ${error.message}`
|
||||
);
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -1084,14 +1172,69 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
||||
onProgress: (completed, total, result) => {
|
||||
const percentComplete = Math.round((completed / total) * 100);
|
||||
if (result) {
|
||||
console.log(`Mirrored issue "${result.title}" (${completed}/${total}, ${percentComplete}%)`);
|
||||
console.log(
|
||||
`Mirrored issue "${result.title}" (${completed}/${total}, ${percentComplete}%)`
|
||||
);
|
||||
}
|
||||
},
|
||||
onRetry: (issue, error, attempt) => {
|
||||
console.log(`Retrying issue "${issue.title}" (attempt ${attempt}): ${error.message}`);
|
||||
}
|
||||
console.log(
|
||||
`Retrying issue "${issue.title}" (attempt ${attempt}): ${error.message}`
|
||||
);
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
console.log(`Completed mirroring ${filteredIssues.length} issues for ${repository.fullName}`);
|
||||
console.log(
|
||||
`Completed mirroring ${filteredIssues.length} issues for ${repository.fullName}`
|
||||
);
|
||||
};
|
||||
|
||||
export async function mirrorGitHubReleasesToGitea({
|
||||
octokit,
|
||||
repository,
|
||||
config,
|
||||
}: {
|
||||
octokit: Octokit;
|
||||
repository: Repository;
|
||||
config: Partial<Config>;
|
||||
}) {
|
||||
if (
|
||||
!config.giteaConfig?.username ||
|
||||
!config.giteaConfig?.token ||
|
||||
!config.giteaConfig?.url
|
||||
) {
|
||||
throw new Error("Gitea config is incomplete for mirroring releases.");
|
||||
}
|
||||
|
||||
const repoOwner = getGiteaRepoOwner({
|
||||
config,
|
||||
repository,
|
||||
});
|
||||
|
||||
const { url, token } = config.giteaConfig;
|
||||
|
||||
const releases = await octokit.rest.repos.listReleases({
|
||||
owner: repository.owner,
|
||||
repo: repository.name,
|
||||
});
|
||||
|
||||
for (const release of releases.data) {
|
||||
await httpPost(
|
||||
`${url}/api/v1/repos/${repoOwner}/${repository.name}/releases`,
|
||||
{
|
||||
tag_name: release.tag_name,
|
||||
target: release.target_commitish,
|
||||
title: release.name || release.tag_name,
|
||||
note: release.body || "",
|
||||
draft: release.draft,
|
||||
prerelease: release.prerelease,
|
||||
},
|
||||
{
|
||||
Authorization: `token ${token}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`✅ Mirrored ${releases.data.length} GitHub releases to Gitea`);
|
||||
}
|
||||
@@ -238,6 +238,7 @@ export const GET: APIRoute = async ({ request }) => {
|
||||
skipForks: false,
|
||||
privateRepositories: false,
|
||||
mirrorIssues: false,
|
||||
mirrorWiki: false,
|
||||
mirrorStarred: true,
|
||||
useSpecificUser: false,
|
||||
preserveOrgStructure: true,
|
||||
|
||||
@@ -1,35 +1,75 @@
|
||||
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
|
||||
import type { MirrorRepoRequest } from "@/types/mirror";
|
||||
|
||||
// Create a mock POST function
|
||||
const mockPOST = mock(async ({ request }) => {
|
||||
const body = await request.json();
|
||||
|
||||
// Check for missing userId or repositoryIds
|
||||
if (!body.userId || !body.repositoryIds) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: "Missing userId or repositoryIds."
|
||||
}),
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// Success case
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: "Repository mirroring started",
|
||||
batchId: "test-batch-id"
|
||||
}),
|
||||
{ status: 200 }
|
||||
);
|
||||
});
|
||||
|
||||
// Create a mock module
|
||||
const mockModule = {
|
||||
POST: mockPOST
|
||||
// Mock the database module
|
||||
const mockDb = {
|
||||
select: mock(() => ({
|
||||
from: mock(() => ({
|
||||
where: mock(() => ({
|
||||
limit: mock(() => Promise.resolve([{
|
||||
id: "config-id",
|
||||
userId: "user-id",
|
||||
githubConfig: {
|
||||
token: "github-token",
|
||||
preserveOrgStructure: false,
|
||||
mirrorIssues: false
|
||||
},
|
||||
giteaConfig: {
|
||||
url: "https://gitea.example.com",
|
||||
token: "gitea-token",
|
||||
username: "giteauser"
|
||||
}
|
||||
}]))
|
||||
}))
|
||||
}))
|
||||
}))
|
||||
};
|
||||
|
||||
mock.module("@/lib/db", () => ({
|
||||
db: mockDb,
|
||||
configs: {},
|
||||
repositories: {}
|
||||
}));
|
||||
|
||||
// Mock the gitea module
|
||||
const mockMirrorGithubRepoToGitea = mock(() => Promise.resolve());
|
||||
const mockMirrorGitHubOrgRepoToGiteaOrg = mock(() => Promise.resolve());
|
||||
|
||||
mock.module("@/lib/gitea", () => ({
|
||||
mirrorGithubRepoToGitea: mockMirrorGithubRepoToGitea,
|
||||
mirrorGitHubOrgRepoToGiteaOrg: mockMirrorGitHubOrgRepoToGiteaOrg
|
||||
}));
|
||||
|
||||
// Mock the github module
|
||||
const mockCreateGitHubClient = mock(() => ({}));
|
||||
|
||||
mock.module("@/lib/github", () => ({
|
||||
createGitHubClient: mockCreateGitHubClient
|
||||
}));
|
||||
|
||||
// Mock the concurrency module
|
||||
const mockProcessWithResilience = mock(() => Promise.resolve([]));
|
||||
|
||||
mock.module("@/lib/utils/concurrency", () => ({
|
||||
processWithResilience: mockProcessWithResilience
|
||||
}));
|
||||
|
||||
// Mock drizzle-orm
|
||||
mock.module("drizzle-orm", () => ({
|
||||
eq: mock(() => ({})),
|
||||
inArray: mock(() => ({}))
|
||||
}));
|
||||
|
||||
// Mock the types
|
||||
mock.module("@/types/Repository", () => ({
|
||||
repositoryVisibilityEnum: {
|
||||
parse: mock((value: string) => value)
|
||||
},
|
||||
repoStatusEnum: {
|
||||
parse: mock((value: string) => value)
|
||||
}
|
||||
}));
|
||||
|
||||
describe("Repository Mirroring API", () => {
|
||||
// Mock console.log and console.error to prevent test output noise
|
||||
let originalConsoleLog: typeof console.log;
|
||||
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
} from "@/lib/gitea";
|
||||
import { createGitHubClient } from "@/lib/github";
|
||||
import { processWithResilience } from "@/lib/utils/concurrency";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
export const POST: APIRoute = async ({ request }) => {
|
||||
try {
|
||||
@@ -77,9 +76,6 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
// Define the concurrency limit - adjust based on API rate limits
|
||||
const CONCURRENCY_LIMIT = 3;
|
||||
|
||||
// Generate a batch ID to group related repositories
|
||||
const batchId = uuidv4();
|
||||
|
||||
// Process repositories in parallel with resilience to container restarts
|
||||
await processWithResilience(
|
||||
repos,
|
||||
@@ -120,7 +116,6 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
{
|
||||
userId: config.userId || "",
|
||||
jobType: "mirror",
|
||||
batchId,
|
||||
getItemId: (repo) => repo.id,
|
||||
getItemName: (repo) => repo.name,
|
||||
concurrencyLimit: CONCURRENCY_LIMIT,
|
||||
@@ -129,15 +124,19 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
checkpointInterval: 5, // Checkpoint every 5 repositories to reduce event frequency
|
||||
onProgress: (completed, total, result) => {
|
||||
const percentComplete = Math.round((completed / total) * 100);
|
||||
console.log(`Mirroring progress: ${percentComplete}% (${completed}/${total})`);
|
||||
console.log(
|
||||
`Mirroring progress: ${percentComplete}% (${completed}/${total})`
|
||||
);
|
||||
|
||||
if (result) {
|
||||
console.log(`Successfully mirrored repository: ${result.name}`);
|
||||
}
|
||||
},
|
||||
onRetry: (repo, error, attempt) => {
|
||||
console.log(`Retrying repository ${repo.name} (attempt ${attempt}): ${error.message}`);
|
||||
}
|
||||
console.log(
|
||||
`Retrying repository ${repo.name} (attempt ${attempt}): ${error.message}`
|
||||
);
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
@@ -168,7 +167,10 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
// Enhanced error logging for better debugging
|
||||
console.error("=== ERROR MIRRORING REPOSITORIES ===");
|
||||
console.error("Error type:", error?.constructor?.name);
|
||||
console.error("Error message:", error instanceof Error ? error.message : String(error));
|
||||
console.error(
|
||||
"Error message:",
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
|
||||
if (error instanceof Error) {
|
||||
console.error("Error stack:", error.stack);
|
||||
@@ -181,9 +183,11 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
console.error("- Headers:", Object.fromEntries(request.headers.entries()));
|
||||
|
||||
// If it's a JSON parsing error, provide more context
|
||||
if (error instanceof SyntaxError && error.message.includes('JSON')) {
|
||||
if (error instanceof SyntaxError && error.message.includes("JSON")) {
|
||||
console.error("🚨 JSON PARSING ERROR DETECTED:");
|
||||
console.error("This suggests the response from Gitea API is not valid JSON");
|
||||
console.error(
|
||||
"This suggests the response from Gitea API is not valid JSON"
|
||||
);
|
||||
console.error("Common causes:");
|
||||
console.error("- Gitea server returned HTML error page instead of JSON");
|
||||
console.error("- Network connection interrupted");
|
||||
@@ -196,12 +200,14 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: error instanceof Error ? error.message : "An unknown error occurred",
|
||||
error:
|
||||
error instanceof Error ? error.message : "An unknown error occurred",
|
||||
errorType: error?.constructor?.name || "Unknown",
|
||||
timestamp: new Date().toISOString(),
|
||||
troubleshooting: error instanceof SyntaxError && error.message.includes('JSON')
|
||||
? "JSON parsing error detected. Check Gitea server status and logs. Ensure Gitea is returning valid JSON responses."
|
||||
: "Check application logs for more details"
|
||||
troubleshooting:
|
||||
error instanceof SyntaxError && error.message.includes("JSON")
|
||||
? "JSON parsing error detected. Check Gitea server status and logs. Ensure Gitea is returning valid JSON responses."
|
||||
: "Check application logs for more details",
|
||||
}),
|
||||
{ status: 500, headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface GitHubConfig {
|
||||
skipForks: boolean;
|
||||
privateRepositories: boolean;
|
||||
mirrorIssues: boolean;
|
||||
mirrorWiki: boolean;
|
||||
mirrorStarred: boolean;
|
||||
preserveOrgStructure: boolean;
|
||||
skipStarredIssues: boolean;
|
||||
|
||||
Reference in New Issue
Block a user