mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-03-13 22:12:54 +03:00
This commit is contained in:
@@ -124,19 +124,31 @@ export function ConfigTabs() {
|
|||||||
if (!user?.id) return;
|
if (!user?.id) return;
|
||||||
setIsSyncing(true);
|
setIsSyncing(true);
|
||||||
try {
|
try {
|
||||||
const result = await apiRequest<{ success: boolean; message?: string }>(
|
const result = await apiRequest<{ success: boolean; message?: string; failedOrgs?: string[]; recoveredOrgs?: number }>(
|
||||||
`/sync?userId=${user.id}`,
|
`/sync?userId=${user.id}`,
|
||||||
{ method: 'POST' },
|
{ method: 'POST' },
|
||||||
);
|
);
|
||||||
result.success
|
if (result.success) {
|
||||||
? toast.success(
|
toast.success(
|
||||||
'GitHub data imported successfully! Head to the Repositories page to start mirroring.',
|
'GitHub data imported successfully! Head to the Repositories page to start mirroring.',
|
||||||
)
|
);
|
||||||
: toast.error(
|
if (result.failedOrgs && result.failedOrgs.length > 0) {
|
||||||
`Failed to import GitHub data: ${
|
toast.warning(
|
||||||
result.message || 'Unknown error'
|
`${result.failedOrgs.length} org${result.failedOrgs.length > 1 ? 's' : ''} failed to import (${result.failedOrgs.join(', ')}). Check the Organizations tab for details.`,
|
||||||
}`,
|
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
if (result.recoveredOrgs && result.recoveredOrgs > 0) {
|
||||||
|
toast.success(
|
||||||
|
`${result.recoveredOrgs} previously failed org${result.recoveredOrgs > 1 ? 's' : ''} recovered successfully.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
toast.error(
|
||||||
|
`Failed to import GitHub data: ${
|
||||||
|
result.message || 'Unknown error'
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
toast.error(
|
toast.error(
|
||||||
`Error importing GitHub data: ${
|
`Error importing GitHub data: ${
|
||||||
|
|||||||
@@ -248,6 +248,11 @@ export function OrganizationList({
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Error message for failed orgs */}
|
||||||
|
{org.status === "failed" && org.errorMessage && (
|
||||||
|
<p className="text-xs text-destructive line-clamp-2">{org.errorMessage}</p>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Destination override section */}
|
{/* Destination override section */}
|
||||||
<div>
|
<div>
|
||||||
<MirrorDestinationEditor
|
<MirrorDestinationEditor
|
||||||
@@ -304,6 +309,13 @@ export function OrganizationList({
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Error message for failed orgs */}
|
||||||
|
{org.status === "failed" && org.errorMessage && (
|
||||||
|
<div className="mb-4 p-3 rounded-md bg-destructive/10 border border-destructive/20">
|
||||||
|
<p className="text-sm text-destructive">{org.errorMessage}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Repository statistics */}
|
{/* Repository statistics */}
|
||||||
<div className="mb-4">
|
<div className="mb-4">
|
||||||
<div className="flex items-center gap-4 text-sm">
|
<div className="flex items-center gap-4 text-sm">
|
||||||
@@ -313,7 +325,7 @@ export function OrganizationList({
|
|||||||
{org.repositoryCount === 1 ? "repository" : "repositories"}
|
{org.repositoryCount === 1 ? "repository" : "repositories"}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Repository breakdown - only show non-zero counts */}
|
{/* Repository breakdown - only show non-zero counts */}
|
||||||
{(() => {
|
{(() => {
|
||||||
const counts = [];
|
const counts = [];
|
||||||
@@ -326,7 +338,7 @@ export function OrganizationList({
|
|||||||
if (org.forkRepositoryCount && org.forkRepositoryCount > 0) {
|
if (org.forkRepositoryCount && org.forkRepositoryCount > 0) {
|
||||||
counts.push(`${org.forkRepositoryCount} ${org.forkRepositoryCount === 1 ? 'fork' : 'forks'}`);
|
counts.push(`${org.forkRepositoryCount} ${org.forkRepositoryCount === 1 ? 'fork' : 'forks'}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return counts.length > 0 ? (
|
return counts.length > 0 ? (
|
||||||
<div className="flex items-center gap-3 text-xs text-muted-foreground">
|
<div className="flex items-center gap-3 text-xs text-muted-foreground">
|
||||||
{counts.map((count, index) => (
|
{counts.map((count, index) => (
|
||||||
@@ -415,7 +427,7 @@ export function OrganizationList({
|
|||||||
)}
|
)}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Dropdown menu for additional actions */}
|
{/* Dropdown menu for additional actions */}
|
||||||
{org.status !== "mirroring" && (
|
{org.status !== "mirroring" && (
|
||||||
<DropdownMenu>
|
<DropdownMenu>
|
||||||
@@ -426,7 +438,7 @@ export function OrganizationList({
|
|||||||
</DropdownMenuTrigger>
|
</DropdownMenuTrigger>
|
||||||
<DropdownMenuContent align="end">
|
<DropdownMenuContent align="end">
|
||||||
{org.status !== "ignored" && (
|
{org.status !== "ignored" && (
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={() => org.id && onIgnore && onIgnore({ orgId: org.id, ignore: true })}
|
onClick={() => org.id && onIgnore && onIgnore({ orgId: org.id, ignore: true })}
|
||||||
>
|
>
|
||||||
<Ban className="h-4 w-4 mr-2" />
|
<Ban className="h-4 w-4 mr-2" />
|
||||||
@@ -449,7 +461,7 @@ export function OrganizationList({
|
|||||||
</DropdownMenu>
|
</DropdownMenu>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="flex items-center gap-2 justify-center">
|
<div className="flex items-center gap-2 justify-center">
|
||||||
{(() => {
|
{(() => {
|
||||||
const giteaUrl = getGiteaOrgUrl(org);
|
const giteaUrl = getGiteaOrgUrl(org);
|
||||||
|
|||||||
@@ -369,7 +369,7 @@ export async function getGithubOrganizations({
|
|||||||
}: {
|
}: {
|
||||||
octokit: Octokit;
|
octokit: Octokit;
|
||||||
config: Partial<Config>;
|
config: Partial<Config>;
|
||||||
}): Promise<GitOrg[]> {
|
}): Promise<{ organizations: GitOrg[]; failedOrgs: { name: string; avatarUrl: string; reason: string }[] }> {
|
||||||
try {
|
try {
|
||||||
const { data: orgs } = await octokit.orgs.listForAuthenticatedUser({
|
const { data: orgs } = await octokit.orgs.listForAuthenticatedUser({
|
||||||
per_page: 100,
|
per_page: 100,
|
||||||
@@ -392,30 +392,47 @@ export async function getGithubOrganizations({
|
|||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
const organizations = await Promise.all(
|
const failedOrgs: { name: string; avatarUrl: string; reason: string }[] = [];
|
||||||
|
const results = await Promise.all(
|
||||||
filteredOrgs.map(async (org) => {
|
filteredOrgs.map(async (org) => {
|
||||||
const [{ data: orgDetails }, { data: membership }] = await Promise.all([
|
try {
|
||||||
octokit.orgs.get({ org: org.login }),
|
const [{ data: orgDetails }, { data: membership }] = await Promise.all([
|
||||||
octokit.orgs.getMembershipForAuthenticatedUser({ org: org.login }),
|
octokit.orgs.get({ org: org.login }),
|
||||||
]);
|
octokit.orgs.getMembershipForAuthenticatedUser({ org: org.login }),
|
||||||
|
]);
|
||||||
|
|
||||||
const totalRepos =
|
const totalRepos =
|
||||||
orgDetails.public_repos + (orgDetails.total_private_repos ?? 0);
|
orgDetails.public_repos + (orgDetails.total_private_repos ?? 0);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
name: org.login,
|
name: org.login,
|
||||||
avatarUrl: org.avatar_url,
|
avatarUrl: org.avatar_url,
|
||||||
membershipRole: membership.role as MembershipRole,
|
membershipRole: membership.role as MembershipRole,
|
||||||
isIncluded: false,
|
isIncluded: false,
|
||||||
status: "imported" as RepoStatus,
|
status: "imported" as RepoStatus,
|
||||||
repositoryCount: totalRepos,
|
repositoryCount: totalRepos,
|
||||||
createdAt: new Date(),
|
createdAt: new Date(),
|
||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
};
|
};
|
||||||
|
} catch (error: any) {
|
||||||
|
// Capture organizations that return 403 (SAML enforcement, insufficient token scope, etc.)
|
||||||
|
if (error?.status === 403) {
|
||||||
|
const reason = error?.message || "access denied";
|
||||||
|
console.warn(
|
||||||
|
`Failed to import organization ${org.login} - ${reason}`,
|
||||||
|
);
|
||||||
|
failedOrgs.push({ name: org.login, avatarUrl: org.avatar_url, reason });
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
return organizations;
|
return {
|
||||||
|
organizations: results.filter((org): org is NonNullable<typeof org> => org !== null),
|
||||||
|
failedOrgs,
|
||||||
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Error fetching organizations: ${
|
`Error fetching organizations: ${
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { APIRoute } from "astro";
|
import type { APIRoute } from "astro";
|
||||||
import { db, organizations, repositories, configs } from "@/lib/db";
|
import { db, organizations, repositories, configs } from "@/lib/db";
|
||||||
import { eq } from "drizzle-orm";
|
import { eq, and } from "drizzle-orm";
|
||||||
import { v4 as uuidv4 } from "uuid";
|
import { v4 as uuidv4 } from "uuid";
|
||||||
import { createMirrorJob } from "@/lib/helpers";
|
import { createMirrorJob } from "@/lib/helpers";
|
||||||
import {
|
import {
|
||||||
@@ -47,13 +47,14 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
|||||||
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||||
|
|
||||||
// Fetch GitHub data in parallel
|
// Fetch GitHub data in parallel
|
||||||
const [basicAndForkedRepos, starredRepos, gitOrgs] = await Promise.all([
|
const [basicAndForkedRepos, starredRepos, orgResult] = await Promise.all([
|
||||||
getGithubRepositories({ octokit, config }),
|
getGithubRepositories({ octokit, config }),
|
||||||
config.githubConfig?.includeStarred
|
config.githubConfig?.includeStarred
|
||||||
? getGithubStarredRepositories({ octokit, config })
|
? getGithubStarredRepositories({ octokit, config })
|
||||||
: Promise.resolve([]),
|
: Promise.resolve([]),
|
||||||
getGithubOrganizations({ octokit, config }),
|
getGithubOrganizations({ octokit, config }),
|
||||||
]);
|
]);
|
||||||
|
const { organizations: gitOrgs, failedOrgs } = orgResult;
|
||||||
|
|
||||||
// Merge and de-duplicate by fullName, preferring starred variant when duplicated
|
// Merge and de-duplicate by fullName, preferring starred variant when duplicated
|
||||||
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||||
@@ -108,8 +109,27 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
|||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
// Prepare failed org records for DB insertion
|
||||||
|
const failedOrgRecords = failedOrgs.map((org) => ({
|
||||||
|
id: uuidv4(),
|
||||||
|
userId,
|
||||||
|
configId: config.id,
|
||||||
|
name: org.name,
|
||||||
|
normalizedName: org.name.toLowerCase(),
|
||||||
|
avatarUrl: org.avatarUrl,
|
||||||
|
membershipRole: "member" as const,
|
||||||
|
isIncluded: false,
|
||||||
|
status: "failed" as const,
|
||||||
|
errorMessage: org.reason,
|
||||||
|
repositoryCount: 0,
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
}));
|
||||||
|
|
||||||
let insertedRepos: typeof newRepos = [];
|
let insertedRepos: typeof newRepos = [];
|
||||||
let insertedOrgs: typeof newOrgs = [];
|
let insertedOrgs: typeof newOrgs = [];
|
||||||
|
let insertedFailedOrgs: typeof failedOrgRecords = [];
|
||||||
|
let recoveredOrgCount = 0;
|
||||||
|
|
||||||
// Transaction to insert only new items
|
// Transaction to insert only new items
|
||||||
await db.transaction(async (tx) => {
|
await db.transaction(async (tx) => {
|
||||||
@@ -119,18 +139,62 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
|||||||
.from(repositories)
|
.from(repositories)
|
||||||
.where(eq(repositories.userId, userId)),
|
.where(eq(repositories.userId, userId)),
|
||||||
tx
|
tx
|
||||||
.select({ normalizedName: organizations.normalizedName })
|
.select({ normalizedName: organizations.normalizedName, status: organizations.status })
|
||||||
.from(organizations)
|
.from(organizations)
|
||||||
.where(eq(organizations.userId, userId)),
|
.where(eq(organizations.userId, userId)),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const existingRepoNames = new Set(existingRepos.map((r) => r.normalizedFullName));
|
const existingRepoNames = new Set(existingRepos.map((r) => r.normalizedFullName));
|
||||||
const existingOrgNames = new Set(existingOrgs.map((o) => o.normalizedName));
|
const existingOrgMap = new Map(existingOrgs.map((o) => [o.normalizedName, o.status]));
|
||||||
|
|
||||||
insertedRepos = newRepos.filter(
|
insertedRepos = newRepos.filter(
|
||||||
(r) => !existingRepoNames.has(r.normalizedFullName)
|
(r) => !existingRepoNames.has(r.normalizedFullName)
|
||||||
);
|
);
|
||||||
insertedOrgs = newOrgs.filter((o) => !existingOrgNames.has(o.normalizedName));
|
insertedOrgs = newOrgs.filter((o) => !existingOrgMap.has(o.normalizedName));
|
||||||
|
|
||||||
|
// Update previously failed orgs that now succeeded
|
||||||
|
const recoveredOrgs = newOrgs.filter(
|
||||||
|
(o) => existingOrgMap.get(o.normalizedName) === "failed"
|
||||||
|
);
|
||||||
|
for (const org of recoveredOrgs) {
|
||||||
|
await tx
|
||||||
|
.update(organizations)
|
||||||
|
.set({
|
||||||
|
status: "imported",
|
||||||
|
errorMessage: null,
|
||||||
|
repositoryCount: org.repositoryCount,
|
||||||
|
avatarUrl: org.avatarUrl,
|
||||||
|
membershipRole: org.membershipRole,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(organizations.userId, userId),
|
||||||
|
eq(organizations.normalizedName, org.normalizedName),
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
recoveredOrgCount = recoveredOrgs.length;
|
||||||
|
|
||||||
|
// Insert or update failed orgs (only update orgs already in "failed" state — don't overwrite good state)
|
||||||
|
insertedFailedOrgs = failedOrgRecords.filter((o) => !existingOrgMap.has(o.normalizedName));
|
||||||
|
const stillFailedOrgs = failedOrgRecords.filter(
|
||||||
|
(o) => existingOrgMap.get(o.normalizedName) === "failed"
|
||||||
|
);
|
||||||
|
for (const org of stillFailedOrgs) {
|
||||||
|
await tx
|
||||||
|
.update(organizations)
|
||||||
|
.set({
|
||||||
|
errorMessage: org.errorMessage,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(organizations.userId, userId),
|
||||||
|
eq(organizations.normalizedName, org.normalizedName),
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Batch insert repositories to avoid SQLite parameter limit (dynamic by column count)
|
// Batch insert repositories to avoid SQLite parameter limit (dynamic by column count)
|
||||||
const sample = newRepos[0];
|
const sample = newRepos[0];
|
||||||
@@ -148,9 +212,10 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
|||||||
|
|
||||||
// Batch insert organizations (they have fewer fields, so we can use larger batches)
|
// Batch insert organizations (they have fewer fields, so we can use larger batches)
|
||||||
const ORG_BATCH_SIZE = 100;
|
const ORG_BATCH_SIZE = 100;
|
||||||
if (insertedOrgs.length > 0) {
|
const allNewOrgs = [...insertedOrgs, ...insertedFailedOrgs];
|
||||||
for (let i = 0; i < insertedOrgs.length; i += ORG_BATCH_SIZE) {
|
if (allNewOrgs.length > 0) {
|
||||||
const batch = insertedOrgs.slice(i, i + ORG_BATCH_SIZE);
|
for (let i = 0; i < allNewOrgs.length; i += ORG_BATCH_SIZE) {
|
||||||
|
const batch = allNewOrgs.slice(i, i + ORG_BATCH_SIZE);
|
||||||
await tx.insert(organizations).values(batch);
|
await tx.insert(organizations).values(batch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -189,6 +254,8 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
|||||||
newRepositories: insertedRepos.length,
|
newRepositories: insertedRepos.length,
|
||||||
newOrganizations: insertedOrgs.length,
|
newOrganizations: insertedOrgs.length,
|
||||||
skippedDisabledRepositories: allGithubRepos.length - mirrorableGithubRepos.length,
|
skippedDisabledRepositories: allGithubRepos.length - mirrorableGithubRepos.length,
|
||||||
|
failedOrgs: failedOrgs.map((o) => o.name),
|
||||||
|
recoveredOrgs: recoveredOrgCount,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user