Mirror/gitea-mirror
1
0
Fork 0
mirror of https://github.com/RayLabsHQ/gitea-mirror.git synced 2026-03-26 15:47:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix: prevent duplicate orgs and repos

Browse Source
This commit is contained in:
Arunavo Ray
2025-10-27 08:44:45 +05:30
parent af9bc861cf
commit 8d96e176b4
20 changed files with 2800 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/pages/api/sync/index.ts
View File

@@ -66,6 +66,7 @@ export const POST: APIRoute = async ({ request }) => {
configId: config.id,
name: repo.name,
fullName: repo.fullName,
normalizedFullName: repo.fullName.toLowerCase(),
url: repo.url,
cloneUrl: repo.cloneUrl,
owner: repo.owner,
@@ -97,6 +98,7 @@ export const POST: APIRoute = async ({ request }) => {
userId,
configId: config.id,
name: org.name,
normalizedName: org.name.toLowerCase(),
avatarUrl: org.avatarUrl,
membershipRole: org.membershipRole,
isIncluded: false,
@@ -113,22 +115,22 @@ export const POST: APIRoute = async ({ request }) => {
await db.transaction(async (tx) => {
const [existingRepos, existingOrgs] = await Promise.all([
tx
.select({ fullName: repositories.fullName })
.select({ normalizedFullName: repositories.normalizedFullName })
.from(repositories)
.where(eq(repositories.userId, userId)),
tx
.select({ name: organizations.name })
.select({ normalizedName: organizations.normalizedName })
.from(organizations)
.where(eq(organizations.userId, userId)),
]);
const existingRepoNames = new Set(existingRepos.map((r) => r.fullName));
const existingOrgNames = new Set(existingOrgs.map((o) => o.name));
const existingRepoNames = new Set(existingRepos.map((r) => r.normalizedFullName));
const existingOrgNames = new Set(existingOrgs.map((o) => o.normalizedName));
insertedRepos = newRepos.filter(
(r) => !existingRepoNames.has(r.fullName)
(r) => !existingRepoNames.has(r.normalizedFullName)
);
insertedOrgs = newOrgs.filter((o) => !existingOrgNames.has(o.name));
insertedOrgs = newOrgs.filter((o) => !existingOrgNames.has(o.normalizedName));
// Batch insert repositories to avoid SQLite parameter limit (dynamic by column count)
const sample = newRepos[0];
@@ -140,7 +142,7 @@ export const POST: APIRoute = async ({ request }) => {
await tx
.insert(repositories)
.values(batch)
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
.onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] });
}
}

139
src/pages/api/sync/organization.ts
View File

@@ -1,5 +1,4 @@
import type { APIRoute } from "astro";
import { Octokit } from "@octokit/rest";
import { configs, db, organizations, repositories } from "@/lib/db";
import { and, eq } from "drizzle-orm";
import { jsonResponse, createSecureErrorResponse } from "@/lib/utils";
@@ -15,7 +14,7 @@ import { createGitHubClient } from "@/lib/github";
export const POST: APIRoute = async ({ request }) => {
try {
const body: AddOrganizationApiRequest = await request.json();
const { role, org, userId } = body;
const { role, org, userId, force = false } = body;
if (!org || !userId || !role) {
return jsonResponse({
@@ -24,21 +23,58 @@ export const POST: APIRoute = async ({ request }) => {
});
}
// Check if org already exists
const existingOrg = await db
const trimmedOrg = org.trim();
const normalizedOrg = trimmedOrg.toLowerCase();
// Check if org already exists (case-insensitive)
const [existingOrg] = await db
.select()
.from(organizations)
.where(
and(eq(organizations.name, org), eq(organizations.userId, userId))
);
and(
eq(organizations.userId, userId),
eq(organizations.normalizedName, normalizedOrg)
)
)
.limit(1);
if (existingOrg.length > 0) {
if (existingOrg && !force) {
return jsonResponse({
data: {
success: false,
error: "Organization already exists for this user",
},
status: 400,
status: 409,
});
}
if (existingOrg && force) {
const [updatedOrg] = await db
.update(organizations)
.set({
membershipRole: role,
normalizedName: normalizedOrg,
updatedAt: new Date(),
})
.where(eq(organizations.id, existingOrg.id))
.returning();
const resPayload: AddOrganizationApiResponse = {
success: true,
organization: updatedOrg ?? existingOrg,
message: "Organization already exists; using existing record.",
};
return jsonResponse({ data: resPayload, status: 200 });
}
if (existingOrg) {
return jsonResponse({
data: {
success: false,
error: "Organization already exists for this user",
},
status: 409,
});
}
@@ -71,17 +107,21 @@ export const POST: APIRoute = async ({ request }) => {
// Create authenticated Octokit instance with rate limit tracking
const githubUsername = decryptedConfig.githubConfig?.owner || undefined;
const octokit = createGitHubClient(decryptedConfig.githubConfig.token, userId, githubUsername);
const octokit = createGitHubClient(
decryptedConfig.githubConfig.token,
userId,
githubUsername
);
// Fetch org metadata
const { data: orgData } = await octokit.orgs.get({ org });
const { data: orgData } = await octokit.orgs.get({ org: trimmedOrg });
// Fetch repos based on config settings
const allRepos = [];
// Fetch all repos (public, private, and member) to show in UI
const publicRepos = await octokit.paginate(octokit.repos.listForOrg, {
org,
org: trimmedOrg,
type: "public",
per_page: 100,
});
@@ -89,7 +129,7 @@ export const POST: APIRoute = async ({ request }) => {
// Always fetch private repos to show them in the UI
const privateRepos = await octokit.paginate(octokit.repos.listForOrg, {
org,
org: trimmedOrg,
type: "private",
per_page: 100,
});
@@ -97,7 +137,7 @@ export const POST: APIRoute = async ({ request }) => {
// Also fetch member repos (includes private repos the user has access to)
const memberRepos = await octokit.paginate(octokit.repos.listForOrg, {
org,
org: trimmedOrg,
type: "member",
per_page: 100,
});
@@ -107,38 +147,44 @@ export const POST: APIRoute = async ({ request }) => {
allRepos.push(...uniqueMemberRepos);
// Insert repositories
const repoRecords = allRepos.map((repo) => ({
id: uuidv4(),
userId,
configId,
name: repo.name,
fullName: repo.full_name,
url: repo.html_url,
cloneUrl: repo.clone_url ?? "",
owner: repo.owner.login,
organization:
repo.owner.type === "Organization" ? repo.owner.login : null,
mirroredLocation: "",
destinationOrg: null,
isPrivate: repo.private,
isForked: repo.fork,
forkedFrom: null,
hasIssues: repo.has_issues,
isStarred: false,
isArchived: repo.archived,
size: repo.size,
hasLFS: false,
hasSubmodules: false,
language: repo.language ?? null,
description: repo.description ?? null,
defaultBranch: repo.default_branch ?? "main",
visibility: (repo.visibility ?? "public") as RepositoryVisibility,
status: "imported" as RepoStatus,
lastMirrored: null,
errorMessage: null,
createdAt: repo.created_at ? new Date(repo.created_at) : new Date(),
updatedAt: repo.updated_at ? new Date(repo.updated_at) : new Date(),
}));
const repoRecords = allRepos.map((repo) => {
const normalizedOwner = repo.owner.login.trim().toLowerCase();
const normalizedRepoName = repo.name.trim().toLowerCase();
return {
id: uuidv4(),
userId,
configId,
name: repo.name,
fullName: repo.full_name,
normalizedFullName: `${normalizedOwner}/${normalizedRepoName}`,
url: repo.html_url,
cloneUrl: repo.clone_url ?? "",
owner: repo.owner.login,
organization:
repo.owner.type === "Organization" ? repo.owner.login : null,
mirroredLocation: "",
destinationOrg: null,
isPrivate: repo.private,
isForked: repo.fork,
forkedFrom: null,
hasIssues: repo.has_issues,
isStarred: false,
isArchived: repo.archived,
size: repo.size,
hasLFS: false,
hasSubmodules: false,
language: repo.language ?? null,
description: repo.description ?? null,
defaultBranch: repo.default_branch ?? "main",
visibility: (repo.visibility ?? "public") as RepositoryVisibility,
status: "imported" as RepoStatus,
lastMirrored: null,
errorMessage: null,
createdAt: repo.created_at ? new Date(repo.created_at) : new Date(),
updatedAt: repo.updated_at ? new Date(repo.updated_at) : new Date(),
};
});
// Batch insert repositories to avoid SQLite parameter limit
// Compute batch size based on column count
@@ -150,7 +196,7 @@ export const POST: APIRoute = async ({ request }) => {
await db
.insert(repositories)
.values(batch)
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
.onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] });
}
// Insert organization metadata
@@ -159,6 +205,7 @@ export const POST: APIRoute = async ({ request }) => {
userId,
configId,
name: orgData.login,
normalizedName: normalizedOrg,
avatarUrl: orgData.avatar_url,
membershipRole: role,
isIncluded: false,

81
src/pages/api/sync/repository.ts
View File

@@ -15,7 +15,7 @@ import { createMirrorJob } from "@/lib/helpers";
export const POST: APIRoute = async ({ request }) => {
try {
const body: AddRepositoriesApiRequest = await request.json();
const { owner, repo, userId } = body;
const { owner, repo, userId, force = false } = body;
if (!owner || !repo || !userId) {
return new Response(
@@ -27,26 +27,43 @@ export const POST: APIRoute = async ({ request }) => {
);
}
const trimmedOwner = owner.trim();
const trimmedRepo = repo.trim();
if (!trimmedOwner || !trimmedRepo) {
return jsonResponse({
data: {
success: false,
error: "Missing owner, repo, or userId",
},
status: 400,
});
}
const normalizedOwner = trimmedOwner.toLowerCase();
const normalizedRepo = trimmedRepo.toLowerCase();
const normalizedFullName = `${normalizedOwner}/${normalizedRepo}`;
// Check if repository with the same owner, name, and userId already exists
const existingRepo = await db
const [existingRepo] = await db
.select()
.from(repositories)
.where(
and(
eq(repositories.owner, owner),
eq(repositories.name, repo),
eq(repositories.userId, userId)
eq(repositories.userId, userId),
eq(repositories.normalizedFullName, normalizedFullName)
)
);
)
.limit(1);
if (existingRepo.length > 0) {
if (existingRepo && !force) {
return jsonResponse({
data: {
success: false,
error:
"Repository with this name and owner already exists for this user",
},
status: 400,
status: 409,
});
}
@@ -68,14 +85,17 @@ export const POST: APIRoute = async ({ request }) => {
const octokit = new Octokit(); // No auth for public repos
const { data: repoData } = await octokit.rest.repos.get({ owner, repo });
const { data: repoData } = await octokit.rest.repos.get({
owner: trimmedOwner,
repo: trimmedRepo,
});
const metadata = {
id: uuidv4(),
const baseMetadata = {
userId,
configId,
name: repoData.name,
fullName: repoData.full_name,
normalizedFullName,
url: repoData.html_url,
cloneUrl: repoData.clone_url,
owner: repoData.owner.login,
@@ -94,6 +114,37 @@ export const POST: APIRoute = async ({ request }) => {
description: repoData.description ?? null,
defaultBranch: repoData.default_branch,
visibility: (repoData.visibility ?? "public") as RepositoryVisibility,
lastMirrored: existingRepo?.lastMirrored ?? null,
errorMessage: existingRepo?.errorMessage ?? null,
mirroredLocation: existingRepo?.mirroredLocation ?? "",
destinationOrg: existingRepo?.destinationOrg ?? null,
updatedAt: repoData.updated_at
? new Date(repoData.updated_at)
: new Date(),
};
if (existingRepo && force) {
const [updatedRepo] = await db
.update(repositories)
.set({
...baseMetadata,
normalizedFullName,
configId,
})
.where(eq(repositories.id, existingRepo.id))
.returning();
const resPayload: AddRepositoriesApiResponse = {
success: true,
repository: updatedRepo ?? existingRepo,
message: "Repository already exists; metadata refreshed.",
};
return jsonResponse({ data: resPayload, status: 200 });
}
const metadata = {
id: uuidv4(),
status: "imported" as Repository["status"],
lastMirrored: null,
errorMessage: null,
@@ -102,15 +153,13 @@ export const POST: APIRoute = async ({ request }) => {
createdAt: repoData.created_at
? new Date(repoData.created_at)
: new Date(),
updatedAt: repoData.updated_at
? new Date(repoData.updated_at)
: new Date(),
};
...baseMetadata,
} satisfies Repository;
await db
.insert(repositories)
.values(metadata)
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
.onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] });
createMirrorJob({
userId,