fix(sync): batch inserts + normalize nulls to avoid SQLite param mismatch

- Batch repository inserts with dynamic sizing under SQLite 999-param limit
- Normalize undefined → null to keep multi-row insert shapes consistent
- De-duplicate owned + starred repos by fullName (prefer starred variant)
- Enforce uniqueness via (user_id, full_name) + onConflictDoNothing
- Handle starred name collisions (suffix/prefix) across mirror + metadata
- Add repo-utils helpers + tests; guard Octokit.plugin in tests
- Remove manual unique index from entrypoint; rely on drizzle-kit migrations
This commit is contained in:
Arunavo Ray
2025-09-13 23:38:50 +05:30
parent 51a6c8ca58
commit 18ecdbc252
15 changed files with 2439 additions and 154 deletions

View File

@@ -80,25 +80,23 @@ export const POST: APIRoute = async ({ request }) => {
cloneUrl: repoData.clone_url,
owner: repoData.owner.login,
organization:
repoData.owner.type === "Organization"
? repoData.owner.login
: undefined,
repoData.owner.type === "Organization" ? repoData.owner.login : null,
isPrivate: repoData.private,
isForked: repoData.fork,
forkedFrom: undefined,
forkedFrom: null,
hasIssues: repoData.has_issues,
isStarred: false,
isArchived: repoData.archived,
size: repoData.size,
hasLFS: false,
hasSubmodules: false,
language: repoData.language || null,
description: repoData.description || null,
language: repoData.language ?? null,
description: repoData.description ?? null,
defaultBranch: repoData.default_branch,
visibility: (repoData.visibility ?? "public") as RepositoryVisibility,
status: "imported" as Repository["status"],
lastMirrored: undefined,
errorMessage: undefined,
lastMirrored: null,
errorMessage: null,
mirroredLocation: "",
destinationOrg: null,
createdAt: repoData.created_at
@@ -109,7 +107,10 @@ export const POST: APIRoute = async ({ request }) => {
: new Date(),
};
await db.insert(repositories).values(metadata);
await db
.insert(repositories)
.values(metadata)
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
createMirrorJob({
userId,