fix(sync): batch inserts + normalize nulls to avoid SQLite param mismatch

- Batch repository inserts with dynamic sizing under SQLite 999-param limit
- Normalize undefined → null to keep multi-row insert shapes consistent
- De-duplicate owned + starred repos by fullName (prefer starred variant)
- Enforce uniqueness via (user_id, full_name) + onConflictDoNothing
- Handle starred name collisions (suffix/prefix) across mirror + metadata
- Add repo-utils helpers + tests; guard Octokit.plugin in tests
- Remove manual unique index from entrypoint; rely on drizzle-kit migrations
This commit is contained in:
Arunavo Ray
2025-09-13 23:38:50 +05:30
parent 51a6c8ca58
commit 18ecdbc252
15 changed files with 2439 additions and 154 deletions

View File

@@ -0,0 +1,75 @@
import { describe, it, expect } from 'bun:test';
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
import type { GitRepo } from '@/types/Repository';
function sampleRepo(overrides: Partial<GitRepo> = {}): GitRepo {
const base: GitRepo = {
name: 'repo',
fullName: 'owner/repo',
url: 'https://github.com/owner/repo',
cloneUrl: 'https://github.com/owner/repo.git',
owner: 'owner',
organization: undefined,
mirroredLocation: '',
destinationOrg: null,
isPrivate: false,
isForked: false,
forkedFrom: undefined,
hasIssues: true,
isStarred: false,
isArchived: false,
size: 1,
hasLFS: false,
hasSubmodules: false,
language: null,
description: null,
defaultBranch: 'main',
visibility: 'public',
status: 'imported',
lastMirrored: undefined,
errorMessage: undefined,
createdAt: new Date(),
updatedAt: new Date(),
};
return { ...base, ...overrides };
}
describe('mergeGitReposPreferStarred', () => {
it('keeps unique repos', () => {
const basic = [sampleRepo({ fullName: 'a/x', name: 'x' })];
const starred: GitRepo[] = [];
const merged = mergeGitReposPreferStarred(basic, starred);
expect(merged).toHaveLength(1);
expect(merged[0].fullName).toBe('a/x');
});
it('prefers starred when duplicate exists', () => {
const basic = [sampleRepo({ fullName: 'a/x', name: 'x', isStarred: false })];
const starred = [sampleRepo({ fullName: 'a/x', name: 'x', isStarred: true })];
const merged = mergeGitReposPreferStarred(basic, starred);
expect(merged).toHaveLength(1);
expect(merged[0].isStarred).toBe(true);
});
});
describe('normalizeGitRepoToInsert', () => {
it('sets undefined optional fields to null', () => {
const repo = sampleRepo({ organization: undefined, forkedFrom: undefined, language: undefined, description: undefined, lastMirrored: undefined, errorMessage: undefined });
const insert = normalizeGitRepoToInsert(repo, { userId: 'u', configId: 'c' });
expect(insert.organization).toBeNull();
expect(insert.forkedFrom).toBeNull();
expect(insert.language).toBeNull();
expect(insert.description).toBeNull();
expect(insert.lastMirrored).toBeNull();
expect(insert.errorMessage).toBeNull();
});
});
describe('calcBatchSizeForInsert', () => {
it('respects 999 parameter limit', () => {
const batch = calcBatchSizeForInsert(29);
expect(batch).toBeGreaterThan(0);
expect(batch * 29).toBeLessThanOrEqual(999);
});
});