fix: prevent duplicate orgs and repos

This commit is contained in:
Arunavo Ray
2025-10-27 08:44:45 +05:30
parent af9bc861cf
commit 8d96e176b4
20 changed files with 2800 additions and 125 deletions

View File

@@ -127,6 +127,7 @@ export const repositorySchema = z.object({
configId: z.string(),
name: z.string(),
fullName: z.string(),
normalizedFullName: z.string(),
url: z.url(),
cloneUrl: z.url(),
owner: z.string(),
@@ -209,6 +210,7 @@ export const organizationSchema = z.object({
userId: z.string(),
configId: z.string(),
name: z.string(),
normalizedName: z.string(),
avatarUrl: z.string(),
membershipRole: z.enum(["member", "admin", "owner", "billing_manager"]).default("member"),
isIncluded: z.boolean().default(true),
@@ -334,6 +336,7 @@ export const repositories = sqliteTable("repositories", {
.references(() => configs.id),
name: text("name").notNull(),
fullName: text("full_name").notNull(),
normalizedFullName: text("normalized_full_name").notNull(),
url: text("url").notNull(),
cloneUrl: text("clone_url").notNull(),
owner: text("owner").notNull(),
@@ -388,6 +391,7 @@ export const repositories = sqliteTable("repositories", {
index("idx_repositories_is_fork").on(table.isForked),
index("idx_repositories_is_starred").on(table.isStarred),
uniqueIndex("uniq_repositories_user_full_name").on(table.userId, table.fullName),
uniqueIndex("uniq_repositories_user_normalized_full_name").on(table.userId, table.normalizedFullName),
]);
export const mirrorJobs = sqliteTable("mirror_jobs", {
@@ -438,6 +442,7 @@ export const organizations = sqliteTable("organizations", {
.notNull()
.references(() => configs.id),
name: text("name").notNull(),
normalizedName: text("normalized_name").notNull(),
avatarUrl: text("avatar_url").notNull(),
@@ -469,6 +474,7 @@ export const organizations = sqliteTable("organizations", {
index("idx_organizations_config_id").on(table.configId),
index("idx_organizations_status").on(table.status),
index("idx_organizations_is_included").on(table.isIncluded),
uniqueIndex("uniq_organizations_user_normalized_name").on(table.userId, table.normalizedName),
]);
// ===== Better Auth Tables =====

View File

@@ -62,6 +62,7 @@ describe('normalizeGitRepoToInsert', () => {
expect(insert.description).toBeNull();
expect(insert.lastMirrored).toBeNull();
expect(insert.errorMessage).toBeNull();
expect(insert.normalizedFullName).toBe(repo.fullName.toLowerCase());
});
});
@@ -72,4 +73,3 @@ describe('calcBatchSizeForInsert', () => {
expect(batch * 29).toBeLessThanOrEqual(999);
});
});

View File

@@ -33,6 +33,7 @@ export function normalizeGitRepoToInsert(
configId,
name: repo.name,
fullName: repo.fullName,
normalizedFullName: repo.fullName.toLowerCase(),
url: repo.url,
cloneUrl: repo.cloneUrl,
owner: repo.owner,
@@ -68,4 +69,3 @@ export function calcBatchSizeForInsert(columnCount: number, maxParams = 999): nu
const effectiveMax = Math.max(1, maxParams - safety);
return Math.max(1, Math.floor(effectiveMax / columnCount));
}

View File

@@ -99,12 +99,12 @@ async function runScheduledSync(config: any): Promise<void> {
// Check for new repositories
const existingRepos = await db
.select({ fullName: repositories.fullName })
.select({ normalizedFullName: repositories.normalizedFullName })
.from(repositories)
.where(eq(repositories.userId, userId));
const existingRepoNames = new Set(existingRepos.map(r => r.fullName));
const newRepos = allGithubRepos.filter(r => !existingRepoNames.has(r.fullName));
const existingRepoNames = new Set(existingRepos.map(r => r.normalizedFullName));
const newRepos = allGithubRepos.filter(r => !existingRepoNames.has(r.fullName.toLowerCase()));
if (newRepos.length > 0) {
console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId}`);
@@ -123,7 +123,7 @@ async function runScheduledSync(config: any): Promise<void> {
await db
.insert(repositories)
.values(batch)
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
.onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] });
}
console.log(`[Scheduler] Successfully imported ${newRepos.length} new repositories for user ${userId}`);
} else {
@@ -432,12 +432,12 @@ async function performInitialAutoStart(): Promise<void> {
// Check for new repositories
const existingRepos = await db
.select({ fullName: repositories.fullName })
.select({ normalizedFullName: repositories.normalizedFullName })
.from(repositories)
.where(eq(repositories.userId, config.userId));
const existingRepoNames = new Set(existingRepos.map(r => r.fullName));
const reposToImport = allGithubRepos.filter(r => !existingRepoNames.has(r.fullName));
const existingRepoNames = new Set(existingRepos.map(r => r.normalizedFullName));
const reposToImport = allGithubRepos.filter(r => !existingRepoNames.has(r.fullName.toLowerCase()));
if (reposToImport.length > 0) {
console.log(`[Scheduler] Importing ${reposToImport.length} repositories for user ${config.userId}...`);
@@ -456,7 +456,7 @@ async function performInitialAutoStart(): Promise<void> {
await db
.insert(repositories)
.values(batch)
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
.onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] });
}
console.log(`[Scheduler] Successfully imported ${reposToImport.length} repositories`);
} else {