mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2025-12-08 20:46:44 +03:00
fix(sync): batch inserts + normalize nulls to avoid SQLite param mismatch
- Batch repository inserts with dynamic sizing under SQLite 999-param limit - Normalize undefined → null to keep multi-row insert shapes consistent - De-duplicate owned + starred repos by fullName (prefer starred variant) - Enforce uniqueness via (user_id, full_name) + onConflictDoNothing - Handle starred name collisions (suffix/prefix) across mirror + metadata - Add repo-utils helpers + tests; guard Octokit.plugin in tests - Remove manual unique index from entrypoint; rely on drizzle-kit migrations
This commit is contained in:
@@ -172,6 +172,7 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then
|
|||||||
owner TEXT NOT NULL,
|
owner TEXT NOT NULL,
|
||||||
organization TEXT,
|
organization TEXT,
|
||||||
mirrored_location TEXT DEFAULT '',
|
mirrored_location TEXT DEFAULT '',
|
||||||
|
destination_org TEXT,
|
||||||
is_private INTEGER NOT NULL DEFAULT 0,
|
is_private INTEGER NOT NULL DEFAULT 0,
|
||||||
is_fork INTEGER NOT NULL DEFAULT 0,
|
is_fork INTEGER NOT NULL DEFAULT 0,
|
||||||
forked_from TEXT,
|
forked_from TEXT,
|
||||||
@@ -181,6 +182,8 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then
|
|||||||
size INTEGER NOT NULL DEFAULT 0,
|
size INTEGER NOT NULL DEFAULT 0,
|
||||||
has_lfs INTEGER NOT NULL DEFAULT 0,
|
has_lfs INTEGER NOT NULL DEFAULT 0,
|
||||||
has_submodules INTEGER NOT NULL DEFAULT 0,
|
has_submodules INTEGER NOT NULL DEFAULT 0,
|
||||||
|
language TEXT,
|
||||||
|
description TEXT,
|
||||||
default_branch TEXT NOT NULL,
|
default_branch TEXT NOT NULL,
|
||||||
visibility TEXT NOT NULL DEFAULT 'public',
|
visibility TEXT NOT NULL DEFAULT 'public',
|
||||||
status TEXT NOT NULL DEFAULT 'imported',
|
status TEXT NOT NULL DEFAULT 'imported',
|
||||||
@@ -192,6 +195,8 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then
|
|||||||
FOREIGN KEY (config_id) REFERENCES configs(id)
|
FOREIGN KEY (config_id) REFERENCES configs(id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Uniqueness of (user_id, full_name) for repositories is enforced via drizzle migrations
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS organizations (
|
CREATE TABLE IF NOT EXISTS organizations (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
user_id TEXT NOT NULL,
|
user_id TEXT NOT NULL,
|
||||||
|
|||||||
1
drizzle/0005_polite_preak.sql
Normal file
1
drizzle/0005_polite_preak.sql
Normal file
@@ -0,0 +1 @@
|
|||||||
|
CREATE UNIQUE INDEX `uniq_repositories_user_full_name` ON `repositories` (`user_id`,`full_name`);
|
||||||
1941
drizzle/meta/0005_snapshot.json
Normal file
1941
drizzle/meta/0005_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -36,6 +36,13 @@
|
|||||||
"when": 1757392620734,
|
"when": 1757392620734,
|
||||||
"tag": "0004_grey_butterfly",
|
"tag": "0004_grey_butterfly",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 5,
|
||||||
|
"version": "6",
|
||||||
|
"when": 1757786449446,
|
||||||
|
"tag": "0005_polite_preak",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -32,7 +32,14 @@ import {
|
|||||||
Funnel,
|
Funnel,
|
||||||
HardDrive
|
HardDrive
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import type { GitHubConfig, MirrorOptions, AdvancedOptions } from "@/types/config";
|
import type { GitHubConfig, MirrorOptions, AdvancedOptions, DuplicateNameStrategy } from "@/types/config";
|
||||||
|
import {
|
||||||
|
Select,
|
||||||
|
SelectContent,
|
||||||
|
SelectItem,
|
||||||
|
SelectTrigger,
|
||||||
|
SelectValue,
|
||||||
|
} from "@/components/ui/select";
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
|
|
||||||
interface GitHubMirrorSettingsProps {
|
interface GitHubMirrorSettingsProps {
|
||||||
@@ -53,7 +60,7 @@ export function GitHubMirrorSettings({
|
|||||||
onAdvancedOptionsChange,
|
onAdvancedOptionsChange,
|
||||||
}: GitHubMirrorSettingsProps) {
|
}: GitHubMirrorSettingsProps) {
|
||||||
|
|
||||||
const handleGitHubChange = (field: keyof GitHubConfig, value: boolean) => {
|
const handleGitHubChange = (field: keyof GitHubConfig, value: boolean | string) => {
|
||||||
onGitHubConfigChange({ ...githubConfig, [field]: value });
|
onGitHubConfigChange({ ...githubConfig, [field]: value });
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -278,6 +285,34 @@ export function GitHubMirrorSettings({
|
|||||||
</Popover>
|
</Popover>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Duplicate name handling for starred repos */}
|
||||||
|
{githubConfig.mirrorStarred && (
|
||||||
|
<div className="mt-4 space-y-2">
|
||||||
|
<Label htmlFor="duplicate-strategy" className="text-sm">
|
||||||
|
Duplicate name handling
|
||||||
|
</Label>
|
||||||
|
<Select
|
||||||
|
value={githubConfig.starredDuplicateStrategy || "suffix"}
|
||||||
|
onValueChange={(value) => handleGitHubChange('starredDuplicateStrategy', value as DuplicateNameStrategy)}
|
||||||
|
>
|
||||||
|
<SelectTrigger id="duplicate-strategy" className="w-full">
|
||||||
|
<SelectValue placeholder="Select duplicate handling strategy" />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="suffix">
|
||||||
|
Add owner as suffix (awesome-project-user1)
|
||||||
|
</SelectItem>
|
||||||
|
<SelectItem value="prefix">
|
||||||
|
Add owner as prefix (user1-awesome-project)
|
||||||
|
</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
Handle starred repos with duplicate names from different owners. Currently supports suffix and prefix strategies.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core";
|
import { sqliteTable, text, integer, index, uniqueIndex } from "drizzle-orm/sqlite-core";
|
||||||
import { sql } from "drizzle-orm";
|
import { sql } from "drizzle-orm";
|
||||||
|
|
||||||
// ===== Zod Validation Schemas =====
|
// ===== Zod Validation Schemas =====
|
||||||
@@ -28,6 +28,7 @@ export const githubConfigSchema = z.object({
|
|||||||
mirrorStrategy: z.enum(["preserve", "single-org", "flat-user", "mixed"]).default("preserve"),
|
mirrorStrategy: z.enum(["preserve", "single-org", "flat-user", "mixed"]).default("preserve"),
|
||||||
defaultOrg: z.string().optional(),
|
defaultOrg: z.string().optional(),
|
||||||
skipStarredIssues: z.boolean().default(false),
|
skipStarredIssues: z.boolean().default(false),
|
||||||
|
starredDuplicateStrategy: z.enum(["suffix", "prefix", "owner-org"]).default("suffix").optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const giteaConfigSchema = z.object({
|
export const giteaConfigSchema = z.object({
|
||||||
@@ -379,6 +380,7 @@ export const repositories = sqliteTable("repositories", {
|
|||||||
index("idx_repositories_organization").on(table.organization),
|
index("idx_repositories_organization").on(table.organization),
|
||||||
index("idx_repositories_is_fork").on(table.isForked),
|
index("idx_repositories_is_fork").on(table.isForked),
|
||||||
index("idx_repositories_is_starred").on(table.isStarred),
|
index("idx_repositories_is_starred").on(table.isStarred),
|
||||||
|
uniqueIndex("uniq_repositories_user_full_name").on(table.userId, table.fullName),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
export const mirrorJobs = sqliteTable("mirror_jobs", {
|
export const mirrorJobs = sqliteTable("mirror_jobs", {
|
||||||
|
|||||||
262
src/lib/gitea.ts
262
src/lib/gitea.ts
@@ -274,15 +274,37 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
// Get the correct owner based on the strategy (with organization overrides)
|
// Get the correct owner based on the strategy (with organization overrides)
|
||||||
let repoOwner = await getGiteaRepoOwnerAsync({ config, repository });
|
let repoOwner = await getGiteaRepoOwnerAsync({ config, repository });
|
||||||
|
|
||||||
|
// Determine the actual repository name to use (handle duplicates for starred repos)
|
||||||
|
let targetRepoName = repository.name;
|
||||||
|
|
||||||
|
if (repository.isStarred && config.githubConfig) {
|
||||||
|
// Extract GitHub owner from full_name (format: owner/repo)
|
||||||
|
const githubOwner = repository.fullName.split('/')[0];
|
||||||
|
|
||||||
|
targetRepoName = await generateUniqueRepoName({
|
||||||
|
config,
|
||||||
|
orgName: repoOwner,
|
||||||
|
baseName: repository.name,
|
||||||
|
githubOwner,
|
||||||
|
strategy: config.githubConfig.starredDuplicateStrategy,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (targetRepoName !== repository.name) {
|
||||||
|
console.log(
|
||||||
|
`Starred repo ${repository.fullName} will be mirrored as ${repoOwner}/${targetRepoName} to avoid naming conflict`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const isExisting = await isRepoPresentInGitea({
|
const isExisting = await isRepoPresentInGitea({
|
||||||
config,
|
config,
|
||||||
owner: repoOwner,
|
owner: repoOwner,
|
||||||
repoName: repository.name,
|
repoName: targetRepoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (isExisting) {
|
if (isExisting) {
|
||||||
console.log(
|
console.log(
|
||||||
`Repository ${repository.name} already exists in Gitea under ${repoOwner}. Updating database status.`
|
`Repository ${targetRepoName} already exists in Gitea under ${repoOwner}. Updating database status.`
|
||||||
);
|
);
|
||||||
|
|
||||||
// Update database to reflect that the repository is already mirrored
|
// Update database to reflect that the repository is already mirrored
|
||||||
@@ -293,7 +315,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
lastMirrored: new Date(),
|
lastMirrored: new Date(),
|
||||||
errorMessage: null,
|
errorMessage: null,
|
||||||
mirroredLocation: `${repoOwner}/${repository.name}`,
|
mirroredLocation: `${repoOwner}/${targetRepoName}`,
|
||||||
})
|
})
|
||||||
.where(eq(repositories.id, repository.id!));
|
.where(eq(repositories.id, repository.id!));
|
||||||
|
|
||||||
@@ -393,11 +415,11 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
const existingRepo = await getGiteaRepoInfo({
|
const existingRepo = await getGiteaRepoInfo({
|
||||||
config,
|
config,
|
||||||
owner: repoOwner,
|
owner: repoOwner,
|
||||||
repoName: repository.name,
|
repoName: targetRepoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (existingRepo && !existingRepo.mirror) {
|
if (existingRepo && !existingRepo.mirror) {
|
||||||
console.log(`Repository ${repository.name} exists but is not a mirror. Handling...`);
|
console.log(`Repository ${targetRepoName} exists but is not a mirror. Handling...`);
|
||||||
|
|
||||||
// Handle the existing non-mirror repository
|
// Handle the existing non-mirror repository
|
||||||
await handleExistingNonMirrorRepo({
|
await handleExistingNonMirrorRepo({
|
||||||
@@ -408,14 +430,14 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
});
|
});
|
||||||
|
|
||||||
// After handling, proceed with mirror creation
|
// After handling, proceed with mirror creation
|
||||||
console.log(`Proceeding with mirror creation for ${repository.name}`);
|
console.log(`Proceeding with mirror creation for ${targetRepoName}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await httpPost(
|
const response = await httpPost(
|
||||||
apiUrl,
|
apiUrl,
|
||||||
{
|
{
|
||||||
clone_addr: cloneAddress,
|
clone_addr: cloneAddress,
|
||||||
repo_name: repository.name,
|
repo_name: targetRepoName,
|
||||||
mirror: true,
|
mirror: true,
|
||||||
mirror_interval: config.giteaConfig?.mirrorInterval || "8h", // Set mirror interval
|
mirror_interval: config.giteaConfig?.mirrorInterval || "8h", // Set mirror interval
|
||||||
wiki: config.giteaConfig?.wiki || false, // will mirror wiki if it exists
|
wiki: config.giteaConfig?.wiki || false, // will mirror wiki if it exists
|
||||||
@@ -460,6 +482,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: repoOwner,
|
giteaOwner: repoOwner,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored issues for ${repository.name}`);
|
console.log(`[Metadata] Successfully mirrored issues for ${repository.name}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -477,6 +500,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: repoOwner,
|
giteaOwner: repoOwner,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name}`);
|
console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -494,6 +518,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: repoOwner,
|
giteaOwner: repoOwner,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored labels for ${repository.name}`);
|
console.log(`[Metadata] Successfully mirrored labels for ${repository.name}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -511,6 +536,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: repoOwner,
|
giteaOwner: repoOwner,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored milestones for ${repository.name}`);
|
console.log(`[Metadata] Successfully mirrored milestones for ${repository.name}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -519,7 +545,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Repository ${repository.name} mirrored successfully`);
|
console.log(`Repository ${repository.name} mirrored successfully as ${targetRepoName}`);
|
||||||
|
|
||||||
// Mark repos as "mirrored" in DB
|
// Mark repos as "mirrored" in DB
|
||||||
await db
|
await db
|
||||||
@@ -529,7 +555,7 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
lastMirrored: new Date(),
|
lastMirrored: new Date(),
|
||||||
errorMessage: null,
|
errorMessage: null,
|
||||||
mirroredLocation: `${repoOwner}/${repository.name}`,
|
mirroredLocation: `${repoOwner}/${targetRepoName}`,
|
||||||
})
|
})
|
||||||
.where(eq(repositories.id, repository.id!));
|
.where(eq(repositories.id, repository.id!));
|
||||||
|
|
||||||
@@ -538,8 +564,8 @@ export const mirrorGithubRepoToGitea = async ({
|
|||||||
userId: config.userId,
|
userId: config.userId,
|
||||||
repositoryId: repository.id,
|
repositoryId: repository.id,
|
||||||
repositoryName: repository.name,
|
repositoryName: repository.name,
|
||||||
message: `Successfully mirrored repository: ${repository.name}`,
|
message: `Successfully mirrored repository: ${repository.name}${targetRepoName !== repository.name ? ` as ${targetRepoName}` : ''}`,
|
||||||
details: `Repository ${repository.name} was mirrored to Gitea.`,
|
details: `Repository ${repository.fullName} was mirrored to Gitea at ${repoOwner}/${targetRepoName}.`,
|
||||||
status: "mirrored",
|
status: "mirrored",
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -608,6 +634,80 @@ export async function getOrCreateGiteaOrg({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a unique repository name for starred repos with duplicate names
|
||||||
|
*/
|
||||||
|
async function generateUniqueRepoName({
|
||||||
|
config,
|
||||||
|
orgName,
|
||||||
|
baseName,
|
||||||
|
githubOwner,
|
||||||
|
strategy,
|
||||||
|
}: {
|
||||||
|
config: Partial<Config>;
|
||||||
|
orgName: string;
|
||||||
|
baseName: string;
|
||||||
|
githubOwner: string;
|
||||||
|
strategy?: string;
|
||||||
|
}): Promise<string> {
|
||||||
|
const duplicateStrategy = strategy || "suffix";
|
||||||
|
|
||||||
|
// First check if base name is available
|
||||||
|
const baseExists = await isRepoPresentInGitea({
|
||||||
|
config,
|
||||||
|
owner: orgName,
|
||||||
|
repoName: baseName,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!baseExists) {
|
||||||
|
return baseName;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate name based on strategy
|
||||||
|
let candidateName: string;
|
||||||
|
let attempt = 0;
|
||||||
|
const maxAttempts = 10;
|
||||||
|
|
||||||
|
while (attempt < maxAttempts) {
|
||||||
|
switch (duplicateStrategy) {
|
||||||
|
case "prefix":
|
||||||
|
// Prefix with owner: owner-reponame
|
||||||
|
candidateName = attempt === 0
|
||||||
|
? `${githubOwner}-${baseName}`
|
||||||
|
: `${githubOwner}-${baseName}-${attempt}`;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case "owner-org":
|
||||||
|
// This would require creating sub-organizations, not supported in this PR
|
||||||
|
// Fall back to suffix strategy
|
||||||
|
case "suffix":
|
||||||
|
default:
|
||||||
|
// Suffix with owner: reponame-owner
|
||||||
|
candidateName = attempt === 0
|
||||||
|
? `${baseName}-${githubOwner}`
|
||||||
|
: `${baseName}-${githubOwner}-${attempt}`;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const exists = await isRepoPresentInGitea({
|
||||||
|
config,
|
||||||
|
owner: orgName,
|
||||||
|
repoName: candidateName,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!exists) {
|
||||||
|
console.log(`Found unique name for duplicate starred repo: ${candidateName}`);
|
||||||
|
return candidateName;
|
||||||
|
}
|
||||||
|
|
||||||
|
attempt++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If all attempts failed, use timestamp as last resort
|
||||||
|
const timestamp = Date.now();
|
||||||
|
return `${baseName}-${githubOwner}-${timestamp}`;
|
||||||
|
}
|
||||||
|
|
||||||
export async function mirrorGitHubRepoToGiteaOrg({
|
export async function mirrorGitHubRepoToGiteaOrg({
|
||||||
octokit,
|
octokit,
|
||||||
config,
|
config,
|
||||||
@@ -633,15 +733,37 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
// Decrypt config tokens for API usage
|
// Decrypt config tokens for API usage
|
||||||
const decryptedConfig = decryptConfigTokens(config as Config);
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
||||||
|
|
||||||
|
// Determine the actual repository name to use (handle duplicates for starred repos)
|
||||||
|
let targetRepoName = repository.name;
|
||||||
|
|
||||||
|
if (repository.isStarred && config.githubConfig) {
|
||||||
|
// Extract GitHub owner from full_name (format: owner/repo)
|
||||||
|
const githubOwner = repository.fullName.split('/')[0];
|
||||||
|
|
||||||
|
targetRepoName = await generateUniqueRepoName({
|
||||||
|
config,
|
||||||
|
orgName,
|
||||||
|
baseName: repository.name,
|
||||||
|
githubOwner,
|
||||||
|
strategy: config.githubConfig.starredDuplicateStrategy,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (targetRepoName !== repository.name) {
|
||||||
|
console.log(
|
||||||
|
`Starred repo ${repository.fullName} will be mirrored as ${orgName}/${targetRepoName} to avoid naming conflict`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const isExisting = await isRepoPresentInGitea({
|
const isExisting = await isRepoPresentInGitea({
|
||||||
config,
|
config,
|
||||||
owner: orgName,
|
owner: orgName,
|
||||||
repoName: repository.name,
|
repoName: targetRepoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (isExisting) {
|
if (isExisting) {
|
||||||
console.log(
|
console.log(
|
||||||
`Repository ${repository.name} already exists in Gitea organization ${orgName}. Updating database status.`
|
`Repository ${targetRepoName} already exists in Gitea organization ${orgName}. Updating database status.`
|
||||||
);
|
);
|
||||||
|
|
||||||
// Update database to reflect that the repository is already mirrored
|
// Update database to reflect that the repository is already mirrored
|
||||||
@@ -652,7 +774,7 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
lastMirrored: new Date(),
|
lastMirrored: new Date(),
|
||||||
errorMessage: null,
|
errorMessage: null,
|
||||||
mirroredLocation: `${orgName}/${repository.name}`,
|
mirroredLocation: `${orgName}/${targetRepoName}`,
|
||||||
})
|
})
|
||||||
.where(eq(repositories.id, repository.id!));
|
.where(eq(repositories.id, repository.id!));
|
||||||
|
|
||||||
@@ -661,19 +783,19 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
userId: config.userId,
|
userId: config.userId,
|
||||||
repositoryId: repository.id,
|
repositoryId: repository.id,
|
||||||
repositoryName: repository.name,
|
repositoryName: repository.name,
|
||||||
message: `Repository ${repository.name} already exists in Gitea organization ${orgName}`,
|
message: `Repository ${targetRepoName} already exists in Gitea organization ${orgName}`,
|
||||||
details: `Repository ${repository.name} was found to already exist in Gitea organization ${orgName} and database status was updated.`,
|
details: `Repository ${targetRepoName} was found to already exist in Gitea organization ${orgName} and database status was updated.`,
|
||||||
status: "mirrored",
|
status: "mirrored",
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Repository ${repository.name} database status updated to mirrored in organization ${orgName}`
|
`Repository ${targetRepoName} database status updated to mirrored in organization ${orgName}`
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Mirroring repository ${repository.name} to organization ${orgName}`
|
`Mirroring repository ${repository.fullName} to organization ${orgName} as ${targetRepoName}`
|
||||||
);
|
);
|
||||||
|
|
||||||
let cloneAddress = repository.cloneUrl;
|
let cloneAddress = repository.cloneUrl;
|
||||||
@@ -710,7 +832,7 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
{
|
{
|
||||||
clone_addr: cloneAddress,
|
clone_addr: cloneAddress,
|
||||||
uid: giteaOrgId,
|
uid: giteaOrgId,
|
||||||
repo_name: repository.name,
|
repo_name: targetRepoName,
|
||||||
mirror: true,
|
mirror: true,
|
||||||
mirror_interval: config.giteaConfig?.mirrorInterval || "8h", // Set mirror interval
|
mirror_interval: config.giteaConfig?.mirrorInterval || "8h", // Set mirror interval
|
||||||
wiki: config.giteaConfig?.wiki || false, // will mirror wiki if it exists
|
wiki: config.giteaConfig?.wiki || false, // will mirror wiki if it exists
|
||||||
@@ -752,10 +874,11 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: orgName,
|
giteaOwner: orgName,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored issues for ${repository.name} to org ${orgName}`);
|
console.log(`[Metadata] Successfully mirrored issues for ${repository.name} to org ${orgName}/${targetRepoName}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[Metadata] Failed to mirror issues for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`);
|
console.error(`[Metadata] Failed to mirror issues for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
// Continue with other metadata operations even if issues fail
|
// Continue with other metadata operations even if issues fail
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -769,10 +892,11 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: orgName,
|
giteaOwner: orgName,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name} to org ${orgName}`);
|
console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name} to org ${orgName}/${targetRepoName}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[Metadata] Failed to mirror pull requests for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`);
|
console.error(`[Metadata] Failed to mirror pull requests for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
// Continue with other metadata operations even if PRs fail
|
// Continue with other metadata operations even if PRs fail
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -786,10 +910,11 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: orgName,
|
giteaOwner: orgName,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored labels for ${repository.name} to org ${orgName}`);
|
console.log(`[Metadata] Successfully mirrored labels for ${repository.name} to org ${orgName}/${targetRepoName}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[Metadata] Failed to mirror labels for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`);
|
console.error(`[Metadata] Failed to mirror labels for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
// Continue with other metadata operations even if labels fail
|
// Continue with other metadata operations even if labels fail
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -803,16 +928,17 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner: orgName,
|
giteaOwner: orgName,
|
||||||
|
giteaRepoName: targetRepoName,
|
||||||
});
|
});
|
||||||
console.log(`[Metadata] Successfully mirrored milestones for ${repository.name} to org ${orgName}`);
|
console.log(`[Metadata] Successfully mirrored milestones for ${repository.name} to org ${orgName}/${targetRepoName}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[Metadata] Failed to mirror milestones for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`);
|
console.error(`[Metadata] Failed to mirror milestones for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
// Continue with other metadata operations even if milestones fail
|
// Continue with other metadata operations even if milestones fail
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Repository ${repository.name} mirrored successfully to organization ${orgName}`
|
`Repository ${repository.name} mirrored successfully to organization ${orgName} as ${targetRepoName}`
|
||||||
);
|
);
|
||||||
|
|
||||||
// Mark repos as "mirrored" in DB
|
// Mark repos as "mirrored" in DB
|
||||||
@@ -823,7 +949,7 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
lastMirrored: new Date(),
|
lastMirrored: new Date(),
|
||||||
errorMessage: null,
|
errorMessage: null,
|
||||||
mirroredLocation: `${orgName}/${repository.name}`,
|
mirroredLocation: `${orgName}/${targetRepoName}`,
|
||||||
})
|
})
|
||||||
.where(eq(repositories.id, repository.id!));
|
.where(eq(repositories.id, repository.id!));
|
||||||
|
|
||||||
@@ -832,8 +958,8 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
|||||||
userId: config.userId,
|
userId: config.userId,
|
||||||
repositoryId: repository.id,
|
repositoryId: repository.id,
|
||||||
repositoryName: repository.name,
|
repositoryName: repository.name,
|
||||||
message: `Repository ${repository.name} mirrored successfully`,
|
message: `Repository ${repository.name} mirrored successfully${targetRepoName !== repository.name ? ` as ${targetRepoName}` : ''}`,
|
||||||
details: `Repository ${repository.name} was mirrored to Gitea`,
|
details: `Repository ${repository.fullName} was mirrored to Gitea at ${orgName}/${targetRepoName}`,
|
||||||
status: "mirrored",
|
status: "mirrored",
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1149,11 +1275,13 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner,
|
giteaOwner,
|
||||||
|
giteaRepoName,
|
||||||
}: {
|
}: {
|
||||||
config: Partial<Config>;
|
config: Partial<Config>;
|
||||||
octokit: Octokit;
|
octokit: Octokit;
|
||||||
repository: Repository;
|
repository: Repository;
|
||||||
giteaOwner: string;
|
giteaOwner: string;
|
||||||
|
giteaRepoName?: string;
|
||||||
}) => {
|
}) => {
|
||||||
//things covered here are- issue, title, body, labels, comments and assignees
|
//things covered here are- issue, title, body, labels, comments and assignees
|
||||||
if (
|
if (
|
||||||
@@ -1168,23 +1296,26 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
|||||||
// Decrypt config tokens for API usage
|
// Decrypt config tokens for API usage
|
||||||
const decryptedConfig = decryptConfigTokens(config as Config);
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
||||||
|
|
||||||
|
// Use provided giteaRepoName or fall back to repository.name
|
||||||
|
const repoName = giteaRepoName || repository.name;
|
||||||
|
|
||||||
// Log configuration details for debugging
|
// Log configuration details for debugging
|
||||||
console.log(`[Issues] Starting issue mirroring for repository ${repository.name}`);
|
console.log(`[Issues] Starting issue mirroring for repository ${repository.name} as ${repoName}`);
|
||||||
console.log(`[Issues] Gitea URL: ${config.giteaConfig!.url}`);
|
console.log(`[Issues] Gitea URL: ${config.giteaConfig!.url}`);
|
||||||
console.log(`[Issues] Gitea Owner: ${giteaOwner}`);
|
console.log(`[Issues] Gitea Owner: ${giteaOwner}`);
|
||||||
console.log(`[Issues] Gitea Default Owner: ${config.giteaConfig!.defaultOwner}`);
|
console.log(`[Issues] Gitea Default Owner: ${config.giteaConfig!.defaultOwner}`);
|
||||||
|
|
||||||
// Verify the repository exists in Gitea before attempting to mirror metadata
|
// Verify the repository exists in Gitea before attempting to mirror metadata
|
||||||
console.log(`[Issues] Verifying repository ${repository.name} exists at ${giteaOwner}`);
|
console.log(`[Issues] Verifying repository ${repoName} exists at ${giteaOwner}`);
|
||||||
const repoExists = await isRepoPresentInGitea({
|
const repoExists = await isRepoPresentInGitea({
|
||||||
config,
|
config,
|
||||||
owner: giteaOwner,
|
owner: giteaOwner,
|
||||||
repoName: repository.name,
|
repoName: repoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!repoExists) {
|
if (!repoExists) {
|
||||||
console.error(`[Issues] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror issues.`);
|
console.error(`[Issues] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror issues.`);
|
||||||
throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [owner, repo] = repository.fullName.split("/");
|
const [owner, repo] = repository.fullName.split("/");
|
||||||
@@ -1215,7 +1346,7 @@ export const mirrorGitRepoIssuesToGitea = async ({
|
|||||||
|
|
||||||
// Get existing labels from Gitea
|
// Get existing labels from Gitea
|
||||||
const giteaLabelsRes = await httpGet(
|
const giteaLabelsRes = await httpGet(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`,
|
||||||
{
|
{
|
||||||
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
||||||
}
|
}
|
||||||
@@ -1556,11 +1687,13 @@ export async function mirrorGitRepoPullRequestsToGitea({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner,
|
giteaOwner,
|
||||||
|
giteaRepoName,
|
||||||
}: {
|
}: {
|
||||||
config: Partial<Config>;
|
config: Partial<Config>;
|
||||||
octokit: Octokit;
|
octokit: Octokit;
|
||||||
repository: Repository;
|
repository: Repository;
|
||||||
giteaOwner: string;
|
giteaOwner: string;
|
||||||
|
giteaRepoName?: string;
|
||||||
}) {
|
}) {
|
||||||
if (
|
if (
|
||||||
!config.githubConfig?.token ||
|
!config.githubConfig?.token ||
|
||||||
@@ -1574,23 +1707,26 @@ export async function mirrorGitRepoPullRequestsToGitea({
|
|||||||
// Decrypt config tokens for API usage
|
// Decrypt config tokens for API usage
|
||||||
const decryptedConfig = decryptConfigTokens(config as Config);
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
||||||
|
|
||||||
|
// Use provided giteaRepoName or fall back to repository.name
|
||||||
|
const repoName = giteaRepoName || repository.name;
|
||||||
|
|
||||||
// Log configuration details for debugging
|
// Log configuration details for debugging
|
||||||
console.log(`[Pull Requests] Starting PR mirroring for repository ${repository.name}`);
|
console.log(`[Pull Requests] Starting PR mirroring for repository ${repository.name} as ${repoName}`);
|
||||||
console.log(`[Pull Requests] Gitea URL: ${config.giteaConfig!.url}`);
|
console.log(`[Pull Requests] Gitea URL: ${config.giteaConfig!.url}`);
|
||||||
console.log(`[Pull Requests] Gitea Owner: ${giteaOwner}`);
|
console.log(`[Pull Requests] Gitea Owner: ${giteaOwner}`);
|
||||||
console.log(`[Pull Requests] Gitea Default Owner: ${config.giteaConfig!.defaultOwner}`);
|
console.log(`[Pull Requests] Gitea Default Owner: ${config.giteaConfig!.defaultOwner}`);
|
||||||
|
|
||||||
// Verify the repository exists in Gitea before attempting to mirror metadata
|
// Verify the repository exists in Gitea before attempting to mirror metadata
|
||||||
console.log(`[Pull Requests] Verifying repository ${repository.name} exists at ${giteaOwner}`);
|
console.log(`[Pull Requests] Verifying repository ${repoName} exists at ${giteaOwner}`);
|
||||||
const repoExists = await isRepoPresentInGitea({
|
const repoExists = await isRepoPresentInGitea({
|
||||||
config,
|
config,
|
||||||
owner: giteaOwner,
|
owner: giteaOwner,
|
||||||
repoName: repository.name,
|
repoName: repoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!repoExists) {
|
if (!repoExists) {
|
||||||
console.error(`[Pull Requests] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror PRs.`);
|
console.error(`[Pull Requests] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror PRs.`);
|
||||||
throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [owner, repo] = repository.fullName.split("/");
|
const [owner, repo] = repository.fullName.split("/");
|
||||||
@@ -1622,7 +1758,7 @@ export async function mirrorGitRepoPullRequestsToGitea({
|
|||||||
|
|
||||||
// Get existing labels from Gitea and ensure "pull-request" label exists
|
// Get existing labels from Gitea and ensure "pull-request" label exists
|
||||||
const giteaLabelsRes = await httpGet(
|
const giteaLabelsRes = await httpGet(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`,
|
||||||
{
|
{
|
||||||
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
||||||
}
|
}
|
||||||
@@ -1640,7 +1776,7 @@ export async function mirrorGitRepoPullRequestsToGitea({
|
|||||||
} else {
|
} else {
|
||||||
try {
|
try {
|
||||||
const created = await httpPost(
|
const created = await httpPost(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`,
|
||||||
{
|
{
|
||||||
name: "pull-request",
|
name: "pull-request",
|
||||||
color: "#0366d6",
|
color: "#0366d6",
|
||||||
@@ -1744,7 +1880,7 @@ export async function mirrorGitRepoPullRequestsToGitea({
|
|||||||
|
|
||||||
console.log(`[Pull Requests] Creating enriched issue for PR #${pr.number}: ${pr.title}`);
|
console.log(`[Pull Requests] Creating enriched issue for PR #${pr.number}: ${pr.title}`);
|
||||||
await httpPost(
|
await httpPost(
|
||||||
`${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repository.name}/issues`,
|
`${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues`,
|
||||||
issueData,
|
issueData,
|
||||||
{
|
{
|
||||||
Authorization: `token ${decryptedConfig.giteaConfig!.token}`,
|
Authorization: `token ${decryptedConfig.giteaConfig!.token}`,
|
||||||
@@ -1764,7 +1900,7 @@ export async function mirrorGitRepoPullRequestsToGitea({
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await httpPost(
|
await httpPost(
|
||||||
`${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repository.name}/issues`,
|
`${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues`,
|
||||||
basicIssueData,
|
basicIssueData,
|
||||||
{
|
{
|
||||||
Authorization: `token ${decryptedConfig.giteaConfig!.token}`,
|
Authorization: `token ${decryptedConfig.giteaConfig!.token}`,
|
||||||
@@ -1795,11 +1931,13 @@ export async function mirrorGitRepoLabelsToGitea({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner,
|
giteaOwner,
|
||||||
|
giteaRepoName,
|
||||||
}: {
|
}: {
|
||||||
config: Partial<Config>;
|
config: Partial<Config>;
|
||||||
octokit: Octokit;
|
octokit: Octokit;
|
||||||
repository: Repository;
|
repository: Repository;
|
||||||
giteaOwner: string;
|
giteaOwner: string;
|
||||||
|
giteaRepoName?: string;
|
||||||
}) {
|
}) {
|
||||||
if (
|
if (
|
||||||
!config.githubConfig?.token ||
|
!config.githubConfig?.token ||
|
||||||
@@ -1812,17 +1950,20 @@ export async function mirrorGitRepoLabelsToGitea({
|
|||||||
// Decrypt config tokens for API usage
|
// Decrypt config tokens for API usage
|
||||||
const decryptedConfig = decryptConfigTokens(config as Config);
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
||||||
|
|
||||||
|
// Use provided giteaRepoName or fall back to repository.name
|
||||||
|
const repoName = giteaRepoName || repository.name;
|
||||||
|
|
||||||
// Verify the repository exists in Gitea before attempting to mirror metadata
|
// Verify the repository exists in Gitea before attempting to mirror metadata
|
||||||
console.log(`[Labels] Verifying repository ${repository.name} exists at ${giteaOwner}`);
|
console.log(`[Labels] Verifying repository ${repoName} exists at ${giteaOwner}`);
|
||||||
const repoExists = await isRepoPresentInGitea({
|
const repoExists = await isRepoPresentInGitea({
|
||||||
config,
|
config,
|
||||||
owner: giteaOwner,
|
owner: giteaOwner,
|
||||||
repoName: repository.name,
|
repoName: repoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!repoExists) {
|
if (!repoExists) {
|
||||||
console.error(`[Labels] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror labels.`);
|
console.error(`[Labels] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror labels.`);
|
||||||
throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [owner, repo] = repository.fullName.split("/");
|
const [owner, repo] = repository.fullName.split("/");
|
||||||
@@ -1847,7 +1988,7 @@ export async function mirrorGitRepoLabelsToGitea({
|
|||||||
|
|
||||||
// Get existing labels from Gitea
|
// Get existing labels from Gitea
|
||||||
const giteaLabelsRes = await httpGet(
|
const giteaLabelsRes = await httpGet(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`,
|
||||||
{
|
{
|
||||||
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
||||||
}
|
}
|
||||||
@@ -1862,7 +2003,7 @@ export async function mirrorGitRepoLabelsToGitea({
|
|||||||
if (!existingLabels.has(label.name)) {
|
if (!existingLabels.has(label.name)) {
|
||||||
try {
|
try {
|
||||||
await httpPost(
|
await httpPost(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`,
|
||||||
{
|
{
|
||||||
name: label.name,
|
name: label.name,
|
||||||
color: `#${label.color}`,
|
color: `#${label.color}`,
|
||||||
@@ -1889,11 +2030,13 @@ export async function mirrorGitRepoMilestonesToGitea({
|
|||||||
octokit,
|
octokit,
|
||||||
repository,
|
repository,
|
||||||
giteaOwner,
|
giteaOwner,
|
||||||
|
giteaRepoName,
|
||||||
}: {
|
}: {
|
||||||
config: Partial<Config>;
|
config: Partial<Config>;
|
||||||
octokit: Octokit;
|
octokit: Octokit;
|
||||||
repository: Repository;
|
repository: Repository;
|
||||||
giteaOwner: string;
|
giteaOwner: string;
|
||||||
|
giteaRepoName?: string;
|
||||||
}) {
|
}) {
|
||||||
if (
|
if (
|
||||||
!config.githubConfig?.token ||
|
!config.githubConfig?.token ||
|
||||||
@@ -1906,17 +2049,20 @@ export async function mirrorGitRepoMilestonesToGitea({
|
|||||||
// Decrypt config tokens for API usage
|
// Decrypt config tokens for API usage
|
||||||
const decryptedConfig = decryptConfigTokens(config as Config);
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
||||||
|
|
||||||
|
// Use provided giteaRepoName or fall back to repository.name
|
||||||
|
const repoName = giteaRepoName || repository.name;
|
||||||
|
|
||||||
// Verify the repository exists in Gitea before attempting to mirror metadata
|
// Verify the repository exists in Gitea before attempting to mirror metadata
|
||||||
console.log(`[Milestones] Verifying repository ${repository.name} exists at ${giteaOwner}`);
|
console.log(`[Milestones] Verifying repository ${repoName} exists at ${giteaOwner}`);
|
||||||
const repoExists = await isRepoPresentInGitea({
|
const repoExists = await isRepoPresentInGitea({
|
||||||
config,
|
config,
|
||||||
owner: giteaOwner,
|
owner: giteaOwner,
|
||||||
repoName: repository.name,
|
repoName: repoName,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!repoExists) {
|
if (!repoExists) {
|
||||||
console.error(`[Milestones] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror milestones.`);
|
console.error(`[Milestones] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror milestones.`);
|
||||||
throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [owner, repo] = repository.fullName.split("/");
|
const [owner, repo] = repository.fullName.split("/");
|
||||||
@@ -1942,7 +2088,7 @@ export async function mirrorGitRepoMilestonesToGitea({
|
|||||||
|
|
||||||
// Get existing milestones from Gitea
|
// Get existing milestones from Gitea
|
||||||
const giteaMilestonesRes = await httpGet(
|
const giteaMilestonesRes = await httpGet(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/milestones`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/milestones`,
|
||||||
{
|
{
|
||||||
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
Authorization: `token ${decryptedConfig.giteaConfig.token}`,
|
||||||
}
|
}
|
||||||
@@ -1957,7 +2103,7 @@ export async function mirrorGitRepoMilestonesToGitea({
|
|||||||
if (!existingMilestones.has(milestone.title)) {
|
if (!existingMilestones.has(milestone.title)) {
|
||||||
try {
|
try {
|
||||||
await httpPost(
|
await httpPost(
|
||||||
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/milestones`,
|
`${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/milestones`,
|
||||||
{
|
{
|
||||||
title: milestone.title,
|
title: milestone.title,
|
||||||
description: milestone.description || "",
|
description: milestone.description || "",
|
||||||
|
|||||||
@@ -18,8 +18,11 @@ if (process.env.NODE_ENV !== "test") {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extend Octokit with throttling plugin
|
// Extend Octokit with throttling plugin when available (tests may stub Octokit)
|
||||||
const MyOctokit = Octokit.plugin(throttling);
|
// Fallback to base Octokit if .plugin is not present
|
||||||
|
const MyOctokit: any = (Octokit as any)?.plugin?.call
|
||||||
|
? (Octokit as any).plugin(throttling)
|
||||||
|
: Octokit as any;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates an authenticated Octokit instance with rate limit tracking and throttling
|
* Creates an authenticated Octokit instance with rate limit tracking and throttling
|
||||||
|
|||||||
75
src/lib/repo-utils.test.ts
Normal file
75
src/lib/repo-utils.test.ts
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
import { describe, it, expect } from 'bun:test';
|
||||||
|
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
|
||||||
|
import type { GitRepo } from '@/types/Repository';
|
||||||
|
|
||||||
|
function sampleRepo(overrides: Partial<GitRepo> = {}): GitRepo {
|
||||||
|
const base: GitRepo = {
|
||||||
|
name: 'repo',
|
||||||
|
fullName: 'owner/repo',
|
||||||
|
url: 'https://github.com/owner/repo',
|
||||||
|
cloneUrl: 'https://github.com/owner/repo.git',
|
||||||
|
owner: 'owner',
|
||||||
|
organization: undefined,
|
||||||
|
mirroredLocation: '',
|
||||||
|
destinationOrg: null,
|
||||||
|
isPrivate: false,
|
||||||
|
isForked: false,
|
||||||
|
forkedFrom: undefined,
|
||||||
|
hasIssues: true,
|
||||||
|
isStarred: false,
|
||||||
|
isArchived: false,
|
||||||
|
size: 1,
|
||||||
|
hasLFS: false,
|
||||||
|
hasSubmodules: false,
|
||||||
|
language: null,
|
||||||
|
description: null,
|
||||||
|
defaultBranch: 'main',
|
||||||
|
visibility: 'public',
|
||||||
|
status: 'imported',
|
||||||
|
lastMirrored: undefined,
|
||||||
|
errorMessage: undefined,
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
};
|
||||||
|
return { ...base, ...overrides };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('mergeGitReposPreferStarred', () => {
|
||||||
|
it('keeps unique repos', () => {
|
||||||
|
const basic = [sampleRepo({ fullName: 'a/x', name: 'x' })];
|
||||||
|
const starred: GitRepo[] = [];
|
||||||
|
const merged = mergeGitReposPreferStarred(basic, starred);
|
||||||
|
expect(merged).toHaveLength(1);
|
||||||
|
expect(merged[0].fullName).toBe('a/x');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('prefers starred when duplicate exists', () => {
|
||||||
|
const basic = [sampleRepo({ fullName: 'a/x', name: 'x', isStarred: false })];
|
||||||
|
const starred = [sampleRepo({ fullName: 'a/x', name: 'x', isStarred: true })];
|
||||||
|
const merged = mergeGitReposPreferStarred(basic, starred);
|
||||||
|
expect(merged).toHaveLength(1);
|
||||||
|
expect(merged[0].isStarred).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('normalizeGitRepoToInsert', () => {
|
||||||
|
it('sets undefined optional fields to null', () => {
|
||||||
|
const repo = sampleRepo({ organization: undefined, forkedFrom: undefined, language: undefined, description: undefined, lastMirrored: undefined, errorMessage: undefined });
|
||||||
|
const insert = normalizeGitRepoToInsert(repo, { userId: 'u', configId: 'c' });
|
||||||
|
expect(insert.organization).toBeNull();
|
||||||
|
expect(insert.forkedFrom).toBeNull();
|
||||||
|
expect(insert.language).toBeNull();
|
||||||
|
expect(insert.description).toBeNull();
|
||||||
|
expect(insert.lastMirrored).toBeNull();
|
||||||
|
expect(insert.errorMessage).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('calcBatchSizeForInsert', () => {
|
||||||
|
it('respects 999 parameter limit', () => {
|
||||||
|
const batch = calcBatchSizeForInsert(29);
|
||||||
|
expect(batch).toBeGreaterThan(0);
|
||||||
|
expect(batch * 29).toBeLessThanOrEqual(999);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
71
src/lib/repo-utils.ts
Normal file
71
src/lib/repo-utils.ts
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
|
import type { GitRepo } from '@/types/Repository';
|
||||||
|
import { repositories } from '@/lib/db/schema';
|
||||||
|
|
||||||
|
export type RepoInsert = typeof repositories.$inferInsert;
|
||||||
|
|
||||||
|
// Merge lists and de-duplicate by fullName, preferring starred variant when present
|
||||||
|
export function mergeGitReposPreferStarred(
|
||||||
|
basicAndForked: GitRepo[],
|
||||||
|
starred: GitRepo[]
|
||||||
|
): GitRepo[] {
|
||||||
|
const map = new Map<string, GitRepo>();
|
||||||
|
for (const r of [...basicAndForked, ...starred]) {
|
||||||
|
const existing = map.get(r.fullName);
|
||||||
|
if (!existing || (!existing.isStarred && r.isStarred)) {
|
||||||
|
map.set(r.fullName, r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Array.from(map.values());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert a GitRepo to a normalized DB insert object with all nullable fields set
|
||||||
|
export function normalizeGitRepoToInsert(
|
||||||
|
repo: GitRepo,
|
||||||
|
{
|
||||||
|
userId,
|
||||||
|
configId,
|
||||||
|
}: { userId: string; configId: string }
|
||||||
|
): RepoInsert {
|
||||||
|
return {
|
||||||
|
id: uuidv4(),
|
||||||
|
userId,
|
||||||
|
configId,
|
||||||
|
name: repo.name,
|
||||||
|
fullName: repo.fullName,
|
||||||
|
url: repo.url,
|
||||||
|
cloneUrl: repo.cloneUrl,
|
||||||
|
owner: repo.owner,
|
||||||
|
organization: repo.organization ?? null,
|
||||||
|
mirroredLocation: repo.mirroredLocation || '',
|
||||||
|
destinationOrg: repo.destinationOrg || null,
|
||||||
|
isPrivate: repo.isPrivate,
|
||||||
|
isForked: repo.isForked,
|
||||||
|
forkedFrom: repo.forkedFrom ?? null,
|
||||||
|
hasIssues: repo.hasIssues,
|
||||||
|
isStarred: repo.isStarred,
|
||||||
|
isArchived: repo.isArchived,
|
||||||
|
size: repo.size,
|
||||||
|
hasLFS: repo.hasLFS,
|
||||||
|
hasSubmodules: repo.hasSubmodules,
|
||||||
|
language: repo.language ?? null,
|
||||||
|
description: repo.description ?? null,
|
||||||
|
defaultBranch: repo.defaultBranch,
|
||||||
|
visibility: repo.visibility,
|
||||||
|
status: 'imported',
|
||||||
|
lastMirrored: repo.lastMirrored ?? null,
|
||||||
|
errorMessage: repo.errorMessage ?? null,
|
||||||
|
createdAt: repo.createdAt || new Date(),
|
||||||
|
updatedAt: repo.updatedAt || new Date(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute a safe batch size based on SQLite 999-parameter limit
|
||||||
|
export function calcBatchSizeForInsert(columnCount: number, maxParams = 999): number {
|
||||||
|
if (columnCount <= 0) return 1;
|
||||||
|
// Reserve a little headroom in case column count drifts
|
||||||
|
const safety = 0;
|
||||||
|
const effectiveMax = Math.max(1, maxParams - safety);
|
||||||
|
return Math.max(1, Math.floor(effectiveMax / columnCount));
|
||||||
|
}
|
||||||
|
|
||||||
@@ -11,6 +11,7 @@ import { getDecryptedGitHubToken } from '@/lib/utils/config-encryption';
|
|||||||
import { parseInterval, formatDuration } from '@/lib/utils/duration-parser';
|
import { parseInterval, formatDuration } from '@/lib/utils/duration-parser';
|
||||||
import type { Repository } from '@/lib/db/schema';
|
import type { Repository } from '@/lib/db/schema';
|
||||||
import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository';
|
import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository';
|
||||||
|
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
|
||||||
|
|
||||||
let schedulerInterval: NodeJS.Timeout | null = null;
|
let schedulerInterval: NodeJS.Timeout | null = null;
|
||||||
let isSchedulerRunning = false;
|
let isSchedulerRunning = false;
|
||||||
@@ -94,8 +95,7 @@ async function runScheduledSync(config: any): Promise<void> {
|
|||||||
? getGithubStarredRepositories({ octokit, config })
|
? getGithubStarredRepositories({ octokit, config })
|
||||||
: Promise.resolve([]),
|
: Promise.resolve([]),
|
||||||
]);
|
]);
|
||||||
|
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||||
const allGithubRepos = [...basicAndForkedRepos, ...starredRepos];
|
|
||||||
|
|
||||||
// Check for new repositories
|
// Check for new repositories
|
||||||
const existingRepos = await db
|
const existingRepos = await db
|
||||||
@@ -110,37 +110,21 @@ async function runScheduledSync(config: any): Promise<void> {
|
|||||||
console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId}`);
|
console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId}`);
|
||||||
|
|
||||||
// Insert new repositories
|
// Insert new repositories
|
||||||
const reposToInsert = newRepos.map(repo => ({
|
const reposToInsert = newRepos.map(repo =>
|
||||||
id: uuidv4(),
|
normalizeGitRepoToInsert(repo, { userId, configId: config.id })
|
||||||
userId,
|
);
|
||||||
configId: config.id,
|
|
||||||
name: repo.name,
|
|
||||||
fullName: repo.fullName,
|
|
||||||
url: repo.url,
|
|
||||||
cloneUrl: repo.cloneUrl,
|
|
||||||
owner: repo.owner,
|
|
||||||
organization: repo.organization,
|
|
||||||
mirroredLocation: repo.mirroredLocation || "",
|
|
||||||
destinationOrg: repo.destinationOrg || null,
|
|
||||||
isPrivate: repo.isPrivate,
|
|
||||||
isForked: repo.isForked,
|
|
||||||
forkedFrom: repo.forkedFrom,
|
|
||||||
hasIssues: repo.hasIssues,
|
|
||||||
isStarred: repo.isStarred,
|
|
||||||
isArchived: repo.isArchived,
|
|
||||||
size: repo.size,
|
|
||||||
hasLFS: repo.hasLFS,
|
|
||||||
hasSubmodules: repo.hasSubmodules,
|
|
||||||
language: repo.language || null,
|
|
||||||
description: repo.description || null,
|
|
||||||
defaultBranch: repo.defaultBranch,
|
|
||||||
visibility: repo.visibility,
|
|
||||||
status: 'imported',
|
|
||||||
createdAt: new Date(),
|
|
||||||
updatedAt: new Date(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
await db.insert(repositories).values(reposToInsert);
|
// Batch insert to avoid SQLite parameter limit
|
||||||
|
const sample = reposToInsert[0];
|
||||||
|
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||||
|
const BATCH_SIZE = calcBatchSizeForInsert(columnCount);
|
||||||
|
for (let i = 0; i < reposToInsert.length; i += BATCH_SIZE) {
|
||||||
|
const batch = reposToInsert.slice(i, i + BATCH_SIZE);
|
||||||
|
await db
|
||||||
|
.insert(repositories)
|
||||||
|
.values(batch)
|
||||||
|
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
|
||||||
|
}
|
||||||
console.log(`[Scheduler] Successfully imported ${newRepos.length} new repositories for user ${userId}`);
|
console.log(`[Scheduler] Successfully imported ${newRepos.length} new repositories for user ${userId}`);
|
||||||
} else {
|
} else {
|
||||||
console.log(`[Scheduler] No new repositories found for user ${userId}`);
|
console.log(`[Scheduler] No new repositories found for user ${userId}`);
|
||||||
@@ -375,8 +359,7 @@ async function performInitialAutoStart(): Promise<void> {
|
|||||||
? getGithubStarredRepositories({ octokit, config })
|
? getGithubStarredRepositories({ octokit, config })
|
||||||
: Promise.resolve([]),
|
: Promise.resolve([]),
|
||||||
]);
|
]);
|
||||||
|
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||||
const allGithubRepos = [...basicAndForkedRepos, ...starredRepos];
|
|
||||||
|
|
||||||
// Check for new repositories
|
// Check for new repositories
|
||||||
const existingRepos = await db
|
const existingRepos = await db
|
||||||
@@ -391,37 +374,21 @@ async function performInitialAutoStart(): Promise<void> {
|
|||||||
console.log(`[Scheduler] Importing ${reposToImport.length} repositories for user ${config.userId}...`);
|
console.log(`[Scheduler] Importing ${reposToImport.length} repositories for user ${config.userId}...`);
|
||||||
|
|
||||||
// Insert new repositories
|
// Insert new repositories
|
||||||
const reposToInsert = reposToImport.map(repo => ({
|
const reposToInsert = reposToImport.map(repo =>
|
||||||
id: uuidv4(),
|
normalizeGitRepoToInsert(repo, { userId: config.userId, configId: config.id })
|
||||||
userId: config.userId,
|
);
|
||||||
configId: config.id,
|
|
||||||
name: repo.name,
|
|
||||||
fullName: repo.fullName,
|
|
||||||
url: repo.url,
|
|
||||||
cloneUrl: repo.cloneUrl,
|
|
||||||
owner: repo.owner,
|
|
||||||
organization: repo.organization,
|
|
||||||
mirroredLocation: repo.mirroredLocation || "",
|
|
||||||
destinationOrg: repo.destinationOrg || null,
|
|
||||||
isPrivate: repo.isPrivate,
|
|
||||||
isForked: repo.isForked,
|
|
||||||
forkedFrom: repo.forkedFrom,
|
|
||||||
hasIssues: repo.hasIssues,
|
|
||||||
isStarred: repo.isStarred,
|
|
||||||
isArchived: repo.isArchived,
|
|
||||||
size: repo.size,
|
|
||||||
hasLFS: repo.hasLFS,
|
|
||||||
hasSubmodules: repo.hasSubmodules,
|
|
||||||
language: repo.language || null,
|
|
||||||
description: repo.description || null,
|
|
||||||
defaultBranch: repo.defaultBranch,
|
|
||||||
visibility: repo.visibility,
|
|
||||||
status: 'imported',
|
|
||||||
createdAt: new Date(),
|
|
||||||
updatedAt: new Date(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
await db.insert(repositories).values(reposToInsert);
|
// Batch insert to avoid SQLite parameter limit
|
||||||
|
const sample = reposToInsert[0];
|
||||||
|
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||||
|
const BATCH_SIZE = calcBatchSizeForInsert(columnCount);
|
||||||
|
for (let i = 0; i < reposToInsert.length; i += BATCH_SIZE) {
|
||||||
|
const batch = reposToInsert.slice(i, i + BATCH_SIZE);
|
||||||
|
await db
|
||||||
|
.insert(repositories)
|
||||||
|
.values(batch)
|
||||||
|
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
|
||||||
|
}
|
||||||
console.log(`[Scheduler] Successfully imported ${reposToImport.length} repositories`);
|
console.log(`[Scheduler] Successfully imported ${reposToImport.length} repositories`);
|
||||||
} else {
|
} else {
|
||||||
console.log(`[Scheduler] No new repositories to import for user ${config.userId}`);
|
console.log(`[Scheduler] No new repositories to import for user ${config.userId}`);
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
getGithubStarredRepositories,
|
getGithubStarredRepositories,
|
||||||
} from "@/lib/github";
|
} from "@/lib/github";
|
||||||
import { jsonResponse, createSecureErrorResponse } from "@/lib/utils";
|
import { jsonResponse, createSecureErrorResponse } from "@/lib/utils";
|
||||||
|
import { mergeGitReposPreferStarred, calcBatchSizeForInsert } from "@/lib/repo-utils";
|
||||||
import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption";
|
import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption";
|
||||||
|
|
||||||
export const POST: APIRoute = async ({ request }) => {
|
export const POST: APIRoute = async ({ request }) => {
|
||||||
@@ -55,7 +56,8 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
getGithubOrganizations({ octokit, config }),
|
getGithubOrganizations({ octokit, config }),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const allGithubRepos = [...basicAndForkedRepos, ...starredRepos];
|
// Merge and de-duplicate by fullName, preferring starred variant when duplicated
|
||||||
|
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||||
|
|
||||||
// Prepare full list of repos and orgs
|
// Prepare full list of repos and orgs
|
||||||
const newRepos = allGithubRepos.map((repo) => ({
|
const newRepos = allGithubRepos.map((repo) => ({
|
||||||
@@ -67,25 +69,25 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
url: repo.url,
|
url: repo.url,
|
||||||
cloneUrl: repo.cloneUrl,
|
cloneUrl: repo.cloneUrl,
|
||||||
owner: repo.owner,
|
owner: repo.owner,
|
||||||
organization: repo.organization,
|
organization: repo.organization ?? null,
|
||||||
mirroredLocation: repo.mirroredLocation || "",
|
mirroredLocation: repo.mirroredLocation || "",
|
||||||
destinationOrg: repo.destinationOrg || null,
|
destinationOrg: repo.destinationOrg || null,
|
||||||
isPrivate: repo.isPrivate,
|
isPrivate: repo.isPrivate,
|
||||||
isForked: repo.isForked,
|
isForked: repo.isForked,
|
||||||
forkedFrom: repo.forkedFrom,
|
forkedFrom: repo.forkedFrom ?? null,
|
||||||
hasIssues: repo.hasIssues,
|
hasIssues: repo.hasIssues,
|
||||||
isStarred: repo.isStarred,
|
isStarred: repo.isStarred,
|
||||||
isArchived: repo.isArchived,
|
isArchived: repo.isArchived,
|
||||||
size: repo.size,
|
size: repo.size,
|
||||||
hasLFS: repo.hasLFS,
|
hasLFS: repo.hasLFS,
|
||||||
hasSubmodules: repo.hasSubmodules,
|
hasSubmodules: repo.hasSubmodules,
|
||||||
language: repo.language || null,
|
language: repo.language ?? null,
|
||||||
description: repo.description || null,
|
description: repo.description ?? null,
|
||||||
defaultBranch: repo.defaultBranch,
|
defaultBranch: repo.defaultBranch,
|
||||||
visibility: repo.visibility,
|
visibility: repo.visibility,
|
||||||
status: repo.status,
|
status: repo.status,
|
||||||
lastMirrored: repo.lastMirrored,
|
lastMirrored: repo.lastMirrored ?? null,
|
||||||
errorMessage: repo.errorMessage,
|
errorMessage: repo.errorMessage ?? null,
|
||||||
createdAt: repo.createdAt,
|
createdAt: repo.createdAt,
|
||||||
updatedAt: repo.updatedAt,
|
updatedAt: repo.updatedAt,
|
||||||
}));
|
}));
|
||||||
@@ -128,12 +130,27 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
);
|
);
|
||||||
insertedOrgs = newOrgs.filter((o) => !existingOrgNames.has(o.name));
|
insertedOrgs = newOrgs.filter((o) => !existingOrgNames.has(o.name));
|
||||||
|
|
||||||
|
// Batch insert repositories to avoid SQLite parameter limit (dynamic by column count)
|
||||||
|
const sample = newRepos[0];
|
||||||
|
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||||
|
const REPO_BATCH_SIZE = calcBatchSizeForInsert(columnCount);
|
||||||
if (insertedRepos.length > 0) {
|
if (insertedRepos.length > 0) {
|
||||||
await tx.insert(repositories).values(insertedRepos);
|
for (let i = 0; i < insertedRepos.length; i += REPO_BATCH_SIZE) {
|
||||||
|
const batch = insertedRepos.slice(i, i + REPO_BATCH_SIZE);
|
||||||
|
await tx
|
||||||
|
.insert(repositories)
|
||||||
|
.values(batch)
|
||||||
|
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Batch insert organizations (they have fewer fields, so we can use larger batches)
|
||||||
|
const ORG_BATCH_SIZE = 100;
|
||||||
if (insertedOrgs.length > 0) {
|
if (insertedOrgs.length > 0) {
|
||||||
await tx.insert(organizations).values(insertedOrgs);
|
for (let i = 0; i < insertedOrgs.length; i += ORG_BATCH_SIZE) {
|
||||||
|
const batch = insertedOrgs.slice(i, i + ORG_BATCH_SIZE);
|
||||||
|
await tx.insert(organizations).values(batch);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -122,25 +122,36 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
destinationOrg: null,
|
destinationOrg: null,
|
||||||
isPrivate: repo.private,
|
isPrivate: repo.private,
|
||||||
isForked: repo.fork,
|
isForked: repo.fork,
|
||||||
forkedFrom: undefined,
|
forkedFrom: null,
|
||||||
hasIssues: repo.has_issues,
|
hasIssues: repo.has_issues,
|
||||||
isStarred: false,
|
isStarred: false,
|
||||||
isArchived: repo.archived,
|
isArchived: repo.archived,
|
||||||
size: repo.size,
|
size: repo.size,
|
||||||
hasLFS: false,
|
hasLFS: false,
|
||||||
hasSubmodules: false,
|
hasSubmodules: false,
|
||||||
language: repo.language || null,
|
language: repo.language ?? null,
|
||||||
description: repo.description || null,
|
description: repo.description ?? null,
|
||||||
defaultBranch: repo.default_branch ?? "main",
|
defaultBranch: repo.default_branch ?? "main",
|
||||||
visibility: (repo.visibility ?? "public") as RepositoryVisibility,
|
visibility: (repo.visibility ?? "public") as RepositoryVisibility,
|
||||||
status: "imported" as RepoStatus,
|
status: "imported" as RepoStatus,
|
||||||
lastMirrored: undefined,
|
lastMirrored: null,
|
||||||
errorMessage: undefined,
|
errorMessage: null,
|
||||||
createdAt: repo.created_at ? new Date(repo.created_at) : new Date(),
|
createdAt: repo.created_at ? new Date(repo.created_at) : new Date(),
|
||||||
updatedAt: repo.updated_at ? new Date(repo.updated_at) : new Date(),
|
updatedAt: repo.updated_at ? new Date(repo.updated_at) : new Date(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await db.insert(repositories).values(repoRecords);
|
// Batch insert repositories to avoid SQLite parameter limit
|
||||||
|
// Compute batch size based on column count
|
||||||
|
const sample = repoRecords[0];
|
||||||
|
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||||
|
const BATCH_SIZE = Math.max(1, Math.floor(999 / columnCount));
|
||||||
|
for (let i = 0; i < repoRecords.length; i += BATCH_SIZE) {
|
||||||
|
const batch = repoRecords.slice(i, i + BATCH_SIZE);
|
||||||
|
await db
|
||||||
|
.insert(repositories)
|
||||||
|
.values(batch)
|
||||||
|
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
|
||||||
|
}
|
||||||
|
|
||||||
// Insert organization metadata
|
// Insert organization metadata
|
||||||
const organizationRecord = {
|
const organizationRecord = {
|
||||||
|
|||||||
@@ -80,25 +80,23 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
cloneUrl: repoData.clone_url,
|
cloneUrl: repoData.clone_url,
|
||||||
owner: repoData.owner.login,
|
owner: repoData.owner.login,
|
||||||
organization:
|
organization:
|
||||||
repoData.owner.type === "Organization"
|
repoData.owner.type === "Organization" ? repoData.owner.login : null,
|
||||||
? repoData.owner.login
|
|
||||||
: undefined,
|
|
||||||
isPrivate: repoData.private,
|
isPrivate: repoData.private,
|
||||||
isForked: repoData.fork,
|
isForked: repoData.fork,
|
||||||
forkedFrom: undefined,
|
forkedFrom: null,
|
||||||
hasIssues: repoData.has_issues,
|
hasIssues: repoData.has_issues,
|
||||||
isStarred: false,
|
isStarred: false,
|
||||||
isArchived: repoData.archived,
|
isArchived: repoData.archived,
|
||||||
size: repoData.size,
|
size: repoData.size,
|
||||||
hasLFS: false,
|
hasLFS: false,
|
||||||
hasSubmodules: false,
|
hasSubmodules: false,
|
||||||
language: repoData.language || null,
|
language: repoData.language ?? null,
|
||||||
description: repoData.description || null,
|
description: repoData.description ?? null,
|
||||||
defaultBranch: repoData.default_branch,
|
defaultBranch: repoData.default_branch,
|
||||||
visibility: (repoData.visibility ?? "public") as RepositoryVisibility,
|
visibility: (repoData.visibility ?? "public") as RepositoryVisibility,
|
||||||
status: "imported" as Repository["status"],
|
status: "imported" as Repository["status"],
|
||||||
lastMirrored: undefined,
|
lastMirrored: null,
|
||||||
errorMessage: undefined,
|
errorMessage: null,
|
||||||
mirroredLocation: "",
|
mirroredLocation: "",
|
||||||
destinationOrg: null,
|
destinationOrg: null,
|
||||||
createdAt: repoData.created_at
|
createdAt: repoData.created_at
|
||||||
@@ -109,7 +107,10 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
: new Date(),
|
: new Date(),
|
||||||
};
|
};
|
||||||
|
|
||||||
await db.insert(repositories).values(metadata);
|
await db
|
||||||
|
.insert(repositories)
|
||||||
|
.values(metadata)
|
||||||
|
.onConflictDoNothing({ target: [repositories.userId, repositories.fullName] });
|
||||||
|
|
||||||
createMirrorJob({
|
createMirrorJob({
|
||||||
userId,
|
userId,
|
||||||
|
|||||||
@@ -29,11 +29,14 @@ export interface DatabaseCleanupConfig {
|
|||||||
nextRun?: Date;
|
nextRun?: Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type DuplicateNameStrategy = "suffix" | "prefix" | "owner-org";
|
||||||
|
|
||||||
export interface GitHubConfig {
|
export interface GitHubConfig {
|
||||||
username: string;
|
username: string;
|
||||||
token: string;
|
token: string;
|
||||||
privateRepositories: boolean;
|
privateRepositories: boolean;
|
||||||
mirrorStarred: boolean;
|
mirrorStarred: boolean;
|
||||||
|
starredDuplicateStrategy?: DuplicateNameStrategy;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MirrorOptions {
|
export interface MirrorOptions {
|
||||||
|
|||||||
Reference in New Issue
Block a user