mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2025-12-06 11:36:44 +03:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b53a29e71 | ||
|
|
64e73f9ca8 | ||
|
|
7d23894e5f | ||
|
|
8f2a4683c1 | ||
|
|
b5323ff8b4 | ||
|
|
7fee2adb51 | ||
|
|
af139ecb2d | ||
|
|
fb827724b6 | ||
|
|
2812b576d0 | ||
|
|
347188f43d | ||
|
|
56bee451de |
@@ -49,7 +49,7 @@ spec:
|
||||
terminationGracePeriodSeconds: {{ .Values.deployment.terminationGracePeriodSeconds }}
|
||||
containers:
|
||||
- name: gitea-mirror
|
||||
image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:v{{ .Values.image.tag | default .Chart.AppVersion | toString }}
|
||||
image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "v%s" .Chart.AppVersion) }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "3.8.3",
|
||||
"version": "3.8.4",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
|
||||
@@ -208,7 +208,7 @@ export const organizationSchema = z.object({
|
||||
configId: z.string(),
|
||||
name: z.string(),
|
||||
avatarUrl: z.string(),
|
||||
membershipRole: z.enum(["admin", "member", "owner"]).default("member"),
|
||||
membershipRole: z.enum(["member", "admin", "owner", "billing_manager"]).default("member"),
|
||||
isIncluded: z.boolean().default(true),
|
||||
destinationOrg: z.string().optional().nullable(),
|
||||
status: z
|
||||
|
||||
@@ -8,10 +8,19 @@ import { createMockResponse, mockFetch } from "@/tests/mock-fetch";
|
||||
// Mock the isRepoPresentInGitea function
|
||||
const mockIsRepoPresentInGitea = mock(() => Promise.resolve(false));
|
||||
|
||||
let mockDbSelectResult: any[] = [];
|
||||
|
||||
// Mock the database module
|
||||
mock.module("@/lib/db", () => {
|
||||
return {
|
||||
db: {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
where: () => ({
|
||||
limit: () => Promise.resolve(mockDbSelectResult)
|
||||
})
|
||||
})
|
||||
}),
|
||||
update: () => ({
|
||||
set: () => ({
|
||||
where: () => Promise.resolve()
|
||||
@@ -63,6 +72,7 @@ describe("Gitea Repository Mirroring", () => {
|
||||
originalConsoleError = console.error;
|
||||
console.log = mock(() => {});
|
||||
console.error = mock(() => {});
|
||||
mockDbSelectResult = [];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -449,4 +459,37 @@ describe("getGiteaRepoOwner - Organization Override Tests", () => {
|
||||
const result = getGiteaRepoOwner({ config: configWithFlatUser, repository: repo });
|
||||
expect(result).toBe("giteauser");
|
||||
});
|
||||
|
||||
test("getGiteaRepoOwnerAsync honors organization override for owner role", async () => {
|
||||
mockDbSelectResult = [
|
||||
{
|
||||
id: "org-id",
|
||||
userId: "user-id",
|
||||
configId: "config-id",
|
||||
name: "myorg",
|
||||
membershipRole: "owner",
|
||||
status: "imported",
|
||||
destinationOrg: "custom-org",
|
||||
avatarUrl: "https://example.com/avatar.png",
|
||||
isIncluded: true,
|
||||
repositoryCount: 0,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date()
|
||||
}
|
||||
];
|
||||
|
||||
const configWithUser: Partial<Config> = {
|
||||
...baseConfig,
|
||||
userId: "user-id"
|
||||
};
|
||||
|
||||
const repo = { ...baseRepo, organization: "myorg" };
|
||||
|
||||
const result = await getGiteaRepoOwnerAsync({
|
||||
config: configWithUser,
|
||||
repository: repo
|
||||
});
|
||||
|
||||
expect(result).toBe("custom-org");
|
||||
});
|
||||
});
|
||||
|
||||
207
src/lib/gitea.ts
207
src/lib/gitea.ts
@@ -200,6 +200,96 @@ export const isRepoPresentInGitea = async ({
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if a repository is currently being mirrored (in-progress state in database)
|
||||
* This prevents race conditions where multiple concurrent operations try to mirror the same repo
|
||||
*/
|
||||
export const isRepoCurrentlyMirroring = async ({
|
||||
config,
|
||||
repoName,
|
||||
expectedLocation,
|
||||
}: {
|
||||
config: Partial<Config>;
|
||||
repoName: string;
|
||||
expectedLocation?: string; // Format: "owner/repo"
|
||||
}): Promise<boolean> => {
|
||||
try {
|
||||
if (!config.userId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const { or } = await import("drizzle-orm");
|
||||
|
||||
// Check database for any repository with "mirroring" or "syncing" status
|
||||
const inProgressRepos = await db
|
||||
.select()
|
||||
.from(repositories)
|
||||
.where(
|
||||
and(
|
||||
eq(repositories.userId, config.userId),
|
||||
eq(repositories.name, repoName),
|
||||
// Check for in-progress statuses
|
||||
or(
|
||||
eq(repositories.status, "mirroring"),
|
||||
eq(repositories.status, "syncing")
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
if (inProgressRepos.length > 0) {
|
||||
// Check if any of the in-progress repos are stale (stuck for > 2 hours)
|
||||
const TWO_HOURS_MS = 2 * 60 * 60 * 1000;
|
||||
const now = new Date().getTime();
|
||||
|
||||
const activeRepos = inProgressRepos.filter((repo) => {
|
||||
if (!repo.updatedAt) return true; // No timestamp, assume active
|
||||
const updatedTime = new Date(repo.updatedAt).getTime();
|
||||
const isStale = (now - updatedTime) > TWO_HOURS_MS;
|
||||
|
||||
if (isStale) {
|
||||
console.warn(
|
||||
`[Idempotency] Repository ${repo.name} has been in "${repo.status}" status for over 2 hours. ` +
|
||||
`Considering it stale and allowing retry.`
|
||||
);
|
||||
}
|
||||
|
||||
return !isStale;
|
||||
});
|
||||
|
||||
if (activeRepos.length === 0) {
|
||||
console.log(
|
||||
`[Idempotency] All in-progress operations for ${repoName} are stale (>2h). Allowing retry.`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// If we have an expected location, verify it matches
|
||||
if (expectedLocation) {
|
||||
const matchingRepo = activeRepos.find(
|
||||
(repo) => repo.mirroredLocation === expectedLocation
|
||||
);
|
||||
if (matchingRepo) {
|
||||
console.log(
|
||||
`[Idempotency] Repository ${repoName} is already being mirrored at ${expectedLocation}`
|
||||
);
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`[Idempotency] Repository ${repoName} is already being mirrored (${activeRepos.length} in-progress operations found)`
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.error("Error checking if repo is currently mirroring:", error);
|
||||
console.error("Error details:", error);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Helper function to check if a repository exists in Gitea.
|
||||
* First checks the recorded mirroredLocation, then falls back to the expected location.
|
||||
@@ -276,11 +366,11 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
|
||||
// Determine the actual repository name to use (handle duplicates for starred repos)
|
||||
let targetRepoName = repository.name;
|
||||
|
||||
|
||||
if (repository.isStarred && config.githubConfig) {
|
||||
// Extract GitHub owner from full_name (format: owner/repo)
|
||||
const githubOwner = repository.fullName.split('/')[0];
|
||||
|
||||
|
||||
targetRepoName = await generateUniqueRepoName({
|
||||
config,
|
||||
orgName: repoOwner,
|
||||
@@ -288,7 +378,7 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
githubOwner,
|
||||
strategy: config.githubConfig.starredDuplicateStrategy,
|
||||
});
|
||||
|
||||
|
||||
if (targetRepoName !== repository.name) {
|
||||
console.log(
|
||||
`Starred repo ${repository.fullName} will be mirrored as ${repoOwner}/${targetRepoName} to avoid naming conflict`
|
||||
@@ -296,6 +386,23 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
}
|
||||
}
|
||||
|
||||
// IDEMPOTENCY CHECK: Check if this repo is already being mirrored
|
||||
const expectedLocation = `${repoOwner}/${targetRepoName}`;
|
||||
const isCurrentlyMirroring = await isRepoCurrentlyMirroring({
|
||||
config,
|
||||
repoName: targetRepoName,
|
||||
expectedLocation,
|
||||
});
|
||||
|
||||
if (isCurrentlyMirroring) {
|
||||
console.log(
|
||||
`[Idempotency] Skipping ${repository.fullName} - already being mirrored to ${expectedLocation}`
|
||||
);
|
||||
|
||||
// Don't throw an error, just return to allow other repos to continue
|
||||
return;
|
||||
}
|
||||
|
||||
const isExisting = await isRepoPresentInGitea({
|
||||
config,
|
||||
owner: repoOwner,
|
||||
@@ -337,11 +444,30 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
|
||||
console.log(`Mirroring repository ${repository.name}`);
|
||||
|
||||
// DOUBLE-CHECK: Final idempotency check right before updating status
|
||||
// This catches race conditions in the small window between first check and status update
|
||||
const finalCheck = await isRepoCurrentlyMirroring({
|
||||
config,
|
||||
repoName: targetRepoName,
|
||||
expectedLocation,
|
||||
});
|
||||
|
||||
if (finalCheck) {
|
||||
console.log(
|
||||
`[Idempotency] Race condition detected - ${repository.fullName} is now being mirrored by another process. Skipping.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark repos as "mirroring" in DB
|
||||
// CRITICAL: Set mirroredLocation NOW (not after success) so idempotency checks work
|
||||
// This becomes the "target location" - where we intend to mirror to
|
||||
// Without this, the idempotency check can't detect concurrent operations on first mirror
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("mirroring"),
|
||||
mirroredLocation: expectedLocation,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repository.id!));
|
||||
@@ -681,32 +807,32 @@ async function generateUniqueRepoName({
|
||||
strategy?: string;
|
||||
}): Promise<string> {
|
||||
const duplicateStrategy = strategy || "suffix";
|
||||
|
||||
|
||||
// First check if base name is available
|
||||
const baseExists = await isRepoPresentInGitea({
|
||||
config,
|
||||
owner: orgName,
|
||||
repoName: baseName,
|
||||
});
|
||||
|
||||
|
||||
if (!baseExists) {
|
||||
return baseName;
|
||||
}
|
||||
|
||||
|
||||
// Generate name based on strategy
|
||||
let candidateName: string;
|
||||
let attempt = 0;
|
||||
const maxAttempts = 10;
|
||||
|
||||
|
||||
while (attempt < maxAttempts) {
|
||||
switch (duplicateStrategy) {
|
||||
case "prefix":
|
||||
// Prefix with owner: owner-reponame
|
||||
candidateName = attempt === 0
|
||||
candidateName = attempt === 0
|
||||
? `${githubOwner}-${baseName}`
|
||||
: `${githubOwner}-${baseName}-${attempt}`;
|
||||
break;
|
||||
|
||||
|
||||
case "owner-org":
|
||||
// This would require creating sub-organizations, not supported in this PR
|
||||
// Fall back to suffix strategy
|
||||
@@ -718,24 +844,31 @@ async function generateUniqueRepoName({
|
||||
: `${baseName}-${githubOwner}-${attempt}`;
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
const exists = await isRepoPresentInGitea({
|
||||
config,
|
||||
owner: orgName,
|
||||
repoName: candidateName,
|
||||
});
|
||||
|
||||
|
||||
if (!exists) {
|
||||
console.log(`Found unique name for duplicate starred repo: ${candidateName}`);
|
||||
return candidateName;
|
||||
}
|
||||
|
||||
|
||||
attempt++;
|
||||
}
|
||||
|
||||
// If all attempts failed, use timestamp as last resort
|
||||
const timestamp = Date.now();
|
||||
return `${baseName}-${githubOwner}-${timestamp}`;
|
||||
|
||||
// SECURITY FIX: Prevent infinite duplicate creation
|
||||
// Instead of falling back to timestamp (which creates infinite duplicates),
|
||||
// throw an error to prevent hundreds of duplicate repos
|
||||
console.error(`Failed to find unique name for ${baseName} after ${maxAttempts} attempts`);
|
||||
console.error(`Organization: ${orgName}, GitHub Owner: ${githubOwner}, Strategy: ${duplicateStrategy}`);
|
||||
throw new Error(
|
||||
`Unable to generate unique repository name for "${baseName}". ` +
|
||||
`All ${maxAttempts} naming attempts resulted in conflicts. ` +
|
||||
`Please manually resolve the naming conflict or adjust your duplicate strategy.`
|
||||
);
|
||||
}
|
||||
|
||||
export async function mirrorGitHubRepoToGiteaOrg({
|
||||
@@ -765,11 +898,11 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
|
||||
// Determine the actual repository name to use (handle duplicates for starred repos)
|
||||
let targetRepoName = repository.name;
|
||||
|
||||
|
||||
if (repository.isStarred && config.githubConfig) {
|
||||
// Extract GitHub owner from full_name (format: owner/repo)
|
||||
const githubOwner = repository.fullName.split('/')[0];
|
||||
|
||||
|
||||
targetRepoName = await generateUniqueRepoName({
|
||||
config,
|
||||
orgName,
|
||||
@@ -777,7 +910,7 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
githubOwner,
|
||||
strategy: config.githubConfig.starredDuplicateStrategy,
|
||||
});
|
||||
|
||||
|
||||
if (targetRepoName !== repository.name) {
|
||||
console.log(
|
||||
`Starred repo ${repository.fullName} will be mirrored as ${orgName}/${targetRepoName} to avoid naming conflict`
|
||||
@@ -785,6 +918,23 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
}
|
||||
}
|
||||
|
||||
// IDEMPOTENCY CHECK: Check if this repo is already being mirrored
|
||||
const expectedLocation = `${orgName}/${targetRepoName}`;
|
||||
const isCurrentlyMirroring = await isRepoCurrentlyMirroring({
|
||||
config,
|
||||
repoName: targetRepoName,
|
||||
expectedLocation,
|
||||
});
|
||||
|
||||
if (isCurrentlyMirroring) {
|
||||
console.log(
|
||||
`[Idempotency] Skipping ${repository.fullName} - already being mirrored to ${expectedLocation}`
|
||||
);
|
||||
|
||||
// Don't throw an error, just return to allow other repos to continue
|
||||
return;
|
||||
}
|
||||
|
||||
const isExisting = await isRepoPresentInGitea({
|
||||
config,
|
||||
owner: orgName,
|
||||
@@ -831,11 +981,30 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
// Use clean clone URL without embedded credentials (Forgejo 12+ security requirement)
|
||||
const cloneAddress = repository.cloneUrl;
|
||||
|
||||
// DOUBLE-CHECK: Final idempotency check right before updating status
|
||||
// This catches race conditions in the small window between first check and status update
|
||||
const finalCheck = await isRepoCurrentlyMirroring({
|
||||
config,
|
||||
repoName: targetRepoName,
|
||||
expectedLocation,
|
||||
});
|
||||
|
||||
if (finalCheck) {
|
||||
console.log(
|
||||
`[Idempotency] Race condition detected - ${repository.fullName} is now being mirrored by another process. Skipping.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark repos as "mirroring" in DB
|
||||
// CRITICAL: Set mirroredLocation NOW (not after success) so idempotency checks work
|
||||
// This becomes the "target location" - where we intend to mirror to
|
||||
// Without this, the idempotency check can't detect concurrent operations on first mirror
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("mirroring"),
|
||||
mirroredLocation: expectedLocation,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repository.id!));
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { RepoStatus } from "./Repository";
|
||||
export const membershipRoleEnum = z.enum([
|
||||
"member",
|
||||
"admin",
|
||||
"owner",
|
||||
"billing_manager",
|
||||
]);
|
||||
|
||||
|
||||
@@ -9,28 +9,28 @@
|
||||
"astro": "astro"
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^4.3.4",
|
||||
"@astrojs/react": "^4.3.0",
|
||||
"@astrojs/mdx": "^4.3.6",
|
||||
"@astrojs/react": "^4.4.0",
|
||||
"@radix-ui/react-icons": "^1.3.2",
|
||||
"@radix-ui/react-slot": "^1.2.3",
|
||||
"@splinetool/react-spline": "^4.1.0",
|
||||
"@splinetool/runtime": "^1.10.52",
|
||||
"@tailwindcss/vite": "^4.1.12",
|
||||
"@splinetool/runtime": "^1.10.73",
|
||||
"@tailwindcss/vite": "^4.1.14",
|
||||
"@types/canvas-confetti": "^1.9.0",
|
||||
"@types/react": "^19.1.12",
|
||||
"@types/react-dom": "^19.1.9",
|
||||
"astro": "^5.13.4",
|
||||
"@types/react": "^19.2.0",
|
||||
"@types/react-dom": "^19.2.0",
|
||||
"astro": "^5.14.3",
|
||||
"canvas-confetti": "^1.9.3",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"lucide-react": "^0.542.0",
|
||||
"react": "^19.1.1",
|
||||
"react-dom": "^19.1.1",
|
||||
"lucide-react": "^0.544.0",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"tailwind-merge": "^3.3.1",
|
||||
"tailwindcss": "^4.1.12"
|
||||
"tailwindcss": "^4.1.14"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tw-animate-css": "^1.3.7"
|
||||
"tw-animate-css": "^1.4.0"
|
||||
},
|
||||
"packageManager": "pnpm@10.15.0"
|
||||
"packageManager": "pnpm@10.18.0"
|
||||
}
|
||||
1622
www/pnpm-lock.yaml
generated
1622
www/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user