mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-03-13 22:12:54 +03:00
Add pre-sync snapshot protection for mirror rewrites (#190)
* add pre-sync snapshot protection * stabilize test module mocks * fix cross-test gitea mock exports * fix gitea mock strategy behavior
This commit is contained in:
@@ -50,6 +50,10 @@ export function ConfigTabs() {
|
|||||||
starredReposOrg: 'starred',
|
starredReposOrg: 'starred',
|
||||||
starredReposMode: 'dedicated-org',
|
starredReposMode: 'dedicated-org',
|
||||||
preserveOrgStructure: false,
|
preserveOrgStructure: false,
|
||||||
|
backupBeforeSync: true,
|
||||||
|
backupRetentionCount: 20,
|
||||||
|
backupDirectory: 'data/repo-backups',
|
||||||
|
blockSyncOnBackupFailure: true,
|
||||||
},
|
},
|
||||||
scheduleConfig: {
|
scheduleConfig: {
|
||||||
enabled: false, // Don't set defaults here - will be loaded from API
|
enabled: false, // Don't set defaults here - will be loaded from API
|
||||||
|
|||||||
@@ -100,9 +100,16 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const normalizedValue =
|
||||||
|
type === "checkbox"
|
||||||
|
? checked
|
||||||
|
: name === "backupRetentionCount"
|
||||||
|
? Math.max(1, Number.parseInt(value, 10) || 20)
|
||||||
|
: value;
|
||||||
|
|
||||||
const newConfig = {
|
const newConfig = {
|
||||||
...config,
|
...config,
|
||||||
[name]: type === "checkbox" ? checked : value,
|
[name]: normalizedValue,
|
||||||
};
|
};
|
||||||
setConfig(newConfig);
|
setConfig(newConfig);
|
||||||
|
|
||||||
@@ -286,7 +293,77 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g
|
|||||||
if (onAutoSave) onAutoSave(newConfig);
|
if (onAutoSave) onAutoSave(newConfig);
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
<Separator />
|
||||||
|
|
||||||
|
<div className="space-y-4">
|
||||||
|
<h3 className="text-sm font-semibold">Destructive Update Protection</h3>
|
||||||
|
<label className="flex items-start gap-3 text-sm">
|
||||||
|
<input
|
||||||
|
name="backupBeforeSync"
|
||||||
|
type="checkbox"
|
||||||
|
checked={Boolean(config.backupBeforeSync)}
|
||||||
|
onChange={handleChange}
|
||||||
|
className="mt-0.5 rounded border-input"
|
||||||
|
/>
|
||||||
|
<span>
|
||||||
|
Create snapshot before each sync
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
Saves a restore point so force-pushes or rewritten upstream history can be recovered.
|
||||||
|
</p>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
{config.backupBeforeSync && (
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<label htmlFor="gitea-backup-retention" className="block text-sm font-medium mb-1.5">
|
||||||
|
Snapshot retention count
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="gitea-backup-retention"
|
||||||
|
name="backupRetentionCount"
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
value={config.backupRetentionCount ?? 20}
|
||||||
|
onChange={handleChange}
|
||||||
|
className="w-full rounded-md border border-input bg-background px-3 py-2 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label htmlFor="gitea-backup-directory" className="block text-sm font-medium mb-1.5">
|
||||||
|
Snapshot directory
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="gitea-backup-directory"
|
||||||
|
name="backupDirectory"
|
||||||
|
type="text"
|
||||||
|
value={config.backupDirectory || "data/repo-backups"}
|
||||||
|
onChange={handleChange}
|
||||||
|
className="w-full rounded-md border border-input bg-background px-3 py-2 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||||
|
placeholder="data/repo-backups"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<label className="flex items-start gap-3 text-sm">
|
||||||
|
<input
|
||||||
|
name="blockSyncOnBackupFailure"
|
||||||
|
type="checkbox"
|
||||||
|
checked={Boolean(config.blockSyncOnBackupFailure)}
|
||||||
|
onChange={handleChange}
|
||||||
|
className="mt-0.5 rounded border-input"
|
||||||
|
/>
|
||||||
|
<span>
|
||||||
|
Block sync when snapshot fails
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
Recommended for backup-first behavior. If disabled, sync continues even when snapshot creation fails.
|
||||||
|
</p>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* Mobile: Show button at bottom */}
|
{/* Mobile: Show button at bottom */}
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -65,6 +65,10 @@ export const giteaConfigSchema = z.object({
|
|||||||
mirrorPullRequests: z.boolean().default(false),
|
mirrorPullRequests: z.boolean().default(false),
|
||||||
mirrorLabels: z.boolean().default(false),
|
mirrorLabels: z.boolean().default(false),
|
||||||
mirrorMilestones: z.boolean().default(false),
|
mirrorMilestones: z.boolean().default(false),
|
||||||
|
backupBeforeSync: z.boolean().default(true),
|
||||||
|
backupRetentionCount: z.number().int().min(1).default(20),
|
||||||
|
backupDirectory: z.string().optional(),
|
||||||
|
blockSyncOnBackupFailure: z.boolean().default(true),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const scheduleConfigSchema = z.object({
|
export const scheduleConfigSchema = z.object({
|
||||||
|
|||||||
@@ -13,6 +13,11 @@ const mockMirrorGitRepoPullRequestsToGitea = mock(() => Promise.resolve());
|
|||||||
const mockMirrorGitRepoLabelsToGitea = mock(() => Promise.resolve());
|
const mockMirrorGitRepoLabelsToGitea = mock(() => Promise.resolve());
|
||||||
const mockMirrorGitRepoMilestonesToGitea = mock(() => Promise.resolve());
|
const mockMirrorGitRepoMilestonesToGitea = mock(() => Promise.resolve());
|
||||||
const mockGetGiteaRepoOwnerAsync = mock(() => Promise.resolve("starred"));
|
const mockGetGiteaRepoOwnerAsync = mock(() => Promise.resolve("starred"));
|
||||||
|
const mockCreatePreSyncBundleBackup = mock(() =>
|
||||||
|
Promise.resolve({ bundlePath: "/tmp/mock.bundle" })
|
||||||
|
);
|
||||||
|
let mockShouldCreatePreSyncBackup = false;
|
||||||
|
let mockShouldBlockSyncOnBackupFailure = true;
|
||||||
|
|
||||||
// Mock the database module
|
// Mock the database module
|
||||||
const mockDb = {
|
const mockDb = {
|
||||||
@@ -28,8 +33,14 @@ const mockDb = {
|
|||||||
|
|
||||||
mock.module("@/lib/db", () => ({
|
mock.module("@/lib/db", () => ({
|
||||||
db: mockDb,
|
db: mockDb,
|
||||||
|
users: {},
|
||||||
|
configs: {},
|
||||||
|
organizations: {},
|
||||||
mirrorJobs: {},
|
mirrorJobs: {},
|
||||||
repositories: {}
|
repositories: {},
|
||||||
|
events: {},
|
||||||
|
accounts: {},
|
||||||
|
sessions: {},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Mock config encryption
|
// Mock config encryption
|
||||||
@@ -235,6 +246,12 @@ mock.module("@/lib/http-client", () => ({
|
|||||||
HttpError: MockHttpError
|
HttpError: MockHttpError
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
mock.module("@/lib/repo-backup", () => ({
|
||||||
|
createPreSyncBundleBackup: mockCreatePreSyncBundleBackup,
|
||||||
|
shouldCreatePreSyncBackup: () => mockShouldCreatePreSyncBackup,
|
||||||
|
shouldBlockSyncOnBackupFailure: () => mockShouldBlockSyncOnBackupFailure,
|
||||||
|
}));
|
||||||
|
|
||||||
// Now import the modules we're testing
|
// Now import the modules we're testing
|
||||||
import {
|
import {
|
||||||
getGiteaRepoInfo,
|
getGiteaRepoInfo,
|
||||||
@@ -264,6 +281,15 @@ describe("Enhanced Gitea Operations", () => {
|
|||||||
mockMirrorGitRepoMilestonesToGitea.mockClear();
|
mockMirrorGitRepoMilestonesToGitea.mockClear();
|
||||||
mockGetGiteaRepoOwnerAsync.mockClear();
|
mockGetGiteaRepoOwnerAsync.mockClear();
|
||||||
mockGetGiteaRepoOwnerAsync.mockImplementation(() => Promise.resolve("starred"));
|
mockGetGiteaRepoOwnerAsync.mockImplementation(() => Promise.resolve("starred"));
|
||||||
|
mockHttpGet.mockClear();
|
||||||
|
mockHttpPost.mockClear();
|
||||||
|
mockHttpDelete.mockClear();
|
||||||
|
mockCreatePreSyncBundleBackup.mockClear();
|
||||||
|
mockCreatePreSyncBundleBackup.mockImplementation(() =>
|
||||||
|
Promise.resolve({ bundlePath: "/tmp/mock.bundle" })
|
||||||
|
);
|
||||||
|
mockShouldCreatePreSyncBackup = false;
|
||||||
|
mockShouldBlockSyncOnBackupFailure = true;
|
||||||
// Reset tracking variables
|
// Reset tracking variables
|
||||||
orgCheckCount = 0;
|
orgCheckCount = 0;
|
||||||
orgTestContext = "";
|
orgTestContext = "";
|
||||||
@@ -529,6 +555,125 @@ describe("Enhanced Gitea Operations", () => {
|
|||||||
expect(releaseCall.octokit).toBeDefined();
|
expect(releaseCall.octokit).toBeDefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("blocks sync when pre-sync snapshot fails and blocking is enabled", async () => {
|
||||||
|
mockShouldCreatePreSyncBackup = true;
|
||||||
|
mockShouldBlockSyncOnBackupFailure = true;
|
||||||
|
mockCreatePreSyncBundleBackup.mockImplementation(() =>
|
||||||
|
Promise.reject(new Error("simulated backup failure"))
|
||||||
|
);
|
||||||
|
|
||||||
|
const config: Partial<Config> = {
|
||||||
|
userId: "user123",
|
||||||
|
githubConfig: {
|
||||||
|
username: "testuser",
|
||||||
|
token: "github-token",
|
||||||
|
privateRepositories: false,
|
||||||
|
mirrorStarred: true,
|
||||||
|
},
|
||||||
|
giteaConfig: {
|
||||||
|
url: "https://gitea.example.com",
|
||||||
|
token: "encrypted-token",
|
||||||
|
defaultOwner: "testuser",
|
||||||
|
mirrorReleases: false,
|
||||||
|
backupBeforeSync: true,
|
||||||
|
blockSyncOnBackupFailure: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const repository: Repository = {
|
||||||
|
id: "repo456",
|
||||||
|
name: "mirror-repo",
|
||||||
|
fullName: "user/mirror-repo",
|
||||||
|
owner: "user",
|
||||||
|
cloneUrl: "https://github.com/user/mirror-repo.git",
|
||||||
|
isPrivate: false,
|
||||||
|
isStarred: true,
|
||||||
|
status: repoStatusEnum.parse("mirrored"),
|
||||||
|
visibility: "public",
|
||||||
|
userId: "user123",
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
syncGiteaRepoEnhanced(
|
||||||
|
{ config, repository },
|
||||||
|
{
|
||||||
|
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||||
|
mirrorGitHubReleasesToGitea: mockMirrorGitHubReleasesToGitea,
|
||||||
|
mirrorGitRepoIssuesToGitea: mockMirrorGitRepoIssuesToGitea,
|
||||||
|
mirrorGitRepoPullRequestsToGitea: mockMirrorGitRepoPullRequestsToGitea,
|
||||||
|
mirrorGitRepoLabelsToGitea: mockMirrorGitRepoLabelsToGitea,
|
||||||
|
mirrorGitRepoMilestonesToGitea: mockMirrorGitRepoMilestonesToGitea,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
).rejects.toThrow("Snapshot failed; sync blocked to protect history.");
|
||||||
|
|
||||||
|
const mirrorSyncCalls = mockHttpPost.mock.calls.filter((call) =>
|
||||||
|
String(call[0]).includes("/mirror-sync")
|
||||||
|
);
|
||||||
|
expect(mirrorSyncCalls.length).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("continues sync when pre-sync snapshot fails and blocking is disabled", async () => {
|
||||||
|
mockShouldCreatePreSyncBackup = true;
|
||||||
|
mockShouldBlockSyncOnBackupFailure = false;
|
||||||
|
mockCreatePreSyncBundleBackup.mockImplementation(() =>
|
||||||
|
Promise.reject(new Error("simulated backup failure"))
|
||||||
|
);
|
||||||
|
|
||||||
|
const config: Partial<Config> = {
|
||||||
|
userId: "user123",
|
||||||
|
githubConfig: {
|
||||||
|
username: "testuser",
|
||||||
|
token: "github-token",
|
||||||
|
privateRepositories: false,
|
||||||
|
mirrorStarred: true,
|
||||||
|
},
|
||||||
|
giteaConfig: {
|
||||||
|
url: "https://gitea.example.com",
|
||||||
|
token: "encrypted-token",
|
||||||
|
defaultOwner: "testuser",
|
||||||
|
mirrorReleases: false,
|
||||||
|
backupBeforeSync: true,
|
||||||
|
blockSyncOnBackupFailure: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const repository: Repository = {
|
||||||
|
id: "repo457",
|
||||||
|
name: "mirror-repo",
|
||||||
|
fullName: "user/mirror-repo",
|
||||||
|
owner: "user",
|
||||||
|
cloneUrl: "https://github.com/user/mirror-repo.git",
|
||||||
|
isPrivate: false,
|
||||||
|
isStarred: true,
|
||||||
|
status: repoStatusEnum.parse("mirrored"),
|
||||||
|
visibility: "public",
|
||||||
|
userId: "user123",
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await syncGiteaRepoEnhanced(
|
||||||
|
{ config, repository },
|
||||||
|
{
|
||||||
|
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||||
|
mirrorGitHubReleasesToGitea: mockMirrorGitHubReleasesToGitea,
|
||||||
|
mirrorGitRepoIssuesToGitea: mockMirrorGitRepoIssuesToGitea,
|
||||||
|
mirrorGitRepoPullRequestsToGitea: mockMirrorGitRepoPullRequestsToGitea,
|
||||||
|
mirrorGitRepoLabelsToGitea: mockMirrorGitRepoLabelsToGitea,
|
||||||
|
mirrorGitRepoMilestonesToGitea: mockMirrorGitRepoMilestonesToGitea,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({ success: true });
|
||||||
|
const mirrorSyncCalls = mockHttpPost.mock.calls.filter((call) =>
|
||||||
|
String(call[0]).includes("/mirror-sync")
|
||||||
|
);
|
||||||
|
expect(mirrorSyncCalls.length).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
test("mirrors metadata components when enabled and not previously synced", async () => {
|
test("mirrors metadata components when enabled and not previously synced", async () => {
|
||||||
const config: Partial<Config> = {
|
const config: Partial<Config> = {
|
||||||
userId: "user123",
|
userId: "user123",
|
||||||
|
|||||||
@@ -15,6 +15,11 @@ import { httpPost, httpGet, httpPatch, HttpError } from "./http-client";
|
|||||||
import { db, repositories } from "./db";
|
import { db, repositories } from "./db";
|
||||||
import { eq } from "drizzle-orm";
|
import { eq } from "drizzle-orm";
|
||||||
import { repoStatusEnum } from "@/types/Repository";
|
import { repoStatusEnum } from "@/types/Repository";
|
||||||
|
import {
|
||||||
|
createPreSyncBundleBackup,
|
||||||
|
shouldCreatePreSyncBackup,
|
||||||
|
shouldBlockSyncOnBackupFailure,
|
||||||
|
} from "./repo-backup";
|
||||||
import {
|
import {
|
||||||
parseRepositoryMetadataState,
|
parseRepositoryMetadataState,
|
||||||
serializeRepositoryMetadataState,
|
serializeRepositoryMetadataState,
|
||||||
@@ -313,6 +318,61 @@ export async function syncGiteaRepoEnhanced({
|
|||||||
throw new Error(`Repository ${repository.name} is not a mirror. Cannot sync.`);
|
throw new Error(`Repository ${repository.name} is not a mirror. Cannot sync.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (shouldCreatePreSyncBackup(config)) {
|
||||||
|
const cloneUrl =
|
||||||
|
repoInfo.clone_url ||
|
||||||
|
`${config.giteaConfig.url.replace(/\/$/, "")}/${repoOwner}/${repository.name}.git`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const backupResult = await createPreSyncBundleBackup({
|
||||||
|
config,
|
||||||
|
owner: repoOwner,
|
||||||
|
repoName: repository.name,
|
||||||
|
cloneUrl,
|
||||||
|
});
|
||||||
|
|
||||||
|
await createMirrorJob({
|
||||||
|
userId: config.userId,
|
||||||
|
repositoryId: repository.id,
|
||||||
|
repositoryName: repository.name,
|
||||||
|
message: `Snapshot created for ${repository.name}`,
|
||||||
|
details: `Pre-sync snapshot created at ${backupResult.bundlePath}.`,
|
||||||
|
status: "syncing",
|
||||||
|
});
|
||||||
|
} catch (backupError) {
|
||||||
|
const errorMessage =
|
||||||
|
backupError instanceof Error ? backupError.message : String(backupError);
|
||||||
|
|
||||||
|
await createMirrorJob({
|
||||||
|
userId: config.userId,
|
||||||
|
repositoryId: repository.id,
|
||||||
|
repositoryName: repository.name,
|
||||||
|
message: `Snapshot failed for ${repository.name}`,
|
||||||
|
details: `Pre-sync snapshot failed: ${errorMessage}`,
|
||||||
|
status: "failed",
|
||||||
|
});
|
||||||
|
|
||||||
|
if (shouldBlockSyncOnBackupFailure(config)) {
|
||||||
|
await db
|
||||||
|
.update(repositories)
|
||||||
|
.set({
|
||||||
|
status: repoStatusEnum.parse("failed"),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
errorMessage: `Snapshot failed; sync blocked to protect history. ${errorMessage}`,
|
||||||
|
})
|
||||||
|
.where(eq(repositories.id, repository.id!));
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Snapshot failed; sync blocked to protect history. ${errorMessage}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn(
|
||||||
|
`[Sync] Snapshot failed for ${repository.name}, continuing because blockSyncOnBackupFailure=false: ${errorMessage}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update mirror interval if needed
|
// Update mirror interval if needed
|
||||||
if (config.giteaConfig?.mirrorInterval) {
|
if (config.giteaConfig?.mirrorInterval) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -24,9 +24,14 @@ mock.module("@/lib/db", () => {
|
|||||||
values: mock(() => Promise.resolve())
|
values: mock(() => Promise.resolve())
|
||||||
}))
|
}))
|
||||||
},
|
},
|
||||||
|
users: {},
|
||||||
|
configs: {},
|
||||||
repositories: {},
|
repositories: {},
|
||||||
organizations: {},
|
organizations: {},
|
||||||
events: {}
|
events: {},
|
||||||
|
mirrorJobs: {},
|
||||||
|
accounts: {},
|
||||||
|
sessions: {},
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -59,10 +64,16 @@ const mockGetOrCreateGiteaOrg = mock(async ({ orgName, config }: any) => {
|
|||||||
|
|
||||||
const mockMirrorGitHubOrgRepoToGiteaOrg = mock(async () => {});
|
const mockMirrorGitHubOrgRepoToGiteaOrg = mock(async () => {});
|
||||||
const mockIsRepoPresentInGitea = mock(async () => false);
|
const mockIsRepoPresentInGitea = mock(async () => false);
|
||||||
|
const mockMirrorGithubRepoToGitea = mock(async () => {});
|
||||||
|
const mockGetGiteaRepoOwnerAsync = mock(async () => "starred");
|
||||||
|
const mockGetGiteaRepoOwner = mock(() => "starred");
|
||||||
|
|
||||||
mock.module("./gitea", () => ({
|
mock.module("./gitea", () => ({
|
||||||
getOrCreateGiteaOrg: mockGetOrCreateGiteaOrg,
|
getOrCreateGiteaOrg: mockGetOrCreateGiteaOrg,
|
||||||
mirrorGitHubOrgRepoToGiteaOrg: mockMirrorGitHubOrgRepoToGiteaOrg,
|
mirrorGitHubOrgRepoToGiteaOrg: mockMirrorGitHubOrgRepoToGiteaOrg,
|
||||||
|
mirrorGithubRepoToGitea: mockMirrorGithubRepoToGitea,
|
||||||
|
getGiteaRepoOwner: mockGetGiteaRepoOwner,
|
||||||
|
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||||
isRepoPresentInGitea: mockIsRepoPresentInGitea
|
isRepoPresentInGitea: mockIsRepoPresentInGitea
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -226,4 +237,4 @@ describe("Starred Repository Error Handling", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -27,8 +27,14 @@ mock.module("@/lib/db", () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
|
users: {},
|
||||||
|
configs: {},
|
||||||
repositories: {},
|
repositories: {},
|
||||||
organizations: {}
|
organizations: {},
|
||||||
|
mirrorJobs: {},
|
||||||
|
events: {},
|
||||||
|
accounts: {},
|
||||||
|
sessions: {},
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -55,8 +61,50 @@ mock.module("@/lib/http-client", () => {
|
|||||||
|
|
||||||
// Mock the gitea module itself
|
// Mock the gitea module itself
|
||||||
mock.module("./gitea", () => {
|
mock.module("./gitea", () => {
|
||||||
|
const mockGetGiteaRepoOwner = mock(({ config, repository }: any) => {
|
||||||
|
if (repository?.isStarred && config?.githubConfig?.starredReposMode === "preserve-owner") {
|
||||||
|
return repository.organization || repository.owner;
|
||||||
|
}
|
||||||
|
if (repository?.isStarred) {
|
||||||
|
return config?.githubConfig?.starredReposOrg || "starred";
|
||||||
|
}
|
||||||
|
|
||||||
|
const mirrorStrategy =
|
||||||
|
config?.githubConfig?.mirrorStrategy ||
|
||||||
|
(config?.giteaConfig?.preserveOrgStructure ? "preserve" : "flat-user");
|
||||||
|
|
||||||
|
switch (mirrorStrategy) {
|
||||||
|
case "preserve":
|
||||||
|
return repository?.organization || config?.giteaConfig?.defaultOwner || "giteauser";
|
||||||
|
case "single-org":
|
||||||
|
return config?.giteaConfig?.organization || config?.giteaConfig?.defaultOwner || "giteauser";
|
||||||
|
case "mixed":
|
||||||
|
if (repository?.organization) return repository.organization;
|
||||||
|
return config?.giteaConfig?.organization || config?.giteaConfig?.defaultOwner || "giteauser";
|
||||||
|
case "flat-user":
|
||||||
|
default:
|
||||||
|
return config?.giteaConfig?.defaultOwner || "giteauser";
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const mockGetGiteaRepoOwnerAsync = mock(async ({ config, repository }: any) => {
|
||||||
|
if (repository?.isStarred && config?.githubConfig?.starredReposMode === "preserve-owner") {
|
||||||
|
return repository.organization || repository.owner;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (repository?.destinationOrg) {
|
||||||
|
return repository.destinationOrg;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (repository?.organization && mockDbSelectResult[0]?.destinationOrg) {
|
||||||
|
return mockDbSelectResult[0].destinationOrg;
|
||||||
|
}
|
||||||
|
|
||||||
|
return config?.giteaConfig?.defaultOwner || "giteauser";
|
||||||
|
});
|
||||||
return {
|
return {
|
||||||
isRepoPresentInGitea: mockIsRepoPresentInGitea,
|
isRepoPresentInGitea: mockIsRepoPresentInGitea,
|
||||||
|
getGiteaRepoOwner: mockGetGiteaRepoOwner,
|
||||||
|
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||||
mirrorGithubRepoToGitea: mock(async () => {}),
|
mirrorGithubRepoToGitea: mock(async () => {}),
|
||||||
mirrorGitHubOrgRepoToGiteaOrg: mock(async () => {})
|
mirrorGitHubOrgRepoToGiteaOrg: mock(async () => {})
|
||||||
};
|
};
|
||||||
|
|||||||
164
src/lib/repo-backup.ts
Normal file
164
src/lib/repo-backup.ts
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
import { mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { Config } from "@/types/config";
|
||||||
|
import { decryptConfigTokens } from "./utils/config-encryption";
|
||||||
|
|
||||||
|
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||||
|
|
||||||
|
function parseBoolean(value: string | undefined, fallback: boolean): boolean {
|
||||||
|
if (value === undefined) return fallback;
|
||||||
|
return TRUE_VALUES.has(value.trim().toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
function parsePositiveInt(value: string | undefined, fallback: number): number {
|
||||||
|
if (!value) return fallback;
|
||||||
|
const parsed = Number.parseInt(value, 10);
|
||||||
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitizePathSegment(input: string): string {
|
||||||
|
return input.replace(/[^a-zA-Z0-9._-]/g, "_");
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildTimestamp(): string {
|
||||||
|
// Example: 2026-02-25T18-34-22-123Z
|
||||||
|
return new Date().toISOString().replace(/[:.]/g, "-");
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAuthenticatedCloneUrl(cloneUrl: string, token: string): string {
|
||||||
|
const parsed = new URL(cloneUrl);
|
||||||
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||||
|
return cloneUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed.username = process.env.PRE_SYNC_BACKUP_GIT_USERNAME || "oauth2";
|
||||||
|
parsed.password = token;
|
||||||
|
return parsed.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function maskToken(text: string, token: string): string {
|
||||||
|
if (!token) return text;
|
||||||
|
return text.split(token).join("***");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runGit(args: string[], tokenToMask: string): Promise<void> {
|
||||||
|
const proc = Bun.spawn({
|
||||||
|
cmd: ["git", ...args],
|
||||||
|
stdout: "pipe",
|
||||||
|
stderr: "pipe",
|
||||||
|
});
|
||||||
|
|
||||||
|
const [stdout, stderr, exitCode] = await Promise.all([
|
||||||
|
new Response(proc.stdout).text(),
|
||||||
|
new Response(proc.stderr).text(),
|
||||||
|
proc.exited,
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (exitCode !== 0) {
|
||||||
|
const details = [stdout, stderr].filter(Boolean).join("\n").trim();
|
||||||
|
const safeDetails = maskToken(details, tokenToMask);
|
||||||
|
throw new Error(`git command failed: ${safeDetails || "unknown git error"}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function enforceRetention(repoBackupDir: string, keepCount: number): Promise<void> {
|
||||||
|
const entries = await readdir(repoBackupDir);
|
||||||
|
const bundleFiles = entries
|
||||||
|
.filter((name) => name.endsWith(".bundle"))
|
||||||
|
.map((name) => path.join(repoBackupDir, name));
|
||||||
|
|
||||||
|
if (bundleFiles.length <= keepCount) return;
|
||||||
|
|
||||||
|
const filesWithMtime = await Promise.all(
|
||||||
|
bundleFiles.map(async (filePath) => ({
|
||||||
|
filePath,
|
||||||
|
mtimeMs: (await stat(filePath)).mtimeMs,
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
|
filesWithMtime.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
||||||
|
const toDelete = filesWithMtime.slice(keepCount);
|
||||||
|
|
||||||
|
await Promise.all(toDelete.map((entry) => rm(entry.filePath, { force: true })));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPreSyncBackupEnabled(): boolean {
|
||||||
|
return parseBoolean(process.env.PRE_SYNC_BACKUP_ENABLED, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldCreatePreSyncBackup(config: Partial<Config>): boolean {
|
||||||
|
const configSetting = config.giteaConfig?.backupBeforeSync;
|
||||||
|
const fallback = isPreSyncBackupEnabled();
|
||||||
|
return configSetting === undefined ? fallback : Boolean(configSetting);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldBlockSyncOnBackupFailure(config: Partial<Config>): boolean {
|
||||||
|
const configSetting = config.giteaConfig?.blockSyncOnBackupFailure;
|
||||||
|
return configSetting === undefined ? true : Boolean(configSetting);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createPreSyncBundleBackup({
|
||||||
|
config,
|
||||||
|
owner,
|
||||||
|
repoName,
|
||||||
|
cloneUrl,
|
||||||
|
}: {
|
||||||
|
config: Partial<Config>;
|
||||||
|
owner: string;
|
||||||
|
repoName: string;
|
||||||
|
cloneUrl: string;
|
||||||
|
}): Promise<{ bundlePath: string }> {
|
||||||
|
if (!shouldCreatePreSyncBackup(config)) {
|
||||||
|
throw new Error("Pre-sync backup is disabled.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!config.giteaConfig?.token) {
|
||||||
|
throw new Error("Gitea token is required for pre-sync backup.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
||||||
|
const giteaToken = decryptedConfig.giteaConfig?.token;
|
||||||
|
if (!giteaToken) {
|
||||||
|
throw new Error("Decrypted Gitea token is required for pre-sync backup.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupRoot =
|
||||||
|
config.giteaConfig?.backupDirectory?.trim() ||
|
||||||
|
process.env.PRE_SYNC_BACKUP_DIR?.trim() ||
|
||||||
|
path.join(process.cwd(), "data", "repo-backups");
|
||||||
|
const retention = Math.max(
|
||||||
|
1,
|
||||||
|
Number.isFinite(config.giteaConfig?.backupRetentionCount)
|
||||||
|
? Number(config.giteaConfig?.backupRetentionCount)
|
||||||
|
: parsePositiveInt(process.env.PRE_SYNC_BACKUP_KEEP_COUNT, 20)
|
||||||
|
);
|
||||||
|
|
||||||
|
const repoBackupDir = path.join(
|
||||||
|
backupRoot,
|
||||||
|
sanitizePathSegment(config.userId || "unknown-user"),
|
||||||
|
sanitizePathSegment(owner),
|
||||||
|
sanitizePathSegment(repoName)
|
||||||
|
);
|
||||||
|
|
||||||
|
await mkdir(repoBackupDir, { recursive: true });
|
||||||
|
|
||||||
|
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "gitea-mirror-backup-"));
|
||||||
|
const mirrorClonePath = path.join(tmpDir, "repo.git");
|
||||||
|
const bundlePath = path.join(repoBackupDir, `${buildTimestamp()}.bundle`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const authCloneUrl = buildAuthenticatedCloneUrl(cloneUrl, giteaToken);
|
||||||
|
|
||||||
|
await runGit(["clone", "--mirror", authCloneUrl, mirrorClonePath], giteaToken);
|
||||||
|
await runGit(["-C", mirrorClonePath, "bundle", "create", bundlePath, "--all"], giteaToken);
|
||||||
|
|
||||||
|
await enforceRetention(repoBackupDir, retention);
|
||||||
|
return { bundlePath };
|
||||||
|
} finally {
|
||||||
|
await rm(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -93,6 +93,10 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
|
|||||||
forkStrategy: "reference",
|
forkStrategy: "reference",
|
||||||
issueConcurrency: 3,
|
issueConcurrency: 3,
|
||||||
pullRequestConcurrency: 5,
|
pullRequestConcurrency: 5,
|
||||||
|
backupBeforeSync: true,
|
||||||
|
backupRetentionCount: 20,
|
||||||
|
backupDirectory: "data/repo-backups",
|
||||||
|
blockSyncOnBackupFailure: true,
|
||||||
},
|
},
|
||||||
include: [],
|
include: [],
|
||||||
exclude: [],
|
exclude: [],
|
||||||
|
|||||||
@@ -100,6 +100,10 @@ export function mapUiToDbConfig(
|
|||||||
mirrorPullRequests: mirrorOptions.mirrorMetadata && mirrorOptions.metadataComponents.pullRequests,
|
mirrorPullRequests: mirrorOptions.mirrorMetadata && mirrorOptions.metadataComponents.pullRequests,
|
||||||
mirrorLabels: mirrorOptions.mirrorMetadata && mirrorOptions.metadataComponents.labels,
|
mirrorLabels: mirrorOptions.mirrorMetadata && mirrorOptions.metadataComponents.labels,
|
||||||
mirrorMilestones: mirrorOptions.mirrorMetadata && mirrorOptions.metadataComponents.milestones,
|
mirrorMilestones: mirrorOptions.mirrorMetadata && mirrorOptions.metadataComponents.milestones,
|
||||||
|
backupBeforeSync: giteaConfig.backupBeforeSync ?? true,
|
||||||
|
backupRetentionCount: giteaConfig.backupRetentionCount ?? 20,
|
||||||
|
backupDirectory: giteaConfig.backupDirectory?.trim() || undefined,
|
||||||
|
blockSyncOnBackupFailure: giteaConfig.blockSyncOnBackupFailure ?? true,
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -140,6 +144,10 @@ export function mapDbToUiConfig(dbConfig: any): {
|
|||||||
personalReposOrg: undefined, // Not stored in current schema
|
personalReposOrg: undefined, // Not stored in current schema
|
||||||
issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3,
|
issueConcurrency: dbConfig.giteaConfig?.issueConcurrency ?? 3,
|
||||||
pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5,
|
pullRequestConcurrency: dbConfig.giteaConfig?.pullRequestConcurrency ?? 5,
|
||||||
|
backupBeforeSync: dbConfig.giteaConfig?.backupBeforeSync ?? true,
|
||||||
|
backupRetentionCount: dbConfig.giteaConfig?.backupRetentionCount ?? 20,
|
||||||
|
backupDirectory: dbConfig.giteaConfig?.backupDirectory || "data/repo-backups",
|
||||||
|
blockSyncOnBackupFailure: dbConfig.giteaConfig?.blockSyncOnBackupFailure ?? true,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Map mirror options from various database fields
|
// Map mirror options from various database fields
|
||||||
|
|||||||
@@ -62,7 +62,13 @@ const mockRepositories = {};
|
|||||||
mock.module("@/lib/db", () => ({
|
mock.module("@/lib/db", () => ({
|
||||||
db: mockDb,
|
db: mockDb,
|
||||||
configs: mockConfigs,
|
configs: mockConfigs,
|
||||||
repositories: mockRepositories
|
repositories: mockRepositories,
|
||||||
|
users: {},
|
||||||
|
organizations: {},
|
||||||
|
mirrorJobs: {},
|
||||||
|
events: {},
|
||||||
|
accounts: {},
|
||||||
|
sessions: {}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Mock the gitea module
|
// Mock the gitea module
|
||||||
@@ -71,7 +77,10 @@ const mockMirrorGitHubOrgRepoToGiteaOrg = mock(() => Promise.resolve());
|
|||||||
|
|
||||||
mock.module("@/lib/gitea", () => ({
|
mock.module("@/lib/gitea", () => ({
|
||||||
mirrorGithubRepoToGitea: mockMirrorGithubRepoToGitea,
|
mirrorGithubRepoToGitea: mockMirrorGithubRepoToGitea,
|
||||||
mirrorGitHubOrgRepoToGiteaOrg: mockMirrorGitHubOrgRepoToGiteaOrg
|
mirrorGitHubOrgRepoToGiteaOrg: mockMirrorGitHubOrgRepoToGiteaOrg,
|
||||||
|
getGiteaRepoOwnerAsync: mock(() => Promise.resolve("test-owner")),
|
||||||
|
isRepoPresentInGitea: mock(() => Promise.resolve(true)),
|
||||||
|
syncGiteaRepo: mock(() => Promise.resolve({ success: true })),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Mock the github module
|
// Mock the github module
|
||||||
|
|||||||
@@ -18,6 +18,10 @@ export interface GiteaConfig {
|
|||||||
personalReposOrg?: string; // Override destination for personal repos
|
personalReposOrg?: string; // Override destination for personal repos
|
||||||
issueConcurrency?: number;
|
issueConcurrency?: number;
|
||||||
pullRequestConcurrency?: number;
|
pullRequestConcurrency?: number;
|
||||||
|
backupBeforeSync?: boolean;
|
||||||
|
backupRetentionCount?: number;
|
||||||
|
backupDirectory?: string;
|
||||||
|
blockSyncOnBackupFailure?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ScheduleConfig {
|
export interface ScheduleConfig {
|
||||||
|
|||||||
Reference in New Issue
Block a user