Mirror/gitea-mirror
1
0
Fork 0
mirror of https://github.com/RayLabsHQ/gitea-mirror.git synced 2025-12-10 21:46:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: enhance job resilience with new database schema and recovery mechanisms

Browse Source 
- Added new fields to the mirror_jobs table for job resilience, including job_type, batch_id, total_items, completed_items, item_ids, completed_item_ids, in_progress, started_at, completed_at, and last_checkpoint.
- Implemented database migration scripts to update the mirror_jobs table schema.
- Introduced processWithResilience utility for handling item processing with checkpointing and recovery capabilities.
- Updated API routes for mirroring organizations and repositories to utilize the new resilience features.
- Created recovery system to detect and resume interrupted jobs on application startup.
- Added middleware to initialize the recovery system when the server starts.
This commit is contained in:
Arunavo Ray
2025-05-22 14:33:03 +05:30
parent f4bc28e6c2
commit abe3113755
13 changed files with 893 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/pages/api/job/mirror-org.ts
View File

@@ -6,8 +6,8 @@ import { createGitHubClient } from "@/lib/github";
import { mirrorGitHubOrgToGitea } from "@/lib/gitea";
import { repoStatusEnum } from "@/types/Repository";
import { type MembershipRole } from "@/types/organizations";
import { processWithRetry } from "@/lib/utils/concurrency";
import { createMirrorJob } from "@/lib/helpers";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -63,7 +63,7 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Fire async mirroring without blocking response, using parallel processing
// Fire async mirroring without blocking response, using parallel processing with resilience
setTimeout(async () => {
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing in config.");
@@ -76,8 +76,11 @@ export const POST: APIRoute = async ({ request }) => {
// Using a lower concurrency for organizations since each org might contain many repos
const CONCURRENCY_LIMIT = 2;
// Process organizations in parallel with retry capability
await processWithRetry(
// Generate a batch ID to group related organizations
const batchId = uuidv4();
// Process organizations in parallel with resilience to container restarts
await processWithResilience(
orgs,
async (org) => {
// Prepare organization data
@@ -92,16 +95,6 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of mirroring
console.log(`Starting mirror for organization: ${org.name}`);
// Create a mirror job entry to track progress
await createMirrorJob({
userId: config.userId || "",
organizationId: org.id,
organizationName: org.name,
message: `Started mirroring organization: ${org.name}`,
details: `Organization ${org.name} is now in the mirroring queue.`,
status: "mirroring",
});
// Mirror the organization
await mirrorGitHubOrgToGitea({
config,
@@ -112,9 +105,15 @@ export const POST: APIRoute = async ({ request }) => {
return org;
},
{
userId: config.userId || "",
jobType: "mirror",
batchId,
getItemId: (org) => org.id,
getItemName: (org) => org.name,
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 3000,
checkpointInterval: 1, // Checkpoint after each organization
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Organization mirroring progress: ${percentComplete}% (${completed}/${total})`);

29
src/pages/api/job/mirror-repo.ts
View File

@@ -8,8 +8,8 @@ import {
mirrorGitHubOrgRepoToGiteaOrg,
} from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github";
import { processWithRetry } from "@/lib/utils/concurrency";
import { createMirrorJob } from "@/lib/helpers";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -65,7 +65,7 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Start async mirroring in background with parallel processing
// Start async mirroring in background with parallel processing and resilience
setTimeout(async () => {
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing.");
@@ -77,8 +77,11 @@ export const POST: APIRoute = async ({ request }) => {
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 3;
// Process repositories in parallel with retry capability
await processWithRetry(
// Generate a batch ID to group related repositories
const batchId = uuidv4();
// Process repositories in parallel with resilience to container restarts
await processWithResilience(
repos,
async (repo) => {
// Prepare repository data
@@ -96,16 +99,6 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name}`);
// Create a mirror job entry to track progress
await createMirrorJob({
userId: config.userId || "",
repositoryId: repo.id,
repositoryName: repo.name,
message: `Started mirroring repository: ${repo.name}`,
details: `Repository ${repo.name} is now in the mirroring queue.`,
status: "mirroring",
});
// Mirror the repository based on whether it's in an organization
if (repo.organization && config.githubConfig.preserveOrgStructure) {
await mirrorGitHubOrgRepoToGiteaOrg({
@@ -125,9 +118,15 @@ export const POST: APIRoute = async ({ request }) => {
return repo;
},
{
userId: config.userId || "",
jobType: "mirror",
batchId,
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 2000,
checkpointInterval: 1, // Checkpoint after each repository
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Mirroring progress: ${percentComplete}% (${completed}/${total})`);

29
src/pages/api/job/sync-repo.ts
View File

@@ -5,8 +5,8 @@ import { eq, inArray } from "drizzle-orm";
import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import { syncGiteaRepo } from "@/lib/gitea";
import type { SyncRepoResponse } from "@/types/sync";
import { processWithRetry } from "@/lib/utils/concurrency";
import { createMirrorJob } from "@/lib/helpers";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -62,13 +62,16 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Start async mirroring in background with parallel processing
// Start async mirroring in background with parallel processing and resilience
setTimeout(async () => {
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 5;
// Process repositories in parallel with retry capability
await processWithRetry(
// Generate a batch ID to group related repositories
const batchId = uuidv4();
// Process repositories in parallel with resilience to container restarts
await processWithResilience(
repos,
async (repo) => {
// Prepare repository data
@@ -85,16 +88,6 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of syncing
console.log(`Starting sync for repository: ${repo.name}`);
// Create a mirror job entry to track progress
await createMirrorJob({
userId: config.userId || "",
repositoryId: repo.id,
repositoryName: repo.name,
message: `Started syncing repository: ${repo.name}`,
details: `Repository ${repo.name} is now in the syncing queue.`,
status: "syncing",
});
// Sync the repository
await syncGiteaRepo({
config,
@@ -104,9 +97,15 @@ export const POST: APIRoute = async ({ request }) => {
return repo;
},
{
userId: config.userId || "",
jobType: "sync",
batchId,
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 2000,
checkpointInterval: 1, // Checkpoint after each repository
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Syncing progress: ${percentComplete}% (${completed}/${total})`);