From 5add8766a4d3729fa83915e7de5c77904a802493 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Sun, 14 Sep 2025 08:31:31 +0530 Subject: [PATCH] fix(scheduler,config): preserve ENV schedule; add AUTO_MIRROR_REPOS auto-mirroring - Prevent Automation UI from overriding schedule: - mapDbScheduleToUi now parses intervals robustly (cron/duration/seconds) via parseInterval - mapUiScheduleToDb merges with existing config and stores interval as seconds (no lossy cron conversion) - /api/config passes existing scheduleConfig to preserve ENV-sourced values - schedule-sync endpoint uses parseInterval for nextRun calculation - Add AUTO_MIRROR_REPOS support and scheduled auto-mirror phase: - scheduleConfig schema includes autoImport and autoMirror - env-config-loader reads AUTO_MIRROR_REPOS and carries through to DB - scheduler auto-mirrors imported/pending/failed repos when autoMirror is enabled before regular sync - docker-compose and ENV docs updated with AUTO_MIRROR_REPOS - Tests pass and build succeeds --- docker-compose.yml | 1 + docs/ENVIRONMENT_VARIABLES.md | 3 +- src/lib/db/schema.ts | 2 + src/lib/env-config-loader.ts | 9 +++- src/lib/scheduler-service.ts | 69 +++++++++++++++++++++++++ src/lib/utils/config-mapper.ts | 65 ++++++++++------------- src/pages/api/config/index.ts | 5 +- src/pages/api/job/schedule-sync-repo.ts | 14 ++++- 8 files changed, 125 insertions(+), 43 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fad79c9..54f392b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,7 @@ services: - SCHEDULE_ENABLED=${SCHEDULE_ENABLED:-false} - GITEA_MIRROR_INTERVAL=${GITEA_MIRROR_INTERVAL:-8h} - AUTO_IMPORT_REPOS=${AUTO_IMPORT_REPOS:-true} + - AUTO_MIRROR_REPOS=${AUTO_MIRROR_REPOS:-false} # Repository Cleanup Configuration - CLEANUP_DELETE_IF_NOT_IN_GITHUB=${CLEANUP_DELETE_IF_NOT_IN_GITHUB:-false} - CLEANUP_ORPHANED_REPO_ACTION=${CLEANUP_ORPHANED_REPO_ACTION:-archive} diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 99f27e2..75e7068 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -195,6 +195,7 @@ Configure automatic scheduled mirroring. | Variable | Description | Default | Options | |----------|-------------|---------|---------| | `AUTO_IMPORT_REPOS` | Automatically discover and import new GitHub repositories during scheduled syncs | `true` | `true`, `false` | +| `AUTO_MIRROR_REPOS` | Automatically mirror newly imported repositories during scheduled syncs (no manual “Mirror All” required) | `false` | `true`, `false` | | `SCHEDULE_ONLY_MIRROR_UPDATED` | Only mirror repos with updates | `false` | `true`, `false` | | `SCHEDULE_UPDATE_INTERVAL` | Check for updates interval (milliseconds) | `86400000` | Number | | `SCHEDULE_SKIP_RECENTLY_MIRRORED` | Skip recently mirrored repos | `true` | `true`, `false` | @@ -407,4 +408,4 @@ BETTER_AUTH_TRUSTED_ORIGINS=http://localhost:3000,http://192.168.1.100:3000 - `admin:org` (read organization data) - Additional scopes may be required for specific features -For more examples and detailed configuration, see the `.env.example` file in the repository. \ No newline at end of file +For more examples and detailed configuration, see the `.env.example` file in the repository. diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 8e0b085..0f0efcb 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -81,6 +81,8 @@ export const scheduleConfigSchema = z.object({ updateInterval: z.number().default(86400000), skipRecentlyMirrored: z.boolean().default(true), recentThreshold: z.number().default(3600000), + autoImport: z.boolean().default(true), + autoMirror: z.boolean().default(false), lastRun: z.coerce.date().optional(), nextRun: z.coerce.date().optional(), }); diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index ccb929e..ce503de 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -69,6 +69,8 @@ interface EnvConfig { updateInterval?: number; skipRecentlyMirrored?: boolean; recentThreshold?: number; + autoImport?: boolean; + autoMirror?: boolean; }; cleanup: { enabled?: boolean; @@ -157,6 +159,8 @@ function parseEnvConfig(): EnvConfig { updateInterval: process.env.SCHEDULE_UPDATE_INTERVAL ? parseInt(process.env.SCHEDULE_UPDATE_INTERVAL, 10) : undefined, skipRecentlyMirrored: process.env.SCHEDULE_SKIP_RECENTLY_MIRRORED === 'true', recentThreshold: process.env.SCHEDULE_RECENT_THRESHOLD ? parseInt(process.env.SCHEDULE_RECENT_THRESHOLD, 10) : undefined, + autoImport: process.env.AUTO_IMPORT_REPOS !== 'false', + autoMirror: process.env.AUTO_MIRROR_REPOS === 'true', }, cleanup: { enabled: process.env.CLEANUP_ENABLED === 'true' || @@ -301,7 +305,8 @@ export async function initializeConfigFromEnv(): Promise { updateInterval: envConfig.schedule.updateInterval ?? existingConfig?.[0]?.scheduleConfig?.updateInterval ?? 86400000, skipRecentlyMirrored: envConfig.schedule.skipRecentlyMirrored ?? existingConfig?.[0]?.scheduleConfig?.skipRecentlyMirrored ?? true, recentThreshold: envConfig.schedule.recentThreshold ?? existingConfig?.[0]?.scheduleConfig?.recentThreshold ?? 3600000, - autoImport: process.env.AUTO_IMPORT_REPOS !== 'false', // New field for auto-importing new repositories + autoImport: envConfig.schedule.autoImport ?? existingConfig?.[0]?.scheduleConfig?.autoImport ?? true, + autoMirror: envConfig.schedule.autoMirror ?? existingConfig?.[0]?.scheduleConfig?.autoMirror ?? false, lastRun: existingConfig?.[0]?.scheduleConfig?.lastRun || undefined, nextRun: existingConfig?.[0]?.scheduleConfig?.nextRun || undefined, }; @@ -359,4 +364,4 @@ export async function initializeConfigFromEnv(): Promise { console.error('[ENV Config Loader] Failed to initialize configuration from environment:', error); // Don't throw - this is a non-critical initialization } -} \ No newline at end of file +} diff --git a/src/lib/scheduler-service.ts b/src/lib/scheduler-service.ts index b19219c..8a54ea0 100644 --- a/src/lib/scheduler-service.ts +++ b/src/lib/scheduler-service.ts @@ -166,6 +166,75 @@ async function runScheduledSync(config: any): Promise { } } + // Auto-mirror: Mirror imported/pending/failed repositories if enabled + if (scheduleConfig.autoMirror) { + try { + console.log(`[Scheduler] Auto-mirror enabled - checking for repositories to mirror for user ${userId}...`); + const reposNeedingMirror = await db + .select() + .from(repositories) + .where( + and( + eq(repositories.userId, userId), + or( + eq(repositories.status, 'imported'), + eq(repositories.status, 'pending'), + eq(repositories.status, 'failed') + ) + ) + ); + + if (reposNeedingMirror.length > 0) { + console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`); + + // Prepare Octokit client + const decryptedToken = getDecryptedGitHubToken(config); + const { Octokit } = await import('@octokit/rest'); + const octokit = new Octokit({ auth: decryptedToken }); + + // Process repositories in batches + const batchSize = scheduleConfig.batchSize || 10; + const pauseBetweenBatches = scheduleConfig.pauseBetweenBatches || 2000; + for (let i = 0; i < reposNeedingMirror.length; i += batchSize) { + const batch = reposNeedingMirror.slice(i, Math.min(i + batchSize, reposNeedingMirror.length)); + console.log(`[Scheduler] Auto-mirror batch ${Math.floor(i / batchSize) + 1} of ${Math.ceil(reposNeedingMirror.length / batchSize)} (${batch.length} repos)`); + + await Promise.all( + batch.map(async (repo) => { + try { + const repository: Repository = { + ...repo, + status: repoStatusEnum.parse(repo.status), + organization: repo.organization ?? undefined, + lastMirrored: repo.lastMirrored ?? undefined, + errorMessage: repo.errorMessage ?? undefined, + mirroredLocation: repo.mirroredLocation || '', + forkedFrom: repo.forkedFrom ?? undefined, + visibility: repositoryVisibilityEnum.parse(repo.visibility), + }; + + await mirrorGithubRepoToGitea({ octokit, repository, config }); + console.log(`[Scheduler] Auto-mirrored repository: ${repo.fullName}`); + } catch (error) { + console.error(`[Scheduler] Failed to auto-mirror repository ${repo.fullName}:`, error); + } + }) + ); + + // Pause between batches if configured + if (i + batchSize < reposNeedingMirror.length) { + console.log(`[Scheduler] Pausing for ${pauseBetweenBatches}ms before next auto-mirror batch...`); + await new Promise(resolve => setTimeout(resolve, pauseBetweenBatches)); + } + } + } else { + console.log(`[Scheduler] No repositories need initial mirroring`); + } + } catch (mirrorError) { + console.error(`[Scheduler] Error during auto-mirror phase for user ${userId}:`, mirrorError); + } + } + // Get repositories to sync let reposToSync = await db .select() diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index c4018f1..9ae97cd 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -11,6 +11,7 @@ import type { } from "@/types/config"; import { z } from "zod"; import { githubConfigSchema, giteaConfigSchema, scheduleConfigSchema, cleanupConfigSchema } from "@/lib/db/schema"; +import { parseInterval } from "@/lib/utils/duration-parser"; // Use the actual database schema types type DbGitHubConfig = z.infer; @@ -165,27 +166,22 @@ export function mapDbToUiConfig(dbConfig: any): { /** * Maps UI schedule config to database schema */ -export function mapUiScheduleToDb(uiSchedule: any): DbScheduleConfig { +export function mapUiScheduleToDb(uiSchedule: any, existing?: DbScheduleConfig): DbScheduleConfig { + // Preserve existing schedule config and only update fields controlled by the UI + const base: DbScheduleConfig = existing + ? { ...(existing as unknown as DbScheduleConfig) } + : (scheduleConfigSchema.parse({}) as unknown as DbScheduleConfig); + + // Store interval as seconds string to avoid lossy cron conversion + const intervalSeconds = typeof uiSchedule.interval === 'number' && uiSchedule.interval > 0 + ? String(uiSchedule.interval) + : (typeof base.interval === 'string' ? base.interval : String(86400)); + return { - enabled: uiSchedule.enabled || false, - interval: uiSchedule.interval ? `0 */${Math.floor(uiSchedule.interval / 3600)} * * *` : "0 2 * * *", // Convert seconds to cron expression - concurrent: false, - batchSize: 10, - pauseBetweenBatches: 5000, - retryAttempts: 3, - retryDelay: 60000, - timeout: 3600000, - autoRetry: true, - cleanupBeforeMirror: false, - notifyOnFailure: true, - notifyOnSuccess: false, - logLevel: "info", - timezone: "UTC", - onlyMirrorUpdated: false, - updateInterval: 86400000, - skipRecentlyMirrored: true, - recentThreshold: 3600000, - }; + ...base, + enabled: !!uiSchedule.enabled, + interval: intervalSeconds, + } as DbScheduleConfig; } /** @@ -202,23 +198,18 @@ export function mapDbScheduleToUi(dbSchedule: DbScheduleConfig): any { }; } - // Extract hours from cron expression if possible + // Parse interval supporting numbers (seconds), duration strings, and cron let intervalSeconds = 86400; // Default to daily (24 hours) - - if (dbSchedule.interval) { - // Check if it's already a number (seconds), use it directly - if (typeof dbSchedule.interval === 'number') { - intervalSeconds = dbSchedule.interval; - } else if (typeof dbSchedule.interval === 'string') { - // Check if it's a cron expression - const cronMatch = dbSchedule.interval.match(/0 \*\/(\d+) \* \* \*/); - if (cronMatch) { - intervalSeconds = parseInt(cronMatch[1]) * 3600; - } else if (dbSchedule.interval === "0 2 * * *") { - // Daily at 2 AM - intervalSeconds = 86400; - } - } + try { + const ms = parseInterval( + typeof dbSchedule.interval === 'number' + ? dbSchedule.interval + : (dbSchedule.interval as unknown as string) + ); + intervalSeconds = Math.max(1, Math.floor(ms / 1000)); + } catch (_e) { + // Fallback to default if unparsable + intervalSeconds = 86400; } return { @@ -266,4 +257,4 @@ export function mapDbCleanupToUi(dbCleanup: DbCleanupConfig): any { lastRun: dbCleanup.lastRun || null, nextRun: dbCleanup.nextRun || null, }; -} \ No newline at end of file +} diff --git a/src/pages/api/config/index.ts b/src/pages/api/config/index.ts index 31a1801..5b59745 100644 --- a/src/pages/api/config/index.ts +++ b/src/pages/api/config/index.ts @@ -87,7 +87,10 @@ export const POST: APIRoute = async ({ request }) => { } // Map schedule and cleanup configs to database schema - const processedScheduleConfig = mapUiScheduleToDb(scheduleConfig); + const processedScheduleConfig = mapUiScheduleToDb( + scheduleConfig, + existingConfig ? existingConfig.scheduleConfig : undefined + ); const processedCleanupConfig = mapUiCleanupToDb(cleanupConfig); if (existingConfig) { diff --git a/src/pages/api/job/schedule-sync-repo.ts b/src/pages/api/job/schedule-sync-repo.ts index 3aed0c6..3cac25e 100644 --- a/src/pages/api/job/schedule-sync-repo.ts +++ b/src/pages/api/job/schedule-sync-repo.ts @@ -8,6 +8,7 @@ import type { ScheduleSyncRepoResponse, } from "@/types/sync"; import { createSecureErrorResponse } from "@/lib/utils"; +import { parseInterval } from "@/lib/utils/duration-parser"; export const POST: APIRoute = async ({ request }) => { try { @@ -72,8 +73,17 @@ export const POST: APIRoute = async ({ request }) => { // Calculate nextRun and update lastRun and nextRun in the config const currentTime = new Date(); - const interval = config.scheduleConfig?.interval ?? 3600; - const nextRun = new Date(currentTime.getTime() + interval * 1000); + let intervalMs = 3600 * 1000; + try { + intervalMs = parseInterval( + typeof config.scheduleConfig?.interval === 'number' + ? config.scheduleConfig.interval + : (config.scheduleConfig?.interval as unknown as string) || '3600' + ); + } catch { + intervalMs = 3600 * 1000; + } + const nextRun = new Date(currentTime.getTime() + intervalMs); // Update the full giteaConfig object await db