From d0693206c3323ea4af55c9ba98ba9a51be2f3b7f Mon Sep 17 00:00:00 2001 From: ARUNAVO RAY Date: Wed, 4 Mar 2026 08:22:44 +0530 Subject: [PATCH] feat: selective starred repo mirroring with autoMirrorStarred toggle (#208) * feat: add autoMirrorStarred toggle for selective starred repo mirroring (#205) Add `githubConfig.autoMirrorStarred` (default: false) to control whether starred repos are included in automatic mirroring operations. Manual per-repo actions always work regardless of this toggle. Bug fixes: - Cleanup service no longer orphans starred repos when includeStarred is disabled (prevents data loss) - First-boot auto-start now gates initial mirror behind autoMirror config (previously mirrored everything unconditionally) - "Mirror All" button now respects autoMirrorStarred setting - Bulk mirror and getAvailableActions now include pending-approval status Changes span schema, config mapping, env loader, scheduler, cleanup service, UI settings toggle, and repository components. * fix: log activity when repos are auto-imported during scheduled sync Auto-discovered repositories (including newly starred ones) were inserted into the database without creating activity log entries, so they appeared in the dashboard but not in the activity log. * ci: set 10-minute timeout on all CI jobs --- .github/workflows/astro-build-test.yml | 1 + .github/workflows/docker-build.yml | 1 + .github/workflows/e2e-tests.yml | 2 +- .github/workflows/helm-test.yml | 2 + .github/workflows/nix-build.yml | 1 + docs/ENVIRONMENT_VARIABLES.md | 1 + src/components/config/ConfigTabs.tsx | 1 + .../config/GitHubMirrorSettings.tsx | 25 ++++++ src/components/repositories/Repository.tsx | 18 ++-- src/hooks/useConfigStatus.ts | 12 +++ src/lib/db/schema.ts | 1 + src/lib/env-config-loader.ts | 3 + src/lib/repository-cleanup-service.ts | 7 ++ src/lib/scheduler-service.ts | 87 ++++++++++++++++++- src/lib/utils/config-mapper.ts | 2 + src/types/config.ts | 1 + 16 files changed, 152 insertions(+), 13 deletions(-) diff --git a/.github/workflows/astro-build-test.yml b/.github/workflows/astro-build-test.yml index ed386d1..58a94f4 100644 --- a/.github/workflows/astro-build-test.yml +++ b/.github/workflows/astro-build-test.yml @@ -24,6 +24,7 @@ jobs: build-and-test: name: Build and Test Astro Project runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout repository diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d61a24e..d256dec 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -36,6 +36,7 @@ env: jobs: docker: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: write diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 16fea1a..a89d2e4 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -46,7 +46,7 @@ jobs: e2e-tests: name: E2E Integration Tests runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 10 steps: - name: Checkout repository diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index 6758eca..867d417 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -21,6 +21,7 @@ jobs: yamllint: name: Lint YAML runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -35,6 +36,7 @@ jobs: helm-template: name: Helm lint & template runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - name: Setup Helm diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 7d1c9a7..2a6649b 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -24,6 +24,7 @@ permissions: jobs: check: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 487c41f..557a3c8 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -78,6 +78,7 @@ Settings for connecting to and configuring GitHub repository sources. | Variable | Description | Default | Options | |----------|-------------|---------|---------| | `SKIP_STARRED_ISSUES` | Enable lightweight mode for starred repos (skip issues) | `false` | `true`, `false` | +| `AUTO_MIRROR_STARRED` | Automatically mirror starred repos during scheduled syncs and "Mirror All". When `false`, starred repos are imported for browsing but must be mirrored individually. | `false` | `true`, `false` | ## Gitea Configuration diff --git a/src/components/config/ConfigTabs.tsx b/src/components/config/ConfigTabs.tsx index 899a8d8..4fac193 100644 --- a/src/components/config/ConfigTabs.tsx +++ b/src/components/config/ConfigTabs.tsx @@ -83,6 +83,7 @@ export function ConfigTabs() { advancedOptions: { skipForks: false, starredCodeOnly: false, + autoMirrorStarred: false, }, }); const { user } = useAuth(); diff --git a/src/components/config/GitHubMirrorSettings.tsx b/src/components/config/GitHubMirrorSettings.tsx index 1d6985a..f9da909 100644 --- a/src/components/config/GitHubMirrorSettings.tsx +++ b/src/components/config/GitHubMirrorSettings.tsx @@ -287,6 +287,31 @@ export function GitHubMirrorSettings({ + {/* Auto-mirror starred repos toggle */} + {githubConfig.mirrorStarred && ( +
+
+ handleAdvancedChange('autoMirrorStarred', !!checked)} + /> +
+ +

+ When disabled, starred repos are imported for browsing but not automatically mirrored. You can still mirror individual repos manually. +

+
+
+
+ )} + {/* Duplicate name handling for starred repos */} {githubConfig.mirrorStarred && (
diff --git a/src/components/repositories/Repository.tsx b/src/components/repositories/Repository.tsx index b6200b0..f714bee 100644 --- a/src/components/repositories/Repository.tsx +++ b/src/components/repositories/Repository.tsx @@ -56,7 +56,7 @@ export default function Repository() { const [isInitialLoading, setIsInitialLoading] = useState(true); const { user } = useAuth(); const { registerRefreshCallback, isLiveEnabled } = useLiveRefresh(); - const { isGitHubConfigured, isFullyConfigured } = useConfigStatus(); + const { isGitHubConfigured, isFullyConfigured, autoMirrorStarred, githubOwner } = useConfigStatus(); const { navigationKey } = useNavigation(); const { filter, setFilter } = useFilterParams({ searchTerm: "", @@ -233,10 +233,12 @@ export default function Repository() { // Filter out repositories that are already mirroring, mirrored, or ignored const eligibleRepos = repositories.filter( (repo) => - repo.status !== "mirroring" && - repo.status !== "mirrored" && + repo.status !== "mirroring" && + repo.status !== "mirrored" && repo.status !== "ignored" && // Skip ignored repositories - repo.id + repo.id && + // Skip starred repos from other owners when autoMirrorStarred is disabled + !(repo.isStarred && !autoMirrorStarred && repo.owner !== githubOwner) ); if (eligibleRepos.length === 0) { @@ -292,7 +294,7 @@ export default function Repository() { const selectedRepos = repositories.filter(repo => repo.id && selectedRepoIds.has(repo.id)); const eligibleRepos = selectedRepos.filter( - repo => repo.status === "imported" || repo.status === "failed" + repo => repo.status === "imported" || repo.status === "failed" || repo.status === "pending-approval" ); if (eligibleRepos.length === 0) { @@ -301,7 +303,7 @@ export default function Repository() { } const repoIds = eligibleRepos.map(repo => repo.id as string); - + setLoadingRepoIds(prev => { const newSet = new Set(prev); repoIds.forEach(id => newSet.add(id)); @@ -937,7 +939,7 @@ export default function Repository() { const actions = []; // Check if any selected repos can be mirrored - if (selectedRepos.some(repo => repo.status === "imported" || repo.status === "failed")) { + if (selectedRepos.some(repo => repo.status === "imported" || repo.status === "failed" || repo.status === "pending-approval")) { actions.push('mirror'); } @@ -975,7 +977,7 @@ export default function Repository() { const selectedRepos = repositories.filter(repo => repo.id && selectedRepoIds.has(repo.id)); return { - mirror: selectedRepos.filter(repo => repo.status === "imported" || repo.status === "failed").length, + mirror: selectedRepos.filter(repo => repo.status === "imported" || repo.status === "failed" || repo.status === "pending-approval").length, sync: selectedRepos.filter(repo => repo.status === "mirrored" || repo.status === "synced").length, rerunMetadata: selectedRepos.filter(repo => ["mirrored", "synced", "archived"].includes(repo.status)).length, retry: selectedRepos.filter(repo => repo.status === "failed").length, diff --git a/src/hooks/useConfigStatus.ts b/src/hooks/useConfigStatus.ts index b9943fd..a652a4f 100644 --- a/src/hooks/useConfigStatus.ts +++ b/src/hooks/useConfigStatus.ts @@ -9,6 +9,8 @@ interface ConfigStatus { isFullyConfigured: boolean; isLoading: boolean; error: string | null; + autoMirrorStarred: boolean; + githubOwner: string; } // Cache to prevent duplicate API calls across components @@ -33,6 +35,8 @@ export function useConfigStatus(): ConfigStatus { isFullyConfigured: false, isLoading: true, error: null, + autoMirrorStarred: false, + githubOwner: '', }); // Track if this hook has already checked config to prevent multiple calls @@ -46,6 +50,8 @@ export function useConfigStatus(): ConfigStatus { isFullyConfigured: false, isLoading: false, error: 'No user found', + autoMirrorStarred: false, + githubOwner: '', }); return; } @@ -78,6 +84,8 @@ export function useConfigStatus(): ConfigStatus { isFullyConfigured, isLoading: false, error: null, + autoMirrorStarred: configResponse?.advancedOptions?.autoMirrorStarred ?? false, + githubOwner: configResponse?.githubConfig?.username ?? '', }); return; } @@ -119,6 +127,8 @@ export function useConfigStatus(): ConfigStatus { isFullyConfigured, isLoading: false, error: null, + autoMirrorStarred: configResponse?.advancedOptions?.autoMirrorStarred ?? false, + githubOwner: configResponse?.githubConfig?.username ?? '', }); hasCheckedRef.current = true; @@ -129,6 +139,8 @@ export function useConfigStatus(): ConfigStatus { isFullyConfigured: false, isLoading: false, error: error instanceof Error ? error.message : 'Failed to check configuration', + autoMirrorStarred: false, + githubOwner: '', }); hasCheckedRef.current = true; } diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 00a0ba5..0580341 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -29,6 +29,7 @@ export const githubConfigSchema = z.object({ mirrorStrategy: z.enum(["preserve", "single-org", "flat-user", "mixed"]).default("preserve"), defaultOrg: z.string().optional(), starredCodeOnly: z.boolean().default(false), + autoMirrorStarred: z.boolean().default(false), skipStarredIssues: z.boolean().optional(), // Deprecated: kept for backward compatibility, use starredCodeOnly instead starredDuplicateStrategy: z.enum(["suffix", "prefix", "owner-org"]).default("suffix").optional(), }); diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index 6f72382..ef2a7e9 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -22,6 +22,7 @@ interface EnvConfig { preserveOrgStructure?: boolean; onlyMirrorOrgs?: boolean; starredCodeOnly?: boolean; + autoMirrorStarred?: boolean; starredReposOrg?: string; starredReposMode?: 'dedicated-org' | 'preserve-owner'; mirrorStrategy?: 'preserve' | 'single-org' | 'flat-user' | 'mixed'; @@ -113,6 +114,7 @@ function parseEnvConfig(): EnvConfig { preserveOrgStructure: process.env.PRESERVE_ORG_STRUCTURE === 'true', onlyMirrorOrgs: process.env.ONLY_MIRROR_ORGS === 'true', starredCodeOnly: process.env.SKIP_STARRED_ISSUES === 'true', + autoMirrorStarred: process.env.AUTO_MIRROR_STARRED === 'true', starredReposOrg: process.env.STARRED_REPOS_ORG, starredReposMode: process.env.STARRED_REPOS_MODE as 'dedicated-org' | 'preserve-owner', mirrorStrategy: process.env.MIRROR_STRATEGY as 'preserve' | 'single-org' | 'flat-user' | 'mixed', @@ -264,6 +266,7 @@ export async function initializeConfigFromEnv(): Promise { mirrorStrategy, defaultOrg: envConfig.gitea.organization || existingConfig?.[0]?.githubConfig?.defaultOrg || 'github-mirrors', starredCodeOnly: envConfig.github.starredCodeOnly ?? existingConfig?.[0]?.githubConfig?.starredCodeOnly ?? false, + autoMirrorStarred: envConfig.github.autoMirrorStarred ?? existingConfig?.[0]?.githubConfig?.autoMirrorStarred ?? false, }; // Build Gitea config diff --git a/src/lib/repository-cleanup-service.ts b/src/lib/repository-cleanup-service.ts index 549a0a1..df7e6a4 100644 --- a/src/lib/repository-cleanup-service.ts +++ b/src/lib/repository-cleanup-service.ts @@ -79,6 +79,13 @@ async function identifyOrphanedRepositories(config: any): Promise { return false; } + // If starred repos are not being fetched from GitHub, we can't determine + // if a starred repo is orphaned - skip it to prevent data loss + if (repo.isStarred && !config.githubConfig?.includeStarred) { + console.log(`[Repository Cleanup] Skipping starred repo ${repo.fullName} - starred repos not being fetched from GitHub`); + return false; + } + const githubRepo = githubReposByFullName.get(repo.fullName); if (!githubRepo) { return true; diff --git a/src/lib/scheduler-service.ts b/src/lib/scheduler-service.ts index 8e0413d..105847d 100644 --- a/src/lib/scheduler-service.ts +++ b/src/lib/scheduler-service.ts @@ -13,6 +13,7 @@ import type { Repository } from '@/lib/db/schema'; import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository'; import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils'; import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility'; +import { createMirrorJob } from '@/lib/helpers'; let schedulerInterval: NodeJS.Timeout | null = null; let isSchedulerRunning = false; @@ -128,6 +129,19 @@ async function runScheduledSync(config: any): Promise { .onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] }); } console.log(`[Scheduler] Successfully imported ${newRepos.length} new repositories for user ${userId}`); + + // Log activity for each newly imported repo + for (const repo of newRepos) { + const sourceLabel = repo.isStarred ? 'starred' : 'owned'; + await createMirrorJob({ + userId, + repositoryName: repo.fullName, + message: `Auto-imported ${sourceLabel} repository: ${repo.fullName}`, + details: `Repository ${repo.fullName} was discovered and imported during scheduled sync.`, + status: 'imported', + skipDuplicateEvent: true, + }); + } } else { console.log(`[Scheduler] No new repositories found for user ${userId}`); } @@ -176,7 +190,7 @@ async function runScheduledSync(config: any): Promise { if (scheduleConfig.autoMirror) { try { console.log(`[Scheduler] Auto-mirror enabled - checking for repositories to mirror for user ${userId}...`); - const reposNeedingMirror = await db + let reposNeedingMirror = await db .select() .from(repositories) .where( @@ -190,6 +204,19 @@ async function runScheduledSync(config: any): Promise { ) ); + // Filter out starred repos from auto-mirror when autoMirrorStarred is disabled + if (!config.githubConfig?.autoMirrorStarred) { + const githubOwner = config.githubConfig?.owner || ''; + const beforeCount = reposNeedingMirror.length; + reposNeedingMirror = reposNeedingMirror.filter( + repo => !repo.isStarred || repo.owner === githubOwner + ); + const skippedCount = beforeCount - reposNeedingMirror.length; + if (skippedCount > 0) { + console.log(`[Scheduler] Skipped ${skippedCount} starred repositories from auto-mirror (autoMirrorStarred is disabled)`); + } + } + if (reposNeedingMirror.length > 0) { console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`); @@ -484,6 +511,19 @@ async function performInitialAutoStart(): Promise { .onConflictDoNothing({ target: [repositories.userId, repositories.normalizedFullName] }); } console.log(`[Scheduler] Successfully imported ${reposToImport.length} repositories`); + + // Log activity for each newly imported repo + for (const repo of reposToImport) { + const sourceLabel = repo.isStarred ? 'starred' : 'owned'; + await createMirrorJob({ + userId: config.userId, + repositoryName: repo.fullName, + message: `Auto-imported ${sourceLabel} repository: ${repo.fullName}`, + details: `Repository ${repo.fullName} was discovered and imported during auto-start.`, + status: 'imported', + skipDuplicateEvent: true, + }); + } } else { console.log(`[Scheduler] No new repositories to import for user ${config.userId}`); } @@ -491,7 +531,7 @@ async function performInitialAutoStart(): Promise { if (skippedDisabledCount > 0) { console.log(`[Scheduler] Skipped ${skippedDisabledCount} disabled GitHub repositories for user ${config.userId}`); } - + // Check if we already have mirrored repositories (indicating this isn't first run) const mirroredRepos = await db .select() @@ -534,8 +574,34 @@ async function performInitialAutoStart(): Promise { } // Step 2: Trigger mirror for all repositories that need mirroring + // Only auto-mirror if autoMirror is enabled in schedule config + if (!config.scheduleConfig?.autoMirror) { + console.log(`[Scheduler] Step 2: Skipping initial mirror - autoMirror is disabled for user ${config.userId}`); + + // Still update schedule config timestamps + const currentTime2 = new Date(); + const intervalSource2 = config.scheduleConfig?.interval || + config.giteaConfig?.mirrorInterval || + '8h'; + const interval2 = parseScheduleInterval(intervalSource2); + const nextRun2 = new Date(currentTime2.getTime() + interval2); + + await db.update(configs).set({ + scheduleConfig: { + ...config.scheduleConfig, + enabled: true, + lastRun: currentTime2, + nextRun: nextRun2, + }, + updatedAt: currentTime2, + }).where(eq(configs.id, config.id)); + + console.log(`[Scheduler] Scheduling enabled for user ${config.userId}, next sync at ${nextRun2.toISOString()}`); + continue; + } + console.log(`[Scheduler] Step 2: Triggering mirror for repositories that need mirroring...`); - const reposNeedingMirror = await db + let reposNeedingMirror = await db .select() .from(repositories) .where( @@ -548,7 +614,20 @@ async function performInitialAutoStart(): Promise { ) ) ); - + + // Filter out starred repos from auto-mirror when autoMirrorStarred is disabled + if (!config.githubConfig?.autoMirrorStarred) { + const githubOwner = config.githubConfig?.owner || ''; + const beforeCount = reposNeedingMirror.length; + reposNeedingMirror = reposNeedingMirror.filter( + repo => !repo.isStarred || repo.owner === githubOwner + ); + const skippedCount = beforeCount - reposNeedingMirror.length; + if (skippedCount > 0) { + console.log(`[Scheduler] Skipped ${skippedCount} starred repositories from initial auto-mirror (autoMirrorStarred is disabled)`); + } + } + if (reposNeedingMirror.length > 0) { console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need mirroring`); diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 4c2779f..8a91086 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -56,6 +56,7 @@ export function mapUiToDbConfig( // Advanced options starredCodeOnly: advancedOptions.starredCodeOnly, + autoMirrorStarred: advancedOptions.autoMirrorStarred ?? false, }; // Map Gitea config to match database schema @@ -172,6 +173,7 @@ export function mapDbToUiConfig(dbConfig: any): { skipForks: !(dbConfig.githubConfig?.includeForks ?? true), // Invert includeForks to get skipForks // Support both old (skipStarredIssues) and new (starredCodeOnly) field names for backward compatibility starredCodeOnly: dbConfig.githubConfig?.starredCodeOnly ?? (dbConfig.githubConfig as any)?.skipStarredIssues ?? false, + autoMirrorStarred: dbConfig.githubConfig?.autoMirrorStarred ?? false, }; return { diff --git a/src/types/config.ts b/src/types/config.ts index ca25e5d..534bf60 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -75,6 +75,7 @@ export interface MirrorOptions { export interface AdvancedOptions { skipForks: boolean; starredCodeOnly: boolean; + autoMirrorStarred?: boolean; } export interface SaveConfigApiRequest {