From 68108b8383df02409dfef122ad29be0578d6276d Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Tue, 24 Jun 2025 11:02:57 +0530 Subject: [PATCH] feat: add personal repositories organization override and update related configurations --- README.md | 17 +++- src/components/config/GiteaConfigForm.tsx | 6 ++ .../config/OrganizationConfiguration.tsx | 34 ++++++- .../organizations/OrganizationsList.tsx | 12 ++- src/lib/db/index.ts | 27 ++++++ src/lib/db/schema.ts | 4 + src/lib/gitea.test.ts | 90 +++++++++++++++++- src/lib/gitea.ts | 91 +++++++++++++++++-- src/lib/utils/config-mapper.ts | 2 + src/pages/api/job/mirror-repo.ts | 6 +- src/pages/api/job/retry-repo.ts | 6 +- src/types/config.ts | 1 + 12 files changed, 274 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d15873e..789f1dd 100644 --- a/README.md +++ b/README.md @@ -322,9 +322,13 @@ Key configuration options include: Gitea Mirror offers three flexible strategies for organizing your repositories in Gitea: #### 1. **Preserve GitHub Structure** (Default) -- Personal repositories → Your Gitea username -- Organization repositories → Same organization name in Gitea -- Maintains the exact structure from GitHub +- Personal repositories → Your Gitea username (or custom organization) +- Organization repositories → Same organization name in Gitea (with individual overrides) +- Maintains the exact structure from GitHub with optional customization + +**New Override Options:** +- **Personal Repos Override**: Redirect your personal repositories to a custom organization instead of your username +- **Organization Overrides**: Set custom destinations for specific GitHub organizations on their individual cards #### 2. **Single Organization** - All repositories → One designated organization @@ -339,6 +343,13 @@ Gitea Mirror offers three flexible strategies for organizing your repositories i > [!NOTE] > **Starred Repositories**: Regardless of the chosen strategy, starred repositories are always mirrored to a separate organization (default: "starred") to keep them organized separately from your own repositories. +> [!TIP] +> **Example Use Case**: With the "Preserve" strategy and overrides, you can: +> - Mirror personal repos to `username-mirror` organization +> - Keep `company-org` repos in their own `company-org` organization +> - Override `community-scripts` to go to `community-mirrors` organization +> - This gives you complete control while maintaining GitHub's structure as the default + ## 🚀 Development ### Local Development Setup diff --git a/src/components/config/GiteaConfigForm.tsx b/src/components/config/GiteaConfigForm.tsx index 1acc394..6f6f5f9 100644 --- a/src/components/config/GiteaConfigForm.tsx +++ b/src/components/config/GiteaConfigForm.tsx @@ -217,6 +217,7 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g strategy={mirrorStrategy} destinationOrg={config.organization} starredReposOrg={config.starredReposOrg} + personalReposOrg={config.personalReposOrg} visibility={config.visibility} onDestinationOrgChange={(org) => { const newConfig = { ...config, organization: org }; @@ -228,6 +229,11 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g setConfig(newConfig); if (onAutoSave) onAutoSave(newConfig); }} + onPersonalReposOrgChange={(org) => { + const newConfig = { ...config, personalReposOrg: org }; + setConfig(newConfig); + if (onAutoSave) onAutoSave(newConfig); + }} onVisibilityChange={(visibility) => { const newConfig = { ...config, visibility }; setConfig(newConfig); diff --git a/src/components/config/OrganizationConfiguration.tsx b/src/components/config/OrganizationConfiguration.tsx index 48711ee..b04545b 100644 --- a/src/components/config/OrganizationConfiguration.tsx +++ b/src/components/config/OrganizationConfiguration.tsx @@ -15,9 +15,11 @@ interface OrganizationConfigurationProps { strategy: MirrorStrategy; destinationOrg?: string; starredReposOrg?: string; + personalReposOrg?: string; visibility: GiteaOrgVisibility; onDestinationOrgChange: (org: string) => void; onStarredReposOrgChange: (org: string) => void; + onPersonalReposOrgChange: (org: string) => void; onVisibilityChange: (visibility: GiteaOrgVisibility) => void; } @@ -31,9 +33,11 @@ export const OrganizationConfiguration: React.FC strategy, destinationOrg, starredReposOrg, + personalReposOrg, visibility, onDestinationOrgChange, onStarredReposOrgChange, + onPersonalReposOrgChange, onVisibilityChange, }) => { return ( @@ -75,7 +79,7 @@ export const OrganizationConfiguration: React.FC

- {/* Right column - shows destination org for single-org, empty div for others */} + {/* Right column - shows destination org for single-org, personal repos org for preserve, empty div for others */} {strategy === "single-org" ? (
+ ) : strategy === "preserve" ? ( +
+ + onPersonalReposOrgChange(e.target.value)} + placeholder="my-personal-mirrors" + className="" + /> +

+ Override destination for your personal repos +

+
) : ( -
+
)}
diff --git a/src/components/organizations/OrganizationsList.tsx b/src/components/organizations/OrganizationsList.tsx index dbc3efe..1435f99 100644 --- a/src/components/organizations/OrganizationsList.tsx +++ b/src/components/organizations/OrganizationsList.tsx @@ -1,8 +1,9 @@ -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { Card } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; -import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock } from "lucide-react"; +import { Input } from "@/components/ui/input"; +import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, Settings, ArrowRight } from "lucide-react"; import { SiGithub } from "react-icons/si"; import type { Organization } from "@/lib/db/schema"; import type { FilterParams } from "@/types/filter"; @@ -140,6 +141,13 @@ export function OrganizationList({ {org.membershipRole}
+ {/* Show destination override if configured */} + {org.destinationOrg && ( +
+ + Mirrors to: {org.destinationOrg} +
+ )} {StatusIcon && ; + const hasDestinationOrg = orgTableInfo.some(col => col.name === 'destination_org'); + + if (!hasDestinationOrg) { + console.log("🔄 Running migration: Adding destination_org column to organizations table"); + db.exec("ALTER TABLE organizations ADD COLUMN destination_org TEXT"); + console.log("✅ Migration completed: destination_org column added"); + } + } catch (error) { + console.error("❌ Error running migrations:", error); + // Don't throw - migrations should be non-breaking + } +} + /** * Ensure all required tables exist in the database */ @@ -159,6 +182,7 @@ function createTable(db: Database, tableName: string) { last_mirrored INTEGER, error_message TEXT, repository_count INTEGER NOT NULL DEFAULT 0, + destination_org TEXT, created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), FOREIGN KEY (user_id) REFERENCES users(id), @@ -437,6 +461,9 @@ export const organizations = sqliteTable("organizations", { .notNull() .default(true), + // Override destination organization for this GitHub org's repos + destinationOrg: text("destination_org"), + status: text("status").notNull().default("imported"), lastMirrored: integer("last_mirrored", { mode: "timestamp" }), errorMessage: text("error_message"), diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 45887d0..06ed135 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -45,6 +45,7 @@ export const configSchema = z.object({ starredReposOrg: z.string().default("github"), preserveOrgStructure: z.boolean().default(false), mirrorStrategy: z.enum(["preserve", "single-org", "flat-user"]).optional(), + personalReposOrg: z.string().optional(), // Override destination for personal repos }), include: z.array(z.string()).default(["*"]), exclude: z.array(z.string()).default([]), @@ -158,6 +159,9 @@ export const organizationSchema = z.object({ privateRepositoryCount: z.number().optional(), forkRepositoryCount: z.number().optional(), + // Override destination organization for this GitHub org's repos + destinationOrg: z.string().optional(), + createdAt: z.date().default(() => new Date()), updatedAt: z.date().default(() => new Date()), }); diff --git a/src/lib/gitea.test.ts b/src/lib/gitea.test.ts index fc9dd07..6c4183f 100644 --- a/src/lib/gitea.test.ts +++ b/src/lib/gitea.test.ts @@ -1,7 +1,8 @@ import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; import { Octokit } from "@octokit/rest"; import { repoStatusEnum } from "@/types/Repository"; -import { getOrCreateGiteaOrg } from "./gitea"; +import { getOrCreateGiteaOrg, getGiteaRepoOwner, getGiteaRepoOwnerAsync } from "./gitea"; +import type { Config, Repository, Organization } from "./db/schema"; // Mock the isRepoPresentInGitea function const mockIsRepoPresentInGitea = mock(() => Promise.resolve(false)); @@ -291,3 +292,90 @@ describe("Gitea Repository Mirroring", () => { expect(mockGetOrCreateGiteaOrg).toHaveBeenCalled(); }); }); + +describe("getGiteaRepoOwner - Organization Override Tests", () => { + const baseConfig: Partial = { + githubConfig: { + username: "testuser", + token: "token", + preserveOrgStructure: false, + skipForks: false, + privateRepositories: false, + mirrorIssues: false, + mirrorWiki: false, + mirrorStarred: false, + useSpecificUser: false, + includeOrgs: [], + excludeOrgs: [], + mirrorPublicOrgs: false, + publicOrgs: [], + skipStarredIssues: false + }, + giteaConfig: { + username: "giteauser", + url: "https://gitea.example.com", + token: "gitea-token", + organization: "github-mirrors", + visibility: "public", + starredReposOrg: "starred", + preserveOrgStructure: false, + mirrorStrategy: "preserve" + } + }; + + const baseRepo: Repository = { + id: "repo-id", + userId: "user-id", + configId: "config-id", + name: "test-repo", + fullName: "testuser/test-repo", + url: "https://github.com/testuser/test-repo", + cloneUrl: "https://github.com/testuser/test-repo.git", + owner: "testuser", + isPrivate: false, + isForked: false, + hasIssues: true, + isStarred: false, + isArchived: false, + size: 1000, + hasLFS: false, + hasSubmodules: false, + defaultBranch: "main", + visibility: "public", + status: "imported", + mirroredLocation: "", + createdAt: new Date(), + updatedAt: new Date() + }; + + test("starred repos go to starredReposOrg", () => { + const repo = { ...baseRepo, isStarred: true }; + const result = getGiteaRepoOwner({ config: baseConfig, repository: repo }); + expect(result).toBe("starred"); + }); + + test("preserve strategy: personal repos use personalReposOrg override", () => { + const configWithOverride = { + ...baseConfig, + giteaConfig: { + ...baseConfig.giteaConfig!, + personalReposOrg: "my-personal-mirrors" + } + }; + const repo = { ...baseRepo, organization: undefined }; + const result = getGiteaRepoOwner({ config: configWithOverride, repository: repo }); + expect(result).toBe("my-personal-mirrors"); + }); + + test("preserve strategy: personal repos fallback to username when no override", () => { + const repo = { ...baseRepo, organization: undefined }; + const result = getGiteaRepoOwner({ config: baseConfig, repository: repo }); + expect(result).toBe("giteauser"); + }); + + test("preserve strategy: org repos go to same org name", () => { + const repo = { ...baseRepo, organization: "myorg" }; + const result = getGiteaRepoOwner({ config: baseConfig, repository: repo }); + expect(result).toBe("myorg"); + }); +}); diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index c6a5ed9..fa6c33b 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -9,7 +9,81 @@ import type { Organization, Repository } from "./db/schema"; import { httpPost, httpGet } from "./http-client"; import { createMirrorJob } from "./helpers"; import { db, organizations, repositories } from "./db"; -import { eq } from "drizzle-orm"; +import { eq, and } from "drizzle-orm"; + +/** + * Helper function to get organization configuration including destination override + */ +export const getOrganizationConfig = async ({ + orgName, + userId, +}: { + orgName: string; + userId: string; +}): Promise => { + try { + const [orgConfig] = await db + .select() + .from(organizations) + .where(and(eq(organizations.name, orgName), eq(organizations.userId, userId))) + .limit(1); + + return orgConfig || null; + } catch (error) { + console.error(`Error fetching organization config for ${orgName}:`, error); + return null; + } +}; + +/** + * Enhanced async version of getGiteaRepoOwner that supports organization overrides + */ +export const getGiteaRepoOwnerAsync = async ({ + config, + repository, +}: { + config: Partial; + repository: Repository; +}): Promise => { + if (!config.githubConfig || !config.giteaConfig) { + throw new Error("GitHub or Gitea config is required."); + } + + if (!config.giteaConfig.username) { + throw new Error("Gitea username is required."); + } + + if (!config.userId) { + throw new Error("User ID is required for organization overrides."); + } + + // Check if repository is starred - starred repos always go to starredReposOrg (highest priority) + if (repository.isStarred && config.giteaConfig.starredReposOrg) { + return config.giteaConfig.starredReposOrg; + } + + // Check for organization-specific override + if (repository.organization) { + const orgConfig = await getOrganizationConfig({ + orgName: repository.organization, + userId: config.userId, + }); + + if (orgConfig?.destinationOrg) { + console.log(`Using organization override: ${repository.organization} -> ${orgConfig.destinationOrg}`); + return orgConfig.destinationOrg; + } + } + + // Check for personal repos override (when it's user's repo, not an organization) + if (!repository.organization && config.giteaConfig.personalReposOrg) { + console.log(`Using personal repos override: ${config.giteaConfig.personalReposOrg}`); + return config.giteaConfig.personalReposOrg; + } + + // Fall back to existing strategy logic + return getGiteaRepoOwner({ config, repository }); +}; export const getGiteaRepoOwner = ({ config, @@ -37,11 +111,12 @@ export const getGiteaRepoOwner = ({ switch (mirrorStrategy) { case "preserve": - // Keep GitHub structure - org repos go to same org, personal repos to user + // Keep GitHub structure - org repos go to same org, personal repos to user (or override) if (repository.organization) { return repository.organization; } - return config.giteaConfig.username; + // Use personal repos override if configured, otherwise use username + return config.giteaConfig.personalReposOrg || config.giteaConfig.username; case "single-org": // All non-starred repos go to the destination organization @@ -160,8 +235,8 @@ export const mirrorGithubRepoToGitea = async ({ throw new Error("Gitea username is required."); } - // Get the correct owner based on the strategy - const repoOwner = getGiteaRepoOwner({ config, repository }); + // Get the correct owner based on the strategy (with organization overrides) + const repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); const isExisting = await isRepoPresentInGitea({ config, @@ -987,8 +1062,8 @@ export const syncGiteaRepo = async ({ status: repoStatusEnum.parse("syncing"), }); - // Get the expected owner based on current config - const repoOwner = getGiteaRepoOwner({ config, repository }); + // Get the expected owner based on current config (with organization overrides) + const repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); // Check if repo exists at the expected location or alternate location const { present, actualOwner } = await checkRepoLocation({ @@ -1289,7 +1364,7 @@ export async function mirrorGitHubReleasesToGitea({ throw new Error("Gitea config is incomplete for mirroring releases."); } - const repoOwner = getGiteaRepoOwner({ + const repoOwner = await getGiteaRepoOwnerAsync({ config, repository, }); diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 7cd644f..8b157b9 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -36,6 +36,7 @@ interface DbGiteaConfig { starredReposOrg: string; preserveOrgStructure: boolean; mirrorStrategy?: "preserve" | "single-org" | "flat-user"; + personalReposOrg?: string; } /** @@ -106,6 +107,7 @@ export function mapDbToUiConfig(dbConfig: any): { starredReposOrg: dbConfig.giteaConfig?.starredReposOrg || "github", preserveOrgStructure: dbConfig.giteaConfig?.preserveOrgStructure || false, mirrorStrategy: dbConfig.giteaConfig?.mirrorStrategy, + personalReposOrg: dbConfig.giteaConfig?.personalReposOrg, }; const mirrorOptions: MirrorOptions = { diff --git a/src/pages/api/job/mirror-repo.ts b/src/pages/api/job/mirror-repo.ts index 95ee671..4e6acea 100644 --- a/src/pages/api/job/mirror-repo.ts +++ b/src/pages/api/job/mirror-repo.ts @@ -6,7 +6,7 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository"; import { mirrorGithubRepoToGitea, mirrorGitHubOrgRepoToGiteaOrg, - getGiteaRepoOwner, + getGiteaRepoOwnerAsync, } from "@/lib/gitea"; import { createGitHubClient } from "@/lib/github"; import { processWithResilience } from "@/lib/utils/concurrency"; @@ -97,8 +97,8 @@ export const POST: APIRoute = async ({ request }) => { // Log the start of mirroring console.log(`Starting mirror for repository: ${repo.name}`); - // Determine where the repository should be mirrored - const owner = getGiteaRepoOwner({ + // Determine where the repository should be mirrored (with organization overrides) + const owner = await getGiteaRepoOwnerAsync({ config, repository: repoData, }); diff --git a/src/pages/api/job/retry-repo.ts b/src/pages/api/job/retry-repo.ts index fc191a0..f283629 100644 --- a/src/pages/api/job/retry-repo.ts +++ b/src/pages/api/job/retry-repo.ts @@ -1,7 +1,7 @@ import type { APIRoute } from "astro"; import { db, configs, repositories } from "@/lib/db"; import { eq, inArray } from "drizzle-orm"; -import { getGiteaRepoOwner, isRepoPresentInGitea } from "@/lib/gitea"; +import { getGiteaRepoOwnerAsync, isRepoPresentInGitea } from "@/lib/gitea"; import { mirrorGithubRepoToGitea, mirrorGitHubOrgRepoToGiteaOrg, @@ -109,8 +109,8 @@ export const POST: APIRoute = async ({ request }) => { status: "imported", }); - // Determine if the repository exists in Gitea - let owner = getGiteaRepoOwner({ + // Determine if the repository exists in Gitea (with organization overrides) + let owner = await getGiteaRepoOwnerAsync({ config, repository: repoData, }); diff --git a/src/types/config.ts b/src/types/config.ts index 01fbba1..4fbdeb5 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -12,6 +12,7 @@ export interface GiteaConfig { starredReposOrg: string; preserveOrgStructure: boolean; mirrorStrategy?: MirrorStrategy; // New field for the strategy + personalReposOrg?: string; // Override destination for personal repos } export interface ScheduleConfig {