feat: add personal repositories organization override and update related configurations

This commit is contained in:
Arunavo Ray
2025-06-24 11:02:57 +05:30
parent d2bec1d56e
commit 68108b8383
12 changed files with 274 additions and 22 deletions

View File

@@ -322,9 +322,13 @@ Key configuration options include:
Gitea Mirror offers three flexible strategies for organizing your repositories in Gitea: Gitea Mirror offers three flexible strategies for organizing your repositories in Gitea:
#### 1. **Preserve GitHub Structure** (Default) #### 1. **Preserve GitHub Structure** (Default)
- Personal repositories → Your Gitea username - Personal repositories → Your Gitea username (or custom organization)
- Organization repositories → Same organization name in Gitea - Organization repositories → Same organization name in Gitea (with individual overrides)
- Maintains the exact structure from GitHub - Maintains the exact structure from GitHub with optional customization
**New Override Options:**
- **Personal Repos Override**: Redirect your personal repositories to a custom organization instead of your username
- **Organization Overrides**: Set custom destinations for specific GitHub organizations on their individual cards
#### 2. **Single Organization** #### 2. **Single Organization**
- All repositories → One designated organization - All repositories → One designated organization
@@ -339,6 +343,13 @@ Gitea Mirror offers three flexible strategies for organizing your repositories i
> [!NOTE] > [!NOTE]
> **Starred Repositories**: Regardless of the chosen strategy, starred repositories are always mirrored to a separate organization (default: "starred") to keep them organized separately from your own repositories. > **Starred Repositories**: Regardless of the chosen strategy, starred repositories are always mirrored to a separate organization (default: "starred") to keep them organized separately from your own repositories.
> [!TIP]
> **Example Use Case**: With the "Preserve" strategy and overrides, you can:
> - Mirror personal repos to `username-mirror` organization
> - Keep `company-org` repos in their own `company-org` organization
> - Override `community-scripts` to go to `community-mirrors` organization
> - This gives you complete control while maintaining GitHub's structure as the default
## 🚀 Development ## 🚀 Development
### Local Development Setup ### Local Development Setup

View File

@@ -217,6 +217,7 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g
strategy={mirrorStrategy} strategy={mirrorStrategy}
destinationOrg={config.organization} destinationOrg={config.organization}
starredReposOrg={config.starredReposOrg} starredReposOrg={config.starredReposOrg}
personalReposOrg={config.personalReposOrg}
visibility={config.visibility} visibility={config.visibility}
onDestinationOrgChange={(org) => { onDestinationOrgChange={(org) => {
const newConfig = { ...config, organization: org }; const newConfig = { ...config, organization: org };
@@ -228,6 +229,11 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g
setConfig(newConfig); setConfig(newConfig);
if (onAutoSave) onAutoSave(newConfig); if (onAutoSave) onAutoSave(newConfig);
}} }}
onPersonalReposOrgChange={(org) => {
const newConfig = { ...config, personalReposOrg: org };
setConfig(newConfig);
if (onAutoSave) onAutoSave(newConfig);
}}
onVisibilityChange={(visibility) => { onVisibilityChange={(visibility) => {
const newConfig = { ...config, visibility }; const newConfig = { ...config, visibility };
setConfig(newConfig); setConfig(newConfig);

View File

@@ -15,9 +15,11 @@ interface OrganizationConfigurationProps {
strategy: MirrorStrategy; strategy: MirrorStrategy;
destinationOrg?: string; destinationOrg?: string;
starredReposOrg?: string; starredReposOrg?: string;
personalReposOrg?: string;
visibility: GiteaOrgVisibility; visibility: GiteaOrgVisibility;
onDestinationOrgChange: (org: string) => void; onDestinationOrgChange: (org: string) => void;
onStarredReposOrgChange: (org: string) => void; onStarredReposOrgChange: (org: string) => void;
onPersonalReposOrgChange: (org: string) => void;
onVisibilityChange: (visibility: GiteaOrgVisibility) => void; onVisibilityChange: (visibility: GiteaOrgVisibility) => void;
} }
@@ -31,9 +33,11 @@ export const OrganizationConfiguration: React.FC<OrganizationConfigurationProps>
strategy, strategy,
destinationOrg, destinationOrg,
starredReposOrg, starredReposOrg,
personalReposOrg,
visibility, visibility,
onDestinationOrgChange, onDestinationOrgChange,
onStarredReposOrgChange, onStarredReposOrgChange,
onPersonalReposOrgChange,
onVisibilityChange, onVisibilityChange,
}) => { }) => {
return ( return (
@@ -75,7 +79,7 @@ export const OrganizationConfiguration: React.FC<OrganizationConfigurationProps>
</p> </p>
</div> </div>
{/* Right column - shows destination org for single-org, empty div for others */} {/* Right column - shows destination org for single-org, personal repos org for preserve, empty div for others */}
{strategy === "single-org" ? ( {strategy === "single-org" ? (
<div className="space-y-1"> <div className="space-y-1">
<Label htmlFor="destinationOrg" className="text-sm font-normal flex items-center gap-2"> <Label htmlFor="destinationOrg" className="text-sm font-normal flex items-center gap-2">
@@ -102,6 +106,32 @@ export const OrganizationConfiguration: React.FC<OrganizationConfigurationProps>
Organization for consolidated repositories Organization for consolidated repositories
</p> </p>
</div> </div>
) : strategy === "preserve" ? (
<div className="space-y-1">
<Label htmlFor="personalReposOrg" className="text-sm font-normal flex items-center gap-2">
Personal Repos Organization
<TooltipProvider>
<Tooltip>
<TooltipTrigger>
<Info className="h-3.5 w-3.5 text-muted-foreground" />
</TooltipTrigger>
<TooltipContent>
<p>Override where your personal repositories are mirrored (leave empty to use your username)</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</Label>
<Input
id="personalReposOrg"
value={personalReposOrg || ""}
onChange={(e) => onPersonalReposOrgChange(e.target.value)}
placeholder="my-personal-mirrors"
className=""
/>
<p className="text-xs text-muted-foreground mt-1">
Override destination for your personal repos
</p>
</div>
) : ( ) : (
<div className="hidden md:block" /> <div className="hidden md:block" />
)} )}

View File

@@ -1,8 +1,9 @@
import { useMemo } from "react"; import { useMemo, useState } from "react";
import { Card } from "@/components/ui/card"; import { Card } from "@/components/ui/card";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge"; import { Badge } from "@/components/ui/badge";
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock } from "lucide-react"; import { Input } from "@/components/ui/input";
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, Settings, ArrowRight } from "lucide-react";
import { SiGithub } from "react-icons/si"; import { SiGithub } from "react-icons/si";
import type { Organization } from "@/lib/db/schema"; import type { Organization } from "@/lib/db/schema";
import type { FilterParams } from "@/types/filter"; import type { FilterParams } from "@/types/filter";
@@ -140,6 +141,13 @@ export function OrganizationList({
{org.membershipRole} {org.membershipRole}
</span> </span>
</div> </div>
{/* Show destination override if configured */}
{org.destinationOrg && (
<div className="flex items-center gap-1 text-xs text-muted-foreground mt-1">
<ArrowRight className="h-3 w-3" />
<span>Mirrors to: <span className="font-medium">{org.destinationOrg}</span></span>
</div>
)}
</div> </div>
<Badge variant={statusBadge.variant} className="ml-2"> <Badge variant={statusBadge.variant} className="ml-2">
{StatusIcon && <StatusIcon className={cn( {StatusIcon && <StatusIcon className={cn(

View File

@@ -28,11 +28,34 @@ try {
// Ensure all required tables exist // Ensure all required tables exist
ensureTablesExist(sqlite); ensureTablesExist(sqlite);
// Run migrations
runMigrations(sqlite);
} catch (error) { } catch (error) {
console.error("Error opening database:", error); console.error("Error opening database:", error);
throw error; throw error;
} }
/**
* Run database migrations
*/
function runMigrations(db: Database) {
try {
// Migration 1: Add destination_org column to organizations table
const orgTableInfo = db.query("PRAGMA table_info(organizations)").all() as Array<{name: string}>;
const hasDestinationOrg = orgTableInfo.some(col => col.name === 'destination_org');
if (!hasDestinationOrg) {
console.log("🔄 Running migration: Adding destination_org column to organizations table");
db.exec("ALTER TABLE organizations ADD COLUMN destination_org TEXT");
console.log("✅ Migration completed: destination_org column added");
}
} catch (error) {
console.error("❌ Error running migrations:", error);
// Don't throw - migrations should be non-breaking
}
}
/** /**
* Ensure all required tables exist in the database * Ensure all required tables exist in the database
*/ */
@@ -159,6 +182,7 @@ function createTable(db: Database, tableName: string) {
last_mirrored INTEGER, last_mirrored INTEGER,
error_message TEXT, error_message TEXT,
repository_count INTEGER NOT NULL DEFAULT 0, repository_count INTEGER NOT NULL DEFAULT 0,
destination_org TEXT,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id), FOREIGN KEY (user_id) REFERENCES users(id),
@@ -437,6 +461,9 @@ export const organizations = sqliteTable("organizations", {
.notNull() .notNull()
.default(true), .default(true),
// Override destination organization for this GitHub org's repos
destinationOrg: text("destination_org"),
status: text("status").notNull().default("imported"), status: text("status").notNull().default("imported"),
lastMirrored: integer("last_mirrored", { mode: "timestamp" }), lastMirrored: integer("last_mirrored", { mode: "timestamp" }),
errorMessage: text("error_message"), errorMessage: text("error_message"),

View File

@@ -45,6 +45,7 @@ export const configSchema = z.object({
starredReposOrg: z.string().default("github"), starredReposOrg: z.string().default("github"),
preserveOrgStructure: z.boolean().default(false), preserveOrgStructure: z.boolean().default(false),
mirrorStrategy: z.enum(["preserve", "single-org", "flat-user"]).optional(), mirrorStrategy: z.enum(["preserve", "single-org", "flat-user"]).optional(),
personalReposOrg: z.string().optional(), // Override destination for personal repos
}), }),
include: z.array(z.string()).default(["*"]), include: z.array(z.string()).default(["*"]),
exclude: z.array(z.string()).default([]), exclude: z.array(z.string()).default([]),
@@ -158,6 +159,9 @@ export const organizationSchema = z.object({
privateRepositoryCount: z.number().optional(), privateRepositoryCount: z.number().optional(),
forkRepositoryCount: z.number().optional(), forkRepositoryCount: z.number().optional(),
// Override destination organization for this GitHub org's repos
destinationOrg: z.string().optional(),
createdAt: z.date().default(() => new Date()), createdAt: z.date().default(() => new Date()),
updatedAt: z.date().default(() => new Date()), updatedAt: z.date().default(() => new Date()),
}); });

View File

@@ -1,7 +1,8 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
import { Octokit } from "@octokit/rest"; import { Octokit } from "@octokit/rest";
import { repoStatusEnum } from "@/types/Repository"; import { repoStatusEnum } from "@/types/Repository";
import { getOrCreateGiteaOrg } from "./gitea"; import { getOrCreateGiteaOrg, getGiteaRepoOwner, getGiteaRepoOwnerAsync } from "./gitea";
import type { Config, Repository, Organization } from "./db/schema";
// Mock the isRepoPresentInGitea function // Mock the isRepoPresentInGitea function
const mockIsRepoPresentInGitea = mock(() => Promise.resolve(false)); const mockIsRepoPresentInGitea = mock(() => Promise.resolve(false));
@@ -291,3 +292,90 @@ describe("Gitea Repository Mirroring", () => {
expect(mockGetOrCreateGiteaOrg).toHaveBeenCalled(); expect(mockGetOrCreateGiteaOrg).toHaveBeenCalled();
}); });
}); });
describe("getGiteaRepoOwner - Organization Override Tests", () => {
const baseConfig: Partial<Config> = {
githubConfig: {
username: "testuser",
token: "token",
preserveOrgStructure: false,
skipForks: false,
privateRepositories: false,
mirrorIssues: false,
mirrorWiki: false,
mirrorStarred: false,
useSpecificUser: false,
includeOrgs: [],
excludeOrgs: [],
mirrorPublicOrgs: false,
publicOrgs: [],
skipStarredIssues: false
},
giteaConfig: {
username: "giteauser",
url: "https://gitea.example.com",
token: "gitea-token",
organization: "github-mirrors",
visibility: "public",
starredReposOrg: "starred",
preserveOrgStructure: false,
mirrorStrategy: "preserve"
}
};
const baseRepo: Repository = {
id: "repo-id",
userId: "user-id",
configId: "config-id",
name: "test-repo",
fullName: "testuser/test-repo",
url: "https://github.com/testuser/test-repo",
cloneUrl: "https://github.com/testuser/test-repo.git",
owner: "testuser",
isPrivate: false,
isForked: false,
hasIssues: true,
isStarred: false,
isArchived: false,
size: 1000,
hasLFS: false,
hasSubmodules: false,
defaultBranch: "main",
visibility: "public",
status: "imported",
mirroredLocation: "",
createdAt: new Date(),
updatedAt: new Date()
};
test("starred repos go to starredReposOrg", () => {
const repo = { ...baseRepo, isStarred: true };
const result = getGiteaRepoOwner({ config: baseConfig, repository: repo });
expect(result).toBe("starred");
});
test("preserve strategy: personal repos use personalReposOrg override", () => {
const configWithOverride = {
...baseConfig,
giteaConfig: {
...baseConfig.giteaConfig!,
personalReposOrg: "my-personal-mirrors"
}
};
const repo = { ...baseRepo, organization: undefined };
const result = getGiteaRepoOwner({ config: configWithOverride, repository: repo });
expect(result).toBe("my-personal-mirrors");
});
test("preserve strategy: personal repos fallback to username when no override", () => {
const repo = { ...baseRepo, organization: undefined };
const result = getGiteaRepoOwner({ config: baseConfig, repository: repo });
expect(result).toBe("giteauser");
});
test("preserve strategy: org repos go to same org name", () => {
const repo = { ...baseRepo, organization: "myorg" };
const result = getGiteaRepoOwner({ config: baseConfig, repository: repo });
expect(result).toBe("myorg");
});
});

View File

@@ -9,7 +9,81 @@ import type { Organization, Repository } from "./db/schema";
import { httpPost, httpGet } from "./http-client"; import { httpPost, httpGet } from "./http-client";
import { createMirrorJob } from "./helpers"; import { createMirrorJob } from "./helpers";
import { db, organizations, repositories } from "./db"; import { db, organizations, repositories } from "./db";
import { eq } from "drizzle-orm"; import { eq, and } from "drizzle-orm";
/**
* Helper function to get organization configuration including destination override
*/
export const getOrganizationConfig = async ({
orgName,
userId,
}: {
orgName: string;
userId: string;
}): Promise<Organization | null> => {
try {
const [orgConfig] = await db
.select()
.from(organizations)
.where(and(eq(organizations.name, orgName), eq(organizations.userId, userId)))
.limit(1);
return orgConfig || null;
} catch (error) {
console.error(`Error fetching organization config for ${orgName}:`, error);
return null;
}
};
/**
* Enhanced async version of getGiteaRepoOwner that supports organization overrides
*/
export const getGiteaRepoOwnerAsync = async ({
config,
repository,
}: {
config: Partial<Config>;
repository: Repository;
}): Promise<string> => {
if (!config.githubConfig || !config.giteaConfig) {
throw new Error("GitHub or Gitea config is required.");
}
if (!config.giteaConfig.username) {
throw new Error("Gitea username is required.");
}
if (!config.userId) {
throw new Error("User ID is required for organization overrides.");
}
// Check if repository is starred - starred repos always go to starredReposOrg (highest priority)
if (repository.isStarred && config.giteaConfig.starredReposOrg) {
return config.giteaConfig.starredReposOrg;
}
// Check for organization-specific override
if (repository.organization) {
const orgConfig = await getOrganizationConfig({
orgName: repository.organization,
userId: config.userId,
});
if (orgConfig?.destinationOrg) {
console.log(`Using organization override: ${repository.organization} -> ${orgConfig.destinationOrg}`);
return orgConfig.destinationOrg;
}
}
// Check for personal repos override (when it's user's repo, not an organization)
if (!repository.organization && config.giteaConfig.personalReposOrg) {
console.log(`Using personal repos override: ${config.giteaConfig.personalReposOrg}`);
return config.giteaConfig.personalReposOrg;
}
// Fall back to existing strategy logic
return getGiteaRepoOwner({ config, repository });
};
export const getGiteaRepoOwner = ({ export const getGiteaRepoOwner = ({
config, config,
@@ -37,11 +111,12 @@ export const getGiteaRepoOwner = ({
switch (mirrorStrategy) { switch (mirrorStrategy) {
case "preserve": case "preserve":
// Keep GitHub structure - org repos go to same org, personal repos to user // Keep GitHub structure - org repos go to same org, personal repos to user (or override)
if (repository.organization) { if (repository.organization) {
return repository.organization; return repository.organization;
} }
return config.giteaConfig.username; // Use personal repos override if configured, otherwise use username
return config.giteaConfig.personalReposOrg || config.giteaConfig.username;
case "single-org": case "single-org":
// All non-starred repos go to the destination organization // All non-starred repos go to the destination organization
@@ -160,8 +235,8 @@ export const mirrorGithubRepoToGitea = async ({
throw new Error("Gitea username is required."); throw new Error("Gitea username is required.");
} }
// Get the correct owner based on the strategy // Get the correct owner based on the strategy (with organization overrides)
const repoOwner = getGiteaRepoOwner({ config, repository }); const repoOwner = await getGiteaRepoOwnerAsync({ config, repository });
const isExisting = await isRepoPresentInGitea({ const isExisting = await isRepoPresentInGitea({
config, config,
@@ -987,8 +1062,8 @@ export const syncGiteaRepo = async ({
status: repoStatusEnum.parse("syncing"), status: repoStatusEnum.parse("syncing"),
}); });
// Get the expected owner based on current config // Get the expected owner based on current config (with organization overrides)
const repoOwner = getGiteaRepoOwner({ config, repository }); const repoOwner = await getGiteaRepoOwnerAsync({ config, repository });
// Check if repo exists at the expected location or alternate location // Check if repo exists at the expected location or alternate location
const { present, actualOwner } = await checkRepoLocation({ const { present, actualOwner } = await checkRepoLocation({
@@ -1289,7 +1364,7 @@ export async function mirrorGitHubReleasesToGitea({
throw new Error("Gitea config is incomplete for mirroring releases."); throw new Error("Gitea config is incomplete for mirroring releases.");
} }
const repoOwner = getGiteaRepoOwner({ const repoOwner = await getGiteaRepoOwnerAsync({
config, config,
repository, repository,
}); });

View File

@@ -36,6 +36,7 @@ interface DbGiteaConfig {
starredReposOrg: string; starredReposOrg: string;
preserveOrgStructure: boolean; preserveOrgStructure: boolean;
mirrorStrategy?: "preserve" | "single-org" | "flat-user"; mirrorStrategy?: "preserve" | "single-org" | "flat-user";
personalReposOrg?: string;
} }
/** /**
@@ -106,6 +107,7 @@ export function mapDbToUiConfig(dbConfig: any): {
starredReposOrg: dbConfig.giteaConfig?.starredReposOrg || "github", starredReposOrg: dbConfig.giteaConfig?.starredReposOrg || "github",
preserveOrgStructure: dbConfig.giteaConfig?.preserveOrgStructure || false, preserveOrgStructure: dbConfig.giteaConfig?.preserveOrgStructure || false,
mirrorStrategy: dbConfig.giteaConfig?.mirrorStrategy, mirrorStrategy: dbConfig.giteaConfig?.mirrorStrategy,
personalReposOrg: dbConfig.giteaConfig?.personalReposOrg,
}; };
const mirrorOptions: MirrorOptions = { const mirrorOptions: MirrorOptions = {

View File

@@ -6,7 +6,7 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import { import {
mirrorGithubRepoToGitea, mirrorGithubRepoToGitea,
mirrorGitHubOrgRepoToGiteaOrg, mirrorGitHubOrgRepoToGiteaOrg,
getGiteaRepoOwner, getGiteaRepoOwnerAsync,
} from "@/lib/gitea"; } from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github"; import { createGitHubClient } from "@/lib/github";
import { processWithResilience } from "@/lib/utils/concurrency"; import { processWithResilience } from "@/lib/utils/concurrency";
@@ -97,8 +97,8 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of mirroring // Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name}`); console.log(`Starting mirror for repository: ${repo.name}`);
// Determine where the repository should be mirrored // Determine where the repository should be mirrored (with organization overrides)
const owner = getGiteaRepoOwner({ const owner = await getGiteaRepoOwnerAsync({
config, config,
repository: repoData, repository: repoData,
}); });

View File

@@ -1,7 +1,7 @@
import type { APIRoute } from "astro"; import type { APIRoute } from "astro";
import { db, configs, repositories } from "@/lib/db"; import { db, configs, repositories } from "@/lib/db";
import { eq, inArray } from "drizzle-orm"; import { eq, inArray } from "drizzle-orm";
import { getGiteaRepoOwner, isRepoPresentInGitea } from "@/lib/gitea"; import { getGiteaRepoOwnerAsync, isRepoPresentInGitea } from "@/lib/gitea";
import { import {
mirrorGithubRepoToGitea, mirrorGithubRepoToGitea,
mirrorGitHubOrgRepoToGiteaOrg, mirrorGitHubOrgRepoToGiteaOrg,
@@ -109,8 +109,8 @@ export const POST: APIRoute = async ({ request }) => {
status: "imported", status: "imported",
}); });
// Determine if the repository exists in Gitea // Determine if the repository exists in Gitea (with organization overrides)
let owner = getGiteaRepoOwner({ let owner = await getGiteaRepoOwnerAsync({
config, config,
repository: repoData, repository: repoData,
}); });

View File

@@ -12,6 +12,7 @@ export interface GiteaConfig {
starredReposOrg: string; starredReposOrg: string;
preserveOrgStructure: boolean; preserveOrgStructure: boolean;
mirrorStrategy?: MirrorStrategy; // New field for the strategy mirrorStrategy?: MirrorStrategy; // New field for the strategy
personalReposOrg?: string; // Override destination for personal repos
} }
export interface ScheduleConfig { export interface ScheduleConfig {