feat: add personal repositories organization override and update related configurations

This commit is contained in:
Arunavo Ray
2025-06-24 11:02:57 +05:30
parent d2bec1d56e
commit 68108b8383
12 changed files with 274 additions and 22 deletions

View File

@@ -322,9 +322,13 @@ Key configuration options include:
Gitea Mirror offers three flexible strategies for organizing your repositories in Gitea:
#### 1. **Preserve GitHub Structure** (Default)
- Personal repositories → Your Gitea username
- Organization repositories → Same organization name in Gitea
- Maintains the exact structure from GitHub
- Personal repositories → Your Gitea username (or custom organization)
- Organization repositories → Same organization name in Gitea (with individual overrides)
- Maintains the exact structure from GitHub with optional customization
**New Override Options:**
- **Personal Repos Override**: Redirect your personal repositories to a custom organization instead of your username
- **Organization Overrides**: Set custom destinations for specific GitHub organizations on their individual cards
#### 2. **Single Organization**
- All repositories → One designated organization
@@ -339,6 +343,13 @@ Gitea Mirror offers three flexible strategies for organizing your repositories i
> [!NOTE]
> **Starred Repositories**: Regardless of the chosen strategy, starred repositories are always mirrored to a separate organization (default: "starred") to keep them organized separately from your own repositories.
> [!TIP]
> **Example Use Case**: With the "Preserve" strategy and overrides, you can:
> - Mirror personal repos to `username-mirror` organization
> - Keep `company-org` repos in their own `company-org` organization
> - Override `community-scripts` to go to `community-mirrors` organization
> - This gives you complete control while maintaining GitHub's structure as the default
## 🚀 Development
### Local Development Setup

View File

@@ -217,6 +217,7 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g
strategy={mirrorStrategy}
destinationOrg={config.organization}
starredReposOrg={config.starredReposOrg}
personalReposOrg={config.personalReposOrg}
visibility={config.visibility}
onDestinationOrgChange={(org) => {
const newConfig = { ...config, organization: org };
@@ -228,6 +229,11 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g
setConfig(newConfig);
if (onAutoSave) onAutoSave(newConfig);
}}
onPersonalReposOrgChange={(org) => {
const newConfig = { ...config, personalReposOrg: org };
setConfig(newConfig);
if (onAutoSave) onAutoSave(newConfig);
}}
onVisibilityChange={(visibility) => {
const newConfig = { ...config, visibility };
setConfig(newConfig);

View File

@@ -15,9 +15,11 @@ interface OrganizationConfigurationProps {
strategy: MirrorStrategy;
destinationOrg?: string;
starredReposOrg?: string;
personalReposOrg?: string;
visibility: GiteaOrgVisibility;
onDestinationOrgChange: (org: string) => void;
onStarredReposOrgChange: (org: string) => void;
onPersonalReposOrgChange: (org: string) => void;
onVisibilityChange: (visibility: GiteaOrgVisibility) => void;
}
@@ -31,9 +33,11 @@ export const OrganizationConfiguration: React.FC<OrganizationConfigurationProps>
strategy,
destinationOrg,
starredReposOrg,
personalReposOrg,
visibility,
onDestinationOrgChange,
onStarredReposOrgChange,
onPersonalReposOrgChange,
onVisibilityChange,
}) => {
return (
@@ -75,7 +79,7 @@ export const OrganizationConfiguration: React.FC<OrganizationConfigurationProps>
</p>
</div>
{/* Right column - shows destination org for single-org, empty div for others */}
{/* Right column - shows destination org for single-org, personal repos org for preserve, empty div for others */}
{strategy === "single-org" ? (
<div className="space-y-1">
<Label htmlFor="destinationOrg" className="text-sm font-normal flex items-center gap-2">
@@ -102,6 +106,32 @@ export const OrganizationConfiguration: React.FC<OrganizationConfigurationProps>
Organization for consolidated repositories
</p>
</div>
) : strategy === "preserve" ? (
<div className="space-y-1">
<Label htmlFor="personalReposOrg" className="text-sm font-normal flex items-center gap-2">
Personal Repos Organization
<TooltipProvider>
<Tooltip>
<TooltipTrigger>
<Info className="h-3.5 w-3.5 text-muted-foreground" />
</TooltipTrigger>
<TooltipContent>
<p>Override where your personal repositories are mirrored (leave empty to use your username)</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</Label>
<Input
id="personalReposOrg"
value={personalReposOrg || ""}
onChange={(e) => onPersonalReposOrgChange(e.target.value)}
placeholder="my-personal-mirrors"
className=""
/>
<p className="text-xs text-muted-foreground mt-1">
Override destination for your personal repos
</p>
</div>
) : (
<div className="hidden md:block" />
)}

View File

@@ -1,8 +1,9 @@
import { useMemo } from "react";
import { useMemo, useState } from "react";
import { Card } from "@/components/ui/card";
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock } from "lucide-react";
import { Input } from "@/components/ui/input";
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, Settings, ArrowRight } from "lucide-react";
import { SiGithub } from "react-icons/si";
import type { Organization } from "@/lib/db/schema";
import type { FilterParams } from "@/types/filter";
@@ -140,6 +141,13 @@ export function OrganizationList({
{org.membershipRole}
</span>
</div>
{/* Show destination override if configured */}
{org.destinationOrg && (
<div className="flex items-center gap-1 text-xs text-muted-foreground mt-1">
<ArrowRight className="h-3 w-3" />
<span>Mirrors to: <span className="font-medium">{org.destinationOrg}</span></span>
</div>
)}
</div>
<Badge variant={statusBadge.variant} className="ml-2">
{StatusIcon && <StatusIcon className={cn(

View File

@@ -28,11 +28,34 @@ try {
// Ensure all required tables exist
ensureTablesExist(sqlite);
// Run migrations
runMigrations(sqlite);
} catch (error) {
console.error("Error opening database:", error);
throw error;
}
/**
* Run database migrations
*/
function runMigrations(db: Database) {
try {
// Migration 1: Add destination_org column to organizations table
const orgTableInfo = db.query("PRAGMA table_info(organizations)").all() as Array<{name: string}>;
const hasDestinationOrg = orgTableInfo.some(col => col.name === 'destination_org');
if (!hasDestinationOrg) {
console.log("🔄 Running migration: Adding destination_org column to organizations table");
db.exec("ALTER TABLE organizations ADD COLUMN destination_org TEXT");
console.log("✅ Migration completed: destination_org column added");
}
} catch (error) {
console.error("❌ Error running migrations:", error);
// Don't throw - migrations should be non-breaking
}
}
/**
* Ensure all required tables exist in the database
*/
@@ -159,6 +182,7 @@ function createTable(db: Database, tableName: string) {
last_mirrored INTEGER,
error_message TEXT,
repository_count INTEGER NOT NULL DEFAULT 0,
destination_org TEXT,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id),
@@ -437,6 +461,9 @@ export const organizations = sqliteTable("organizations", {
.notNull()
.default(true),
// Override destination organization for this GitHub org's repos
destinationOrg: text("destination_org"),
status: text("status").notNull().default("imported"),
lastMirrored: integer("last_mirrored", { mode: "timestamp" }),
errorMessage: text("error_message"),

View File

@@ -45,6 +45,7 @@ export const configSchema = z.object({
starredReposOrg: z.string().default("github"),
preserveOrgStructure: z.boolean().default(false),
mirrorStrategy: z.enum(["preserve", "single-org", "flat-user"]).optional(),
personalReposOrg: z.string().optional(), // Override destination for personal repos
}),
include: z.array(z.string()).default(["*"]),
exclude: z.array(z.string()).default([]),
@@ -158,6 +159,9 @@ export const organizationSchema = z.object({
privateRepositoryCount: z.number().optional(),
forkRepositoryCount: z.number().optional(),
// Override destination organization for this GitHub org's repos
destinationOrg: z.string().optional(),
createdAt: z.date().default(() => new Date()),
updatedAt: z.date().default(() => new Date()),
});

View File

@@ -1,7 +1,8 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
import { Octokit } from "@octokit/rest";
import { repoStatusEnum } from "@/types/Repository";
import { getOrCreateGiteaOrg } from "./gitea";
import { getOrCreateGiteaOrg, getGiteaRepoOwner, getGiteaRepoOwnerAsync } from "./gitea";
import type { Config, Repository, Organization } from "./db/schema";
// Mock the isRepoPresentInGitea function
const mockIsRepoPresentInGitea = mock(() => Promise.resolve(false));
@@ -291,3 +292,90 @@ describe("Gitea Repository Mirroring", () => {
expect(mockGetOrCreateGiteaOrg).toHaveBeenCalled();
});
});
describe("getGiteaRepoOwner - Organization Override Tests", () => {
const baseConfig: Partial<Config> = {
githubConfig: {
username: "testuser",
token: "token",
preserveOrgStructure: false,
skipForks: false,
privateRepositories: false,
mirrorIssues: false,
mirrorWiki: false,
mirrorStarred: false,
useSpecificUser: false,
includeOrgs: [],
excludeOrgs: [],
mirrorPublicOrgs: false,
publicOrgs: [],
skipStarredIssues: false
},
giteaConfig: {
username: "giteauser",
url: "https://gitea.example.com",
token: "gitea-token",
organization: "github-mirrors",
visibility: "public",
starredReposOrg: "starred",
preserveOrgStructure: false,
mirrorStrategy: "preserve"
}
};
const baseRepo: Repository = {
id: "repo-id",
userId: "user-id",
configId: "config-id",
name: "test-repo",
fullName: "testuser/test-repo",
url: "https://github.com/testuser/test-repo",
cloneUrl: "https://github.com/testuser/test-repo.git",
owner: "testuser",
isPrivate: false,
isForked: false,
hasIssues: true,
isStarred: false,
isArchived: false,
size: 1000,
hasLFS: false,
hasSubmodules: false,
defaultBranch: "main",
visibility: "public",
status: "imported",
mirroredLocation: "",
createdAt: new Date(),
updatedAt: new Date()
};
test("starred repos go to starredReposOrg", () => {
const repo = { ...baseRepo, isStarred: true };
const result = getGiteaRepoOwner({ config: baseConfig, repository: repo });
expect(result).toBe("starred");
});
test("preserve strategy: personal repos use personalReposOrg override", () => {
const configWithOverride = {
...baseConfig,
giteaConfig: {
...baseConfig.giteaConfig!,
personalReposOrg: "my-personal-mirrors"
}
};
const repo = { ...baseRepo, organization: undefined };
const result = getGiteaRepoOwner({ config: configWithOverride, repository: repo });
expect(result).toBe("my-personal-mirrors");
});
test("preserve strategy: personal repos fallback to username when no override", () => {
const repo = { ...baseRepo, organization: undefined };
const result = getGiteaRepoOwner({ config: baseConfig, repository: repo });
expect(result).toBe("giteauser");
});
test("preserve strategy: org repos go to same org name", () => {
const repo = { ...baseRepo, organization: "myorg" };
const result = getGiteaRepoOwner({ config: baseConfig, repository: repo });
expect(result).toBe("myorg");
});
});

View File

@@ -9,7 +9,81 @@ import type { Organization, Repository } from "./db/schema";
import { httpPost, httpGet } from "./http-client";
import { createMirrorJob } from "./helpers";
import { db, organizations, repositories } from "./db";
import { eq } from "drizzle-orm";
import { eq, and } from "drizzle-orm";
/**
* Helper function to get organization configuration including destination override
*/
export const getOrganizationConfig = async ({
orgName,
userId,
}: {
orgName: string;
userId: string;
}): Promise<Organization | null> => {
try {
const [orgConfig] = await db
.select()
.from(organizations)
.where(and(eq(organizations.name, orgName), eq(organizations.userId, userId)))
.limit(1);
return orgConfig || null;
} catch (error) {
console.error(`Error fetching organization config for ${orgName}:`, error);
return null;
}
};
/**
* Enhanced async version of getGiteaRepoOwner that supports organization overrides
*/
export const getGiteaRepoOwnerAsync = async ({
config,
repository,
}: {
config: Partial<Config>;
repository: Repository;
}): Promise<string> => {
if (!config.githubConfig || !config.giteaConfig) {
throw new Error("GitHub or Gitea config is required.");
}
if (!config.giteaConfig.username) {
throw new Error("Gitea username is required.");
}
if (!config.userId) {
throw new Error("User ID is required for organization overrides.");
}
// Check if repository is starred - starred repos always go to starredReposOrg (highest priority)
if (repository.isStarred && config.giteaConfig.starredReposOrg) {
return config.giteaConfig.starredReposOrg;
}
// Check for organization-specific override
if (repository.organization) {
const orgConfig = await getOrganizationConfig({
orgName: repository.organization,
userId: config.userId,
});
if (orgConfig?.destinationOrg) {
console.log(`Using organization override: ${repository.organization} -> ${orgConfig.destinationOrg}`);
return orgConfig.destinationOrg;
}
}
// Check for personal repos override (when it's user's repo, not an organization)
if (!repository.organization && config.giteaConfig.personalReposOrg) {
console.log(`Using personal repos override: ${config.giteaConfig.personalReposOrg}`);
return config.giteaConfig.personalReposOrg;
}
// Fall back to existing strategy logic
return getGiteaRepoOwner({ config, repository });
};
export const getGiteaRepoOwner = ({
config,
@@ -37,11 +111,12 @@ export const getGiteaRepoOwner = ({
switch (mirrorStrategy) {
case "preserve":
// Keep GitHub structure - org repos go to same org, personal repos to user
// Keep GitHub structure - org repos go to same org, personal repos to user (or override)
if (repository.organization) {
return repository.organization;
}
return config.giteaConfig.username;
// Use personal repos override if configured, otherwise use username
return config.giteaConfig.personalReposOrg || config.giteaConfig.username;
case "single-org":
// All non-starred repos go to the destination organization
@@ -160,8 +235,8 @@ export const mirrorGithubRepoToGitea = async ({
throw new Error("Gitea username is required.");
}
// Get the correct owner based on the strategy
const repoOwner = getGiteaRepoOwner({ config, repository });
// Get the correct owner based on the strategy (with organization overrides)
const repoOwner = await getGiteaRepoOwnerAsync({ config, repository });
const isExisting = await isRepoPresentInGitea({
config,
@@ -987,8 +1062,8 @@ export const syncGiteaRepo = async ({
status: repoStatusEnum.parse("syncing"),
});
// Get the expected owner based on current config
const repoOwner = getGiteaRepoOwner({ config, repository });
// Get the expected owner based on current config (with organization overrides)
const repoOwner = await getGiteaRepoOwnerAsync({ config, repository });
// Check if repo exists at the expected location or alternate location
const { present, actualOwner } = await checkRepoLocation({
@@ -1289,7 +1364,7 @@ export async function mirrorGitHubReleasesToGitea({
throw new Error("Gitea config is incomplete for mirroring releases.");
}
const repoOwner = getGiteaRepoOwner({
const repoOwner = await getGiteaRepoOwnerAsync({
config,
repository,
});

View File

@@ -36,6 +36,7 @@ interface DbGiteaConfig {
starredReposOrg: string;
preserveOrgStructure: boolean;
mirrorStrategy?: "preserve" | "single-org" | "flat-user";
personalReposOrg?: string;
}
/**
@@ -106,6 +107,7 @@ export function mapDbToUiConfig(dbConfig: any): {
starredReposOrg: dbConfig.giteaConfig?.starredReposOrg || "github",
preserveOrgStructure: dbConfig.giteaConfig?.preserveOrgStructure || false,
mirrorStrategy: dbConfig.giteaConfig?.mirrorStrategy,
personalReposOrg: dbConfig.giteaConfig?.personalReposOrg,
};
const mirrorOptions: MirrorOptions = {

View File

@@ -6,7 +6,7 @@ import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import {
mirrorGithubRepoToGitea,
mirrorGitHubOrgRepoToGiteaOrg,
getGiteaRepoOwner,
getGiteaRepoOwnerAsync,
} from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github";
import { processWithResilience } from "@/lib/utils/concurrency";
@@ -97,8 +97,8 @@ export const POST: APIRoute = async ({ request }) => {
// Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name}`);
// Determine where the repository should be mirrored
const owner = getGiteaRepoOwner({
// Determine where the repository should be mirrored (with organization overrides)
const owner = await getGiteaRepoOwnerAsync({
config,
repository: repoData,
});

View File

@@ -1,7 +1,7 @@
import type { APIRoute } from "astro";
import { db, configs, repositories } from "@/lib/db";
import { eq, inArray } from "drizzle-orm";
import { getGiteaRepoOwner, isRepoPresentInGitea } from "@/lib/gitea";
import { getGiteaRepoOwnerAsync, isRepoPresentInGitea } from "@/lib/gitea";
import {
mirrorGithubRepoToGitea,
mirrorGitHubOrgRepoToGiteaOrg,
@@ -109,8 +109,8 @@ export const POST: APIRoute = async ({ request }) => {
status: "imported",
});
// Determine if the repository exists in Gitea
let owner = getGiteaRepoOwner({
// Determine if the repository exists in Gitea (with organization overrides)
let owner = await getGiteaRepoOwnerAsync({
config,
repository: repoData,
});

View File

@@ -12,6 +12,7 @@ export interface GiteaConfig {
starredReposOrg: string;
preserveOrgStructure: boolean;
mirrorStrategy?: MirrorStrategy; // New field for the strategy
personalReposOrg?: string; // Override destination for personal repos
}
export interface ScheduleConfig {