diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index a3d061a..0a649e0 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -172,6 +172,7 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then owner TEXT NOT NULL, organization TEXT, mirrored_location TEXT DEFAULT '', + destination_org TEXT, is_private INTEGER NOT NULL DEFAULT 0, is_fork INTEGER NOT NULL DEFAULT 0, forked_from TEXT, @@ -181,6 +182,8 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then size INTEGER NOT NULL DEFAULT 0, has_lfs INTEGER NOT NULL DEFAULT 0, has_submodules INTEGER NOT NULL DEFAULT 0, + language TEXT, + description TEXT, default_branch TEXT NOT NULL, visibility TEXT NOT NULL DEFAULT 'public', status TEXT NOT NULL DEFAULT 'imported', @@ -192,6 +195,8 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then FOREIGN KEY (config_id) REFERENCES configs(id) ); + -- Uniqueness of (user_id, full_name) for repositories is enforced via drizzle migrations + CREATE TABLE IF NOT EXISTS organizations ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, diff --git a/drizzle/0005_polite_preak.sql b/drizzle/0005_polite_preak.sql new file mode 100644 index 0000000..f1b8fb2 --- /dev/null +++ b/drizzle/0005_polite_preak.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX `uniq_repositories_user_full_name` ON `repositories` (`user_id`,`full_name`); \ No newline at end of file diff --git a/drizzle/meta/0005_snapshot.json b/drizzle/meta/0005_snapshot.json new file mode 100644 index 0000000..799e42e --- /dev/null +++ b/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1941 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "4b92c9e0-363d-4da1-8d8d-c2e6c31cbb70", + "prevId": "284b8db7-ef64-4311-b890-0251bcbe5866", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_user_id": { + "name": "provider_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_accounts_account_id": { + "name": "idx_accounts_account_id", + "columns": [ + "account_id" + ], + "isUnique": false + }, + "idx_accounts_user_id": { + "name": "idx_accounts_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_accounts_provider": { + "name": "idx_accounts_provider", + "columns": [ + "provider_id", + "provider_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "configs": { + "name": "configs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "github_config": { + "name": "github_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "gitea_config": { + "name": "gitea_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "include": { + "name": "include", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[\"*\"]'" + }, + "exclude": { + "name": "exclude", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "schedule_config": { + "name": "schedule_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cleanup_config": { + "name": "cleanup_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": { + "configs_user_id_users_id_fk": { + "name": "configs_user_id_users_id_fk", + "tableFrom": "configs", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "read": { + "name": "read", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_events_user_channel": { + "name": "idx_events_user_channel", + "columns": [ + "user_id", + "channel" + ], + "isUnique": false + }, + "idx_events_created_at": { + "name": "idx_events_created_at", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "idx_events_read": { + "name": "idx_events_read", + "columns": [ + "read" + ], + "isUnique": false + } + }, + "foreignKeys": { + "events_user_id_users_id_fk": { + "name": "events_user_id_users_id_fk", + "tableFrom": "events", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mirror_jobs": { + "name": "mirror_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_name": { + "name": "repository_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_name": { + "name": "organization_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "job_type": { + "name": "job_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mirror'" + }, + "batch_id": { + "name": "batch_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_items": { + "name": "total_items", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_items": { + "name": "completed_items", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "item_ids": { + "name": "item_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_item_ids": { + "name": "completed_item_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'[]'" + }, + "in_progress": { + "name": "in_progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_mirror_jobs_user_id": { + "name": "idx_mirror_jobs_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_mirror_jobs_batch_id": { + "name": "idx_mirror_jobs_batch_id", + "columns": [ + "batch_id" + ], + "isUnique": false + }, + "idx_mirror_jobs_in_progress": { + "name": "idx_mirror_jobs_in_progress", + "columns": [ + "in_progress" + ], + "isUnique": false + }, + "idx_mirror_jobs_job_type": { + "name": "idx_mirror_jobs_job_type", + "columns": [ + "job_type" + ], + "isUnique": false + }, + "idx_mirror_jobs_timestamp": { + "name": "idx_mirror_jobs_timestamp", + "columns": [ + "timestamp" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mirror_jobs_user_id_users_id_fk": { + "name": "mirror_jobs_user_id_users_id_fk", + "tableFrom": "mirror_jobs", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_access_tokens": { + "name": "oauth_access_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_oauth_access_tokens_access_token": { + "name": "idx_oauth_access_tokens_access_token", + "columns": [ + "access_token" + ], + "isUnique": false + }, + "idx_oauth_access_tokens_user_id": { + "name": "idx_oauth_access_tokens_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_oauth_access_tokens_client_id": { + "name": "idx_oauth_access_tokens_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_access_tokens_user_id_users_id_fk": { + "name": "oauth_access_tokens_user_id_users_id_fk", + "tableFrom": "oauth_access_tokens", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_applications": { + "name": "oauth_applications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_urls": { + "name": "redirect_urls", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disabled": { + "name": "disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "oauth_applications_client_id_unique": { + "name": "oauth_applications_client_id_unique", + "columns": [ + "client_id" + ], + "isUnique": true + }, + "idx_oauth_applications_client_id": { + "name": "idx_oauth_applications_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_applications_user_id": { + "name": "idx_oauth_applications_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_consent": { + "name": "oauth_consent", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consent_given": { + "name": "consent_given", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_oauth_consent_user_id": { + "name": "idx_oauth_consent_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_oauth_consent_client_id": { + "name": "idx_oauth_consent_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_consent_user_client": { + "name": "idx_oauth_consent_user_client", + "columns": [ + "user_id", + "client_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_consent_user_id_users_id_fk": { + "name": "oauth_consent_user_id_users_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "membership_role": { + "name": "membership_role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "is_included": { + "name": "is_included", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "destination_org": { + "name": "destination_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "last_mirrored": { + "name": "last_mirrored", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_count": { + "name": "repository_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_repository_count": { + "name": "public_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_repository_count": { + "name": "private_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fork_repository_count": { + "name": "fork_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_organizations_user_id": { + "name": "idx_organizations_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_organizations_config_id": { + "name": "idx_organizations_config_id", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "idx_organizations_status": { + "name": "idx_organizations_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_organizations_is_included": { + "name": "idx_organizations_is_included", + "columns": [ + "is_included" + ], + "isUnique": false + } + }, + "foreignKeys": { + "organizations_user_id_users_id_fk": { + "name": "organizations_user_id_users_id_fk", + "tableFrom": "organizations", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "organizations_config_id_configs_id_fk": { + "name": "organizations_config_id_configs_id_fk", + "tableFrom": "organizations", + "tableTo": "configs", + "columnsFrom": [ + "config_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rate_limits": { + "name": "rate_limits", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'github'" + }, + "limit": { + "name": "limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reset": { + "name": "reset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retry_after": { + "name": "retry_after", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'ok'" + }, + "last_checked": { + "name": "last_checked", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_rate_limits_user_provider": { + "name": "idx_rate_limits_user_provider", + "columns": [ + "user_id", + "provider" + ], + "isUnique": false + }, + "idx_rate_limits_status": { + "name": "idx_rate_limits_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "rate_limits_user_id_users_id_fk": { + "name": "rate_limits_user_id_users_id_fk", + "tableFrom": "rate_limits", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "clone_url": { + "name": "clone_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization": { + "name": "organization", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mirrored_location": { + "name": "mirrored_location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "is_private": { + "name": "is_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_fork": { + "name": "is_fork", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_issues": { + "name": "has_issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_starred": { + "name": "is_starred", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_archived": { + "name": "is_archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_lfs": { + "name": "has_lfs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "has_submodules": { + "name": "has_submodules", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "last_mirrored": { + "name": "last_mirrored", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination_org": { + "name": "destination_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_repositories_user_id": { + "name": "idx_repositories_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_repositories_config_id": { + "name": "idx_repositories_config_id", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "idx_repositories_status": { + "name": "idx_repositories_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_repositories_owner": { + "name": "idx_repositories_owner", + "columns": [ + "owner" + ], + "isUnique": false + }, + "idx_repositories_organization": { + "name": "idx_repositories_organization", + "columns": [ + "organization" + ], + "isUnique": false + }, + "idx_repositories_is_fork": { + "name": "idx_repositories_is_fork", + "columns": [ + "is_fork" + ], + "isUnique": false + }, + "idx_repositories_is_starred": { + "name": "idx_repositories_is_starred", + "columns": [ + "is_starred" + ], + "isUnique": false + }, + "uniq_repositories_user_full_name": { + "name": "uniq_repositories_user_full_name", + "columns": [ + "user_id", + "full_name" + ], + "isUnique": true + } + }, + "foreignKeys": { + "repositories_user_id_users_id_fk": { + "name": "repositories_user_id_users_id_fk", + "tableFrom": "repositories", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "repositories_config_id_configs_id_fk": { + "name": "repositories_config_id_configs_id_fk", + "tableFrom": "repositories", + "tableTo": "configs", + "columnsFrom": [ + "config_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_sessions_user_id": { + "name": "idx_sessions_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_sessions_token": { + "name": "idx_sessions_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_sessions_expires_at": { + "name": "idx_sessions_expires_at", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sso_providers": { + "name": "sso_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "columns": [ + "provider_id" + ], + "isUnique": true + }, + "idx_sso_providers_provider_id": { + "name": "idx_sso_providers_provider_id", + "columns": [ + "provider_id" + ], + "isUnique": false + }, + "idx_sso_providers_domain": { + "name": "idx_sso_providers_domain", + "columns": [ + "domain" + ], + "isUnique": false + }, + "idx_sso_providers_issuer": { + "name": "idx_sso_providers_issuer", + "columns": [ + "issuer" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_email_unique": { + "name": "users_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_verification_tokens_token": { + "name": "idx_verification_tokens_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_verification_tokens_identifier": { + "name": "idx_verification_tokens_identifier", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verifications": { + "name": "verifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_verifications_identifier": { + "name": "idx_verifications_identifier", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 8f2126f..ba5f7f0 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1757392620734, "tag": "0004_grey_butterfly", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1757786449446, + "tag": "0005_polite_preak", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/components/config/GitHubMirrorSettings.tsx b/src/components/config/GitHubMirrorSettings.tsx index 9fd67c3..006c88a 100644 --- a/src/components/config/GitHubMirrorSettings.tsx +++ b/src/components/config/GitHubMirrorSettings.tsx @@ -32,7 +32,14 @@ import { Funnel, HardDrive } from "lucide-react"; -import type { GitHubConfig, MirrorOptions, AdvancedOptions } from "@/types/config"; +import type { GitHubConfig, MirrorOptions, AdvancedOptions, DuplicateNameStrategy } from "@/types/config"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { cn } from "@/lib/utils"; interface GitHubMirrorSettingsProps { @@ -53,7 +60,7 @@ export function GitHubMirrorSettings({ onAdvancedOptionsChange, }: GitHubMirrorSettingsProps) { - const handleGitHubChange = (field: keyof GitHubConfig, value: boolean) => { + const handleGitHubChange = (field: keyof GitHubConfig, value: boolean | string) => { onGitHubConfigChange({ ...githubConfig, [field]: value }); }; @@ -278,6 +285,34 @@ export function GitHubMirrorSettings({ + + {/* Duplicate name handling for starred repos */} + {githubConfig.mirrorStarred && ( +
+ + +

+ Handle starred repos with duplicate names from different owners. Currently supports suffix and prefix strategies. +

+
+ )} @@ -596,4 +631,4 @@ export function GitHubMirrorSettings({ ); -} \ No newline at end of file +} diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 17b2153..9f87831 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core"; +import { sqliteTable, text, integer, index, uniqueIndex } from "drizzle-orm/sqlite-core"; import { sql } from "drizzle-orm"; // ===== Zod Validation Schemas ===== @@ -28,6 +28,7 @@ export const githubConfigSchema = z.object({ mirrorStrategy: z.enum(["preserve", "single-org", "flat-user", "mixed"]).default("preserve"), defaultOrg: z.string().optional(), skipStarredIssues: z.boolean().default(false), + starredDuplicateStrategy: z.enum(["suffix", "prefix", "owner-org"]).default("suffix").optional(), }); export const giteaConfigSchema = z.object({ @@ -379,6 +380,7 @@ export const repositories = sqliteTable("repositories", { index("idx_repositories_organization").on(table.organization), index("idx_repositories_is_fork").on(table.isForked), index("idx_repositories_is_starred").on(table.isStarred), + uniqueIndex("uniq_repositories_user_full_name").on(table.userId, table.fullName), ]); export const mirrorJobs = sqliteTable("mirror_jobs", { @@ -674,4 +676,4 @@ export type Repository = z.infer; export type MirrorJob = z.infer; export type Organization = z.infer; export type Event = z.infer; -export type RateLimit = z.infer; \ No newline at end of file +export type RateLimit = z.infer; diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index 211ad42..ee80997 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -274,15 +274,37 @@ export const mirrorGithubRepoToGitea = async ({ // Get the correct owner based on the strategy (with organization overrides) let repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); + // Determine the actual repository name to use (handle duplicates for starred repos) + let targetRepoName = repository.name; + + if (repository.isStarred && config.githubConfig) { + // Extract GitHub owner from full_name (format: owner/repo) + const githubOwner = repository.fullName.split('/')[0]; + + targetRepoName = await generateUniqueRepoName({ + config, + orgName: repoOwner, + baseName: repository.name, + githubOwner, + strategy: config.githubConfig.starredDuplicateStrategy, + }); + + if (targetRepoName !== repository.name) { + console.log( + `Starred repo ${repository.fullName} will be mirrored as ${repoOwner}/${targetRepoName} to avoid naming conflict` + ); + } + } + const isExisting = await isRepoPresentInGitea({ config, owner: repoOwner, - repoName: repository.name, + repoName: targetRepoName, }); if (isExisting) { console.log( - `Repository ${repository.name} already exists in Gitea under ${repoOwner}. Updating database status.` + `Repository ${targetRepoName} already exists in Gitea under ${repoOwner}. Updating database status.` ); // Update database to reflect that the repository is already mirrored @@ -293,7 +315,7 @@ export const mirrorGithubRepoToGitea = async ({ updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, - mirroredLocation: `${repoOwner}/${repository.name}`, + mirroredLocation: `${repoOwner}/${targetRepoName}`, }) .where(eq(repositories.id, repository.id!)); @@ -393,11 +415,11 @@ export const mirrorGithubRepoToGitea = async ({ const existingRepo = await getGiteaRepoInfo({ config, owner: repoOwner, - repoName: repository.name, + repoName: targetRepoName, }); if (existingRepo && !existingRepo.mirror) { - console.log(`Repository ${repository.name} exists but is not a mirror. Handling...`); + console.log(`Repository ${targetRepoName} exists but is not a mirror. Handling...`); // Handle the existing non-mirror repository await handleExistingNonMirrorRepo({ @@ -408,14 +430,14 @@ export const mirrorGithubRepoToGitea = async ({ }); // After handling, proceed with mirror creation - console.log(`Proceeding with mirror creation for ${repository.name}`); + console.log(`Proceeding with mirror creation for ${targetRepoName}`); } const response = await httpPost( apiUrl, { clone_addr: cloneAddress, - repo_name: repository.name, + repo_name: targetRepoName, mirror: true, mirror_interval: config.giteaConfig?.mirrorInterval || "8h", // Set mirror interval wiki: config.giteaConfig?.wiki || false, // will mirror wiki if it exists @@ -460,6 +482,7 @@ export const mirrorGithubRepoToGitea = async ({ octokit, repository, giteaOwner: repoOwner, + giteaRepoName: targetRepoName, }); console.log(`[Metadata] Successfully mirrored issues for ${repository.name}`); } catch (error) { @@ -477,6 +500,7 @@ export const mirrorGithubRepoToGitea = async ({ octokit, repository, giteaOwner: repoOwner, + giteaRepoName: targetRepoName, }); console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name}`); } catch (error) { @@ -494,6 +518,7 @@ export const mirrorGithubRepoToGitea = async ({ octokit, repository, giteaOwner: repoOwner, + giteaRepoName: targetRepoName, }); console.log(`[Metadata] Successfully mirrored labels for ${repository.name}`); } catch (error) { @@ -511,6 +536,7 @@ export const mirrorGithubRepoToGitea = async ({ octokit, repository, giteaOwner: repoOwner, + giteaRepoName: targetRepoName, }); console.log(`[Metadata] Successfully mirrored milestones for ${repository.name}`); } catch (error) { @@ -519,7 +545,7 @@ export const mirrorGithubRepoToGitea = async ({ } } - console.log(`Repository ${repository.name} mirrored successfully`); + console.log(`Repository ${repository.name} mirrored successfully as ${targetRepoName}`); // Mark repos as "mirrored" in DB await db @@ -529,7 +555,7 @@ export const mirrorGithubRepoToGitea = async ({ updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, - mirroredLocation: `${repoOwner}/${repository.name}`, + mirroredLocation: `${repoOwner}/${targetRepoName}`, }) .where(eq(repositories.id, repository.id!)); @@ -538,8 +564,8 @@ export const mirrorGithubRepoToGitea = async ({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, - message: `Successfully mirrored repository: ${repository.name}`, - details: `Repository ${repository.name} was mirrored to Gitea.`, + message: `Successfully mirrored repository: ${repository.name}${targetRepoName !== repository.name ? ` as ${targetRepoName}` : ''}`, + details: `Repository ${repository.fullName} was mirrored to Gitea at ${repoOwner}/${targetRepoName}.`, status: "mirrored", }); @@ -608,6 +634,80 @@ export async function getOrCreateGiteaOrg({ } } +/** + * Generate a unique repository name for starred repos with duplicate names + */ +async function generateUniqueRepoName({ + config, + orgName, + baseName, + githubOwner, + strategy, +}: { + config: Partial; + orgName: string; + baseName: string; + githubOwner: string; + strategy?: string; +}): Promise { + const duplicateStrategy = strategy || "suffix"; + + // First check if base name is available + const baseExists = await isRepoPresentInGitea({ + config, + owner: orgName, + repoName: baseName, + }); + + if (!baseExists) { + return baseName; + } + + // Generate name based on strategy + let candidateName: string; + let attempt = 0; + const maxAttempts = 10; + + while (attempt < maxAttempts) { + switch (duplicateStrategy) { + case "prefix": + // Prefix with owner: owner-reponame + candidateName = attempt === 0 + ? `${githubOwner}-${baseName}` + : `${githubOwner}-${baseName}-${attempt}`; + break; + + case "owner-org": + // This would require creating sub-organizations, not supported in this PR + // Fall back to suffix strategy + case "suffix": + default: + // Suffix with owner: reponame-owner + candidateName = attempt === 0 + ? `${baseName}-${githubOwner}` + : `${baseName}-${githubOwner}-${attempt}`; + break; + } + + const exists = await isRepoPresentInGitea({ + config, + owner: orgName, + repoName: candidateName, + }); + + if (!exists) { + console.log(`Found unique name for duplicate starred repo: ${candidateName}`); + return candidateName; + } + + attempt++; + } + + // If all attempts failed, use timestamp as last resort + const timestamp = Date.now(); + return `${baseName}-${githubOwner}-${timestamp}`; +} + export async function mirrorGitHubRepoToGiteaOrg({ octokit, config, @@ -633,15 +733,37 @@ export async function mirrorGitHubRepoToGiteaOrg({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Determine the actual repository name to use (handle duplicates for starred repos) + let targetRepoName = repository.name; + + if (repository.isStarred && config.githubConfig) { + // Extract GitHub owner from full_name (format: owner/repo) + const githubOwner = repository.fullName.split('/')[0]; + + targetRepoName = await generateUniqueRepoName({ + config, + orgName, + baseName: repository.name, + githubOwner, + strategy: config.githubConfig.starredDuplicateStrategy, + }); + + if (targetRepoName !== repository.name) { + console.log( + `Starred repo ${repository.fullName} will be mirrored as ${orgName}/${targetRepoName} to avoid naming conflict` + ); + } + } + const isExisting = await isRepoPresentInGitea({ config, owner: orgName, - repoName: repository.name, + repoName: targetRepoName, }); if (isExisting) { console.log( - `Repository ${repository.name} already exists in Gitea organization ${orgName}. Updating database status.` + `Repository ${targetRepoName} already exists in Gitea organization ${orgName}. Updating database status.` ); // Update database to reflect that the repository is already mirrored @@ -652,7 +774,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, - mirroredLocation: `${orgName}/${repository.name}`, + mirroredLocation: `${orgName}/${targetRepoName}`, }) .where(eq(repositories.id, repository.id!)); @@ -661,19 +783,19 @@ export async function mirrorGitHubRepoToGiteaOrg({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, - message: `Repository ${repository.name} already exists in Gitea organization ${orgName}`, - details: `Repository ${repository.name} was found to already exist in Gitea organization ${orgName} and database status was updated.`, + message: `Repository ${targetRepoName} already exists in Gitea organization ${orgName}`, + details: `Repository ${targetRepoName} was found to already exist in Gitea organization ${orgName} and database status was updated.`, status: "mirrored", }); console.log( - `Repository ${repository.name} database status updated to mirrored in organization ${orgName}` + `Repository ${targetRepoName} database status updated to mirrored in organization ${orgName}` ); return; } console.log( - `Mirroring repository ${repository.name} to organization ${orgName}` + `Mirroring repository ${repository.fullName} to organization ${orgName} as ${targetRepoName}` ); let cloneAddress = repository.cloneUrl; @@ -710,7 +832,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ { clone_addr: cloneAddress, uid: giteaOrgId, - repo_name: repository.name, + repo_name: targetRepoName, mirror: true, mirror_interval: config.giteaConfig?.mirrorInterval || "8h", // Set mirror interval wiki: config.giteaConfig?.wiki || false, // will mirror wiki if it exists @@ -752,10 +874,11 @@ export async function mirrorGitHubRepoToGiteaOrg({ octokit, repository, giteaOwner: orgName, + giteaRepoName: targetRepoName, }); - console.log(`[Metadata] Successfully mirrored issues for ${repository.name} to org ${orgName}`); + console.log(`[Metadata] Successfully mirrored issues for ${repository.name} to org ${orgName}/${targetRepoName}`); } catch (error) { - console.error(`[Metadata] Failed to mirror issues for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`); + console.error(`[Metadata] Failed to mirror issues for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`); // Continue with other metadata operations even if issues fail } } @@ -769,10 +892,11 @@ export async function mirrorGitHubRepoToGiteaOrg({ octokit, repository, giteaOwner: orgName, + giteaRepoName: targetRepoName, }); - console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name} to org ${orgName}`); + console.log(`[Metadata] Successfully mirrored pull requests for ${repository.name} to org ${orgName}/${targetRepoName}`); } catch (error) { - console.error(`[Metadata] Failed to mirror pull requests for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`); + console.error(`[Metadata] Failed to mirror pull requests for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`); // Continue with other metadata operations even if PRs fail } } @@ -786,10 +910,11 @@ export async function mirrorGitHubRepoToGiteaOrg({ octokit, repository, giteaOwner: orgName, + giteaRepoName: targetRepoName, }); - console.log(`[Metadata] Successfully mirrored labels for ${repository.name} to org ${orgName}`); + console.log(`[Metadata] Successfully mirrored labels for ${repository.name} to org ${orgName}/${targetRepoName}`); } catch (error) { - console.error(`[Metadata] Failed to mirror labels for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`); + console.error(`[Metadata] Failed to mirror labels for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`); // Continue with other metadata operations even if labels fail } } @@ -803,16 +928,17 @@ export async function mirrorGitHubRepoToGiteaOrg({ octokit, repository, giteaOwner: orgName, + giteaRepoName: targetRepoName, }); - console.log(`[Metadata] Successfully mirrored milestones for ${repository.name} to org ${orgName}`); + console.log(`[Metadata] Successfully mirrored milestones for ${repository.name} to org ${orgName}/${targetRepoName}`); } catch (error) { - console.error(`[Metadata] Failed to mirror milestones for ${repository.name} to org ${orgName}: ${error instanceof Error ? error.message : String(error)}`); + console.error(`[Metadata] Failed to mirror milestones for ${repository.name} to org ${orgName}/${targetRepoName}: ${error instanceof Error ? error.message : String(error)}`); // Continue with other metadata operations even if milestones fail } } console.log( - `Repository ${repository.name} mirrored successfully to organization ${orgName}` + `Repository ${repository.name} mirrored successfully to organization ${orgName} as ${targetRepoName}` ); // Mark repos as "mirrored" in DB @@ -823,7 +949,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ updatedAt: new Date(), lastMirrored: new Date(), errorMessage: null, - mirroredLocation: `${orgName}/${repository.name}`, + mirroredLocation: `${orgName}/${targetRepoName}`, }) .where(eq(repositories.id, repository.id!)); @@ -832,8 +958,8 @@ export async function mirrorGitHubRepoToGiteaOrg({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, - message: `Repository ${repository.name} mirrored successfully`, - details: `Repository ${repository.name} was mirrored to Gitea`, + message: `Repository ${repository.name} mirrored successfully${targetRepoName !== repository.name ? ` as ${targetRepoName}` : ''}`, + details: `Repository ${repository.fullName} was mirrored to Gitea at ${orgName}/${targetRepoName}`, status: "mirrored", }); @@ -1149,11 +1275,13 @@ export const mirrorGitRepoIssuesToGitea = async ({ octokit, repository, giteaOwner, + giteaRepoName, }: { config: Partial; octokit: Octokit; repository: Repository; giteaOwner: string; + giteaRepoName?: string; }) => { //things covered here are- issue, title, body, labels, comments and assignees if ( @@ -1168,23 +1296,26 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Use provided giteaRepoName or fall back to repository.name + const repoName = giteaRepoName || repository.name; + // Log configuration details for debugging - console.log(`[Issues] Starting issue mirroring for repository ${repository.name}`); + console.log(`[Issues] Starting issue mirroring for repository ${repository.name} as ${repoName}`); console.log(`[Issues] Gitea URL: ${config.giteaConfig!.url}`); console.log(`[Issues] Gitea Owner: ${giteaOwner}`); console.log(`[Issues] Gitea Default Owner: ${config.giteaConfig!.defaultOwner}`); // Verify the repository exists in Gitea before attempting to mirror metadata - console.log(`[Issues] Verifying repository ${repository.name} exists at ${giteaOwner}`); + console.log(`[Issues] Verifying repository ${repoName} exists at ${giteaOwner}`); const repoExists = await isRepoPresentInGitea({ config, owner: giteaOwner, - repoName: repository.name, + repoName: repoName, }); if (!repoExists) { - console.error(`[Issues] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror issues.`); - throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); + console.error(`[Issues] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror issues.`); + throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); } const [owner, repo] = repository.fullName.split("/"); @@ -1215,7 +1346,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ // Get existing labels from Gitea const giteaLabelsRes = await httpGet( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`, { Authorization: `token ${decryptedConfig.giteaConfig.token}`, } @@ -1556,11 +1687,13 @@ export async function mirrorGitRepoPullRequestsToGitea({ octokit, repository, giteaOwner, + giteaRepoName, }: { config: Partial; octokit: Octokit; repository: Repository; giteaOwner: string; + giteaRepoName?: string; }) { if ( !config.githubConfig?.token || @@ -1574,23 +1707,26 @@ export async function mirrorGitRepoPullRequestsToGitea({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Use provided giteaRepoName or fall back to repository.name + const repoName = giteaRepoName || repository.name; + // Log configuration details for debugging - console.log(`[Pull Requests] Starting PR mirroring for repository ${repository.name}`); + console.log(`[Pull Requests] Starting PR mirroring for repository ${repository.name} as ${repoName}`); console.log(`[Pull Requests] Gitea URL: ${config.giteaConfig!.url}`); console.log(`[Pull Requests] Gitea Owner: ${giteaOwner}`); console.log(`[Pull Requests] Gitea Default Owner: ${config.giteaConfig!.defaultOwner}`); // Verify the repository exists in Gitea before attempting to mirror metadata - console.log(`[Pull Requests] Verifying repository ${repository.name} exists at ${giteaOwner}`); + console.log(`[Pull Requests] Verifying repository ${repoName} exists at ${giteaOwner}`); const repoExists = await isRepoPresentInGitea({ config, owner: giteaOwner, - repoName: repository.name, + repoName: repoName, }); if (!repoExists) { - console.error(`[Pull Requests] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror PRs.`); - throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); + console.error(`[Pull Requests] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror PRs.`); + throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); } const [owner, repo] = repository.fullName.split("/"); @@ -1622,7 +1758,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ // Get existing labels from Gitea and ensure "pull-request" label exists const giteaLabelsRes = await httpGet( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`, { Authorization: `token ${decryptedConfig.giteaConfig.token}`, } @@ -1640,7 +1776,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ } else { try { const created = await httpPost( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`, { name: "pull-request", color: "#0366d6", @@ -1744,7 +1880,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ console.log(`[Pull Requests] Creating enriched issue for PR #${pr.number}: ${pr.title}`); await httpPost( - `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repository.name}/issues`, + `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues`, issueData, { Authorization: `token ${decryptedConfig.giteaConfig!.token}`, @@ -1764,7 +1900,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ try { await httpPost( - `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repository.name}/issues`, + `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues`, basicIssueData, { Authorization: `token ${decryptedConfig.giteaConfig!.token}`, @@ -1795,11 +1931,13 @@ export async function mirrorGitRepoLabelsToGitea({ octokit, repository, giteaOwner, + giteaRepoName, }: { config: Partial; octokit: Octokit; repository: Repository; giteaOwner: string; + giteaRepoName?: string; }) { if ( !config.githubConfig?.token || @@ -1812,17 +1950,20 @@ export async function mirrorGitRepoLabelsToGitea({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Use provided giteaRepoName or fall back to repository.name + const repoName = giteaRepoName || repository.name; + // Verify the repository exists in Gitea before attempting to mirror metadata - console.log(`[Labels] Verifying repository ${repository.name} exists at ${giteaOwner}`); + console.log(`[Labels] Verifying repository ${repoName} exists at ${giteaOwner}`); const repoExists = await isRepoPresentInGitea({ config, owner: giteaOwner, - repoName: repository.name, + repoName: repoName, }); if (!repoExists) { - console.error(`[Labels] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror labels.`); - throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); + console.error(`[Labels] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror labels.`); + throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); } const [owner, repo] = repository.fullName.split("/"); @@ -1847,7 +1988,7 @@ export async function mirrorGitRepoLabelsToGitea({ // Get existing labels from Gitea const giteaLabelsRes = await httpGet( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`, { Authorization: `token ${decryptedConfig.giteaConfig.token}`, } @@ -1862,7 +2003,7 @@ export async function mirrorGitRepoLabelsToGitea({ if (!existingLabels.has(label.name)) { try { await httpPost( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/labels`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/labels`, { name: label.name, color: `#${label.color}`, @@ -1889,11 +2030,13 @@ export async function mirrorGitRepoMilestonesToGitea({ octokit, repository, giteaOwner, + giteaRepoName, }: { config: Partial; octokit: Octokit; repository: Repository; giteaOwner: string; + giteaRepoName?: string; }) { if ( !config.githubConfig?.token || @@ -1906,17 +2049,20 @@ export async function mirrorGitRepoMilestonesToGitea({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Use provided giteaRepoName or fall back to repository.name + const repoName = giteaRepoName || repository.name; + // Verify the repository exists in Gitea before attempting to mirror metadata - console.log(`[Milestones] Verifying repository ${repository.name} exists at ${giteaOwner}`); + console.log(`[Milestones] Verifying repository ${repoName} exists at ${giteaOwner}`); const repoExists = await isRepoPresentInGitea({ config, owner: giteaOwner, - repoName: repository.name, + repoName: repoName, }); if (!repoExists) { - console.error(`[Milestones] Repository ${repository.name} not found at ${giteaOwner}. Cannot mirror milestones.`); - throw new Error(`Repository ${repository.name} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); + console.error(`[Milestones] Repository ${repoName} not found at ${giteaOwner}. Cannot mirror milestones.`); + throw new Error(`Repository ${repoName} does not exist in Gitea at ${giteaOwner}. Please ensure the repository is mirrored first.`); } const [owner, repo] = repository.fullName.split("/"); @@ -1942,7 +2088,7 @@ export async function mirrorGitRepoMilestonesToGitea({ // Get existing milestones from Gitea const giteaMilestonesRes = await httpGet( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/milestones`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/milestones`, { Authorization: `token ${decryptedConfig.giteaConfig.token}`, } @@ -1957,7 +2103,7 @@ export async function mirrorGitRepoMilestonesToGitea({ if (!existingMilestones.has(milestone.title)) { try { await httpPost( - `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repository.name}/milestones`, + `${config.giteaConfig.url}/api/v1/repos/${giteaOwner}/${repoName}/milestones`, { title: milestone.title, description: milestone.description || "", diff --git a/src/lib/github.ts b/src/lib/github.ts index cb0b1fb..d79cd1b 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -18,8 +18,11 @@ if (process.env.NODE_ENV !== "test") { } } -// Extend Octokit with throttling plugin -const MyOctokit = Octokit.plugin(throttling); +// Extend Octokit with throttling plugin when available (tests may stub Octokit) +// Fallback to base Octokit if .plugin is not present +const MyOctokit: any = (Octokit as any)?.plugin?.call + ? (Octokit as any).plugin(throttling) + : Octokit as any; /** * Creates an authenticated Octokit instance with rate limit tracking and throttling diff --git a/src/lib/repo-utils.test.ts b/src/lib/repo-utils.test.ts new file mode 100644 index 0000000..6265bd8 --- /dev/null +++ b/src/lib/repo-utils.test.ts @@ -0,0 +1,75 @@ +import { describe, it, expect } from 'bun:test'; +import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils'; +import type { GitRepo } from '@/types/Repository'; + +function sampleRepo(overrides: Partial = {}): GitRepo { + const base: GitRepo = { + name: 'repo', + fullName: 'owner/repo', + url: 'https://github.com/owner/repo', + cloneUrl: 'https://github.com/owner/repo.git', + owner: 'owner', + organization: undefined, + mirroredLocation: '', + destinationOrg: null, + isPrivate: false, + isForked: false, + forkedFrom: undefined, + hasIssues: true, + isStarred: false, + isArchived: false, + size: 1, + hasLFS: false, + hasSubmodules: false, + language: null, + description: null, + defaultBranch: 'main', + visibility: 'public', + status: 'imported', + lastMirrored: undefined, + errorMessage: undefined, + createdAt: new Date(), + updatedAt: new Date(), + }; + return { ...base, ...overrides }; +} + +describe('mergeGitReposPreferStarred', () => { + it('keeps unique repos', () => { + const basic = [sampleRepo({ fullName: 'a/x', name: 'x' })]; + const starred: GitRepo[] = []; + const merged = mergeGitReposPreferStarred(basic, starred); + expect(merged).toHaveLength(1); + expect(merged[0].fullName).toBe('a/x'); + }); + + it('prefers starred when duplicate exists', () => { + const basic = [sampleRepo({ fullName: 'a/x', name: 'x', isStarred: false })]; + const starred = [sampleRepo({ fullName: 'a/x', name: 'x', isStarred: true })]; + const merged = mergeGitReposPreferStarred(basic, starred); + expect(merged).toHaveLength(1); + expect(merged[0].isStarred).toBe(true); + }); +}); + +describe('normalizeGitRepoToInsert', () => { + it('sets undefined optional fields to null', () => { + const repo = sampleRepo({ organization: undefined, forkedFrom: undefined, language: undefined, description: undefined, lastMirrored: undefined, errorMessage: undefined }); + const insert = normalizeGitRepoToInsert(repo, { userId: 'u', configId: 'c' }); + expect(insert.organization).toBeNull(); + expect(insert.forkedFrom).toBeNull(); + expect(insert.language).toBeNull(); + expect(insert.description).toBeNull(); + expect(insert.lastMirrored).toBeNull(); + expect(insert.errorMessage).toBeNull(); + }); +}); + +describe('calcBatchSizeForInsert', () => { + it('respects 999 parameter limit', () => { + const batch = calcBatchSizeForInsert(29); + expect(batch).toBeGreaterThan(0); + expect(batch * 29).toBeLessThanOrEqual(999); + }); +}); + diff --git a/src/lib/repo-utils.ts b/src/lib/repo-utils.ts new file mode 100644 index 0000000..e33ce43 --- /dev/null +++ b/src/lib/repo-utils.ts @@ -0,0 +1,71 @@ +import { v4 as uuidv4 } from 'uuid'; +import type { GitRepo } from '@/types/Repository'; +import { repositories } from '@/lib/db/schema'; + +export type RepoInsert = typeof repositories.$inferInsert; + +// Merge lists and de-duplicate by fullName, preferring starred variant when present +export function mergeGitReposPreferStarred( + basicAndForked: GitRepo[], + starred: GitRepo[] +): GitRepo[] { + const map = new Map(); + for (const r of [...basicAndForked, ...starred]) { + const existing = map.get(r.fullName); + if (!existing || (!existing.isStarred && r.isStarred)) { + map.set(r.fullName, r); + } + } + return Array.from(map.values()); +} + +// Convert a GitRepo to a normalized DB insert object with all nullable fields set +export function normalizeGitRepoToInsert( + repo: GitRepo, + { + userId, + configId, + }: { userId: string; configId: string } +): RepoInsert { + return { + id: uuidv4(), + userId, + configId, + name: repo.name, + fullName: repo.fullName, + url: repo.url, + cloneUrl: repo.cloneUrl, + owner: repo.owner, + organization: repo.organization ?? null, + mirroredLocation: repo.mirroredLocation || '', + destinationOrg: repo.destinationOrg || null, + isPrivate: repo.isPrivate, + isForked: repo.isForked, + forkedFrom: repo.forkedFrom ?? null, + hasIssues: repo.hasIssues, + isStarred: repo.isStarred, + isArchived: repo.isArchived, + size: repo.size, + hasLFS: repo.hasLFS, + hasSubmodules: repo.hasSubmodules, + language: repo.language ?? null, + description: repo.description ?? null, + defaultBranch: repo.defaultBranch, + visibility: repo.visibility, + status: 'imported', + lastMirrored: repo.lastMirrored ?? null, + errorMessage: repo.errorMessage ?? null, + createdAt: repo.createdAt || new Date(), + updatedAt: repo.updatedAt || new Date(), + }; +} + +// Compute a safe batch size based on SQLite 999-parameter limit +export function calcBatchSizeForInsert(columnCount: number, maxParams = 999): number { + if (columnCount <= 0) return 1; + // Reserve a little headroom in case column count drifts + const safety = 0; + const effectiveMax = Math.max(1, maxParams - safety); + return Math.max(1, Math.floor(effectiveMax / columnCount)); +} + diff --git a/src/lib/scheduler-service.ts b/src/lib/scheduler-service.ts index d51e954..b19219c 100644 --- a/src/lib/scheduler-service.ts +++ b/src/lib/scheduler-service.ts @@ -11,6 +11,7 @@ import { getDecryptedGitHubToken } from '@/lib/utils/config-encryption'; import { parseInterval, formatDuration } from '@/lib/utils/duration-parser'; import type { Repository } from '@/lib/db/schema'; import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository'; +import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils'; let schedulerInterval: NodeJS.Timeout | null = null; let isSchedulerRunning = false; @@ -94,8 +95,7 @@ async function runScheduledSync(config: any): Promise { ? getGithubStarredRepositories({ octokit, config }) : Promise.resolve([]), ]); - - const allGithubRepos = [...basicAndForkedRepos, ...starredRepos]; + const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); // Check for new repositories const existingRepos = await db @@ -110,37 +110,21 @@ async function runScheduledSync(config: any): Promise { console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId}`); // Insert new repositories - const reposToInsert = newRepos.map(repo => ({ - id: uuidv4(), - userId, - configId: config.id, - name: repo.name, - fullName: repo.fullName, - url: repo.url, - cloneUrl: repo.cloneUrl, - owner: repo.owner, - organization: repo.organization, - mirroredLocation: repo.mirroredLocation || "", - destinationOrg: repo.destinationOrg || null, - isPrivate: repo.isPrivate, - isForked: repo.isForked, - forkedFrom: repo.forkedFrom, - hasIssues: repo.hasIssues, - isStarred: repo.isStarred, - isArchived: repo.isArchived, - size: repo.size, - hasLFS: repo.hasLFS, - hasSubmodules: repo.hasSubmodules, - language: repo.language || null, - description: repo.description || null, - defaultBranch: repo.defaultBranch, - visibility: repo.visibility, - status: 'imported', - createdAt: new Date(), - updatedAt: new Date(), - })); + const reposToInsert = newRepos.map(repo => + normalizeGitRepoToInsert(repo, { userId, configId: config.id }) + ); - await db.insert(repositories).values(reposToInsert); + // Batch insert to avoid SQLite parameter limit + const sample = reposToInsert[0]; + const columnCount = Object.keys(sample ?? {}).length || 1; + const BATCH_SIZE = calcBatchSizeForInsert(columnCount); + for (let i = 0; i < reposToInsert.length; i += BATCH_SIZE) { + const batch = reposToInsert.slice(i, i + BATCH_SIZE); + await db + .insert(repositories) + .values(batch) + .onConflictDoNothing({ target: [repositories.userId, repositories.fullName] }); + } console.log(`[Scheduler] Successfully imported ${newRepos.length} new repositories for user ${userId}`); } else { console.log(`[Scheduler] No new repositories found for user ${userId}`); @@ -375,8 +359,7 @@ async function performInitialAutoStart(): Promise { ? getGithubStarredRepositories({ octokit, config }) : Promise.resolve([]), ]); - - const allGithubRepos = [...basicAndForkedRepos, ...starredRepos]; + const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); // Check for new repositories const existingRepos = await db @@ -391,37 +374,21 @@ async function performInitialAutoStart(): Promise { console.log(`[Scheduler] Importing ${reposToImport.length} repositories for user ${config.userId}...`); // Insert new repositories - const reposToInsert = reposToImport.map(repo => ({ - id: uuidv4(), - userId: config.userId, - configId: config.id, - name: repo.name, - fullName: repo.fullName, - url: repo.url, - cloneUrl: repo.cloneUrl, - owner: repo.owner, - organization: repo.organization, - mirroredLocation: repo.mirroredLocation || "", - destinationOrg: repo.destinationOrg || null, - isPrivate: repo.isPrivate, - isForked: repo.isForked, - forkedFrom: repo.forkedFrom, - hasIssues: repo.hasIssues, - isStarred: repo.isStarred, - isArchived: repo.isArchived, - size: repo.size, - hasLFS: repo.hasLFS, - hasSubmodules: repo.hasSubmodules, - language: repo.language || null, - description: repo.description || null, - defaultBranch: repo.defaultBranch, - visibility: repo.visibility, - status: 'imported', - createdAt: new Date(), - updatedAt: new Date(), - })); + const reposToInsert = reposToImport.map(repo => + normalizeGitRepoToInsert(repo, { userId: config.userId, configId: config.id }) + ); - await db.insert(repositories).values(reposToInsert); + // Batch insert to avoid SQLite parameter limit + const sample = reposToInsert[0]; + const columnCount = Object.keys(sample ?? {}).length || 1; + const BATCH_SIZE = calcBatchSizeForInsert(columnCount); + for (let i = 0; i < reposToInsert.length; i += BATCH_SIZE) { + const batch = reposToInsert.slice(i, i + BATCH_SIZE); + await db + .insert(repositories) + .values(batch) + .onConflictDoNothing({ target: [repositories.userId, repositories.fullName] }); + } console.log(`[Scheduler] Successfully imported ${reposToImport.length} repositories`); } else { console.log(`[Scheduler] No new repositories to import for user ${config.userId}`); @@ -697,4 +664,4 @@ export function stopSchedulerService(): void { */ export function isSchedulerServiceRunning(): boolean { return schedulerInterval !== null; -} \ No newline at end of file +} diff --git a/src/pages/api/sync/index.ts b/src/pages/api/sync/index.ts index 2840d5d..1f08fdd 100644 --- a/src/pages/api/sync/index.ts +++ b/src/pages/api/sync/index.ts @@ -10,6 +10,7 @@ import { getGithubStarredRepositories, } from "@/lib/github"; import { jsonResponse, createSecureErrorResponse } from "@/lib/utils"; +import { mergeGitReposPreferStarred, calcBatchSizeForInsert } from "@/lib/repo-utils"; import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption"; export const POST: APIRoute = async ({ request }) => { @@ -55,7 +56,8 @@ export const POST: APIRoute = async ({ request }) => { getGithubOrganizations({ octokit, config }), ]); - const allGithubRepos = [...basicAndForkedRepos, ...starredRepos]; + // Merge and de-duplicate by fullName, preferring starred variant when duplicated + const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); // Prepare full list of repos and orgs const newRepos = allGithubRepos.map((repo) => ({ @@ -67,25 +69,25 @@ export const POST: APIRoute = async ({ request }) => { url: repo.url, cloneUrl: repo.cloneUrl, owner: repo.owner, - organization: repo.organization, + organization: repo.organization ?? null, mirroredLocation: repo.mirroredLocation || "", destinationOrg: repo.destinationOrg || null, isPrivate: repo.isPrivate, isForked: repo.isForked, - forkedFrom: repo.forkedFrom, + forkedFrom: repo.forkedFrom ?? null, hasIssues: repo.hasIssues, isStarred: repo.isStarred, isArchived: repo.isArchived, size: repo.size, hasLFS: repo.hasLFS, hasSubmodules: repo.hasSubmodules, - language: repo.language || null, - description: repo.description || null, + language: repo.language ?? null, + description: repo.description ?? null, defaultBranch: repo.defaultBranch, visibility: repo.visibility, status: repo.status, - lastMirrored: repo.lastMirrored, - errorMessage: repo.errorMessage, + lastMirrored: repo.lastMirrored ?? null, + errorMessage: repo.errorMessage ?? null, createdAt: repo.createdAt, updatedAt: repo.updatedAt, })); @@ -128,12 +130,27 @@ export const POST: APIRoute = async ({ request }) => { ); insertedOrgs = newOrgs.filter((o) => !existingOrgNames.has(o.name)); + // Batch insert repositories to avoid SQLite parameter limit (dynamic by column count) + const sample = newRepos[0]; + const columnCount = Object.keys(sample ?? {}).length || 1; + const REPO_BATCH_SIZE = calcBatchSizeForInsert(columnCount); if (insertedRepos.length > 0) { - await tx.insert(repositories).values(insertedRepos); + for (let i = 0; i < insertedRepos.length; i += REPO_BATCH_SIZE) { + const batch = insertedRepos.slice(i, i + REPO_BATCH_SIZE); + await tx + .insert(repositories) + .values(batch) + .onConflictDoNothing({ target: [repositories.userId, repositories.fullName] }); + } } + // Batch insert organizations (they have fewer fields, so we can use larger batches) + const ORG_BATCH_SIZE = 100; if (insertedOrgs.length > 0) { - await tx.insert(organizations).values(insertedOrgs); + for (let i = 0; i < insertedOrgs.length; i += ORG_BATCH_SIZE) { + const batch = insertedOrgs.slice(i, i + ORG_BATCH_SIZE); + await tx.insert(organizations).values(batch); + } } }); diff --git a/src/pages/api/sync/organization.ts b/src/pages/api/sync/organization.ts index 9638333..6f40e1d 100644 --- a/src/pages/api/sync/organization.ts +++ b/src/pages/api/sync/organization.ts @@ -122,25 +122,36 @@ export const POST: APIRoute = async ({ request }) => { destinationOrg: null, isPrivate: repo.private, isForked: repo.fork, - forkedFrom: undefined, + forkedFrom: null, hasIssues: repo.has_issues, isStarred: false, isArchived: repo.archived, size: repo.size, hasLFS: false, hasSubmodules: false, - language: repo.language || null, - description: repo.description || null, + language: repo.language ?? null, + description: repo.description ?? null, defaultBranch: repo.default_branch ?? "main", visibility: (repo.visibility ?? "public") as RepositoryVisibility, status: "imported" as RepoStatus, - lastMirrored: undefined, - errorMessage: undefined, + lastMirrored: null, + errorMessage: null, createdAt: repo.created_at ? new Date(repo.created_at) : new Date(), updatedAt: repo.updated_at ? new Date(repo.updated_at) : new Date(), })); - await db.insert(repositories).values(repoRecords); + // Batch insert repositories to avoid SQLite parameter limit + // Compute batch size based on column count + const sample = repoRecords[0]; + const columnCount = Object.keys(sample ?? {}).length || 1; + const BATCH_SIZE = Math.max(1, Math.floor(999 / columnCount)); + for (let i = 0; i < repoRecords.length; i += BATCH_SIZE) { + const batch = repoRecords.slice(i, i + BATCH_SIZE); + await db + .insert(repositories) + .values(batch) + .onConflictDoNothing({ target: [repositories.userId, repositories.fullName] }); + } // Insert organization metadata const organizationRecord = { diff --git a/src/pages/api/sync/repository.ts b/src/pages/api/sync/repository.ts index 3d09eac..fed0590 100644 --- a/src/pages/api/sync/repository.ts +++ b/src/pages/api/sync/repository.ts @@ -80,25 +80,23 @@ export const POST: APIRoute = async ({ request }) => { cloneUrl: repoData.clone_url, owner: repoData.owner.login, organization: - repoData.owner.type === "Organization" - ? repoData.owner.login - : undefined, + repoData.owner.type === "Organization" ? repoData.owner.login : null, isPrivate: repoData.private, isForked: repoData.fork, - forkedFrom: undefined, + forkedFrom: null, hasIssues: repoData.has_issues, isStarred: false, isArchived: repoData.archived, size: repoData.size, hasLFS: false, hasSubmodules: false, - language: repoData.language || null, - description: repoData.description || null, + language: repoData.language ?? null, + description: repoData.description ?? null, defaultBranch: repoData.default_branch, visibility: (repoData.visibility ?? "public") as RepositoryVisibility, status: "imported" as Repository["status"], - lastMirrored: undefined, - errorMessage: undefined, + lastMirrored: null, + errorMessage: null, mirroredLocation: "", destinationOrg: null, createdAt: repoData.created_at @@ -109,7 +107,10 @@ export const POST: APIRoute = async ({ request }) => { : new Date(), }; - await db.insert(repositories).values(metadata); + await db + .insert(repositories) + .values(metadata) + .onConflictDoNothing({ target: [repositories.userId, repositories.fullName] }); createMirrorJob({ userId, diff --git a/src/types/config.ts b/src/types/config.ts index 240ac76..98bf15a 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -29,11 +29,14 @@ export interface DatabaseCleanupConfig { nextRun?: Date; } +export type DuplicateNameStrategy = "suffix" | "prefix" | "owner-org"; + export interface GitHubConfig { username: string; token: string; privateRepositories: boolean; mirrorStarred: boolean; + starredDuplicateStrategy?: DuplicateNameStrategy; } export interface MirrorOptions {