diff --git a/bun.lock b/bun.lock index 8c22ba0..6290a07 100644 --- a/bun.lock +++ b/bun.lock @@ -9,6 +9,7 @@ "@astrojs/node": "9.4.3", "@astrojs/react": "^4.3.1", "@better-auth/sso": "^1.3.8", + "@octokit/plugin-throttling": "^11.0.1", "@octokit/rest": "^22.0.0", "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-avatar": "^1.1.10", @@ -19,6 +20,7 @@ "@radix-ui/react-hover-card": "^1.1.15", "@radix-ui/react-label": "^2.1.7", "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-select": "^2.2.6", @@ -329,6 +331,8 @@ "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@16.0.0", "", { "dependencies": { "@octokit/types": "^14.1.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g=="], + "@octokit/plugin-throttling": ["@octokit/plugin-throttling@11.0.1", "", { "dependencies": { "@octokit/types": "^14.0.0", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": "^7.0.0" } }, "sha512-S+EVhy52D/272L7up58dr3FNSMXWuNZolkL4zMJBNIfIxyZuUcczsQAU4b5w6dewJXnKYVgSHSV5wxitMSW1kw=="], + "@octokit/request": ["@octokit/request@10.0.2", "", { "dependencies": { "@octokit/endpoint": "^11.0.0", "@octokit/request-error": "^7.0.0", "@octokit/types": "^14.0.0", "fast-content-type-parse": "^3.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-iYj4SJG/2bbhh+iIpFmG5u49DtJ4lipQ+aPakjL9OKpsGY93wM8w06gvFbEQxcMsZcCvk5th5KkIm2m8o14aWA=="], "@octokit/request-error": ["@octokit/request-error@7.0.0", "", { "dependencies": { "@octokit/types": "^14.0.0" } }, "sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg=="], @@ -399,6 +403,8 @@ "@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.3", "", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="], + "@radix-ui/react-progress": ["@radix-ui/react-progress@1.1.7", "", { "dependencies": { "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-vPdg/tF6YC/ynuBIJlk1mm7Le0VgW6ub6J2UWnTQ7/D23KXcPI1qy+0vBkgKgd38RCMJavBXpB83HPNFMTb0Fg=="], + "@radix-ui/react-radio-group": ["@radix-ui/react-radio-group@1.3.8", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ=="], "@radix-ui/react-roving-focus": ["@radix-ui/react-roving-focus@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA=="], @@ -691,6 +697,8 @@ "body-parser": ["body-parser@1.20.3", "", { "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.13.0", "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" } }, "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g=="], + "bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="], + "boxen": ["boxen@8.0.1", "", { "dependencies": { "ansi-align": "^3.0.1", "camelcase": "^8.0.0", "chalk": "^5.3.0", "cli-boxes": "^3.0.0", "string-width": "^7.2.0", "type-fest": "^4.21.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0" } }, "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], diff --git a/drizzle/0004_grey_butterfly.sql b/drizzle/0004_grey_butterfly.sql new file mode 100644 index 0000000..60fe22a --- /dev/null +++ b/drizzle/0004_grey_butterfly.sql @@ -0,0 +1,18 @@ +CREATE TABLE `rate_limits` ( + `id` text PRIMARY KEY NOT NULL, + `user_id` text NOT NULL, + `provider` text DEFAULT 'github' NOT NULL, + `limit` integer NOT NULL, + `remaining` integer NOT NULL, + `used` integer NOT NULL, + `reset` integer NOT NULL, + `retry_after` integer, + `status` text DEFAULT 'ok' NOT NULL, + `last_checked` integer NOT NULL, + `created_at` integer DEFAULT (unixepoch()) NOT NULL, + `updated_at` integer DEFAULT (unixepoch()) NOT NULL, + FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE INDEX `idx_rate_limits_user_provider` ON `rate_limits` (`user_id`,`provider`);--> statement-breakpoint +CREATE INDEX `idx_rate_limits_status` ON `rate_limits` (`status`); \ No newline at end of file diff --git a/drizzle/meta/0004_snapshot.json b/drizzle/meta/0004_snapshot.json new file mode 100644 index 0000000..69fb081 --- /dev/null +++ b/drizzle/meta/0004_snapshot.json @@ -0,0 +1,1933 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "284b8db7-ef64-4311-b890-0251bcbe5866", + "prevId": "376b0dbc-5fd4-4251-979f-f5cc2d1455b3", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_user_id": { + "name": "provider_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_accounts_account_id": { + "name": "idx_accounts_account_id", + "columns": [ + "account_id" + ], + "isUnique": false + }, + "idx_accounts_user_id": { + "name": "idx_accounts_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_accounts_provider": { + "name": "idx_accounts_provider", + "columns": [ + "provider_id", + "provider_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "configs": { + "name": "configs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "github_config": { + "name": "github_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "gitea_config": { + "name": "gitea_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "include": { + "name": "include", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[\"*\"]'" + }, + "exclude": { + "name": "exclude", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "schedule_config": { + "name": "schedule_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cleanup_config": { + "name": "cleanup_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": { + "configs_user_id_users_id_fk": { + "name": "configs_user_id_users_id_fk", + "tableFrom": "configs", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "read": { + "name": "read", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_events_user_channel": { + "name": "idx_events_user_channel", + "columns": [ + "user_id", + "channel" + ], + "isUnique": false + }, + "idx_events_created_at": { + "name": "idx_events_created_at", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "idx_events_read": { + "name": "idx_events_read", + "columns": [ + "read" + ], + "isUnique": false + } + }, + "foreignKeys": { + "events_user_id_users_id_fk": { + "name": "events_user_id_users_id_fk", + "tableFrom": "events", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mirror_jobs": { + "name": "mirror_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_name": { + "name": "repository_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_name": { + "name": "organization_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "job_type": { + "name": "job_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mirror'" + }, + "batch_id": { + "name": "batch_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_items": { + "name": "total_items", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_items": { + "name": "completed_items", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "item_ids": { + "name": "item_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_item_ids": { + "name": "completed_item_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'[]'" + }, + "in_progress": { + "name": "in_progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_mirror_jobs_user_id": { + "name": "idx_mirror_jobs_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_mirror_jobs_batch_id": { + "name": "idx_mirror_jobs_batch_id", + "columns": [ + "batch_id" + ], + "isUnique": false + }, + "idx_mirror_jobs_in_progress": { + "name": "idx_mirror_jobs_in_progress", + "columns": [ + "in_progress" + ], + "isUnique": false + }, + "idx_mirror_jobs_job_type": { + "name": "idx_mirror_jobs_job_type", + "columns": [ + "job_type" + ], + "isUnique": false + }, + "idx_mirror_jobs_timestamp": { + "name": "idx_mirror_jobs_timestamp", + "columns": [ + "timestamp" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mirror_jobs_user_id_users_id_fk": { + "name": "mirror_jobs_user_id_users_id_fk", + "tableFrom": "mirror_jobs", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_access_tokens": { + "name": "oauth_access_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_oauth_access_tokens_access_token": { + "name": "idx_oauth_access_tokens_access_token", + "columns": [ + "access_token" + ], + "isUnique": false + }, + "idx_oauth_access_tokens_user_id": { + "name": "idx_oauth_access_tokens_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_oauth_access_tokens_client_id": { + "name": "idx_oauth_access_tokens_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_access_tokens_user_id_users_id_fk": { + "name": "oauth_access_tokens_user_id_users_id_fk", + "tableFrom": "oauth_access_tokens", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_applications": { + "name": "oauth_applications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_urls": { + "name": "redirect_urls", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disabled": { + "name": "disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "oauth_applications_client_id_unique": { + "name": "oauth_applications_client_id_unique", + "columns": [ + "client_id" + ], + "isUnique": true + }, + "idx_oauth_applications_client_id": { + "name": "idx_oauth_applications_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_applications_user_id": { + "name": "idx_oauth_applications_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_consent": { + "name": "oauth_consent", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consent_given": { + "name": "consent_given", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_oauth_consent_user_id": { + "name": "idx_oauth_consent_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_oauth_consent_client_id": { + "name": "idx_oauth_consent_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_consent_user_client": { + "name": "idx_oauth_consent_user_client", + "columns": [ + "user_id", + "client_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_consent_user_id_users_id_fk": { + "name": "oauth_consent_user_id_users_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "membership_role": { + "name": "membership_role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "is_included": { + "name": "is_included", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "destination_org": { + "name": "destination_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "last_mirrored": { + "name": "last_mirrored", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_count": { + "name": "repository_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_repository_count": { + "name": "public_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_repository_count": { + "name": "private_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fork_repository_count": { + "name": "fork_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_organizations_user_id": { + "name": "idx_organizations_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_organizations_config_id": { + "name": "idx_organizations_config_id", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "idx_organizations_status": { + "name": "idx_organizations_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_organizations_is_included": { + "name": "idx_organizations_is_included", + "columns": [ + "is_included" + ], + "isUnique": false + } + }, + "foreignKeys": { + "organizations_user_id_users_id_fk": { + "name": "organizations_user_id_users_id_fk", + "tableFrom": "organizations", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "organizations_config_id_configs_id_fk": { + "name": "organizations_config_id_configs_id_fk", + "tableFrom": "organizations", + "tableTo": "configs", + "columnsFrom": [ + "config_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rate_limits": { + "name": "rate_limits", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'github'" + }, + "limit": { + "name": "limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reset": { + "name": "reset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retry_after": { + "name": "retry_after", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'ok'" + }, + "last_checked": { + "name": "last_checked", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_rate_limits_user_provider": { + "name": "idx_rate_limits_user_provider", + "columns": [ + "user_id", + "provider" + ], + "isUnique": false + }, + "idx_rate_limits_status": { + "name": "idx_rate_limits_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "rate_limits_user_id_users_id_fk": { + "name": "rate_limits_user_id_users_id_fk", + "tableFrom": "rate_limits", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "clone_url": { + "name": "clone_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization": { + "name": "organization", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mirrored_location": { + "name": "mirrored_location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "is_private": { + "name": "is_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_fork": { + "name": "is_fork", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_issues": { + "name": "has_issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_starred": { + "name": "is_starred", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_archived": { + "name": "is_archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_lfs": { + "name": "has_lfs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "has_submodules": { + "name": "has_submodules", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "last_mirrored": { + "name": "last_mirrored", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination_org": { + "name": "destination_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_repositories_user_id": { + "name": "idx_repositories_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_repositories_config_id": { + "name": "idx_repositories_config_id", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "idx_repositories_status": { + "name": "idx_repositories_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_repositories_owner": { + "name": "idx_repositories_owner", + "columns": [ + "owner" + ], + "isUnique": false + }, + "idx_repositories_organization": { + "name": "idx_repositories_organization", + "columns": [ + "organization" + ], + "isUnique": false + }, + "idx_repositories_is_fork": { + "name": "idx_repositories_is_fork", + "columns": [ + "is_fork" + ], + "isUnique": false + }, + "idx_repositories_is_starred": { + "name": "idx_repositories_is_starred", + "columns": [ + "is_starred" + ], + "isUnique": false + } + }, + "foreignKeys": { + "repositories_user_id_users_id_fk": { + "name": "repositories_user_id_users_id_fk", + "tableFrom": "repositories", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "repositories_config_id_configs_id_fk": { + "name": "repositories_config_id_configs_id_fk", + "tableFrom": "repositories", + "tableTo": "configs", + "columnsFrom": [ + "config_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_sessions_user_id": { + "name": "idx_sessions_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_sessions_token": { + "name": "idx_sessions_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_sessions_expires_at": { + "name": "idx_sessions_expires_at", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sso_providers": { + "name": "sso_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "columns": [ + "provider_id" + ], + "isUnique": true + }, + "idx_sso_providers_provider_id": { + "name": "idx_sso_providers_provider_id", + "columns": [ + "provider_id" + ], + "isUnique": false + }, + "idx_sso_providers_domain": { + "name": "idx_sso_providers_domain", + "columns": [ + "domain" + ], + "isUnique": false + }, + "idx_sso_providers_issuer": { + "name": "idx_sso_providers_issuer", + "columns": [ + "issuer" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_email_unique": { + "name": "users_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_verification_tokens_token": { + "name": "idx_verification_tokens_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_verification_tokens_identifier": { + "name": "idx_verification_tokens_identifier", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verifications": { + "name": "verifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_verifications_identifier": { + "name": "idx_verifications_identifier", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index ddc0a39..8f2126f 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1757390828679, "tag": "0003_open_spacker_dave", "breakpoints": true + }, + { + "idx": 4, + "version": "6", + "when": 1757392620734, + "tag": "0004_grey_butterfly", + "breakpoints": true } ] } \ No newline at end of file diff --git a/package.json b/package.json index c29c6fb..b1963d9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "gitea-mirror", "type": "module", - "version": "3.5.4", + "version": "3.6.0", "engines": { "bun": ">=1.2.9" }, @@ -47,6 +47,7 @@ "@astrojs/node": "9.4.3", "@astrojs/react": "^4.3.1", "@better-auth/sso": "^1.3.8", + "@octokit/plugin-throttling": "^11.0.1", "@octokit/rest": "^22.0.0", "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-avatar": "^1.1.10", @@ -57,6 +58,7 @@ "@radix-ui/react-hover-card": "^1.1.15", "@radix-ui/react-label": "^2.1.7", "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-select": "^2.2.6", diff --git a/src/components/dashboard/Dashboard.tsx b/src/components/dashboard/Dashboard.tsx index a7ac653..08cfdef 100644 --- a/src/components/dashboard/Dashboard.tsx +++ b/src/components/dashboard/Dashboard.tsx @@ -9,6 +9,7 @@ import { apiRequest, showErrorToast } from "@/lib/utils"; import type { DashboardApiResponse } from "@/types/dashboard"; import { useSSE } from "@/hooks/useSEE"; import { toast } from "sonner"; +import { useEffect as useEffectForToasts } from "react"; import { Skeleton } from "@/components/ui/skeleton"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { useLiveRefresh } from "@/hooks/useLiveRefresh"; @@ -105,6 +106,51 @@ export function Dashboard() { onMessage: handleNewMessage, }); + // Setup rate limit event listener for toast notifications + useEffectForToasts(() => { + if (!user?.id) return; + + const eventSource = new EventSource(`/api/events?userId=${user.id}`); + + eventSource.addEventListener("rate-limit", (event) => { + try { + const data = JSON.parse(event.data); + + switch (data.type) { + case "warning": + // 80% threshold warning + toast.warning("GitHub API Rate Limit Warning", { + description: data.message, + duration: 8000, + }); + break; + + case "exceeded": + // 100% rate limit exceeded + toast.error("GitHub API Rate Limit Exceeded", { + description: data.message, + duration: 10000, + }); + break; + + case "resumed": + // Rate limit reset notification + toast.success("Rate Limit Reset", { + description: "API operations have resumed.", + duration: 5000, + }); + break; + } + } catch (error) { + console.error("Error parsing rate limit event:", error); + } + }); + + return () => { + eventSource.close(); + }; + }, [user?.id]); + // Extract fetchDashboardData as a stable callback const fetchDashboardData = useCallback(async (showToast = false) => { try { diff --git a/src/components/ui/progress.tsx b/src/components/ui/progress.tsx new file mode 100644 index 0000000..e3c1840 --- /dev/null +++ b/src/components/ui/progress.tsx @@ -0,0 +1,30 @@ +import * as React from "react" +import * as ProgressPrimitive from "@radix-ui/react-progress" + +import { cn } from "@/lib/utils" + +interface ProgressProps extends React.ComponentPropsWithoutRef { + indicatorClassName?: string +} + +const Progress = React.forwardRef< + React.ElementRef, + ProgressProps +>(({ className, value, indicatorClassName, ...props }, ref) => ( + + + +)) +Progress.displayName = ProgressPrimitive.Root.displayName + +export { Progress } \ No newline at end of file diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index 4b30868..634ae4a 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -82,5 +82,6 @@ export { oauthApplications, oauthAccessTokens, oauthConsent, - ssoProviders + ssoProviders, + rateLimits } from "./schema"; diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index d50cfc0..17b2153 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -626,10 +626,52 @@ export const ssoProviders = sqliteTable("sso_providers", { index("idx_sso_providers_issuer").on(table.issuer), ]); +// ===== Rate Limit Tracking ===== + +export const rateLimitSchema = z.object({ + id: z.string(), + userId: z.string(), + provider: z.enum(["github", "gitea"]).default("github"), + limit: z.number(), + remaining: z.number(), + used: z.number(), + reset: z.coerce.date(), + retryAfter: z.number().optional(), // seconds to wait + status: z.enum(["ok", "warning", "limited", "exceeded"]).default("ok"), + lastChecked: z.coerce.date(), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); + +export const rateLimits = sqliteTable("rate_limits", { + id: text("id").primaryKey(), + userId: text("user_id") + .notNull() + .references(() => users.id), + provider: text("provider").notNull().default("github"), + limit: integer("limit").notNull(), + remaining: integer("remaining").notNull(), + used: integer("used").notNull(), + reset: integer("reset", { mode: "timestamp" }).notNull(), + retryAfter: integer("retry_after"), // seconds to wait + status: text("status").notNull().default("ok"), + lastChecked: integer("last_checked", { mode: "timestamp" }).notNull(), + createdAt: integer("created_at", { mode: "timestamp" }) + .notNull() + .default(sql`(unixepoch())`), + updatedAt: integer("updated_at", { mode: "timestamp" }) + .notNull() + .default(sql`(unixepoch())`), +}, (table) => [ + index("idx_rate_limits_user_provider").on(table.userId, table.provider), + index("idx_rate_limits_status").on(table.status), +]); + // Export type definitions export type User = z.infer; export type Config = z.infer; export type Repository = z.infer; export type MirrorJob = z.infer; export type Organization = z.infer; -export type Event = z.infer; \ No newline at end of file +export type Event = z.infer; +export type RateLimit = z.infer; \ No newline at end of file diff --git a/src/lib/github.ts b/src/lib/github.ts index 0bd6d41..cb0b1fb 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -1,15 +1,176 @@ import type { GitOrg, MembershipRole } from "@/types/organizations"; import type { GitRepo, RepoStatus } from "@/types/Repository"; import { Octokit } from "@octokit/rest"; +import { throttling } from "@octokit/plugin-throttling"; import type { Config } from "@/types/config"; +// Conditionally import rate limit manager (not available in test environment) +let RateLimitManager: any = null; +let publishEvent: any = null; + +if (process.env.NODE_ENV !== "test") { + try { + const rateLimitModule = await import("@/lib/rate-limit-manager"); + RateLimitManager = rateLimitModule.RateLimitManager; + const eventsModule = await import("@/lib/events"); + publishEvent = eventsModule.publishEvent; + } catch (error) { + console.warn("Rate limit manager not available:", error); + } +} + +// Extend Octokit with throttling plugin +const MyOctokit = Octokit.plugin(throttling); /** - * Creates an authenticated Octokit instance + * Creates an authenticated Octokit instance with rate limit tracking and throttling */ -export function createGitHubClient(token: string): Octokit { - return new Octokit({ - auth: token, +export function createGitHubClient(token: string, userId?: string, username?: string): Octokit { + // Create a proper User-Agent to identify our application + // This helps GitHub understand our traffic patterns and can provide better rate limits + const userAgent = username + ? `gitea-mirror/3.5.4 (user:${username})` + : "gitea-mirror/3.5.4"; + + const octokit = new MyOctokit({ + auth: token, // Always use token for authentication (5000 req/hr vs 60 for unauthenticated) + userAgent, // Identify our application and user + baseUrl: "https://api.github.com", // Explicitly set the API endpoint + log: { + debug: () => {}, + info: console.log, + warn: console.warn, + error: console.error, + }, + request: { + // Add default headers for better identification + headers: { + accept: "application/vnd.github.v3+json", + "x-github-api-version": "2022-11-28", // Use a stable API version + }, + }, + throttle: { + onRateLimit: async (retryAfter: number, options: any, octokit: any, retryCount: number) => { + const isSearch = options.url.includes("/search/"); + const maxRetries = isSearch ? 5 : 3; // Search endpoints get more retries + + console.warn( + `[GitHub] Rate limit hit for ${options.method} ${options.url}. Retry ${retryCount + 1}/${maxRetries}` + ); + + // Update rate limit status and notify UI (if available) + if (userId && RateLimitManager) { + await RateLimitManager.updateFromResponse(userId, { + "retry-after": retryAfter.toString(), + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": (Date.now() / 1000 + retryAfter).toString(), + }); + } + + if (userId && publishEvent) { + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "rate-limited", + provider: "github", + retryAfter, + retryCount, + endpoint: options.url, + message: `Rate limit hit. Waiting ${retryAfter}s before retry ${retryCount + 1}/${maxRetries}...`, + }, + }); + } + + // Retry with exponential backoff + if (retryCount < maxRetries) { + console.log(`[GitHub] Waiting ${retryAfter}s before retry...`); + return true; + } + + // Max retries reached + console.error(`[GitHub] Max retries (${maxRetries}) reached for ${options.url}`); + return false; + }, + onSecondaryRateLimit: async (retryAfter: number, options: any, octokit: any, retryCount: number) => { + console.warn( + `[GitHub] Secondary rate limit hit for ${options.method} ${options.url}` + ); + + // Update status and notify UI (if available) + if (userId && publishEvent) { + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "secondary-limited", + provider: "github", + retryAfter, + retryCount, + endpoint: options.url, + message: `Secondary rate limit hit. Waiting ${retryAfter}s...`, + }, + }); + } + + // Retry up to 2 times for secondary rate limits + if (retryCount < 2) { + console.log(`[GitHub] Waiting ${retryAfter}s for secondary rate limit...`); + return true; + } + + return false; + }, + // Throttle options to prevent hitting limits + fallbackSecondaryRateRetryAfter: 60, // Wait 60s on secondary rate limit + minimumSecondaryRateRetryAfter: 5, // Min 5s wait + retryAfterBaseValue: 1000, // Base retry in ms + }, }); + + // Add additional rate limit tracking if userId is provided and RateLimitManager is available + if (userId && RateLimitManager) { + octokit.hook.after("request", async (response: any, options: any) => { + // Update rate limit from response headers + if (response.headers) { + await RateLimitManager.updateFromResponse(userId, response.headers); + } + }); + + octokit.hook.error("request", async (error: any, options: any) => { + // Handle rate limit errors + if (error.status === 403 || error.status === 429) { + const message = error.message || ""; + + if (message.includes("rate limit") || message.includes("API rate limit")) { + console.error(`[GitHub] Rate limit error for user ${userId}: ${message}`); + + // Update rate limit status from error response (if available) + if (error.response?.headers && RateLimitManager) { + await RateLimitManager.updateFromResponse(userId, error.response.headers); + } + + // Create error event for UI (if available) + if (publishEvent) { + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "error", + provider: "github", + error: message, + endpoint: options.url, + message: `Rate limit exceeded: ${message}`, + }, + }); + } + } + } + + throw error; + }); + } + + return octokit; } /** diff --git a/src/lib/rate-limit-manager.ts b/src/lib/rate-limit-manager.ts new file mode 100644 index 0000000..9a2ecb9 --- /dev/null +++ b/src/lib/rate-limit-manager.ts @@ -0,0 +1,422 @@ +import { db, rateLimits } from "@/lib/db"; +import { eq, and } from "drizzle-orm"; +import { v4 as uuidv4 } from "uuid"; +import type { Octokit } from "@octokit/rest"; +import { publishEvent } from "@/lib/events"; + +type RateLimitStatus = "ok" | "warning" | "limited" | "exceeded"; + +interface RateLimitInfo { + limit: number; + remaining: number; + used: number; + reset: Date; + retryAfter?: number; + status: RateLimitStatus; +} + +interface RateLimitHeaders { + "x-ratelimit-limit"?: string; + "x-ratelimit-remaining"?: string; + "x-ratelimit-used"?: string; + "x-ratelimit-reset"?: string; + "retry-after"?: string; +} + +/** + * Rate limit manager for GitHub API + * + * GitHub API Limits for authenticated users: + * - Primary: 5,000 requests per hour + * - Secondary: 900 points per minute (GET = 1 point, mutations = more) + * - Concurrent: Maximum 100 concurrent requests (recommended: 5-20) + * + * For repositories with many issues/PRs: + * - Each issue = 1 request to fetch + * - Each PR = 1 request to fetch + * - Comments = Additional requests per issue/PR + * - Better to limit by total requests rather than repositories + */ +export class RateLimitManager { + private static readonly WARNING_THRESHOLD = 0.2; // Warn when 20% remaining (80% used) + private static readonly PAUSE_THRESHOLD = 0.05; // Pause when 5% remaining + private static readonly MIN_REQUESTS_BUFFER = 100; // Keep at least 100 requests as buffer + private static lastNotifiedThreshold: Map = new Map(); // Track last notification per user + + /** + * Check current rate limit status from GitHub + */ + static async checkGitHubRateLimit(octokit: Octokit, userId: string): Promise { + try { + const { data } = await octokit.rateLimit.get(); + const core = data.rate; + + const info: RateLimitInfo = { + limit: core.limit, + remaining: core.remaining, + used: core.used, + reset: new Date(core.reset * 1000), + status: this.calculateStatus(core.remaining, core.limit), + }; + + // Update database + await this.updateRateLimit(userId, "github", info); + + return info; + } catch (error) { + console.error("Failed to check GitHub rate limit:", error); + // Return last known status from database if API check fails + return await this.getLastKnownStatus(userId, "github"); + } + } + + /** + * Extract rate limit info from response headers + */ + static parseRateLimitHeaders(headers: RateLimitHeaders): Partial { + const info: Partial = {}; + + if (headers["x-ratelimit-limit"]) { + info.limit = parseInt(headers["x-ratelimit-limit"], 10); + } + if (headers["x-ratelimit-remaining"]) { + info.remaining = parseInt(headers["x-ratelimit-remaining"], 10); + } + if (headers["x-ratelimit-used"]) { + info.used = parseInt(headers["x-ratelimit-used"], 10); + } + if (headers["x-ratelimit-reset"]) { + info.reset = new Date(parseInt(headers["x-ratelimit-reset"], 10) * 1000); + } + if (headers["retry-after"]) { + info.retryAfter = parseInt(headers["retry-after"], 10); + } + + if (info.remaining !== undefined && info.limit !== undefined) { + info.status = this.calculateStatus(info.remaining, info.limit); + } + + return info; + } + + /** + * Update rate limit info from API response + */ + static async updateFromResponse(userId: string, headers: RateLimitHeaders): Promise { + const info = this.parseRateLimitHeaders(headers); + if (Object.keys(info).length > 0) { + await this.updateRateLimit(userId, "github", info as RateLimitInfo); + } + } + + /** + * Calculate rate limit status based on remaining requests + */ + static calculateStatus(remaining: number, limit: number): RateLimitStatus { + const ratio = remaining / limit; + + if (remaining === 0) return "exceeded"; + if (remaining < this.MIN_REQUESTS_BUFFER || ratio < this.PAUSE_THRESHOLD) return "limited"; + if (ratio < this.WARNING_THRESHOLD) return "warning"; + return "ok"; + } + + /** + * Check if we should pause operations + */ + static async shouldPause(userId: string, provider: "github" | "gitea" = "github"): Promise { + const status = await this.getLastKnownStatus(userId, provider); + return status.status === "limited" || status.status === "exceeded"; + } + + /** + * Calculate wait time until rate limit resets + */ + static calculateWaitTime(reset: Date, retryAfter?: number): number { + if (retryAfter) { + return retryAfter * 1000; // Convert to milliseconds + } + + const now = new Date(); + const waitTime = reset.getTime() - now.getTime(); + return Math.max(0, waitTime); + } + + /** + * Wait until rate limit resets + */ + static async waitForReset(userId: string, provider: "github" | "gitea" = "github"): Promise { + const status = await this.getLastKnownStatus(userId, provider); + + if (status.status === "ok" || status.status === "warning") { + return; // No need to wait + } + + const waitTime = this.calculateWaitTime(status.reset, status.retryAfter); + + if (waitTime > 0) { + console.log(`[RateLimit] Waiting ${Math.ceil(waitTime / 1000)}s for rate limit reset...`); + + // Create event for UI notification + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "waiting", + provider, + waitTime, + resetAt: status.reset, + message: `API rate limit reached. Waiting ${Math.ceil(waitTime / 1000)} seconds before resuming...`, + }, + }); + + // Wait + await new Promise(resolve => setTimeout(resolve, waitTime)); + + // Update status after waiting + await this.updateRateLimit(userId, provider, { + ...status, + status: "ok", + remaining: status.limit, + used: 0, + }); + + // Notify that we've resumed + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "resumed", + provider, + message: "Rate limit reset. Resuming operations...", + }, + }); + } + } + + /** + * Update rate limit info in database + */ + private static async updateRateLimit( + userId: string, + provider: "github" | "gitea", + info: RateLimitInfo + ): Promise { + const existing = await db + .select() + .from(rateLimits) + .where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, provider))) + .limit(1); + + const data = { + userId, + provider, + limit: info.limit, + remaining: info.remaining, + used: info.used, + reset: info.reset, + retryAfter: info.retryAfter, + status: info.status, + lastChecked: new Date(), + updatedAt: new Date(), + }; + + if (existing.length > 0) { + await db + .update(rateLimits) + .set(data) + .where(eq(rateLimits.id, existing[0].id)); + } else { + await db.insert(rateLimits).values({ + id: uuidv4(), + ...data, + createdAt: new Date(), + }); + } + + // Only send notifications at specific thresholds to avoid spam + const usedPercentage = ((info.limit - info.remaining) / info.limit) * 100; + const userKey = `${userId}-${provider}`; + const lastNotified = this.lastNotifiedThreshold.get(userKey) || 0; + + // Notify at 80% usage (20% remaining) + if (usedPercentage >= 80 && usedPercentage < 100 && lastNotified < 80) { + this.lastNotifiedThreshold.set(userKey, 80); + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "warning", + provider, + status: info.status, + remaining: info.remaining, + limit: info.limit, + usedPercentage: Math.round(usedPercentage), + message: `GitHub API rate limit at ${Math.round(usedPercentage)}%. ${info.remaining} requests remaining.`, + }, + }); + console.log(`[RateLimit] 80% threshold reached for user ${userId}: ${info.remaining}/${info.limit} requests remaining`); + } + + // Notify at 100% usage (0 remaining) + if (info.remaining === 0 && lastNotified < 100) { + this.lastNotifiedThreshold.set(userKey, 100); + const resetTime = new Date(info.reset); + const minutesUntilReset = Math.ceil((resetTime.getTime() - Date.now()) / 60000); + await publishEvent({ + userId, + channel: "rate-limit", + payload: { + type: "exceeded", + provider, + status: "exceeded", + remaining: 0, + limit: info.limit, + usedPercentage: 100, + reset: info.reset, + message: `GitHub API rate limit exceeded. Will automatically resume in ${minutesUntilReset} minutes.`, + }, + }); + console.log(`[RateLimit] 100% rate limit exceeded for user ${userId}. Resets at ${resetTime.toLocaleTimeString()}`); + } + + // Reset notification threshold when rate limit resets + if (info.remaining > info.limit * 0.5 && lastNotified > 0) { + this.lastNotifiedThreshold.delete(userKey); + } + } + + /** + * Get last known rate limit status from database + */ + private static async getLastKnownStatus( + userId: string, + provider: "github" | "gitea" + ): Promise { + const [result] = await db + .select() + .from(rateLimits) + .where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, provider))) + .limit(1); + + if (result) { + return { + limit: result.limit, + remaining: result.remaining, + used: result.used, + reset: result.reset, + retryAfter: result.retryAfter ?? undefined, + status: result.status as RateLimitStatus, + }; + } + + // Return default if no data + return { + limit: 5000, + remaining: 5000, + used: 0, + reset: new Date(Date.now() + 3600000), // 1 hour from now + status: "ok", + }; + } + + /** + * Get human-readable status message + */ + private static getStatusMessage(info: RateLimitInfo): string { + const percentage = Math.round((info.remaining / info.limit) * 100); + + switch (info.status) { + case "exceeded": + return `API rate limit exceeded. Resets at ${info.reset.toLocaleTimeString()}.`; + case "limited": + return `API rate limit critical: Only ${info.remaining} requests remaining (${percentage}%). Pausing operations...`; + case "warning": + return `API rate limit warning: ${info.remaining} requests remaining (${percentage}%).`; + default: + return `API rate limit healthy: ${info.remaining}/${info.limit} requests remaining.`; + } + } + + /** + * Smart retry with exponential backoff for rate-limited requests + */ + static async retryWithBackoff( + fn: () => Promise, + userId: string, + maxRetries: number = 3 + ): Promise { + let lastError: any; + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + // Check if we should pause before attempting + if (await this.shouldPause(userId)) { + await this.waitForReset(userId); + } + + return await fn(); + } catch (error: any) { + lastError = error; + + // Check if it's a rate limit error + if (error.status === 403 && error.message?.includes("rate limit")) { + console.log(`[RateLimit] Rate limit hit on attempt ${attempt + 1}/${maxRetries}`); + + // Parse rate limit headers from error response if available + if (error.response?.headers) { + await this.updateFromResponse(userId, error.response.headers); + } + + // Wait for reset + await this.waitForReset(userId); + } else if (error.status === 429) { + // Too Many Requests - use exponential backoff + const backoffTime = Math.min(1000 * Math.pow(2, attempt), 30000); // Max 30s + console.log(`[RateLimit] Too many requests, backing off ${backoffTime}ms`); + await new Promise(resolve => setTimeout(resolve, backoffTime)); + } else { + // Not a rate limit error, throw immediately + throw error; + } + } + } + + throw lastError; + } +} + +/** + * Middleware to check rate limits before making API calls + */ +export async function withRateLimitCheck( + userId: string, + operation: () => Promise, + operationName: string = "API call" +): Promise { + // Check if we should pause + if (await RateLimitManager.shouldPause(userId)) { + console.log(`[RateLimit] Pausing ${operationName} due to rate limit`); + await RateLimitManager.waitForReset(userId); + } + + // Execute with retry logic + return await RateLimitManager.retryWithBackoff(operation, userId); +} + +/** + * Hook to update rate limits from Octokit responses + */ +export function createOctokitRateLimitPlugin(userId: string) { + return { + hook: (request: any, options: any) => { + return request(options).then((response: any) => { + // Update rate limit from response headers + if (response.headers) { + RateLimitManager.updateFromResponse(userId, response.headers).catch(console.error); + } + return response; + }); + }, + }; +} \ No newline at end of file diff --git a/src/lib/recovery.ts b/src/lib/recovery.ts index 08dc522..d0be12a 100644 --- a/src/lib/recovery.ts +++ b/src/lib/recovery.ts @@ -260,11 +260,13 @@ async function recoverMirrorJob(job: any, remainingItemIds: string[]) { throw new Error('GitHub token not found in configuration'); } - // Create GitHub client with error handling + // Create GitHub client with error handling and rate limit tracking let octokit; try { const decryptedToken = getDecryptedGitHubToken(config); - octokit = createGitHubClient(decryptedToken); + const githubUsername = config.githubConfig?.owner || undefined; + const userId = config.userId || undefined; + octokit = createGitHubClient(decryptedToken, userId, githubUsername); } catch (error) { throw new Error(`Failed to create GitHub client: ${error instanceof Error ? error.message : String(error)}`); } diff --git a/src/lib/repository-cleanup-service.ts b/src/lib/repository-cleanup-service.ts index 93b6698..10d2bb7 100644 --- a/src/lib/repository-cleanup-service.ts +++ b/src/lib/repository-cleanup-service.ts @@ -23,9 +23,10 @@ async function identifyOrphanedRepositories(config: any): Promise { const userId = config.userId; try { - // Get current GitHub repositories + // Get current GitHub repositories with rate limit tracking const decryptedToken = getDecryptedGitHubToken(config); - const octokit = createGitHubClient(decryptedToken); + const githubUsername = config.githubConfig?.owner || undefined; + const octokit = createGitHubClient(decryptedToken, userId, githubUsername); let allGithubRepos = []; let githubApiAccessible = true; diff --git a/src/lib/utils/concurrency.ts b/src/lib/utils/concurrency.ts index 47e54d3..595e4cc 100644 --- a/src/lib/utils/concurrency.ts +++ b/src/lib/utils/concurrency.ts @@ -10,7 +10,7 @@ export async function processInParallel( items: T[], processItem: (item: T) => Promise, - concurrencyLimit: number = 5, + concurrencyLimit: number = 5, // Safe default for GitHub API (max 100 concurrent, but 5-10 recommended) onProgress?: (completed: number, total: number, result?: R) => void ): Promise { const results: R[] = []; diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index 5f8ccce..768fe56 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -93,7 +93,7 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default enabled: scheduleEnabled, interval: scheduleInterval, concurrent: false, - batchSize: 10, + batchSize: 5, // Reduced from 10 to be more conservative with GitHub API limits lastRun: null, nextRun: scheduleEnabled ? new Date(Date.now() + scheduleInterval * 1000) : null, }, diff --git a/src/pages/api/events/index.ts b/src/pages/api/events/index.ts new file mode 100644 index 0000000..dd26e73 --- /dev/null +++ b/src/pages/api/events/index.ts @@ -0,0 +1,69 @@ +import type { APIRoute } from "astro"; +import { getNewEvents } from "@/lib/events"; + +export const GET: APIRoute = async ({ request }) => { + const url = new URL(request.url); + const userId = url.searchParams.get("userId"); + + if (!userId) { + return new Response("Missing userId", { status: 400 }); + } + + // Create a new ReadableStream for SSE + const stream = new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + let lastEventTime = new Date(); + + // Send initial connection message + controller.enqueue(encoder.encode(": connected\n\n")); + + // Poll for new events every 2 seconds + const pollInterval = setInterval(async () => { + try { + // Get new rate limit events + const newEvents = await getNewEvents({ + userId, + channel: "rate-limit", + lastEventTime, + }); + + // Send each new event + for (const event of newEvents) { + const message = `event: rate-limit\ndata: ${JSON.stringify(event.payload)}\n\n`; + controller.enqueue(encoder.encode(message)); + lastEventTime = new Date(event.createdAt); + } + } catch (error) { + console.error("Error polling for events:", error); + } + }, 2000); // Poll every 2 seconds + + // Send heartbeat every 30 seconds to keep connection alive + const heartbeatInterval = setInterval(() => { + try { + controller.enqueue(encoder.encode(": heartbeat\n\n")); + } catch (error) { + clearInterval(heartbeatInterval); + clearInterval(pollInterval); + } + }, 30000); + + // Cleanup on close + request.signal.addEventListener("abort", () => { + clearInterval(pollInterval); + clearInterval(heartbeatInterval); + controller.close(); + }); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", // Disable nginx buffering + }, + }); +}; \ No newline at end of file diff --git a/src/pages/api/job/mirror-org.ts b/src/pages/api/job/mirror-org.ts index d9328aa..488b80d 100644 --- a/src/pages/api/job/mirror-org.ts +++ b/src/pages/api/job/mirror-org.ts @@ -71,9 +71,10 @@ export const POST: APIRoute = async ({ request }) => { throw new Error("GitHub token is missing in config."); } - // Create a single Octokit instance to be reused + // Create a single Octokit instance to be reused with rate limit tracking const decryptedToken = getDecryptedGitHubToken(config); - const octokit = createGitHubClient(decryptedToken); + const githubUsername = config.githubConfig?.owner || undefined; + const octokit = createGitHubClient(decryptedToken, userId, githubUsername); // Define the concurrency limit - adjust based on API rate limits // Using a lower concurrency for organizations since each org might contain many repos diff --git a/src/pages/api/job/mirror-repo.ts b/src/pages/api/job/mirror-repo.ts index fb9476c..af14932 100644 --- a/src/pages/api/job/mirror-repo.ts +++ b/src/pages/api/job/mirror-repo.ts @@ -73,9 +73,10 @@ export const POST: APIRoute = async ({ request }) => { throw new Error("GitHub token is missing."); } - // Create a single Octokit instance to be reused + // Create a single Octokit instance to be reused with rate limit tracking const decryptedToken = getDecryptedGitHubToken(config); - const octokit = createGitHubClient(decryptedToken); + const githubUsername = config.githubConfig?.owner || undefined; + const octokit = createGitHubClient(decryptedToken, userId, githubUsername); // Define the concurrency limit - adjust based on API rate limits const CONCURRENCY_LIMIT = 3; diff --git a/src/pages/api/job/retry-repo.ts b/src/pages/api/job/retry-repo.ts index ddc6476..81192e6 100644 --- a/src/pages/api/job/retry-repo.ts +++ b/src/pages/api/job/retry-repo.ts @@ -71,12 +71,13 @@ export const POST: APIRoute = async ({ request }) => { // Start background retry with parallel processing setTimeout(async () => { - // Create a single Octokit instance to be reused if needed + // Create a single Octokit instance to be reused if needed with rate limit tracking const decryptedToken = config.githubConfig.token ? getDecryptedGitHubToken(config) : null; + const githubUsername = config.githubConfig?.owner || undefined; const octokit = decryptedToken - ? createGitHubClient(decryptedToken) + ? createGitHubClient(decryptedToken, userId, githubUsername) : null; // Define the concurrency limit - adjust based on API rate limits diff --git a/src/pages/api/rate-limit/index.ts b/src/pages/api/rate-limit/index.ts new file mode 100644 index 0000000..dabb45e --- /dev/null +++ b/src/pages/api/rate-limit/index.ts @@ -0,0 +1,104 @@ +import type { APIRoute } from "astro"; +import { db, rateLimits } from "@/lib/db"; +import { eq, and, desc } from "drizzle-orm"; +import { jsonResponse, createSecureErrorResponse } from "@/lib/utils"; +import { RateLimitManager } from "@/lib/rate-limit-manager"; +import { createGitHubClient } from "@/lib/github"; +import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption"; +import { configs } from "@/lib/db"; + +export const GET: APIRoute = async ({ request }) => { + const url = new URL(request.url); + const userId = url.searchParams.get("userId"); + const refresh = url.searchParams.get("refresh") === "true"; + + if (!userId) { + return jsonResponse({ + data: { error: "Missing userId" }, + status: 400, + }); + } + + try { + // If refresh is requested, fetch current rate limit from GitHub + if (refresh) { + const [config] = await db + .select() + .from(configs) + .where(eq(configs.userId, userId)) + .limit(1); + + if (config && config.githubConfig?.token) { + const decryptedToken = getDecryptedGitHubToken(config); + const githubUsername = config.githubConfig?.owner || undefined; + const octokit = createGitHubClient(decryptedToken, userId, githubUsername); + + // This will update the rate limit in the database + await RateLimitManager.checkGitHubRateLimit(octokit, userId); + } + } + + // Get rate limit status from database + const [rateLimit] = await db + .select() + .from(rateLimits) + .where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, "github"))) + .orderBy(desc(rateLimits.updatedAt)) + .limit(1); + + if (!rateLimit) { + return jsonResponse({ + data: { + limit: 5000, + remaining: 5000, + used: 0, + reset: new Date(Date.now() + 3600000), // 1 hour from now + status: "ok", + lastChecked: new Date(), + message: "No rate limit data available yet", + }, + }); + } + + // Calculate percentage + const percentage = Math.round((rateLimit.remaining / rateLimit.limit) * 100); + + // Calculate time until reset + const now = new Date(); + const resetTime = new Date(rateLimit.reset); + const timeUntilReset = Math.max(0, resetTime.getTime() - now.getTime()); + const minutesUntilReset = Math.ceil(timeUntilReset / 60000); + + let message = ""; + switch (rateLimit.status) { + case "exceeded": + message = `Rate limit exceeded. Resets in ${minutesUntilReset} minutes.`; + break; + case "limited": + message = `Rate limit critical: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`; + break; + case "warning": + message = `Rate limit warning: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`; + break; + default: + message = `Rate limit healthy: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`; + } + + return jsonResponse({ + data: { + limit: rateLimit.limit, + remaining: rateLimit.remaining, + used: rateLimit.used, + reset: rateLimit.reset, + retryAfter: rateLimit.retryAfter, + status: rateLimit.status, + lastChecked: rateLimit.lastChecked, + percentage, + minutesUntilReset, + message, + }, + }); + } catch (error) { + return createSecureErrorResponse(error, "rate limit check", 500); + } +}; \ No newline at end of file diff --git a/src/pages/api/sync/index.ts b/src/pages/api/sync/index.ts index a65da9b..2840d5d 100644 --- a/src/pages/api/sync/index.ts +++ b/src/pages/api/sync/index.ts @@ -43,7 +43,8 @@ export const POST: APIRoute = async ({ request }) => { // Decrypt the GitHub token before using it const decryptedToken = getDecryptedGitHubToken(config); - const octokit = createGitHubClient(decryptedToken); + const githubUsername = config.githubConfig?.owner || undefined; + const octokit = createGitHubClient(decryptedToken, userId, githubUsername); // Fetch GitHub data in parallel const [basicAndForkedRepos, starredRepos, gitOrgs] = await Promise.all([ diff --git a/src/pages/api/sync/organization.ts b/src/pages/api/sync/organization.ts index 9a8db84..9638333 100644 --- a/src/pages/api/sync/organization.ts +++ b/src/pages/api/sync/organization.ts @@ -69,8 +69,9 @@ export const POST: APIRoute = async ({ request }) => { }); } - // Create authenticated Octokit instance - const octokit = createGitHubClient(decryptedConfig.githubConfig.token); + // Create authenticated Octokit instance with rate limit tracking + const githubUsername = decryptedConfig.githubConfig?.owner || undefined; + const octokit = createGitHubClient(decryptedConfig.githubConfig.token, userId, githubUsername); // Fetch org metadata const { data: orgData } = await octokit.orgs.get({ org });