mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2025-12-06 11:36:44 +03:00
Added Github API rate limiting
- Implemented comprehensive GitHub API rate limit handling:
- Integrated @octokit/plugin-throttling for automatic retry with exponential backoff
- Added RateLimitManager service to track and enforce rate limits
- Store rate limit status in database for persistence across restarts
- Automatic pause and resume when limits are exceeded
- Proper user identification for 5000 req/hr authenticated limit (vs 60 unauthenticated)
- Improved rate limit UI/UX:
- Removed intrusive rate limit card from dashboard
- Toast notifications only at critical thresholds (80% and 100% usage)
- All rate limit events logged for debugging
- Optimized for GitHub's API constraints:
- Reduced default batch size from 10 to 5 repositories
- Added documentation about GitHub's 100 concurrent request limit
- Better handling of repositories with many issues/PRs
This commit is contained in:
Arunavo Ray
8
bun.lock
8
bun.lock
@@ -9,6 +9,7 @@
|
||||
"@astrojs/node": "9.4.3",
|
||||
"@astrojs/react": "^4.3.1",
|
||||
"@better-auth/sso": "^1.3.8",
|
||||
"@octokit/plugin-throttling": "^11.0.1",
|
||||
"@octokit/rest": "^22.0.0",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-avatar": "^1.1.10",
|
||||
@@ -19,6 +20,7 @@
|
||||
"@radix-ui/react-hover-card": "^1.1.15",
|
||||
"@radix-ui/react-label": "^2.1.7",
|
||||
"@radix-ui/react-popover": "^1.1.15",
|
||||
"@radix-ui/react-progress": "^1.1.7",
|
||||
"@radix-ui/react-radio-group": "^1.3.8",
|
||||
"@radix-ui/react-scroll-area": "^1.2.10",
|
||||
"@radix-ui/react-select": "^2.2.6",
|
||||
@@ -329,6 +331,8 @@
|
||||
|
||||
"@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@16.0.0", "", { "dependencies": { "@octokit/types": "^14.1.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g=="],
|
||||
|
||||
"@octokit/plugin-throttling": ["@octokit/plugin-throttling@11.0.1", "", { "dependencies": { "@octokit/types": "^14.0.0", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": "^7.0.0" } }, "sha512-S+EVhy52D/272L7up58dr3FNSMXWuNZolkL4zMJBNIfIxyZuUcczsQAU4b5w6dewJXnKYVgSHSV5wxitMSW1kw=="],
|
||||
|
||||
"@octokit/request": ["@octokit/request@10.0.2", "", { "dependencies": { "@octokit/endpoint": "^11.0.0", "@octokit/request-error": "^7.0.0", "@octokit/types": "^14.0.0", "fast-content-type-parse": "^3.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-iYj4SJG/2bbhh+iIpFmG5u49DtJ4lipQ+aPakjL9OKpsGY93wM8w06gvFbEQxcMsZcCvk5th5KkIm2m8o14aWA=="],
|
||||
|
||||
"@octokit/request-error": ["@octokit/request-error@7.0.0", "", { "dependencies": { "@octokit/types": "^14.0.0" } }, "sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg=="],
|
||||
@@ -399,6 +403,8 @@
|
||||
|
||||
"@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.3", "", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="],
|
||||
|
||||
"@radix-ui/react-progress": ["@radix-ui/react-progress@1.1.7", "", { "dependencies": { "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-vPdg/tF6YC/ynuBIJlk1mm7Le0VgW6ub6J2UWnTQ7/D23KXcPI1qy+0vBkgKgd38RCMJavBXpB83HPNFMTb0Fg=="],
|
||||
|
||||
"@radix-ui/react-radio-group": ["@radix-ui/react-radio-group@1.3.8", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ=="],
|
||||
|
||||
"@radix-ui/react-roving-focus": ["@radix-ui/react-roving-focus@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA=="],
|
||||
@@ -691,6 +697,8 @@
|
||||
|
||||
"body-parser": ["body-parser@1.20.3", "", { "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.13.0", "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" } }, "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g=="],
|
||||
|
||||
"bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="],
|
||||
|
||||
"boxen": ["boxen@8.0.1", "", { "dependencies": { "ansi-align": "^3.0.1", "camelcase": "^8.0.0", "chalk": "^5.3.0", "cli-boxes": "^3.0.0", "string-width": "^7.2.0", "type-fest": "^4.21.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0" } }, "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw=="],
|
||||
|
||||
"braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="],
|
||||
|
||||
18
drizzle/0004_grey_butterfly.sql
Normal file
18
drizzle/0004_grey_butterfly.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE `rate_limits` (
|
||||
`id` text PRIMARY KEY NOT NULL,
|
||||
`user_id` text NOT NULL,
|
||||
`provider` text DEFAULT 'github' NOT NULL,
|
||||
`limit` integer NOT NULL,
|
||||
`remaining` integer NOT NULL,
|
||||
`used` integer NOT NULL,
|
||||
`reset` integer NOT NULL,
|
||||
`retry_after` integer,
|
||||
`status` text DEFAULT 'ok' NOT NULL,
|
||||
`last_checked` integer NOT NULL,
|
||||
`created_at` integer DEFAULT (unixepoch()) NOT NULL,
|
||||
`updated_at` integer DEFAULT (unixepoch()) NOT NULL,
|
||||
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `idx_rate_limits_user_provider` ON `rate_limits` (`user_id`,`provider`);--> statement-breakpoint
|
||||
CREATE INDEX `idx_rate_limits_status` ON `rate_limits` (`status`);
|
||||
1933
drizzle/meta/0004_snapshot.json
Normal file
1933
drizzle/meta/0004_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,13 @@
|
||||
"when": 1757390828679,
|
||||
"tag": "0003_open_spacker_dave",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 4,
|
||||
"version": "6",
|
||||
"when": 1757392620734,
|
||||
"tag": "0004_grey_butterfly",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "3.5.4",
|
||||
"version": "3.6.0",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
@@ -47,6 +47,7 @@
|
||||
"@astrojs/node": "9.4.3",
|
||||
"@astrojs/react": "^4.3.1",
|
||||
"@better-auth/sso": "^1.3.8",
|
||||
"@octokit/plugin-throttling": "^11.0.1",
|
||||
"@octokit/rest": "^22.0.0",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-avatar": "^1.1.10",
|
||||
@@ -57,6 +58,7 @@
|
||||
"@radix-ui/react-hover-card": "^1.1.15",
|
||||
"@radix-ui/react-label": "^2.1.7",
|
||||
"@radix-ui/react-popover": "^1.1.15",
|
||||
"@radix-ui/react-progress": "^1.1.7",
|
||||
"@radix-ui/react-radio-group": "^1.3.8",
|
||||
"@radix-ui/react-scroll-area": "^1.2.10",
|
||||
"@radix-ui/react-select": "^2.2.6",
|
||||
|
||||
@@ -9,6 +9,7 @@ import { apiRequest, showErrorToast } from "@/lib/utils";
|
||||
import type { DashboardApiResponse } from "@/types/dashboard";
|
||||
import { useSSE } from "@/hooks/useSEE";
|
||||
import { toast } from "sonner";
|
||||
import { useEffect as useEffectForToasts } from "react";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
|
||||
@@ -105,6 +106,51 @@ export function Dashboard() {
|
||||
onMessage: handleNewMessage,
|
||||
});
|
||||
|
||||
// Setup rate limit event listener for toast notifications
|
||||
useEffectForToasts(() => {
|
||||
if (!user?.id) return;
|
||||
|
||||
const eventSource = new EventSource(`/api/events?userId=${user.id}`);
|
||||
|
||||
eventSource.addEventListener("rate-limit", (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
switch (data.type) {
|
||||
case "warning":
|
||||
// 80% threshold warning
|
||||
toast.warning("GitHub API Rate Limit Warning", {
|
||||
description: data.message,
|
||||
duration: 8000,
|
||||
});
|
||||
break;
|
||||
|
||||
case "exceeded":
|
||||
// 100% rate limit exceeded
|
||||
toast.error("GitHub API Rate Limit Exceeded", {
|
||||
description: data.message,
|
||||
duration: 10000,
|
||||
});
|
||||
break;
|
||||
|
||||
case "resumed":
|
||||
// Rate limit reset notification
|
||||
toast.success("Rate Limit Reset", {
|
||||
description: "API operations have resumed.",
|
||||
duration: 5000,
|
||||
});
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error parsing rate limit event:", error);
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
eventSource.close();
|
||||
};
|
||||
}, [user?.id]);
|
||||
|
||||
// Extract fetchDashboardData as a stable callback
|
||||
const fetchDashboardData = useCallback(async (showToast = false) => {
|
||||
try {
|
||||
|
||||
30
src/components/ui/progress.tsx
Normal file
30
src/components/ui/progress.tsx
Normal file
@@ -0,0 +1,30 @@
|
||||
import * as React from "react"
|
||||
import * as ProgressPrimitive from "@radix-ui/react-progress"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
interface ProgressProps extends React.ComponentPropsWithoutRef<typeof ProgressPrimitive.Root> {
|
||||
indicatorClassName?: string
|
||||
}
|
||||
|
||||
const Progress = React.forwardRef<
|
||||
React.ElementRef<typeof ProgressPrimitive.Root>,
|
||||
ProgressProps
|
||||
>(({ className, value, indicatorClassName, ...props }, ref) => (
|
||||
<ProgressPrimitive.Root
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"relative h-4 w-full overflow-hidden rounded-full bg-secondary",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<ProgressPrimitive.Indicator
|
||||
className={cn("h-full w-full flex-1 bg-primary transition-all", indicatorClassName)}
|
||||
style={{ transform: `translateX(-${100 - (value || 0)}%)` }}
|
||||
/>
|
||||
</ProgressPrimitive.Root>
|
||||
))
|
||||
Progress.displayName = ProgressPrimitive.Root.displayName
|
||||
|
||||
export { Progress }
|
||||
@@ -82,5 +82,6 @@ export {
|
||||
oauthApplications,
|
||||
oauthAccessTokens,
|
||||
oauthConsent,
|
||||
ssoProviders
|
||||
ssoProviders,
|
||||
rateLimits
|
||||
} from "./schema";
|
||||
|
||||
@@ -626,10 +626,52 @@ export const ssoProviders = sqliteTable("sso_providers", {
|
||||
index("idx_sso_providers_issuer").on(table.issuer),
|
||||
]);
|
||||
|
||||
// ===== Rate Limit Tracking =====
|
||||
|
||||
export const rateLimitSchema = z.object({
|
||||
id: z.string(),
|
||||
userId: z.string(),
|
||||
provider: z.enum(["github", "gitea"]).default("github"),
|
||||
limit: z.number(),
|
||||
remaining: z.number(),
|
||||
used: z.number(),
|
||||
reset: z.coerce.date(),
|
||||
retryAfter: z.number().optional(), // seconds to wait
|
||||
status: z.enum(["ok", "warning", "limited", "exceeded"]).default("ok"),
|
||||
lastChecked: z.coerce.date(),
|
||||
createdAt: z.coerce.date(),
|
||||
updatedAt: z.coerce.date(),
|
||||
});
|
||||
|
||||
export const rateLimits = sqliteTable("rate_limits", {
|
||||
id: text("id").primaryKey(),
|
||||
userId: text("user_id")
|
||||
.notNull()
|
||||
.references(() => users.id),
|
||||
provider: text("provider").notNull().default("github"),
|
||||
limit: integer("limit").notNull(),
|
||||
remaining: integer("remaining").notNull(),
|
||||
used: integer("used").notNull(),
|
||||
reset: integer("reset", { mode: "timestamp" }).notNull(),
|
||||
retryAfter: integer("retry_after"), // seconds to wait
|
||||
status: text("status").notNull().default("ok"),
|
||||
lastChecked: integer("last_checked", { mode: "timestamp" }).notNull(),
|
||||
createdAt: integer("created_at", { mode: "timestamp" })
|
||||
.notNull()
|
||||
.default(sql`(unixepoch())`),
|
||||
updatedAt: integer("updated_at", { mode: "timestamp" })
|
||||
.notNull()
|
||||
.default(sql`(unixepoch())`),
|
||||
}, (table) => [
|
||||
index("idx_rate_limits_user_provider").on(table.userId, table.provider),
|
||||
index("idx_rate_limits_status").on(table.status),
|
||||
]);
|
||||
|
||||
// Export type definitions
|
||||
export type User = z.infer<typeof userSchema>;
|
||||
export type Config = z.infer<typeof configSchema>;
|
||||
export type Repository = z.infer<typeof repositorySchema>;
|
||||
export type MirrorJob = z.infer<typeof mirrorJobSchema>;
|
||||
export type Organization = z.infer<typeof organizationSchema>;
|
||||
export type Event = z.infer<typeof eventSchema>;
|
||||
export type Event = z.infer<typeof eventSchema>;
|
||||
export type RateLimit = z.infer<typeof rateLimitSchema>;
|
||||
@@ -1,15 +1,176 @@
|
||||
import type { GitOrg, MembershipRole } from "@/types/organizations";
|
||||
import type { GitRepo, RepoStatus } from "@/types/Repository";
|
||||
import { Octokit } from "@octokit/rest";
|
||||
import { throttling } from "@octokit/plugin-throttling";
|
||||
import type { Config } from "@/types/config";
|
||||
// Conditionally import rate limit manager (not available in test environment)
|
||||
let RateLimitManager: any = null;
|
||||
let publishEvent: any = null;
|
||||
|
||||
if (process.env.NODE_ENV !== "test") {
|
||||
try {
|
||||
const rateLimitModule = await import("@/lib/rate-limit-manager");
|
||||
RateLimitManager = rateLimitModule.RateLimitManager;
|
||||
const eventsModule = await import("@/lib/events");
|
||||
publishEvent = eventsModule.publishEvent;
|
||||
} catch (error) {
|
||||
console.warn("Rate limit manager not available:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Extend Octokit with throttling plugin
|
||||
const MyOctokit = Octokit.plugin(throttling);
|
||||
|
||||
/**
|
||||
* Creates an authenticated Octokit instance
|
||||
* Creates an authenticated Octokit instance with rate limit tracking and throttling
|
||||
*/
|
||||
export function createGitHubClient(token: string): Octokit {
|
||||
return new Octokit({
|
||||
auth: token,
|
||||
export function createGitHubClient(token: string, userId?: string, username?: string): Octokit {
|
||||
// Create a proper User-Agent to identify our application
|
||||
// This helps GitHub understand our traffic patterns and can provide better rate limits
|
||||
const userAgent = username
|
||||
? `gitea-mirror/3.5.4 (user:${username})`
|
||||
: "gitea-mirror/3.5.4";
|
||||
|
||||
const octokit = new MyOctokit({
|
||||
auth: token, // Always use token for authentication (5000 req/hr vs 60 for unauthenticated)
|
||||
userAgent, // Identify our application and user
|
||||
baseUrl: "https://api.github.com", // Explicitly set the API endpoint
|
||||
log: {
|
||||
debug: () => {},
|
||||
info: console.log,
|
||||
warn: console.warn,
|
||||
error: console.error,
|
||||
},
|
||||
request: {
|
||||
// Add default headers for better identification
|
||||
headers: {
|
||||
accept: "application/vnd.github.v3+json",
|
||||
"x-github-api-version": "2022-11-28", // Use a stable API version
|
||||
},
|
||||
},
|
||||
throttle: {
|
||||
onRateLimit: async (retryAfter: number, options: any, octokit: any, retryCount: number) => {
|
||||
const isSearch = options.url.includes("/search/");
|
||||
const maxRetries = isSearch ? 5 : 3; // Search endpoints get more retries
|
||||
|
||||
console.warn(
|
||||
`[GitHub] Rate limit hit for ${options.method} ${options.url}. Retry ${retryCount + 1}/${maxRetries}`
|
||||
);
|
||||
|
||||
// Update rate limit status and notify UI (if available)
|
||||
if (userId && RateLimitManager) {
|
||||
await RateLimitManager.updateFromResponse(userId, {
|
||||
"retry-after": retryAfter.toString(),
|
||||
"x-ratelimit-remaining": "0",
|
||||
"x-ratelimit-reset": (Date.now() / 1000 + retryAfter).toString(),
|
||||
});
|
||||
}
|
||||
|
||||
if (userId && publishEvent) {
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "rate-limited",
|
||||
provider: "github",
|
||||
retryAfter,
|
||||
retryCount,
|
||||
endpoint: options.url,
|
||||
message: `Rate limit hit. Waiting ${retryAfter}s before retry ${retryCount + 1}/${maxRetries}...`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Retry with exponential backoff
|
||||
if (retryCount < maxRetries) {
|
||||
console.log(`[GitHub] Waiting ${retryAfter}s before retry...`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Max retries reached
|
||||
console.error(`[GitHub] Max retries (${maxRetries}) reached for ${options.url}`);
|
||||
return false;
|
||||
},
|
||||
onSecondaryRateLimit: async (retryAfter: number, options: any, octokit: any, retryCount: number) => {
|
||||
console.warn(
|
||||
`[GitHub] Secondary rate limit hit for ${options.method} ${options.url}`
|
||||
);
|
||||
|
||||
// Update status and notify UI (if available)
|
||||
if (userId && publishEvent) {
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "secondary-limited",
|
||||
provider: "github",
|
||||
retryAfter,
|
||||
retryCount,
|
||||
endpoint: options.url,
|
||||
message: `Secondary rate limit hit. Waiting ${retryAfter}s...`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Retry up to 2 times for secondary rate limits
|
||||
if (retryCount < 2) {
|
||||
console.log(`[GitHub] Waiting ${retryAfter}s for secondary rate limit...`);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
},
|
||||
// Throttle options to prevent hitting limits
|
||||
fallbackSecondaryRateRetryAfter: 60, // Wait 60s on secondary rate limit
|
||||
minimumSecondaryRateRetryAfter: 5, // Min 5s wait
|
||||
retryAfterBaseValue: 1000, // Base retry in ms
|
||||
},
|
||||
});
|
||||
|
||||
// Add additional rate limit tracking if userId is provided and RateLimitManager is available
|
||||
if (userId && RateLimitManager) {
|
||||
octokit.hook.after("request", async (response: any, options: any) => {
|
||||
// Update rate limit from response headers
|
||||
if (response.headers) {
|
||||
await RateLimitManager.updateFromResponse(userId, response.headers);
|
||||
}
|
||||
});
|
||||
|
||||
octokit.hook.error("request", async (error: any, options: any) => {
|
||||
// Handle rate limit errors
|
||||
if (error.status === 403 || error.status === 429) {
|
||||
const message = error.message || "";
|
||||
|
||||
if (message.includes("rate limit") || message.includes("API rate limit")) {
|
||||
console.error(`[GitHub] Rate limit error for user ${userId}: ${message}`);
|
||||
|
||||
// Update rate limit status from error response (if available)
|
||||
if (error.response?.headers && RateLimitManager) {
|
||||
await RateLimitManager.updateFromResponse(userId, error.response.headers);
|
||||
}
|
||||
|
||||
// Create error event for UI (if available)
|
||||
if (publishEvent) {
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "error",
|
||||
provider: "github",
|
||||
error: message,
|
||||
endpoint: options.url,
|
||||
message: `Rate limit exceeded: ${message}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
return octokit;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
422
src/lib/rate-limit-manager.ts
Normal file
422
src/lib/rate-limit-manager.ts
Normal file
@@ -0,0 +1,422 @@
|
||||
import { db, rateLimits } from "@/lib/db";
|
||||
import { eq, and } from "drizzle-orm";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type { Octokit } from "@octokit/rest";
|
||||
import { publishEvent } from "@/lib/events";
|
||||
|
||||
type RateLimitStatus = "ok" | "warning" | "limited" | "exceeded";
|
||||
|
||||
interface RateLimitInfo {
|
||||
limit: number;
|
||||
remaining: number;
|
||||
used: number;
|
||||
reset: Date;
|
||||
retryAfter?: number;
|
||||
status: RateLimitStatus;
|
||||
}
|
||||
|
||||
interface RateLimitHeaders {
|
||||
"x-ratelimit-limit"?: string;
|
||||
"x-ratelimit-remaining"?: string;
|
||||
"x-ratelimit-used"?: string;
|
||||
"x-ratelimit-reset"?: string;
|
||||
"retry-after"?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate limit manager for GitHub API
|
||||
*
|
||||
* GitHub API Limits for authenticated users:
|
||||
* - Primary: 5,000 requests per hour
|
||||
* - Secondary: 900 points per minute (GET = 1 point, mutations = more)
|
||||
* - Concurrent: Maximum 100 concurrent requests (recommended: 5-20)
|
||||
*
|
||||
* For repositories with many issues/PRs:
|
||||
* - Each issue = 1 request to fetch
|
||||
* - Each PR = 1 request to fetch
|
||||
* - Comments = Additional requests per issue/PR
|
||||
* - Better to limit by total requests rather than repositories
|
||||
*/
|
||||
export class RateLimitManager {
|
||||
private static readonly WARNING_THRESHOLD = 0.2; // Warn when 20% remaining (80% used)
|
||||
private static readonly PAUSE_THRESHOLD = 0.05; // Pause when 5% remaining
|
||||
private static readonly MIN_REQUESTS_BUFFER = 100; // Keep at least 100 requests as buffer
|
||||
private static lastNotifiedThreshold: Map<string, number> = new Map(); // Track last notification per user
|
||||
|
||||
/**
|
||||
* Check current rate limit status from GitHub
|
||||
*/
|
||||
static async checkGitHubRateLimit(octokit: Octokit, userId: string): Promise<RateLimitInfo> {
|
||||
try {
|
||||
const { data } = await octokit.rateLimit.get();
|
||||
const core = data.rate;
|
||||
|
||||
const info: RateLimitInfo = {
|
||||
limit: core.limit,
|
||||
remaining: core.remaining,
|
||||
used: core.used,
|
||||
reset: new Date(core.reset * 1000),
|
||||
status: this.calculateStatus(core.remaining, core.limit),
|
||||
};
|
||||
|
||||
// Update database
|
||||
await this.updateRateLimit(userId, "github", info);
|
||||
|
||||
return info;
|
||||
} catch (error) {
|
||||
console.error("Failed to check GitHub rate limit:", error);
|
||||
// Return last known status from database if API check fails
|
||||
return await this.getLastKnownStatus(userId, "github");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract rate limit info from response headers
|
||||
*/
|
||||
static parseRateLimitHeaders(headers: RateLimitHeaders): Partial<RateLimitInfo> {
|
||||
const info: Partial<RateLimitInfo> = {};
|
||||
|
||||
if (headers["x-ratelimit-limit"]) {
|
||||
info.limit = parseInt(headers["x-ratelimit-limit"], 10);
|
||||
}
|
||||
if (headers["x-ratelimit-remaining"]) {
|
||||
info.remaining = parseInt(headers["x-ratelimit-remaining"], 10);
|
||||
}
|
||||
if (headers["x-ratelimit-used"]) {
|
||||
info.used = parseInt(headers["x-ratelimit-used"], 10);
|
||||
}
|
||||
if (headers["x-ratelimit-reset"]) {
|
||||
info.reset = new Date(parseInt(headers["x-ratelimit-reset"], 10) * 1000);
|
||||
}
|
||||
if (headers["retry-after"]) {
|
||||
info.retryAfter = parseInt(headers["retry-after"], 10);
|
||||
}
|
||||
|
||||
if (info.remaining !== undefined && info.limit !== undefined) {
|
||||
info.status = this.calculateStatus(info.remaining, info.limit);
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update rate limit info from API response
|
||||
*/
|
||||
static async updateFromResponse(userId: string, headers: RateLimitHeaders): Promise<void> {
|
||||
const info = this.parseRateLimitHeaders(headers);
|
||||
if (Object.keys(info).length > 0) {
|
||||
await this.updateRateLimit(userId, "github", info as RateLimitInfo);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate rate limit status based on remaining requests
|
||||
*/
|
||||
static calculateStatus(remaining: number, limit: number): RateLimitStatus {
|
||||
const ratio = remaining / limit;
|
||||
|
||||
if (remaining === 0) return "exceeded";
|
||||
if (remaining < this.MIN_REQUESTS_BUFFER || ratio < this.PAUSE_THRESHOLD) return "limited";
|
||||
if (ratio < this.WARNING_THRESHOLD) return "warning";
|
||||
return "ok";
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should pause operations
|
||||
*/
|
||||
static async shouldPause(userId: string, provider: "github" | "gitea" = "github"): Promise<boolean> {
|
||||
const status = await this.getLastKnownStatus(userId, provider);
|
||||
return status.status === "limited" || status.status === "exceeded";
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate wait time until rate limit resets
|
||||
*/
|
||||
static calculateWaitTime(reset: Date, retryAfter?: number): number {
|
||||
if (retryAfter) {
|
||||
return retryAfter * 1000; // Convert to milliseconds
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const waitTime = reset.getTime() - now.getTime();
|
||||
return Math.max(0, waitTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until rate limit resets
|
||||
*/
|
||||
static async waitForReset(userId: string, provider: "github" | "gitea" = "github"): Promise<void> {
|
||||
const status = await this.getLastKnownStatus(userId, provider);
|
||||
|
||||
if (status.status === "ok" || status.status === "warning") {
|
||||
return; // No need to wait
|
||||
}
|
||||
|
||||
const waitTime = this.calculateWaitTime(status.reset, status.retryAfter);
|
||||
|
||||
if (waitTime > 0) {
|
||||
console.log(`[RateLimit] Waiting ${Math.ceil(waitTime / 1000)}s for rate limit reset...`);
|
||||
|
||||
// Create event for UI notification
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "waiting",
|
||||
provider,
|
||||
waitTime,
|
||||
resetAt: status.reset,
|
||||
message: `API rate limit reached. Waiting ${Math.ceil(waitTime / 1000)} seconds before resuming...`,
|
||||
},
|
||||
});
|
||||
|
||||
// Wait
|
||||
await new Promise(resolve => setTimeout(resolve, waitTime));
|
||||
|
||||
// Update status after waiting
|
||||
await this.updateRateLimit(userId, provider, {
|
||||
...status,
|
||||
status: "ok",
|
||||
remaining: status.limit,
|
||||
used: 0,
|
||||
});
|
||||
|
||||
// Notify that we've resumed
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "resumed",
|
||||
provider,
|
||||
message: "Rate limit reset. Resuming operations...",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update rate limit info in database
|
||||
*/
|
||||
private static async updateRateLimit(
|
||||
userId: string,
|
||||
provider: "github" | "gitea",
|
||||
info: RateLimitInfo
|
||||
): Promise<void> {
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(rateLimits)
|
||||
.where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, provider)))
|
||||
.limit(1);
|
||||
|
||||
const data = {
|
||||
userId,
|
||||
provider,
|
||||
limit: info.limit,
|
||||
remaining: info.remaining,
|
||||
used: info.used,
|
||||
reset: info.reset,
|
||||
retryAfter: info.retryAfter,
|
||||
status: info.status,
|
||||
lastChecked: new Date(),
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
if (existing.length > 0) {
|
||||
await db
|
||||
.update(rateLimits)
|
||||
.set(data)
|
||||
.where(eq(rateLimits.id, existing[0].id));
|
||||
} else {
|
||||
await db.insert(rateLimits).values({
|
||||
id: uuidv4(),
|
||||
...data,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
// Only send notifications at specific thresholds to avoid spam
|
||||
const usedPercentage = ((info.limit - info.remaining) / info.limit) * 100;
|
||||
const userKey = `${userId}-${provider}`;
|
||||
const lastNotified = this.lastNotifiedThreshold.get(userKey) || 0;
|
||||
|
||||
// Notify at 80% usage (20% remaining)
|
||||
if (usedPercentage >= 80 && usedPercentage < 100 && lastNotified < 80) {
|
||||
this.lastNotifiedThreshold.set(userKey, 80);
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "warning",
|
||||
provider,
|
||||
status: info.status,
|
||||
remaining: info.remaining,
|
||||
limit: info.limit,
|
||||
usedPercentage: Math.round(usedPercentage),
|
||||
message: `GitHub API rate limit at ${Math.round(usedPercentage)}%. ${info.remaining} requests remaining.`,
|
||||
},
|
||||
});
|
||||
console.log(`[RateLimit] 80% threshold reached for user ${userId}: ${info.remaining}/${info.limit} requests remaining`);
|
||||
}
|
||||
|
||||
// Notify at 100% usage (0 remaining)
|
||||
if (info.remaining === 0 && lastNotified < 100) {
|
||||
this.lastNotifiedThreshold.set(userKey, 100);
|
||||
const resetTime = new Date(info.reset);
|
||||
const minutesUntilReset = Math.ceil((resetTime.getTime() - Date.now()) / 60000);
|
||||
await publishEvent({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
payload: {
|
||||
type: "exceeded",
|
||||
provider,
|
||||
status: "exceeded",
|
||||
remaining: 0,
|
||||
limit: info.limit,
|
||||
usedPercentage: 100,
|
||||
reset: info.reset,
|
||||
message: `GitHub API rate limit exceeded. Will automatically resume in ${minutesUntilReset} minutes.`,
|
||||
},
|
||||
});
|
||||
console.log(`[RateLimit] 100% rate limit exceeded for user ${userId}. Resets at ${resetTime.toLocaleTimeString()}`);
|
||||
}
|
||||
|
||||
// Reset notification threshold when rate limit resets
|
||||
if (info.remaining > info.limit * 0.5 && lastNotified > 0) {
|
||||
this.lastNotifiedThreshold.delete(userKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get last known rate limit status from database
|
||||
*/
|
||||
private static async getLastKnownStatus(
|
||||
userId: string,
|
||||
provider: "github" | "gitea"
|
||||
): Promise<RateLimitInfo> {
|
||||
const [result] = await db
|
||||
.select()
|
||||
.from(rateLimits)
|
||||
.where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, provider)))
|
||||
.limit(1);
|
||||
|
||||
if (result) {
|
||||
return {
|
||||
limit: result.limit,
|
||||
remaining: result.remaining,
|
||||
used: result.used,
|
||||
reset: result.reset,
|
||||
retryAfter: result.retryAfter ?? undefined,
|
||||
status: result.status as RateLimitStatus,
|
||||
};
|
||||
}
|
||||
|
||||
// Return default if no data
|
||||
return {
|
||||
limit: 5000,
|
||||
remaining: 5000,
|
||||
used: 0,
|
||||
reset: new Date(Date.now() + 3600000), // 1 hour from now
|
||||
status: "ok",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable status message
|
||||
*/
|
||||
private static getStatusMessage(info: RateLimitInfo): string {
|
||||
const percentage = Math.round((info.remaining / info.limit) * 100);
|
||||
|
||||
switch (info.status) {
|
||||
case "exceeded":
|
||||
return `API rate limit exceeded. Resets at ${info.reset.toLocaleTimeString()}.`;
|
||||
case "limited":
|
||||
return `API rate limit critical: Only ${info.remaining} requests remaining (${percentage}%). Pausing operations...`;
|
||||
case "warning":
|
||||
return `API rate limit warning: ${info.remaining} requests remaining (${percentage}%).`;
|
||||
default:
|
||||
return `API rate limit healthy: ${info.remaining}/${info.limit} requests remaining.`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart retry with exponential backoff for rate-limited requests
|
||||
*/
|
||||
static async retryWithBackoff<T>(
|
||||
fn: () => Promise<T>,
|
||||
userId: string,
|
||||
maxRetries: number = 3
|
||||
): Promise<T> {
|
||||
let lastError: any;
|
||||
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
try {
|
||||
// Check if we should pause before attempting
|
||||
if (await this.shouldPause(userId)) {
|
||||
await this.waitForReset(userId);
|
||||
}
|
||||
|
||||
return await fn();
|
||||
} catch (error: any) {
|
||||
lastError = error;
|
||||
|
||||
// Check if it's a rate limit error
|
||||
if (error.status === 403 && error.message?.includes("rate limit")) {
|
||||
console.log(`[RateLimit] Rate limit hit on attempt ${attempt + 1}/${maxRetries}`);
|
||||
|
||||
// Parse rate limit headers from error response if available
|
||||
if (error.response?.headers) {
|
||||
await this.updateFromResponse(userId, error.response.headers);
|
||||
}
|
||||
|
||||
// Wait for reset
|
||||
await this.waitForReset(userId);
|
||||
} else if (error.status === 429) {
|
||||
// Too Many Requests - use exponential backoff
|
||||
const backoffTime = Math.min(1000 * Math.pow(2, attempt), 30000); // Max 30s
|
||||
console.log(`[RateLimit] Too many requests, backing off ${backoffTime}ms`);
|
||||
await new Promise(resolve => setTimeout(resolve, backoffTime));
|
||||
} else {
|
||||
// Not a rate limit error, throw immediately
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Middleware to check rate limits before making API calls
|
||||
*/
|
||||
export async function withRateLimitCheck<T>(
|
||||
userId: string,
|
||||
operation: () => Promise<T>,
|
||||
operationName: string = "API call"
|
||||
): Promise<T> {
|
||||
// Check if we should pause
|
||||
if (await RateLimitManager.shouldPause(userId)) {
|
||||
console.log(`[RateLimit] Pausing ${operationName} due to rate limit`);
|
||||
await RateLimitManager.waitForReset(userId);
|
||||
}
|
||||
|
||||
// Execute with retry logic
|
||||
return await RateLimitManager.retryWithBackoff(operation, userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to update rate limits from Octokit responses
|
||||
*/
|
||||
export function createOctokitRateLimitPlugin(userId: string) {
|
||||
return {
|
||||
hook: (request: any, options: any) => {
|
||||
return request(options).then((response: any) => {
|
||||
// Update rate limit from response headers
|
||||
if (response.headers) {
|
||||
RateLimitManager.updateFromResponse(userId, response.headers).catch(console.error);
|
||||
}
|
||||
return response;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -260,11 +260,13 @@ async function recoverMirrorJob(job: any, remainingItemIds: string[]) {
|
||||
throw new Error('GitHub token not found in configuration');
|
||||
}
|
||||
|
||||
// Create GitHub client with error handling
|
||||
// Create GitHub client with error handling and rate limit tracking
|
||||
let octokit;
|
||||
try {
|
||||
const decryptedToken = getDecryptedGitHubToken(config);
|
||||
octokit = createGitHubClient(decryptedToken);
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const userId = config.userId || undefined;
|
||||
octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to create GitHub client: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
|
||||
@@ -23,9 +23,10 @@ async function identifyOrphanedRepositories(config: any): Promise<any[]> {
|
||||
const userId = config.userId;
|
||||
|
||||
try {
|
||||
// Get current GitHub repositories
|
||||
// Get current GitHub repositories with rate limit tracking
|
||||
const decryptedToken = getDecryptedGitHubToken(config);
|
||||
const octokit = createGitHubClient(decryptedToken);
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||
|
||||
let allGithubRepos = [];
|
||||
let githubApiAccessible = true;
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
export async function processInParallel<T, R>(
|
||||
items: T[],
|
||||
processItem: (item: T) => Promise<R>,
|
||||
concurrencyLimit: number = 5,
|
||||
concurrencyLimit: number = 5, // Safe default for GitHub API (max 100 concurrent, but 5-10 recommended)
|
||||
onProgress?: (completed: number, total: number, result?: R) => void
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
|
||||
@@ -93,7 +93,7 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
|
||||
enabled: scheduleEnabled,
|
||||
interval: scheduleInterval,
|
||||
concurrent: false,
|
||||
batchSize: 10,
|
||||
batchSize: 5, // Reduced from 10 to be more conservative with GitHub API limits
|
||||
lastRun: null,
|
||||
nextRun: scheduleEnabled ? new Date(Date.now() + scheduleInterval * 1000) : null,
|
||||
},
|
||||
|
||||
69
src/pages/api/events/index.ts
Normal file
69
src/pages/api/events/index.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
import type { APIRoute } from "astro";
|
||||
import { getNewEvents } from "@/lib/events";
|
||||
|
||||
export const GET: APIRoute = async ({ request }) => {
|
||||
const url = new URL(request.url);
|
||||
const userId = url.searchParams.get("userId");
|
||||
|
||||
if (!userId) {
|
||||
return new Response("Missing userId", { status: 400 });
|
||||
}
|
||||
|
||||
// Create a new ReadableStream for SSE
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
const encoder = new TextEncoder();
|
||||
let lastEventTime = new Date();
|
||||
|
||||
// Send initial connection message
|
||||
controller.enqueue(encoder.encode(": connected\n\n"));
|
||||
|
||||
// Poll for new events every 2 seconds
|
||||
const pollInterval = setInterval(async () => {
|
||||
try {
|
||||
// Get new rate limit events
|
||||
const newEvents = await getNewEvents({
|
||||
userId,
|
||||
channel: "rate-limit",
|
||||
lastEventTime,
|
||||
});
|
||||
|
||||
// Send each new event
|
||||
for (const event of newEvents) {
|
||||
const message = `event: rate-limit\ndata: ${JSON.stringify(event.payload)}\n\n`;
|
||||
controller.enqueue(encoder.encode(message));
|
||||
lastEventTime = new Date(event.createdAt);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error polling for events:", error);
|
||||
}
|
||||
}, 2000); // Poll every 2 seconds
|
||||
|
||||
// Send heartbeat every 30 seconds to keep connection alive
|
||||
const heartbeatInterval = setInterval(() => {
|
||||
try {
|
||||
controller.enqueue(encoder.encode(": heartbeat\n\n"));
|
||||
} catch (error) {
|
||||
clearInterval(heartbeatInterval);
|
||||
clearInterval(pollInterval);
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
// Cleanup on close
|
||||
request.signal.addEventListener("abort", () => {
|
||||
clearInterval(pollInterval);
|
||||
clearInterval(heartbeatInterval);
|
||||
controller.close();
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no", // Disable nginx buffering
|
||||
},
|
||||
});
|
||||
};
|
||||
@@ -71,9 +71,10 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
throw new Error("GitHub token is missing in config.");
|
||||
}
|
||||
|
||||
// Create a single Octokit instance to be reused
|
||||
// Create a single Octokit instance to be reused with rate limit tracking
|
||||
const decryptedToken = getDecryptedGitHubToken(config);
|
||||
const octokit = createGitHubClient(decryptedToken);
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||
|
||||
// Define the concurrency limit - adjust based on API rate limits
|
||||
// Using a lower concurrency for organizations since each org might contain many repos
|
||||
|
||||
@@ -73,9 +73,10 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
throw new Error("GitHub token is missing.");
|
||||
}
|
||||
|
||||
// Create a single Octokit instance to be reused
|
||||
// Create a single Octokit instance to be reused with rate limit tracking
|
||||
const decryptedToken = getDecryptedGitHubToken(config);
|
||||
const octokit = createGitHubClient(decryptedToken);
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||
|
||||
// Define the concurrency limit - adjust based on API rate limits
|
||||
const CONCURRENCY_LIMIT = 3;
|
||||
|
||||
@@ -71,12 +71,13 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
|
||||
// Start background retry with parallel processing
|
||||
setTimeout(async () => {
|
||||
// Create a single Octokit instance to be reused if needed
|
||||
// Create a single Octokit instance to be reused if needed with rate limit tracking
|
||||
const decryptedToken = config.githubConfig.token
|
||||
? getDecryptedGitHubToken(config)
|
||||
: null;
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const octokit = decryptedToken
|
||||
? createGitHubClient(decryptedToken)
|
||||
? createGitHubClient(decryptedToken, userId, githubUsername)
|
||||
: null;
|
||||
|
||||
// Define the concurrency limit - adjust based on API rate limits
|
||||
|
||||
104
src/pages/api/rate-limit/index.ts
Normal file
104
src/pages/api/rate-limit/index.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import type { APIRoute } from "astro";
|
||||
import { db, rateLimits } from "@/lib/db";
|
||||
import { eq, and, desc } from "drizzle-orm";
|
||||
import { jsonResponse, createSecureErrorResponse } from "@/lib/utils";
|
||||
import { RateLimitManager } from "@/lib/rate-limit-manager";
|
||||
import { createGitHubClient } from "@/lib/github";
|
||||
import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption";
|
||||
import { configs } from "@/lib/db";
|
||||
|
||||
export const GET: APIRoute = async ({ request }) => {
|
||||
const url = new URL(request.url);
|
||||
const userId = url.searchParams.get("userId");
|
||||
const refresh = url.searchParams.get("refresh") === "true";
|
||||
|
||||
if (!userId) {
|
||||
return jsonResponse({
|
||||
data: { error: "Missing userId" },
|
||||
status: 400,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// If refresh is requested, fetch current rate limit from GitHub
|
||||
if (refresh) {
|
||||
const [config] = await db
|
||||
.select()
|
||||
.from(configs)
|
||||
.where(eq(configs.userId, userId))
|
||||
.limit(1);
|
||||
|
||||
if (config && config.githubConfig?.token) {
|
||||
const decryptedToken = getDecryptedGitHubToken(config);
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||
|
||||
// This will update the rate limit in the database
|
||||
await RateLimitManager.checkGitHubRateLimit(octokit, userId);
|
||||
}
|
||||
}
|
||||
|
||||
// Get rate limit status from database
|
||||
const [rateLimit] = await db
|
||||
.select()
|
||||
.from(rateLimits)
|
||||
.where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, "github")))
|
||||
.orderBy(desc(rateLimits.updatedAt))
|
||||
.limit(1);
|
||||
|
||||
if (!rateLimit) {
|
||||
return jsonResponse({
|
||||
data: {
|
||||
limit: 5000,
|
||||
remaining: 5000,
|
||||
used: 0,
|
||||
reset: new Date(Date.now() + 3600000), // 1 hour from now
|
||||
status: "ok",
|
||||
lastChecked: new Date(),
|
||||
message: "No rate limit data available yet",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Calculate percentage
|
||||
const percentage = Math.round((rateLimit.remaining / rateLimit.limit) * 100);
|
||||
|
||||
// Calculate time until reset
|
||||
const now = new Date();
|
||||
const resetTime = new Date(rateLimit.reset);
|
||||
const timeUntilReset = Math.max(0, resetTime.getTime() - now.getTime());
|
||||
const minutesUntilReset = Math.ceil(timeUntilReset / 60000);
|
||||
|
||||
let message = "";
|
||||
switch (rateLimit.status) {
|
||||
case "exceeded":
|
||||
message = `Rate limit exceeded. Resets in ${minutesUntilReset} minutes.`;
|
||||
break;
|
||||
case "limited":
|
||||
message = `Rate limit critical: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`;
|
||||
break;
|
||||
case "warning":
|
||||
message = `Rate limit warning: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`;
|
||||
break;
|
||||
default:
|
||||
message = `Rate limit healthy: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`;
|
||||
}
|
||||
|
||||
return jsonResponse({
|
||||
data: {
|
||||
limit: rateLimit.limit,
|
||||
remaining: rateLimit.remaining,
|
||||
used: rateLimit.used,
|
||||
reset: rateLimit.reset,
|
||||
retryAfter: rateLimit.retryAfter,
|
||||
status: rateLimit.status,
|
||||
lastChecked: rateLimit.lastChecked,
|
||||
percentage,
|
||||
minutesUntilReset,
|
||||
message,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return createSecureErrorResponse(error, "rate limit check", 500);
|
||||
}
|
||||
};
|
||||
@@ -43,7 +43,8 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
|
||||
// Decrypt the GitHub token before using it
|
||||
const decryptedToken = getDecryptedGitHubToken(config);
|
||||
const octokit = createGitHubClient(decryptedToken);
|
||||
const githubUsername = config.githubConfig?.owner || undefined;
|
||||
const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
|
||||
|
||||
// Fetch GitHub data in parallel
|
||||
const [basicAndForkedRepos, starredRepos, gitOrgs] = await Promise.all([
|
||||
|
||||
@@ -69,8 +69,9 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
});
|
||||
}
|
||||
|
||||
// Create authenticated Octokit instance
|
||||
const octokit = createGitHubClient(decryptedConfig.githubConfig.token);
|
||||
// Create authenticated Octokit instance with rate limit tracking
|
||||
const githubUsername = decryptedConfig.githubConfig?.owner || undefined;
|
||||
const octokit = createGitHubClient(decryptedConfig.githubConfig.token, userId, githubUsername);
|
||||
|
||||
// Fetch org metadata
|
||||
const { data: orgData } = await octokit.orgs.get({ org });
|
||||
|
||||
Reference in New Issue
Block a user