Added Github API rate limiting

- Implemented comprehensive GitHub API rate limit handling:
    - Integrated @octokit/plugin-throttling for automatic retry with exponential backoff
    - Added RateLimitManager service to track and enforce rate limits
    - Store rate limit status in database for persistence across restarts
    - Automatic pause and resume when limits are exceeded
    - Proper user identification for 5000 req/hr authenticated limit (vs 60 unauthenticated)

  - Improved rate limit UI/UX:
    - Removed intrusive rate limit card from dashboard
    - Toast notifications only at critical thresholds (80% and 100% usage)
    - All rate limit events logged for debugging

  - Optimized for GitHub's API constraints:
    - Reduced default batch size from 10 to 5 repositories
    - Added documentation about GitHub's 100 concurrent request limit
    - Better handling of repositories with many issues/PRs

src/pages/api/events/index.ts

@@ -0,0 +1,69 @@
+import type { APIRoute } from "astro";
+import { getNewEvents } from "@/lib/events";
+
+export const GET: APIRoute = async ({ request }) => {
+  const url = new URL(request.url);
+  const userId = url.searchParams.get("userId");
+
+  if (!userId) {
+    return new Response("Missing userId", { status: 400 });
+  }
+
+  // Create a new ReadableStream for SSE
+  const stream = new ReadableStream({
+    start(controller) {
+      const encoder = new TextEncoder();
+      let lastEventTime = new Date();
+
+      // Send initial connection message
+      controller.enqueue(encoder.encode(": connected\n\n"));
+
+      // Poll for new events every 2 seconds
+      const pollInterval = setInterval(async () => {
+        try {
+          // Get new rate limit events
+          const newEvents = await getNewEvents({
+            userId,
+            channel: "rate-limit",
+            lastEventTime,
+          });
+
+          // Send each new event
+          for (const event of newEvents) {
+            const message = `event: rate-limit\ndata: ${JSON.stringify(event.payload)}\n\n`;
+            controller.enqueue(encoder.encode(message));
+            lastEventTime = new Date(event.createdAt);
+          }
+        } catch (error) {
+          console.error("Error polling for events:", error);
+        }
+      }, 2000); // Poll every 2 seconds
+
+      // Send heartbeat every 30 seconds to keep connection alive
+      const heartbeatInterval = setInterval(() => {
+        try {
+          controller.enqueue(encoder.encode(": heartbeat\n\n"));
+        } catch (error) {
+          clearInterval(heartbeatInterval);
+          clearInterval(pollInterval);
+        }
+      }, 30000);
+
+      // Cleanup on close
+      request.signal.addEventListener("abort", () => {
+        clearInterval(pollInterval);
+        clearInterval(heartbeatInterval);
+        controller.close();
+      });
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      "Content-Type": "text/event-stream",
+      "Cache-Control": "no-cache",
+      "Connection": "keep-alive",
+      "X-Accel-Buffering": "no", // Disable nginx buffering
+    },
+  });
+};

src/pages/api/job/mirror-org.ts

@@ -71,9 +71,10 @@ export const POST: APIRoute = async ({ request }) => {
         throw new Error("GitHub token is missing in config.");
       }
 
-      // Create a single Octokit instance to be reused
+      // Create a single Octokit instance to be reused with rate limit tracking
       const decryptedToken = getDecryptedGitHubToken(config);
-      const octokit = createGitHubClient(decryptedToken);
+      const githubUsername = config.githubConfig?.owner || undefined;
+      const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
 
       // Define the concurrency limit - adjust based on API rate limits
       // Using a lower concurrency for organizations since each org might contain many repos

src/pages/api/job/mirror-repo.ts

@@ -73,9 +73,10 @@ export const POST: APIRoute = async ({ request }) => {
         throw new Error("GitHub token is missing.");
       }
 
-      // Create a single Octokit instance to be reused
+      // Create a single Octokit instance to be reused with rate limit tracking
       const decryptedToken = getDecryptedGitHubToken(config);
-      const octokit = createGitHubClient(decryptedToken);
+      const githubUsername = config.githubConfig?.owner || undefined;
+      const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
 
       // Define the concurrency limit - adjust based on API rate limits
       const CONCURRENCY_LIMIT = 3;

src/pages/api/job/retry-repo.ts

@@ -71,12 +71,13 @@ export const POST: APIRoute = async ({ request }) => {
 
     // Start background retry with parallel processing
     setTimeout(async () => {
-      // Create a single Octokit instance to be reused if needed
+      // Create a single Octokit instance to be reused if needed with rate limit tracking
       const decryptedToken = config.githubConfig.token
         ? getDecryptedGitHubToken(config)
         : null;
+      const githubUsername = config.githubConfig?.owner || undefined;
       const octokit = decryptedToken
-        ? createGitHubClient(decryptedToken)
+        ? createGitHubClient(decryptedToken, userId, githubUsername)
         : null;
 
       // Define the concurrency limit - adjust based on API rate limits

src/pages/api/rate-limit/index.ts

@@ -0,0 +1,104 @@
+import type { APIRoute } from "astro";
+import { db, rateLimits } from "@/lib/db";
+import { eq, and, desc } from "drizzle-orm";
+import { jsonResponse, createSecureErrorResponse } from "@/lib/utils";
+import { RateLimitManager } from "@/lib/rate-limit-manager";
+import { createGitHubClient } from "@/lib/github";
+import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption";
+import { configs } from "@/lib/db";
+
+export const GET: APIRoute = async ({ request }) => {
+  const url = new URL(request.url);
+  const userId = url.searchParams.get("userId");
+  const refresh = url.searchParams.get("refresh") === "true";
+
+  if (!userId) {
+    return jsonResponse({
+      data: { error: "Missing userId" },
+      status: 400,
+    });
+  }
+
+  try {
+    // If refresh is requested, fetch current rate limit from GitHub
+    if (refresh) {
+      const [config] = await db
+        .select()
+        .from(configs)
+        .where(eq(configs.userId, userId))
+        .limit(1);
+
+      if (config && config.githubConfig?.token) {
+        const decryptedToken = getDecryptedGitHubToken(config);
+        const githubUsername = config.githubConfig?.owner || undefined;
+        const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
+        
+        // This will update the rate limit in the database
+        await RateLimitManager.checkGitHubRateLimit(octokit, userId);
+      }
+    }
+
+    // Get rate limit status from database
+    const [rateLimit] = await db
+      .select()
+      .from(rateLimits)
+      .where(and(eq(rateLimits.userId, userId), eq(rateLimits.provider, "github")))
+      .orderBy(desc(rateLimits.updatedAt))
+      .limit(1);
+
+    if (!rateLimit) {
+      return jsonResponse({
+        data: {
+          limit: 5000,
+          remaining: 5000,
+          used: 0,
+          reset: new Date(Date.now() + 3600000), // 1 hour from now
+          status: "ok",
+          lastChecked: new Date(),
+          message: "No rate limit data available yet",
+        },
+      });
+    }
+
+    // Calculate percentage
+    const percentage = Math.round((rateLimit.remaining / rateLimit.limit) * 100);
+    
+    // Calculate time until reset
+    const now = new Date();
+    const resetTime = new Date(rateLimit.reset);
+    const timeUntilReset = Math.max(0, resetTime.getTime() - now.getTime());
+    const minutesUntilReset = Math.ceil(timeUntilReset / 60000);
+
+    let message = "";
+    switch (rateLimit.status) {
+      case "exceeded":
+        message = `Rate limit exceeded. Resets in ${minutesUntilReset} minutes.`;
+        break;
+      case "limited":
+        message = `Rate limit critical: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`;
+        break;
+      case "warning":
+        message = `Rate limit warning: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`;
+        break;
+      default:
+        message = `Rate limit healthy: ${rateLimit.remaining}/${rateLimit.limit} (${percentage}%)`;
+    }
+
+    return jsonResponse({
+      data: {
+        limit: rateLimit.limit,
+        remaining: rateLimit.remaining,
+        used: rateLimit.used,
+        reset: rateLimit.reset,
+        retryAfter: rateLimit.retryAfter,
+        status: rateLimit.status,
+        lastChecked: rateLimit.lastChecked,
+        percentage,
+        minutesUntilReset,
+        message,
+      },
+    });
+  } catch (error) {
+    return createSecureErrorResponse(error, "rate limit check", 500);
+  }
+};

src/pages/api/sync/index.ts

@@ -43,7 +43,8 @@ export const POST: APIRoute = async ({ request }) => {
 
     // Decrypt the GitHub token before using it
     const decryptedToken = getDecryptedGitHubToken(config);
-    const octokit = createGitHubClient(decryptedToken);
+    const githubUsername = config.githubConfig?.owner || undefined;
+    const octokit = createGitHubClient(decryptedToken, userId, githubUsername);
 
     // Fetch GitHub data in parallel
     const [basicAndForkedRepos, starredRepos, gitOrgs] = await Promise.all([

src/pages/api/sync/organization.ts

@@ -69,8 +69,9 @@ export const POST: APIRoute = async ({ request }) => {
       });
     }
     
-    // Create authenticated Octokit instance
-    const octokit = createGitHubClient(decryptedConfig.githubConfig.token);
+    // Create authenticated Octokit instance with rate limit tracking
+    const githubUsername = decryptedConfig.githubConfig?.owner || undefined;
+    const octokit = createGitHubClient(decryptedConfig.githubConfig.token, userId, githubUsername);
 
     // Fetch org metadata
     const { data: orgData } = await octokit.orgs.get({ org });