Compare commits

...

2 Commits

Author SHA1 Message Date
Arunavo Ray
99336e2607 chore: bump version to 2.9.2 2025-05-28 10:15:43 +05:30
Arunavo Ray
cba421d606 feat: enhance error logging for better debugging of JSON parsing issues
- Add comprehensive error logging in mirror-repo API endpoint
- Enhance HTTP client error handling with detailed response information
- Improve concurrency utility error reporting with context
- Add specific detection and guidance for JSON parsing errors
- Include troubleshooting information in error responses
- Update tests to accommodate enhanced logging

This will help users diagnose issues like 'JSON Parse error: Unexpected EOF'
by providing detailed information about what responses are being received
from the Gitea API and what might be causing the failures.
2025-05-28 10:13:41 +05:30
5 changed files with 115 additions and 44 deletions

View File

@@ -1,7 +1,7 @@
{ {
"name": "gitea-mirror", "name": "gitea-mirror",
"type": "module", "type": "module",
"version": "2.9.1", "version": "2.9.2",
"engines": { "engines": {
"bun": ">=1.2.9" "bun": ">=1.2.9"
}, },

View File

@@ -43,7 +43,7 @@ export async function httpRequest<T = any>(
if (!response.ok) { if (!response.ok) {
let errorMessage = `HTTP ${response.status}: ${response.statusText}`; let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
let responseText = ''; let responseText = '';
try { try {
responseText = await responseClone.text(); responseText = await responseClone.text();
if (responseText) { if (responseText) {
@@ -70,9 +70,19 @@ export async function httpRequest<T = any>(
data = await response.json(); data = await response.json();
} catch (jsonError) { } catch (jsonError) {
const responseText = await responseClone.text(); const responseText = await responseClone.text();
console.error(`Failed to parse JSON response: ${responseText}`);
// Enhanced JSON parsing error logging
console.error("=== JSON PARSING ERROR ===");
console.error("URL:", url);
console.error("Status:", response.status, response.statusText);
console.error("Content-Type:", contentType);
console.error("Response length:", responseText.length);
console.error("Response preview (first 500 chars):", responseText.substring(0, 500));
console.error("JSON Error:", jsonError instanceof Error ? jsonError.message : String(jsonError));
console.error("========================");
throw new HttpError( throw new HttpError(
`Failed to parse JSON response: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`, `Failed to parse JSON response from ${url}: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}. Response: ${responseText.substring(0, 200)}${responseText.length > 200 ? '...' : ''}`,
response.status, response.status,
response.statusText, response.statusText,
responseText responseText

View File

@@ -5,19 +5,19 @@ describe("processInParallel", () => {
test("processes items in parallel with concurrency control", async () => { test("processes items in parallel with concurrency control", async () => {
// Create an array of numbers to process // Create an array of numbers to process
const items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; const items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
// Create a mock function to track execution // Create a mock function to track execution
const processItem = mock(async (item: number) => { const processItem = mock(async (item: number) => {
// Simulate async work // Simulate async work
await new Promise(resolve => setTimeout(resolve, 10)); await new Promise(resolve => setTimeout(resolve, 10));
return item * 2; return item * 2;
}); });
// Create a mock progress callback // Create a mock progress callback
const onProgress = mock((completed: number, total: number, result?: number) => { const onProgress = mock((completed: number, total: number, result?: number) => {
// Progress tracking // Progress tracking
}); });
// Process the items with a concurrency limit of 3 // Process the items with a concurrency limit of 3
const results = await processInParallel( const results = await processInParallel(
items, items,
@@ -25,25 +25,25 @@ describe("processInParallel", () => {
3, 3,
onProgress onProgress
); );
// Verify results // Verify results
expect(results).toEqual([2, 4, 6, 8, 10, 12, 14, 16, 18, 20]); expect(results).toEqual([2, 4, 6, 8, 10, 12, 14, 16, 18, 20]);
// Verify that processItem was called for each item // Verify that processItem was called for each item
expect(processItem).toHaveBeenCalledTimes(10); expect(processItem).toHaveBeenCalledTimes(10);
// Verify that onProgress was called for each item // Verify that onProgress was called for each item
expect(onProgress).toHaveBeenCalledTimes(10); expect(onProgress).toHaveBeenCalledTimes(10);
// Verify the last call to onProgress had the correct completed/total values // Verify the last call to onProgress had the correct completed/total values
expect(onProgress.mock.calls[9][0]).toBe(10); // completed expect(onProgress.mock.calls[9][0]).toBe(10); // completed
expect(onProgress.mock.calls[9][1]).toBe(10); // total expect(onProgress.mock.calls[9][1]).toBe(10); // total
}); });
test("handles errors in processing", async () => { test("handles errors in processing", async () => {
// Create an array of numbers to process // Create an array of numbers to process
const items = [1, 2, 3, 4, 5]; const items = [1, 2, 3, 4, 5];
// Create a mock function that throws an error for item 3 // Create a mock function that throws an error for item 3
const processItem = mock(async (item: number) => { const processItem = mock(async (item: number) => {
if (item === 3) { if (item === 3) {
@@ -51,24 +51,24 @@ describe("processInParallel", () => {
} }
return item * 2; return item * 2;
}); });
// Create a spy for console.error // Create a spy for console.error
const originalConsoleError = console.error; const originalConsoleError = console.error;
const consoleErrorMock = mock(() => {}); const consoleErrorMock = mock(() => {});
console.error = consoleErrorMock; console.error = consoleErrorMock;
try { try {
// Process the items // Process the items
const results = await processInParallel(items, processItem); const results = await processInParallel(items, processItem);
// Verify results (should have 4 items, missing the one that errored) // Verify results (should have 4 items, missing the one that errored)
expect(results).toEqual([2, 4, 8, 10]); expect(results).toEqual([2, 4, 8, 10]);
// Verify that processItem was called for each item // Verify that processItem was called for each item
expect(processItem).toHaveBeenCalledTimes(5); expect(processItem).toHaveBeenCalledTimes(5);
// Verify that console.error was called once // Verify that console.error was called (enhanced logging calls it multiple times)
expect(consoleErrorMock).toHaveBeenCalledTimes(1); expect(consoleErrorMock).toHaveBeenCalled();
} finally { } finally {
// Restore console.error // Restore console.error
console.error = originalConsoleError; console.error = originalConsoleError;
@@ -80,51 +80,51 @@ describe("processWithRetry", () => {
test("retries failed operations", async () => { test("retries failed operations", async () => {
// Create an array of numbers to process // Create an array of numbers to process
const items = [1, 2, 3]; const items = [1, 2, 3];
// Create a counter to track retry attempts // Create a counter to track retry attempts
const attemptCounts: Record<number, number> = { 1: 0, 2: 0, 3: 0 }; const attemptCounts: Record<number, number> = { 1: 0, 2: 0, 3: 0 };
// Create a mock function that fails on first attempt for item 2 // Create a mock function that fails on first attempt for item 2
const processItem = mock(async (item: number) => { const processItem = mock(async (item: number) => {
attemptCounts[item]++; attemptCounts[item]++;
if (item === 2 && attemptCounts[item] === 1) { if (item === 2 && attemptCounts[item] === 1) {
throw new Error("Temporary error"); throw new Error("Temporary error");
} }
return item * 2; return item * 2;
}); });
// Create a mock for the onRetry callback // Create a mock for the onRetry callback
const onRetry = mock((item: number, error: Error, attempt: number) => { const onRetry = mock((item: number, error: Error, attempt: number) => {
// Retry tracking // Retry tracking
}); });
// Process the items with retry // Process the items with retry
const results = await processWithRetry(items, processItem, { const results = await processWithRetry(items, processItem, {
maxRetries: 2, maxRetries: 2,
retryDelay: 10, retryDelay: 10,
onRetry, onRetry,
}); });
// Verify results // Verify results
expect(results).toEqual([2, 4, 6]); expect(results).toEqual([2, 4, 6]);
// Verify that item 2 was retried once // Verify that item 2 was retried once
expect(attemptCounts[1]).toBe(1); // No retries expect(attemptCounts[1]).toBe(1); // No retries
expect(attemptCounts[2]).toBe(2); // One retry expect(attemptCounts[2]).toBe(2); // One retry
expect(attemptCounts[3]).toBe(1); // No retries expect(attemptCounts[3]).toBe(1); // No retries
// Verify that onRetry was called once // Verify that onRetry was called once
expect(onRetry).toHaveBeenCalledTimes(1); expect(onRetry).toHaveBeenCalledTimes(1);
expect(onRetry.mock.calls[0][0]).toBe(2); // item expect(onRetry.mock.calls[0][0]).toBe(2); // item
expect(onRetry.mock.calls[0][2]).toBe(1); // attempt expect(onRetry.mock.calls[0][2]).toBe(1); // attempt
}); });
test("gives up after max retries", async () => { test("gives up after max retries", async () => {
// Create an array of numbers to process // Create an array of numbers to process
const items = [1, 2]; const items = [1, 2];
// Create a mock function that always fails for item 2 // Create a mock function that always fails for item 2
const processItem = mock(async (item: number) => { const processItem = mock(async (item: number) => {
if (item === 2) { if (item === 2) {
@@ -132,17 +132,17 @@ describe("processWithRetry", () => {
} }
return item * 2; return item * 2;
}); });
// Create a mock for the onRetry callback // Create a mock for the onRetry callback
const onRetry = mock((item: number, error: Error, attempt: number) => { const onRetry = mock((item: number, error: Error, attempt: number) => {
// Retry tracking // Retry tracking
}); });
// Create a spy for console.error // Create a spy for console.error
const originalConsoleError = console.error; const originalConsoleError = console.error;
const consoleErrorMock = mock(() => {}); const consoleErrorMock = mock(() => {});
console.error = consoleErrorMock; console.error = consoleErrorMock;
try { try {
// Process the items with retry // Process the items with retry
const results = await processWithRetry(items, processItem, { const results = await processWithRetry(items, processItem, {
@@ -150,15 +150,15 @@ describe("processWithRetry", () => {
retryDelay: 10, retryDelay: 10,
onRetry, onRetry,
}); });
// Verify results (should have 1 item, missing the one that errored) // Verify results (should have 1 item, missing the one that errored)
expect(results).toEqual([2]); expect(results).toEqual([2]);
// Verify that onRetry was called twice (for 2 retry attempts) // Verify that onRetry was called twice (for 2 retry attempts)
expect(onRetry).toHaveBeenCalledTimes(2); expect(onRetry).toHaveBeenCalledTimes(2);
// Verify that console.error was called once // Verify that console.error was called (enhanced logging calls it multiple times)
expect(consoleErrorMock).toHaveBeenCalledTimes(1); expect(consoleErrorMock).toHaveBeenCalled();
} finally { } finally {
// Restore console.error // Restore console.error
console.error = originalConsoleError; console.error = originalConsoleError;

View File

@@ -46,11 +46,25 @@ export async function processInParallel<T, R>(
const batchResults = await Promise.allSettled(batchPromises); const batchResults = await Promise.allSettled(batchPromises);
// Process results and handle errors // Process results and handle errors
for (const result of batchResults) { for (let j = 0; j < batchResults.length; j++) {
const result = batchResults[j];
if (result.status === 'fulfilled') { if (result.status === 'fulfilled') {
results.push(result.value); results.push(result.value);
} else { } else {
console.error('Error processing item:', result.reason); const itemIndex = i + j;
console.error("=== BATCH ITEM PROCESSING ERROR ===");
console.error("Batch index:", Math.floor(i / concurrencyLimit));
console.error("Item index in batch:", j);
console.error("Global item index:", itemIndex);
console.error("Error type:", result.reason?.constructor?.name);
console.error("Error message:", result.reason instanceof Error ? result.reason.message : String(result.reason));
if (result.reason instanceof Error && result.reason.message.includes('JSON')) {
console.error("🚨 JSON parsing error in batch processing");
console.error("This indicates an API response issue from Gitea");
}
console.error("==================================");
} }
} }
} }
@@ -139,6 +153,21 @@ export async function processWithRetry<T, R>(
const delay = retryDelay * Math.pow(2, attempt - 1); const delay = retryDelay * Math.pow(2, attempt - 1);
await new Promise(resolve => setTimeout(resolve, delay)); await new Promise(resolve => setTimeout(resolve, delay));
} else { } else {
// Enhanced error logging for final failure
console.error("=== ITEM PROCESSING FAILED (MAX RETRIES EXCEEDED) ===");
console.error("Item:", getItemId ? getItemId(item) : 'unknown');
console.error("Error type:", lastError.constructor.name);
console.error("Error message:", lastError.message);
console.error("Attempts made:", maxRetries + 1);
if (lastError.message.includes('JSON')) {
console.error("🚨 JSON-related error detected in item processing");
console.error("This suggests an issue with API responses from Gitea");
}
console.error("Stack trace:", lastError.stack);
console.error("================================================");
throw lastError; throw lastError;
} }
} }

View File

@@ -165,11 +165,43 @@ export const POST: APIRoute = async ({ request }) => {
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
}); });
} catch (error) { } catch (error) {
console.error("Error mirroring repositories:", error); // Enhanced error logging for better debugging
console.error("=== ERROR MIRRORING REPOSITORIES ===");
console.error("Error type:", error?.constructor?.name);
console.error("Error message:", error instanceof Error ? error.message : String(error));
if (error instanceof Error) {
console.error("Error stack:", error.stack);
}
// Log additional context
console.error("Request details:");
console.error("- URL:", request.url);
console.error("- Method:", request.method);
console.error("- Headers:", Object.fromEntries(request.headers.entries()));
// If it's a JSON parsing error, provide more context
if (error instanceof SyntaxError && error.message.includes('JSON')) {
console.error("🚨 JSON PARSING ERROR DETECTED:");
console.error("This suggests the response from Gitea API is not valid JSON");
console.error("Common causes:");
console.error("- Gitea server returned HTML error page instead of JSON");
console.error("- Network connection interrupted");
console.error("- Gitea server is down or misconfigured");
console.error("- Authentication token is invalid");
console.error("Check your Gitea server logs and configuration");
}
console.error("=====================================");
return new Response( return new Response(
JSON.stringify({ JSON.stringify({
error: error: error instanceof Error ? error.message : "An unknown error occurred",
error instanceof Error ? error.message : "An unknown error occurred", errorType: error?.constructor?.name || "Unknown",
timestamp: new Date().toISOString(),
troubleshooting: error instanceof SyntaxError && error.message.includes('JSON')
? "JSON parsing error detected. Check Gitea server status and logs. Ensure Gitea is returning valid JSON responses."
: "Check application logs for more details"
}), }),
{ status: 500, headers: { "Content-Type": "application/json" } } { status: 500, headers: { "Content-Type": "application/json" } }
); );