mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2025-12-07 20:16:46 +03:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
99336e2607 | ||
|
|
cba421d606 |
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "gitea-mirror",
|
"name": "gitea-mirror",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"version": "2.9.1",
|
"version": "2.9.2",
|
||||||
"engines": {
|
"engines": {
|
||||||
"bun": ">=1.2.9"
|
"bun": ">=1.2.9"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ export async function httpRequest<T = any>(
|
|||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
|
let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
|
||||||
let responseText = '';
|
let responseText = '';
|
||||||
|
|
||||||
try {
|
try {
|
||||||
responseText = await responseClone.text();
|
responseText = await responseClone.text();
|
||||||
if (responseText) {
|
if (responseText) {
|
||||||
@@ -70,9 +70,19 @@ export async function httpRequest<T = any>(
|
|||||||
data = await response.json();
|
data = await response.json();
|
||||||
} catch (jsonError) {
|
} catch (jsonError) {
|
||||||
const responseText = await responseClone.text();
|
const responseText = await responseClone.text();
|
||||||
console.error(`Failed to parse JSON response: ${responseText}`);
|
|
||||||
|
// Enhanced JSON parsing error logging
|
||||||
|
console.error("=== JSON PARSING ERROR ===");
|
||||||
|
console.error("URL:", url);
|
||||||
|
console.error("Status:", response.status, response.statusText);
|
||||||
|
console.error("Content-Type:", contentType);
|
||||||
|
console.error("Response length:", responseText.length);
|
||||||
|
console.error("Response preview (first 500 chars):", responseText.substring(0, 500));
|
||||||
|
console.error("JSON Error:", jsonError instanceof Error ? jsonError.message : String(jsonError));
|
||||||
|
console.error("========================");
|
||||||
|
|
||||||
throw new HttpError(
|
throw new HttpError(
|
||||||
`Failed to parse JSON response: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`,
|
`Failed to parse JSON response from ${url}: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}. Response: ${responseText.substring(0, 200)}${responseText.length > 200 ? '...' : ''}`,
|
||||||
response.status,
|
response.status,
|
||||||
response.statusText,
|
response.statusText,
|
||||||
responseText
|
responseText
|
||||||
|
|||||||
@@ -5,19 +5,19 @@ describe("processInParallel", () => {
|
|||||||
test("processes items in parallel with concurrency control", async () => {
|
test("processes items in parallel with concurrency control", async () => {
|
||||||
// Create an array of numbers to process
|
// Create an array of numbers to process
|
||||||
const items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
|
const items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
|
||||||
|
|
||||||
// Create a mock function to track execution
|
// Create a mock function to track execution
|
||||||
const processItem = mock(async (item: number) => {
|
const processItem = mock(async (item: number) => {
|
||||||
// Simulate async work
|
// Simulate async work
|
||||||
await new Promise(resolve => setTimeout(resolve, 10));
|
await new Promise(resolve => setTimeout(resolve, 10));
|
||||||
return item * 2;
|
return item * 2;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a mock progress callback
|
// Create a mock progress callback
|
||||||
const onProgress = mock((completed: number, total: number, result?: number) => {
|
const onProgress = mock((completed: number, total: number, result?: number) => {
|
||||||
// Progress tracking
|
// Progress tracking
|
||||||
});
|
});
|
||||||
|
|
||||||
// Process the items with a concurrency limit of 3
|
// Process the items with a concurrency limit of 3
|
||||||
const results = await processInParallel(
|
const results = await processInParallel(
|
||||||
items,
|
items,
|
||||||
@@ -25,25 +25,25 @@ describe("processInParallel", () => {
|
|||||||
3,
|
3,
|
||||||
onProgress
|
onProgress
|
||||||
);
|
);
|
||||||
|
|
||||||
// Verify results
|
// Verify results
|
||||||
expect(results).toEqual([2, 4, 6, 8, 10, 12, 14, 16, 18, 20]);
|
expect(results).toEqual([2, 4, 6, 8, 10, 12, 14, 16, 18, 20]);
|
||||||
|
|
||||||
// Verify that processItem was called for each item
|
// Verify that processItem was called for each item
|
||||||
expect(processItem).toHaveBeenCalledTimes(10);
|
expect(processItem).toHaveBeenCalledTimes(10);
|
||||||
|
|
||||||
// Verify that onProgress was called for each item
|
// Verify that onProgress was called for each item
|
||||||
expect(onProgress).toHaveBeenCalledTimes(10);
|
expect(onProgress).toHaveBeenCalledTimes(10);
|
||||||
|
|
||||||
// Verify the last call to onProgress had the correct completed/total values
|
// Verify the last call to onProgress had the correct completed/total values
|
||||||
expect(onProgress.mock.calls[9][0]).toBe(10); // completed
|
expect(onProgress.mock.calls[9][0]).toBe(10); // completed
|
||||||
expect(onProgress.mock.calls[9][1]).toBe(10); // total
|
expect(onProgress.mock.calls[9][1]).toBe(10); // total
|
||||||
});
|
});
|
||||||
|
|
||||||
test("handles errors in processing", async () => {
|
test("handles errors in processing", async () => {
|
||||||
// Create an array of numbers to process
|
// Create an array of numbers to process
|
||||||
const items = [1, 2, 3, 4, 5];
|
const items = [1, 2, 3, 4, 5];
|
||||||
|
|
||||||
// Create a mock function that throws an error for item 3
|
// Create a mock function that throws an error for item 3
|
||||||
const processItem = mock(async (item: number) => {
|
const processItem = mock(async (item: number) => {
|
||||||
if (item === 3) {
|
if (item === 3) {
|
||||||
@@ -51,24 +51,24 @@ describe("processInParallel", () => {
|
|||||||
}
|
}
|
||||||
return item * 2;
|
return item * 2;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a spy for console.error
|
// Create a spy for console.error
|
||||||
const originalConsoleError = console.error;
|
const originalConsoleError = console.error;
|
||||||
const consoleErrorMock = mock(() => {});
|
const consoleErrorMock = mock(() => {});
|
||||||
console.error = consoleErrorMock;
|
console.error = consoleErrorMock;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Process the items
|
// Process the items
|
||||||
const results = await processInParallel(items, processItem);
|
const results = await processInParallel(items, processItem);
|
||||||
|
|
||||||
// Verify results (should have 4 items, missing the one that errored)
|
// Verify results (should have 4 items, missing the one that errored)
|
||||||
expect(results).toEqual([2, 4, 8, 10]);
|
expect(results).toEqual([2, 4, 8, 10]);
|
||||||
|
|
||||||
// Verify that processItem was called for each item
|
// Verify that processItem was called for each item
|
||||||
expect(processItem).toHaveBeenCalledTimes(5);
|
expect(processItem).toHaveBeenCalledTimes(5);
|
||||||
|
|
||||||
// Verify that console.error was called once
|
// Verify that console.error was called (enhanced logging calls it multiple times)
|
||||||
expect(consoleErrorMock).toHaveBeenCalledTimes(1);
|
expect(consoleErrorMock).toHaveBeenCalled();
|
||||||
} finally {
|
} finally {
|
||||||
// Restore console.error
|
// Restore console.error
|
||||||
console.error = originalConsoleError;
|
console.error = originalConsoleError;
|
||||||
@@ -80,51 +80,51 @@ describe("processWithRetry", () => {
|
|||||||
test("retries failed operations", async () => {
|
test("retries failed operations", async () => {
|
||||||
// Create an array of numbers to process
|
// Create an array of numbers to process
|
||||||
const items = [1, 2, 3];
|
const items = [1, 2, 3];
|
||||||
|
|
||||||
// Create a counter to track retry attempts
|
// Create a counter to track retry attempts
|
||||||
const attemptCounts: Record<number, number> = { 1: 0, 2: 0, 3: 0 };
|
const attemptCounts: Record<number, number> = { 1: 0, 2: 0, 3: 0 };
|
||||||
|
|
||||||
// Create a mock function that fails on first attempt for item 2
|
// Create a mock function that fails on first attempt for item 2
|
||||||
const processItem = mock(async (item: number) => {
|
const processItem = mock(async (item: number) => {
|
||||||
attemptCounts[item]++;
|
attemptCounts[item]++;
|
||||||
|
|
||||||
if (item === 2 && attemptCounts[item] === 1) {
|
if (item === 2 && attemptCounts[item] === 1) {
|
||||||
throw new Error("Temporary error");
|
throw new Error("Temporary error");
|
||||||
}
|
}
|
||||||
|
|
||||||
return item * 2;
|
return item * 2;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a mock for the onRetry callback
|
// Create a mock for the onRetry callback
|
||||||
const onRetry = mock((item: number, error: Error, attempt: number) => {
|
const onRetry = mock((item: number, error: Error, attempt: number) => {
|
||||||
// Retry tracking
|
// Retry tracking
|
||||||
});
|
});
|
||||||
|
|
||||||
// Process the items with retry
|
// Process the items with retry
|
||||||
const results = await processWithRetry(items, processItem, {
|
const results = await processWithRetry(items, processItem, {
|
||||||
maxRetries: 2,
|
maxRetries: 2,
|
||||||
retryDelay: 10,
|
retryDelay: 10,
|
||||||
onRetry,
|
onRetry,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Verify results
|
// Verify results
|
||||||
expect(results).toEqual([2, 4, 6]);
|
expect(results).toEqual([2, 4, 6]);
|
||||||
|
|
||||||
// Verify that item 2 was retried once
|
// Verify that item 2 was retried once
|
||||||
expect(attemptCounts[1]).toBe(1); // No retries
|
expect(attemptCounts[1]).toBe(1); // No retries
|
||||||
expect(attemptCounts[2]).toBe(2); // One retry
|
expect(attemptCounts[2]).toBe(2); // One retry
|
||||||
expect(attemptCounts[3]).toBe(1); // No retries
|
expect(attemptCounts[3]).toBe(1); // No retries
|
||||||
|
|
||||||
// Verify that onRetry was called once
|
// Verify that onRetry was called once
|
||||||
expect(onRetry).toHaveBeenCalledTimes(1);
|
expect(onRetry).toHaveBeenCalledTimes(1);
|
||||||
expect(onRetry.mock.calls[0][0]).toBe(2); // item
|
expect(onRetry.mock.calls[0][0]).toBe(2); // item
|
||||||
expect(onRetry.mock.calls[0][2]).toBe(1); // attempt
|
expect(onRetry.mock.calls[0][2]).toBe(1); // attempt
|
||||||
});
|
});
|
||||||
|
|
||||||
test("gives up after max retries", async () => {
|
test("gives up after max retries", async () => {
|
||||||
// Create an array of numbers to process
|
// Create an array of numbers to process
|
||||||
const items = [1, 2];
|
const items = [1, 2];
|
||||||
|
|
||||||
// Create a mock function that always fails for item 2
|
// Create a mock function that always fails for item 2
|
||||||
const processItem = mock(async (item: number) => {
|
const processItem = mock(async (item: number) => {
|
||||||
if (item === 2) {
|
if (item === 2) {
|
||||||
@@ -132,17 +132,17 @@ describe("processWithRetry", () => {
|
|||||||
}
|
}
|
||||||
return item * 2;
|
return item * 2;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a mock for the onRetry callback
|
// Create a mock for the onRetry callback
|
||||||
const onRetry = mock((item: number, error: Error, attempt: number) => {
|
const onRetry = mock((item: number, error: Error, attempt: number) => {
|
||||||
// Retry tracking
|
// Retry tracking
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a spy for console.error
|
// Create a spy for console.error
|
||||||
const originalConsoleError = console.error;
|
const originalConsoleError = console.error;
|
||||||
const consoleErrorMock = mock(() => {});
|
const consoleErrorMock = mock(() => {});
|
||||||
console.error = consoleErrorMock;
|
console.error = consoleErrorMock;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Process the items with retry
|
// Process the items with retry
|
||||||
const results = await processWithRetry(items, processItem, {
|
const results = await processWithRetry(items, processItem, {
|
||||||
@@ -150,15 +150,15 @@ describe("processWithRetry", () => {
|
|||||||
retryDelay: 10,
|
retryDelay: 10,
|
||||||
onRetry,
|
onRetry,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Verify results (should have 1 item, missing the one that errored)
|
// Verify results (should have 1 item, missing the one that errored)
|
||||||
expect(results).toEqual([2]);
|
expect(results).toEqual([2]);
|
||||||
|
|
||||||
// Verify that onRetry was called twice (for 2 retry attempts)
|
// Verify that onRetry was called twice (for 2 retry attempts)
|
||||||
expect(onRetry).toHaveBeenCalledTimes(2);
|
expect(onRetry).toHaveBeenCalledTimes(2);
|
||||||
|
|
||||||
// Verify that console.error was called once
|
// Verify that console.error was called (enhanced logging calls it multiple times)
|
||||||
expect(consoleErrorMock).toHaveBeenCalledTimes(1);
|
expect(consoleErrorMock).toHaveBeenCalled();
|
||||||
} finally {
|
} finally {
|
||||||
// Restore console.error
|
// Restore console.error
|
||||||
console.error = originalConsoleError;
|
console.error = originalConsoleError;
|
||||||
|
|||||||
@@ -46,11 +46,25 @@ export async function processInParallel<T, R>(
|
|||||||
const batchResults = await Promise.allSettled(batchPromises);
|
const batchResults = await Promise.allSettled(batchPromises);
|
||||||
|
|
||||||
// Process results and handle errors
|
// Process results and handle errors
|
||||||
for (const result of batchResults) {
|
for (let j = 0; j < batchResults.length; j++) {
|
||||||
|
const result = batchResults[j];
|
||||||
if (result.status === 'fulfilled') {
|
if (result.status === 'fulfilled') {
|
||||||
results.push(result.value);
|
results.push(result.value);
|
||||||
} else {
|
} else {
|
||||||
console.error('Error processing item:', result.reason);
|
const itemIndex = i + j;
|
||||||
|
console.error("=== BATCH ITEM PROCESSING ERROR ===");
|
||||||
|
console.error("Batch index:", Math.floor(i / concurrencyLimit));
|
||||||
|
console.error("Item index in batch:", j);
|
||||||
|
console.error("Global item index:", itemIndex);
|
||||||
|
console.error("Error type:", result.reason?.constructor?.name);
|
||||||
|
console.error("Error message:", result.reason instanceof Error ? result.reason.message : String(result.reason));
|
||||||
|
|
||||||
|
if (result.reason instanceof Error && result.reason.message.includes('JSON')) {
|
||||||
|
console.error("🚨 JSON parsing error in batch processing");
|
||||||
|
console.error("This indicates an API response issue from Gitea");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error("==================================");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -139,6 +153,21 @@ export async function processWithRetry<T, R>(
|
|||||||
const delay = retryDelay * Math.pow(2, attempt - 1);
|
const delay = retryDelay * Math.pow(2, attempt - 1);
|
||||||
await new Promise(resolve => setTimeout(resolve, delay));
|
await new Promise(resolve => setTimeout(resolve, delay));
|
||||||
} else {
|
} else {
|
||||||
|
// Enhanced error logging for final failure
|
||||||
|
console.error("=== ITEM PROCESSING FAILED (MAX RETRIES EXCEEDED) ===");
|
||||||
|
console.error("Item:", getItemId ? getItemId(item) : 'unknown');
|
||||||
|
console.error("Error type:", lastError.constructor.name);
|
||||||
|
console.error("Error message:", lastError.message);
|
||||||
|
console.error("Attempts made:", maxRetries + 1);
|
||||||
|
|
||||||
|
if (lastError.message.includes('JSON')) {
|
||||||
|
console.error("🚨 JSON-related error detected in item processing");
|
||||||
|
console.error("This suggests an issue with API responses from Gitea");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error("Stack trace:", lastError.stack);
|
||||||
|
console.error("================================================");
|
||||||
|
|
||||||
throw lastError;
|
throw lastError;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,11 +165,43 @@ export const POST: APIRoute = async ({ request }) => {
|
|||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error mirroring repositories:", error);
|
// Enhanced error logging for better debugging
|
||||||
|
console.error("=== ERROR MIRRORING REPOSITORIES ===");
|
||||||
|
console.error("Error type:", error?.constructor?.name);
|
||||||
|
console.error("Error message:", error instanceof Error ? error.message : String(error));
|
||||||
|
|
||||||
|
if (error instanceof Error) {
|
||||||
|
console.error("Error stack:", error.stack);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log additional context
|
||||||
|
console.error("Request details:");
|
||||||
|
console.error("- URL:", request.url);
|
||||||
|
console.error("- Method:", request.method);
|
||||||
|
console.error("- Headers:", Object.fromEntries(request.headers.entries()));
|
||||||
|
|
||||||
|
// If it's a JSON parsing error, provide more context
|
||||||
|
if (error instanceof SyntaxError && error.message.includes('JSON')) {
|
||||||
|
console.error("🚨 JSON PARSING ERROR DETECTED:");
|
||||||
|
console.error("This suggests the response from Gitea API is not valid JSON");
|
||||||
|
console.error("Common causes:");
|
||||||
|
console.error("- Gitea server returned HTML error page instead of JSON");
|
||||||
|
console.error("- Network connection interrupted");
|
||||||
|
console.error("- Gitea server is down or misconfigured");
|
||||||
|
console.error("- Authentication token is invalid");
|
||||||
|
console.error("Check your Gitea server logs and configuration");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error("=====================================");
|
||||||
|
|
||||||
return new Response(
|
return new Response(
|
||||||
JSON.stringify({
|
JSON.stringify({
|
||||||
error:
|
error: error instanceof Error ? error.message : "An unknown error occurred",
|
||||||
error instanceof Error ? error.message : "An unknown error occurred",
|
errorType: error?.constructor?.name || "Unknown",
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
troubleshooting: error instanceof SyntaxError && error.message.includes('JSON')
|
||||||
|
? "JSON parsing error detected. Check Gitea server status and logs. Ensure Gitea is returning valid JSON responses."
|
||||||
|
: "Check application logs for more details"
|
||||||
}),
|
}),
|
||||||
{ status: 500, headers: { "Content-Type": "application/json" } }
|
{ status: 500, headers: { "Content-Type": "application/json" } }
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user