Files
gitea-mirror/src/middleware.ts
Arunavo Ray daf4ab6a93 feat: Implement graceful shutdown and enhanced job recovery
- Added shutdown handler in docker-entrypoint.sh to manage application termination signals.
- Introduced shutdown manager to track active jobs and ensure state persistence during shutdown.
- Enhanced cleanup service to support stopping and status retrieval.
- Integrated signal handlers for proper response to termination signals (SIGTERM, SIGINT, SIGHUP).
- Updated middleware to initialize shutdown manager and cleanup service.
- Created integration tests for graceful shutdown functionality, verifying job state preservation and recovery.
- Documented graceful shutdown process and configuration in GRACEFUL_SHUTDOWN.md and SHUTDOWN_PROCESS.md.
- Added new scripts for testing shutdown behavior and cleanup.
2025-05-24 23:06:28 +05:30

97 lines
3.5 KiB
TypeScript

import { defineMiddleware } from 'astro:middleware';
import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from './lib/recovery';
import { startCleanupService, stopCleanupService } from './lib/cleanup-service';
import { initializeShutdownManager, registerShutdownCallback } from './lib/shutdown-manager';
import { setupSignalHandlers } from './lib/signal-handlers';
// Flag to track if recovery has been initialized
let recoveryInitialized = false;
let recoveryAttempted = false;
let cleanupServiceStarted = false;
let shutdownManagerInitialized = false;
export const onRequest = defineMiddleware(async (context, next) => {
// Initialize shutdown manager and signal handlers first
if (!shutdownManagerInitialized) {
try {
console.log('🔧 Initializing shutdown manager and signal handlers...');
initializeShutdownManager();
setupSignalHandlers();
shutdownManagerInitialized = true;
console.log('✅ Shutdown manager and signal handlers initialized');
} catch (error) {
console.error('❌ Failed to initialize shutdown manager:', error);
// Continue anyway - this shouldn't block the application
}
}
// Initialize recovery system only once when the server starts
// This is a fallback in case the startup script didn't run
if (!recoveryInitialized && !recoveryAttempted) {
recoveryAttempted = true;
try {
// Check if recovery is actually needed before attempting
const needsRecovery = await hasJobsNeedingRecovery();
if (needsRecovery) {
console.log('⚠️ Middleware detected jobs needing recovery (startup script may not have run)');
console.log('Attempting recovery from middleware...');
// Run recovery with a shorter timeout since this is during request handling
const recoveryResult = await Promise.race([
initializeRecovery({
skipIfRecentAttempt: true,
maxRetries: 2,
retryDelay: 3000,
}),
new Promise<boolean>((_, reject) => {
setTimeout(() => reject(new Error('Middleware recovery timeout')), 15000);
})
]);
if (recoveryResult) {
console.log('✅ Middleware recovery completed successfully');
} else {
console.log('⚠️ Middleware recovery completed with some issues');
}
} else {
console.log('✅ No recovery needed (startup script likely handled it)');
}
recoveryInitialized = true;
} catch (error) {
console.error('⚠️ Middleware recovery failed or timed out:', error);
console.log('Application will continue, but some jobs may remain interrupted');
// Log recovery status for debugging
const status = getRecoveryStatus();
console.log('Recovery status:', status);
recoveryInitialized = true; // Mark as attempted to avoid retries
}
}
// Start cleanup service only once after recovery is complete
if (recoveryInitialized && !cleanupServiceStarted) {
try {
console.log('Starting automatic database cleanup service...');
startCleanupService();
// Register cleanup service shutdown callback
registerShutdownCallback(async () => {
console.log('🛑 Shutting down cleanup service...');
stopCleanupService();
});
cleanupServiceStarted = true;
} catch (error) {
console.error('Failed to start cleanup service:', error);
// Don't fail the request if cleanup service fails to start
}
}
// Continue with the request
return next();
});