mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2025-12-07 20:16:46 +03:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
20a771f340 | ||
|
|
d925b3c155 | ||
|
|
47e1c7b493 | ||
|
|
d7ce2a6908 | ||
|
|
4efe741c64 | ||
|
|
773842fa72 | ||
|
|
90944a40c6 |
@@ -30,3 +30,9 @@ JWT_SECRET=change-this-to-a-secure-random-string-in-production
|
||||
# GITEA_ORGANIZATION=github-mirrors
|
||||
# GITEA_ORG_VISIBILITY=public
|
||||
# DELAY=3600
|
||||
|
||||
# Optional Database Cleanup Configuration (configured via web UI)
|
||||
# These environment variables are optional and only used as defaults
|
||||
# Users can configure cleanup settings through the web interface
|
||||
# CLEANUP_ENABLED=false
|
||||
# CLEANUP_RETENTION_DAYS=7
|
||||
|
||||
30
README.md
30
README.md
@@ -481,7 +481,7 @@ Try the following steps:
|
||||
> docker compose up -d
|
||||
> ```
|
||||
>
|
||||
> This setup includes automatic database maintenance that runs daily to clean up old events and mirror jobs, preventing the database from growing too large. You can customize the retention periods by setting the `EVENTS_RETENTION_DAYS` and `JOBS_RETENTION_DAYS` environment variables.
|
||||
> This setup provides a complete containerized deployment for the Gitea Mirror application.
|
||||
|
||||
|
||||
#### Database Maintenance
|
||||
@@ -498,35 +498,9 @@ Try the following steps:
|
||||
>
|
||||
> # Reset user accounts (for development)
|
||||
> bun run reset-users
|
||||
>
|
||||
> # Clean up old events (keeps last 7 days by default)
|
||||
> bun run cleanup-events
|
||||
>
|
||||
> # Clean up old events with custom retention period (e.g., 30 days)
|
||||
> bun run cleanup-events 30
|
||||
>
|
||||
> # Clean up old mirror jobs (keeps last 7 days by default)
|
||||
> bun run cleanup-jobs
|
||||
>
|
||||
> # Clean up old mirror jobs with custom retention period (e.g., 30 days)
|
||||
> bun run cleanup-jobs 30
|
||||
>
|
||||
> # Clean up both events and mirror jobs
|
||||
> bun run cleanup-all
|
||||
> ```
|
||||
>
|
||||
> For automated maintenance, consider setting up cron jobs to run the cleanup scripts periodically:
|
||||
>
|
||||
> ```bash
|
||||
> # Add these to your crontab
|
||||
> # Clean up events daily at 2 AM
|
||||
> 0 2 * * * cd /path/to/gitea-mirror && bun run cleanup-events
|
||||
>
|
||||
> # Clean up mirror jobs daily at 3 AM
|
||||
> 0 3 * * * cd /path/to/gitea-mirror && bun run cleanup-jobs
|
||||
> ```
|
||||
>
|
||||
> **Note:** When using Docker, these cleanup jobs are automatically scheduled inside the container with the default retention period of 7 days. You can customize the retention periods by setting the `EVENTS_RETENTION_DAYS` and `JOBS_RETENTION_DAYS` environment variables in your docker-compose file.
|
||||
> **Note:** For cleaning up old activities and events, use the cleanup button in the Activity Log page of the web interface.
|
||||
|
||||
|
||||
> [!NOTE]
|
||||
|
||||
@@ -41,9 +41,6 @@ services:
|
||||
- GITEA_ORGANIZATION=${GITEA_ORGANIZATION:-github-mirrors}
|
||||
- GITEA_ORG_VISIBILITY=${GITEA_ORG_VISIBILITY:-public}
|
||||
- DELAY=${DELAY:-3600}
|
||||
# Database maintenance settings
|
||||
- EVENTS_RETENTION_DAYS=${EVENTS_RETENTION_DAYS:-7}
|
||||
- JOBS_RETENTION_DAYS=${JOBS_RETENTION_DAYS:-7}
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/api/health"]
|
||||
interval: 30s
|
||||
|
||||
@@ -30,24 +30,7 @@ if [ "$JWT_SECRET" = "your-secret-key-change-this-in-production" ] || [ -z "$JWT
|
||||
echo "JWT_SECRET has been set to a secure random value"
|
||||
fi
|
||||
|
||||
# Set up automatic database cleanup cron job
|
||||
# Default to 7 days retention for events and mirror jobs unless specified by environment variables
|
||||
EVENTS_RETENTION_DAYS=${EVENTS_RETENTION_DAYS:-7}
|
||||
JOBS_RETENTION_DAYS=${JOBS_RETENTION_DAYS:-7}
|
||||
|
||||
# Create cron directory if it doesn't exist
|
||||
mkdir -p /app/data/cron
|
||||
|
||||
# Create the cron job file
|
||||
cat > /app/data/cron/cleanup-cron <<EOF
|
||||
# Run event cleanup daily at 2 AM
|
||||
0 2 * * * cd /app && bun dist/scripts/cleanup-events.js ${EVENTS_RETENTION_DAYS} >> /app/data/cleanup-events.log 2>&1
|
||||
|
||||
# Run mirror jobs cleanup daily at 3 AM
|
||||
0 3 * * * cd /app && bun dist/scripts/cleanup-mirror-jobs.js ${JOBS_RETENTION_DAYS} >> /app/data/cleanup-mirror-jobs.log 2>&1
|
||||
|
||||
# Empty line at the end is required for cron to work properly
|
||||
EOF
|
||||
|
||||
# Skip dependency installation entirely for pre-built images
|
||||
# Dependencies are already installed during the Docker build process
|
||||
@@ -223,32 +206,7 @@ if [ -f "package.json" ]; then
|
||||
echo "Setting application version: $npm_package_version"
|
||||
fi
|
||||
|
||||
# Set up cron if it's available
|
||||
if command -v crontab >/dev/null 2>&1; then
|
||||
echo "Setting up automatic database cleanup cron jobs..."
|
||||
# Install cron if not already installed
|
||||
if ! command -v cron >/dev/null 2>&1; then
|
||||
echo "Installing cron..."
|
||||
apt-get update && apt-get install -y cron
|
||||
fi
|
||||
|
||||
# Install the cron job
|
||||
crontab /app/data/cron/cleanup-cron
|
||||
|
||||
# Start cron service
|
||||
if command -v service >/dev/null 2>&1; then
|
||||
service cron start
|
||||
echo "Cron service started"
|
||||
elif command -v cron >/dev/null 2>&1; then
|
||||
cron
|
||||
echo "Cron daemon started"
|
||||
else
|
||||
echo "Warning: Could not start cron service. Automatic database cleanup will not run."
|
||||
fi
|
||||
else
|
||||
echo "Warning: crontab command not found. Automatic database cleanup will not be set up."
|
||||
echo "Consider setting up external scheduled tasks to run cleanup scripts."
|
||||
fi
|
||||
|
||||
# Run startup recovery to handle any interrupted jobs
|
||||
echo "Running startup recovery..."
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "2.6.0",
|
||||
"version": "2.8.0",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
@@ -17,9 +17,7 @@
|
||||
"check-db": "bun scripts/manage-db.ts check",
|
||||
"fix-db": "bun scripts/manage-db.ts fix",
|
||||
"reset-users": "bun scripts/manage-db.ts reset-users",
|
||||
"cleanup-events": "bun scripts/cleanup-events.ts",
|
||||
"cleanup-jobs": "bun scripts/cleanup-mirror-jobs.ts",
|
||||
"cleanup-all": "bun scripts/cleanup-events.ts && bun scripts/cleanup-mirror-jobs.ts",
|
||||
|
||||
"startup-recovery": "bun scripts/startup-recovery.ts",
|
||||
"startup-recovery-force": "bun scripts/startup-recovery.ts --force",
|
||||
"test-recovery": "bun scripts/test-recovery.ts",
|
||||
|
||||
@@ -64,19 +64,7 @@ The following scripts help manage events in the SQLite database:
|
||||
|
||||
|
||||
|
||||
### Event Cleanup (cleanup-events.ts)
|
||||
|
||||
Removes old events and duplicate events from the database to prevent it from growing too large.
|
||||
|
||||
```bash
|
||||
# Remove events older than 7 days (default) and duplicates
|
||||
bun scripts/cleanup-events.ts
|
||||
|
||||
# Remove events older than X days and duplicates
|
||||
bun scripts/cleanup-events.ts 14
|
||||
```
|
||||
|
||||
This script can be scheduled to run periodically (e.g., daily) using cron or another scheduler. When using Docker, this is automatically scheduled to run daily.
|
||||
|
||||
### Remove Duplicate Events (remove-duplicate-events.ts)
|
||||
|
||||
@@ -90,19 +78,7 @@ bun scripts/remove-duplicate-events.ts
|
||||
bun scripts/remove-duplicate-events.ts <userId>
|
||||
```
|
||||
|
||||
### Mirror Jobs Cleanup (cleanup-mirror-jobs.ts)
|
||||
|
||||
Removes old mirror jobs from the database to prevent it from growing too large.
|
||||
|
||||
```bash
|
||||
# Remove mirror jobs older than 7 days (default)
|
||||
bun scripts/cleanup-mirror-jobs.ts
|
||||
|
||||
# Remove mirror jobs older than X days
|
||||
bun scripts/cleanup-mirror-jobs.ts 14
|
||||
```
|
||||
|
||||
This script can be scheduled to run periodically (e.g., daily) using cron or another scheduler. When using Docker, this is automatically scheduled to run daily.
|
||||
|
||||
### Fix Interrupted Jobs (fix-interrupted-jobs.ts)
|
||||
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Script to clean up old events from the database
|
||||
* This script should be run periodically (e.g., daily) to prevent the events table from growing too large
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/cleanup-events.ts [days]
|
||||
*
|
||||
* Where [days] is the number of days to keep events (default: 7)
|
||||
*/
|
||||
|
||||
import { cleanupOldEvents, removeDuplicateEvents } from "../src/lib/events";
|
||||
|
||||
// Parse command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const daysToKeep = args.length > 0 ? parseInt(args[0], 10) : 7;
|
||||
|
||||
if (isNaN(daysToKeep) || daysToKeep < 1) {
|
||||
console.error("Error: Days to keep must be a positive number");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function runCleanup() {
|
||||
try {
|
||||
console.log(`Starting event cleanup (retention: ${daysToKeep} days)...`);
|
||||
|
||||
// First, remove duplicate events
|
||||
console.log("Step 1: Removing duplicate events...");
|
||||
const duplicateResult = await removeDuplicateEvents();
|
||||
console.log(`- Duplicate events removed: ${duplicateResult.duplicatesRemoved}`);
|
||||
|
||||
// Then, clean up old events
|
||||
console.log("Step 2: Cleaning up old events...");
|
||||
const result = await cleanupOldEvents(daysToKeep);
|
||||
|
||||
console.log(`Cleanup summary:`);
|
||||
console.log(`- Duplicate events removed: ${duplicateResult.duplicatesRemoved}`);
|
||||
console.log(`- Read events deleted: ${result.readEventsDeleted}`);
|
||||
console.log(`- Unread events deleted: ${result.unreadEventsDeleted}`);
|
||||
console.log(`- Total events deleted: ${result.readEventsDeleted + result.unreadEventsDeleted + duplicateResult.duplicatesRemoved}`);
|
||||
|
||||
console.log("Event cleanup completed successfully");
|
||||
} catch (error) {
|
||||
console.error("Error running event cleanup:", error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the cleanup
|
||||
runCleanup();
|
||||
@@ -1,102 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Script to clean up old mirror jobs from the database
|
||||
* This script should be run periodically (e.g., daily) to prevent the mirror_jobs table from growing too large
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/cleanup-mirror-jobs.ts [days]
|
||||
*
|
||||
* Where [days] is the number of days to keep mirror jobs (default: 7)
|
||||
*/
|
||||
|
||||
import { db, mirrorJobs } from "../src/lib/db";
|
||||
import { lt, and, eq } from "drizzle-orm";
|
||||
|
||||
// Parse command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const daysToKeep = args.length > 0 ? parseInt(args[0], 10) : 7;
|
||||
|
||||
if (isNaN(daysToKeep) || daysToKeep < 1) {
|
||||
console.error("Error: Days to keep must be a positive number");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up old mirror jobs to prevent the database from growing too large
|
||||
* Should be called periodically (e.g., daily via a cron job)
|
||||
*
|
||||
* @param maxAgeInDays Number of days to keep mirror jobs (default: 7)
|
||||
* @returns Object containing the number of completed and in-progress jobs deleted
|
||||
*/
|
||||
async function cleanupOldMirrorJobs(
|
||||
maxAgeInDays: number = 7
|
||||
): Promise<{ completedJobsDeleted: number; inProgressJobsDeleted: number }> {
|
||||
try {
|
||||
console.log(`Cleaning up mirror jobs older than ${maxAgeInDays} days...`);
|
||||
|
||||
// Calculate the cutoff date for completed jobs
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - maxAgeInDays);
|
||||
|
||||
// Delete completed jobs older than the cutoff date
|
||||
// Only delete jobs that are not in progress (inProgress = false)
|
||||
const completedResult = await db
|
||||
.delete(mirrorJobs)
|
||||
.where(
|
||||
and(
|
||||
eq(mirrorJobs.inProgress, false),
|
||||
lt(mirrorJobs.timestamp, cutoffDate)
|
||||
)
|
||||
);
|
||||
|
||||
const completedJobsDeleted = completedResult.changes || 0;
|
||||
console.log(`Deleted ${completedJobsDeleted} completed mirror jobs`);
|
||||
|
||||
// Calculate a much older cutoff date for in-progress jobs (3x the retention period)
|
||||
// This is to handle jobs that might have been abandoned or crashed
|
||||
const inProgressCutoffDate = new Date();
|
||||
inProgressCutoffDate.setDate(inProgressCutoffDate.getDate() - (maxAgeInDays * 3));
|
||||
|
||||
// Delete in-progress jobs that are significantly older
|
||||
// This helps clean up jobs that might have been abandoned due to crashes
|
||||
const inProgressResult = await db
|
||||
.delete(mirrorJobs)
|
||||
.where(
|
||||
and(
|
||||
eq(mirrorJobs.inProgress, true),
|
||||
lt(mirrorJobs.timestamp, inProgressCutoffDate)
|
||||
)
|
||||
);
|
||||
|
||||
const inProgressJobsDeleted = inProgressResult.changes || 0;
|
||||
console.log(`Deleted ${inProgressJobsDeleted} abandoned in-progress mirror jobs`);
|
||||
|
||||
return { completedJobsDeleted, inProgressJobsDeleted };
|
||||
} catch (error) {
|
||||
console.error("Error cleaning up old mirror jobs:", error);
|
||||
return { completedJobsDeleted: 0, inProgressJobsDeleted: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// Run the cleanup
|
||||
async function runCleanup() {
|
||||
try {
|
||||
console.log(`Starting mirror jobs cleanup (retention: ${daysToKeep} days)...`);
|
||||
|
||||
// Call the cleanupOldMirrorJobs function
|
||||
const result = await cleanupOldMirrorJobs(daysToKeep);
|
||||
|
||||
console.log(`Cleanup summary:`);
|
||||
console.log(`- Completed jobs deleted: ${result.completedJobsDeleted}`);
|
||||
console.log(`- Abandoned in-progress jobs deleted: ${result.inProgressJobsDeleted}`);
|
||||
console.log(`- Total jobs deleted: ${result.completedJobsDeleted + result.inProgressJobsDeleted}`);
|
||||
|
||||
console.log("Mirror jobs cleanup completed successfully");
|
||||
} catch (error) {
|
||||
console.error("Error running mirror jobs cleanup:", error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the cleanup
|
||||
runCleanup();
|
||||
@@ -197,6 +197,33 @@ async function ensureTablesExist() {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Migration: Add cleanup_config column to existing configs table
|
||||
try {
|
||||
const db = new Database(dbPath);
|
||||
|
||||
// Check if cleanup_config column exists
|
||||
const tableInfo = db.query(`PRAGMA table_info(configs)`).all();
|
||||
const hasCleanupConfig = tableInfo.some((column: any) => column.name === 'cleanup_config');
|
||||
|
||||
if (!hasCleanupConfig) {
|
||||
console.log("Adding cleanup_config column to configs table...");
|
||||
|
||||
// Add the column with a default value
|
||||
const defaultCleanupConfig = JSON.stringify({
|
||||
enabled: false,
|
||||
retentionDays: 7,
|
||||
lastRun: null,
|
||||
nextRun: null,
|
||||
});
|
||||
|
||||
db.exec(`ALTER TABLE configs ADD COLUMN cleanup_config TEXT NOT NULL DEFAULT '${defaultCleanupConfig}'`);
|
||||
console.log("✅ cleanup_config column added successfully.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("❌ Error during cleanup_config migration:", error);
|
||||
// Don't exit here as this is not critical for basic functionality
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -328,6 +355,7 @@ async function initializeDatabase() {
|
||||
include TEXT NOT NULL DEFAULT '["*"]',
|
||||
exclude TEXT NOT NULL DEFAULT '[]',
|
||||
schedule_config TEXT NOT NULL,
|
||||
cleanup_config TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
|
||||
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
|
||||
FOREIGN KEY (user_id) REFERENCES users(id)
|
||||
@@ -459,10 +487,16 @@ async function initializeDatabase() {
|
||||
lastRun: null,
|
||||
nextRun: null,
|
||||
});
|
||||
const cleanupConfig = JSON.stringify({
|
||||
enabled: false,
|
||||
retentionDays: 7,
|
||||
lastRun: null,
|
||||
nextRun: null,
|
||||
});
|
||||
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, cleanup_config, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
stmt.run(
|
||||
@@ -475,6 +509,7 @@ async function initializeDatabase() {
|
||||
include,
|
||||
exclude,
|
||||
scheduleConfig,
|
||||
cleanupConfig,
|
||||
Date.now(),
|
||||
Date.now()
|
||||
);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useEffect, useState, useCallback, useRef } from 'react';
|
||||
import { GitHubConfigForm } from './GitHubConfigForm';
|
||||
import { GiteaConfigForm } from './GiteaConfigForm';
|
||||
import { ScheduleConfigForm } from './ScheduleConfigForm';
|
||||
import { DatabaseCleanupConfigForm } from './DatabaseCleanupConfigForm';
|
||||
import type {
|
||||
ConfigApiResponse,
|
||||
GiteaConfig,
|
||||
@@ -9,6 +10,7 @@ import type {
|
||||
SaveConfigApiRequest,
|
||||
SaveConfigApiResponse,
|
||||
ScheduleConfig,
|
||||
DatabaseCleanupConfig,
|
||||
} from '@/types/config';
|
||||
import { Button } from '../ui/button';
|
||||
import { useAuth } from '@/hooks/useAuth';
|
||||
@@ -22,6 +24,7 @@ type ConfigState = {
|
||||
githubConfig: GitHubConfig;
|
||||
giteaConfig: GiteaConfig;
|
||||
scheduleConfig: ScheduleConfig;
|
||||
cleanupConfig: DatabaseCleanupConfig;
|
||||
};
|
||||
|
||||
export function ConfigTabs() {
|
||||
@@ -48,13 +51,19 @@ export function ConfigTabs() {
|
||||
enabled: false,
|
||||
interval: 3600,
|
||||
},
|
||||
cleanupConfig: {
|
||||
enabled: false,
|
||||
retentionDays: 7,
|
||||
},
|
||||
});
|
||||
const { user, refreshUser } = useAuth();
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [isSyncing, setIsSyncing] = useState<boolean>(false);
|
||||
const [isConfigSaved, setIsConfigSaved] = useState<boolean>(false);
|
||||
const [isAutoSaving, setIsAutoSaving] = useState<boolean>(false);
|
||||
const autoSaveTimeoutRef = useRef<NodeJS.Timeout | null>(null);
|
||||
const [isAutoSavingSchedule, setIsAutoSavingSchedule] = useState<boolean>(false);
|
||||
const [isAutoSavingCleanup, setIsAutoSavingCleanup] = useState<boolean>(false);
|
||||
const autoSaveScheduleTimeoutRef = useRef<NodeJS.Timeout | null>(null);
|
||||
const autoSaveCleanupTimeoutRef = useRef<NodeJS.Timeout | null>(null);
|
||||
|
||||
const isConfigFormValid = (): boolean => {
|
||||
const { githubConfig, giteaConfig } = config;
|
||||
@@ -107,6 +116,7 @@ export function ConfigTabs() {
|
||||
githubConfig: config.githubConfig,
|
||||
giteaConfig: config.giteaConfig,
|
||||
scheduleConfig: config.scheduleConfig,
|
||||
cleanupConfig: config.cleanupConfig,
|
||||
};
|
||||
try {
|
||||
const response = await fetch('/api/config', {
|
||||
@@ -142,19 +152,20 @@ export function ConfigTabs() {
|
||||
if (!user?.id || !isConfigSaved) return; // Only auto-save if config was previously saved
|
||||
|
||||
// Clear any existing timeout
|
||||
if (autoSaveTimeoutRef.current) {
|
||||
clearTimeout(autoSaveTimeoutRef.current);
|
||||
if (autoSaveScheduleTimeoutRef.current) {
|
||||
clearTimeout(autoSaveScheduleTimeoutRef.current);
|
||||
}
|
||||
|
||||
// Debounce the auto-save to prevent excessive API calls
|
||||
autoSaveTimeoutRef.current = setTimeout(async () => {
|
||||
setIsAutoSaving(true);
|
||||
autoSaveScheduleTimeoutRef.current = setTimeout(async () => {
|
||||
setIsAutoSavingSchedule(true);
|
||||
|
||||
const reqPayload: SaveConfigApiRequest = {
|
||||
userId: user.id!,
|
||||
githubConfig: config.githubConfig,
|
||||
giteaConfig: config.giteaConfig,
|
||||
scheduleConfig: scheduleConfig,
|
||||
cleanupConfig: config.cleanupConfig,
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -184,16 +195,71 @@ export function ConfigTabs() {
|
||||
{ duration: 3000 }
|
||||
);
|
||||
} finally {
|
||||
setIsAutoSaving(false);
|
||||
setIsAutoSavingSchedule(false);
|
||||
}
|
||||
}, 500); // 500ms debounce
|
||||
}, [user?.id, isConfigSaved, config.githubConfig, config.giteaConfig]);
|
||||
}, [user?.id, isConfigSaved, config.githubConfig, config.giteaConfig, config.cleanupConfig]);
|
||||
|
||||
// Cleanup timeout on unmount
|
||||
// Auto-save function specifically for cleanup config changes
|
||||
const autoSaveCleanupConfig = useCallback(async (cleanupConfig: DatabaseCleanupConfig) => {
|
||||
if (!user?.id || !isConfigSaved) return; // Only auto-save if config was previously saved
|
||||
|
||||
// Clear any existing timeout
|
||||
if (autoSaveCleanupTimeoutRef.current) {
|
||||
clearTimeout(autoSaveCleanupTimeoutRef.current);
|
||||
}
|
||||
|
||||
// Debounce the auto-save to prevent excessive API calls
|
||||
autoSaveCleanupTimeoutRef.current = setTimeout(async () => {
|
||||
setIsAutoSavingCleanup(true);
|
||||
|
||||
const reqPayload: SaveConfigApiRequest = {
|
||||
userId: user.id!,
|
||||
githubConfig: config.githubConfig,
|
||||
giteaConfig: config.giteaConfig,
|
||||
scheduleConfig: config.scheduleConfig,
|
||||
cleanupConfig: cleanupConfig,
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/config', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(reqPayload),
|
||||
});
|
||||
const result: SaveConfigApiResponse = await response.json();
|
||||
|
||||
if (result.success) {
|
||||
// Silent success - no toast for auto-save
|
||||
// Invalidate config cache so other components get fresh data
|
||||
invalidateConfigCache();
|
||||
} else {
|
||||
toast.error(
|
||||
`Auto-save failed: ${result.message || 'Unknown error'}`,
|
||||
{ duration: 3000 }
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
toast.error(
|
||||
`Auto-save error: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
{ duration: 3000 }
|
||||
);
|
||||
} finally {
|
||||
setIsAutoSavingCleanup(false);
|
||||
}
|
||||
}, 500); // 500ms debounce
|
||||
}, [user?.id, isConfigSaved, config.githubConfig, config.giteaConfig, config.scheduleConfig]);
|
||||
|
||||
// Cleanup timeouts on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (autoSaveTimeoutRef.current) {
|
||||
clearTimeout(autoSaveTimeoutRef.current);
|
||||
if (autoSaveScheduleTimeoutRef.current) {
|
||||
clearTimeout(autoSaveScheduleTimeoutRef.current);
|
||||
}
|
||||
if (autoSaveCleanupTimeoutRef.current) {
|
||||
clearTimeout(autoSaveCleanupTimeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
@@ -216,6 +282,8 @@ export function ConfigTabs() {
|
||||
response.giteaConfig || config.giteaConfig,
|
||||
scheduleConfig:
|
||||
response.scheduleConfig || config.scheduleConfig,
|
||||
cleanupConfig:
|
||||
response.cleanupConfig || config.cleanupConfig,
|
||||
});
|
||||
if (response.id) setIsConfigSaved(true);
|
||||
}
|
||||
@@ -273,11 +341,20 @@ export function ConfigTabs() {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="border rounded-lg p-4">
|
||||
<div className="space-y-4">
|
||||
<Skeleton className="h-8 w-48" />
|
||||
<Skeleton className="h-16 w-full" />
|
||||
<Skeleton className="h-8 w-32" />
|
||||
<div className="flex gap-x-4">
|
||||
<div className="w-1/2 border rounded-lg p-4">
|
||||
<div className="space-y-4">
|
||||
<Skeleton className="h-8 w-48" />
|
||||
<Skeleton className="h-16 w-full" />
|
||||
<Skeleton className="h-8 w-32" />
|
||||
</div>
|
||||
</div>
|
||||
<div className="w-1/2 border rounded-lg p-4">
|
||||
<div className="space-y-4">
|
||||
<Skeleton className="h-8 w-48" />
|
||||
<Skeleton className="h-16 w-full" />
|
||||
<Skeleton className="h-8 w-32" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -368,20 +445,40 @@ export function ConfigTabs() {
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
<ScheduleConfigForm
|
||||
config={config.scheduleConfig}
|
||||
setConfig={update =>
|
||||
setConfig(prev => ({
|
||||
...prev,
|
||||
scheduleConfig:
|
||||
typeof update === 'function'
|
||||
? update(prev.scheduleConfig)
|
||||
: update,
|
||||
}))
|
||||
}
|
||||
onAutoSave={autoSaveScheduleConfig}
|
||||
isAutoSaving={isAutoSaving}
|
||||
/>
|
||||
<div className="flex gap-x-4">
|
||||
<div className="w-1/2">
|
||||
<ScheduleConfigForm
|
||||
config={config.scheduleConfig}
|
||||
setConfig={update =>
|
||||
setConfig(prev => ({
|
||||
...prev,
|
||||
scheduleConfig:
|
||||
typeof update === 'function'
|
||||
? update(prev.scheduleConfig)
|
||||
: update,
|
||||
}))
|
||||
}
|
||||
onAutoSave={autoSaveScheduleConfig}
|
||||
isAutoSaving={isAutoSavingSchedule}
|
||||
/>
|
||||
</div>
|
||||
<div className="w-1/2">
|
||||
<DatabaseCleanupConfigForm
|
||||
config={config.cleanupConfig}
|
||||
setConfig={update =>
|
||||
setConfig(prev => ({
|
||||
...prev,
|
||||
cleanupConfig:
|
||||
typeof update === 'function'
|
||||
? update(prev.cleanupConfig)
|
||||
: update,
|
||||
}))
|
||||
}
|
||||
onAutoSave={autoSaveCleanupConfig}
|
||||
isAutoSaving={isAutoSavingCleanup}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
146
src/components/config/DatabaseCleanupConfigForm.tsx
Normal file
146
src/components/config/DatabaseCleanupConfigForm.tsx
Normal file
@@ -0,0 +1,146 @@
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Checkbox } from "../ui/checkbox";
|
||||
import type { DatabaseCleanupConfig } from "@/types/config";
|
||||
import { formatDate } from "@/lib/utils";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "../ui/select";
|
||||
import { RefreshCw, Database } from "lucide-react";
|
||||
|
||||
interface DatabaseCleanupConfigFormProps {
|
||||
config: DatabaseCleanupConfig;
|
||||
setConfig: React.Dispatch<React.SetStateAction<DatabaseCleanupConfig>>;
|
||||
onAutoSave?: (config: DatabaseCleanupConfig) => void;
|
||||
isAutoSaving?: boolean;
|
||||
}
|
||||
|
||||
export function DatabaseCleanupConfigForm({
|
||||
config,
|
||||
setConfig,
|
||||
onAutoSave,
|
||||
isAutoSaving = false,
|
||||
}: DatabaseCleanupConfigFormProps) {
|
||||
const handleChange = (
|
||||
e: React.ChangeEvent<HTMLInputElement | HTMLSelectElement>
|
||||
) => {
|
||||
const { name, value, type } = e.target;
|
||||
const newConfig = {
|
||||
...config,
|
||||
[name]:
|
||||
type === "checkbox" ? (e.target as HTMLInputElement).checked : value,
|
||||
};
|
||||
setConfig(newConfig);
|
||||
|
||||
// Trigger auto-save for cleanup config changes
|
||||
if (onAutoSave) {
|
||||
onAutoSave(newConfig);
|
||||
}
|
||||
};
|
||||
|
||||
// Predefined retention periods
|
||||
const retentionOptions: { value: number; label: string }[] = [
|
||||
{ value: 1, label: "1 day" },
|
||||
{ value: 3, label: "3 days" },
|
||||
{ value: 7, label: "7 days" },
|
||||
{ value: 14, label: "14 days" },
|
||||
{ value: 30, label: "30 days" },
|
||||
{ value: 60, label: "60 days" },
|
||||
{ value: 90, label: "90 days" },
|
||||
];
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardContent className="pt-6 relative">
|
||||
{isAutoSaving && (
|
||||
<div className="absolute top-4 right-4 flex items-center text-sm text-muted-foreground">
|
||||
<RefreshCw className="h-3 w-3 animate-spin mr-1" />
|
||||
<span className="text-xs">Auto-saving...</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex flex-col gap-y-4">
|
||||
<div className="flex items-center">
|
||||
<Checkbox
|
||||
id="cleanup-enabled"
|
||||
name="enabled"
|
||||
checked={config.enabled}
|
||||
onCheckedChange={(checked) =>
|
||||
handleChange({
|
||||
target: {
|
||||
name: "enabled",
|
||||
type: "checkbox",
|
||||
checked: Boolean(checked),
|
||||
value: "",
|
||||
},
|
||||
} as React.ChangeEvent<HTMLInputElement>)
|
||||
}
|
||||
/>
|
||||
<label
|
||||
htmlFor="cleanup-enabled"
|
||||
className="select-none ml-2 block text-sm font-medium"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<Database className="h-4 w-4" />
|
||||
Enable Automatic Database Cleanup
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{config.enabled && (
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-2">
|
||||
Retention Period
|
||||
</label>
|
||||
|
||||
<Select
|
||||
name="retentionDays"
|
||||
value={String(config.retentionDays)}
|
||||
onValueChange={(value) =>
|
||||
handleChange({
|
||||
target: { name: "retentionDays", value },
|
||||
} as React.ChangeEvent<HTMLInputElement>)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full border border-input dark:bg-background dark:hover:bg-background">
|
||||
<SelectValue placeholder="Select retention period" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="bg-background text-foreground border border-input shadow-sm">
|
||||
{retentionOptions.map((option) => (
|
||||
<SelectItem
|
||||
key={option.value}
|
||||
value={option.value.toString()}
|
||||
className="cursor-pointer text-sm px-3 py-2 hover:bg-accent focus:bg-accent focus:text-accent-foreground"
|
||||
>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
Activities and events older than this period will be automatically deleted.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Last Run</label>
|
||||
<div className="text-sm">
|
||||
{config.lastRun ? formatDate(config.lastRun) : "Never"}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{config.nextRun && config.enabled && (
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Next Run</label>
|
||||
<div className="text-sm">{formatDate(config.nextRun)}</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -90,51 +90,53 @@ export function ScheduleConfigForm({
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label
|
||||
htmlFor="interval"
|
||||
className="block text-sm font-medium mb-1.5"
|
||||
>
|
||||
Mirroring Interval
|
||||
</label>
|
||||
|
||||
<Select
|
||||
name="interval"
|
||||
value={String(config.interval)}
|
||||
onValueChange={(value) =>
|
||||
handleChange({
|
||||
target: { name: "interval", value },
|
||||
} as React.ChangeEvent<HTMLInputElement>)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full border border-input dark:bg-background dark:hover:bg-background">
|
||||
<SelectValue placeholder="Select interval" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="bg-background text-foreground border border-input shadow-sm">
|
||||
{intervals.map((interval) => (
|
||||
<SelectItem
|
||||
key={interval.value}
|
||||
value={interval.value.toString()}
|
||||
className="cursor-pointer text-sm px-3 py-2 hover:bg-accent focus:bg-accent focus:text-accent-foreground"
|
||||
>
|
||||
{interval.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
How often the mirroring process should run.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{config.lastRun && (
|
||||
{config.enabled && (
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Last Run</label>
|
||||
<div className="text-sm">{formatDate(config.lastRun)}</div>
|
||||
<label
|
||||
htmlFor="interval"
|
||||
className="block text-sm font-medium mb-1.5"
|
||||
>
|
||||
Mirroring Interval
|
||||
</label>
|
||||
|
||||
<Select
|
||||
name="interval"
|
||||
value={String(config.interval)}
|
||||
onValueChange={(value) =>
|
||||
handleChange({
|
||||
target: { name: "interval", value },
|
||||
} as React.ChangeEvent<HTMLInputElement>)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full border border-input dark:bg-background dark:hover:bg-background">
|
||||
<SelectValue placeholder="Select interval" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="bg-background text-foreground border border-input shadow-sm">
|
||||
{intervals.map((interval) => (
|
||||
<SelectItem
|
||||
key={interval.value}
|
||||
value={interval.value.toString()}
|
||||
className="cursor-pointer text-sm px-3 py-2 hover:bg-accent focus:bg-accent focus:text-accent-foreground"
|
||||
>
|
||||
{interval.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
How often the mirroring process should run.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Last Run</label>
|
||||
<div className="text-sm">
|
||||
{config.lastRun ? formatDate(config.lastRun) : "Never"}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{config.nextRun && config.enabled && (
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-1">Next Run</label>
|
||||
|
||||
@@ -150,20 +150,11 @@ Events in Gitea Mirror (such as repository mirroring operations) are stored in t
|
||||
# View all events in the database
|
||||
bun scripts/check-events.ts
|
||||
|
||||
# Clean up old events (default: older than 7 days)
|
||||
bun scripts/cleanup-events.ts
|
||||
|
||||
# Clean up old mirror jobs (default: older than 7 days)
|
||||
bun scripts/cleanup-mirror-jobs.ts
|
||||
|
||||
# Clean up both events and mirror jobs
|
||||
bun run cleanup-all
|
||||
|
||||
# Mark all events as read
|
||||
bun scripts/mark-events-read.ts
|
||||
```
|
||||
|
||||
When using Docker, database cleanup is automatically scheduled to run daily. You can customize the retention periods by setting the `EVENTS_RETENTION_DAYS` and `JOBS_RETENTION_DAYS` environment variables in your docker-compose file.
|
||||
For cleaning up old activities and events, use the cleanup button in the Activity Log page of the web interface.
|
||||
|
||||
### Health Check Endpoint
|
||||
|
||||
|
||||
@@ -179,4 +179,4 @@ After your initial setup:
|
||||
- Check out the [Configuration Guide](/configuration) for advanced settings
|
||||
- Review the [Architecture Documentation](/architecture) to understand the system
|
||||
- For server deployments, set up monitoring using the health check endpoint
|
||||
- Consider setting up a cron job to clean up old events: `bun scripts/cleanup-events.ts`
|
||||
- Use the cleanup button in the Activity Log page to manage old events and activities
|
||||
|
||||
191
src/lib/cleanup-service.ts
Normal file
191
src/lib/cleanup-service.ts
Normal file
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* Background cleanup service for automatic database maintenance
|
||||
* This service runs periodically to clean up old events and mirror jobs
|
||||
* based on user configuration settings
|
||||
*/
|
||||
|
||||
import { db, configs, events, mirrorJobs } from "@/lib/db";
|
||||
import { eq, lt, and } from "drizzle-orm";
|
||||
|
||||
interface CleanupResult {
|
||||
userId: string;
|
||||
eventsDeleted: number;
|
||||
mirrorJobsDeleted: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old events and mirror jobs for a specific user
|
||||
*/
|
||||
async function cleanupForUser(userId: string, retentionDays: number): Promise<CleanupResult> {
|
||||
try {
|
||||
console.log(`Running cleanup for user ${userId} with ${retentionDays} days retention`);
|
||||
|
||||
// Calculate cutoff date
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
let eventsDeleted = 0;
|
||||
let mirrorJobsDeleted = 0;
|
||||
|
||||
// Clean up old events
|
||||
const eventsResult = await db
|
||||
.delete(events)
|
||||
.where(
|
||||
and(
|
||||
eq(events.userId, userId),
|
||||
lt(events.createdAt, cutoffDate)
|
||||
)
|
||||
);
|
||||
eventsDeleted = eventsResult.changes || 0;
|
||||
|
||||
// Clean up old mirror jobs (only completed ones)
|
||||
const jobsResult = await db
|
||||
.delete(mirrorJobs)
|
||||
.where(
|
||||
and(
|
||||
eq(mirrorJobs.userId, userId),
|
||||
eq(mirrorJobs.inProgress, false),
|
||||
lt(mirrorJobs.timestamp, cutoffDate)
|
||||
)
|
||||
);
|
||||
mirrorJobsDeleted = jobsResult.changes || 0;
|
||||
|
||||
console.log(`Cleanup completed for user ${userId}: ${eventsDeleted} events, ${mirrorJobsDeleted} jobs deleted`);
|
||||
|
||||
return {
|
||||
userId,
|
||||
eventsDeleted,
|
||||
mirrorJobsDeleted,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error during cleanup for user ${userId}:`, error);
|
||||
return {
|
||||
userId,
|
||||
eventsDeleted: 0,
|
||||
mirrorJobsDeleted: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the cleanup configuration with last run time and calculate next run
|
||||
*/
|
||||
async function updateCleanupConfig(userId: string, cleanupConfig: any) {
|
||||
try {
|
||||
const now = new Date();
|
||||
const nextRun = new Date(now.getTime() + 24 * 60 * 60 * 1000); // Next day
|
||||
|
||||
const updatedConfig = {
|
||||
...cleanupConfig,
|
||||
lastRun: now,
|
||||
nextRun: nextRun,
|
||||
};
|
||||
|
||||
await db
|
||||
.update(configs)
|
||||
.set({
|
||||
cleanupConfig: updatedConfig,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(configs.userId, userId));
|
||||
|
||||
console.log(`Updated cleanup config for user ${userId}, next run: ${nextRun.toISOString()}`);
|
||||
} catch (error) {
|
||||
console.error(`Error updating cleanup config for user ${userId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run automatic cleanup for all users with cleanup enabled
|
||||
*/
|
||||
export async function runAutomaticCleanup(): Promise<CleanupResult[]> {
|
||||
try {
|
||||
console.log('Starting automatic cleanup service...');
|
||||
|
||||
// Get all users with cleanup enabled
|
||||
const userConfigs = await db
|
||||
.select()
|
||||
.from(configs)
|
||||
.where(eq(configs.isActive, true));
|
||||
|
||||
const results: CleanupResult[] = [];
|
||||
const now = new Date();
|
||||
|
||||
for (const config of userConfigs) {
|
||||
try {
|
||||
const cleanupConfig = config.cleanupConfig;
|
||||
|
||||
// Skip if cleanup is not enabled
|
||||
if (!cleanupConfig?.enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if it's time to run cleanup
|
||||
const nextRun = cleanupConfig.nextRun ? new Date(cleanupConfig.nextRun) : null;
|
||||
|
||||
// If nextRun is null or in the past, run cleanup
|
||||
if (!nextRun || now >= nextRun) {
|
||||
const result = await cleanupForUser(config.userId, cleanupConfig.retentionDays || 7);
|
||||
results.push(result);
|
||||
|
||||
// Update the cleanup config with new run times
|
||||
await updateCleanupConfig(config.userId, cleanupConfig);
|
||||
} else {
|
||||
console.log(`Skipping cleanup for user ${config.userId}, next run: ${nextRun.toISOString()}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error processing cleanup for user ${config.userId}:`, error);
|
||||
results.push({
|
||||
userId: config.userId,
|
||||
eventsDeleted: 0,
|
||||
mirrorJobsDeleted: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Automatic cleanup completed. Processed ${results.length} users.`);
|
||||
return results;
|
||||
} catch (error) {
|
||||
console.error('Error in automatic cleanup service:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the cleanup service with periodic execution
|
||||
* This should be called when the application starts
|
||||
*/
|
||||
export function startCleanupService() {
|
||||
console.log('Starting background cleanup service...');
|
||||
|
||||
// Run cleanup every hour
|
||||
const CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour in milliseconds
|
||||
|
||||
// Run initial cleanup after 5 minutes to allow app to fully start
|
||||
setTimeout(() => {
|
||||
runAutomaticCleanup().catch(error => {
|
||||
console.error('Error in initial cleanup run:', error);
|
||||
});
|
||||
}, 5 * 60 * 1000); // 5 minutes
|
||||
|
||||
// Set up periodic cleanup
|
||||
setInterval(() => {
|
||||
runAutomaticCleanup().catch(error => {
|
||||
console.error('Error in periodic cleanup run:', error);
|
||||
});
|
||||
}, CLEANUP_INTERVAL);
|
||||
|
||||
console.log(`✅ Cleanup service started. Will run every ${CLEANUP_INTERVAL / 1000 / 60} minutes.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the cleanup service (for testing or shutdown)
|
||||
*/
|
||||
export function stopCleanupService() {
|
||||
// Note: In a real implementation, you'd want to track the interval ID
|
||||
// and clear it here. For now, this is a placeholder.
|
||||
console.log('Cleanup service stop requested (not implemented)');
|
||||
}
|
||||
@@ -81,6 +81,7 @@ export const events = sqliteTable("events", {
|
||||
const githubSchema = configSchema.shape.githubConfig;
|
||||
const giteaSchema = configSchema.shape.giteaConfig;
|
||||
const scheduleSchema = configSchema.shape.scheduleConfig;
|
||||
const cleanupSchema = configSchema.shape.cleanupConfig;
|
||||
|
||||
export const configs = sqliteTable("configs", {
|
||||
id: text("id").primaryKey(),
|
||||
@@ -112,6 +113,10 @@ export const configs = sqliteTable("configs", {
|
||||
.$type<z.infer<typeof scheduleSchema>>()
|
||||
.notNull(),
|
||||
|
||||
cleanupConfig: text("cleanup_config", { mode: "json" })
|
||||
.$type<z.infer<typeof cleanupSchema>>()
|
||||
.notNull(),
|
||||
|
||||
createdAt: integer("created_at", { mode: "timestamp" })
|
||||
.notNull()
|
||||
.default(new Date()),
|
||||
|
||||
@@ -52,6 +52,12 @@ export const configSchema = z.object({
|
||||
lastRun: z.date().optional(),
|
||||
nextRun: z.date().optional(),
|
||||
}),
|
||||
cleanupConfig: z.object({
|
||||
enabled: z.boolean().default(false),
|
||||
retentionDays: z.number().min(1).default(7), // in days
|
||||
lastRun: z.date().optional(),
|
||||
nextRun: z.date().optional(),
|
||||
}),
|
||||
createdAt: z.date().default(() => new Date()),
|
||||
updatedAt: z.date().default(() => new Date()),
|
||||
});
|
||||
|
||||
@@ -87,36 +87,27 @@ export async function getNewEvents({
|
||||
lastEventTime?: Date;
|
||||
}): Promise<any[]> {
|
||||
try {
|
||||
console.log(`Getting new events for user ${userId} in channel ${channel}`);
|
||||
if (lastEventTime) {
|
||||
console.log(`Looking for events after ${lastEventTime.toISOString()}`);
|
||||
}
|
||||
|
||||
// Build the query
|
||||
let query = db
|
||||
.select()
|
||||
.from(events)
|
||||
.where(
|
||||
and(
|
||||
eq(events.userId, userId),
|
||||
eq(events.channel, channel),
|
||||
eq(events.read, false)
|
||||
)
|
||||
)
|
||||
.orderBy(events.createdAt);
|
||||
// Build the query conditions
|
||||
const conditions = [
|
||||
eq(events.userId, userId),
|
||||
eq(events.channel, channel),
|
||||
eq(events.read, false)
|
||||
];
|
||||
|
||||
// Add time filter if provided
|
||||
if (lastEventTime) {
|
||||
query = query.where(gt(events.createdAt, lastEventTime));
|
||||
conditions.push(gt(events.createdAt, lastEventTime));
|
||||
}
|
||||
|
||||
// Execute the query
|
||||
const newEvents = await query;
|
||||
console.log(`Found ${newEvents.length} new events`);
|
||||
const newEvents = await db
|
||||
.select()
|
||||
.from(events)
|
||||
.where(and(...conditions))
|
||||
.orderBy(events.createdAt);
|
||||
|
||||
// Mark events as read
|
||||
if (newEvents.length > 0) {
|
||||
console.log(`Marking ${newEvents.length} events as read`);
|
||||
await db
|
||||
.update(events)
|
||||
.set({ read: true })
|
||||
@@ -149,14 +140,11 @@ export async function removeDuplicateEvents(userId?: string): Promise<{ duplicat
|
||||
console.log("Removing duplicate events...");
|
||||
|
||||
// Build the base query
|
||||
let query = db.select().from(events);
|
||||
if (userId) {
|
||||
query = query.where(eq(events.userId, userId));
|
||||
}
|
||||
const allEvents = userId
|
||||
? await db.select().from(events).where(eq(events.userId, userId))
|
||||
: await db.select().from(events);
|
||||
|
||||
const allEvents = await query;
|
||||
const duplicateIds: string[] = [];
|
||||
const seenKeys = new Set<string>();
|
||||
|
||||
// Group events by user and channel, then check for duplicates
|
||||
const eventsByUserChannel = new Map<string, typeof allEvents>();
|
||||
@@ -214,7 +202,7 @@ export async function removeDuplicateEvents(userId?: string): Promise<{ duplicat
|
||||
|
||||
/**
|
||||
* Cleans up old events to prevent the database from growing too large
|
||||
* Should be called periodically (e.g., daily via a cron job)
|
||||
* This function is used by the cleanup button in the Activity Log page
|
||||
*
|
||||
* @param maxAgeInDays Number of days to keep events (default: 7)
|
||||
* @param cleanupUnreadAfterDays Number of days after which to clean up unread events (default: 2x maxAgeInDays)
|
||||
@@ -241,7 +229,7 @@ export async function cleanupOldEvents(
|
||||
)
|
||||
);
|
||||
|
||||
const readEventsDeleted = readResult.changes || 0;
|
||||
const readEventsDeleted = (readResult as any).changes || 0;
|
||||
console.log(`Deleted ${readEventsDeleted} read events`);
|
||||
|
||||
// Calculate the cutoff date for unread events (default to 2x the retention period)
|
||||
@@ -259,7 +247,7 @@ export async function cleanupOldEvents(
|
||||
)
|
||||
);
|
||||
|
||||
const unreadEventsDeleted = unreadResult.changes || 0;
|
||||
const unreadEventsDeleted = (unreadResult as any).changes || 0;
|
||||
console.log(`Deleted ${unreadEventsDeleted} unread events`);
|
||||
|
||||
return { readEventsDeleted, unreadEventsDeleted };
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { defineMiddleware } from 'astro:middleware';
|
||||
import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from './lib/recovery';
|
||||
import { startCleanupService } from './lib/cleanup-service';
|
||||
|
||||
// Flag to track if recovery has been initialized
|
||||
let recoveryInitialized = false;
|
||||
let recoveryAttempted = false;
|
||||
let cleanupServiceStarted = false;
|
||||
|
||||
export const onRequest = defineMiddleware(async (context, next) => {
|
||||
// Initialize recovery system only once when the server starts
|
||||
@@ -53,6 +55,18 @@ export const onRequest = defineMiddleware(async (context, next) => {
|
||||
}
|
||||
}
|
||||
|
||||
// Start cleanup service only once after recovery is complete
|
||||
if (recoveryInitialized && !cleanupServiceStarted) {
|
||||
try {
|
||||
console.log('Starting automatic database cleanup service...');
|
||||
startCleanupService();
|
||||
cleanupServiceStarted = true;
|
||||
} catch (error) {
|
||||
console.error('Failed to start cleanup service:', error);
|
||||
// Don't fail the request if cleanup service fails to start
|
||||
}
|
||||
}
|
||||
|
||||
// Continue with the request
|
||||
return next();
|
||||
});
|
||||
|
||||
72
src/pages/api/cleanup/auto.ts
Normal file
72
src/pages/api/cleanup/auto.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* API endpoint to manually trigger automatic cleanup
|
||||
* This is useful for testing and debugging the cleanup service
|
||||
*/
|
||||
|
||||
import type { APIRoute } from 'astro';
|
||||
import { runAutomaticCleanup } from '@/lib/cleanup-service';
|
||||
|
||||
export const POST: APIRoute = async ({ request }) => {
|
||||
try {
|
||||
console.log('Manual cleanup trigger requested');
|
||||
|
||||
// Run the automatic cleanup
|
||||
const results = await runAutomaticCleanup();
|
||||
|
||||
// Calculate totals
|
||||
const totalEventsDeleted = results.reduce((sum, result) => sum + result.eventsDeleted, 0);
|
||||
const totalJobsDeleted = results.reduce((sum, result) => sum + result.mirrorJobsDeleted, 0);
|
||||
const errors = results.filter(result => result.error);
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: 'Automatic cleanup completed',
|
||||
results: {
|
||||
usersProcessed: results.length,
|
||||
totalEventsDeleted,
|
||||
totalJobsDeleted,
|
||||
errors: errors.length,
|
||||
details: results,
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Error in manual cleanup trigger:', error);
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
message: 'Failed to run automatic cleanup',
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
}),
|
||||
{
|
||||
status: 500,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const GET: APIRoute = async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
message: 'Use POST method to trigger cleanup',
|
||||
}),
|
||||
{
|
||||
status: 405,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
);
|
||||
};
|
||||
@@ -6,14 +6,14 @@ import { eq } from "drizzle-orm";
|
||||
export const POST: APIRoute = async ({ request }) => {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const { userId, githubConfig, giteaConfig, scheduleConfig } = body;
|
||||
const { userId, githubConfig, giteaConfig, scheduleConfig, cleanupConfig } = body;
|
||||
|
||||
if (!userId || !githubConfig || !giteaConfig || !scheduleConfig) {
|
||||
if (!userId || !githubConfig || !giteaConfig || !scheduleConfig || !cleanupConfig) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
message:
|
||||
"userId, githubConfig, giteaConfig, and scheduleConfig are required.",
|
||||
"userId, githubConfig, giteaConfig, scheduleConfig, and cleanupConfig are required.",
|
||||
}),
|
||||
{
|
||||
status: 400,
|
||||
@@ -64,6 +64,7 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
githubConfig,
|
||||
giteaConfig,
|
||||
scheduleConfig,
|
||||
cleanupConfig,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(configs.id, existingConfig.id));
|
||||
@@ -113,6 +114,7 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
include: [],
|
||||
exclude: [],
|
||||
scheduleConfig,
|
||||
cleanupConfig,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
@@ -197,6 +199,12 @@ export const GET: APIRoute = async ({ request }) => {
|
||||
lastRun: null,
|
||||
nextRun: null,
|
||||
},
|
||||
cleanupConfig: {
|
||||
enabled: false,
|
||||
retentionDays: 7,
|
||||
lastRun: null,
|
||||
nextRun: null,
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
|
||||
import { GET } from "./health";
|
||||
import * as dbModule from "@/lib/db";
|
||||
import os from "os";
|
||||
|
||||
// Mock the database module
|
||||
mock.module("@/lib/db", () => {
|
||||
return {
|
||||
db: {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
limit: () => Promise.resolve([{ test: 1 }])
|
||||
})
|
||||
})
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Mock the os functions individually
|
||||
const originalPlatform = os.platform;
|
||||
const originalVersion = os.version;
|
||||
const originalArch = os.arch;
|
||||
const originalTotalmem = os.totalmem;
|
||||
const originalFreemem = os.freemem;
|
||||
|
||||
describe("Health API Endpoint", () => {
|
||||
beforeEach(() => {
|
||||
// Mock os functions
|
||||
os.platform = mock(() => "test-platform");
|
||||
os.version = mock(() => "test-version");
|
||||
os.arch = mock(() => "test-arch");
|
||||
os.totalmem = mock(() => 16 * 1024 * 1024 * 1024); // 16GB
|
||||
os.freemem = mock(() => 8 * 1024 * 1024 * 1024); // 8GB
|
||||
|
||||
// Mock process.memoryUsage
|
||||
process.memoryUsage = mock(() => ({
|
||||
rss: 100 * 1024 * 1024, // 100MB
|
||||
heapTotal: 50 * 1024 * 1024, // 50MB
|
||||
heapUsed: 30 * 1024 * 1024, // 30MB
|
||||
external: 10 * 1024 * 1024, // 10MB
|
||||
arrayBuffers: 5 * 1024 * 1024, // 5MB
|
||||
}));
|
||||
|
||||
// Mock process.env
|
||||
process.env.npm_package_version = "2.1.0";
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// Restore original os functions
|
||||
os.platform = originalPlatform;
|
||||
os.version = originalVersion;
|
||||
os.arch = originalArch;
|
||||
os.totalmem = originalTotalmem;
|
||||
os.freemem = originalFreemem;
|
||||
});
|
||||
|
||||
test("returns a successful health check response", async () => {
|
||||
const response = await GET({ request: new Request("http://localhost/api/health") } as any);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Check the structure of the response
|
||||
expect(data.status).toBe("ok");
|
||||
expect(data.timestamp).toBeDefined();
|
||||
expect(data.version).toBe("2.1.0");
|
||||
|
||||
// Check database status
|
||||
expect(data.database.connected).toBe(true);
|
||||
|
||||
// Check system info
|
||||
expect(data.system.os.platform).toBe("test-platform");
|
||||
expect(data.system.os.version).toBe("test-version");
|
||||
expect(data.system.os.arch).toBe("test-arch");
|
||||
|
||||
// Check memory info
|
||||
expect(data.system.memory.rss).toBe("100 MB");
|
||||
expect(data.system.memory.heapTotal).toBe("50 MB");
|
||||
expect(data.system.memory.heapUsed).toBe("30 MB");
|
||||
expect(data.system.memory.systemTotal).toBe("16 GB");
|
||||
expect(data.system.memory.systemFree).toBe("8 GB");
|
||||
|
||||
// Check uptime
|
||||
expect(data.system.uptime.startTime).toBeDefined();
|
||||
expect(data.system.uptime.uptimeMs).toBeGreaterThanOrEqual(0);
|
||||
expect(data.system.uptime.formatted).toBeDefined();
|
||||
});
|
||||
|
||||
test("handles database connection failures", async () => {
|
||||
// Mock database failure
|
||||
mock.module("@/lib/db", () => {
|
||||
return {
|
||||
db: {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
limit: () => Promise.reject(new Error("Database connection error"))
|
||||
})
|
||||
})
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Mock console.error to prevent test output noise
|
||||
const originalConsoleError = console.error;
|
||||
console.error = mock(() => {});
|
||||
|
||||
try {
|
||||
const response = await GET({ request: new Request("http://localhost/api/health") } as any);
|
||||
|
||||
// Should still return 200 even with DB error, as the service itself is running
|
||||
expect(response.status).toBe(200);
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Status should still be ok since the service is running
|
||||
expect(data.status).toBe("ok");
|
||||
|
||||
// Database should show as disconnected
|
||||
expect(data.database.connected).toBe(false);
|
||||
expect(data.database.message).toBe("Database connection error");
|
||||
} finally {
|
||||
// Restore console.error
|
||||
console.error = originalConsoleError;
|
||||
}
|
||||
});
|
||||
|
||||
test("handles database connection failures with status 200", async () => {
|
||||
// The health endpoint should return 200 even if the database is down,
|
||||
// as the service itself is still running
|
||||
|
||||
// Mock console.error to prevent test output noise
|
||||
const originalConsoleError = console.error;
|
||||
console.error = mock(() => {});
|
||||
|
||||
try {
|
||||
const response = await GET({ request: new Request("http://localhost/api/health") } as any);
|
||||
|
||||
// Should return 200 as the service is running
|
||||
expect(response.status).toBe(200);
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Status should be ok
|
||||
expect(data.status).toBe("ok");
|
||||
|
||||
// Database should show as disconnected
|
||||
expect(data.database.connected).toBe(false);
|
||||
} finally {
|
||||
// Restore console.error
|
||||
console.error = originalConsoleError;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -34,8 +34,6 @@ export const GET: APIRoute = async ({ request }) => {
|
||||
if (isClosed) return;
|
||||
|
||||
try {
|
||||
console.log(`Polling for events for user ${userId} in channel ${channel}`);
|
||||
|
||||
// Get new events from SQLite
|
||||
const events = await getNewEvents({
|
||||
userId,
|
||||
@@ -43,8 +41,6 @@ export const GET: APIRoute = async ({ request }) => {
|
||||
lastEventTime,
|
||||
});
|
||||
|
||||
console.log(`Found ${events.length} new events`);
|
||||
|
||||
// Send events to client
|
||||
if (events.length > 0) {
|
||||
// Update last event time
|
||||
@@ -52,7 +48,6 @@ export const GET: APIRoute = async ({ request }) => {
|
||||
|
||||
// Send each event to the client
|
||||
for (const event of events) {
|
||||
console.log(`Sending event: ${JSON.stringify(event.payload)}`);
|
||||
sendMessage(`data: ${JSON.stringify(event.payload)}\n\n`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,13 @@ export interface ScheduleConfig {
|
||||
nextRun?: Date;
|
||||
}
|
||||
|
||||
export interface DatabaseCleanupConfig {
|
||||
enabled: boolean;
|
||||
retentionDays: number;
|
||||
lastRun?: Date;
|
||||
nextRun?: Date;
|
||||
}
|
||||
|
||||
export interface GitHubConfig {
|
||||
username: string;
|
||||
token: string;
|
||||
@@ -34,6 +41,7 @@ export interface SaveConfigApiRequest {
|
||||
githubConfig: GitHubConfig;
|
||||
giteaConfig: GiteaConfig;
|
||||
scheduleConfig: ScheduleConfig;
|
||||
cleanupConfig: DatabaseCleanupConfig;
|
||||
}
|
||||
|
||||
export interface SaveConfigApiResponse {
|
||||
@@ -55,6 +63,7 @@ export interface ConfigApiResponse {
|
||||
githubConfig: GitHubConfig;
|
||||
giteaConfig: GiteaConfig;
|
||||
scheduleConfig: ScheduleConfig;
|
||||
cleanupConfig: DatabaseCleanupConfig;
|
||||
include: string[];
|
||||
exclude: string[];
|
||||
createdAt: Date;
|
||||
|
||||
Reference in New Issue
Block a user