From 04e8b817d3a67d3a5c456dd8267cf340236cb1e8 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Wed, 21 May 2025 02:25:05 +0530 Subject: [PATCH] feat: add event cleanup scripts and Docker Compose setup for automated maintenance --- README.md | 26 +++++++++++++++++++++ crontab | 4 ++++ docker-compose.homelab.yml | 38 +++++++++++++++++++++++++++++++ package.json | 1 + scripts/cleanup-events.ts | 43 +++++++++++++++++++++++++++++++++++ scripts/make-events-old.ts | 29 ++++++++++++++++++++++++ scripts/mark-events-read.ts | 27 ++++++++++++++++++++++ src/lib/events.ts | 45 +++++++++++++++++++++++++++++++------ src/pages/api/sse/index.ts | 2 +- 9 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 crontab create mode 100644 docker-compose.homelab.yml create mode 100644 scripts/cleanup-events.ts create mode 100644 scripts/make-events-old.ts create mode 100644 scripts/mark-events-read.ts diff --git a/README.md b/README.md index 1d3bd39..a161749 100644 --- a/README.md +++ b/README.md @@ -445,6 +445,19 @@ Try the following steps: > -v gitea-mirror-data:/app/data \ > ghcr.io/arunavo4/gitea-mirror:latest > ``` +> +> For homelab/self-hosted setups, you can use the provided Docker Compose file with automatic event cleanup: +> +> ```bash +> # Clone the repository +> git clone https://github.com/arunavo4/gitea-mirror.git +> cd gitea-mirror +> +> # Start the application with Docker Compose +> docker-compose -f docker-compose.homelab.yml up -d +> ``` +> +> This setup includes a cron job that runs daily to clean up old events and prevent the database from growing too large. #### Database Maintenance @@ -461,6 +474,19 @@ Try the following steps: > > # Reset user accounts (for development) > bun run reset-users +> +> # Clean up old events (keeps last 7 days by default) +> bun run cleanup-events +> +> # Clean up old events with custom retention period (e.g., 30 days) +> bun run cleanup-events 30 +> ``` +> +> For automated maintenance, consider setting up a cron job to run the cleanup script periodically: +> +> ```bash +> # Add this to your crontab (runs daily at 2 AM) +> 0 2 * * * cd /path/to/gitea-mirror && bun run cleanup-events > ``` diff --git a/crontab b/crontab new file mode 100644 index 0000000..83d41f3 --- /dev/null +++ b/crontab @@ -0,0 +1,4 @@ +# Run event cleanup daily at 2 AM +0 2 * * * cd /app && bun run cleanup-events 30 >> /app/data/cleanup-events.log 2>&1 + +# Empty line at the end is required for cron to work properly diff --git a/docker-compose.homelab.yml b/docker-compose.homelab.yml new file mode 100644 index 0000000..dcef182 --- /dev/null +++ b/docker-compose.homelab.yml @@ -0,0 +1,38 @@ +version: '3.8' + +services: + gitea-mirror: + image: ghcr.io/arunavo4/gitea-mirror:latest + container_name: gitea-mirror + restart: unless-stopped + ports: + - "4321:4321" + volumes: + - gitea-mirror-data:/app/data + # Mount the crontab file + - ./crontab:/etc/cron.d/gitea-mirror-cron + environment: + - NODE_ENV=production + - HOST=0.0.0.0 + - PORT=4321 + - DATABASE_URL=sqlite://data/gitea-mirror.db + - DELAY=${DELAY:-3600} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4321/health"] + interval: 1m + timeout: 10s + retries: 3 + start_period: 30s + # Install cron in the container and set up the cron job + command: > + sh -c " + apt-get update && apt-get install -y cron curl && + chmod 0644 /etc/cron.d/gitea-mirror-cron && + crontab /etc/cron.d/gitea-mirror-cron && + service cron start && + bun dist/server/entry.mjs + " + +# Define named volumes for database persistence +volumes: + gitea-mirror-data: # Database volume diff --git a/package.json b/package.json index dc0be06..791e179 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "reset-users": "bun scripts/manage-db.ts reset-users", "migrate-db": "bun scripts/migrate-db.ts", "cleanup-redis": "bun scripts/cleanup-redis.ts", + "cleanup-events": "bun scripts/cleanup-events.ts", "preview": "bunx --bun astro preview", "start": "bun dist/server/entry.mjs", "start:fresh": "bun run cleanup-db && bun run manage-db init && bun dist/server/entry.mjs", diff --git a/scripts/cleanup-events.ts b/scripts/cleanup-events.ts new file mode 100644 index 0000000..7fafa4a --- /dev/null +++ b/scripts/cleanup-events.ts @@ -0,0 +1,43 @@ +#!/usr/bin/env bun +/** + * Script to clean up old events from the database + * This script should be run periodically (e.g., daily) to prevent the events table from growing too large + * + * Usage: + * bun scripts/cleanup-events.ts [days] + * + * Where [days] is the number of days to keep events (default: 7) + */ + +import { cleanupOldEvents } from "../src/lib/events"; + +// Parse command line arguments +const args = process.argv.slice(2); +const daysToKeep = args.length > 0 ? parseInt(args[0], 10) : 7; + +if (isNaN(daysToKeep) || daysToKeep < 1) { + console.error("Error: Days to keep must be a positive number"); + process.exit(1); +} + +async function runCleanup() { + try { + console.log(`Starting event cleanup (retention: ${daysToKeep} days)...`); + + // Call the cleanupOldEvents function from the events module + const result = await cleanupOldEvents(daysToKeep); + + console.log(`Cleanup summary:`); + console.log(`- Read events deleted: ${result.readEventsDeleted}`); + console.log(`- Unread events deleted: ${result.unreadEventsDeleted}`); + console.log(`- Total events deleted: ${result.readEventsDeleted + result.unreadEventsDeleted}`); + + console.log("Event cleanup completed successfully"); + } catch (error) { + console.error("Error running event cleanup:", error); + process.exit(1); + } +} + +// Run the cleanup +runCleanup(); diff --git a/scripts/make-events-old.ts b/scripts/make-events-old.ts new file mode 100644 index 0000000..1b3ff9a --- /dev/null +++ b/scripts/make-events-old.ts @@ -0,0 +1,29 @@ +#!/usr/bin/env bun +/** + * Script to make events appear older for testing cleanup + */ + +import { db, events } from "../src/lib/db"; + +async function makeEventsOld() { + try { + console.log("Making events appear older..."); + + // Calculate a timestamp from 2 days ago + const oldDate = new Date(); + oldDate.setDate(oldDate.getDate() - 2); + + // Update all events to have an older timestamp + const result = await db + .update(events) + .set({ createdAt: oldDate }); + + console.log(`Updated ${result.changes || 0} events to appear older`); + } catch (error) { + console.error("Error updating event timestamps:", error); + process.exit(1); + } +} + +// Run the function +makeEventsOld(); diff --git a/scripts/mark-events-read.ts b/scripts/mark-events-read.ts new file mode 100644 index 0000000..aff5454 --- /dev/null +++ b/scripts/mark-events-read.ts @@ -0,0 +1,27 @@ +#!/usr/bin/env bun +/** + * Script to mark all events as read + */ + +import { db, events } from "../src/lib/db"; +import { eq } from "drizzle-orm"; + +async function markEventsAsRead() { + try { + console.log("Marking all events as read..."); + + // Update all events to mark them as read + const result = await db + .update(events) + .set({ read: true }) + .where(eq(events.read, false)); + + console.log(`Marked ${result.changes || 0} events as read`); + } catch (error) { + console.error("Error marking events as read:", error); + process.exit(1); + } +} + +// Run the function +markEventsAsRead(); diff --git a/src/lib/events.ts b/src/lib/events.ts index 901f055..91591d3 100644 --- a/src/lib/events.ts +++ b/src/lib/events.ts @@ -1,6 +1,6 @@ import { v4 as uuidv4 } from "uuid"; import { db, events } from "./db"; -import { eq, and, gt } from "drizzle-orm"; +import { eq, and, gt, lt } from "drizzle-orm"; /** * Publishes an event to a specific channel for a user @@ -106,25 +106,56 @@ export async function getNewEvents({ /** * Cleans up old events to prevent the database from growing too large * Should be called periodically (e.g., daily via a cron job) + * + * @param maxAgeInDays Number of days to keep events (default: 7) + * @param cleanupUnreadAfterDays Number of days after which to clean up unread events (default: 2x maxAgeInDays) + * @returns Object containing the number of read and unread events deleted */ -export async function cleanupOldEvents(maxAgeInDays: number = 7): Promise { +export async function cleanupOldEvents( + maxAgeInDays: number = 7, + cleanupUnreadAfterDays?: number +): Promise<{ readEventsDeleted: number; unreadEventsDeleted: number }> { try { + console.log(`Cleaning up events older than ${maxAgeInDays} days...`); + + // Calculate the cutoff date for read events const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - maxAgeInDays); - // Delete events older than the cutoff date - const result = await db + // Delete read events older than the cutoff date + const readResult = await db .delete(events) .where( and( eq(events.read, true), - gt(cutoffDate, events.createdAt) + lt(events.createdAt, cutoffDate) ) ); - return result.changes || 0; + const readEventsDeleted = readResult.changes || 0; + console.log(`Deleted ${readEventsDeleted} read events`); + + // Calculate the cutoff date for unread events (default to 2x the retention period) + const unreadCutoffDate = new Date(); + const unreadMaxAge = cleanupUnreadAfterDays || (maxAgeInDays * 2); + unreadCutoffDate.setDate(unreadCutoffDate.getDate() - unreadMaxAge); + + // Delete unread events that are significantly older + const unreadResult = await db + .delete(events) + .where( + and( + eq(events.read, false), + lt(events.createdAt, unreadCutoffDate) + ) + ); + + const unreadEventsDeleted = unreadResult.changes || 0; + console.log(`Deleted ${unreadEventsDeleted} unread events`); + + return { readEventsDeleted, unreadEventsDeleted }; } catch (error) { console.error("Error cleaning up old events:", error); - return 0; + return { readEventsDeleted: 0, unreadEventsDeleted: 0 }; } } diff --git a/src/pages/api/sse/index.ts b/src/pages/api/sse/index.ts index bd0e801..c029e5f 100644 --- a/src/pages/api/sse/index.ts +++ b/src/pages/api/sse/index.ts @@ -11,7 +11,7 @@ export const GET: APIRoute = async ({ request }) => { const channel = `mirror-status:${userId}`; let isClosed = false; - const POLL_INTERVAL = 2000; // Poll every 2 seconds + const POLL_INTERVAL = 5000; // Poll every 5 seconds (reduced from 2 seconds for low-traffic usage) const stream = new ReadableStream({ start(controller) {