Compare commits

...

51 Commits

Author SHA1 Message Date
Arunavo Ray
20a771f340 chore: bump version to 2.8.0 2025-05-24 18:49:30 +05:30
Arunavo Ray
d925b3c155 refactor: Remove unnecessary console logs from event polling and retrieval 2025-05-24 18:41:55 +05:30
Arunavo Ray
47e1c7b493 feat: Implement automatic database cleanup feature with configuration options and API support 2025-05-24 18:33:59 +05:30
Arunavo Ray
d7ce2a6908 feat: Refactor database cleanup process by removing scripts and updating documentation to use the Activity Log for event management 2025-05-24 17:58:37 +05:30
Arunavo Ray
4efe741c64 Bump version to 2.7.0 2025-05-24 17:39:23 +05:30
Arunavo Ray
773842fa72 feat: Improve cron job setup for automatic database cleanup with better error handling 2025-05-24 17:37:36 +05:30
Arunavo Ray
90944a40c6 feat: Remove health API tests to streamline codebase 2025-05-24 16:14:22 +05:30
Arunavo Ray
f436737efb Bump version to 2.6.0 2025-05-24 15:41:12 +05:30
Arunavo Ray
a4e4afdaaf feat: Enhance RepositoryTable layout by adding Links column and adjusting action button alignment 2025-05-24 15:32:59 +05:30
Arunavo Ray
cbef04d4b4 feat: Add Gitea configuration hook and enhance repository list with Gitea links 2025-05-24 15:18:31 +05:30
Arunavo Ray
a988be1028 feat: Implement comprehensive job recovery and resume process improvements
- Added a startup recovery script to handle interrupted jobs before application startup.
- Enhanced recovery system with database connection validation and stale job cleanup.
- Improved middleware to check for recovery needs and handle recovery during requests.
- Updated health check endpoint to include recovery system status and metrics.
- Introduced test scripts for verifying recovery functionality and job state management.
- Enhanced logging and error handling throughout the recovery process.
2025-05-24 13:45:25 +05:30
Arunavo Ray
98610482ae feat: enhance event management by adding duplicate removal, cleanup functionality, and improving activity logging 2025-05-24 13:25:58 +05:30
Arunavo Ray
546db472e5 feat: enhance navigation handling by updating navigation key and improving page state management 2025-05-24 12:52:02 +05:30
Arunavo Ray
70b3e412ad feat: implement navigation context and enhance component loading states across the application 2025-05-24 12:51:57 +05:30
Arunavo Ray
a3ac31795c refactor: remove problematic useEffect to prevent circular dependencies and optimize user data fetching 2025-05-24 12:24:04 +05:30
Arunavo Ray
f41fb9b91f feat: adjust height of RepositoryTable and add status bar to display filtered repository count 2025-05-24 11:37:05 +05:30
Arunavo Ray
0b568a3b37 feat: implement auto-save functionality for schedule config changes and enhance UI with loading indicator 2025-05-24 11:31:40 +05:30
Arunavo Ray
a1da82a718 refactor: simplify ConfigCardSkeleton structure and enhance layout in ConfigTabs component 2025-05-24 10:53:33 +05:30
Arunavo Ray
645d495e80 refactor: remove Docker configuration generation and clipboard copy functionality from ConfigTabs component 2025-05-24 10:48:43 +05:30
Arunavo Ray
0890ed0bb8 feat: add live refresh functionality and configuration status hooks; enhance UI components with new switch and refresh features 2025-05-24 10:24:25 +05:30
Arunavo Ray
fc985f29df fix: update base image version in Dockerfile and remove cron installation 2025-05-23 16:07:42 +05:30
Arunavo Ray
7d32112369 feat: implement automatic database cleanup with cron jobs for events and mirror jobs 2025-05-23 12:15:34 +05:30
Arunavo Ray
3bb85a4cdb chore: bump version to 2.5.4 2025-05-23 11:43:36 +05:30
Arunavo Ray
30182544ba feat: enhance ActivityLog and ActivityList components with key generation and filtering improvements 2025-05-23 11:29:39 +05:30
Arunavo Ray
fb73f33aeb fix: update healthcheck endpoints for Gitea services in docker-compose files 2025-05-22 22:48:33 +05:30
Arunavo Ray
48f63bdfc8 Release v2.5.3 2025-05-22 21:59:21 +05:30
Arunavo Ray
e2506a874e feat: enhance JWT_SECRET handling with auto-generation and persistence 2025-05-22 20:58:22 +05:30
Arunavo Ray
b67473ec7e refactor: update Proxmox LXC deployment instructions and replace deprecated script 2025-05-22 20:35:18 +05:30
Arunavo Ray
4ca4356ad1 Release v2.5.2 2025-05-22 19:54:07 +05:30
Arunavo Ray
3136a2120d feat: extract version from package.json and set as environment variable 2025-05-22 19:48:42 +05:30
Arunavo Ray
615ebd5079 Release 2.5.1: Fix Docker entrypoint to prevent unnecessary bun install 2025-05-22 19:33:07 +05:30
Arunavo Ray
6e48d3f86c refactor: remove bun setup for pre-built images and streamline database initialization scripts 2025-05-22 19:29:15 +05:30
Arunavo Ray
c5de7e616d chore: bump version to 2.5.0 2025-05-22 18:53:43 +05:30
Arunavo Ray
309f8c4341 feat: add version information component and integrate version check in health API 2025-05-22 18:51:11 +05:30
ARUNAVO RAY
0c596ac241 Merge pull request #15 from arunavo4/parallel-jobs
Implement parallel processing
2025-05-22 18:18:14 +05:30
Arunavo Ray
894be88a28 feat: migrate testing framework to Bun and update test configurations
- Updated GitHub Actions workflow to use Bun's test runner and coverage reporting.
- Added comprehensive testing documentation for the Gitea Mirror project.
- Refactored test scripts in package.json to align with Bun's testing commands.
- Created new test files for database, Gitea, GitHub, health, and mirroring APIs.
- Implemented mock functions for API tests to handle various scenarios and responses.
- Established a test setup file for consistent test environment configuration.
2025-05-22 18:08:51 +05:30
Arunavo Ray
6ab7f0a5a0 fix: add missing --bun flag to vitest and astro build commands 2025-05-22 16:44:14 +05:30
Arunavo Ray
abe3113755 feat: enhance job resilience with new database schema and recovery mechanisms
- Added new fields to the mirror_jobs table for job resilience, including job_type, batch_id, total_items, completed_items, item_ids, completed_item_ids, in_progress, started_at, completed_at, and last_checkpoint.
- Implemented database migration scripts to update the mirror_jobs table schema.
- Introduced processWithResilience utility for handling item processing with checkpointing and recovery capabilities.
- Updated API routes for mirroring organizations and repositories to utilize the new resilience features.
- Created recovery system to detect and resume interrupted jobs on application startup.
- Added middleware to initialize the recovery system when the server starts.
2025-05-22 14:33:03 +05:30
Arunavo Ray
f4bc28e6c2 Implement parallel processing with retry logic for repository mirroring and syncing operations 2025-05-22 13:28:28 +05:30
Arunavo Ray
aaf8dc6fe4 Bump version to 2.1.0 2025-05-22 11:05:07 +05:30
ARUNAVO RAY
cda78bc0f5 Merge pull request #13 from arunavo4/fix-bun-permissions-in-lxc
Fix Bun permissions issue in LXC container installer
2025-05-22 09:15:49 +05:30
Arunavo Ray
9ccd656734 Update documentation: revise architecture, configuration, and quick start guides to reflect new features and deployment options 2025-05-22 09:12:32 +05:30
Arunavo Ray
8b5c5d8ed2 Update README to include event management scripts and LXC deployment details 2025-05-22 09:06:47 +05:30
Arunavo Ray
1ab642c9e7 Update LXC deployment scripts: replace installer script with Proxmox-specific script and update README references 2025-05-22 09:04:05 +05:30
Arunavo Ray
1eae725535 Update LXC deployment guide references and remove outdated documentation 2025-05-22 08:56:53 +05:30
Arunavo Ray
5bf52c806f Update README and add LXC deployment guide; enhance LXC installer scripts 2025-05-22 08:53:19 +05:30
Arunavo Ray
a15178d2cd Implement health check API and update health check commands in Docker and Docker Compose 2025-05-22 08:39:07 +05:30
Arunavo Ray
32ef9124a7 Remove sqlite3 dependency from package.json 2025-05-22 08:12:26 +05:30
Arunavo Ray
161685b966 Add directory permission check before creating symlink in systemd service setup 2025-05-21 22:30:15 +05:30
Arunavo Ray
0cf95b2a0e Improve error handling and permission checks in LXC installer 2025-05-21 22:26:50 +05:30
Arunavo Ray
c896194aeb Fix Bun permissions issue in LXC container installer 2025-05-21 22:19:43 +05:30
80 changed files with 6813 additions and 1539 deletions

View File

@@ -30,3 +30,9 @@ JWT_SECRET=change-this-to-a-secure-random-string-in-production
# GITEA_ORGANIZATION=github-mirrors
# GITEA_ORG_VISIBILITY=public
# DELAY=3600
# Optional Database Cleanup Configuration (configured via web UI)
# These environment variables are optional and only used as defaults
# Users can configure cleanup settings through the web interface
# CLEANUP_ENABLED=false
# CLEANUP_RETENTION_DAYS=7

View File

@@ -38,10 +38,10 @@ jobs:
bun install
- name: Run tests
run: bunx vitest run
run: bun test --coverage
- name: Build Astro project
run: bunx astro build
run: bunx --bun astro build
- name: Upload build artifacts
uses: actions/upload-artifact@v4

42
CHANGELOG.md Normal file
View File

@@ -0,0 +1,42 @@
# Changelog
All notable changes to the Gitea Mirror project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [2.5.3] - 2025-05-22
### Added
- Enhanced JWT_SECRET handling with auto-generation and persistence for improved security
- Updated Proxmox LXC deployment instructions and replaced deprecated script
## [2.5.2] - 2024-11-22
### Fixed
- Fixed version information in health API for Docker deployments by setting npm_package_version environment variable in entrypoint script
## [2.5.1] - 2024-10-01
### Fixed
- Fixed Docker entrypoint script to prevent unnecessary `bun install` on container startup
- Removed redundant dependency installation in Docker containers for pre-built images
- Fixed "PathAlreadyExists" errors during container initialization
### Changed
- Improved database initialization in Docker entrypoint script
- Added additional checks for TypeScript versions of database management scripts
## [2.5.0] - 2024-09-15
Initial public release with core functionality:
### Added
- GitHub to Gitea repository mirroring
- User authentication and management
- Dashboard with mirroring statistics
- Configuration management for mirroring settings
- Support for organization mirroring
- Automated mirroring with configurable schedules
- Docker multi-architecture support (amd64, arm64)
- LXC container deployment scripts

View File

@@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.4
FROM oven/bun:1.2.9-alpine AS base
FROM oven/bun:1.2.14-alpine AS base
WORKDIR /app
RUN apk add --no-cache libc6-compat python3 make g++ gcc wget sqlite
RUN apk add --no-cache libc6-compat python3 make g++ gcc wget sqlite openssl
# ----------------------------
FROM base AS deps
@@ -49,6 +49,6 @@ VOLUME /app/data
EXPOSE 4321
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://localhost:4321/ || exit 1
CMD wget --no-verbose --tries=1 --spider http://localhost:4321/api/health || exit 1
ENTRYPOINT ["./docker-entrypoint.sh"]

View File

@@ -14,13 +14,17 @@
```bash
# Using Docker (recommended)
docker compose --profile production up -d
docker compose up -d
# Using Bun
bun run setup && bun run dev
# Using LXC on Proxmox
curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-installer.sh | bash
# Using LXC Containers
# For Proxmox VE (online) - Community script by Tobias ([CrazyWolf13](https://github.com/CrazyWolf13))
curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/install/gitea-mirror-install.sh | bash
# For local testing (offline-friendly)
sudo LOCAL_REPO_DIR=~/Development/gitea-mirror ./scripts/gitea-mirror-lxc-local.sh
````
See the [LXC Container Deployment Guide](scripts/README-lxc.md).
@@ -111,7 +115,7 @@ Gitea Mirror provides multi-architecture Docker images that work on both ARM64 (
```bash
# Start the application using Docker Compose
docker compose --profile production up -d
docker compose up -d
# For development mode (requires configuration)
# Ensure you have run bun run setup first
@@ -158,26 +162,43 @@ cp .env.example .env
./scripts/build-docker.sh --push
# Then run with Docker Compose
docker compose --profile production up -d
docker compose up -d
```
See [Docker build documentation](./scripts/README-docker.md) for more details.
##### Using LXC Containers (for Proxmox Homelab Setups)
##### Using LXC Containers
Gitea Mirror can be deployed on Proxmox LXC containers, which is ideal for homelab setups:
Gitea Mirror offers two deployment options for LXC containers:
**1. Proxmox VE (online, recommended for production)**
```bash
# One-command installation on an Ubuntu 22.04 LXC container
curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-installer.sh | bash
# One-command installation on Proxmox VE
# Uses the community-maintained script by Tobias ([CrazyWolf13](https://github.com/CrazyWolf13))
# at [community-scripts/ProxmoxVED](https://github.com/community-scripts/ProxmoxVED)
curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/install/gitea-mirror-install.sh | bash
```
The installer script:
- Downloads the Gitea Mirror repository
- Installs all dependencies including Bun
- Builds the application
- Sets up a systemd service
- Starts the application
**2. Local testing (offline-friendly, works on developer laptops)**
```bash
# Download the script
curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-local.sh -o gitea-mirror-lxc-local.sh
chmod +x gitea-mirror-lxc-local.sh
# Run with your local repo directory
sudo LOCAL_REPO_DIR=~/Development/gitea-mirror ./gitea-mirror-lxc-local.sh
```
Both scripts:
- Set up a privileged Ubuntu 22.04 LXC container
- Install Bun runtime environment
- Build the application
- Configure a systemd service
- Start the service automatically
The application includes a health check endpoint at `/api/health` for monitoring.
See the [LXC Container Deployment Guide](scripts/README-lxc.md) for detailed instructions.
@@ -212,8 +233,10 @@ The Docker container can be configured with the following environment variables:
- `DATABASE_URL`: SQLite database URL (default: `file:data/gitea-mirror.db`)
- `HOST`: Host to bind to (default: `0.0.0.0`)
- `PORT`: Port to listen on (default: `4321`)
- `JWT_SECRET`: Secret key for JWT token generation (important for security)
- `JWT_SECRET`: Secret key for JWT token generation (auto-generated if not provided)
> [!TIP]
> For security, Gitea Mirror will automatically generate a secure random JWT secret on first run if one isn't provided or if the default value is used. This generated secret is stored in the data directory for persistence across container restarts.
#### Manual Installation
@@ -379,7 +402,7 @@ docker compose -f docker-compose.dev.yml up -d
- **Backend**: Bun
- **Database**: SQLite (handles both data storage and event notifications)
- **API Integration**: GitHub API (Octokit), Gitea API
- **Deployment Options**: Docker containers, Proxmox LXC containers
- **Deployment Options**: Docker containers, LXC containers (Proxmox VE and local testing)
## Contributing
@@ -447,7 +470,7 @@ Try the following steps:
> ghcr.io/arunavo4/gitea-mirror:latest
> ```
>
> For homelab/self-hosted setups, you can use the provided Docker Compose file with automatic event cleanup:
> For homelab/self-hosted setups, you can use the standard Docker Compose file which includes automatic database cleanup:
>
> ```bash
> # Clone the repository
@@ -455,10 +478,10 @@ Try the following steps:
> cd gitea-mirror
>
> # Start the application with Docker Compose
> docker-compose -f docker-compose.homelab.yml up -d
> docker compose up -d
> ```
>
> This setup includes a cron job that runs daily to clean up old events and prevent the database from growing too large.
> This setup provides a complete containerized deployment for the Gitea Mirror application.
#### Database Maintenance
@@ -475,20 +498,9 @@ Try the following steps:
>
> # Reset user accounts (for development)
> bun run reset-users
>
> # Clean up old events (keeps last 7 days by default)
> bun run cleanup-events
>
> # Clean up old events with custom retention period (e.g., 30 days)
> bun run cleanup-events 30
> ```
>
> For automated maintenance, consider setting up a cron job to run the cleanup script periodically:
>
> ```bash
> # Add this to your crontab (runs daily at 2 AM)
> 0 2 * * * cd /path/to/gitea-mirror && bun run cleanup-events
> ```
> **Note:** For cleaning up old activities and events, use the cleanup button in the Activity Log page of the web interface.
> [!NOTE]

102
bun.lock
View File

@@ -17,6 +17,7 @@
"@radix-ui/react-radio-group": "^1.3.6",
"@radix-ui/react-select": "^2.2.4",
"@radix-ui/react-slot": "^1.2.2",
"@radix-ui/react-switch": "^1.2.5",
"@radix-ui/react-tabs": "^1.1.11",
"@radix-ui/react-tooltip": "^1.2.6",
"@tailwindcss/vite": "^4.1.7",
@@ -40,7 +41,6 @@
"react-dom": "^19.1.0",
"react-icons": "^5.5.0",
"sonner": "^2.0.3",
"sqlite3": "^5.1.7",
"superagent": "^10.2.1",
"tailwind-merge": "^3.3.0",
"tailwindcss": "^4.1.7",
@@ -335,6 +335,8 @@
"@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.2", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-y7TBO4xN4Y94FvcWIOIh18fM4R1A8S4q1jhoz4PNzOoHsFcN8pogcFmZrTYAm4F9VRUrWP/Mw7xSKybIeRI+CQ=="],
"@radix-ui/react-switch": ["@radix-ui/react-switch@1.2.5", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-5ijLkak6ZMylXsaImpZ8u4Rlf5grRmoc0p0QeX9VJtlrM4f5m3nCTX8tWga/zOA8PZYIR/t0p2Mnvd7InrJ6yQ=="],
"@radix-ui/react-tabs": ["@radix-ui/react-tabs@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.2", "@radix-ui/react-roving-focus": "1.1.9", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-4FiKSVoXqPP/KfzlB7lwwqoFV6EPwkrrqGp9cUYXjwDYHhvpnqq79P+EPHKcdoTE7Rl8w/+6s9rTlsfXHES9GA=="],
"@radix-ui/react-tooltip": ["@radix-ui/react-tooltip@1.2.6", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.9", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.6", "@radix-ui/react-portal": "1.1.8", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.2", "@radix-ui/react-slot": "1.2.2", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-visually-hidden": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-zYb+9dc9tkoN2JjBDIIPLQtk3gGyz8FMKoqYTb8EMVQ5a5hBcdHPECrsZVI4NpPAUOixhkoqg7Hj5ry5USowfA=="],
@@ -637,7 +639,7 @@
"chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="],
"chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="],
"chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
"ci-info": ["ci-info@4.2.0", "", {}, "sha512-cYY9mypksY8NRqgDB1XD1RiJL338v/551niynFTGkZOO2LHuB2OmOYxDIe/ttN9AHwrqdum1360G3ald0W9kCg=="],
@@ -1187,7 +1189,7 @@
"minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],
"minipass": ["minipass@5.0.0", "", {}, "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ=="],
"minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="],
"minipass-collect": ["minipass-collect@1.0.2", "", { "dependencies": { "minipass": "^3.0.0" } }, "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA=="],
@@ -1199,9 +1201,9 @@
"minipass-sized": ["minipass-sized@1.0.3", "", { "dependencies": { "minipass": "^3.0.0" } }, "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g=="],
"minizlib": ["minizlib@2.1.2", "", { "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" } }, "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg=="],
"minizlib": ["minizlib@3.0.2", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA=="],
"mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
"mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="],
"mkdirp-classic": ["mkdirp-classic@0.5.3", "", {}, "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="],
@@ -1491,7 +1493,7 @@
"tapable": ["tapable@2.2.2", "", {}, "sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg=="],
"tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="],
"tar": ["tar@7.4.3", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" } }, "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw=="],
"tar-fs": ["tar-fs@2.1.2", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA=="],
@@ -1643,7 +1645,7 @@
"xxhash-wasm": ["xxhash-wasm@1.1.0", "", {}, "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA=="],
"yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="],
"yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="],
@@ -1667,15 +1669,15 @@
"@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
"@isaacs/fs-minipass/minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="],
"@npmcli/move-file/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
"@octokit/plugin-paginate-rest/@octokit/types": ["@octokit/types@13.10.0", "", { "dependencies": { "@octokit/openapi-types": "^24.2.0" } }, "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA=="],
"@octokit/plugin-rest-endpoint-methods/@octokit/types": ["@octokit/types@13.10.0", "", { "dependencies": { "@octokit/openapi-types": "^24.2.0" } }, "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA=="],
"@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="],
"@radix-ui/react-switch/@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.3", "", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="],
"@tailwindcss/oxide/tar": ["tar@7.4.3", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" } }, "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw=="],
"@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="],
"@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.4.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.0.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g=="],
@@ -1701,10 +1703,16 @@
"boxen/chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="],
"cacache/chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="],
"cacache/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="],
"cacache/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"cacache/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
"cacache/tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="],
"fs-minipass/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"gauge/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
@@ -1725,16 +1733,18 @@
"minipass-fetch/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"minipass-fetch/minizlib": ["minizlib@2.1.2", "", { "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" } }, "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg=="],
"minipass-flush/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"minipass-pipeline/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"minipass-sized/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"node-fetch/whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
"node-gyp/tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="],
"parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="],
"pretty-format/ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="],
@@ -1745,6 +1755,8 @@
"socks-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="],
"sqlite3/tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="],
"ssri/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"strip-ansi/ansi-regex": ["ansi-regex@6.1.0", "", {}, "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA=="],
@@ -1761,34 +1773,82 @@
"@octokit/plugin-rest-endpoint-methods/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@24.2.0", "", {}, "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg=="],
"@tailwindcss/oxide/tar/chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
"@tailwindcss/oxide/tar/minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="],
"@tailwindcss/oxide/tar/minizlib": ["minizlib@3.0.2", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA=="],
"@tailwindcss/oxide/tar/mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="],
"@tailwindcss/oxide/tar/yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="],
"@radix-ui/react-switch/@radix-ui/react-primitive/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="],
"ansi-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
"ansi-align/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
"cacache/lru-cache/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"cacache/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"cacache/tar/minipass": ["minipass@5.0.0", "", {}, "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ=="],
"cacache/tar/minizlib": ["minizlib@2.1.2", "", { "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" } }, "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg=="],
"cacache/tar/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"fs-minipass/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"gauge/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
"make-fetch-happen/http-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="],
"make-fetch-happen/https-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="],
"make-fetch-happen/lru-cache/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"make-fetch-happen/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"minipass-collect/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"minipass-fetch/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"minipass-fetch/minizlib/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"minipass-flush/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"minipass-pipeline/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"minipass-sized/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
"node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
"node-gyp/tar/chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="],
"node-gyp/tar/minipass": ["minipass@5.0.0", "", {}, "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ=="],
"node-gyp/tar/minizlib": ["minizlib@2.1.2", "", { "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" } }, "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg=="],
"node-gyp/tar/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
"node-gyp/tar/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"send/mime-types/mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="],
"sqlite3/tar/chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="],
"sqlite3/tar/minipass": ["minipass@5.0.0", "", {}, "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ=="],
"sqlite3/tar/minizlib": ["minizlib@2.1.2", "", { "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" } }, "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg=="],
"sqlite3/tar/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
"sqlite3/tar/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"ssri/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],
"wide-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
"wide-align/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
"cacache/tar/minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"node-gyp/tar/minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
"sqlite3/tar/minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
}
}

View File

@@ -1,4 +0,0 @@
# Run event cleanup daily at 2 AM
0 2 * * * cd /app && bun run cleanup-events 30 >> /app/data/cleanup-events.log 2>&1
# Empty line at the end is required for cron to work properly

View File

@@ -28,7 +28,7 @@ services:
networks:
- gitea-network
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/"]
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/api/healthz"]
interval: 30s
timeout: 5s
retries: 3
@@ -75,7 +75,7 @@ services:
- GITEA_ORG_VISIBILITY=${GITEA_ORG_VISIBILITY:-public}
- DELAY=${DELAY:-3600}
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:4321/"]
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:4321/api/health"]
interval: 30s
timeout: 5s
retries: 3

View File

@@ -1,38 +0,0 @@
version: '3.8'
services:
gitea-mirror:
image: ghcr.io/arunavo4/gitea-mirror:latest
container_name: gitea-mirror
restart: unless-stopped
ports:
- "4321:4321"
volumes:
- gitea-mirror-data:/app/data
# Mount the crontab file
- ./crontab:/etc/cron.d/gitea-mirror-cron
environment:
- NODE_ENV=production
- HOST=0.0.0.0
- PORT=4321
- DATABASE_URL=sqlite://data/gitea-mirror.db
- DELAY=${DELAY:-3600}
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:4321/health"]
interval: 1m
timeout: 10s
retries: 3
start_period: 30s
# Install cron in the container and set up the cron job
command: >
sh -c "
apt-get update && apt-get install -y cron curl &&
chmod 0644 /etc/cron.d/gitea-mirror-cron &&
crontab /etc/cron.d/gitea-mirror-cron &&
service cron start &&
bun dist/server/entry.mjs
"
# Define named volumes for database persistence
volumes:
gitea-mirror-data: # Database volume

View File

@@ -1,8 +1,7 @@
# Gitea Mirror deployment configuration
# - production: Standard deployment with real data
# Standard deployment with automatic database maintenance
services:
# Production service with real data
gitea-mirror:
image: ${DOCKER_REGISTRY:-ghcr.io}/${DOCKER_IMAGE:-arunavo4/gitea-mirror}:${DOCKER_TAG:-latest}
build:
@@ -43,12 +42,11 @@ services:
- GITEA_ORG_VISIBILITY=${GITEA_ORG_VISIBILITY:-public}
- DELAY=${DELAY:-3600}
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/"]
test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:4321/api/health"]
interval: 30s
timeout: 10s
retries: 5
start_period: 15s
profiles: ["production"]
# Define named volumes for database persistence
volumes:

View File

@@ -5,12 +5,36 @@ set -e
# Ensure data directory exists
mkdir -p /app/data
# If bun is available, run setup (for dev images)
if command -v bun >/dev/null 2>&1; then
echo "Running bun setup (if needed)..."
bun run setup || true
# Generate a secure JWT secret if one isn't provided or is using the default value
JWT_SECRET_FILE="/app/data/.jwt_secret"
if [ "$JWT_SECRET" = "your-secret-key-change-this-in-production" ] || [ -z "$JWT_SECRET" ]; then
# Check if we have a previously generated secret
if [ -f "$JWT_SECRET_FILE" ]; then
echo "Using previously generated JWT secret"
export JWT_SECRET=$(cat "$JWT_SECRET_FILE")
else
echo "Generating a secure random JWT secret"
# Try to generate a secure random string using OpenSSL
if command -v openssl >/dev/null 2>&1; then
GENERATED_SECRET=$(openssl rand -hex 32)
else
# Fallback to using /dev/urandom if openssl is not available
echo "OpenSSL not found, using fallback method for random generation"
GENERATED_SECRET=$(head -c 32 /dev/urandom | sha256sum | cut -d' ' -f1)
fi
export JWT_SECRET="$GENERATED_SECRET"
# Save the secret to a file for persistence across container restarts
echo "$GENERATED_SECRET" > "$JWT_SECRET_FILE"
chmod 600 "$JWT_SECRET_FILE"
fi
echo "JWT_SECRET has been set to a secure random value"
fi
# Skip dependency installation entirely for pre-built images
# Dependencies are already installed during the Docker build process
# Initialize the database if it doesn't exist
if [ ! -f "/app/data/gitea-mirror.db" ]; then
echo "Initializing database..."
@@ -18,6 +42,8 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then
bun dist/scripts/init-db.js
elif [ -f "dist/scripts/manage-db.js" ]; then
bun dist/scripts/manage-db.js init
elif [ -f "scripts/manage-db.ts" ]; then
bun scripts/manage-db.ts init
else
echo "Warning: Could not find database initialization scripts in dist/scripts."
echo "Creating and initializing database manually..."
@@ -111,9 +137,28 @@ if [ ! -f "/app/data/gitea-mirror.db" ]; then
status TEXT NOT NULL DEFAULT 'imported',
message TEXT NOT NULL,
timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- New fields for job resilience
job_type TEXT NOT NULL DEFAULT 'mirror',
batch_id TEXT,
total_items INTEGER,
completed_items INTEGER DEFAULT 0,
item_ids TEXT, -- JSON array as text
completed_item_ids TEXT DEFAULT '[]', -- JSON array as text
in_progress INTEGER NOT NULL DEFAULT 0, -- Boolean as integer
started_at TIMESTAMP,
completed_at TIMESTAMP,
last_checkpoint TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_user_id ON mirror_jobs(user_id);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_batch_id ON mirror_jobs(batch_id);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_in_progress ON mirror_jobs(in_progress);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_job_type ON mirror_jobs(job_type);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_timestamp ON mirror_jobs(timestamp);
CREATE TABLE IF NOT EXISTS events (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
@@ -136,10 +181,55 @@ else
bun dist/scripts/fix-db-issues.js
elif [ -f "dist/scripts/manage-db.js" ]; then
bun dist/scripts/manage-db.js fix
elif [ -f "scripts/manage-db.ts" ]; then
bun scripts/manage-db.ts fix
fi
# Since the application is not used by anyone yet, we've removed the schema updates and migrations
echo "Database already exists, no migrations needed."
# Run database migrations
echo "Running database migrations..."
# Update mirror_jobs table with new columns for resilience
if [ -f "dist/scripts/update-mirror-jobs-table.js" ]; then
echo "Updating mirror_jobs table..."
bun dist/scripts/update-mirror-jobs-table.js
elif [ -f "scripts/update-mirror-jobs-table.ts" ]; then
echo "Updating mirror_jobs table using TypeScript script..."
bun scripts/update-mirror-jobs-table.ts
else
echo "Warning: Could not find mirror_jobs table update script."
fi
fi
# Extract version from package.json and set as environment variable
if [ -f "package.json" ]; then
export npm_package_version=$(grep -o '"version": *"[^"]*"' package.json | cut -d'"' -f4)
echo "Setting application version: $npm_package_version"
fi
# Run startup recovery to handle any interrupted jobs
echo "Running startup recovery..."
if [ -f "dist/scripts/startup-recovery.js" ]; then
echo "Running startup recovery using compiled script..."
bun dist/scripts/startup-recovery.js --timeout=30000
RECOVERY_EXIT_CODE=$?
elif [ -f "scripts/startup-recovery.ts" ]; then
echo "Running startup recovery using TypeScript script..."
bun scripts/startup-recovery.ts --timeout=30000
RECOVERY_EXIT_CODE=$?
else
echo "Warning: Startup recovery script not found. Skipping recovery."
RECOVERY_EXIT_CODE=0
fi
# Log recovery result
if [ $RECOVERY_EXIT_CODE -eq 0 ]; then
echo "✅ Startup recovery completed successfully"
elif [ $RECOVERY_EXIT_CODE -eq 1 ]; then
echo "⚠️ Startup recovery completed with warnings"
else
echo "❌ Startup recovery failed with exit code $RECOVERY_EXIT_CODE"
fi
# Start the application

View File

@@ -0,0 +1,170 @@
# Job Recovery and Resume Process Improvements
This document outlines the comprehensive improvements made to the job recovery and resume process to make it more robust to application restarts, container restarts, and application crashes.
## Problems Addressed
The original recovery system had several critical issues:
1. **Middleware-based initialization**: Recovery only ran when the first request came in
2. **Database connection issues**: No validation of database connectivity before recovery attempts
3. **Limited error handling**: Insufficient error handling for various failure scenarios
4. **No startup recovery**: No mechanism to handle recovery before serving requests
5. **Incomplete job state management**: Jobs could remain in inconsistent states
6. **No retry mechanisms**: Single-attempt recovery with no fallback strategies
## Improvements Implemented
### 1. Enhanced Recovery System (`src/lib/recovery.ts`)
#### New Features:
- **Database connection validation** before attempting recovery
- **Stale job cleanup** for jobs older than 24 hours
- **Retry mechanisms** with configurable attempts and delays
- **Individual job error handling** to prevent one failed job from stopping recovery
- **Recovery state tracking** to prevent concurrent recovery attempts
- **Enhanced logging** with detailed job information
#### Key Functions:
- `initializeRecovery()` - Main recovery function with enhanced error handling
- `validateDatabaseConnection()` - Ensures database is accessible
- `cleanupStaleJobs()` - Removes jobs that are too old to recover
- `getRecoveryStatus()` - Returns current recovery system status
- `forceRecovery()` - Bypasses recent attempt checks
- `hasJobsNeedingRecovery()` - Checks if recovery is needed
### 2. Startup Recovery Script (`scripts/startup-recovery.ts`)
A dedicated script that runs recovery before the application starts serving requests:
#### Features:
- **Timeout protection** (default: 30 seconds)
- **Force recovery option** to bypass recent attempt checks
- **Graceful signal handling** (SIGINT, SIGTERM)
- **Detailed logging** with progress indicators
- **Exit codes** for different scenarios (success, warnings, errors)
#### Usage:
```bash
bun scripts/startup-recovery.ts [--force] [--timeout=30000]
```
### 3. Improved Middleware (`src/middleware.ts`)
The middleware now serves as a fallback recovery mechanism:
#### Changes:
- **Checks if recovery is needed** before attempting
- **Shorter timeout** (15 seconds) for request-time recovery
- **Better error handling** with status logging
- **Prevents multiple attempts** with proper state tracking
### 4. Enhanced Database Queries (`src/lib/helpers.ts`)
#### Improvements:
- **Proper Drizzle ORM syntax** for all database queries
- **Enhanced interrupted job detection** with multiple criteria:
- Jobs with no recent checkpoint (10+ minutes)
- Jobs running too long (2+ hours)
- **Detailed logging** of found interrupted jobs
- **Better error handling** for database operations
### 5. Docker Integration (`docker-entrypoint.sh`)
#### Changes:
- **Automatic startup recovery** runs before application start
- **Exit code handling** with appropriate logging
- **Fallback mechanisms** if recovery script is not found
- **Non-blocking execution** - application starts even if recovery fails
### 6. Health Check Integration (`src/pages/api/health.ts`)
#### New Features:
- **Recovery system status** in health endpoint
- **Job recovery metrics** (jobs needing recovery, recovery in progress)
- **Overall health status** considers recovery state
- **Detailed recovery information** for monitoring
### 7. Testing Infrastructure (`scripts/test-recovery.ts`)
A comprehensive test script to verify recovery functionality:
#### Features:
- **Creates test interrupted jobs** with realistic scenarios
- **Verifies recovery detection** and execution
- **Checks final job states** after recovery
- **Cleanup functionality** for test data
- **Comprehensive logging** of test progress
## Configuration Options
### Recovery System Options:
- `maxRetries`: Number of recovery attempts (default: 3)
- `retryDelay`: Delay between attempts in ms (default: 5000)
- `skipIfRecentAttempt`: Skip if recent attempt made (default: true)
### Startup Recovery Options:
- `--force`: Force recovery even if recent attempt was made
- `--timeout`: Maximum time to wait for recovery (default: 30000ms)
## Usage Examples
### Manual Recovery:
```bash
# Run startup recovery
bun run startup-recovery
# Force recovery
bun run startup-recovery-force
# Test recovery system
bun run test-recovery
# Clean up test data
bun run test-recovery-cleanup
```
### Programmatic Usage:
```typescript
import { initializeRecovery, hasJobsNeedingRecovery } from '@/lib/recovery';
// Check if recovery is needed
const needsRecovery = await hasJobsNeedingRecovery();
// Run recovery with custom options
const success = await initializeRecovery({
maxRetries: 5,
retryDelay: 3000,
skipIfRecentAttempt: false
});
```
## Monitoring and Observability
### Health Check Endpoint:
- **URL**: `/api/health`
- **Recovery Status**: Included in response
- **Monitoring**: Can be used with external monitoring systems
### Log Messages:
- **Startup**: Clear indicators of recovery attempts and results
- **Progress**: Detailed logging of recovery steps
- **Errors**: Comprehensive error information for debugging
## Benefits
1. **Reliability**: Jobs are automatically recovered after application restarts
2. **Resilience**: Multiple retry mechanisms and fallback strategies
3. **Observability**: Comprehensive logging and health check integration
4. **Performance**: Efficient detection and processing of interrupted jobs
5. **Maintainability**: Clear separation of concerns and modular design
6. **Testing**: Built-in testing infrastructure for verification
## Migration Notes
- **Backward Compatible**: All existing functionality is preserved
- **Automatic**: Recovery runs automatically on startup
- **Configurable**: All timeouts and retry counts can be adjusted
- **Monitoring**: Health checks now include recovery status
This comprehensive improvement ensures that the gitea-mirror application can reliably handle job recovery in all deployment scenarios, from development to production container environments.

127
docs/testing.md Normal file
View File

@@ -0,0 +1,127 @@
# Testing in Gitea Mirror
This document provides guidance on testing in the Gitea Mirror project.
## Current Status
The project now uses Bun's built-in test runner, which is Jest-compatible and provides a fast, reliable testing experience. We've migrated away from Vitest due to compatibility issues with Bun.
## Running Tests
To run tests, use the following commands:
```bash
# Run all tests
bun test
# Run tests in watch mode (automatically re-run when files change)
bun test --watch
# Run tests with coverage reporting
bun test --coverage
```
## Test File Naming Conventions
Bun's test runner automatically discovers test files that match the following patterns:
- `*.test.{js|jsx|ts|tsx}`
- `*_test.{js|jsx|ts|tsx}`
- `*.spec.{js|jsx|ts|tsx}`
- `*_spec.{js|jsx|ts|tsx}`
## Writing Tests
The project uses Bun's test runner with a Jest-compatible API. Here's an example test:
```typescript
// example.test.ts
import { describe, test, expect } from "bun:test";
describe("Example Test", () => {
test("should pass", () => {
expect(true).toBe(true);
});
});
```
### Testing React Components
For testing React components, we use React Testing Library:
```typescript
// component.test.tsx
import { describe, test, expect } from "bun:test";
import { render, screen } from "@testing-library/react";
import MyComponent from "../components/MyComponent";
describe("MyComponent", () => {
test("renders correctly", () => {
render(<MyComponent />);
expect(screen.getByText("Hello World")).toBeInTheDocument();
});
});
```
## Test Setup
The test setup is defined in `src/tests/setup.bun.ts` and includes:
- Automatic cleanup after each test
- Setup for any global test environment needs
## Mocking
Bun's test runner provides built-in mocking capabilities:
```typescript
import { test, expect, mock } from "bun:test";
// Create a mock function
const mockFn = mock(() => "mocked value");
test("mock function", () => {
const result = mockFn();
expect(result).toBe("mocked value");
expect(mockFn).toHaveBeenCalled();
});
// Mock a module
mock.module("./some-module", () => {
return {
someFunction: () => "mocked module function"
};
});
```
## CI Integration
The CI workflow has been updated to use Bun's test runner. Tests are automatically run as part of the CI pipeline.
## Test Coverage
To generate test coverage reports, run:
```bash
bun test --coverage
```
This will generate a coverage report in the `coverage` directory.
## Types of Tests
The project includes several types of tests:
1. **Unit Tests**: Testing individual functions and utilities
2. **API Tests**: Testing API endpoints
3. **Component Tests**: Testing React components
4. **Integration Tests**: Testing how components work together
## Future Improvements
When expanding the test suite, consider:
1. Adding more comprehensive API endpoint tests
2. Increasing component test coverage
3. Setting up end-to-end tests with a tool like Playwright
4. Adding performance tests for critical paths

View File

@@ -1,27 +1,33 @@
{
"name": "gitea-mirror",
"type": "module",
"version": "2.0.0",
"version": "2.8.0",
"engines": {
"bun": ">=1.2.9"
},
"scripts": {
"setup": "bun install && bun run manage-db init",
"setup": "bun install && bun run manage-db init && bun run update-db",
"dev": "bunx --bun astro dev",
"dev:clean": "bun run cleanup-db && bun run manage-db init && bunx --bun astro dev",
"dev:clean": "bun run cleanup-db && bun run manage-db init && bun run update-db && bunx --bun astro dev",
"build": "bunx --bun astro build",
"cleanup-db": "rm -f gitea-mirror.db data/gitea-mirror.db",
"manage-db": "bun scripts/manage-db.ts",
"init-db": "bun scripts/manage-db.ts init",
"update-db": "bun scripts/update-mirror-jobs-table.ts",
"check-db": "bun scripts/manage-db.ts check",
"fix-db": "bun scripts/manage-db.ts fix",
"reset-users": "bun scripts/manage-db.ts reset-users",
"cleanup-events": "bun scripts/cleanup-events.ts",
"startup-recovery": "bun scripts/startup-recovery.ts",
"startup-recovery-force": "bun scripts/startup-recovery.ts --force",
"test-recovery": "bun scripts/test-recovery.ts",
"test-recovery-cleanup": "bun scripts/test-recovery.ts --cleanup",
"preview": "bunx --bun astro preview",
"start": "bun dist/server/entry.mjs",
"start:fresh": "bun run cleanup-db && bun run manage-db init && bun dist/server/entry.mjs",
"test": "bunx --bun vitest run",
"test:watch": "bunx --bun vitest",
"start:fresh": "bun run cleanup-db && bun run manage-db init && bun run update-db && bun dist/server/entry.mjs",
"test": "bun test",
"test:watch": "bun test --watch",
"test:coverage": "bun test --coverage",
"astro": "bunx --bun astro"
},
"dependencies": {
@@ -38,6 +44,7 @@
"@radix-ui/react-radio-group": "^1.3.6",
"@radix-ui/react-select": "^2.2.4",
"@radix-ui/react-slot": "^1.2.2",
"@radix-ui/react-switch": "^1.2.5",
"@radix-ui/react-tabs": "^1.1.11",
"@radix-ui/react-tooltip": "^1.2.6",
"@tailwindcss/vite": "^4.1.7",
@@ -61,7 +68,6 @@
"react-dom": "^19.1.0",
"react-icons": "^5.5.0",
"sonner": "^2.0.3",
"sqlite3": "^5.1.7",
"superagent": "^10.2.1",
"tailwind-merge": "^3.3.0",
"tailwindcss": "^4.1.7",

View File

@@ -47,12 +47,12 @@ The script uses environment variables from the `.env` file in the project root:
# First build the image
./scripts/build-docker.sh --load
# Then run using docker-compose for development
docker-compose -f ../docker-compose.dev.yml up -d
# Or for production
docker-compose --profile production up -d
docker compose up -d
```
## Diagnostics Script

View File

@@ -1,109 +1,118 @@
# LXC Container Deployment Guide
This guide explains how to deploy the Gitea Mirror application on Proxmox LXC containers while keeping your existing Docker containers.
## Overview
Run **Gitea Mirror** in an isolated LXC container, either:
## Prerequisites
1. **Online, on a Proxmox VE host** script pulls everything from GitHub
2. **Offline / LAN-only, on a developer laptop** script pushes your local checkout + Bun ZIP
- Proxmox VE installed and configured
- Basic knowledge of LXC containers and Proxmox
- Access to Proxmox web interface or CLI
---
## Creating an LXC Container
## 1. Proxmox VE (online, recommended for prod)
1. In Proxmox web interface, create a new LXC container:
- Choose Ubuntu 22.04 as the template
- Allocate appropriate resources (2GB RAM, 2 CPU cores recommended)
- At least 10GB of disk space
- Configure networking as needed
### Prerequisites
* Proxmox VE node with the default `vmbr0` bridge
* Root shell on the node
* Ubuntu 22.04 LXC template present (`pveam update && pveam download ...`)
2. Start the container and get a shell (either via Proxmox web console or SSH)
### One-command install
## Deploying Gitea Mirror
```bash
# Community-maintained script for Proxmox VE by Tobias ([CrazyWolf13](https://github.com/CrazyWolf13))
# at [community-scripts/ProxmoxVED](https://github.com/community-scripts/ProxmoxVED)
sudo bash -c "$(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/install/gitea-mirror-install.sh)"
```
### Option 1: One-Command Installation (Recommended)
What it does:
This method allows you to install Gitea Mirror with a single command, without having to copy files manually:
* Uses the community-maintained script from ProxmoxVED
* Installs dependencies and Bun runtime
* Clones & builds `arunavo4/gitea-mirror`
* Creates a systemd service and starts it
* Sets up a random `JWT_SECRET` for security
1. SSH into your LXC container:
```bash
ssh root@lxc-container-ip
```
Browse to:
2. Run the installer script directly:
```bash
curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-installer.sh | bash
```
```
http://<container-ip>:4321
```
3. The installer will:
- Download the Gitea Mirror repository
- Install all dependencies including Bun
- Build the application
- Set up a systemd service
- Start the application
- Display access information
---
### Option 2: Manual Setup
## 2. Local testing (LXD on a workstation, works offline)
If you prefer to set up manually or the automatic script doesn't work for your environment:
### Prerequisites
1. Install dependencies:
```bash
apt update
apt install -y curl git sqlite3 build-essential
```
* `lxd` installed (`sudo apt install lxd`; `lxd init --auto`)
* Your repo cloned locally e.g. `~/Development/gitea-mirror`
* Bun ZIP downloaded once:
`https://github.com/oven-sh/bun/releases/latest/download/bun-linux-x64.zip`
2. Install Bun:
```bash
curl -fsSL https://bun.sh/install | bash
export BUN_INSTALL="/root/.bun"
export PATH="$BUN_INSTALL/bin:$PATH"
```
### Offline installer script
3. Clone or copy your project:
```bash
git clone https://github.com/yourusername/gitea-mirror.git /opt/gitea-mirror
cd /opt/gitea-mirror
```
```bash
git clone https://github.com/arunavo4/gitea-mirror.git # if not already
curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-local.sh -o gitea-mirror-lxc-local.sh
chmod +x gitea-mirror-lxc-local.sh
4. Build and initialize:
```bash
bun install
bun run build
bun run manage-db init
```
sudo LOCAL_REPO_DIR=~/Development/gitea-mirror \
./gitea-mirror-lxc-local.sh
```
5. Create a systemd service manually:
```bash
nano /etc/systemd/system/gitea-mirror.service
# Add the service configuration as shown below:
What it does:
[Unit]
Description=Gitea Mirror
After=network.target
* Launches privileged LXC `gitea-test` (`lxc launch ubuntu:22.04 ...`)
* Pushes **Bun ZIP** + tarred **local repo** into `/opt`
* Unpacks, builds, initializes DB
* Symlinks both `bun` and `bunx``/usr/local/bin`
* Creates a root systemd unit and starts it
[Service]
Type=simple
WorkingDirectory=/opt/gitea-mirror
ExecStart=/root/.bun/bin/bun dist/server/entry.mjs
Restart=on-failure
RestartSec=10
User=gitea-mirror
Group=gitea-mirror
Environment=NODE_ENV=production
Environment=HOST=0.0.0.0
Environment=PORT=4321
Environment=DATABASE_URL=file:data/gitea-mirror.db
Environment=JWT_SECRET=your-secure-secret-key
Access from host:
[Install]
WantedBy=multi-user.target
```
```
http://$(lxc exec gitea-test -- hostname -I | awk '{print $1}'):4321
```
6. Enable and start the service:
```bash
systemctl enable gitea-mirror.service
systemctl start gitea-mirror.service
```
(Optional) forward to host localhost:
```bash
sudo lxc config device add gitea-test mirror proxy \
listen=tcp:0.0.0.0:4321 connect=tcp:127.0.0.1:4321
```
---
## Health-check endpoint
Gitea Mirror includes a built-in health check endpoint at `/api/health` that provides:
- System status and uptime
- Database connectivity check
- Memory usage statistics
- Environment information
You can use this endpoint for monitoring your deployment:
```bash
# Basic check (returns 200 OK if healthy)
curl -I http://<container-ip>:4321/api/health
# Detailed health information (JSON)
curl http://<container-ip>:4321/api/health
```
---
## Troubleshooting
| Check | Command |
| -------------- | ----------------------------------------------------- |
| Service status | `systemctl status gitea-mirror` |
| Live logs | `journalctl -u gitea-mirror -f` |
| Verify Bun | `bun --version && bunx --version` |
| DB perms | `chown -R root:root /opt/gitea-mirror/data` (Proxmox) |
---
## Connecting LXC and Docker Containers
@@ -121,32 +130,3 @@ If you need your LXC container to communicate with Docker containers:
```
3. In Proxmox, edit the LXC container's network configuration to use this bridge.
## Accessing the Application
Once deployed, you can access the Gitea Mirror application at:
```
http://lxc-container-ip:4321
```
## Troubleshooting
- Check service status:
```bash
systemctl status gitea-mirror
```
- View logs:
```bash
journalctl -u gitea-mirror -f
```
- If the service fails to start, check permissions on the data directory:
```bash
chown -R gitea-mirror:gitea-mirror /opt/gitea-mirror/data
```
- Verify Bun is installed correctly:
```bash
bun --version
```

View File

@@ -1,12 +1,14 @@
# Scripts Directory
This folder contains utility scripts for database management.
This folder contains utility scripts for database management, event management, Docker builds, and LXC container deployment.
## Database Management Tool (manage-db.ts)
## Database Management
### Database Management Tool (manage-db.ts)
This is a consolidated database management tool that handles all database-related operations. It combines the functionality of the previous separate scripts into a single, more intelligent script that can check, fix, and initialize the database as needed.
### Features
#### Features
- **Check Mode**: Validates the existence and integrity of the database
- **Init Mode**: Creates the database only if it doesn't already exist
@@ -14,12 +16,12 @@ This is a consolidated database management tool that handles all database-relate
- **Reset Users Mode**: Removes all users and their data
- **Auto Mode**: Automatically checks, fixes, and initializes the database if needed
## Running the Database Management Tool
#### Running the Database Management Tool
You can execute the database management tool using your package manager with various commands:
```bash
# Checks database status (default action if no command is specified, equivalent to 'bun run check-db')
# Checks database status (default action if no command is specified)
bun run manage-db
# Check database status
@@ -37,9 +39,6 @@ bun run db-auto
# Reset all users (for testing signup flow)
bun run reset-users
# Update the database schema to the latest version
bun run update-schema
# Remove database files completely
bun run cleanup-db
@@ -53,6 +52,89 @@ bun run dev:clean
bun run start:fresh
```
## Database File Location
#### Database File Location
The database file should be located in the `./data/gitea-mirror.db` directory. If the file is found in the root directory, the fix mode will move it to the correct location.
## Event Management
The following scripts help manage events in the SQLite database:
> **Note**: For a more user-friendly approach, you can use the cleanup button in the Activity Log page of the web interface to delete all activities with a single click.
### Remove Duplicate Events (remove-duplicate-events.ts)
Specifically removes duplicate events based on deduplication keys without affecting old events.
```bash
# Remove duplicate events for all users
bun scripts/remove-duplicate-events.ts
# Remove duplicate events for a specific user
bun scripts/remove-duplicate-events.ts <userId>
```
### Fix Interrupted Jobs (fix-interrupted-jobs.ts)
Fixes interrupted jobs that might be preventing cleanup by marking them as failed.
```bash
# Fix all interrupted jobs
bun scripts/fix-interrupted-jobs.ts
# Fix interrupted jobs for a specific user
bun scripts/fix-interrupted-jobs.ts <userId>
```
Use this script if you're having trouble cleaning up activities due to "interrupted" jobs that won't delete.
### Startup Recovery (startup-recovery.ts)
Runs job recovery during application startup to handle any interrupted jobs from previous runs.
```bash
# Run startup recovery (normal mode)
bun scripts/startup-recovery.ts
# Force recovery even if recent attempt was made
bun scripts/startup-recovery.ts --force
# Set custom timeout (default: 30000ms)
bun scripts/startup-recovery.ts --timeout=60000
# Using npm scripts
bun run startup-recovery
bun run startup-recovery-force
```
This script is automatically run by the Docker entrypoint during container startup. It ensures that any jobs interrupted by container restarts or application crashes are properly recovered or marked as failed.
## Deployment Scripts
### Docker Deployment
- **build-docker.sh**: Builds the Docker image for the application
- **docker-diagnostics.sh**: Provides diagnostic information for Docker deployments
### LXC Container Deployment
Two deployment options are available for LXC containers:
1. **Proxmox VE (online)**: Using the community-maintained script by Tobias ([CrazyWolf13](https://github.com/CrazyWolf13))
- Author: Tobias ([CrazyWolf13](https://github.com/CrazyWolf13))
- Available at: [community-scripts/ProxmoxVED](https://github.com/community-scripts/ProxmoxVED/blob/main/install/gitea-mirror-install.sh)
- Pulls everything from GitHub
- Creates a privileged container with the application
- Sets up systemd service
2. **gitea-mirror-lxc-local.sh**: For offline/LAN-only deployment on a developer laptop
- Pushes your local checkout + Bun ZIP to the container
- Useful for testing without internet access
For detailed instructions on LXC deployment, see [README-lxc.md](./README-lxc.md).

View File

@@ -1,38 +0,0 @@
#!/usr/bin/env bun
/**
* Script to check events in the database
*/
import { Database } from "bun:sqlite";
import path from "path";
import fs from "fs";
// Define the database path
const dataDir = path.join(process.cwd(), "data");
if (!fs.existsSync(dataDir)) {
console.error("Data directory not found:", dataDir);
process.exit(1);
}
const dbPath = path.join(dataDir, "gitea-mirror.db");
if (!fs.existsSync(dbPath)) {
console.error("Database file not found:", dbPath);
process.exit(1);
}
// Open the database
const db = new Database(dbPath);
// Check if the events table exists
const tableExists = db.query("SELECT name FROM sqlite_master WHERE type='table' AND name='events'").get();
if (!tableExists) {
console.error("Events table does not exist");
process.exit(1);
}
// Get all events
const events = db.query("SELECT * FROM events").all();
console.log("Events in the database:");
console.log(JSON.stringify(events, null, 2));

View File

@@ -1,43 +0,0 @@
#!/usr/bin/env bun
/**
* Script to clean up old events from the database
* This script should be run periodically (e.g., daily) to prevent the events table from growing too large
*
* Usage:
* bun scripts/cleanup-events.ts [days]
*
* Where [days] is the number of days to keep events (default: 7)
*/
import { cleanupOldEvents } from "../src/lib/events";
// Parse command line arguments
const args = process.argv.slice(2);
const daysToKeep = args.length > 0 ? parseInt(args[0], 10) : 7;
if (isNaN(daysToKeep) || daysToKeep < 1) {
console.error("Error: Days to keep must be a positive number");
process.exit(1);
}
async function runCleanup() {
try {
console.log(`Starting event cleanup (retention: ${daysToKeep} days)...`);
// Call the cleanupOldEvents function from the events module
const result = await cleanupOldEvents(daysToKeep);
console.log(`Cleanup summary:`);
console.log(`- Read events deleted: ${result.readEventsDeleted}`);
console.log(`- Unread events deleted: ${result.unreadEventsDeleted}`);
console.log(`- Total events deleted: ${result.readEventsDeleted + result.unreadEventsDeleted}`);
console.log("Event cleanup completed successfully");
} catch (error) {
console.error("Error running event cleanup:", error);
process.exit(1);
}
}
// Run the cleanup
runCleanup();

View File

@@ -0,0 +1,74 @@
#!/usr/bin/env bun
/**
* Script to fix interrupted jobs that might be preventing cleanup
* This script marks all in-progress jobs as failed to allow them to be deleted
*
* Usage:
* bun scripts/fix-interrupted-jobs.ts [userId]
*
* Where [userId] is optional - if provided, only fixes jobs for that user
*/
import { db, mirrorJobs } from "../src/lib/db";
import { eq } from "drizzle-orm";
// Parse command line arguments
const args = process.argv.slice(2);
const userId = args.length > 0 ? args[0] : undefined;
async function fixInterruptedJobs() {
try {
console.log("Checking for interrupted jobs...");
// Build the query
let query = db
.select()
.from(mirrorJobs)
.where(eq(mirrorJobs.inProgress, true));
if (userId) {
console.log(`Filtering for user: ${userId}`);
query = query.where(eq(mirrorJobs.userId, userId));
}
// Find all in-progress jobs
const inProgressJobs = await query;
if (inProgressJobs.length === 0) {
console.log("No interrupted jobs found.");
return;
}
console.log(`Found ${inProgressJobs.length} interrupted jobs:`);
inProgressJobs.forEach(job => {
console.log(`- Job ${job.id}: ${job.message} (${job.repositoryName || job.organizationName || 'Unknown'})`);
});
// Mark all in-progress jobs as failed
let updateQuery = db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: "Job interrupted and marked as failed by cleanup script"
})
.where(eq(mirrorJobs.inProgress, true));
if (userId) {
updateQuery = updateQuery.where(eq(mirrorJobs.userId, userId));
}
await updateQuery;
console.log(`✅ Successfully marked ${inProgressJobs.length} interrupted jobs as failed.`);
console.log("These jobs can now be deleted through the normal cleanup process.");
} catch (error) {
console.error("Error fixing interrupted jobs:", error);
process.exit(1);
}
}
// Run the fix
fixInterruptedJobs();

View File

@@ -1,188 +0,0 @@
#!/bin/bash
# Gitea Mirror LXC Container Installer
# This is a self-contained script to install Gitea Mirror in an LXC container
# Usage: curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-installer.sh | bash
set -e
# Configuration variables - change these as needed
INSTALL_DIR="/opt/gitea-mirror"
REPO_URL="https://github.com/arunavo4/gitea-mirror.git"
SERVICE_USER="gitea-mirror"
PORT=4321
# Color codes for better readability
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Print banner
echo -e "${BLUE}"
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ ║"
echo "║ Gitea Mirror LXC Container Installer ║"
echo "║ ║"
echo "╚════════════════════════════════════════════════════════════╝"
echo -e "${NC}"
# Ensure script is run as root
if [ "$(id -u)" -ne 0 ]; then
echo -e "${RED}This script must be run as root${NC}" >&2
exit 1
fi
echo -e "${GREEN}Starting Gitea Mirror installation...${NC}"
# Check if we're in an LXC container
if [ -d /proc/vz ] && [ ! -d /proc/bc ]; then
echo -e "${YELLOW}Running in an OpenVZ container. Some features may not work.${NC}"
elif [ -f /proc/1/environ ] && grep -q container=lxc /proc/1/environ; then
echo -e "${GREEN}Running in an LXC container. Good!${NC}"
else
echo -e "${YELLOW}Not running in a container. This script is designed for LXC containers.${NC}"
read -p "Continue anyway? (y/n) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo -e "${RED}Installation aborted.${NC}"
exit 1
fi
fi
# Install dependencies
echo -e "${BLUE}Step 1/7: Installing dependencies...${NC}"
apt update
apt install -y curl git sqlite3 build-essential openssl
# Create service user
echo -e "${BLUE}Step 2/7: Creating service user...${NC}"
if id "$SERVICE_USER" &>/dev/null; then
echo -e "${YELLOW}User $SERVICE_USER already exists${NC}"
else
useradd -m -s /bin/bash "$SERVICE_USER"
echo -e "${GREEN}Created user $SERVICE_USER${NC}"
fi
# Install Bun
echo -e "${BLUE}Step 3/7: Installing Bun runtime...${NC}"
if command -v bun >/dev/null 2>&1; then
echo -e "${YELLOW}Bun is already installed${NC}"
bun --version
else
echo -e "${GREEN}Installing Bun...${NC}"
curl -fsSL https://bun.sh/install | bash
export BUN_INSTALL=${BUN_INSTALL:-"/root/.bun"}
export PATH="$BUN_INSTALL/bin:$PATH"
echo -e "${GREEN}Bun installed successfully${NC}"
bun --version
fi
# Clone repository
echo -e "${BLUE}Step 4/7: Downloading Gitea Mirror...${NC}"
if [ -d "$INSTALL_DIR" ]; then
echo -e "${YELLOW}Directory $INSTALL_DIR already exists${NC}"
read -p "Update existing installation? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
cd "$INSTALL_DIR"
git pull
echo -e "${GREEN}Repository updated${NC}"
else
echo -e "${YELLOW}Using existing installation${NC}"
fi
else
echo -e "${GREEN}Cloning repository...${NC}"
git clone "$REPO_URL" "$INSTALL_DIR"
echo -e "${GREEN}Repository cloned to $INSTALL_DIR${NC}"
fi
# Set up application
echo -e "${BLUE}Step 5/7: Setting up application...${NC}"
cd "$INSTALL_DIR"
# Create data directory with proper permissions
mkdir -p data
chown -R "$SERVICE_USER:$SERVICE_USER" data
# Install dependencies and build
echo -e "${GREEN}Installing dependencies and building application...${NC}"
bun install
bun run build
# Initialize database if it doesn't exist
echo -e "${GREEN}Initializing database...${NC}"
if [ ! -f "data/gitea-mirror.db" ]; then
bun run manage-db init
chown "$SERVICE_USER:$SERVICE_USER" data/gitea-mirror.db
fi
# Generate a random JWT secret if not provided
JWT_SECRET=${JWT_SECRET:-$(openssl rand -hex 32)}
# Create systemd service
echo -e "${BLUE}Step 6/7: Creating systemd service...${NC}"
# Store Bun path in a variable for better maintainability
BUN_PATH=$(command -v bun)
echo -e "${GREEN}Using Bun from: $BUN_PATH${NC}"
cat >/etc/systemd/system/gitea-mirror.service <<SERVICE
[Unit]
Description=Gitea Mirror
After=network.target
[Service]
Type=simple
WorkingDirectory=$INSTALL_DIR
ExecStart=$BUN_PATH dist/server/entry.mjs
Restart=on-failure
RestartSec=10
User=$SERVICE_USER
Group=$SERVICE_USER
Environment=NODE_ENV=production
Environment=HOST=0.0.0.0
Environment=PORT=$PORT
Environment=DATABASE_URL=file:data/gitea-mirror.db
Environment=JWT_SECRET=${JWT_SECRET}
[Install]
WantedBy=multi-user.target
SERVICE
# Start service
echo -e "${BLUE}Step 7/7: Starting service...${NC}"
systemctl daemon-reload
systemctl enable gitea-mirror.service
systemctl start gitea-mirror.service
# Check if service started successfully
if systemctl is-active --quiet gitea-mirror.service; then
echo -e "${GREEN}Gitea Mirror service started successfully!${NC}"
else
echo -e "${RED}Failed to start Gitea Mirror service. Check logs with: journalctl -u gitea-mirror${NC}"
exit 1
fi
# Get IP address
IP_ADDRESS=$(hostname -I | awk '{print $1}')
# Print success message
echo -e "${GREEN}"
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ ║"
echo "║ Gitea Mirror Installation Complete ║"
echo "║ ║"
echo "╚════════════════════════════════════════════════════════════╝"
echo -e "${NC}"
echo -e "${GREEN}Gitea Mirror is now running at: http://$IP_ADDRESS:$PORT${NC}"
echo
echo -e "${YELLOW}Important security information:${NC}"
echo -e "JWT_SECRET: ${JWT_SECRET}"
echo -e "${YELLOW}Please save this JWT_SECRET in a secure location.${NC}"
echo
echo -e "${BLUE}To check service status:${NC} systemctl status gitea-mirror"
echo -e "${BLUE}To view logs:${NC} journalctl -u gitea-mirror -f"
echo -e "${BLUE}Data directory:${NC} $INSTALL_DIR/data"
echo
echo -e "${GREEN}Thank you for installing Gitea Mirror!${NC}"

View File

@@ -0,0 +1,86 @@
#!/usr/bin/env bash
# gitea-mirror-lxc-local.sh (offline, local repo, verbose)
set -euo pipefail
CONTAINER="gitea-test"
IMAGE="ubuntu:22.04"
INSTALL_DIR="/opt/gitea-mirror"
PORT=4321
JWT_SECRET="$(openssl rand -hex 32)"
BUN_ZIP="/tmp/bun-linux-x64.zip"
BUN_URL="https://github.com/oven-sh/bun/releases/latest/download/bun-linux-x64.zip"
LOCAL_REPO_DIR="${LOCAL_REPO_DIR:-./gitea-mirror}"
REPO_TAR="/tmp/gitea-mirror-local.tar.gz"
need() { command -v "$1" >/dev/null || { echo "Missing $1"; exit 1; }; }
need curl; need lxc; need tar; need unzip
# ── build host artefacts ────────────────────────────────────────────────
[[ -d $LOCAL_REPO_DIR ]] || { echo "❌ LOCAL_REPO_DIR not found"; exit 1; }
[[ -f $LOCAL_REPO_DIR/package.json ]] || { echo "❌ package.json missing"; exit 1; }
[[ -f $BUN_ZIP ]] || curl -L --retry 5 --retry-delay 5 -o "$BUN_ZIP" "$BUN_URL"
tar -czf "$REPO_TAR" -C "$(dirname "$LOCAL_REPO_DIR")" "$(basename "$LOCAL_REPO_DIR")"
# ── ensure container exists ─────────────────────────────────────────────
lxd init --auto >/dev/null 2>&1 || true
lxc info "$CONTAINER" >/dev/null 2>&1 || lxc launch "$IMAGE" "$CONTAINER"
echo "🔧 installing base packages…"
sudo lxc exec "$CONTAINER" -- bash -c 'set -ex; apt update; apt install -y unzip tar openssl sqlite3'
echo "⬆️ pushing artefacts…"
sudo lxc file push "$BUN_ZIP" "$CONTAINER/opt/"
sudo lxc file push "$REPO_TAR" "$CONTAINER/opt/"
echo "📦 unpacking Bun + repo…"
sudo lxc exec "$CONTAINER" -- bash -ex <<'IN'
cd /opt
# Bun
unzip -oq bun-linux-x64.zip -d bun
BIN=$(find /opt/bun -type f -name bun -perm -111 | head -n1)
ln -sf "$BIN" /usr/local/bin/bun # bun
ln -sf "$BIN" /usr/local/bin/bunx # bunx shim
# Repo
rm -rf /opt/gitea-mirror
mkdir -p /opt/gitea-mirror
tar -xzf gitea-mirror-local.tar.gz --strip-components=1 -C /opt/gitea-mirror
IN
echo "🏗️ bun install / build…"
sudo lxc exec "$CONTAINER" -- bash -ex <<'IN'
cd /opt/gitea-mirror
bun install
bun run build
bun run manage-db init
IN
echo "📝 systemd unit…"
sudo lxc exec "$CONTAINER" -- bash -ex <<IN
cat >/etc/systemd/system/gitea-mirror.service <<SERVICE
[Unit]
Description=Gitea Mirror
After=network.target
[Service]
Type=simple
WorkingDirectory=$INSTALL_DIR
ExecStart=/usr/local/bin/bun dist/server/entry.mjs
Restart=on-failure
RestartSec=10
Environment=NODE_ENV=production
Environment=HOST=0.0.0.0
Environment=PORT=$PORT
Environment=DATABASE_URL=file:data/gitea-mirror.db
Environment=JWT_SECRET=$JWT_SECRET
[Install]
WantedBy=multi-user.target
SERVICE
systemctl daemon-reload
systemctl enable gitea-mirror
systemctl restart gitea-mirror
IN
echo -e "\n✅ finished; service status:"
sudo lxc exec "$CONTAINER" -- systemctl status gitea-mirror --no-pager

View File

@@ -1,29 +0,0 @@
#!/usr/bin/env bun
/**
* Script to make events appear older for testing cleanup
*/
import { db, events } from "../src/lib/db";
async function makeEventsOld() {
try {
console.log("Making events appear older...");
// Calculate a timestamp from 2 days ago
const oldDate = new Date();
oldDate.setDate(oldDate.getDate() - 2);
// Update all events to have an older timestamp
const result = await db
.update(events)
.set({ createdAt: oldDate });
console.log(`Updated ${result.changes || 0} events to appear older`);
} catch (error) {
console.error("Error updating event timestamps:", error);
process.exit(1);
}
}
// Run the function
makeEventsOld();

View File

@@ -145,9 +145,31 @@ async function ensureTablesExist() {
status TEXT NOT NULL DEFAULT 'imported',
message TEXT NOT NULL,
timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- New fields for job resilience
job_type TEXT NOT NULL DEFAULT 'mirror',
batch_id TEXT,
total_items INTEGER,
completed_items INTEGER DEFAULT 0,
item_ids TEXT, -- JSON array as text
completed_item_ids TEXT DEFAULT '[]', -- JSON array as text
in_progress INTEGER NOT NULL DEFAULT 0, -- Boolean as integer
started_at TIMESTAMP,
completed_at TIMESTAMP,
last_checkpoint TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
)
`);
// Create indexes for better performance
db.exec(`
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_user_id ON mirror_jobs(user_id);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_batch_id ON mirror_jobs(batch_id);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_in_progress ON mirror_jobs(in_progress);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_job_type ON mirror_jobs(job_type);
CREATE INDEX IF NOT EXISTS idx_mirror_jobs_timestamp ON mirror_jobs(timestamp);
`);
break;
case "events":
db.exec(`
@@ -175,6 +197,33 @@ async function ensureTablesExist() {
process.exit(1);
}
}
// Migration: Add cleanup_config column to existing configs table
try {
const db = new Database(dbPath);
// Check if cleanup_config column exists
const tableInfo = db.query(`PRAGMA table_info(configs)`).all();
const hasCleanupConfig = tableInfo.some((column: any) => column.name === 'cleanup_config');
if (!hasCleanupConfig) {
console.log("Adding cleanup_config column to configs table...");
// Add the column with a default value
const defaultCleanupConfig = JSON.stringify({
enabled: false,
retentionDays: 7,
lastRun: null,
nextRun: null,
});
db.exec(`ALTER TABLE configs ADD COLUMN cleanup_config TEXT NOT NULL DEFAULT '${defaultCleanupConfig}'`);
console.log("✅ cleanup_config column added successfully.");
}
} catch (error) {
console.error("❌ Error during cleanup_config migration:", error);
// Don't exit here as this is not critical for basic functionality
}
}
/**
@@ -306,6 +355,7 @@ async function initializeDatabase() {
include TEXT NOT NULL DEFAULT '["*"]',
exclude TEXT NOT NULL DEFAULT '[]',
schedule_config TEXT NOT NULL,
cleanup_config TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
FOREIGN KEY (user_id) REFERENCES users(id)
@@ -437,10 +487,16 @@ async function initializeDatabase() {
lastRun: null,
nextRun: null,
});
const cleanupConfig = JSON.stringify({
enabled: false,
retentionDays: 7,
lastRun: null,
nextRun: null,
});
const stmt = db.prepare(`
INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, include, exclude, schedule_config, cleanup_config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
stmt.run(
@@ -453,6 +509,7 @@ async function initializeDatabase() {
include,
exclude,
scheduleConfig,
cleanupConfig,
Date.now(),
Date.now()
);

View File

@@ -1,27 +0,0 @@
#!/usr/bin/env bun
/**
* Script to mark all events as read
*/
import { db, events } from "../src/lib/db";
import { eq } from "drizzle-orm";
async function markEventsAsRead() {
try {
console.log("Marking all events as read...");
// Update all events to mark them as read
const result = await db
.update(events)
.set({ read: true })
.where(eq(events.read, false));
console.log(`Marked ${result.changes || 0} events as read`);
} catch (error) {
console.error("Error marking events as read:", error);
process.exit(1);
}
}
// Run the function
markEventsAsRead();

View File

@@ -0,0 +1,44 @@
#!/usr/bin/env bun
/**
* Script to remove duplicate events from the database
* This script identifies and removes events with duplicate deduplication keys
*
* Usage:
* bun scripts/remove-duplicate-events.ts [userId]
*
* Where [userId] is optional - if provided, only removes duplicates for that user
*/
import { removeDuplicateEvents } from "../src/lib/events";
// Parse command line arguments
const args = process.argv.slice(2);
const userId = args.length > 0 ? args[0] : undefined;
async function runDuplicateRemoval() {
try {
if (userId) {
console.log(`Starting duplicate event removal for user: ${userId}...`);
} else {
console.log("Starting duplicate event removal for all users...");
}
// Call the removeDuplicateEvents function
const result = await removeDuplicateEvents(userId);
console.log(`Duplicate removal summary:`);
console.log(`- Duplicate events removed: ${result.duplicatesRemoved}`);
if (result.duplicatesRemoved > 0) {
console.log("Duplicate event removal completed successfully");
} else {
console.log("No duplicate events found");
}
} catch (error) {
console.error("Error running duplicate event removal:", error);
process.exit(1);
}
}
// Run the duplicate removal
runDuplicateRemoval();

113
scripts/startup-recovery.ts Normal file
View File

@@ -0,0 +1,113 @@
#!/usr/bin/env bun
/**
* Startup recovery script
* This script runs job recovery before the application starts serving requests
* It ensures that any interrupted jobs from previous runs are properly handled
*
* Usage:
* bun scripts/startup-recovery.ts [--force] [--timeout=30000]
*
* Options:
* --force: Force recovery even if a recent attempt was made
* --timeout: Maximum time to wait for recovery (in milliseconds, default: 30000)
*/
import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from "../src/lib/recovery";
// Parse command line arguments
const args = process.argv.slice(2);
const forceRecovery = args.includes('--force');
const timeoutArg = args.find(arg => arg.startsWith('--timeout='));
const timeout = timeoutArg ? parseInt(timeoutArg.split('=')[1], 10) : 30000;
if (isNaN(timeout) || timeout < 1000) {
console.error("Error: Timeout must be at least 1000ms");
process.exit(1);
}
async function runStartupRecovery() {
console.log('=== Gitea Mirror Startup Recovery ===');
console.log(`Timeout: ${timeout}ms`);
console.log(`Force recovery: ${forceRecovery}`);
console.log('');
const startTime = Date.now();
try {
// Set up timeout
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => {
reject(new Error(`Recovery timeout after ${timeout}ms`));
}, timeout);
});
// Check if recovery is needed first
console.log('Checking if recovery is needed...');
const needsRecovery = await hasJobsNeedingRecovery();
if (!needsRecovery) {
console.log('✅ No jobs need recovery. Startup can proceed.');
process.exit(0);
}
console.log('⚠️ Jobs found that need recovery. Starting recovery process...');
// Run recovery with timeout
const recoveryPromise = initializeRecovery({
skipIfRecentAttempt: !forceRecovery,
maxRetries: 3,
retryDelay: 5000,
});
const recoveryResult = await Promise.race([recoveryPromise, timeoutPromise]);
const endTime = Date.now();
const duration = endTime - startTime;
if (recoveryResult) {
console.log(`✅ Recovery completed successfully in ${duration}ms`);
console.log('Application startup can proceed.');
process.exit(0);
} else {
console.log(`⚠️ Recovery completed with some failures in ${duration}ms`);
console.log('Application startup can proceed, but some jobs may have failed.');
process.exit(0);
}
} catch (error) {
const endTime = Date.now();
const duration = endTime - startTime;
if (error instanceof Error && error.message.includes('timeout')) {
console.error(`❌ Recovery timed out after ${duration}ms`);
console.error('Application will start anyway, but some jobs may remain interrupted.');
// Get current recovery status
const status = getRecoveryStatus();
console.log('Recovery status:', status);
// Exit with warning code but allow startup to continue
process.exit(1);
} else {
console.error(`❌ Recovery failed after ${duration}ms:`, error);
console.error('Application will start anyway, but recovery was unsuccessful.');
// Exit with error code but allow startup to continue
process.exit(1);
}
}
}
// Handle process signals gracefully
process.on('SIGINT', () => {
console.log('\n⚠ Recovery interrupted by SIGINT');
process.exit(130);
});
process.on('SIGTERM', () => {
console.log('\n⚠ Recovery interrupted by SIGTERM');
process.exit(143);
});
// Run the startup recovery
runStartupRecovery();

183
scripts/test-recovery.ts Normal file
View File

@@ -0,0 +1,183 @@
#!/usr/bin/env bun
/**
* Test script for the recovery system
* This script creates test jobs and verifies that the recovery system can handle them
*
* Usage:
* bun scripts/test-recovery.ts [--cleanup]
*
* Options:
* --cleanup: Clean up test jobs after testing
*/
import { db, mirrorJobs } from "../src/lib/db";
import { createMirrorJob } from "../src/lib/helpers";
import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from "../src/lib/recovery";
import { eq } from "drizzle-orm";
import { v4 as uuidv4 } from "uuid";
// Parse command line arguments
const args = process.argv.slice(2);
const cleanup = args.includes('--cleanup');
// Test configuration
const TEST_USER_ID = "test-user-recovery";
const TEST_BATCH_ID = "test-batch-recovery";
async function runRecoveryTest() {
console.log('=== Recovery System Test ===');
console.log(`Cleanup mode: ${cleanup}`);
console.log('');
try {
if (cleanup) {
await cleanupTestJobs();
return;
}
// Step 1: Create test jobs that simulate interrupted state
console.log('Step 1: Creating test interrupted jobs...');
await createTestInterruptedJobs();
// Step 2: Check if recovery system detects them
console.log('Step 2: Checking if recovery system detects interrupted jobs...');
const needsRecovery = await hasJobsNeedingRecovery();
console.log(`Jobs needing recovery: ${needsRecovery}`);
if (!needsRecovery) {
console.log('❌ Recovery system did not detect interrupted jobs');
return;
}
// Step 3: Get recovery status
console.log('Step 3: Getting recovery status...');
const status = getRecoveryStatus();
console.log('Recovery status:', status);
// Step 4: Run recovery
console.log('Step 4: Running recovery...');
const recoveryResult = await initializeRecovery({
skipIfRecentAttempt: false,
maxRetries: 2,
retryDelay: 2000,
});
console.log(`Recovery result: ${recoveryResult}`);
// Step 5: Verify recovery completed
console.log('Step 5: Verifying recovery completed...');
const stillNeedsRecovery = await hasJobsNeedingRecovery();
console.log(`Jobs still needing recovery: ${stillNeedsRecovery}`);
// Step 6: Check final job states
console.log('Step 6: Checking final job states...');
await checkTestJobStates();
console.log('');
console.log('✅ Recovery test completed successfully!');
console.log('Run with --cleanup to remove test jobs');
} catch (error) {
console.error('❌ Recovery test failed:', error);
process.exit(1);
}
}
/**
* Create test jobs that simulate interrupted state
*/
async function createTestInterruptedJobs() {
const testJobs = [
{
repositoryId: uuidv4(),
repositoryName: "test-repo-1",
message: "Test mirror job 1",
status: "mirroring" as const,
jobType: "mirror" as const,
},
{
repositoryId: uuidv4(),
repositoryName: "test-repo-2",
message: "Test sync job 2",
status: "syncing" as const,
jobType: "sync" as const,
},
];
for (const job of testJobs) {
const jobId = await createMirrorJob({
userId: TEST_USER_ID,
repositoryId: job.repositoryId,
repositoryName: job.repositoryName,
message: job.message,
status: job.status,
jobType: job.jobType,
batchId: TEST_BATCH_ID,
totalItems: 5,
itemIds: [job.repositoryId, uuidv4(), uuidv4(), uuidv4(), uuidv4()],
inProgress: true,
skipDuplicateEvent: true,
});
// Manually set the job to look interrupted (old timestamp)
const oldTimestamp = new Date();
oldTimestamp.setMinutes(oldTimestamp.getMinutes() - 15); // 15 minutes ago
await db
.update(mirrorJobs)
.set({
startedAt: oldTimestamp,
lastCheckpoint: oldTimestamp,
})
.where(eq(mirrorJobs.id, jobId));
console.log(`Created test job: ${jobId} (${job.repositoryName})`);
}
}
/**
* Check the final states of test jobs
*/
async function checkTestJobStates() {
const testJobs = await db
.select()
.from(mirrorJobs)
.where(eq(mirrorJobs.userId, TEST_USER_ID));
console.log(`Found ${testJobs.length} test jobs:`);
for (const job of testJobs) {
console.log(`- Job ${job.id}: ${job.status} (inProgress: ${job.inProgress})`);
console.log(` Message: ${job.message}`);
console.log(` Started: ${job.startedAt ? new Date(job.startedAt).toISOString() : 'never'}`);
console.log(` Completed: ${job.completedAt ? new Date(job.completedAt).toISOString() : 'never'}`);
console.log('');
}
}
/**
* Clean up test jobs
*/
async function cleanupTestJobs() {
console.log('Cleaning up test jobs...');
const result = await db
.delete(mirrorJobs)
.where(eq(mirrorJobs.userId, TEST_USER_ID));
console.log('✅ Test jobs cleaned up successfully');
}
// Handle process signals gracefully
process.on('SIGINT', () => {
console.log('\n⚠ Test interrupted by SIGINT');
process.exit(130);
});
process.on('SIGTERM', () => {
console.log('\n⚠ Test interrupted by SIGTERM');
process.exit(143);
});
// Run the test
runRecoveryTest();

View File

@@ -1,16 +1,18 @@
import { useMemo, useRef, useState, useEffect } from "react";
import { useVirtualizer } from "@tanstack/react-virtual";
import type { MirrorJob } from "@/lib/db/schema";
import Fuse from "fuse.js";
import { Button } from "../ui/button";
import { RefreshCw } from "lucide-react";
import { Card } from "../ui/card";
import { formatDate, getStatusColor } from "@/lib/utils";
import { Skeleton } from "../ui/skeleton";
import type { FilterParams } from "@/types/filter";
import { useEffect, useMemo, useRef, useState } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
import type { MirrorJob } from '@/lib/db/schema';
import Fuse from 'fuse.js';
import { Button } from '../ui/button';
import { RefreshCw } from 'lucide-react';
import { Card } from '../ui/card';
import { formatDate, getStatusColor } from '@/lib/utils';
import { Skeleton } from '../ui/skeleton';
import type { FilterParams } from '@/types/filter';
type MirrorJobWithKey = MirrorJob & { _rowKey: string };
interface ActivityListProps {
activities: MirrorJob[];
activities: MirrorJobWithKey[];
isLoading: boolean;
filter: FilterParams;
setFilter: (filter: FilterParams) => void;
@@ -22,38 +24,44 @@ export default function ActivityList({
filter,
setFilter,
}: ActivityListProps) {
const [expandedItems, setExpandedItems] = useState<Set<string>>(new Set());
const [expandedItems, setExpandedItems] = useState<Set<string>>(
() => new Set(),
);
const parentRef = useRef<HTMLDivElement>(null);
const rowRefs = useRef<Map<string, HTMLDivElement | null>>(new Map());
// We keep the ref only for possible future scroll-to-row logic.
const rowRefs = useRef<Map<string, HTMLDivElement | null>>(new Map()); // eslint-disable-line @typescript-eslint/no-unused-vars
const filteredActivities = useMemo(() => {
let result = activities;
if (filter.status) {
result = result.filter((activity) => activity.status === filter.status);
result = result.filter((a) => a.status === filter.status);
}
if (filter.type) {
if (filter.type === 'repository') {
result = result.filter((activity) => !!activity.repositoryId);
} else if (filter.type === 'organization') {
result = result.filter((activity) => !!activity.organizationId);
}
result =
filter.type === 'repository'
? result.filter((a) => !!a.repositoryId)
: filter.type === 'organization'
? result.filter((a) => !!a.organizationId)
: result;
}
if (filter.name) {
result = result.filter((activity) =>
activity.repositoryName === filter.name ||
activity.organizationName === filter.name
result = result.filter(
(a) =>
a.repositoryName === filter.name ||
a.organizationName === filter.name,
);
}
if (filter.searchTerm) {
const fuse = new Fuse(result, {
keys: ["message", "details", "organizationName", "repositoryName"],
keys: ['message', 'details', 'organizationName', 'repositoryName'],
threshold: 0.3,
});
result = fuse.search(filter.searchTerm).map((res) => res.item);
result = fuse.search(filter.searchTerm).map((r) => r.item);
}
return result;
@@ -62,10 +70,8 @@ export default function ActivityList({
const virtualizer = useVirtualizer({
count: filteredActivities.length,
getScrollElement: () => parentRef.current,
estimateSize: (index) => {
const activity = filteredActivities[index];
return expandedItems.has(activity.id || "") ? 217 : 120;
},
estimateSize: (idx) =>
expandedItems.has(filteredActivities[idx]._rowKey) ? 217 : 120,
overscan: 5,
measureElement: (el) => el.getBoundingClientRect().height + 8,
});
@@ -74,118 +80,127 @@ export default function ActivityList({
virtualizer.measure();
}, [expandedItems, virtualizer]);
return isLoading ? (
<div className="flex flex-col gap-y-4">
{Array.from({ length: 5 }, (_, index) => (
<Skeleton key={index} className="h-28 w-full rounded-md" />
))}
</div>
) : filteredActivities.length === 0 ? (
<div className="flex flex-col items-center justify-center py-12 text-center">
<RefreshCw className="h-12 w-12 text-muted-foreground mb-4" />
<h3 className="text-lg font-medium">No activities found</h3>
<p className="text-sm text-muted-foreground mt-1 mb-4 max-w-md">
{filter.searchTerm || filter.status || filter.type || filter.name
? "Try adjusting your search or filter criteria."
: "No mirroring activities have been recorded yet."}
</p>
{filter.searchTerm || filter.status || filter.type || filter.name ? (
<Button
variant="outline"
onClick={() => {
setFilter({ searchTerm: "", status: "", type: "", name: "" });
}}
>
Clear Filters
</Button>
) : (
<Button>
<RefreshCw className="h-4 w-4 mr-2" />
Refresh
</Button>
)}
</div>
) : (
/* ------------------------------ render ------------------------------ */
if (isLoading) {
return (
<div className='flex flex-col gap-y-4'>
{Array.from({ length: 5 }, (_, i) => (
<Skeleton key={i} className='h-28 w-full rounded-md' />
))}
</div>
);
}
if (filteredActivities.length === 0) {
const hasFilter =
filter.searchTerm || filter.status || filter.type || filter.name;
return (
<div className='flex flex-col items-center justify-center py-12 text-center'>
<RefreshCw className='mb-4 h-12 w-12 text-muted-foreground' />
<h3 className='text-lg font-medium'>No activities found</h3>
<p className='mt-1 mb-4 max-w-md text-sm text-muted-foreground'>
{hasFilter
? 'Try adjusting your search or filter criteria.'
: 'No mirroring activities have been recorded yet.'}
</p>
{hasFilter && (
<Button
variant='outline'
onClick={() =>
setFilter({ searchTerm: '', status: '', type: '', name: '' })
}
>
Clear Filters
</Button>
)}
</div>
);
}
return (
<Card
className="border rounded-md max-h-[calc(100dvh-191px)] overflow-y-auto relative"
ref={parentRef}
className='relative max-h-[calc(100dvh-191px)] overflow-y-auto rounded-md border'
>
<div
style={{
height: virtualizer.getTotalSize(),
position: "relative",
width: "100%",
position: 'relative',
width: '100%',
}}
>
{virtualizer.getVirtualItems().map((virtualRow) => {
const activity = filteredActivities[virtualRow.index];
const isExpanded = expandedItems.has(activity.id || "");
const key = activity.id || String(virtualRow.index);
{virtualizer.getVirtualItems().map((vRow) => {
const activity = filteredActivities[vRow.index];
const isExpanded = expandedItems.has(activity._rowKey);
return (
<div
key={key}
key={activity._rowKey}
ref={(node) => {
if (node) {
rowRefs.current.set(key, node);
virtualizer.measureElement(node);
}
rowRefs.current.set(activity._rowKey, node);
if (node) virtualizer.measureElement(node);
}}
style={{
position: "absolute",
position: 'absolute',
top: 0,
left: 0,
width: "100%",
transform: `translateY(${virtualRow.start}px)`,
paddingBottom: "8px",
width: '100%',
transform: `translateY(${vRow.start}px)`,
paddingBottom: '8px',
}}
className="border-b px-4 pt-4"
className='border-b px-4 pt-4'
>
<div className="flex items-start gap-4">
<div className="relative mt-2">
<div className='flex items-start gap-4'>
<div className='relative mt-2'>
<div
className={`h-2 w-2 rounded-full ${getStatusColor(
activity.status
activity.status,
)}`}
/>
</div>
<div className="flex-1">
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between mb-1">
<p className="font-medium">{activity.message}</p>
<p className="text-sm text-muted-foreground">
<div className='flex-1'>
<div className='mb-1 flex flex-col sm:flex-row sm:items-center sm:justify-between'>
<p className='font-medium'>{activity.message}</p>
<p className='text-sm text-muted-foreground'>
{formatDate(activity.timestamp)}
</p>
</div>
{activity.repositoryName && (
<p className="text-sm text-muted-foreground mb-2">
<p className='mb-2 text-sm text-muted-foreground'>
Repository: {activity.repositoryName}
</p>
)}
{activity.organizationName && (
<p className="text-sm text-muted-foreground mb-2">
<p className='mb-2 text-sm text-muted-foreground'>
Organization: {activity.organizationName}
</p>
)}
{activity.details && (
<div className="mt-2">
<div className='mt-2'>
<Button
variant="ghost"
onClick={() => {
const newSet = new Set(expandedItems);
const id = activity.id || "";
newSet.has(id) ? newSet.delete(id) : newSet.add(id);
setExpandedItems(newSet);
}}
className="text-xs h-7 px-2"
variant='ghost'
className='h-7 px-2 text-xs'
onClick={() =>
setExpandedItems((prev) => {
const next = new Set(prev);
next.has(activity._rowKey)
? next.delete(activity._rowKey)
: next.add(activity._rowKey);
return next;
})
}
>
{isExpanded ? "Hide Details" : "Show Details"}
{isExpanded ? 'Hide Details' : 'Show Details'}
</Button>
{isExpanded && (
<pre className="mt-2 p-3 bg-muted rounded-md text-xs overflow-auto whitespace-pre-wrap min-h-[100px]">
<pre className='mt-2 min-h-[100px] whitespace-pre-wrap overflow-auto rounded-md bg-muted p-3 text-xs'>
{activity.details}
</pre>
)}

View File

@@ -1,289 +1,419 @@
import { useCallback, useEffect, useState } from "react";
import { Button } from "@/components/ui/button";
import { Search, Download, RefreshCw, ChevronDown } from "lucide-react";
import { useCallback, useEffect, useState, useRef } from 'react';
import { Button } from '@/components/ui/button';
import { ChevronDown, Download, RefreshCw, Search, Trash2 } from 'lucide-react';
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "../ui/dropdown-menu";
import { apiRequest, formatDate } from "@/lib/utils";
import { useAuth } from "@/hooks/useAuth";
import type { MirrorJob } from "@/lib/db/schema";
import type { ActivityApiResponse } from "@/types/activities";
} from '../ui/dropdown-menu';
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from '../ui/dialog';
import { apiRequest, formatDate } from '@/lib/utils';
import { useAuth } from '@/hooks/useAuth';
import type { MirrorJob } from '@/lib/db/schema';
import type { ActivityApiResponse } from '@/types/activities';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "../ui/select";
import { repoStatusEnum, type RepoStatus } from "@/types/Repository";
import ActivityList from "./ActivityList";
import { ActivityNameCombobox } from "./ActivityNameCombobox";
import { useSSE } from "@/hooks/useSEE";
import { useFilterParams } from "@/hooks/useFilterParams";
import { toast } from "sonner";
} from '../ui/select';
import { repoStatusEnum, type RepoStatus } from '@/types/Repository';
import ActivityList from './ActivityList';
import { ActivityNameCombobox } from './ActivityNameCombobox';
import { useSSE } from '@/hooks/useSEE';
import { useFilterParams } from '@/hooks/useFilterParams';
import { toast } from 'sonner';
import { useLiveRefresh } from '@/hooks/useLiveRefresh';
import { useConfigStatus } from '@/hooks/useConfigStatus';
import { useNavigation } from '@/components/layout/MainLayout';
type MirrorJobWithKey = MirrorJob & { _rowKey: string };
// Maximum number of activities to keep in memory to prevent performance issues
const MAX_ACTIVITIES = 1000;
// More robust key generation to prevent collisions
function genKey(job: MirrorJob, index?: number): string {
const baseId = job.id || `temp-${Date.now()}-${Math.random().toString(36).slice(2)}`;
const timestamp = job.timestamp instanceof Date ? job.timestamp.getTime() : new Date(job.timestamp).getTime();
const indexSuffix = index !== undefined ? `-${index}` : '';
return `${baseId}-${timestamp}${indexSuffix}`;
}
// Create a deep clone without structuredClone for better browser compatibility
function deepClone<T>(obj: T): T {
if (obj === null || typeof obj !== 'object') return obj;
if (obj instanceof Date) return new Date(obj.getTime()) as T;
if (Array.isArray(obj)) return obj.map(item => deepClone(item)) as T;
const cloned = {} as T;
for (const key in obj) {
if (Object.prototype.hasOwnProperty.call(obj, key)) {
cloned[key] = deepClone(obj[key]);
}
}
return cloned;
}
export function ActivityLog() {
const { user } = useAuth();
const [activities, setActivities] = useState<MirrorJob[]>([]);
const [isLoading, setIsLoading] = useState<boolean>(false);
const { filter, setFilter } = useFilterParams({
searchTerm: "",
status: "",
type: "",
name: "",
});
const { registerRefreshCallback } = useLiveRefresh();
const { isFullyConfigured } = useConfigStatus();
const { navigationKey } = useNavigation();
const handleNewMessage = useCallback((data: MirrorJob) => {
setActivities((prevActivities) => [data, ...prevActivities]);
const [activities, setActivities] = useState<MirrorJobWithKey[]>([]);
const [isLoading, setIsLoading] = useState(false);
const [showCleanupDialog, setShowCleanupDialog] = useState(false);
console.log("Received new log:", data);
// Ref to track if component is mounted to prevent state updates after unmount
const isMountedRef = useRef(true);
useEffect(() => {
return () => {
isMountedRef.current = false;
};
}, []);
const { filter, setFilter } = useFilterParams({
searchTerm: '',
status: '',
type: '',
name: '',
});
/* ----------------------------- SSE hook ----------------------------- */
const handleNewMessage = useCallback((data: MirrorJob) => {
if (!isMountedRef.current) return;
setActivities((prev) => {
// Create a deep clone of the new activity
const clonedData = deepClone(data);
// Check if this activity already exists to prevent duplicates
const existingIndex = prev.findIndex(activity =>
activity.id === clonedData.id ||
(activity.repositoryId === clonedData.repositoryId &&
activity.organizationId === clonedData.organizationId &&
activity.message === clonedData.message &&
Math.abs(new Date(activity.timestamp).getTime() - new Date(clonedData.timestamp).getTime()) < 1000)
);
if (existingIndex !== -1) {
// Update existing activity instead of adding duplicate
const updated = [...prev];
updated[existingIndex] = {
...clonedData,
_rowKey: prev[existingIndex]._rowKey, // Keep the same key
};
return updated;
}
// Add new activity with unique key
const withKey: MirrorJobWithKey = {
...clonedData,
_rowKey: genKey(clonedData, prev.length),
};
// Limit the number of activities to prevent memory issues
const newActivities = [withKey, ...prev];
return newActivities.slice(0, MAX_ACTIVITIES);
});
}, []);
// Use the SSE hook
const { connected } = useSSE({
userId: user?.id,
onMessage: handleNewMessage,
});
/* ------------------------- initial fetch --------------------------- */
const fetchActivities = useCallback(async () => {
if (!user) return false;
if (!user?.id) return false;
try {
setIsLoading(true);
const response = await apiRequest<ActivityApiResponse>(
const res = await apiRequest<ActivityApiResponse>(
`/activities?userId=${user.id}`,
{
method: "GET",
}
{ method: 'GET' },
);
if (response.success) {
setActivities(response.activities);
return true;
} else {
toast.error(response.message || "Failed to fetch activities.");
if (!res.success) {
toast.error(res.message ?? 'Failed to fetch activities.');
return false;
}
} catch (error) {
toast.error(
error instanceof Error ? error.message : "Failed to fetch activities."
);
return false;
} finally {
setIsLoading(false);
}
}, [user]);
useEffect(() => {
fetchActivities();
}, [fetchActivities]);
// Process activities with robust cloning and unique keys
const data: MirrorJobWithKey[] = res.activities.map((activity, index) => {
const clonedActivity = deepClone(activity);
return {
...clonedActivity,
_rowKey: genKey(clonedActivity, index),
};
});
const handleRefreshActivities = async () => {
const success = await fetchActivities();
if (success) {
toast.success("Activities refreshed successfully.");
}
};
// Sort by timestamp (newest first) to ensure consistent ordering
data.sort((a, b) => {
const timeA = new Date(a.timestamp).getTime();
const timeB = new Date(b.timestamp).getTime();
return timeB - timeA;
});
// Get the currently filtered activities
const getFilteredActivities = () => {
return activities.filter(activity => {
let isIncluded = true;
if (filter.status) {
isIncluded = isIncluded && activity.status === filter.status;
if (isMountedRef.current) {
setActivities(data);
}
if (filter.type) {
if (filter.type === 'repository') {
isIncluded = isIncluded && !!activity.repositoryId;
} else if (filter.type === 'organization') {
isIncluded = isIncluded && !!activity.organizationId;
}
}
if (filter.name) {
isIncluded = isIncluded && (
activity.repositoryName === filter.name ||
activity.organizationName === filter.name
return true;
} catch (err) {
if (isMountedRef.current) {
toast.error(
err instanceof Error ? err.message : 'Failed to fetch activities.',
);
}
return false;
} finally {
if (isMountedRef.current) {
setIsLoading(false);
}
}
}, [user?.id]); // Only depend on user.id, not entire user object
// Note: We're not applying the search term filter here as that would require
// re-implementing the Fuse.js search logic
useEffect(() => {
// Reset loading state when component becomes active
setIsLoading(true);
fetchActivities();
}, [fetchActivities, navigationKey]); // Include navigationKey to trigger on navigation
return isIncluded;
// Register with global live refresh system
useEffect(() => {
// Only register for live refresh if configuration is complete
// Activity logs can exist from previous runs, but new activities won't be generated without config
if (!isFullyConfigured) {
return;
}
const unregister = registerRefreshCallback(() => {
fetchActivities();
});
return unregister;
}, [registerRefreshCallback, fetchActivities, isFullyConfigured]);
/* ---------------------- filtering + exporting ---------------------- */
const applyLightFilter = (list: MirrorJobWithKey[]) => {
return list.filter((a) => {
if (filter.status && a.status !== filter.status) return false;
if (filter.type === 'repository' && !a.repositoryId) return false;
if (filter.type === 'organization' && !a.organizationId) return false;
if (
filter.name &&
a.repositoryName !== filter.name &&
a.organizationName !== filter.name
) {
return false;
}
return true;
});
};
// Function to export activities as CSV
const exportAsCSV = () => {
const filteredActivities = getFilteredActivities();
const rows = applyLightFilter(activities);
if (!rows.length) return toast.error('No activities to export.');
if (filteredActivities.length === 0) {
toast.error("No activities to export.");
return;
}
// Create CSV content
const headers = ["Timestamp", "Message", "Status", "Repository", "Organization", "Details"];
const csvRows = [
headers.join(","),
...filteredActivities.map(activity => {
const formattedDate = formatDate(activity.timestamp);
// Escape fields that might contain commas or quotes
const escapeCsvField = (field: string | null | undefined) => {
if (!field) return '';
if (field.includes(',') || field.includes('"') || field.includes('\n')) {
return `"${field.replace(/"/g, '""')}"`;
}
return field;
};
return [
formattedDate,
escapeCsvField(activity.message),
activity.status,
escapeCsvField(activity.repositoryName || ''),
escapeCsvField(activity.organizationName || ''),
escapeCsvField(activity.details || '')
].join(',');
})
const headers = [
'Timestamp',
'Message',
'Status',
'Repository',
'Organization',
'Details',
];
const csvContent = csvRows.join('\n');
const escape = (v: string | null | undefined) =>
v && /[,\"\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v ?? '';
// Download the CSV file
downloadFile(csvContent, 'text/csv;charset=utf-8;', 'activity_log_export.csv');
const csv = [
headers.join(','),
...rows.map((a) =>
[
formatDate(a.timestamp),
escape(a.message),
a.status,
escape(a.repositoryName),
escape(a.organizationName),
escape(a.details),
].join(','),
),
].join('\n');
toast.success("Activity log exported as CSV successfully.");
downloadFile(csv, 'text/csv;charset=utf-8;', 'activity_log_export.csv');
toast.success('CSV exported.');
};
// Function to export activities as JSON
const exportAsJSON = () => {
const filteredActivities = getFilteredActivities();
const rows = applyLightFilter(activities);
if (!rows.length) return toast.error('No activities to export.');
if (filteredActivities.length === 0) {
toast.error("No activities to export.");
return;
}
const json = JSON.stringify(
rows.map((a) => ({
...a,
formattedTime: formatDate(a.timestamp),
})),
null,
2,
);
// Format the activities for export (removing any sensitive or unnecessary fields if needed)
const activitiesForExport = filteredActivities.map(activity => ({
id: activity.id,
timestamp: activity.timestamp,
formattedTime: formatDate(activity.timestamp),
message: activity.message,
status: activity.status,
repositoryId: activity.repositoryId,
repositoryName: activity.repositoryName,
organizationId: activity.organizationId,
organizationName: activity.organizationName,
details: activity.details
}));
const jsonContent = JSON.stringify(activitiesForExport, null, 2);
// Download the JSON file
downloadFile(jsonContent, 'application/json', 'activity_log_export.json');
toast.success("Activity log exported as JSON successfully.");
downloadFile(json, 'application/json', 'activity_log_export.json');
toast.success('JSON exported.');
};
// Generic function to download a file
const downloadFile = (content: string, mimeType: string, filename: string) => {
// Add date to filename
const date = new Date();
const dateStr = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
const filenameWithDate = filename.replace('.', `_${dateStr}.`);
// Create a download link
const blob = new Blob([content], { type: mimeType });
const url = URL.createObjectURL(blob);
const downloadFile = (
content: string,
mime: string,
filename: string,
): void => {
const date = new Date().toISOString().slice(0, 10); // yyyy-mm-dd
const link = document.createElement('a');
link.href = url;
link.setAttribute('download', filenameWithDate);
document.body.appendChild(link);
link.href = URL.createObjectURL(new Blob([content], { type: mime }));
link.download = filename.replace('.', `_${date}.`);
link.click();
document.body.removeChild(link);
};
const handleCleanupClick = () => {
setShowCleanupDialog(true);
};
const confirmCleanup = async () => {
if (!user?.id) return;
try {
setIsLoading(true);
setShowCleanupDialog(false);
// Use fetch directly to avoid potential axios issues
const response = await fetch('/api/activities/cleanup', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ userId: user.id }),
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({ error: 'Unknown error occurred' }));
throw new Error(errorData.error || `HTTP ${response.status}: ${response.statusText}`);
}
const res = await response.json();
if (res.success) {
// Clear the activities from the UI
setActivities([]);
toast.success(`All activities cleaned up successfully. Deleted ${res.result.mirrorJobsDeleted} mirror jobs and ${res.result.eventsDeleted} events.`);
} else {
toast.error(res.error || 'Failed to cleanup activities.');
}
} catch (error) {
console.error('Error cleaning up activities:', error);
toast.error(error instanceof Error ? error.message : 'Failed to cleanup activities.');
} finally {
setIsLoading(false);
}
};
const cancelCleanup = () => {
setShowCleanupDialog(false);
};
/* ------------------------------ UI ------------------------------ */
return (
<div className="flex flex-col gap-y-8">
<div className="flex flex-row items-center gap-4 w-full">
<div className="relative flex-1">
<Search className="absolute left-2 top-2.5 h-4 w-4 text-muted-foreground" />
<div className='flex flex-col gap-y-8'>
<div className='flex w-full flex-row items-center gap-4'>
{/* search input */}
<div className='relative flex-1'>
<Search className='absolute left-2 top-2.5 h-4 w-4 text-muted-foreground' />
<input
type="text"
placeholder="Search activities..."
className="pl-8 h-9 w-full rounded-md border border-input bg-background px-3 py-1 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
type='text'
placeholder='Search activities...'
className='h-9 w-full rounded-md border border-input bg-background px-3 py-1 pl-8 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring'
value={filter.searchTerm}
onChange={(e) =>
setFilter((prev) => ({ ...prev, searchTerm: e.target.value }))
setFilter((prev) => ({
...prev,
searchTerm: e.target.value,
}))
}
/>
</div>
{/* status select */}
<Select
value={filter.status || "all"}
onValueChange={(value) =>
setFilter((prev) => ({
...prev,
status: value === "all" ? "" : (value as RepoStatus),
value={filter.status || 'all'}
onValueChange={(v) =>
setFilter((p) => ({
...p,
status: v === 'all' ? '' : (v as RepoStatus),
}))
}
>
<SelectTrigger className="w-[140px] h-9 max-h-9">
<SelectValue placeholder="All Status" />
<SelectTrigger className='h-9 w-[140px] max-h-9'>
<SelectValue placeholder='All Status' />
</SelectTrigger>
<SelectContent>
{["all", ...repoStatusEnum.options].map((status) => (
<SelectItem key={status} value={status}>
{status === "all"
? "All Status"
: status.charAt(0).toUpperCase() + status.slice(1)}
{['all', ...repoStatusEnum.options].map((s) => (
<SelectItem key={s} value={s}>
{s === 'all' ? 'All Status' : s[0].toUpperCase() + s.slice(1)}
</SelectItem>
))}
</SelectContent>
</Select>
{/* Repository/Organization Name Combobox */}
{/* repo/org name combobox */}
<ActivityNameCombobox
activities={activities}
value={filter.name || ""}
onChange={(name: string) => setFilter((prev) => ({ ...prev, name }))}
value={filter.name || ''}
onChange={(name) => setFilter((p) => ({ ...p, name }))}
/>
{/* Filter by type: repository/org/all */}
{/* type select */}
<Select
value={filter.type || "all"}
onValueChange={(value) =>
setFilter((prev) => ({
...prev,
type: value === "all" ? "" : value,
}))
value={filter.type || 'all'}
onValueChange={(v) =>
setFilter((p) => ({ ...p, type: v === 'all' ? '' : v }))
}
>
<SelectTrigger className="w-[140px] h-9 max-h-9">
<SelectValue placeholder="All Types" />
<SelectTrigger className='h-9 w-[140px] max-h-9'>
<SelectValue placeholder='All Types' />
</SelectTrigger>
<SelectContent>
{['all', 'repository', 'organization'].map((type) => (
<SelectItem key={type} value={type}>
{type === 'all' ? 'All Types' : type.charAt(0).toUpperCase() + type.slice(1)}
{['all', 'repository', 'organization'].map((t) => (
<SelectItem key={t} value={t}>
{t === 'all' ? 'All Types' : t[0].toUpperCase() + t.slice(1)}
</SelectItem>
))}
</SelectContent>
</Select>
{/* export dropdown */}
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="outline" className="flex items-center gap-1">
<Download className="h-4 w-4 mr-1" />
<Button variant='outline' className='flex items-center gap-1'>
<Download className='mr-1 h-4 w-4' />
Export
<ChevronDown className="h-4 w-4 ml-1" />
<ChevronDown className='ml-1 h-4 w-4' />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent>
@@ -295,19 +425,60 @@ export function ActivityLog() {
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<Button onClick={handleRefreshActivities}>
<RefreshCw className="h-4 w-4 mr-2" />
Refresh
{/* refresh */}
<Button
variant="outline"
size="icon"
onClick={() => fetchActivities()}
title="Refresh activity log"
>
<RefreshCw className='h-4 w-4' />
</Button>
{/* cleanup all activities */}
<Button
variant="outline"
size="icon"
onClick={handleCleanupClick}
title="Delete all activities"
className="text-destructive hover:text-destructive"
>
<Trash2 className='h-4 w-4' />
</Button>
</div>
<div className="flex flex-col gap-y-6">
<ActivityList
activities={activities}
isLoading={isLoading || !connected}
filter={filter}
setFilter={setFilter}
/>
</div>
{/* activity list */}
<ActivityList
activities={applyLightFilter(activities)}
isLoading={isLoading || !connected}
filter={filter}
setFilter={setFilter}
/>
{/* cleanup confirmation dialog */}
<Dialog open={showCleanupDialog} onOpenChange={setShowCleanupDialog}>
<DialogContent>
<DialogHeader>
<DialogTitle>Delete All Activities</DialogTitle>
<DialogDescription>
Are you sure you want to delete ALL activities? This action cannot be undone and will remove all mirror jobs and events from the database.
</DialogDescription>
</DialogHeader>
<DialogFooter>
<Button variant="outline" onClick={cancelCleanup}>
Cancel
</Button>
<Button
variant="destructive"
onClick={confirmCleanup}
disabled={isLoading}
>
{isLoading ? 'Deleting...' : 'Delete All Activities'}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</div>
);
}

View File

@@ -1,14 +1,8 @@
import { useEffect, useState } from 'react';
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from '@/components/ui/card';
import { useEffect, useState, useCallback, useRef } from 'react';
import { GitHubConfigForm } from './GitHubConfigForm';
import { GiteaConfigForm } from './GiteaConfigForm';
import { ScheduleConfigForm } from './ScheduleConfigForm';
import { DatabaseCleanupConfigForm } from './DatabaseCleanupConfigForm';
import type {
ConfigApiResponse,
GiteaConfig,
@@ -16,18 +10,21 @@ import type {
SaveConfigApiRequest,
SaveConfigApiResponse,
ScheduleConfig,
DatabaseCleanupConfig,
} from '@/types/config';
import { Button } from '../ui/button';
import { useAuth } from '@/hooks/useAuth';
import { apiRequest } from '@/lib/utils';
import { Copy, CopyCheck, RefreshCw } from 'lucide-react';
import { RefreshCw } from 'lucide-react';
import { toast } from 'sonner';
import { Skeleton } from '@/components/ui/skeleton';
import { invalidateConfigCache } from '@/hooks/useConfigStatus';
type ConfigState = {
githubConfig: GitHubConfig;
giteaConfig: GiteaConfig;
scheduleConfig: ScheduleConfig;
cleanupConfig: DatabaseCleanupConfig;
};
export function ConfigTabs() {
@@ -54,13 +51,19 @@ export function ConfigTabs() {
enabled: false,
interval: 3600,
},
cleanupConfig: {
enabled: false,
retentionDays: 7,
},
});
const { user, refreshUser } = useAuth();
const [isLoading, setIsLoading] = useState(true);
const [dockerCode, setDockerCode] = useState<string>('');
const [isCopied, setIsCopied] = useState<boolean>(false);
const [isSyncing, setIsSyncing] = useState<boolean>(false);
const [isConfigSaved, setIsConfigSaved] = useState<boolean>(false);
const [isAutoSavingSchedule, setIsAutoSavingSchedule] = useState<boolean>(false);
const [isAutoSavingCleanup, setIsAutoSavingCleanup] = useState<boolean>(false);
const autoSaveScheduleTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const autoSaveCleanupTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const isConfigFormValid = (): boolean => {
const { githubConfig, giteaConfig } = config;
@@ -75,26 +78,8 @@ export function ConfigTabs() {
return isGitHubValid && isGiteaValid;
};
useEffect(() => {
const updateLastAndNextRun = () => {
const lastRun = config.scheduleConfig.lastRun
? new Date(config.scheduleConfig.lastRun)
: new Date();
const intervalInSeconds = config.scheduleConfig.interval;
const nextRun = new Date(
lastRun.getTime() + intervalInSeconds * 1000,
);
setConfig(prev => ({
...prev,
scheduleConfig: {
...prev.scheduleConfig,
lastRun,
nextRun,
},
}));
};
updateLastAndNextRun();
}, [config.scheduleConfig.interval]);
// Removed the problematic useEffect that was causing circular dependencies
// The lastRun and nextRun should be managed by the backend and fetched via API
const handleImportGitHubData = async () => {
if (!user?.id) return;
@@ -131,6 +116,7 @@ export function ConfigTabs() {
githubConfig: config.githubConfig,
giteaConfig: config.giteaConfig,
scheduleConfig: config.scheduleConfig,
cleanupConfig: config.cleanupConfig,
};
try {
const response = await fetch('/api/config', {
@@ -142,6 +128,8 @@ export function ConfigTabs() {
if (result.success) {
await refreshUser();
setIsConfigSaved(true);
// Invalidate config cache so other components get fresh data
invalidateConfigCache();
toast.success(
'Configuration saved successfully! Now import your GitHub data to begin.',
);
@@ -159,8 +147,125 @@ export function ConfigTabs() {
}
};
// Auto-save function specifically for schedule config changes
const autoSaveScheduleConfig = useCallback(async (scheduleConfig: ScheduleConfig) => {
if (!user?.id || !isConfigSaved) return; // Only auto-save if config was previously saved
// Clear any existing timeout
if (autoSaveScheduleTimeoutRef.current) {
clearTimeout(autoSaveScheduleTimeoutRef.current);
}
// Debounce the auto-save to prevent excessive API calls
autoSaveScheduleTimeoutRef.current = setTimeout(async () => {
setIsAutoSavingSchedule(true);
const reqPayload: SaveConfigApiRequest = {
userId: user.id!,
githubConfig: config.githubConfig,
giteaConfig: config.giteaConfig,
scheduleConfig: scheduleConfig,
cleanupConfig: config.cleanupConfig,
};
try {
const response = await fetch('/api/config', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(reqPayload),
});
const result: SaveConfigApiResponse = await response.json();
if (result.success) {
// Silent success - no toast for auto-save
// Removed refreshUser() call to prevent page reload
// Invalidate config cache so other components get fresh data
invalidateConfigCache();
} else {
toast.error(
`Auto-save failed: ${result.message || 'Unknown error'}`,
{ duration: 3000 }
);
}
} catch (error) {
toast.error(
`Auto-save error: ${
error instanceof Error ? error.message : String(error)
}`,
{ duration: 3000 }
);
} finally {
setIsAutoSavingSchedule(false);
}
}, 500); // 500ms debounce
}, [user?.id, isConfigSaved, config.githubConfig, config.giteaConfig, config.cleanupConfig]);
// Auto-save function specifically for cleanup config changes
const autoSaveCleanupConfig = useCallback(async (cleanupConfig: DatabaseCleanupConfig) => {
if (!user?.id || !isConfigSaved) return; // Only auto-save if config was previously saved
// Clear any existing timeout
if (autoSaveCleanupTimeoutRef.current) {
clearTimeout(autoSaveCleanupTimeoutRef.current);
}
// Debounce the auto-save to prevent excessive API calls
autoSaveCleanupTimeoutRef.current = setTimeout(async () => {
setIsAutoSavingCleanup(true);
const reqPayload: SaveConfigApiRequest = {
userId: user.id!,
githubConfig: config.githubConfig,
giteaConfig: config.giteaConfig,
scheduleConfig: config.scheduleConfig,
cleanupConfig: cleanupConfig,
};
try {
const response = await fetch('/api/config', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(reqPayload),
});
const result: SaveConfigApiResponse = await response.json();
if (result.success) {
// Silent success - no toast for auto-save
// Invalidate config cache so other components get fresh data
invalidateConfigCache();
} else {
toast.error(
`Auto-save failed: ${result.message || 'Unknown error'}`,
{ duration: 3000 }
);
}
} catch (error) {
toast.error(
`Auto-save error: ${
error instanceof Error ? error.message : String(error)
}`,
{ duration: 3000 }
);
} finally {
setIsAutoSavingCleanup(false);
}
}, 500); // 500ms debounce
}, [user?.id, isConfigSaved, config.githubConfig, config.giteaConfig, config.scheduleConfig]);
// Cleanup timeouts on unmount
useEffect(() => {
if (!user) return;
return () => {
if (autoSaveScheduleTimeoutRef.current) {
clearTimeout(autoSaveScheduleTimeoutRef.current);
}
if (autoSaveCleanupTimeoutRef.current) {
clearTimeout(autoSaveCleanupTimeoutRef.current);
}
};
}, []);
useEffect(() => {
if (!user?.id) return;
const fetchConfig = async () => {
setIsLoading(true);
@@ -177,6 +282,8 @@ export function ConfigTabs() {
response.giteaConfig || config.giteaConfig,
scheduleConfig:
response.scheduleConfig || config.scheduleConfig,
cleanupConfig:
response.cleanupConfig || config.cleanupConfig,
});
if (response.id) setIsConfigSaved(true);
}
@@ -190,84 +297,59 @@ export function ConfigTabs() {
};
fetchConfig();
}, [user]);
useEffect(() => {
const generateDockerCode = () => `
services:
gitea-mirror:
image: arunavo4/gitea-mirror:latest
restart: unless-stopped
container_name: gitea-mirror
environment:
- GITHUB_USERNAME=${config.githubConfig.username}
- GITEA_URL=${config.giteaConfig.url}
- GITEA_TOKEN=${config.giteaConfig.token}
- GITHUB_TOKEN=${config.githubConfig.token}
- SKIP_FORKS=${config.githubConfig.skipForks}
- PRIVATE_REPOSITORIES=${config.githubConfig.privateRepositories}
- MIRROR_ISSUES=${config.githubConfig.mirrorIssues}
- MIRROR_STARRED=${config.githubConfig.mirrorStarred}
- PRESERVE_ORG_STRUCTURE=${config.githubConfig.preserveOrgStructure}
- SKIP_STARRED_ISSUES=${config.githubConfig.skipStarredIssues}
- GITEA_ORGANIZATION=${config.giteaConfig.organization}
- GITEA_ORG_VISIBILITY=${config.giteaConfig.visibility}
- DELAY=${config.scheduleConfig.interval}`;
setDockerCode(generateDockerCode());
}, [config]);
const handleCopyToClipboard = (text: string) => {
navigator.clipboard.writeText(text).then(
() => {
setIsCopied(true);
toast.success('Docker configuration copied to clipboard!');
setTimeout(() => setIsCopied(false), 2000);
},
() => toast.error('Could not copy text to clipboard.'),
);
};
}, [user?.id]); // Only depend on user.id, not the entire user object
function ConfigCardSkeleton() {
return (
<Card>
<CardHeader className="flex-row justify-between">
<div className="flex flex-col gap-y-1.5 m-0">
<Skeleton className="h-6 w-48" />
<div className="space-y-6">
{/* Header section */}
<div className="flex flex-row justify-between items-start">
<div className="flex flex-col gap-y-1.5">
<Skeleton className="h-8 w-48" />
<Skeleton className="h-4 w-72" />
</div>
<div className="flex gap-x-4">
<Skeleton className="h-10 w-36" />
<Skeleton className="h-10 w-36" />
</div>
</CardHeader>
<CardContent>
<div className="flex flex-col gap-y-4">
<div className="flex gap-x-4">
<div className="w-1/2 border rounded-lg p-4">
<div className="flex justify-between items-center mb-4">
<Skeleton className="h-6 w-40" />
<Skeleton className="h-9 w-32" />
</div>
<div className="space-y-4">
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
<Skeleton className="h-32 w-full" />
</div>
</div>
{/* Content section */}
<div className="flex flex-col gap-y-4">
<div className="flex gap-x-4">
<div className="w-1/2 border rounded-lg p-4">
<div className="flex justify-between items-center mb-4">
<Skeleton className="h-6 w-40" />
<Skeleton className="h-9 w-32" />
</div>
<div className="w-1/2 border rounded-lg p-4">
<div className="flex justify-between items-center mb-4">
<Skeleton className="h-6 w-40" />
<Skeleton className="h-9 w-32" />
</div>
<div className="space-y-4">
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
</div>
<div className="space-y-4">
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
<Skeleton className="h-32 w-full" />
</div>
</div>
<div className="border rounded-lg p-4">
<div className="w-1/2 border rounded-lg p-4">
<div className="flex justify-between items-center mb-4">
<Skeleton className="h-6 w-40" />
<Skeleton className="h-9 w-32" />
</div>
<div className="space-y-4">
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
<Skeleton className="h-20 w-full" />
</div>
</div>
</div>
<div className="flex gap-x-4">
<div className="w-1/2 border rounded-lg p-4">
<div className="space-y-4">
<Skeleton className="h-8 w-48" />
<Skeleton className="h-16 w-full" />
<Skeleton className="h-8 w-32" />
</div>
</div>
<div className="w-1/2 border rounded-lg p-4">
<div className="space-y-4">
<Skeleton className="h-8 w-48" />
<Skeleton className="h-16 w-full" />
@@ -275,107 +357,96 @@ services:
</div>
</div>
</div>
</CardContent>
</Card>
);
}
function DockerConfigSkeleton() {
return (
<Card>
<CardHeader>
<Skeleton className="h-6 w-40" />
<Skeleton className="h-4 w-64" />
</CardHeader>
<CardContent className="relative">
<Skeleton className="h-8 w-8 absolute top-4 right-10 rounded-md" />
<Skeleton className="h-48 w-full rounded-md" />
</CardContent>
</Card>
</div>
</div>
);
}
return isLoading ? (
<div className="flex flex-col gap-y-6">
<div className="space-y-6">
<ConfigCardSkeleton />
<DockerConfigSkeleton />
</div>
) : (
<div className="flex flex-col gap-y-6">
<Card>
<CardHeader className="flex-row justify-between">
<div className="flex flex-col gap-y-1.5 m-0">
<CardTitle>Configuration Settings</CardTitle>
<CardDescription>
Configure your GitHub and Gitea connections, and set up automatic
mirroring.
</CardDescription>
</div>
<div className="flex gap-x-4">
<Button
onClick={handleImportGitHubData}
disabled={isSyncing || !isConfigSaved}
title={
!isConfigSaved
? 'Save configuration first'
: isSyncing
? 'Import in progress'
: 'Import GitHub Data'
}
>
{isSyncing ? (
<>
<RefreshCw className="h-4 w-4 animate-spin mr-1" />
Import GitHub Data
</>
) : (
<>
<RefreshCw className="h-4 w-4 mr-1" />
Import GitHub Data
</>
)}
</Button>
<Button
onClick={handleSaveConfig}
disabled={!isConfigFormValid()}
title={
!isConfigFormValid()
? 'Please fill all required fields'
: 'Save Configuration'
}
>
Save Configuration
</Button>
</div>
</CardHeader>
<CardContent>
<div className="flex flex-col gap-y-4">
<div className="flex gap-x-4">
<GitHubConfigForm
config={config.githubConfig}
setConfig={update =>
setConfig(prev => ({
...prev,
githubConfig:
typeof update === 'function'
? update(prev.githubConfig)
: update,
}))
}
/>
<GiteaConfigForm
config={config.giteaConfig}
setConfig={update =>
setConfig(prev => ({
...prev,
giteaConfig:
typeof update === 'function'
? update(prev.giteaConfig)
: update,
}))
}
/>
</div>
<div className="space-y-6">
{/* Header section */}
<div className="flex flex-row justify-between items-start">
<div className="flex flex-col gap-y-1.5">
<h1 className="text-2xl font-semibold leading-none tracking-tight">
Configuration Settings
</h1>
<p className="text-sm text-muted-foreground">
Configure your GitHub and Gitea connections, and set up automatic
mirroring.
</p>
</div>
<div className="flex gap-x-4">
<Button
onClick={handleImportGitHubData}
disabled={isSyncing || !isConfigSaved}
title={
!isConfigSaved
? 'Save configuration first'
: isSyncing
? 'Import in progress'
: 'Import GitHub Data'
}
>
{isSyncing ? (
<>
<RefreshCw className="h-4 w-4 animate-spin mr-1" />
Import GitHub Data
</>
) : (
<>
<RefreshCw className="h-4 w-4 mr-1" />
Import GitHub Data
</>
)}
</Button>
<Button
onClick={handleSaveConfig}
disabled={!isConfigFormValid()}
title={
!isConfigFormValid()
? 'Please fill all required fields'
: 'Save Configuration'
}
>
Save Configuration
</Button>
</div>
</div>
{/* Content section */}
<div className="flex flex-col gap-y-4">
<div className="flex gap-x-4">
<GitHubConfigForm
config={config.githubConfig}
setConfig={update =>
setConfig(prev => ({
...prev,
githubConfig:
typeof update === 'function'
? update(prev.githubConfig)
: update,
}))
}
/>
<GiteaConfigForm
config={config.giteaConfig}
setConfig={update =>
setConfig(prev => ({
...prev,
giteaConfig:
typeof update === 'function'
? update(prev.giteaConfig)
: update,
}))
}
/>
</div>
<div className="flex gap-x-4">
<div className="w-1/2">
<ScheduleConfigForm
config={config.scheduleConfig}
setConfig={update =>
@@ -387,35 +458,28 @@ services:
: update,
}))
}
onAutoSave={autoSaveScheduleConfig}
isAutoSaving={isAutoSavingSchedule}
/>
</div>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Docker Configuration</CardTitle>
<CardDescription>
Equivalent Docker configuration for your current settings.
</CardDescription>
</CardHeader>
<CardContent className="relative">
<Button
variant="outline"
size="icon"
className="absolute top-4 right-10"
onClick={() => handleCopyToClipboard(dockerCode)}
>
{isCopied ? (
<CopyCheck className="text-green-500" />
) : (
<Copy className="text-muted-foreground" />
)}
</Button>
<pre className="bg-muted p-4 rounded-md overflow-auto text-sm">
{dockerCode}
</pre>
</CardContent>
</Card>
<div className="w-1/2">
<DatabaseCleanupConfigForm
config={config.cleanupConfig}
setConfig={update =>
setConfig(prev => ({
...prev,
cleanupConfig:
typeof update === 'function'
? update(prev.cleanupConfig)
: update,
}))
}
onAutoSave={autoSaveCleanupConfig}
isAutoSaving={isAutoSavingCleanup}
/>
</div>
</div>
</div>
</div>
);
}

View File

@@ -0,0 +1,146 @@
import { Card, CardContent } from "@/components/ui/card";
import { Checkbox } from "../ui/checkbox";
import type { DatabaseCleanupConfig } from "@/types/config";
import { formatDate } from "@/lib/utils";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "../ui/select";
import { RefreshCw, Database } from "lucide-react";
interface DatabaseCleanupConfigFormProps {
config: DatabaseCleanupConfig;
setConfig: React.Dispatch<React.SetStateAction<DatabaseCleanupConfig>>;
onAutoSave?: (config: DatabaseCleanupConfig) => void;
isAutoSaving?: boolean;
}
export function DatabaseCleanupConfigForm({
config,
setConfig,
onAutoSave,
isAutoSaving = false,
}: DatabaseCleanupConfigFormProps) {
const handleChange = (
e: React.ChangeEvent<HTMLInputElement | HTMLSelectElement>
) => {
const { name, value, type } = e.target;
const newConfig = {
...config,
[name]:
type === "checkbox" ? (e.target as HTMLInputElement).checked : value,
};
setConfig(newConfig);
// Trigger auto-save for cleanup config changes
if (onAutoSave) {
onAutoSave(newConfig);
}
};
// Predefined retention periods
const retentionOptions: { value: number; label: string }[] = [
{ value: 1, label: "1 day" },
{ value: 3, label: "3 days" },
{ value: 7, label: "7 days" },
{ value: 14, label: "14 days" },
{ value: 30, label: "30 days" },
{ value: 60, label: "60 days" },
{ value: 90, label: "90 days" },
];
return (
<Card>
<CardContent className="pt-6 relative">
{isAutoSaving && (
<div className="absolute top-4 right-4 flex items-center text-sm text-muted-foreground">
<RefreshCw className="h-3 w-3 animate-spin mr-1" />
<span className="text-xs">Auto-saving...</span>
</div>
)}
<div className="flex flex-col gap-y-4">
<div className="flex items-center">
<Checkbox
id="cleanup-enabled"
name="enabled"
checked={config.enabled}
onCheckedChange={(checked) =>
handleChange({
target: {
name: "enabled",
type: "checkbox",
checked: Boolean(checked),
value: "",
},
} as React.ChangeEvent<HTMLInputElement>)
}
/>
<label
htmlFor="cleanup-enabled"
className="select-none ml-2 block text-sm font-medium"
>
<div className="flex items-center gap-2">
<Database className="h-4 w-4" />
Enable Automatic Database Cleanup
</div>
</label>
</div>
{config.enabled && (
<div>
<label className="block text-sm font-medium mb-2">
Retention Period
</label>
<Select
name="retentionDays"
value={String(config.retentionDays)}
onValueChange={(value) =>
handleChange({
target: { name: "retentionDays", value },
} as React.ChangeEvent<HTMLInputElement>)
}
>
<SelectTrigger className="w-full border border-input dark:bg-background dark:hover:bg-background">
<SelectValue placeholder="Select retention period" />
</SelectTrigger>
<SelectContent className="bg-background text-foreground border border-input shadow-sm">
{retentionOptions.map((option) => (
<SelectItem
key={option.value}
value={option.value.toString()}
className="cursor-pointer text-sm px-3 py-2 hover:bg-accent focus:bg-accent focus:text-accent-foreground"
>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-xs text-muted-foreground mt-1">
Activities and events older than this period will be automatically deleted.
</p>
</div>
)}
<div>
<label className="block text-sm font-medium mb-1">Last Run</label>
<div className="text-sm">
{config.lastRun ? formatDate(config.lastRun) : "Never"}
</div>
</div>
{config.nextRun && config.enabled && (
<div>
<label className="block text-sm font-medium mb-1">Next Run</label>
<div className="text-sm">{formatDate(config.nextRun)}</div>
</div>
)}
</div>
</CardContent>
</Card>
);
}

View File

@@ -9,33 +9,36 @@ import {
SelectTrigger,
SelectValue,
} from "../ui/select";
import { RefreshCw } from "lucide-react";
interface ScheduleConfigFormProps {
config: ScheduleConfig;
setConfig: React.Dispatch<React.SetStateAction<ScheduleConfig>>;
onAutoSave?: (config: ScheduleConfig) => void;
isAutoSaving?: boolean;
}
export function ScheduleConfigForm({
config,
setConfig,
onAutoSave,
isAutoSaving = false,
}: ScheduleConfigFormProps) {
const handleChange = (
e: React.ChangeEvent<HTMLInputElement | HTMLSelectElement>
) => {
const { name, value, type } = e.target;
setConfig({
const newConfig = {
...config,
[name]:
type === "checkbox" ? (e.target as HTMLInputElement).checked : value,
});
};
};
setConfig(newConfig);
// Convert seconds to human-readable format
const formatInterval = (seconds: number): string => {
if (seconds < 60) return `${seconds} seconds`;
if (seconds < 3600) return `${Math.floor(seconds / 60)} minutes`;
if (seconds < 86400) return `${Math.floor(seconds / 3600)} hours`;
return `${Math.floor(seconds / 86400)} days`;
// Trigger auto-save for schedule config changes
if (onAutoSave) {
onAutoSave(newConfig);
}
};
// Predefined intervals
@@ -55,7 +58,13 @@ export function ScheduleConfigForm({
return (
<Card>
<CardContent className="pt-6">
<CardContent className="pt-6 relative">
{isAutoSaving && (
<div className="absolute top-4 right-4 flex items-center text-sm text-muted-foreground">
<RefreshCw className="h-3 w-3 animate-spin mr-1" />
<span className="text-xs">Auto-saving...</span>
</div>
)}
<div className="flex flex-col gap-y-4">
<div className="flex items-center">
<Checkbox
@@ -81,51 +90,53 @@ export function ScheduleConfigForm({
</label>
</div>
<div>
<label
htmlFor="interval"
className="block text-sm font-medium mb-1.5"
>
Mirroring Interval
</label>
<Select
name="interval"
value={String(config.interval)}
onValueChange={(value) =>
handleChange({
target: { name: "interval", value },
} as React.ChangeEvent<HTMLInputElement>)
}
>
<SelectTrigger className="w-full border border-input dark:bg-background dark:hover:bg-background">
<SelectValue placeholder="Select interval" />
</SelectTrigger>
<SelectContent className="bg-background text-foreground border border-input shadow-sm">
{intervals.map((interval) => (
<SelectItem
key={interval.value}
value={interval.value.toString()}
className="cursor-pointer text-sm px-3 py-2 hover:bg-accent focus:bg-accent focus:text-accent-foreground"
>
{interval.label}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-xs text-muted-foreground mt-1">
How often the mirroring process should run.
</p>
</div>
{config.lastRun && (
{config.enabled && (
<div>
<label className="block text-sm font-medium mb-1">Last Run</label>
<div className="text-sm">{formatDate(config.lastRun)}</div>
<label
htmlFor="interval"
className="block text-sm font-medium mb-1.5"
>
Mirroring Interval
</label>
<Select
name="interval"
value={String(config.interval)}
onValueChange={(value) =>
handleChange({
target: { name: "interval", value },
} as React.ChangeEvent<HTMLInputElement>)
}
>
<SelectTrigger className="w-full border border-input dark:bg-background dark:hover:bg-background">
<SelectValue placeholder="Select interval" />
</SelectTrigger>
<SelectContent className="bg-background text-foreground border border-input shadow-sm">
{intervals.map((interval) => (
<SelectItem
key={interval.value}
value={interval.value.toString()}
className="cursor-pointer text-sm px-3 py-2 hover:bg-accent focus:bg-accent focus:text-accent-foreground"
>
{interval.label}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-xs text-muted-foreground mt-1">
How often the mirroring process should run.
</p>
</div>
)}
<div>
<label className="block text-sm font-medium mb-1">Last Run</label>
<div className="text-sm">
{config.lastRun ? formatDate(config.lastRun) : "Never"}
</div>
</div>
{config.nextRun && config.enabled && (
<div>
<label className="block text-sm font-medium mb-1">Next Run</label>

View File

@@ -2,7 +2,7 @@ import { StatusCard } from "./StatusCard";
import { RecentActivity } from "./RecentActivity";
import { RepositoryList } from "./RepositoryList";
import { GitFork, Clock, FlipHorizontal, Building2 } from "lucide-react";
import { useCallback, useEffect, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import type { MirrorJob, Organization, Repository } from "@/lib/db/schema";
import { useAuth } from "@/hooks/useAuth";
import { apiRequest } from "@/lib/utils";
@@ -11,9 +11,18 @@ import { useSSE } from "@/hooks/useSEE";
import { toast } from "sonner";
import { Skeleton } from "@/components/ui/skeleton";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
import { usePageVisibility } from "@/hooks/usePageVisibility";
import { useConfigStatus } from "@/hooks/useConfigStatus";
import { useNavigation } from "@/components/layout/MainLayout";
export function Dashboard() {
const { user } = useAuth();
const { registerRefreshCallback } = useLiveRefresh();
const isPageVisible = usePageVisibility();
const { isFullyConfigured } = useConfigStatus();
const { navigationKey } = useNavigation();
const [repositories, setRepositories] = useState<Repository[]>([]);
const [organizations, setOrganizations] = useState<Organization[]>([]);
const [activities, setActivities] = useState<MirrorJob[]>([]);
@@ -23,6 +32,10 @@ export function Dashboard() {
const [mirroredCount, setMirroredCount] = useState<number>(0);
const [lastSync, setLastSync] = useState<Date | null>(null);
// Dashboard auto-refresh timer (30 seconds)
const dashboardTimerRef = useRef<NodeJS.Timeout | null>(null);
const DASHBOARD_REFRESH_INTERVAL = 30000; // 30 seconds
// Create a stable callback using useCallback
const handleNewMessage = useCallback((data: MirrorJob) => {
if (data.repositoryId) {
@@ -54,44 +67,101 @@ export function Dashboard() {
onMessage: handleNewMessage,
});
// Extract fetchDashboardData as a stable callback
const fetchDashboardData = useCallback(async (showToast = false) => {
try {
if (!user?.id) {
return false;
}
// Don't fetch data if configuration is not complete
if (!isFullyConfigured) {
if (showToast) {
toast.info("Please configure GitHub and Gitea settings first");
}
return false;
}
const response = await apiRequest<DashboardApiResponse>(
`/dashboard?userId=${user.id}`,
{
method: "GET",
}
);
if (response.success) {
setRepositories(response.repositories);
setOrganizations(response.organizations);
setActivities(response.activities);
setRepoCount(response.repoCount);
setOrgCount(response.orgCount);
setMirroredCount(response.mirroredCount);
setLastSync(response.lastSync);
if (showToast) {
toast.success("Dashboard data refreshed successfully");
}
return true;
} else {
toast.error(response.error || "Error fetching dashboard data");
return false;
}
} catch (error) {
toast.error(
error instanceof Error
? error.message
: "Error fetching dashboard data"
);
return false;
} finally {
setIsLoading(false);
}
}, [user?.id, isFullyConfigured]); // Only depend on user.id, not entire user object
// Initial data fetch and reset loading state when component becomes active
useEffect(() => {
const fetchDashboardData = async () => {
try {
if (!user || !user.id) {
return;
}
// Reset loading state when component mounts or becomes active
setIsLoading(true);
fetchDashboardData();
}, [fetchDashboardData, navigationKey]); // Include navigationKey to trigger on navigation
const response = await apiRequest<DashboardApiResponse>(
`/dashboard?userId=${user.id}`,
{
method: "GET",
}
);
// Setup dashboard auto-refresh (30 seconds) and register with live refresh
useEffect(() => {
// Clear any existing timer
if (dashboardTimerRef.current) {
clearInterval(dashboardTimerRef.current);
dashboardTimerRef.current = null;
}
if (response.success) {
setRepositories(response.repositories);
setOrganizations(response.organizations);
setActivities(response.activities);
setRepoCount(response.repoCount);
setOrgCount(response.orgCount);
setMirroredCount(response.mirroredCount);
setLastSync(response.lastSync);
} else {
toast.error(response.error || "Error fetching dashboard data");
}
} catch (error) {
toast.error(
error instanceof Error
? error.message
: "Error fetching dashboard data"
);
} finally {
setIsLoading(false);
// Set up 30-second auto-refresh only when page is visible and configuration is complete
if (isPageVisible && isFullyConfigured) {
dashboardTimerRef.current = setInterval(() => {
fetchDashboardData();
}, DASHBOARD_REFRESH_INTERVAL);
}
// Cleanup on unmount or when page becomes invisible
return () => {
if (dashboardTimerRef.current) {
clearInterval(dashboardTimerRef.current);
dashboardTimerRef.current = null;
}
};
}, [isPageVisible, isFullyConfigured, fetchDashboardData]);
fetchDashboardData();
}, [user]);
// Register with global live refresh system
useEffect(() => {
// Only register if configuration is complete
if (!isFullyConfigured) {
return;
}
const unregister = registerRefreshCallback(() => {
fetchDashboardData();
});
return unregister;
}, [registerRefreshCallback, fetchDashboardData, isFullyConfigured]);
// Status Card Skeleton component
function StatusCardSkeleton() {
@@ -150,6 +220,7 @@ export function Dashboard() {
</div>
) : (
<div className="flex flex-col gap-y-6">
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
<StatusCard
title="Total Repositories"

View File

@@ -1,15 +1,50 @@
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Button } from "@/components/ui/button";
import { GitFork } from "lucide-react";
import { SiGithub } from "react-icons/si";
import { SiGithub, SiGitea } from "react-icons/si";
import type { Repository } from "@/lib/db/schema";
import { getStatusColor } from "@/lib/utils";
import { useGiteaConfig } from "@/hooks/useGiteaConfig";
interface RepositoryListProps {
repositories: Repository[];
}
export function RepositoryList({ repositories }: RepositoryListProps) {
const { giteaConfig } = useGiteaConfig();
// Helper function to construct Gitea repository URL
const getGiteaRepoUrl = (repository: Repository): string | null => {
if (!giteaConfig?.url) {
return null;
}
// Only provide Gitea links for repositories that have been or are being mirrored
const validStatuses = ['mirroring', 'mirrored', 'syncing', 'synced'];
if (!validStatuses.includes(repository.status)) {
return null;
}
// Use mirroredLocation if available, otherwise construct from repository data
let repoPath: string;
if (repository.mirroredLocation) {
repoPath = repository.mirroredLocation;
} else {
// Fallback: construct the path based on repository data
// If repository has organization and preserveOrgStructure would be true, use org
// Otherwise use the repository owner
const owner = repository.organization || repository.owner;
repoPath = `${owner}/${repository.name}`;
}
// Ensure the base URL doesn't have a trailing slash
const baseUrl = giteaConfig.url.endsWith('/')
? giteaConfig.url.slice(0, -1)
: giteaConfig.url;
return `${baseUrl}/${repoPath}`;
};
return (
<Card className="w-full">
{/* calculating the max height based non the other elements and sizing styles */}
@@ -69,14 +104,48 @@ export function RepositoryList({ repositories }: RepositoryListProps) {
{/* setting the minimum width to 3rem corresponding to the largest status (mirrored) so that all are left alligned */}
{repo.status}
</span>
<Button variant="ghost" size="icon">
<GitFork className="h-4 w-4" />
</Button>
{(() => {
const giteaUrl = getGiteaRepoUrl(repo);
// Determine tooltip based on status and configuration
let tooltip: string;
if (!giteaConfig?.url) {
tooltip = "Gitea not configured";
} else if (repo.status === 'imported') {
tooltip = "Repository not yet mirrored to Gitea";
} else if (repo.status === 'failed') {
tooltip = "Repository mirroring failed";
} else if (repo.status === 'mirroring') {
tooltip = "Repository is being mirrored to Gitea";
} else if (giteaUrl) {
tooltip = "View on Gitea";
} else {
tooltip = "Gitea repository not available";
}
return giteaUrl ? (
<Button variant="ghost" size="icon" asChild>
<a
href={giteaUrl}
target="_blank"
rel="noopener noreferrer"
title={tooltip}
>
<SiGitea className="h-4 w-4" />
</a>
</Button>
) : (
<Button variant="ghost" size="icon" disabled title={tooltip}>
<SiGitea className="h-4 w-4" />
</Button>
);
})()}
<Button variant="ghost" size="icon" asChild>
<a
href={repo.url}
target="_blank"
rel="noopener noreferrer"
title="View on GitHub"
>
<SiGithub className="h-4 w-4" />
</a>

View File

@@ -5,9 +5,30 @@ import { ModeToggle } from "@/components/theme/ModeToggle";
import { Avatar, AvatarFallback, AvatarImage } from "../ui/avatar";
import { toast } from "sonner";
import { Skeleton } from "@/components/ui/skeleton";
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
import { useConfigStatus } from "@/hooks/useConfigStatus";
export function Header() {
interface HeaderProps {
currentPage?: "dashboard" | "repositories" | "organizations" | "configuration" | "activity-log";
onNavigate?: (page: string) => void;
}
export function Header({ currentPage, onNavigate }: HeaderProps) {
const { user, logout, isLoading } = useAuth();
const { isLiveEnabled, toggleLive } = useLiveRefresh();
const { isFullyConfigured, isLoading: configLoading } = useConfigStatus();
// Show Live button on all pages except configuration
const showLiveButton = currentPage && currentPage !== "configuration";
// Determine button state and tooltip
const isLiveActive = isLiveEnabled && isFullyConfigured;
const getTooltip = () => {
if (!isFullyConfigured && !configLoading) {
return 'Configure GitHub and Gitea settings to enable live refresh';
}
return isLiveEnabled ? 'Disable live refresh' : 'Enable live refresh';
};
const handleLogout = async () => {
toast.success("Logged out successfully");
@@ -29,12 +50,40 @@ export function Header() {
return (
<header className="border-b bg-background">
<div className="flex h-[4.5rem] items-center justify-between px-6">
<a href="/" className="flex items-center gap-2 py-1">
<button
onClick={() => {
if (currentPage !== 'dashboard') {
window.history.pushState({}, '', '/');
onNavigate?.('dashboard');
}
}}
className="flex items-center gap-2 py-1 hover:opacity-80 transition-opacity"
>
<SiGitea className="h-6 w-6" />
<span className="text-xl font-bold">Gitea Mirror</span>
</a>
</button>
<div className="flex items-center gap-4">
{showLiveButton && (
<Button
variant="outline"
size="lg"
className={`flex items-center gap-2 ${!isFullyConfigured && !configLoading ? 'opacity-50 cursor-not-allowed' : ''}`}
onClick={isFullyConfigured || configLoading ? toggleLive : undefined}
title={getTooltip()}
disabled={!isFullyConfigured && !configLoading}
>
<div className={`w-3 h-3 rounded-full ${
configLoading
? 'bg-yellow-400 animate-pulse'
: isLiveActive
? 'bg-emerald-400 animate-pulse'
: 'bg-gray-500'
}`} />
<span>LIVE</span>
</Button>
)}
<ModeToggle />
{isLoading ? (

View File

@@ -1,3 +1,4 @@
import { useState, useEffect, createContext, useContext } from "react";
import { Header } from "./Header";
import { Sidebar } from "./Sidebar";
import { Dashboard } from "@/components/dashboard/Dashboard";
@@ -9,6 +10,12 @@ import { Organization } from "../organizations/Organization";
import { Toaster } from "@/components/ui/sonner";
import { useAuth } from "@/hooks/useAuth";
import { useRepoSync } from "@/hooks/useSyncRepo";
import { useConfigStatus } from "@/hooks/useConfigStatus";
// Navigation context to signal when navigation happens
const NavigationContext = createContext<{ navigationKey: number }>({ navigationKey: 0 });
export const useNavigation = () => useContext(NavigationContext);
interface AppProps {
page:
@@ -32,8 +39,12 @@ export default function App({ page }: AppProps) {
);
}
function AppWithProviders({ page }: AppProps) {
const { user } = useAuth();
function AppWithProviders({ page: initialPage }: AppProps) {
const { user, isLoading: authLoading } = useAuth();
const { isLoading: configLoading } = useConfigStatus();
const [currentPage, setCurrentPage] = useState<AppProps['page']>(initialPage);
const [navigationKey, setNavigationKey] = useState(0);
useRepoSync({
userId: user?.id,
enabled: user?.syncEnabled,
@@ -42,20 +53,65 @@ function AppWithProviders({ page }: AppProps) {
nextSync: user?.nextSync,
});
// Handle navigation from sidebar
const handleNavigation = (pageName: string) => {
setCurrentPage(pageName as AppProps['page']);
// Increment navigation key to force components to refresh their loading state
setNavigationKey(prev => prev + 1);
};
// Handle browser back/forward navigation
useEffect(() => {
const handlePopState = () => {
const path = window.location.pathname;
const pageMap: Record<string, AppProps['page']> = {
'/': 'dashboard',
'/repositories': 'repositories',
'/organizations': 'organizations',
'/config': 'configuration',
'/activity': 'activity-log'
};
const pageName = pageMap[path] || 'dashboard';
setCurrentPage(pageName);
// Also increment navigation key for browser navigation to trigger loading states
setNavigationKey(prev => prev + 1);
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []);
// Show loading state only during initial auth/config loading
const isInitialLoading = authLoading || (configLoading && !user);
if (isInitialLoading) {
return (
<main className="flex min-h-screen flex-col items-center justify-center">
<div className="flex flex-col items-center gap-4">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary"></div>
<p className="text-sm text-muted-foreground">Loading...</p>
</div>
</main>
);
}
return (
<main className="flex min-h-screen flex-col">
<Header />
<div className="flex flex-1">
<Sidebar />
<section className="flex-1 p-6 overflow-y-auto h-[calc(100dvh-4.55rem)]">
{page === "dashboard" && <Dashboard />}
{page === "repositories" && <Repository />}
{page === "organizations" && <Organization />}
{page === "configuration" && <ConfigTabs />}
{page === "activity-log" && <ActivityLog />}
</section>
</div>
<Toaster />
</main>
<NavigationContext.Provider value={{ navigationKey }}>
<main className="flex min-h-screen flex-col">
<Header currentPage={currentPage} onNavigate={handleNavigation} />
<div className="flex flex-1">
<Sidebar onNavigate={handleNavigation} />
<section className="flex-1 p-6 overflow-y-auto h-[calc(100dvh-4.55rem)]">
{currentPage === "dashboard" && <Dashboard />}
{currentPage === "repositories" && <Repository />}
{currentPage === "organizations" && <Organization />}
{currentPage === "configuration" && <ConfigTabs />}
{currentPage === "activity-log" && <ActivityLog />}
</section>
</div>
<Toaster />
</main>
</NavigationContext.Provider>
);
}

View File

@@ -1,13 +1,16 @@
import * as React from "react";
import { AuthProvider } from "@/hooks/useAuth";
import { TooltipProvider } from "@/components/ui/tooltip";
import { LiveRefreshProvider } from "@/hooks/useLiveRefresh";
export default function Providers({ children }: { children: React.ReactNode }) {
return (
<AuthProvider>
<TooltipProvider>
{children}
</TooltipProvider>
<LiveRefreshProvider>
<TooltipProvider>
{children}
</TooltipProvider>
</LiveRefreshProvider>
</AuthProvider>
);
}

View File

@@ -2,12 +2,14 @@ import { useEffect, useState } from "react";
import { cn } from "@/lib/utils";
import { ExternalLink } from "lucide-react";
import { links } from "@/data/Sidebar";
import { VersionInfo } from "./VersionInfo";
interface SidebarProps {
className?: string;
onNavigate?: (page: string) => void;
}
export function Sidebar({ className }: SidebarProps) {
export function Sidebar({ className, onNavigate }: SidebarProps) {
const [currentPath, setCurrentPath] = useState<string>("");
useEffect(() => {
@@ -17,20 +19,53 @@ export function Sidebar({ className }: SidebarProps) {
console.log("Hydrated path:", path); // Should log now
}, []);
// Listen for URL changes (browser back/forward)
useEffect(() => {
const handlePopState = () => {
setCurrentPath(window.location.pathname);
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []);
const handleNavigation = (href: string, event: React.MouseEvent) => {
event.preventDefault();
// Don't navigate if already on the same page
if (currentPath === href) return;
// Update URL without page reload
window.history.pushState({}, '', href);
setCurrentPath(href);
// Map href to page name for the parent component
const pageMap: Record<string, string> = {
'/': 'dashboard',
'/repositories': 'repositories',
'/organizations': 'organizations',
'/config': 'configuration',
'/activity': 'activity-log'
};
const pageName = pageMap[href] || 'dashboard';
onNavigate?.(pageName);
};
return (
<aside className={cn("w-64 border-r bg-background", className)}>
<div className="flex flex-col h-full py-4">
<div className="flex flex-col h-full pt-4">
<nav className="flex flex-col gap-y-1 pl-2 pr-3">
{links.map((link, index) => {
const isActive = currentPath === link.href;
const Icon = link.icon;
return (
<a
<button
key={index}
href={link.href}
onClick={(e) => handleNavigation(link.href, e)}
className={cn(
"flex items-center gap-3 rounded-md px-3 py-2 text-sm font-medium transition-colors",
"flex items-center gap-3 rounded-md px-3 py-2 text-sm font-medium transition-colors w-full text-left",
isActive
? "bg-primary text-primary-foreground"
: "text-muted-foreground hover:bg-accent hover:text-accent-foreground"
@@ -38,7 +73,7 @@ export function Sidebar({ className }: SidebarProps) {
>
<Icon className="h-4 w-4" />
{link.label}
</a>
</button>
);
})}
</nav>
@@ -59,6 +94,7 @@ export function Sidebar({ className }: SidebarProps) {
<ExternalLink className="h-3 w-3" />
</a>
</div>
<VersionInfo />
</div>
</div>
</aside>

View File

@@ -0,0 +1,49 @@
import { useEffect, useState } from "react";
import { healthApi } from "@/lib/api";
export function VersionInfo() {
const [versionInfo, setVersionInfo] = useState<{
current: string;
latest: string;
updateAvailable: boolean;
}>({
current: "loading...",
latest: "",
updateAvailable: false
});
useEffect(() => {
const fetchVersion = async () => {
try {
const healthData = await healthApi.check();
setVersionInfo({
current: healthData.version || "unknown",
latest: healthData.latestVersion || "unknown",
updateAvailable: healthData.updateAvailable || false
});
} catch (error) {
console.error("Failed to fetch version:", error);
setVersionInfo({
current: "unknown",
latest: "",
updateAvailable: false
});
}
};
fetchVersion();
}, []);
return (
<div className="text-xs text-muted-foreground text-center pt-2 pb-3 border-t border-border mt-2">
{versionInfo.updateAvailable ? (
<div className="flex flex-col">
<span>v{versionInfo.current}</span>
<span className="text-primary">v{versionInfo.latest} available</span>
</div>
) : (
<span>v{versionInfo.current}</span>
)}
</div>
);
}

View File

@@ -24,12 +24,18 @@ import type { MirrorOrgRequest, MirrorOrgResponse } from "@/types/mirror";
import { useSSE } from "@/hooks/useSEE";
import { useFilterParams } from "@/hooks/useFilterParams";
import { toast } from "sonner";
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
import { useConfigStatus } from "@/hooks/useConfigStatus";
import { useNavigation } from "@/components/layout/MainLayout";
export function Organization() {
const [organizations, setOrganizations] = useState<Organization[]>([]);
const [isLoading, setIsLoading] = useState<boolean>(true);
const [isDialogOpen, setIsDialogOpen] = useState<boolean>(false);
const { user } = useAuth();
const { registerRefreshCallback } = useLiveRefresh();
const { isGitHubConfigured } = useConfigStatus();
const { navigationKey } = useNavigation();
const { filter, setFilter } = useFilterParams({
searchTerm: "",
membershipRole: "",
@@ -59,7 +65,13 @@ export function Organization() {
});
const fetchOrganizations = useCallback(async () => {
if (!user || !user.id) {
if (!user?.id) {
return false;
}
// Don't fetch organizations if GitHub is not configured
if (!isGitHubConfigured) {
setIsLoading(false);
return false;
}
@@ -88,11 +100,27 @@ export function Organization() {
} finally {
setIsLoading(false);
}
}, [user]);
}, [user?.id, isGitHubConfigured]); // Only depend on user.id, not entire user object
useEffect(() => {
// Reset loading state when component becomes active
setIsLoading(true);
fetchOrganizations();
}, [fetchOrganizations]);
}, [fetchOrganizations, navigationKey]); // Include navigationKey to trigger on navigation
// Register with global live refresh system
useEffect(() => {
// Only register for live refresh if GitHub is configured
if (!isGitHubConfigured) {
return;
}
const unregister = registerRefreshCallback(() => {
fetchOrganizations();
});
return unregister;
}, [registerRefreshCallback, fetchOrganizations, isGitHubConfigured]);
const handleRefresh = async () => {
const success = await fetchOrganizations();
@@ -342,9 +370,13 @@ export function Organization() {
</SelectContent>
</Select>
<Button variant="default" onClick={handleRefresh}>
<RefreshCw className="h-4 w-4 mr-2" />
Refresh
<Button
variant="outline"
size="icon"
onClick={handleRefresh}
title="Refresh organizations"
>
<RefreshCw className="h-4 w-4" />
</Button>
<Button

View File

@@ -27,13 +27,18 @@ import type { SyncRepoRequest, SyncRepoResponse } from "@/types/sync";
import { OwnerCombobox, OrganizationCombobox } from "./RepositoryComboboxes";
import type { RetryRepoRequest, RetryRepoResponse } from "@/types/retry";
import AddRepositoryDialog from "./AddRepositoryDialog";
import type { ConfigApiResponse } from "@/types/config";
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
import { useConfigStatus } from "@/hooks/useConfigStatus";
import { useNavigation } from "@/components/layout/MainLayout";
export default function Repository() {
const [repositories, setRepositories] = useState<Repository[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [isGitHubConfigured, setIsGitHubConfigured] = useState<boolean>(true);
const { user } = useAuth();
const { registerRefreshCallback } = useLiveRefresh();
const { isGitHubConfigured } = useConfigStatus();
const { navigationKey } = useNavigation();
const { filter, setFilter } = useFilterParams({
searchTerm: "",
status: "",
@@ -76,27 +81,15 @@ export default function Repository() {
});
const fetchRepositories = useCallback(async () => {
if (!user) return;
if (!user?.id) return;
// Don't fetch repositories if GitHub is not configured or still loading config
if (!isGitHubConfigured) {
setIsLoading(false);
return false;
}
// First, check if GitHub is configured by fetching the user's config
try {
const configResponse = await apiRequest<ConfigApiResponse>(
`/config?userId=${user.id}`,
{
method: "GET",
}
);
// Check if GitHub credentials are configured
if (!configResponse?.githubConfig?.username || !configResponse?.githubConfig?.token) {
setIsLoading(false);
setIsGitHubConfigured(false);
// Don't show error toast for unconfigured GitHub - just return silently
return false;
}
// GitHub is configured
setIsGitHubConfigured(true);
setIsLoading(true);
const response = await apiRequest<RepositoryApiResponse>(
@@ -121,11 +114,27 @@ export default function Repository() {
} finally {
setIsLoading(false);
}
}, [user]);
}, [user?.id, isGitHubConfigured]); // Only depend on user.id, not entire user object
useEffect(() => {
// Reset loading state when component becomes active
setIsLoading(true);
fetchRepositories();
}, [fetchRepositories]);
}, [fetchRepositories, navigationKey]); // Include navigationKey to trigger on navigation
// Register with global live refresh system
useEffect(() => {
// Only register for live refresh if GitHub is configured
if (!isGitHubConfigured) {
return;
}
const unregister = registerRefreshCallback(() => {
fetchRepositories();
});
return unregister;
}, [registerRefreshCallback, fetchRepositories, isGitHubConfigured]);
const handleRefresh = async () => {
const success = await fetchRepositories();
@@ -442,9 +451,13 @@ export default function Repository() {
</SelectContent>
</Select>
<Button variant="default" onClick={handleRefresh}>
<RefreshCw className="h-4 w-4 mr-2" />
Refresh
<Button
variant="outline"
size="icon"
onClick={handleRefresh}
title="Refresh repositories"
>
<RefreshCw className="h-4 w-4" />
</Button>
<Button
@@ -465,7 +478,11 @@ export default function Repository() {
</p>
<Button
variant="default"
onClick={() => window.location.href = "/config"}
onClick={() => {
window.history.pushState({}, '', '/config');
// We need to trigger a page change event for the navigation system
window.dispatchEvent(new PopStateEvent('popstate'));
}}
>
Go to Configuration
</Button>

View File

@@ -2,12 +2,13 @@ import { useMemo, useRef } from "react";
import Fuse from "fuse.js";
import { useVirtualizer } from "@tanstack/react-virtual";
import { GitFork, RefreshCw, RotateCcw } from "lucide-react";
import { SiGithub } from "react-icons/si";
import { SiGithub, SiGitea } from "react-icons/si";
import type { Repository } from "@/lib/db/schema";
import { Button } from "@/components/ui/button";
import { formatDate, getStatusColor } from "@/lib/utils";
import type { FilterParams } from "@/types/filter";
import { Skeleton } from "@/components/ui/skeleton";
import { useGiteaConfig } from "@/hooks/useGiteaConfig";
interface RepositoryTableProps {
repositories: Repository[];
@@ -31,6 +32,37 @@ export default function RepositoryTable({
loadingRepoIds,
}: RepositoryTableProps) {
const tableParentRef = useRef<HTMLDivElement>(null);
const { giteaConfig } = useGiteaConfig();
// Helper function to construct Gitea repository URL
const getGiteaRepoUrl = (repository: Repository): string | null => {
if (!giteaConfig?.url) {
return null;
}
// Only provide Gitea links for repositories that have been or are being mirrored
const validStatuses = ['mirroring', 'mirrored', 'syncing', 'synced'];
if (!validStatuses.includes(repository.status)) {
return null;
}
// Use mirroredLocation if available, otherwise construct from repository data
let repoPath: string;
if (repository.mirroredLocation) {
repoPath = repository.mirroredLocation;
} else {
// Fallback: construct the path based on repository data
const owner = repository.organization || repository.owner;
repoPath = `${owner}/${repository.name}`;
}
// Ensure the base URL doesn't have a trailing slash
const baseUrl = giteaConfig.url.endsWith('/')
? giteaConfig.url.slice(0, -1)
: giteaConfig.url;
return `${baseUrl}/${repoPath}`;
};
const hasAnyFilter = Object.values(filter).some(
(val) => val?.toString().trim() !== ""
@@ -85,9 +117,12 @@ export default function RepositoryTable({
Last Mirrored
</div>
<div className="h-full p-3 text-sm font-medium flex-[1]">Status</div>
<div className="h-full p-3 text-sm font-medium flex-[1] text-right">
<div className="h-full p-3 text-sm font-medium flex-[1]">
Actions
</div>
<div className="h-full p-3 text-sm font-medium flex-[0.8] text-center">
Links
</div>
</div>
{Array.from({ length: 5 }).map((_, i) => (
@@ -110,7 +145,10 @@ export default function RepositoryTable({
<div className="h-full p-3 text-sm font-medium flex-[1]">
<Skeleton className="h-full w-full" />
</div>
<div className="h-full p-3 text-sm font-medium flex-[1] text-right">
<div className="h-full p-3 text-sm font-medium flex-[1]">
<Skeleton className="h-full w-full" />
</div>
<div className="h-full p-3 text-sm font-medium flex-[0.8] text-center">
<Skeleton className="h-full w-full" />
</div>
</div>
@@ -158,15 +196,18 @@ export default function RepositoryTable({
Last Mirrored
</div>
<div className="h-full p-3 text-sm font-medium flex-[1]">Status</div>
<div className="h-full p-3 text-sm font-medium flex-[1] text-right">
<div className="h-full p-3 text-sm font-medium flex-[1]">
Actions
</div>
<div className="h-full p-3 text-sm font-medium flex-[0.8] text-center">
Links
</div>
</div>
{/* table body wrapper (for a parent in virtualization) */}
<div
ref={tableParentRef}
className="flex flex-col max-h-[calc(100dvh-236px)] overflow-y-auto" //the height is set according to the other contents
className="flex flex-col max-h-[calc(100dvh-276px)] overflow-y-auto" //adjusted height to account for status bar
>
<div
style={{
@@ -238,61 +279,60 @@ export default function RepositoryTable({
</div>
{/* Actions */}
<div className="h-full p-3 flex items-center justify-end gap-x-2 flex-[1]">
{/* {repo.status === "mirrored" ||
repo.status === "syncing" ||
repo.status === "synced" ? (
<Button
variant="ghost"
disabled={repo.status === "syncing" || isLoading}
onClick={() => onSync({ repoId: repo.id ?? "" })}
>
{isLoading ? (
<>
<RefreshCw className="h-4 w-4 animate-spin mr-1" />
Sync
</>
) : (
<>
<RefreshCw className="h-4 w-4 mr-1" />
Sync
</>
)}
</Button>
) : (
<Button
variant="ghost"
disabled={repo.status === "mirroring" || isLoading}
onClick={() => onMirror({ repoId: repo.id ?? "" })}
>
{isLoading ? (
<>
<RefreshCw className="h-4 w-4 animate-spin mr-1" />
Mirror
</>
) : (
<>
<RefreshCw className="h-4 w-4 mr-1" />
Mirror
</>
)}
</Button>
)} */}
<div className="h-full p-3 flex items-center justify-start flex-[1]">
<RepoActionButton
repo={{ id: repo.id ?? "", status: repo.status }}
isLoading={isLoading}
onMirror={({ repoId }) =>
onMirror({ repoId: repo.id ?? "" })
}
onSync={({ repoId }) => onSync({ repoId: repo.id ?? "" })}
onRetry={({ repoId }) => onRetry({ repoId: repo.id ?? "" })}
onMirror={() => onMirror({ repoId: repo.id ?? "" })}
onSync={() => onSync({ repoId: repo.id ?? "" })}
onRetry={() => onRetry({ repoId: repo.id ?? "" })}
/>
</div>
{/* Links */}
<div className="h-full p-3 flex items-center justify-center gap-x-2 flex-[0.8]">
{(() => {
const giteaUrl = getGiteaRepoUrl(repo);
// Determine tooltip based on status and configuration
let tooltip: string;
if (!giteaConfig?.url) {
tooltip = "Gitea not configured";
} else if (repo.status === 'imported') {
tooltip = "Repository not yet mirrored to Gitea";
} else if (repo.status === 'failed') {
tooltip = "Repository mirroring failed";
} else if (repo.status === 'mirroring') {
tooltip = "Repository is being mirrored to Gitea";
} else if (giteaUrl) {
tooltip = "View on Gitea";
} else {
tooltip = "Gitea repository not available";
}
return giteaUrl ? (
<Button variant="ghost" size="icon" asChild>
<a
href={giteaUrl}
target="_blank"
rel="noopener noreferrer"
title={tooltip}
>
<SiGitea className="h-4 w-4" />
</a>
</Button>
) : (
<Button variant="ghost" size="icon" disabled title={tooltip}>
<SiGitea className="h-4 w-4" />
</Button>
);
})()}
<Button variant="ghost" size="icon" asChild>
<a
href={repo.url}
target="_blank"
rel="noopener noreferrer"
title="View on GitHub"
>
<SiGithub className="h-4 w-4" />
</a>
@@ -303,6 +343,23 @@ export default function RepositoryTable({
})}
</div>
</div>
{/* Status Bar */}
<div className="h-[40px] flex items-center justify-between border-t bg-muted/30 px-3">
<div className="flex items-center gap-2">
<div className="h-1.5 w-1.5 rounded-full bg-primary" />
<span className="text-sm font-medium text-foreground">
{hasAnyFilter
? `Showing ${filteredRepositories.length} of ${repositories.length} repositories`
: `${repositories.length} ${repositories.length === 1 ? 'repository' : 'repositories'} total`}
</span>
</div>
{hasAnyFilter && (
<span className="text-xs text-muted-foreground">
Filters applied
</span>
)}
</div>
</div>
);
}
@@ -316,12 +373,10 @@ function RepoActionButton({
}: {
repo: { id: string; status: string };
isLoading: boolean;
onMirror: ({ repoId }: { repoId: string }) => void;
onSync: ({ repoId }: { repoId: string }) => void;
onRetry: ({ repoId }: { repoId: string }) => void;
onMirror: () => void;
onSync: () => void;
onRetry: () => void;
}) {
const repoId = repo.id ?? "";
let label = "";
let icon = <></>;
let onClick = () => {};
@@ -330,23 +385,28 @@ function RepoActionButton({
if (repo.status === "failed") {
label = "Retry";
icon = <RotateCcw className="h-4 w-4 mr-1" />;
onClick = () => onRetry({ repoId });
onClick = onRetry;
} else if (["mirrored", "synced", "syncing"].includes(repo.status)) {
label = "Sync";
icon = <RefreshCw className="h-4 w-4 mr-1" />;
onClick = () => onSync({ repoId });
onClick = onSync;
disabled ||= repo.status === "syncing";
} else if (["imported", "mirroring"].includes(repo.status)) {
label = "Mirror";
icon = <RefreshCw className="h-4 w-4 mr-1" />;
onClick = () => onMirror({ repoId });
icon = <GitFork className="h-4 w-4 mr-1" />;
onClick = onMirror;
disabled ||= repo.status === "mirroring";
} else {
return null; // unsupported status
}
return (
<Button variant="ghost" disabled={disabled} onClick={onClick}>
<Button
variant="ghost"
disabled={disabled}
onClick={onClick}
className="min-w-[80px] justify-start"
>
{isLoading ? (
<>
<RefreshCw className="h-4 w-4 animate-spin mr-1" />

View File

@@ -0,0 +1,29 @@
import * as React from "react"
import * as SwitchPrimitive from "@radix-ui/react-switch"
import { cn } from "@/lib/utils"
function Switch({
className,
...props
}: React.ComponentProps<typeof SwitchPrimitive.Root>) {
return (
<SwitchPrimitive.Root
data-slot="switch"
className={cn(
"peer data-[state=checked]:bg-primary data-[state=unchecked]:bg-input focus-visible:border-ring focus-visible:ring-ring/50 dark:data-[state=unchecked]:bg-input/80 inline-flex h-[1.15rem] w-8 shrink-0 items-center rounded-full border border-transparent shadow-xs transition-all outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
className
)}
{...props}
>
<SwitchPrimitive.Thumb
data-slot="switch-thumb"
className={cn(
"bg-background dark:data-[state=unchecked]:bg-foreground dark:data-[state=checked]:bg-primary-foreground pointer-events-none block size-4 rounded-full ring-0 transition-transform data-[state=checked]:translate-x-[calc(100%-2px)] data-[state=unchecked]:translate-x-0"
)}
/>
</SwitchPrimitive.Root>
)
}
export { Switch }

View File

@@ -2,7 +2,7 @@
title: "Architecture"
description: "Comprehensive overview of the Gitea Mirror application architecture."
order: 1
updatedDate: 2023-10-15
updatedDate: 2025-05-22
---
<div class="mb-6">
@@ -21,17 +21,18 @@ The application is built using:
- <span class="font-semibold text-foreground">Astro</span>: Web framework for the frontend
- <span class="font-semibold text-foreground">React</span>: Component library for interactive UI elements
- <span class="font-semibold text-foreground">Shadcn UI</span>: UI component library built on Tailwind CSS
- <span class="font-semibold text-foreground">SQLite</span>: Database for storing configuration and state
- <span class="font-semibold text-foreground">SQLite</span>: Database for storing configuration, state, and events
- <span class="font-semibold text-foreground">Bun</span>: Runtime environment for the backend
- <span class="font-semibold text-foreground">Drizzle ORM</span>: Type-safe ORM for database interactions
## Architecture Diagram
```mermaid
graph TD
subgraph "Gitea Mirror"
Frontend["Frontend<br/>(Astro)"]
Frontend["Frontend<br/>(Astro + React)"]
Backend["Backend<br/>(Bun)"]
Database["Database<br/>(SQLite)"]
Database["Database<br/>(SQLite + Drizzle)"]
Frontend <--> Backend
Backend <--> Database
@@ -70,14 +71,15 @@ The backend is built with Bun and provides API endpoints for the frontend to int
- Mirroring operations
- Database interactions
### Database (SQLite)
### Database (SQLite + Drizzle ORM)
SQLite is used for data persistence, storing:
SQLite with Bun's native SQLite driver is used for data persistence, with Drizzle ORM providing type-safe database interactions. The database stores:
- User accounts and authentication data
- GitHub and Gitea configuration
- Repository and organization information
- Mirroring job history and status
- Event notifications and their read status
## Data Flow
@@ -93,11 +95,29 @@ SQLite is used for data persistence, storing:
gitea-mirror/
├── src/ # Source code
│ ├── components/ # React components
│ ├── content/ # Documentation and content
│ ├── layouts/ # Astro layout components
│ ├── lib/ # Utility functions and database
│ ├── pages/ # Astro pages and API routes
│ └── styles/ # CSS and Tailwind styles
├── public/ # Static assets
├── data/ # Database and persistent data
── docker/ # Docker configuration
── docker/ # Docker configuration
└── scripts/ # Utility scripts for deployment and maintenance
├── gitea-mirror-lxc-local.sh # Local LXC deployment script
└── manage-db.ts # Database management tool
```
## Deployment Options
Gitea Mirror supports multiple deployment options:
1. **Docker**: Run as a containerized application using Docker and docker-compose
2. **LXC Containers**: Deploy in Linux Containers (LXC) on Proxmox VE (using community script by [Tobias/CrazyWolf13](https://github.com/CrazyWolf13)) or local workstations
3. **Native**: Run directly on the host system using Bun runtime
Each deployment method has its own advantages:
- **Docker**: Isolation, easy updates, consistent environment
- **LXC**: Lightweight virtualization, better performance than Docker, system-level isolation
- **Native**: Best performance, direct access to system resources

View File

@@ -2,7 +2,7 @@
title: "Configuration"
description: "Guide to configuring Gitea Mirror for your environment."
order: 2
updatedDate: 2023-10-15
updatedDate: 2025-05-22
---
<div class="mb-6">
@@ -24,14 +24,16 @@ The following environment variables can be used to configure Gitea Mirror:
| Variable | Description | Default Value | Example |
|----------|-------------|---------------|---------|
| `NODE_ENV` | Runtime environment (development, production, test) | `development` | `production` |
| `DATABASE_URL` | SQLite database URL | `sqlite://data/gitea-mirror.db` | `sqlite://path/to/your/database.db` |
| `JWT_SECRET` | Secret key for JWT authentication | `your-secret-key-change-this-in-production` | `your-secure-random-string` |
| `DATABASE_URL` | SQLite database URL | `file:data/gitea-mirror.db` | `file:path/to/your/database.db` |
| `JWT_SECRET` | Secret key for JWT authentication | Auto-generated secure random string | `your-secure-random-string` |
| `HOST` | Server host | `localhost` | `0.0.0.0` |
| `PORT` | Server port | `3000` | `8080` |
| `PORT` | Server port | `4321` | `8080` |
### Important Security Note
In production environments, you should always set a strong, unique `JWT_SECRET` to ensure secure authentication.
The application will automatically generate a secure random `JWT_SECRET` on first run if one isn't provided or if the default value is used. This generated secret is stored in the data directory for persistence across container restarts.
While this auto-generation feature provides good security by default, you can still explicitly set your own `JWT_SECRET` for complete control over your deployment.
## Web UI Configuration
@@ -118,3 +120,57 @@ Example patterns:
- `*` - All repositories
- `org-name/*` - All repositories in a specific organization
- `username/repo-name` - A specific repository
### Database Management
Gitea Mirror includes several database management tools that can be run from the command line:
```bash
# Initialize the database (only if it doesn't exist)
bun run init-db
# Check database status
bun run check-db
# Fix database location issues
bun run fix-db
# Reset all users (for testing signup flow)
bun run reset-users
# Remove database files completely
bun run cleanup-db
```
### Event Management
Events in Gitea Mirror (such as repository mirroring operations) are stored in the SQLite database. You can manage these events using the following scripts:
```bash
# View all events in the database
bun scripts/check-events.ts
# Mark all events as read
bun scripts/mark-events-read.ts
```
For cleaning up old activities and events, use the cleanup button in the Activity Log page of the web interface.
### Health Check Endpoint
Gitea Mirror includes a built-in health check endpoint at `/api/health` that provides:
- System status and uptime
- Database connectivity check
- Memory usage statistics
- Environment information
You can use this endpoint for monitoring your deployment:
```bash
# Basic check (returns 200 OK if healthy)
curl -I http://your-server:port/api/health
# Detailed health information (JSON)
curl http://your-server:port/api/health
```

View File

@@ -2,7 +2,7 @@
title: "Quick Start Guide"
description: "Get started with Gitea Mirror quickly."
order: 3
updatedDate: 2023-10-15
updatedDate: 2025-05-22
---
<div class="mb-6">
@@ -16,13 +16,16 @@ Before you begin, make sure you have:
1. <span class="font-semibold text-foreground">A GitHub account with a personal access token</span>
2. <span class="font-semibold text-foreground">A Gitea instance with an access token</span>
3. <span class="font-semibold text-foreground">Docker and docker-compose (recommended) or Bun 1.2.9+ installed</span>
3. <span class="font-semibold text-foreground">One of the following:</span>
- Docker and docker-compose (for Docker deployment)
- Bun 1.2.9+ (for native deployment)
- Proxmox VE or LXD (for LXC container deployment)
## Installation Options
Choose the installation method that works best for your environment.
### Using Docker (Recommended)
### Using Docker (Recommended for most users)
Docker provides the easiest way to get started with minimal configuration.
@@ -34,12 +37,12 @@ Docker provides the easiest way to get started with minimal configuration.
2. Start the application in production mode:
```bash
docker-compose --profile production up -d
docker compose up -d
```
3. Access the application at [http://localhost:4321](http://localhost:4321)
### Manual Installation
### Using Bun (Native Installation)
If you prefer to run the application directly on your system:
@@ -62,6 +65,11 @@ If you prefer to run the application directly on your system:
bun run dev
```
Note: For Bun-specific features, use:
```bash
bunx --bun astro dev
```
**Production Mode:**
```bash
bun run build
@@ -70,6 +78,44 @@ If you prefer to run the application directly on your system:
4. Access the application at [http://localhost:4321](http://localhost:4321)
### Using LXC Containers (Recommended for server deployments)
#### Proxmox VE (Online Installation)
For deploying on a Proxmox VE host with internet access:
```bash
# Optional env overrides: CTID HOSTNAME STORAGE DISK_SIZE CORES MEMORY BRIDGE IP_CONF
sudo bash -c "$(curl -fsSL https://raw.githubusercontent.com/arunavo4/gitea-mirror/main/scripts/gitea-mirror-lxc-proxmox.sh)"
```
This script:
- Creates a privileged LXC container
- Installs Bun and dependencies
- Clones and builds the application
- Sets up a systemd service
#### Local LXD (Offline-friendly Installation)
For testing on a local workstation or in environments without internet access:
1. Clone the repository locally:
```bash
git clone https://github.com/arunavo4/gitea-mirror.git
```
2. Download the Bun installer once:
```bash
curl -L -o /tmp/bun-linux-x64.zip https://github.com/oven-sh/bun/releases/latest/download/bun-linux-x64.zip
```
3. Run the local LXC installer:
```bash
sudo LOCAL_REPO_DIR=~/path/to/gitea-mirror ./gitea-mirror/scripts/gitea-mirror-lxc-local.sh
```
For more details on LXC deployment, see the [LXC Container Deployment Guide](https://github.com/arunavo4/gitea-mirror/blob/main/scripts/README-lxc.md).
## Initial Configuration
Follow these steps to configure Gitea Mirror for first use:
@@ -116,7 +162,12 @@ If you encounter any issues:
- Check the Activity Log for detailed error messages
- Verify your GitHub and Gitea tokens have the correct permissions
- Ensure your Gitea instance is accessible from the machine running Gitea Mirror
- For Docker installations, check container logs with `docker logs gitea-mirror`
- Check logs based on your deployment method:
- Docker: `docker logs gitea-mirror`
- Native: Check the terminal output or system logs
- LXC: `systemctl status gitea-mirror` or `journalctl -u gitea-mirror -f`
- Use the health check endpoint to verify system status: `curl http://your-server:4321/api/health`
- For database issues, try the database management tools: `bun run check-db` or `bun run fix-db`
## Next Steps
@@ -125,3 +176,7 @@ After your initial setup:
- Explore the dashboard for an overview of your mirroring status
- Set up automatic mirroring schedules for hands-off operation
- Configure organization mirroring for team repositories
- Check out the [Configuration Guide](/configuration) for advanced settings
- Review the [Architecture Documentation](/architecture) to understand the system
- For server deployments, set up monitoring using the health check endpoint
- Use the cleanup button in the Activity Log page to manage old events and activities

View File

@@ -0,0 +1,154 @@
import { useCallback, useEffect, useState, useRef } from 'react';
import { useAuth } from './useAuth';
import { apiRequest } from '@/lib/utils';
import type { ConfigApiResponse } from '@/types/config';
interface ConfigStatus {
isGitHubConfigured: boolean;
isGiteaConfigured: boolean;
isFullyConfigured: boolean;
isLoading: boolean;
error: string | null;
}
// Cache to prevent duplicate API calls across components
let configCache: { data: ConfigApiResponse | null; timestamp: number; userId: string | null } = {
data: null,
timestamp: 0,
userId: null
};
const CACHE_DURATION = 30000; // 30 seconds cache
/**
* Hook to check if GitHub and Gitea are properly configured
* Returns configuration status and prevents unnecessary API calls when not configured
* Uses caching to prevent duplicate API calls across components
*/
export function useConfigStatus(): ConfigStatus {
const { user } = useAuth();
const [configStatus, setConfigStatus] = useState<ConfigStatus>({
isGitHubConfigured: false,
isGiteaConfigured: false,
isFullyConfigured: false,
isLoading: true,
error: null,
});
// Track if this hook has already checked config to prevent multiple calls
const hasCheckedRef = useRef(false);
const checkConfiguration = useCallback(async () => {
if (!user?.id) {
setConfigStatus({
isGitHubConfigured: false,
isGiteaConfigured: false,
isFullyConfigured: false,
isLoading: false,
error: 'No user found',
});
return;
}
// Check cache first
const now = Date.now();
const isCacheValid = configCache.data &&
configCache.userId === user.id &&
(now - configCache.timestamp) < CACHE_DURATION;
if (isCacheValid && hasCheckedRef.current) {
const configResponse = configCache.data!;
const isGitHubConfigured = !!(
configResponse?.githubConfig?.username &&
configResponse?.githubConfig?.token
);
const isGiteaConfigured = !!(
configResponse?.giteaConfig?.url &&
configResponse?.giteaConfig?.username &&
configResponse?.giteaConfig?.token
);
const isFullyConfigured = isGitHubConfigured && isGiteaConfigured;
setConfigStatus({
isGitHubConfigured,
isGiteaConfigured,
isFullyConfigured,
isLoading: false,
error: null,
});
return;
}
try {
// Only show loading if we haven't checked before or cache is invalid
if (!hasCheckedRef.current) {
setConfigStatus(prev => ({ ...prev, isLoading: true, error: null }));
}
const configResponse = await apiRequest<ConfigApiResponse>(
`/config?userId=${user.id}`,
{ method: 'GET' }
);
// Update cache
configCache = {
data: configResponse,
timestamp: now,
userId: user.id
};
const isGitHubConfigured = !!(
configResponse?.githubConfig?.username &&
configResponse?.githubConfig?.token
);
const isGiteaConfigured = !!(
configResponse?.giteaConfig?.url &&
configResponse?.giteaConfig?.username &&
configResponse?.giteaConfig?.token
);
const isFullyConfigured = isGitHubConfigured && isGiteaConfigured;
setConfigStatus({
isGitHubConfigured,
isGiteaConfigured,
isFullyConfigured,
isLoading: false,
error: null,
});
hasCheckedRef.current = true;
} catch (error) {
setConfigStatus({
isGitHubConfigured: false,
isGiteaConfigured: false,
isFullyConfigured: false,
isLoading: false,
error: error instanceof Error ? error.message : 'Failed to check configuration',
});
hasCheckedRef.current = true;
}
}, [user?.id]);
useEffect(() => {
checkConfiguration();
}, [checkConfiguration]);
return configStatus;
}
// Export function to invalidate cache when config is updated
export function invalidateConfigCache() {
configCache = { data: null, timestamp: 0, userId: null };
}
// Export function to get cached config data for other hooks
export function getCachedConfig(): ConfigApiResponse | null {
const now = Date.now();
const isCacheValid = configCache.data && (now - configCache.timestamp) < CACHE_DURATION;
return isCacheValid ? configCache.data : null;
}

View File

@@ -0,0 +1,73 @@
import { useCallback, useEffect, useState } from 'react';
import { useAuth } from './useAuth';
import { apiRequest } from '@/lib/utils';
import type { ConfigApiResponse, GiteaConfig } from '@/types/config';
import { getCachedConfig } from './useConfigStatus';
interface GiteaConfigHook {
giteaConfig: GiteaConfig | null;
isLoading: boolean;
error: string | null;
}
/**
* Hook to get Gitea configuration data
* Uses the same cache as useConfigStatus to prevent duplicate API calls
*/
export function useGiteaConfig(): GiteaConfigHook {
const { user } = useAuth();
const [giteaConfigState, setGiteaConfigState] = useState<GiteaConfigHook>({
giteaConfig: null,
isLoading: true,
error: null,
});
const fetchGiteaConfig = useCallback(async () => {
if (!user?.id) {
setGiteaConfigState({
giteaConfig: null,
isLoading: false,
error: 'User not authenticated',
});
return;
}
// Try to get from cache first
const cachedConfig = getCachedConfig();
if (cachedConfig) {
setGiteaConfigState({
giteaConfig: cachedConfig.giteaConfig || null,
isLoading: false,
error: null,
});
return;
}
try {
setGiteaConfigState(prev => ({ ...prev, isLoading: true, error: null }));
const configResponse = await apiRequest<ConfigApiResponse>(
`/config?userId=${user.id}`,
{ method: 'GET' }
);
setGiteaConfigState({
giteaConfig: configResponse?.giteaConfig || null,
isLoading: false,
error: null,
});
} catch (error) {
setGiteaConfigState({
giteaConfig: null,
isLoading: false,
error: error instanceof Error ? error.message : 'Failed to fetch Gitea configuration',
});
}
}, [user?.id]);
useEffect(() => {
fetchGiteaConfig();
}, [fetchGiteaConfig]);
return giteaConfigState;
}

102
src/hooks/useLiveRefresh.ts Normal file
View File

@@ -0,0 +1,102 @@
import * as React from "react";
import { useState, useEffect, createContext, useContext, useCallback, useRef } from "react";
import { usePageVisibility } from "./usePageVisibility";
import { useConfigStatus } from "./useConfigStatus";
interface LiveRefreshContextType {
isLiveEnabled: boolean;
toggleLive: () => void;
registerRefreshCallback: (callback: () => void) => () => void;
}
const LiveRefreshContext = createContext<LiveRefreshContextType | undefined>(undefined);
const LIVE_REFRESH_INTERVAL = 3000; // 3 seconds
const SESSION_STORAGE_KEY = 'gitea-mirror-live-refresh';
export function LiveRefreshProvider({ children }: { children: React.ReactNode }) {
const [isLiveEnabled, setIsLiveEnabled] = useState<boolean>(false);
const isPageVisible = usePageVisibility();
const { isFullyConfigured } = useConfigStatus();
const refreshCallbacksRef = useRef<Set<() => void>>(new Set());
const intervalRef = useRef<NodeJS.Timeout | null>(null);
// Load initial state from session storage
useEffect(() => {
const savedState = sessionStorage.getItem(SESSION_STORAGE_KEY);
if (savedState === 'true') {
setIsLiveEnabled(true);
}
}, []);
// Save state to session storage whenever it changes
useEffect(() => {
sessionStorage.setItem(SESSION_STORAGE_KEY, isLiveEnabled.toString());
}, [isLiveEnabled]);
// Execute all registered refresh callbacks
const executeRefreshCallbacks = useCallback(() => {
refreshCallbacksRef.current.forEach(callback => {
try {
callback();
} catch (error) {
console.error('Error executing refresh callback:', error);
}
});
}, []);
// Setup/cleanup the refresh interval
useEffect(() => {
// Clear existing interval
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
// Only set up interval if live is enabled, page is visible, and configuration is complete
if (isLiveEnabled && isPageVisible && isFullyConfigured) {
intervalRef.current = setInterval(executeRefreshCallbacks, LIVE_REFRESH_INTERVAL);
}
// Cleanup on unmount
return () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
};
}, [isLiveEnabled, isPageVisible, isFullyConfigured, executeRefreshCallbacks]);
const toggleLive = useCallback(() => {
setIsLiveEnabled(prev => !prev);
}, []);
const registerRefreshCallback = useCallback((callback: () => void) => {
refreshCallbacksRef.current.add(callback);
// Return cleanup function
return () => {
refreshCallbacksRef.current.delete(callback);
};
}, []);
const contextValue = {
isLiveEnabled,
toggleLive,
registerRefreshCallback,
};
return React.createElement(
LiveRefreshContext.Provider,
{ value: contextValue },
children
);
}
export function useLiveRefresh() {
const context = useContext(LiveRefreshContext);
if (context === undefined) {
throw new Error("useLiveRefresh must be used within a LiveRefreshProvider");
}
return context;
}

View File

@@ -0,0 +1,28 @@
import { useEffect, useState } from 'react';
/**
* Hook to detect if the page/tab is currently visible
* Returns false when user switches to another tab or minimizes the window
*/
export function usePageVisibility(): boolean {
const [isVisible, setIsVisible] = useState<boolean>(true);
useEffect(() => {
const handleVisibilityChange = () => {
setIsVisible(!document.hidden);
};
// Set initial state
setIsVisible(!document.hidden);
// Listen for visibility changes
document.addEventListener('visibilitychange', handleVisibilityChange);
// Cleanup
return () => {
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, []);
return isVisible;
}

View File

@@ -88,3 +88,84 @@ export const giteaApi = {
body: JSON.stringify({ url, token }),
}),
};
// Health API
export interface HealthResponse {
status: "ok" | "error";
timestamp: string;
version: string;
latestVersion: string;
updateAvailable: boolean;
database: {
connected: boolean;
message: string;
};
system: {
uptime: {
startTime: string;
uptimeMs: number;
formatted: string;
};
memory: {
rss: string;
heapTotal: string;
heapUsed: string;
external: string;
systemTotal: string;
systemFree: string;
};
os: {
platform: string;
version: string;
arch: string;
};
env: string;
};
error?: string;
}
export const healthApi = {
check: async (): Promise<HealthResponse> => {
try {
const response = await fetch(`${API_BASE}/health`);
if (!response.ok) {
const errorData = await response.json().catch(() => ({
status: "error",
error: "Failed to parse error response",
}));
return {
...errorData,
status: "error",
timestamp: new Date().toISOString(),
} as HealthResponse;
}
return await response.json();
} catch (error) {
return {
status: "error",
timestamp: new Date().toISOString(),
error: error instanceof Error ? error.message : "Unknown error checking health",
version: "unknown",
latestVersion: "unknown",
updateAvailable: false,
database: { connected: false, message: "Failed to connect to API" },
system: {
uptime: { startTime: "", uptimeMs: 0, formatted: "N/A" },
memory: {
rss: "N/A",
heapTotal: "N/A",
heapUsed: "N/A",
external: "N/A",
systemTotal: "N/A",
systemFree: "N/A",
},
os: { platform: "", version: "", arch: "" },
env: "",
},
};
}
},
};

191
src/lib/cleanup-service.ts Normal file
View File

@@ -0,0 +1,191 @@
/**
* Background cleanup service for automatic database maintenance
* This service runs periodically to clean up old events and mirror jobs
* based on user configuration settings
*/
import { db, configs, events, mirrorJobs } from "@/lib/db";
import { eq, lt, and } from "drizzle-orm";
interface CleanupResult {
userId: string;
eventsDeleted: number;
mirrorJobsDeleted: number;
error?: string;
}
/**
* Clean up old events and mirror jobs for a specific user
*/
async function cleanupForUser(userId: string, retentionDays: number): Promise<CleanupResult> {
try {
console.log(`Running cleanup for user ${userId} with ${retentionDays} days retention`);
// Calculate cutoff date
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
let eventsDeleted = 0;
let mirrorJobsDeleted = 0;
// Clean up old events
const eventsResult = await db
.delete(events)
.where(
and(
eq(events.userId, userId),
lt(events.createdAt, cutoffDate)
)
);
eventsDeleted = eventsResult.changes || 0;
// Clean up old mirror jobs (only completed ones)
const jobsResult = await db
.delete(mirrorJobs)
.where(
and(
eq(mirrorJobs.userId, userId),
eq(mirrorJobs.inProgress, false),
lt(mirrorJobs.timestamp, cutoffDate)
)
);
mirrorJobsDeleted = jobsResult.changes || 0;
console.log(`Cleanup completed for user ${userId}: ${eventsDeleted} events, ${mirrorJobsDeleted} jobs deleted`);
return {
userId,
eventsDeleted,
mirrorJobsDeleted,
};
} catch (error) {
console.error(`Error during cleanup for user ${userId}:`, error);
return {
userId,
eventsDeleted: 0,
mirrorJobsDeleted: 0,
error: error instanceof Error ? error.message : 'Unknown error',
};
}
}
/**
* Update the cleanup configuration with last run time and calculate next run
*/
async function updateCleanupConfig(userId: string, cleanupConfig: any) {
try {
const now = new Date();
const nextRun = new Date(now.getTime() + 24 * 60 * 60 * 1000); // Next day
const updatedConfig = {
...cleanupConfig,
lastRun: now,
nextRun: nextRun,
};
await db
.update(configs)
.set({
cleanupConfig: updatedConfig,
updatedAt: now,
})
.where(eq(configs.userId, userId));
console.log(`Updated cleanup config for user ${userId}, next run: ${nextRun.toISOString()}`);
} catch (error) {
console.error(`Error updating cleanup config for user ${userId}:`, error);
}
}
/**
* Run automatic cleanup for all users with cleanup enabled
*/
export async function runAutomaticCleanup(): Promise<CleanupResult[]> {
try {
console.log('Starting automatic cleanup service...');
// Get all users with cleanup enabled
const userConfigs = await db
.select()
.from(configs)
.where(eq(configs.isActive, true));
const results: CleanupResult[] = [];
const now = new Date();
for (const config of userConfigs) {
try {
const cleanupConfig = config.cleanupConfig;
// Skip if cleanup is not enabled
if (!cleanupConfig?.enabled) {
continue;
}
// Check if it's time to run cleanup
const nextRun = cleanupConfig.nextRun ? new Date(cleanupConfig.nextRun) : null;
// If nextRun is null or in the past, run cleanup
if (!nextRun || now >= nextRun) {
const result = await cleanupForUser(config.userId, cleanupConfig.retentionDays || 7);
results.push(result);
// Update the cleanup config with new run times
await updateCleanupConfig(config.userId, cleanupConfig);
} else {
console.log(`Skipping cleanup for user ${config.userId}, next run: ${nextRun.toISOString()}`);
}
} catch (error) {
console.error(`Error processing cleanup for user ${config.userId}:`, error);
results.push({
userId: config.userId,
eventsDeleted: 0,
mirrorJobsDeleted: 0,
error: error instanceof Error ? error.message : 'Unknown error',
});
}
}
console.log(`Automatic cleanup completed. Processed ${results.length} users.`);
return results;
} catch (error) {
console.error('Error in automatic cleanup service:', error);
return [];
}
}
/**
* Start the cleanup service with periodic execution
* This should be called when the application starts
*/
export function startCleanupService() {
console.log('Starting background cleanup service...');
// Run cleanup every hour
const CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour in milliseconds
// Run initial cleanup after 5 minutes to allow app to fully start
setTimeout(() => {
runAutomaticCleanup().catch(error => {
console.error('Error in initial cleanup run:', error);
});
}, 5 * 60 * 1000); // 5 minutes
// Set up periodic cleanup
setInterval(() => {
runAutomaticCleanup().catch(error => {
console.error('Error in periodic cleanup run:', error);
});
}, CLEANUP_INTERVAL);
console.log(`✅ Cleanup service started. Will run every ${CLEANUP_INTERVAL / 1000 / 60} minutes.`);
}
/**
* Stop the cleanup service (for testing or shutdown)
*/
export function stopCleanupService() {
// Note: In a real implementation, you'd want to track the interval ID
// and clear it here. For now, this is a placeholder.
console.log('Cleanup service stop requested (not implemented)');
}

42
src/lib/db/index.test.ts Normal file
View File

@@ -0,0 +1,42 @@
import { describe, test, expect, mock, beforeAll, afterAll } from "bun:test";
import { drizzle } from "drizzle-orm/bun-sqlite";
// Silence console logs during tests
let originalConsoleLog: typeof console.log;
beforeAll(() => {
// Save original console.log
originalConsoleLog = console.log;
// Replace with no-op function
console.log = () => {};
});
afterAll(() => {
// Restore original console.log
console.log = originalConsoleLog;
});
// Mock the database module
mock.module("bun:sqlite", () => {
return {
Database: mock(function() {
return {
query: mock(() => ({
all: mock(() => []),
run: mock(() => ({}))
}))
};
})
};
});
// Mock the database tables
describe("Database Schema", () => {
test("database connection can be created", async () => {
// Import the db from the module
const { db } = await import("./index");
// Check that db is defined
expect(db).toBeDefined();
});
});

View File

@@ -81,6 +81,7 @@ export const events = sqliteTable("events", {
const githubSchema = configSchema.shape.githubConfig;
const giteaSchema = configSchema.shape.giteaConfig;
const scheduleSchema = configSchema.shape.scheduleConfig;
const cleanupSchema = configSchema.shape.cleanupConfig;
export const configs = sqliteTable("configs", {
id: text("id").primaryKey(),
@@ -112,6 +113,10 @@ export const configs = sqliteTable("configs", {
.$type<z.infer<typeof scheduleSchema>>()
.notNull(),
cleanupConfig: text("cleanup_config", { mode: "json" })
.$type<z.infer<typeof cleanupSchema>>()
.notNull(),
createdAt: integer("created_at", { mode: "timestamp" })
.notNull()
.default(new Date()),
@@ -189,6 +194,18 @@ export const mirrorJobs = sqliteTable("mirror_jobs", {
timestamp: integer("timestamp", { mode: "timestamp" })
.notNull()
.default(new Date()),
// New fields for job resilience
jobType: text("job_type").notNull().default("mirror"),
batchId: text("batch_id"),
totalItems: integer("total_items"),
completedItems: integer("completed_items").default(0),
itemIds: text("item_ids", { mode: "json" }).$type<string[]>(),
completedItemIds: text("completed_item_ids", { mode: "json" }).$type<string[]>().default([]),
inProgress: integer("in_progress", { mode: "boolean" }).notNull().default(false),
startedAt: integer("started_at", { mode: "timestamp" }),
completedAt: integer("completed_at", { mode: "timestamp" }),
lastCheckpoint: integer("last_checkpoint", { mode: "timestamp" }),
});
export const organizations = sqliteTable("organizations", {

View File

@@ -52,6 +52,12 @@ export const configSchema = z.object({
lastRun: z.date().optional(),
nextRun: z.date().optional(),
}),
cleanupConfig: z.object({
enabled: z.boolean().default(false),
retentionDays: z.number().min(1).default(7), // in days
lastRun: z.date().optional(),
nextRun: z.date().optional(),
}),
createdAt: z.date().default(() => new Date()),
updatedAt: z.date().default(() => new Date()),
});
@@ -111,6 +117,18 @@ export const mirrorJobSchema = z.object({
status: repoStatusEnum.default("imported"),
message: z.string(),
timestamp: z.date().default(() => new Date()),
// New fields for job resilience
jobType: z.enum(["mirror", "sync", "retry"]).default("mirror"),
batchId: z.string().uuid().optional(), // Group related jobs together
totalItems: z.number().optional(), // Total number of items to process
completedItems: z.number().optional(), // Number of items completed
itemIds: z.array(z.string()).optional(), // IDs of items to process
completedItemIds: z.array(z.string()).optional(), // IDs of completed items
inProgress: z.boolean().default(false), // Whether the job is currently running
startedAt: z.date().optional(), // When the job started
completedAt: z.date().optional(), // When the job completed
lastCheckpoint: z.date().optional(), // Last time progress was saved
});
export type MirrorJob = z.infer<typeof mirrorJobSchema>;

View File

@@ -1,6 +1,6 @@
import { v4 as uuidv4 } from "uuid";
import { db, events } from "./db";
import { eq, and, gt, lt } from "drizzle-orm";
import { eq, and, gt, lt, inArray } from "drizzle-orm";
/**
* Publishes an event to a specific channel for a user
@@ -10,21 +10,58 @@ export async function publishEvent({
userId,
channel,
payload,
deduplicationKey,
}: {
userId: string;
channel: string;
payload: any;
deduplicationKey?: string; // Optional key to prevent duplicate events
}): Promise<string> {
try {
const eventId = uuidv4();
console.log(`Publishing event to channel ${channel} for user ${userId}`);
// Check for duplicate events if deduplication key is provided
if (deduplicationKey) {
const existingEvent = await db
.select()
.from(events)
.where(
and(
eq(events.userId, userId),
eq(events.channel, channel),
eq(events.read, false)
)
)
.limit(10); // Check recent unread events
// Check if any existing event has the same deduplication key in payload
const isDuplicate = existingEvent.some(event => {
try {
const eventPayload = JSON.parse(event.payload as string);
return eventPayload.deduplicationKey === deduplicationKey;
} catch {
return false;
}
});
if (isDuplicate) {
console.log(`Skipping duplicate event with key: ${deduplicationKey}`);
return eventId; // Return a valid ID but don't create the event
}
}
// Add deduplication key to payload if provided
const eventPayload = deduplicationKey
? { ...payload, deduplicationKey }
: payload;
// Insert the event into the SQLite database
await db.insert(events).values({
id: eventId,
userId,
channel,
payload: JSON.stringify(payload),
payload: JSON.stringify(eventPayload),
createdAt: new Date(),
});
@@ -50,36 +87,27 @@ export async function getNewEvents({
lastEventTime?: Date;
}): Promise<any[]> {
try {
console.log(`Getting new events for user ${userId} in channel ${channel}`);
if (lastEventTime) {
console.log(`Looking for events after ${lastEventTime.toISOString()}`);
}
// Build the query
let query = db
.select()
.from(events)
.where(
and(
eq(events.userId, userId),
eq(events.channel, channel),
eq(events.read, false)
)
)
.orderBy(events.createdAt);
// Build the query conditions
const conditions = [
eq(events.userId, userId),
eq(events.channel, channel),
eq(events.read, false)
];
// Add time filter if provided
if (lastEventTime) {
query = query.where(gt(events.createdAt, lastEventTime));
conditions.push(gt(events.createdAt, lastEventTime));
}
// Execute the query
const newEvents = await query;
console.log(`Found ${newEvents.length} new events`);
const newEvents = await db
.select()
.from(events)
.where(and(...conditions))
.orderBy(events.createdAt);
// Mark events as read
if (newEvents.length > 0) {
console.log(`Marking ${newEvents.length} events as read`);
await db
.update(events)
.set({ read: true })
@@ -103,9 +131,78 @@ export async function getNewEvents({
}
}
/**
* Removes duplicate events based on deduplication keys
* This can be called periodically to clean up any duplicates that may have slipped through
*/
export async function removeDuplicateEvents(userId?: string): Promise<{ duplicatesRemoved: number }> {
try {
console.log("Removing duplicate events...");
// Build the base query
const allEvents = userId
? await db.select().from(events).where(eq(events.userId, userId))
: await db.select().from(events);
const duplicateIds: string[] = [];
// Group events by user and channel, then check for duplicates
const eventsByUserChannel = new Map<string, typeof allEvents>();
for (const event of allEvents) {
const key = `${event.userId}-${event.channel}`;
if (!eventsByUserChannel.has(key)) {
eventsByUserChannel.set(key, []);
}
eventsByUserChannel.get(key)!.push(event);
}
// Check each group for duplicates
for (const [, events] of eventsByUserChannel) {
const channelSeenKeys = new Set<string>();
// Sort by creation time (keep the earliest)
events.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
for (const event of events) {
try {
const payload = JSON.parse(event.payload as string);
if (payload.deduplicationKey) {
if (channelSeenKeys.has(payload.deduplicationKey)) {
duplicateIds.push(event.id);
} else {
channelSeenKeys.add(payload.deduplicationKey);
}
}
} catch {
// Skip events with invalid JSON
}
}
}
// Remove duplicates
if (duplicateIds.length > 0) {
console.log(`Removing ${duplicateIds.length} duplicate events`);
// Delete in batches to avoid query size limits
const batchSize = 100;
for (let i = 0; i < duplicateIds.length; i += batchSize) {
const batch = duplicateIds.slice(i, i + batchSize);
await db.delete(events).where(inArray(events.id, batch));
}
}
console.log(`Removed ${duplicateIds.length} duplicate events`);
return { duplicatesRemoved: duplicateIds.length };
} catch (error) {
console.error("Error removing duplicate events:", error);
return { duplicatesRemoved: 0 };
}
}
/**
* Cleans up old events to prevent the database from growing too large
* Should be called periodically (e.g., daily via a cron job)
* This function is used by the cleanup button in the Activity Log page
*
* @param maxAgeInDays Number of days to keep events (default: 7)
* @param cleanupUnreadAfterDays Number of days after which to clean up unread events (default: 2x maxAgeInDays)
@@ -132,7 +229,7 @@ export async function cleanupOldEvents(
)
);
const readEventsDeleted = readResult.changes || 0;
const readEventsDeleted = (readResult as any).changes || 0;
console.log(`Deleted ${readEventsDeleted} read events`);
// Calculate the cutoff date for unread events (default to 2x the retention period)
@@ -150,7 +247,7 @@ export async function cleanupOldEvents(
)
);
const unreadEventsDeleted = unreadResult.changes || 0;
const unreadEventsDeleted = (unreadResult as any).changes || 0;
console.log(`Deleted ${unreadEventsDeleted} unread events`);
return { readEventsDeleted, unreadEventsDeleted };

120
src/lib/gitea.test.ts Normal file
View File

@@ -0,0 +1,120 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
import { Octokit } from "@octokit/rest";
import { repoStatusEnum } from "@/types/Repository";
// Mock the isRepoPresentInGitea function
const mockIsRepoPresentInGitea = mock(() => Promise.resolve(false));
// Mock the database module
mock.module("@/lib/db", () => {
return {
db: {
update: () => ({
set: () => ({
where: () => Promise.resolve()
})
})
},
repositories: {},
organizations: {}
};
});
// Mock the helpers module
mock.module("@/lib/helpers", () => {
return {
createMirrorJob: mock(() => Promise.resolve("job-id"))
};
});
// Mock superagent
mock.module("superagent", () => {
const mockPost = mock(() => ({
set: () => ({
set: () => ({
send: () => Promise.resolve({ body: { id: 123 } })
})
})
}));
const mockGet = mock(() => ({
set: () => Promise.resolve({ body: [] })
}));
return {
post: mockPost,
get: mockGet
};
});
// Mock the gitea module itself
mock.module("./gitea", () => {
return {
isRepoPresentInGitea: mockIsRepoPresentInGitea,
mirrorGithubRepoToGitea: mock(async () => {}),
mirrorGitHubOrgRepoToGiteaOrg: mock(async () => {})
};
});
describe("Gitea Repository Mirroring", () => {
// Mock console.log and console.error to prevent test output noise
let originalConsoleLog: typeof console.log;
let originalConsoleError: typeof console.error;
beforeEach(() => {
originalConsoleLog = console.log;
originalConsoleError = console.error;
console.log = mock(() => {});
console.error = mock(() => {});
});
afterEach(() => {
console.log = originalConsoleLog;
console.error = originalConsoleError;
});
test("mirrorGithubRepoToGitea handles private repositories correctly", async () => {
// Import the mocked function
const { mirrorGithubRepoToGitea } = await import("./gitea");
// Create mock Octokit instance
const octokit = {} as Octokit;
// Create mock repository (private)
const repository = {
id: "repo-id",
name: "test-repo",
fullName: "testuser/test-repo",
url: "https://github.com/testuser/test-repo",
cloneUrl: "https://github.com/testuser/test-repo.git",
owner: "testuser",
isPrivate: true,
status: repoStatusEnum.parse("imported")
};
// Create mock config
const config = {
id: "config-id",
userId: "user-id",
githubConfig: {
token: "github-token",
mirrorIssues: false
},
giteaConfig: {
url: "https://gitea.example.com",
token: "gitea-token",
username: "giteauser"
}
};
// Call the function
await mirrorGithubRepoToGitea({
octokit,
repository: repository as any,
config
});
// Check that the function was called
expect(mirrorGithubRepoToGitea).toHaveBeenCalled();
});
});

View File

@@ -295,15 +295,8 @@ export async function getOrCreateGiteaOrg({
if (orgRes.ok) {
const org = await orgRes.json();
await createMirrorJob({
userId: config.userId,
organizationId: org.id,
organizationName: orgName,
status: "imported",
message: `Organization ${orgName} fetched successfully`,
details: `Organization ${orgName} was fetched from GitHub`,
});
// Note: Organization events are handled by the main mirroring process
// to avoid duplicate events
return org.id;
}
@@ -325,13 +318,8 @@ export async function getOrCreateGiteaOrg({
throw new Error(`Failed to create Gitea org: ${await createRes.text()}`);
}
await createMirrorJob({
userId: config.userId,
organizationName: orgName,
status: "imported",
message: `Organization ${orgName} created successfully`,
details: `Organization ${orgName} was created in Gitea`,
});
// Note: Organization creation events are handled by the main mirroring process
// to avoid duplicate events
const newOrg = await createRes.json();
return newOrg.id;
@@ -417,15 +405,8 @@ export async function mirrorGitHubRepoToGiteaOrg({
})
.where(eq(repositories.id, repository.id!));
// Append log for "mirroring" status
await createMirrorJob({
userId: config.userId,
repositoryId: repository.id,
repositoryName: repository.name,
message: `Started mirroring repository: ${repository.name}`,
details: `Repository ${repository.name} is now in the mirroring state.`,
status: "mirroring",
});
// Note: "mirroring" status events are handled by the concurrency system
// to avoid duplicate events during batch operations
const apiUrl = `${config.giteaConfig.url}/api/v1/repos/migrate`;
@@ -601,11 +582,22 @@ export async function mirrorGitHubOrgToGitea({
.from(repositories)
.where(eq(repositories.organization, organization.name));
for (const repo of orgRepos) {
await mirrorGitHubRepoToGiteaOrg({
octokit,
config,
repository: {
if (orgRepos.length === 0) {
console.log(`No repositories found for organization ${organization.name}`);
return;
}
console.log(`Mirroring ${orgRepos.length} repositories for organization ${organization.name}`);
// Import the processWithRetry function
const { processWithRetry } = await import("@/lib/utils/concurrency");
// Process repositories in parallel with concurrency control
await processWithRetry(
orgRepos,
async (repo) => {
// Prepare repository data
const repoData = {
...repo,
status: repo.status as RepoStatus,
visibility: repo.visibility as RepositoryVisibility,
@@ -614,11 +606,37 @@ export async function mirrorGitHubOrgToGitea({
organization: repo.organization ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
mirroredLocation: repo.mirroredLocation || "",
};
// Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name} in organization ${organization.name}`);
// Mirror the repository
await mirrorGitHubRepoToGiteaOrg({
octokit,
config,
repository: repoData,
giteaOrgId,
orgName: organization.name,
});
return repo;
},
{
concurrencyLimit: 3, // Process 3 repositories at a time
maxRetries: 2,
retryDelay: 2000,
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
if (result) {
console.log(`Mirrored repository "${result.name}" in organization ${organization.name} (${completed}/${total}, ${percentComplete}%)`);
}
},
giteaOrgId,
orgName: organization.name,
});
}
onRetry: (repo, error, attempt) => {
console.log(`Retrying repository ${repo.name} in organization ${organization.name} (attempt ${attempt}): ${error.message}`);
}
}
);
console.log(`Organization ${organization.name} mirrored successfully`);
@@ -837,7 +855,15 @@ export const mirrorGitRepoIssuesToGitea = async ({
(res) => res.data
);
console.log(`Mirroring ${issues.length} issues from ${repository.fullName}`);
// Filter out pull requests
const filteredIssues = issues.filter(issue => !(issue as any).pull_request);
console.log(`Mirroring ${filteredIssues.length} issues from ${repository.fullName}`);
if (filteredIssues.length === 0) {
console.log(`No issues to mirror for ${repository.fullName}`);
return;
}
// Get existing labels from Gitea
const giteaLabelsRes = await superagent
@@ -851,58 +877,60 @@ export const mirrorGitRepoIssuesToGitea = async ({
giteaLabels.map((label: any) => [label.name, label.id])
);
for (const issue of issues) {
if ((issue as any).pull_request) {
continue;
}
// Import the processWithRetry function
const { processWithRetry } = await import("@/lib/utils/concurrency");
const githubLabelNames =
issue.labels
?.map((l) => (typeof l === "string" ? l : l.name))
.filter((l): l is string => !!l) || [];
// Process issues in parallel with concurrency control
await processWithRetry(
filteredIssues,
async (issue) => {
const githubLabelNames =
issue.labels
?.map((l) => (typeof l === "string" ? l : l.name))
.filter((l): l is string => !!l) || [];
const giteaLabelIds: number[] = [];
const giteaLabelIds: number[] = [];
// Resolve or create labels in Gitea
for (const name of githubLabelNames) {
if (labelMap.has(name)) {
giteaLabelIds.push(labelMap.get(name)!);
} else {
try {
const created = await superagent
.post(
`${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/labels`
)
.set("Authorization", `token ${config.giteaConfig.token}`)
.send({ name, color: "#ededed" }); // Default color
// Resolve or create labels in Gitea
for (const name of githubLabelNames) {
if (labelMap.has(name)) {
giteaLabelIds.push(labelMap.get(name)!);
} else {
try {
const created = await superagent
.post(
`${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/labels`
)
.set("Authorization", `token ${config.giteaConfig.token}`)
.send({ name, color: "#ededed" }); // Default color
labelMap.set(name, created.body.id);
giteaLabelIds.push(created.body.id);
} catch (labelErr) {
console.error(
`Failed to create label "${name}" in Gitea: ${labelErr}`
);
labelMap.set(name, created.body.id);
giteaLabelIds.push(created.body.id);
} catch (labelErr) {
console.error(
`Failed to create label "${name}" in Gitea: ${labelErr}`
);
}
}
}
}
const originalAssignees =
issue.assignees && issue.assignees.length > 0
? `\n\nOriginally assigned to: ${issue.assignees
.map((a) => `@${a.login}`)
.join(", ")} on GitHub.`
: "";
const originalAssignees =
issue.assignees && issue.assignees.length > 0
? `\n\nOriginally assigned to: ${issue.assignees
.map((a) => `@${a.login}`)
.join(", ")} on GitHub.`
: "";
const issuePayload: any = {
title: issue.title,
body: `Originally created by @${
issue.user?.login
} on GitHub.${originalAssignees}\n\n${issue.body || ""}`,
closed: issue.state === "closed",
labels: giteaLabelIds,
};
const issuePayload: any = {
title: issue.title,
body: `Originally created by @${
issue.user?.login
} on GitHub.${originalAssignees}\n\n${issue.body || ""}`,
closed: issue.state === "closed",
labels: giteaLabelIds,
};
try {
// Create the issue in Gitea
const createdIssue = await superagent
.post(
`${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/issues`
@@ -922,41 +950,49 @@ export const mirrorGitRepoIssuesToGitea = async ({
(res) => res.data
);
for (const comment of comments) {
try {
await superagent
.post(
`${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/issues/${createdIssue.body.number}/comments`
)
.set("Authorization", `token ${config.giteaConfig.token}`)
.send({
body: `@${comment.user?.login} commented on GitHub:\n\n${comment.body}`,
});
} catch (commentErr) {
console.error(
`Failed to copy comment to Gitea for issue "${issue.title}": ${
commentErr instanceof Error
? commentErr.message
: String(commentErr)
}`
);
}
// Process comments in parallel with concurrency control
if (comments.length > 0) {
await processWithRetry(
comments,
async (comment) => {
await superagent
.post(
`${config.giteaConfig.url}/api/v1/repos/${repoOrigin}/${repository.name}/issues/${createdIssue.body.number}/comments`
)
.set("Authorization", `token ${config.giteaConfig.token}`)
.send({
body: `@${comment.user?.login} commented on GitHub:\n\n${comment.body}`,
});
return comment;
},
{
concurrencyLimit: 5,
maxRetries: 2,
retryDelay: 1000,
onRetry: (comment, error, attempt) => {
console.log(`Retrying comment (attempt ${attempt}): ${error.message}`);
}
}
);
}
} catch (err) {
if (err instanceof Error && (err as any).response) {
console.error(
`Failed to create issue "${issue.title}" in Gitea: ${err.message}`
);
console.error(
`Response body: ${JSON.stringify((err as any).response.body)}`
);
} else {
console.error(
`Failed to create issue "${issue.title}" in Gitea: ${
err instanceof Error ? err.message : String(err)
}`
);
return issue;
},
{
concurrencyLimit: 3, // Process 3 issues at a time
maxRetries: 2,
retryDelay: 2000,
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
if (result) {
console.log(`Mirrored issue "${result.title}" (${completed}/${total}, ${percentComplete}%)`);
}
},
onRetry: (issue, error, attempt) => {
console.log(`Retrying issue "${issue.title}" (attempt ${attempt}): ${error.message}`);
}
}
}
);
console.log(`Completed mirroring ${filteredIssues.length} issues for ${repository.fullName}`);
};

View File

@@ -1,5 +1,6 @@
import type { RepoStatus } from "@/types/Repository";
import { db, mirrorJobs } from "./db";
import { eq, and, or, lt, isNull } from "drizzle-orm";
import { v4 as uuidv4 } from "uuid";
import { publishEvent } from "./events";
@@ -12,6 +13,12 @@ export async function createMirrorJob({
message,
status,
details,
jobType,
batchId,
totalItems,
itemIds,
inProgress,
skipDuplicateEvent,
}: {
userId: string;
organizationId?: string;
@@ -21,6 +28,12 @@ export async function createMirrorJob({
details?: string;
message: string;
status: RepoStatus;
jobType?: "mirror" | "sync" | "retry";
batchId?: string;
totalItems?: number;
itemIds?: string[];
inProgress?: boolean;
skipDuplicateEvent?: boolean; // Option to skip event publishing for internal operations
}) {
const jobId = uuidv4();
const currentTimestamp = new Date();
@@ -32,24 +45,49 @@ export async function createMirrorJob({
repositoryName,
organizationId,
organizationName,
configId: uuidv4(),
details,
message: message,
status: status,
timestamp: currentTimestamp,
// New resilience fields
jobType: jobType || "mirror",
batchId: batchId || undefined,
totalItems: totalItems || undefined,
completedItems: 0,
itemIds: itemIds || undefined,
completedItemIds: [],
inProgress: inProgress !== undefined ? inProgress : false,
startedAt: inProgress ? currentTimestamp : undefined,
completedAt: undefined,
lastCheckpoint: undefined,
};
try {
// Insert the job into the database
await db.insert(mirrorJobs).values(job);
// Publish the event using SQLite instead of Redis
const channel = `mirror-status:${userId}`;
await publishEvent({
userId,
channel,
payload: job
});
// Publish the event using SQLite instead of Redis (unless skipped)
if (!skipDuplicateEvent) {
const channel = `mirror-status:${userId}`;
// Create deduplication key based on the operation
let deduplicationKey: string | undefined;
if (repositoryId && status) {
deduplicationKey = `repo-${repositoryId}-${status}`;
} else if (organizationId && status) {
deduplicationKey = `org-${organizationId}-${status}`;
} else if (batchId) {
deduplicationKey = `batch-${batchId}-${status}`;
}
await publishEvent({
userId,
channel,
payload: job,
deduplicationKey
});
}
return jobId;
} catch (error) {
@@ -57,3 +95,217 @@ export async function createMirrorJob({
throw new Error("Error creating mirror job");
}
}
/**
* Updates the progress of a mirror job
*/
export async function updateMirrorJobProgress({
jobId,
completedItemId,
status,
message,
details,
inProgress,
isCompleted,
}: {
jobId: string;
completedItemId?: string;
status?: RepoStatus;
message?: string;
details?: string;
inProgress?: boolean;
isCompleted?: boolean;
}) {
try {
// Get the current job
const [job] = await db
.select()
.from(mirrorJobs)
.where(eq(mirrorJobs.id, jobId));
if (!job) {
throw new Error(`Mirror job with ID ${jobId} not found`);
}
// Update the job with new progress
const updates: Record<string, any> = {
lastCheckpoint: new Date(),
};
// Add completed item if provided
if (completedItemId) {
const completedItemIds = job.completedItemIds || [];
if (!completedItemIds.includes(completedItemId)) {
updates.completedItemIds = [...completedItemIds, completedItemId];
updates.completedItems = (job.completedItems || 0) + 1;
}
}
// Update status if provided
if (status) {
updates.status = status;
}
// Update message if provided
if (message) {
updates.message = message;
}
// Update details if provided
if (details) {
updates.details = details;
}
// Update in-progress status if provided
if (inProgress !== undefined) {
updates.inProgress = inProgress;
}
// Mark as completed if specified
if (isCompleted) {
updates.inProgress = false;
updates.completedAt = new Date();
}
// Update the job in the database
await db
.update(mirrorJobs)
.set(updates)
.where(eq(mirrorJobs.id, jobId));
// Publish the event with deduplication
const updatedJob = {
...job,
...updates,
};
// Create deduplication key for progress updates
let deduplicationKey: string | undefined;
if (completedItemId) {
deduplicationKey = `progress-${jobId}-${completedItemId}`;
} else if (isCompleted) {
deduplicationKey = `completed-${jobId}`;
} else {
deduplicationKey = `update-${jobId}-${Date.now()}`;
}
await publishEvent({
userId: job.userId,
channel: `mirror-status:${job.userId}`,
payload: updatedJob,
deduplicationKey
});
return updatedJob;
} catch (error) {
console.error("Error updating mirror job progress:", error);
throw new Error("Error updating mirror job progress");
}
}
/**
* Finds interrupted jobs that need to be resumed with enhanced criteria
*/
export async function findInterruptedJobs() {
try {
// Find jobs that are marked as in-progress but haven't been updated recently
const cutoffTime = new Date();
cutoffTime.setMinutes(cutoffTime.getMinutes() - 10); // Consider jobs inactive after 10 minutes without updates
// Also check for jobs that have been running for too long (over 2 hours)
const staleCutoffTime = new Date();
staleCutoffTime.setHours(staleCutoffTime.getHours() - 2);
const interruptedJobs = await db
.select()
.from(mirrorJobs)
.where(
and(
eq(mirrorJobs.inProgress, true),
or(
// Jobs with no recent checkpoint
or(isNull(mirrorJobs.lastCheckpoint), lt(mirrorJobs.lastCheckpoint, cutoffTime)),
// Jobs that started too long ago (likely stale)
lt(mirrorJobs.startedAt, staleCutoffTime)
)
)
);
// Log details about found jobs for debugging
if (interruptedJobs.length > 0) {
console.log(`Found ${interruptedJobs.length} interrupted jobs:`);
interruptedJobs.forEach(job => {
const lastCheckpoint = job.lastCheckpoint ? new Date(job.lastCheckpoint).toISOString() : 'never';
const startedAt = job.startedAt ? new Date(job.startedAt).toISOString() : 'unknown';
console.log(`- Job ${job.id}: ${job.jobType} (started: ${startedAt}, last checkpoint: ${lastCheckpoint})`);
});
}
return interruptedJobs;
} catch (error) {
console.error("Error finding interrupted jobs:", error);
return [];
}
}
/**
* Resumes an interrupted job
*/
export async function resumeInterruptedJob(job: any) {
try {
console.log(`Resuming interrupted job: ${job.id}`);
// Skip if job doesn't have the necessary data to resume
if (!job.itemIds || !job.completedItemIds) {
console.log(`Cannot resume job ${job.id}: missing item data`);
// Mark the job as failed
await updateMirrorJobProgress({
jobId: job.id,
status: "failed",
message: "Job interrupted and could not be resumed",
details: "The job was interrupted and did not have enough information to resume",
inProgress: false,
isCompleted: true,
});
return null;
}
// Calculate remaining items
const remainingItemIds = job.itemIds.filter(
(id: string) => !job.completedItemIds.includes(id)
);
if (remainingItemIds.length === 0) {
console.log(`Job ${job.id} has no remaining items, marking as completed`);
// Mark the job as completed
await updateMirrorJobProgress({
jobId: job.id,
status: "mirrored",
message: "Job completed after resuming",
inProgress: false,
isCompleted: true,
});
return null;
}
// Update the job to show it's being resumed
await updateMirrorJobProgress({
jobId: job.id,
message: `Resuming job with ${remainingItemIds.length} remaining items`,
details: `Job was interrupted and is being resumed. ${job.completedItemIds.length} of ${job.itemIds.length} items were already processed.`,
inProgress: true,
});
return {
job,
remainingItemIds,
};
} catch (error) {
console.error(`Error resuming job ${job.id}:`, error);
return null;
}
}

504
src/lib/recovery.ts Normal file
View File

@@ -0,0 +1,504 @@
/**
* Recovery mechanism for interrupted jobs
* This module handles detecting and resuming jobs that were interrupted by container restarts
*/
import { findInterruptedJobs, resumeInterruptedJob } from './helpers';
import { db, repositories, organizations, mirrorJobs } from './db';
import { eq, and, lt } from 'drizzle-orm';
import { mirrorGithubRepoToGitea, mirrorGitHubOrgRepoToGiteaOrg, syncGiteaRepo } from './gitea';
import { createGitHubClient } from './github';
import { processWithResilience } from './utils/concurrency';
import { repositoryVisibilityEnum, repoStatusEnum } from '@/types/Repository';
import type { Repository } from './db/schema';
// Recovery state tracking
let recoveryInProgress = false;
let lastRecoveryAttempt: Date | null = null;
/**
* Validates database connection before attempting recovery
*/
async function validateDatabaseConnection(): Promise<boolean> {
try {
// Simple query to test database connectivity
await db.select().from(mirrorJobs).limit(1);
return true;
} catch (error) {
console.error('Database connection validation failed:', error);
return false;
}
}
/**
* Cleans up stale jobs that are too old to recover
*/
async function cleanupStaleJobs(): Promise<void> {
try {
const staleThreshold = new Date();
staleThreshold.setHours(staleThreshold.getHours() - 24); // Jobs older than 24 hours
const staleJobs = await db
.select()
.from(mirrorJobs)
.where(
and(
eq(mirrorJobs.inProgress, true),
lt(mirrorJobs.startedAt, staleThreshold)
)
);
if (staleJobs.length > 0) {
console.log(`Found ${staleJobs.length} stale jobs to clean up`);
// Mark stale jobs as failed
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: "Job marked as failed due to being stale (older than 24 hours)"
})
.where(
and(
eq(mirrorJobs.inProgress, true),
lt(mirrorJobs.startedAt, staleThreshold)
)
);
console.log(`Cleaned up ${staleJobs.length} stale jobs`);
}
} catch (error) {
console.error('Error cleaning up stale jobs:', error);
}
}
/**
* Initialize the recovery system with enhanced error handling and resilience
* This should be called when the application starts
*/
export async function initializeRecovery(options: {
maxRetries?: number;
retryDelay?: number;
skipIfRecentAttempt?: boolean;
} = {}): Promise<boolean> {
const { maxRetries = 3, retryDelay = 5000, skipIfRecentAttempt = true } = options;
// Prevent concurrent recovery attempts
if (recoveryInProgress) {
console.log('Recovery already in progress, skipping...');
return false;
}
// Skip if recent attempt (within last 5 minutes) unless forced
if (skipIfRecentAttempt && lastRecoveryAttempt) {
const timeSinceLastAttempt = Date.now() - lastRecoveryAttempt.getTime();
if (timeSinceLastAttempt < 5 * 60 * 1000) {
console.log('Recent recovery attempt detected, skipping...');
return false;
}
}
recoveryInProgress = true;
lastRecoveryAttempt = new Date();
console.log('Initializing recovery system...');
let attempt = 0;
while (attempt < maxRetries) {
try {
attempt++;
console.log(`Recovery attempt ${attempt}/${maxRetries}`);
// Validate database connection first
const dbConnected = await validateDatabaseConnection();
if (!dbConnected) {
throw new Error('Database connection validation failed');
}
// Clean up stale jobs first
await cleanupStaleJobs();
// Find interrupted jobs
const interruptedJobs = await findInterruptedJobs();
if (interruptedJobs.length === 0) {
console.log('No interrupted jobs found.');
recoveryInProgress = false;
return true;
}
console.log(`Found ${interruptedJobs.length} interrupted jobs. Starting recovery...`);
// Process each interrupted job with individual error handling
let successCount = 0;
let failureCount = 0;
for (const job of interruptedJobs) {
try {
const resumeData = await resumeInterruptedJob(job);
if (!resumeData) {
console.log(`Job ${job.id} could not be resumed.`);
failureCount++;
continue;
}
const { job: updatedJob, remainingItemIds } = resumeData;
// Handle different job types
switch (updatedJob.jobType) {
case 'mirror':
await recoverMirrorJob(updatedJob, remainingItemIds);
break;
case 'sync':
await recoverSyncJob(updatedJob, remainingItemIds);
break;
case 'retry':
await recoverRetryJob(updatedJob, remainingItemIds);
break;
default:
console.log(`Unknown job type: ${updatedJob.jobType}`);
failureCount++;
continue;
}
successCount++;
} catch (jobError) {
console.error(`Error recovering individual job ${job.id}:`, jobError);
failureCount++;
// Mark the job as failed if recovery fails
try {
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: `Job recovery failed: ${jobError instanceof Error ? jobError.message : String(jobError)}`
})
.where(eq(mirrorJobs.id, job.id));
} catch (updateError) {
console.error(`Failed to mark job ${job.id} as failed:`, updateError);
}
}
}
console.log(`Recovery process completed. Success: ${successCount}, Failures: ${failureCount}`);
recoveryInProgress = false;
return true;
} catch (error) {
console.error(`Recovery attempt ${attempt} failed:`, error);
if (attempt < maxRetries) {
console.log(`Retrying in ${retryDelay}ms...`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
} else {
console.error('All recovery attempts failed');
recoveryInProgress = false;
return false;
}
}
}
recoveryInProgress = false;
return false;
}
/**
* Recover a mirror job with enhanced error handling
*/
async function recoverMirrorJob(job: any, remainingItemIds: string[]) {
console.log(`Recovering mirror job ${job.id} with ${remainingItemIds.length} remaining items`);
try {
// Get the config for this user with better error handling
const configs = await db
.select()
.from(repositories)
.where(eq(repositories.userId, job.userId))
.limit(1);
if (configs.length === 0) {
throw new Error(`No configuration found for user ${job.userId}`);
}
const config = configs[0];
if (!config.configId) {
throw new Error(`Configuration missing configId for user ${job.userId}`);
}
// Get repositories to process with validation
const repos = await db
.select()
.from(repositories)
.where(eq(repositories.id, remainingItemIds));
if (repos.length === 0) {
console.warn(`No repositories found for remaining item IDs: ${remainingItemIds.join(', ')}`);
// Mark job as completed since there's nothing to process
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "mirrored",
message: "Job completed - no repositories found to process"
})
.where(eq(mirrorJobs.id, job.id));
return;
}
console.log(`Found ${repos.length} repositories to process for recovery`);
// Validate GitHub configuration before creating client
if (!config.githubConfig?.token) {
throw new Error('GitHub token not found in configuration');
}
// Create GitHub client with error handling
let octokit;
try {
octokit = createGitHubClient(config.githubConfig.token);
} catch (error) {
throw new Error(`Failed to create GitHub client: ${error instanceof Error ? error.message : String(error)}`);
}
// Process repositories with resilience and reduced concurrency for recovery
await processWithResilience(
repos,
async (repo) => {
// Prepare repository data with validation
const repoData = {
...repo,
status: repoStatusEnum.parse("imported"),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility || "public"),
mirroredLocation: repo.mirroredLocation || "",
};
// Mirror the repository based on whether it's in an organization
if (repo.organization && config.githubConfig.preserveOrgStructure) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
octokit,
orgName: repo.organization,
repository: repoData,
});
} else {
await mirrorGithubRepoToGitea({
octokit,
repository: repoData,
config,
});
}
return repo;
},
{
userId: job.userId,
jobType: 'mirror',
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
resumeFromJobId: job.id,
concurrencyLimit: 2, // Reduced concurrency for recovery to be more stable
maxRetries: 3, // Increased retries for recovery
retryDelay: 3000, // Longer delay for recovery
}
);
console.log(`Successfully recovered mirror job ${job.id}`);
} catch (error) {
console.error(`Error recovering mirror job ${job.id}:`, error);
// Mark the job as failed
try {
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: `Mirror job recovery failed: ${error instanceof Error ? error.message : String(error)}`
})
.where(eq(mirrorJobs.id, job.id));
} catch (updateError) {
console.error(`Failed to mark mirror job ${job.id} as failed:`, updateError);
}
throw error; // Re-throw to be handled by the caller
}
}
/**
* Recover a sync job with enhanced error handling
*/
async function recoverSyncJob(job: any, remainingItemIds: string[]) {
console.log(`Recovering sync job ${job.id} with ${remainingItemIds.length} remaining items`);
try {
// Get the config for this user with better error handling
const configs = await db
.select()
.from(repositories)
.where(eq(repositories.userId, job.userId))
.limit(1);
if (configs.length === 0) {
throw new Error(`No configuration found for user ${job.userId}`);
}
const config = configs[0];
if (!config.configId) {
throw new Error(`Configuration missing configId for user ${job.userId}`);
}
// Get repositories to process with validation
const repos = await db
.select()
.from(repositories)
.where(eq(repositories.id, remainingItemIds));
if (repos.length === 0) {
console.warn(`No repositories found for remaining item IDs: ${remainingItemIds.join(', ')}`);
// Mark job as completed since there's nothing to process
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "mirrored",
message: "Job completed - no repositories found to process"
})
.where(eq(mirrorJobs.id, job.id));
return;
}
console.log(`Found ${repos.length} repositories to process for sync recovery`);
// Process repositories with resilience and reduced concurrency for recovery
await processWithResilience(
repos,
async (repo) => {
// Prepare repository data with validation
const repoData = {
...repo,
status: repoStatusEnum.parse(repo.status || "imported"),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility || "public"),
};
// Sync the repository
await syncGiteaRepo({
config,
repository: repoData,
});
return repo;
},
{
userId: job.userId,
jobType: 'sync',
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
resumeFromJobId: job.id,
concurrencyLimit: 3, // Reduced concurrency for recovery
maxRetries: 3, // Increased retries for recovery
retryDelay: 3000, // Longer delay for recovery
}
);
console.log(`Successfully recovered sync job ${job.id}`);
} catch (error) {
console.error(`Error recovering sync job ${job.id}:`, error);
// Mark the job as failed
try {
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: `Sync job recovery failed: ${error instanceof Error ? error.message : String(error)}`
})
.where(eq(mirrorJobs.id, job.id));
} catch (updateError) {
console.error(`Failed to mark sync job ${job.id} as failed:`, updateError);
}
throw error; // Re-throw to be handled by the caller
}
}
/**
* Recover a retry job with enhanced error handling
*/
async function recoverRetryJob(job: any, remainingItemIds: string[]) {
console.log(`Recovering retry job ${job.id} with ${remainingItemIds.length} remaining items`);
try {
// For now, retry jobs are treated similarly to mirror jobs
// In the future, this could have specific retry logic
await recoverMirrorJob(job, remainingItemIds);
console.log(`Successfully recovered retry job ${job.id}`);
} catch (error) {
console.error(`Error recovering retry job ${job.id}:`, error);
// Mark the job as failed
try {
await db
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: `Retry job recovery failed: ${error instanceof Error ? error.message : String(error)}`
})
.where(eq(mirrorJobs.id, job.id));
} catch (updateError) {
console.error(`Failed to mark retry job ${job.id} as failed:`, updateError);
}
throw error; // Re-throw to be handled by the caller
}
}
/**
* Get recovery system status
*/
export function getRecoveryStatus() {
return {
inProgress: recoveryInProgress,
lastAttempt: lastRecoveryAttempt,
};
}
/**
* Force recovery to run (bypassing recent attempt check)
*/
export async function forceRecovery(): Promise<boolean> {
return initializeRecovery({ skipIfRecentAttempt: false });
}
/**
* Check if there are any jobs that need recovery
*/
export async function hasJobsNeedingRecovery(): Promise<boolean> {
try {
const interruptedJobs = await findInterruptedJobs();
return interruptedJobs.length > 0;
} catch (error) {
console.error('Error checking for jobs needing recovery:', error);
return false;
}
}

110
src/lib/utils.test.ts Normal file
View File

@@ -0,0 +1,110 @@
import { describe, test, expect } from "bun:test";
import { jsonResponse, formatDate, truncate, safeParse } from "./utils";
describe("jsonResponse", () => {
test("creates a Response with JSON content", () => {
const data = { message: "Hello, world!" };
const response = jsonResponse({ data });
expect(response).toBeInstanceOf(Response);
expect(response.status).toBe(200);
expect(response.headers.get("Content-Type")).toBe("application/json");
});
test("uses the provided status code", () => {
const data = { error: "Not found" };
const response = jsonResponse({ data, status: 404 });
expect(response.status).toBe(404);
});
test("correctly serializes complex objects", async () => {
const now = new Date();
const data = {
message: "Complex object",
date: now,
nested: { foo: "bar" },
array: [1, 2, 3]
};
const response = jsonResponse({ data });
const responseBody = await response.json();
expect(responseBody).toEqual({
message: "Complex object",
date: now.toISOString(),
nested: { foo: "bar" },
array: [1, 2, 3]
});
});
});
describe("formatDate", () => {
test("formats a date object", () => {
const date = new Date("2023-01-15T12:30:45Z");
const formatted = formatDate(date);
// The exact format might depend on the locale, so we'll check for parts
expect(formatted).toContain("2023");
expect(formatted).toContain("January");
expect(formatted).toContain("15");
});
test("formats a date string", () => {
const dateStr = "2023-01-15T12:30:45Z";
const formatted = formatDate(dateStr);
expect(formatted).toContain("2023");
expect(formatted).toContain("January");
expect(formatted).toContain("15");
});
test("returns 'Never' for null or undefined", () => {
expect(formatDate(null)).toBe("Never");
expect(formatDate(undefined)).toBe("Never");
});
});
describe("truncate", () => {
test("truncates a string that exceeds the length", () => {
const str = "This is a long string that needs truncation";
const truncated = truncate(str, 10);
expect(truncated).toBe("This is a ...");
expect(truncated.length).toBe(13); // 10 chars + "..."
});
test("does not truncate a string that is shorter than the length", () => {
const str = "Short";
const truncated = truncate(str, 10);
expect(truncated).toBe("Short");
});
test("handles empty strings", () => {
expect(truncate("", 10)).toBe("");
});
});
describe("safeParse", () => {
test("parses valid JSON strings", () => {
const jsonStr = '{"name":"John","age":30}';
const parsed = safeParse(jsonStr);
expect(parsed).toEqual({ name: "John", age: 30 });
});
test("returns undefined for invalid JSON strings", () => {
const invalidJson = '{"name":"John",age:30}'; // Missing quotes around age
const parsed = safeParse(invalidJson);
expect(parsed).toBeUndefined();
});
test("returns the original value for non-string inputs", () => {
const obj = { name: "John", age: 30 };
const parsed = safeParse(obj);
expect(parsed).toBe(obj);
});
});

View File

@@ -0,0 +1,167 @@
import { describe, test, expect, mock } from "bun:test";
import { processInParallel, processWithRetry } from "./concurrency";
describe("processInParallel", () => {
test("processes items in parallel with concurrency control", async () => {
// Create an array of numbers to process
const items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
// Create a mock function to track execution
const processItem = mock(async (item: number) => {
// Simulate async work
await new Promise(resolve => setTimeout(resolve, 10));
return item * 2;
});
// Create a mock progress callback
const onProgress = mock((completed: number, total: number, result?: number) => {
// Progress tracking
});
// Process the items with a concurrency limit of 3
const results = await processInParallel(
items,
processItem,
3,
onProgress
);
// Verify results
expect(results).toEqual([2, 4, 6, 8, 10, 12, 14, 16, 18, 20]);
// Verify that processItem was called for each item
expect(processItem).toHaveBeenCalledTimes(10);
// Verify that onProgress was called for each item
expect(onProgress).toHaveBeenCalledTimes(10);
// Verify the last call to onProgress had the correct completed/total values
expect(onProgress.mock.calls[9][0]).toBe(10); // completed
expect(onProgress.mock.calls[9][1]).toBe(10); // total
});
test("handles errors in processing", async () => {
// Create an array of numbers to process
const items = [1, 2, 3, 4, 5];
// Create a mock function that throws an error for item 3
const processItem = mock(async (item: number) => {
if (item === 3) {
throw new Error("Test error");
}
return item * 2;
});
// Create a spy for console.error
const originalConsoleError = console.error;
const consoleErrorMock = mock(() => {});
console.error = consoleErrorMock;
try {
// Process the items
const results = await processInParallel(items, processItem);
// Verify results (should have 4 items, missing the one that errored)
expect(results).toEqual([2, 4, 8, 10]);
// Verify that processItem was called for each item
expect(processItem).toHaveBeenCalledTimes(5);
// Verify that console.error was called once
expect(consoleErrorMock).toHaveBeenCalledTimes(1);
} finally {
// Restore console.error
console.error = originalConsoleError;
}
});
});
describe("processWithRetry", () => {
test("retries failed operations", async () => {
// Create an array of numbers to process
const items = [1, 2, 3];
// Create a counter to track retry attempts
const attemptCounts: Record<number, number> = { 1: 0, 2: 0, 3: 0 };
// Create a mock function that fails on first attempt for item 2
const processItem = mock(async (item: number) => {
attemptCounts[item]++;
if (item === 2 && attemptCounts[item] === 1) {
throw new Error("Temporary error");
}
return item * 2;
});
// Create a mock for the onRetry callback
const onRetry = mock((item: number, error: Error, attempt: number) => {
// Retry tracking
});
// Process the items with retry
const results = await processWithRetry(items, processItem, {
maxRetries: 2,
retryDelay: 10,
onRetry,
});
// Verify results
expect(results).toEqual([2, 4, 6]);
// Verify that item 2 was retried once
expect(attemptCounts[1]).toBe(1); // No retries
expect(attemptCounts[2]).toBe(2); // One retry
expect(attemptCounts[3]).toBe(1); // No retries
// Verify that onRetry was called once
expect(onRetry).toHaveBeenCalledTimes(1);
expect(onRetry.mock.calls[0][0]).toBe(2); // item
expect(onRetry.mock.calls[0][2]).toBe(1); // attempt
});
test("gives up after max retries", async () => {
// Create an array of numbers to process
const items = [1, 2];
// Create a mock function that always fails for item 2
const processItem = mock(async (item: number) => {
if (item === 2) {
throw new Error("Persistent error");
}
return item * 2;
});
// Create a mock for the onRetry callback
const onRetry = mock((item: number, error: Error, attempt: number) => {
// Retry tracking
});
// Create a spy for console.error
const originalConsoleError = console.error;
const consoleErrorMock = mock(() => {});
console.error = consoleErrorMock;
try {
// Process the items with retry
const results = await processWithRetry(items, processItem, {
maxRetries: 2,
retryDelay: 10,
onRetry,
});
// Verify results (should have 1 item, missing the one that errored)
expect(results).toEqual([2]);
// Verify that onRetry was called twice (for 2 retry attempts)
expect(onRetry).toHaveBeenCalledTimes(2);
// Verify that console.error was called once
expect(consoleErrorMock).toHaveBeenCalledTimes(1);
} finally {
// Restore console.error
console.error = originalConsoleError;
}
});
});

View File

@@ -0,0 +1,292 @@
/**
* Utility for processing items in parallel with concurrency control
*
* @param items Array of items to process
* @param processItem Function to process each item
* @param concurrencyLimit Maximum number of concurrent operations
* @param onProgress Optional callback for progress updates
* @returns Promise that resolves when all items are processed
*/
export async function processInParallel<T, R>(
items: T[],
processItem: (item: T) => Promise<R>,
concurrencyLimit: number = 5,
onProgress?: (completed: number, total: number, result?: R) => void
): Promise<R[]> {
const results: R[] = [];
let completed = 0;
const total = items.length;
// Process items in batches to control concurrency
for (let i = 0; i < total; i += concurrencyLimit) {
const batch = items.slice(i, i + concurrencyLimit);
const batchPromises = batch.map(async (item) => {
try {
const result = await processItem(item);
completed++;
if (onProgress) {
onProgress(completed, total, result);
}
return result;
} catch (error) {
completed++;
if (onProgress) {
onProgress(completed, total);
}
throw error;
}
});
// Wait for the current batch to complete before starting the next batch
const batchResults = await Promise.allSettled(batchPromises);
// Process results and handle errors
for (const result of batchResults) {
if (result.status === 'fulfilled') {
results.push(result.value);
} else {
console.error('Error processing item:', result.reason);
}
}
}
return results;
}
/**
* Utility for processing items in parallel with automatic retry for failed operations
*
* @param items Array of items to process
* @param processItem Function to process each item
* @param options Configuration options
* @returns Promise that resolves when all items are processed
*/
export async function processWithRetry<T, R>(
items: T[],
processItem: (item: T) => Promise<R>,
options: {
concurrencyLimit?: number;
maxRetries?: number;
retryDelay?: number;
onProgress?: (completed: number, total: number, result?: R) => void;
onRetry?: (item: T, error: Error, attempt: number) => void;
jobId?: string; // Optional job ID for checkpointing
getItemId?: (item: T) => string; // Function to get a unique ID for each item
onCheckpoint?: (jobId: string, completedItemId: string) => Promise<void>; // Callback for checkpointing
checkpointInterval?: number; // How many items to process before checkpointing
} = {}
): Promise<R[]> {
const {
concurrencyLimit = 5,
maxRetries = 3,
retryDelay = 1000,
onProgress,
onRetry,
jobId,
getItemId,
onCheckpoint,
checkpointInterval = 1 // Default to checkpointing after each item
} = options;
// Track checkpoint counter
let itemsProcessedSinceLastCheckpoint = 0;
// Wrap the process function with retry logic
const processWithRetryLogic = async (item: T): Promise<R> => {
let lastError: Error | null = null;
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
try {
const result = await processItem(item);
// Handle checkpointing if enabled
if (jobId && getItemId && onCheckpoint) {
const itemId = getItemId(item);
itemsProcessedSinceLastCheckpoint++;
// Checkpoint based on the interval
if (itemsProcessedSinceLastCheckpoint >= checkpointInterval) {
await onCheckpoint(jobId, itemId);
itemsProcessedSinceLastCheckpoint = 0;
}
}
return result;
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
if (attempt <= maxRetries) {
if (onRetry) {
onRetry(item, lastError, attempt);
}
// Exponential backoff
const delay = retryDelay * Math.pow(2, attempt - 1);
await new Promise(resolve => setTimeout(resolve, delay));
} else {
throw lastError;
}
}
}
// This should never be reached due to the throw in the catch block
throw lastError || new Error('Unknown error occurred');
};
const results = await processInParallel(
items,
processWithRetryLogic,
concurrencyLimit,
onProgress
);
// Final checkpoint if there are remaining items since the last checkpoint
if (jobId && getItemId && onCheckpoint && itemsProcessedSinceLastCheckpoint > 0) {
// We don't have a specific item ID for the final checkpoint, so we'll use a placeholder
await onCheckpoint(jobId, 'final');
}
return results;
}
/**
* Process items in parallel with resilience to container restarts
* This version supports resuming from a previous checkpoint
*/
export async function processWithResilience<T, R>(
items: T[],
processItem: (item: T) => Promise<R>,
options: {
concurrencyLimit?: number;
maxRetries?: number;
retryDelay?: number;
onProgress?: (completed: number, total: number, result?: R) => void;
onRetry?: (item: T, error: Error, attempt: number) => void;
userId: string; // Required for creating mirror jobs
jobType: "mirror" | "sync" | "retry";
getItemId: (item: T) => string; // Required function to get a unique ID for each item
getItemName: (item: T) => string; // Required function to get a display name for each item
checkpointInterval?: number;
resumeFromJobId?: string; // Optional job ID to resume from
}
): Promise<R[]> {
const {
userId,
jobType,
getItemId,
getItemName,
resumeFromJobId,
checkpointInterval = 10, // Increased from 5 to reduce event frequency
...otherOptions
} = options;
// Import helpers for job management
const { createMirrorJob, updateMirrorJobProgress } = await import('@/lib/helpers');
// Get item IDs for all items
const allItemIds = items.map(getItemId);
// Create or resume a job
let jobId: string;
let completedItemIds: string[] = [];
let itemsToProcess = [...items];
if (resumeFromJobId) {
// We're resuming an existing job
jobId = resumeFromJobId;
// Get the job from the database to find completed items
const { db, mirrorJobs } = await import('@/lib/db');
const { eq } = await import('drizzle-orm');
const [job] = await db
.select()
.from(mirrorJobs)
.where(eq(mirrorJobs.id, resumeFromJobId));
if (job && job.completedItemIds) {
completedItemIds = job.completedItemIds;
// Filter out already completed items
itemsToProcess = items.filter(item => !completedItemIds.includes(getItemId(item)));
console.log(`Resuming job ${jobId} with ${itemsToProcess.length} remaining items`);
// Update the job to show it's being resumed
await updateMirrorJobProgress({
jobId,
message: `Resuming job with ${itemsToProcess.length} remaining items`,
details: `Job is being resumed. ${completedItemIds.length} of ${items.length} items were already processed.`,
inProgress: true,
});
}
} else {
// Create a new job
jobId = await createMirrorJob({
userId,
message: `Started ${jobType} job with ${items.length} items`,
details: `Processing ${items.length} items in parallel with checkpointing`,
status: "mirroring",
jobType,
totalItems: items.length,
itemIds: allItemIds,
inProgress: true,
});
console.log(`Created new job ${jobId} with ${items.length} items`);
}
// Define the checkpoint function
const onCheckpoint = async (jobId: string, completedItemId: string) => {
const itemName = items.find(item => getItemId(item) === completedItemId)
? getItemName(items.find(item => getItemId(item) === completedItemId)!)
: 'unknown';
await updateMirrorJobProgress({
jobId,
completedItemId,
message: `Processed item: ${itemName}`,
});
};
try {
// Process the items with checkpointing
const results = await processWithRetry(
itemsToProcess,
processItem,
{
...otherOptions,
jobId,
getItemId,
onCheckpoint,
checkpointInterval,
}
);
// Mark the job as completed
await updateMirrorJobProgress({
jobId,
status: "mirrored",
message: `Completed ${jobType} job with ${items.length} items`,
inProgress: false,
isCompleted: true,
});
return results;
} catch (error) {
// Mark the job as failed
await updateMirrorJobProgress({
jobId,
status: "failed",
message: `Failed ${jobType} job: ${error instanceof Error ? error.message : String(error)}`,
inProgress: false,
isCompleted: true,
});
throw error;
}
}

72
src/middleware.ts Normal file
View File

@@ -0,0 +1,72 @@
import { defineMiddleware } from 'astro:middleware';
import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from './lib/recovery';
import { startCleanupService } from './lib/cleanup-service';
// Flag to track if recovery has been initialized
let recoveryInitialized = false;
let recoveryAttempted = false;
let cleanupServiceStarted = false;
export const onRequest = defineMiddleware(async (context, next) => {
// Initialize recovery system only once when the server starts
// This is a fallback in case the startup script didn't run
if (!recoveryInitialized && !recoveryAttempted) {
recoveryAttempted = true;
try {
// Check if recovery is actually needed before attempting
const needsRecovery = await hasJobsNeedingRecovery();
if (needsRecovery) {
console.log('⚠️ Middleware detected jobs needing recovery (startup script may not have run)');
console.log('Attempting recovery from middleware...');
// Run recovery with a shorter timeout since this is during request handling
const recoveryResult = await Promise.race([
initializeRecovery({
skipIfRecentAttempt: true,
maxRetries: 2,
retryDelay: 3000,
}),
new Promise<boolean>((_, reject) => {
setTimeout(() => reject(new Error('Middleware recovery timeout')), 15000);
})
]);
if (recoveryResult) {
console.log('✅ Middleware recovery completed successfully');
} else {
console.log('⚠️ Middleware recovery completed with some issues');
}
} else {
console.log('✅ No recovery needed (startup script likely handled it)');
}
recoveryInitialized = true;
} catch (error) {
console.error('⚠️ Middleware recovery failed or timed out:', error);
console.log('Application will continue, but some jobs may remain interrupted');
// Log recovery status for debugging
const status = getRecoveryStatus();
console.log('Recovery status:', status);
recoveryInitialized = true; // Mark as attempted to avoid retries
}
}
// Start cleanup service only once after recovery is complete
if (recoveryInitialized && !cleanupServiceStarted) {
try {
console.log('Starting automatic database cleanup service...');
startCleanupService();
cleanupServiceStarted = true;
} catch (error) {
console.error('Failed to start cleanup service:', error);
// Don't fail the request if cleanup service fails to start
}
}
// Continue with the request
return next();
});

View File

@@ -0,0 +1,115 @@
import type { APIRoute } from "astro";
import { db, mirrorJobs, events } from "@/lib/db";
import { eq, count } from "drizzle-orm";
export const POST: APIRoute = async ({ request }) => {
try {
let body;
try {
body = await request.json();
} catch (jsonError) {
console.error("Invalid JSON in request body:", jsonError);
return new Response(
JSON.stringify({ error: "Invalid JSON in request body." }),
{ status: 400, headers: { "Content-Type": "application/json" } }
);
}
const { userId } = body || {};
if (!userId) {
return new Response(
JSON.stringify({ error: "Missing 'userId' in request body." }),
{ status: 400, headers: { "Content-Type": "application/json" } }
);
}
// Start a transaction to ensure all operations succeed or fail together
const result = await db.transaction(async (tx) => {
// Count activities before deletion
const mirrorJobsCountResult = await tx
.select({ count: count() })
.from(mirrorJobs)
.where(eq(mirrorJobs.userId, userId));
const eventsCountResult = await tx
.select({ count: count() })
.from(events)
.where(eq(events.userId, userId));
const totalMirrorJobs = mirrorJobsCountResult[0]?.count || 0;
const totalEvents = eventsCountResult[0]?.count || 0;
console.log(`Found ${totalMirrorJobs} mirror jobs and ${totalEvents} events to delete for user ${userId}`);
// First, mark all in-progress jobs as completed/failed to allow deletion
await tx
.update(mirrorJobs)
.set({
inProgress: false,
completedAt: new Date(),
status: "failed",
message: "Job interrupted and cleaned up by user"
})
.where(eq(mirrorJobs.userId, userId));
console.log(`Updated in-progress jobs to allow deletion`);
// Delete all mirror jobs for the user (now that none are in progress)
await tx
.delete(mirrorJobs)
.where(eq(mirrorJobs.userId, userId));
// Delete all events for the user
await tx
.delete(events)
.where(eq(events.userId, userId));
return {
mirrorJobsDeleted: totalMirrorJobs,
eventsDeleted: totalEvents,
totalMirrorJobs,
totalEvents,
};
});
console.log(`Cleaned up activities for user ${userId}:`, result);
return new Response(
JSON.stringify({
success: true,
message: "All activities cleaned up successfully.",
result: {
mirrorJobsDeleted: result.mirrorJobsDeleted,
eventsDeleted: result.eventsDeleted,
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
} catch (error) {
console.error("Error cleaning up activities:", error);
// Provide more specific error messages
let errorMessage = "An unknown error occurred.";
if (error instanceof Error) {
errorMessage = error.message;
// Check for common database errors
if (error.message.includes("FOREIGN KEY constraint failed")) {
errorMessage = "Cannot delete activities due to database constraints. Some jobs may still be referenced by other records.";
} else if (error.message.includes("database is locked")) {
errorMessage = "Database is currently locked. Please try again in a moment.";
} else if (error.message.includes("no such table")) {
errorMessage = "Database tables are missing. Please check your database setup.";
}
}
return new Response(
JSON.stringify({
success: false,
error: errorMessage,
}),
{ status: 500, headers: { "Content-Type": "application/json" } }
);
}
};

View File

@@ -0,0 +1,72 @@
/**
* API endpoint to manually trigger automatic cleanup
* This is useful for testing and debugging the cleanup service
*/
import type { APIRoute } from 'astro';
import { runAutomaticCleanup } from '@/lib/cleanup-service';
export const POST: APIRoute = async ({ request }) => {
try {
console.log('Manual cleanup trigger requested');
// Run the automatic cleanup
const results = await runAutomaticCleanup();
// Calculate totals
const totalEventsDeleted = results.reduce((sum, result) => sum + result.eventsDeleted, 0);
const totalJobsDeleted = results.reduce((sum, result) => sum + result.mirrorJobsDeleted, 0);
const errors = results.filter(result => result.error);
return new Response(
JSON.stringify({
success: true,
message: 'Automatic cleanup completed',
results: {
usersProcessed: results.length,
totalEventsDeleted,
totalJobsDeleted,
errors: errors.length,
details: results,
},
}),
{
status: 200,
headers: {
'Content-Type': 'application/json',
},
}
);
} catch (error) {
console.error('Error in manual cleanup trigger:', error);
return new Response(
JSON.stringify({
success: false,
message: 'Failed to run automatic cleanup',
error: error instanceof Error ? error.message : 'Unknown error',
}),
{
status: 500,
headers: {
'Content-Type': 'application/json',
},
}
);
}
};
export const GET: APIRoute = async () => {
return new Response(
JSON.stringify({
success: false,
message: 'Use POST method to trigger cleanup',
}),
{
status: 405,
headers: {
'Content-Type': 'application/json',
},
}
);
};

View File

@@ -6,14 +6,14 @@ import { eq } from "drizzle-orm";
export const POST: APIRoute = async ({ request }) => {
try {
const body = await request.json();
const { userId, githubConfig, giteaConfig, scheduleConfig } = body;
const { userId, githubConfig, giteaConfig, scheduleConfig, cleanupConfig } = body;
if (!userId || !githubConfig || !giteaConfig || !scheduleConfig) {
if (!userId || !githubConfig || !giteaConfig || !scheduleConfig || !cleanupConfig) {
return new Response(
JSON.stringify({
success: false,
message:
"userId, githubConfig, giteaConfig, and scheduleConfig are required.",
"userId, githubConfig, giteaConfig, scheduleConfig, and cleanupConfig are required.",
}),
{
status: 400,
@@ -64,6 +64,7 @@ export const POST: APIRoute = async ({ request }) => {
githubConfig,
giteaConfig,
scheduleConfig,
cleanupConfig,
updatedAt: new Date(),
})
.where(eq(configs.id, existingConfig.id));
@@ -113,6 +114,7 @@ export const POST: APIRoute = async ({ request }) => {
include: [],
exclude: [],
scheduleConfig,
cleanupConfig,
createdAt: new Date(),
updatedAt: new Date(),
});
@@ -197,6 +199,12 @@ export const GET: APIRoute = async ({ request }) => {
lastRun: null,
nextRun: null,
},
cleanupConfig: {
enabled: false,
retentionDays: 7,
lastRun: null,
nextRun: null,
},
}),
{
status: 200,

View File

@@ -0,0 +1,187 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
import axios from "axios";
// Mock the POST function
const mockPOST = mock(async ({ request }) => {
const body = await request.json();
// Check for missing URL or token
if (!body.url || !body.token) {
return new Response(
JSON.stringify({
success: false,
message: "Gitea URL and token are required"
}),
{ status: 400 }
);
}
// Check for username mismatch
if (body.username && body.username !== "giteauser") {
return new Response(
JSON.stringify({
success: false,
message: "Token belongs to giteauser, not " + body.username
}),
{ status: 400 }
);
}
// Handle invalid token
if (body.token === "invalid-token") {
return new Response(
JSON.stringify({
success: false,
message: "Invalid Gitea token"
}),
{ status: 401 }
);
}
// Success case
return new Response(
JSON.stringify({
success: true,
message: "Successfully connected to Gitea as giteauser",
user: {
login: "giteauser",
name: "Gitea User",
avatar_url: "https://gitea.example.com/avatar.png"
}
}),
{ status: 200 }
);
});
// Mock the module
mock.module("./test-connection", () => {
return {
POST: mockPOST
};
});
// Import after mocking
import { POST } from "./test-connection";
describe("Gitea Test Connection API", () => {
// Mock console.error to prevent test output noise
let originalConsoleError: typeof console.error;
beforeEach(() => {
originalConsoleError = console.error;
console.error = mock(() => {});
});
afterEach(() => {
console.error = originalConsoleError;
});
test("returns 400 if url or token is missing", async () => {
// Test missing URL
const requestMissingUrl = new Request("http://localhost/api/gitea/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
token: "valid-token"
})
});
const responseMissingUrl = await POST({ request: requestMissingUrl } as any);
expect(responseMissingUrl.status).toBe(400);
const dataMissingUrl = await responseMissingUrl.json();
expect(dataMissingUrl.success).toBe(false);
expect(dataMissingUrl.message).toBe("Gitea URL and token are required");
// Test missing token
const requestMissingToken = new Request("http://localhost/api/gitea/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
url: "https://gitea.example.com"
})
});
const responseMissingToken = await POST({ request: requestMissingToken } as any);
expect(responseMissingToken.status).toBe(400);
const dataMissingToken = await responseMissingToken.json();
expect(dataMissingToken.success).toBe(false);
expect(dataMissingToken.message).toBe("Gitea URL and token are required");
});
test("returns 200 with user data on successful connection", async () => {
const request = new Request("http://localhost/api/gitea/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
url: "https://gitea.example.com",
token: "valid-token"
})
});
const response = await POST({ request } as any);
expect(response.status).toBe(200);
const data = await response.json();
expect(data.success).toBe(true);
expect(data.message).toBe("Successfully connected to Gitea as giteauser");
expect(data.user).toEqual({
login: "giteauser",
name: "Gitea User",
avatar_url: "https://gitea.example.com/avatar.png"
});
});
test("returns 400 if username doesn't match authenticated user", async () => {
const request = new Request("http://localhost/api/gitea/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
url: "https://gitea.example.com",
token: "valid-token",
username: "differentuser"
})
});
const response = await POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.success).toBe(false);
expect(data.message).toBe("Token belongs to giteauser, not differentuser");
});
test("handles authentication errors", async () => {
const request = new Request("http://localhost/api/gitea/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
url: "https://gitea.example.com",
token: "invalid-token"
})
});
const response = await POST({ request } as any);
expect(response.status).toBe(401);
const data = await response.json();
expect(data.success).toBe(false);
expect(data.message).toBe("Invalid Gitea token");
});
});

View File

@@ -0,0 +1,133 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
import { POST } from "./test-connection";
import { Octokit } from "@octokit/rest";
// Mock the Octokit class
mock.module("@octokit/rest", () => {
return {
Octokit: mock(function() {
return {
users: {
getAuthenticated: mock(() => Promise.resolve({
data: {
login: "testuser",
name: "Test User",
avatar_url: "https://example.com/avatar.png"
}
}))
}
};
})
};
});
describe("GitHub Test Connection API", () => {
// Mock console.error to prevent test output noise
let originalConsoleError: typeof console.error;
beforeEach(() => {
originalConsoleError = console.error;
console.error = mock(() => {});
});
afterEach(() => {
console.error = originalConsoleError;
});
test("returns 400 if token is missing", async () => {
const request = new Request("http://localhost/api/github/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({})
});
const response = await POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.success).toBe(false);
expect(data.message).toBe("GitHub token is required");
});
test("returns 200 with user data on successful connection", async () => {
const request = new Request("http://localhost/api/github/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
token: "valid-token"
})
});
const response = await POST({ request } as any);
expect(response.status).toBe(200);
const data = await response.json();
expect(data.success).toBe(true);
expect(data.message).toBe("Successfully connected to GitHub as testuser");
expect(data.user).toEqual({
login: "testuser",
name: "Test User",
avatar_url: "https://example.com/avatar.png"
});
});
test("returns 400 if username doesn't match authenticated user", async () => {
const request = new Request("http://localhost/api/github/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
token: "valid-token",
username: "differentuser"
})
});
const response = await POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.success).toBe(false);
expect(data.message).toBe("Token belongs to testuser, not differentuser");
});
test("handles authentication errors", async () => {
// Mock Octokit to throw an error
mock.module("@octokit/rest", () => {
return {
Octokit: mock(function() {
return {
users: {
getAuthenticated: mock(() => Promise.reject(new Error("Bad credentials")))
}
};
})
};
});
const request = new Request("http://localhost/api/github/test-connection", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
token: "invalid-token"
})
});
const response = await POST({ request } as any);
expect(response.status).toBe(500);
const data = await response.json();
expect(data.success).toBe(false);
expect(data.message).toContain("Bad credentials");
});
});

222
src/pages/api/health.ts Normal file
View File

@@ -0,0 +1,222 @@
import type { APIRoute } from "astro";
import { jsonResponse } from "@/lib/utils";
import { db } from "@/lib/db";
import { ENV } from "@/lib/config";
import { getRecoveryStatus, hasJobsNeedingRecovery } from "@/lib/recovery";
import os from "os";
import axios from "axios";
// Track when the server started
const serverStartTime = new Date();
// Cache for the latest version to avoid frequent GitHub API calls
interface VersionCache {
latestVersion: string;
timestamp: number;
}
let versionCache: VersionCache | null = null;
const CACHE_TTL = 3600000; // 1 hour in milliseconds
export const GET: APIRoute = async () => {
try {
// Check database connection by running a simple query
const dbStatus = await checkDatabaseConnection();
// Get system information
const systemInfo = {
uptime: getUptime(),
memory: getMemoryUsage(),
os: {
platform: os.platform(),
version: os.version(),
arch: os.arch(),
},
env: ENV.NODE_ENV,
};
// Get current and latest versions
const currentVersion = process.env.npm_package_version || "unknown";
const latestVersion = await checkLatestVersion();
// Get recovery system status
const recoveryStatus = await getRecoverySystemStatus();
// Determine overall health status
let overallStatus = "ok";
if (!dbStatus.connected) {
overallStatus = "error";
} else if (recoveryStatus.jobsNeedingRecovery > 0 && !recoveryStatus.inProgress) {
overallStatus = "degraded";
}
// Build response
const healthData = {
status: overallStatus,
timestamp: new Date().toISOString(),
version: currentVersion,
latestVersion: latestVersion,
updateAvailable: latestVersion !== "unknown" &&
currentVersion !== "unknown" &&
latestVersion !== currentVersion,
database: dbStatus,
recovery: recoveryStatus,
system: systemInfo,
};
return jsonResponse({
data: healthData,
status: 200,
});
} catch (error) {
console.error("Health check failed:", error);
return jsonResponse({
data: {
status: "error",
timestamp: new Date().toISOString(),
error: error instanceof Error ? error.message : "Unknown error",
version: process.env.npm_package_version || "unknown",
latestVersion: "unknown",
updateAvailable: false,
},
status: 503, // Service Unavailable
});
}
};
/**
* Check database connection by running a simple query
*/
async function checkDatabaseConnection() {
try {
// Run a simple query to check if the database is accessible
const result = await db.select({ test: sql`1` }).from(sql`sqlite_master`).limit(1);
return {
connected: true,
message: "Database connection successful",
};
} catch (error) {
console.error("Database connection check failed:", error);
return {
connected: false,
message: error instanceof Error ? error.message : "Database connection failed",
};
}
}
/**
* Get recovery system status
*/
async function getRecoverySystemStatus() {
try {
const recoveryStatus = getRecoveryStatus();
const needsRecovery = await hasJobsNeedingRecovery();
return {
status: needsRecovery ? 'jobs-pending' : 'healthy',
inProgress: recoveryStatus.inProgress,
lastAttempt: recoveryStatus.lastAttempt?.toISOString() || null,
jobsNeedingRecovery: needsRecovery ? 1 : 0, // Simplified count for health check
message: needsRecovery
? 'Jobs found that need recovery'
: 'No jobs need recovery',
};
} catch (error) {
console.error('Recovery system status check failed:', error);
return {
status: 'error',
inProgress: false,
lastAttempt: null,
jobsNeedingRecovery: -1,
message: error instanceof Error ? error.message : 'Recovery status check failed',
};
}
}
/**
* Get server uptime information
*/
function getUptime() {
const now = new Date();
const uptimeMs = now.getTime() - serverStartTime.getTime();
// Convert to human-readable format
const seconds = Math.floor(uptimeMs / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
const days = Math.floor(hours / 24);
return {
startTime: serverStartTime.toISOString(),
uptimeMs,
formatted: `${days}d ${hours % 24}h ${minutes % 60}m ${seconds % 60}s`,
};
}
/**
* Get memory usage information
*/
function getMemoryUsage() {
const memoryUsage = process.memoryUsage();
return {
rss: formatBytes(memoryUsage.rss),
heapTotal: formatBytes(memoryUsage.heapTotal),
heapUsed: formatBytes(memoryUsage.heapUsed),
external: formatBytes(memoryUsage.external),
systemTotal: formatBytes(os.totalmem()),
systemFree: formatBytes(os.freemem()),
};
}
/**
* Format bytes to human-readable format
*/
function formatBytes(bytes: number): string {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
/**
* Check for the latest version from GitHub releases
*/
async function checkLatestVersion(): Promise<string> {
// Return cached version if available and not expired
if (versionCache && (Date.now() - versionCache.timestamp) < CACHE_TTL) {
return versionCache.latestVersion;
}
try {
// Fetch the latest release from GitHub
const response = await axios.get(
'https://api.github.com/repos/arunavo4/gitea-mirror/releases/latest',
{ headers: { 'Accept': 'application/vnd.github.v3+json' } }
);
// Extract version from tag_name (remove 'v' prefix if present)
const latestVersion = response.data.tag_name.replace(/^v/, '');
// Update cache
versionCache = {
latestVersion,
timestamp: Date.now()
};
return latestVersion;
} catch (error) {
console.error('Failed to check for latest version:', error);
return 'unknown';
}
}
// Import sql tag for raw SQL queries
import { sql } from "drizzle-orm";

View File

@@ -0,0 +1,109 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
// Create a mock POST function
const mockPOST = mock(async ({ request }) => {
const body = await request.json();
// Check for missing userId or organizationIds
if (!body.userId || !body.organizationIds) {
return new Response(
JSON.stringify({
error: "Missing userId or organizationIds."
}),
{ status: 400 }
);
}
// Success case
return new Response(
JSON.stringify({
success: true,
message: "Organization mirroring started",
batchId: "test-batch-id"
}),
{ status: 200 }
);
});
// Create a mock module
const mockModule = {
POST: mockPOST
};
describe("Organization Mirroring API", () => {
// Mock console.log and console.error to prevent test output noise
let originalConsoleLog: typeof console.log;
let originalConsoleError: typeof console.error;
beforeEach(() => {
originalConsoleLog = console.log;
originalConsoleError = console.error;
console.log = mock(() => {});
console.error = mock(() => {});
});
afterEach(() => {
console.log = originalConsoleLog;
console.error = originalConsoleError;
});
test("returns 400 if userId is missing", async () => {
const request = new Request("http://localhost/api/job/mirror-org", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
organizationIds: ["org-id-1", "org-id-2"]
})
});
const response = await mockModule.POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.error).toBe("Missing userId or organizationIds.");
});
test("returns 400 if organizationIds is missing", async () => {
const request = new Request("http://localhost/api/job/mirror-org", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
userId: "user-id"
})
});
const response = await mockModule.POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.error).toBe("Missing userId or organizationIds.");
});
test("returns 200 and starts mirroring organizations", async () => {
const request = new Request("http://localhost/api/job/mirror-org", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
userId: "user-id",
organizationIds: ["org-id-1", "org-id-2"]
})
});
const response = await mockModule.POST({ request } as any);
expect(response.status).toBe(200);
const data = await response.json();
expect(data.success).toBe(true);
expect(data.message).toBe("Organization mirroring started");
expect(data.batchId).toBe("test-batch-id");
});
});

View File

@@ -6,6 +6,8 @@ import { createGitHubClient } from "@/lib/github";
import { mirrorGitHubOrgToGitea } from "@/lib/gitea";
import { repoStatusEnum } from "@/types/Repository";
import { type MembershipRole } from "@/types/organizations";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -61,31 +63,72 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Fire async mirroring without blocking response
// Fire async mirroring without blocking response, using parallel processing with resilience
setTimeout(async () => {
for (const org of orgs) {
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing in config.");
}
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing in config.");
}
const octokit = createGitHubClient(config.githubConfig.token);
// Create a single Octokit instance to be reused
const octokit = createGitHubClient(config.githubConfig.token);
try {
// Define the concurrency limit - adjust based on API rate limits
// Using a lower concurrency for organizations since each org might contain many repos
const CONCURRENCY_LIMIT = 2;
// Generate a batch ID to group related organizations
const batchId = uuidv4();
// Process organizations in parallel with resilience to container restarts
await processWithResilience(
orgs,
async (org) => {
// Prepare organization data
const orgData = {
...org,
status: repoStatusEnum.parse("imported"),
membershipRole: org.membershipRole as MembershipRole,
lastMirrored: org.lastMirrored ?? undefined,
errorMessage: org.errorMessage ?? undefined,
};
// Log the start of mirroring
console.log(`Starting mirror for organization: ${org.name}`);
// Mirror the organization
await mirrorGitHubOrgToGitea({
config,
octokit,
organization: {
...org,
status: repoStatusEnum.parse("imported"),
membershipRole: org.membershipRole as MembershipRole,
lastMirrored: org.lastMirrored ?? undefined,
errorMessage: org.errorMessage ?? undefined,
},
organization: orgData,
});
} catch (error) {
console.error(`Mirror failed for organization ${org.name}:`, error);
return org;
},
{
userId: config.userId || "",
jobType: "mirror",
batchId,
getItemId: (org) => org.id,
getItemName: (org) => org.name,
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 3000,
checkpointInterval: 1, // Checkpoint after each organization
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Organization mirroring progress: ${percentComplete}% (${completed}/${total})`);
if (result) {
console.log(`Successfully mirrored organization: ${result.name}`);
}
},
onRetry: (org, error, attempt) => {
console.log(`Retrying organization ${org.name} (attempt ${attempt}): ${error.message}`);
}
}
}
);
console.log("All organization mirroring tasks completed");
}, 0);
const responsePayload: MirrorOrgResponse = {

View File

@@ -0,0 +1,109 @@
import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test";
// Create a mock POST function
const mockPOST = mock(async ({ request }) => {
const body = await request.json();
// Check for missing userId or repositoryIds
if (!body.userId || !body.repositoryIds) {
return new Response(
JSON.stringify({
error: "Missing userId or repositoryIds."
}),
{ status: 400 }
);
}
// Success case
return new Response(
JSON.stringify({
success: true,
message: "Repository mirroring started",
batchId: "test-batch-id"
}),
{ status: 200 }
);
});
// Create a mock module
const mockModule = {
POST: mockPOST
};
describe("Repository Mirroring API", () => {
// Mock console.log and console.error to prevent test output noise
let originalConsoleLog: typeof console.log;
let originalConsoleError: typeof console.error;
beforeEach(() => {
originalConsoleLog = console.log;
originalConsoleError = console.error;
console.log = mock(() => {});
console.error = mock(() => {});
});
afterEach(() => {
console.log = originalConsoleLog;
console.error = originalConsoleError;
});
test("returns 400 if userId is missing", async () => {
const request = new Request("http://localhost/api/job/mirror-repo", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
repositoryIds: ["repo-id-1", "repo-id-2"]
})
});
const response = await mockModule.POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.error).toBe("Missing userId or repositoryIds.");
});
test("returns 400 if repositoryIds is missing", async () => {
const request = new Request("http://localhost/api/job/mirror-repo", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
userId: "user-id"
})
});
const response = await mockModule.POST({ request } as any);
expect(response.status).toBe(400);
const data = await response.json();
expect(data.error).toBe("Missing userId or repositoryIds.");
});
test("returns 200 and starts mirroring repositories", async () => {
const request = new Request("http://localhost/api/job/mirror-repo", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
userId: "user-id",
repositoryIds: ["repo-id-1", "repo-id-2"]
})
});
const response = await mockModule.POST({ request } as any);
expect(response.status).toBe(200);
const data = await response.json();
expect(data.success).toBe(true);
expect(data.message).toBe("Repository mirroring started");
expect(data.batchId).toBe("test-batch-id");
});
});

View File

@@ -8,6 +8,8 @@ import {
mirrorGitHubOrgRepoToGiteaOrg,
} from "@/lib/gitea";
import { createGitHubClient } from "@/lib/github";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -63,52 +65,83 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Start async mirroring in background
// Start async mirroring in background with parallel processing and resilience
setTimeout(async () => {
for (const repo of repos) {
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing.");
}
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing.");
}
const octokit = createGitHubClient(config.githubConfig.token);
// Create a single Octokit instance to be reused
const octokit = createGitHubClient(config.githubConfig.token);
try {
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 3;
// Generate a batch ID to group related repositories
const batchId = uuidv4();
// Process repositories in parallel with resilience to container restarts
await processWithResilience(
repos,
async (repo) => {
// Prepare repository data
const repoData = {
...repo,
status: repoStatusEnum.parse("imported"),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
};
// Log the start of mirroring
console.log(`Starting mirror for repository: ${repo.name}`);
// Mirror the repository based on whether it's in an organization
if (repo.organization && config.githubConfig.preserveOrgStructure) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
octokit,
orgName: repo.organization,
repository: {
...repo,
status: repoStatusEnum.parse("imported"),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
},
repository: repoData,
});
} else {
await mirrorGithubRepoToGitea({
octokit,
repository: {
...repo,
status: repoStatusEnum.parse("imported"),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
mirroredLocation: repo.mirroredLocation || "",
},
repository: repoData,
config,
});
}
} catch (error) {
console.error(`Mirror failed for repo ${repo.name}:`, error);
return repo;
},
{
userId: config.userId || "",
jobType: "mirror",
batchId,
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 2000,
checkpointInterval: 5, // Checkpoint every 5 repositories to reduce event frequency
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Mirroring progress: ${percentComplete}% (${completed}/${total})`);
if (result) {
console.log(`Successfully mirrored repository: ${result.name}`);
}
},
onRetry: (repo, error, attempt) => {
console.log(`Retrying repository ${repo.name} (attempt ${attempt}): ${error.message}`);
}
}
}
);
console.log("All repository mirroring tasks completed");
}, 0);
const responsePayload: MirrorRepoResponse = {

View File

@@ -10,6 +10,8 @@ import {
import { createGitHubClient } from "@/lib/github";
import { repoStatusEnum, repositoryVisibilityEnum } from "@/types/Repository";
import type { RetryRepoRequest, RetryRepoResponse } from "@/types/retry";
import { processWithRetry } from "@/lib/utils/concurrency";
import { createMirrorJob } from "@/lib/helpers";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -65,10 +67,21 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Start background retry
// Start background retry with parallel processing
setTimeout(async () => {
for (const repo of repos) {
try {
// Create a single Octokit instance to be reused if needed
const octokit = config.githubConfig.token
? createGitHubClient(config.githubConfig.token)
: null;
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 3;
// Process repositories in parallel with retry capability
await processWithRetry(
repos,
async (repo) => {
// Prepare repository data
const visibility = repositoryVisibilityEnum.parse(repo.visibility);
const status = repoStatusEnum.parse(repo.status);
const repoData = {
@@ -81,6 +94,20 @@ export const POST: APIRoute = async ({ request }) => {
forkedFrom: repo.forkedFrom ?? undefined,
};
// Log the start of retry operation
console.log(`Starting retry for repository: ${repo.name}`);
// Create a mirror job entry to track progress
await createMirrorJob({
userId: config.userId || "",
repositoryId: repo.id,
repositoryName: repo.name,
message: `Started retry operation for repository: ${repo.name}`,
details: `Repository ${repo.name} is now in the retry queue.`,
status: "imported",
});
// Determine if the repository exists in Gitea
let owner = getGiteaRepoOwner({
config,
repository: repoData,
@@ -93,16 +120,21 @@ export const POST: APIRoute = async ({ request }) => {
});
if (present) {
// If the repository exists, sync it
await syncGiteaRepo({ config, repository: repoData });
console.log(`Synced existing repo: ${repo.name}`);
} else {
// If the repository doesn't exist, mirror it
if (!config.githubConfig.token) {
throw new Error("GitHub token is missing.");
}
if (!octokit) {
throw new Error("Octokit client is not initialized.");
}
console.log(`Importing repo: ${repo.name} ${owner}`);
const octokit = createGitHubClient(config.githubConfig.token);
if (repo.organization && config.githubConfig.preserveOrgStructure) {
await mirrorGitHubOrgRepoToGiteaOrg({
config,
@@ -124,10 +156,28 @@ export const POST: APIRoute = async ({ request }) => {
});
}
}
} catch (err) {
console.error(`Failed to retry repo ${repo.name}:`, err);
return repo;
},
{
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 2000,
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Retry progress: ${percentComplete}% (${completed}/${total})`);
if (result) {
console.log(`Successfully processed repository: ${result.name}`);
}
},
onRetry: (repo, error, attempt) => {
console.log(`Retrying repository ${repo.name} (attempt ${attempt}): ${error.message}`);
}
}
}
);
console.log("All repository retry tasks completed");
}, 0);
const responsePayload: RetryRepoResponse = {

View File

@@ -5,6 +5,8 @@ import { eq, inArray } from "drizzle-orm";
import { repositoryVisibilityEnum, repoStatusEnum } from "@/types/Repository";
import { syncGiteaRepo } from "@/lib/gitea";
import type { SyncRepoResponse } from "@/types/sync";
import { processWithResilience } from "@/lib/utils/concurrency";
import { v4 as uuidv4 } from "uuid";
export const POST: APIRoute = async ({ request }) => {
try {
@@ -60,26 +62,65 @@ export const POST: APIRoute = async ({ request }) => {
);
}
// Start async mirroring in background
// Start async mirroring in background with parallel processing and resilience
setTimeout(async () => {
for (const repo of repos) {
try {
// Define the concurrency limit - adjust based on API rate limits
const CONCURRENCY_LIMIT = 5;
// Generate a batch ID to group related repositories
const batchId = uuidv4();
// Process repositories in parallel with resilience to container restarts
await processWithResilience(
repos,
async (repo) => {
// Prepare repository data
const repoData = {
...repo,
status: repoStatusEnum.parse(repo.status),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
};
// Log the start of syncing
console.log(`Starting sync for repository: ${repo.name}`);
// Sync the repository
await syncGiteaRepo({
config,
repository: {
...repo,
status: repoStatusEnum.parse(repo.status),
organization: repo.organization ?? undefined,
lastMirrored: repo.lastMirrored ?? undefined,
errorMessage: repo.errorMessage ?? undefined,
forkedFrom: repo.forkedFrom ?? undefined,
visibility: repositoryVisibilityEnum.parse(repo.visibility),
},
repository: repoData,
});
} catch (error) {
console.error(`Sync failed for repo ${repo.name}:`, error);
return repo;
},
{
userId: config.userId || "",
jobType: "sync",
batchId,
getItemId: (repo) => repo.id,
getItemName: (repo) => repo.name,
concurrencyLimit: CONCURRENCY_LIMIT,
maxRetries: 2,
retryDelay: 2000,
checkpointInterval: 1, // Checkpoint after each repository
onProgress: (completed, total, result) => {
const percentComplete = Math.round((completed / total) * 100);
console.log(`Syncing progress: ${percentComplete}% (${completed}/${total})`);
if (result) {
console.log(`Successfully synced repository: ${result.name}`);
}
},
onRetry: (repo, error, attempt) => {
console.log(`Retrying sync for repository ${repo.name} (attempt ${attempt}): ${error.message}`);
}
}
}
);
console.log("All repository syncing tasks completed");
}, 0);
const responsePayload: SyncRepoResponse = {

View File

@@ -34,8 +34,6 @@ export const GET: APIRoute = async ({ request }) => {
if (isClosed) return;
try {
console.log(`Polling for events for user ${userId} in channel ${channel}`);
// Get new events from SQLite
const events = await getNewEvents({
userId,
@@ -43,8 +41,6 @@ export const GET: APIRoute = async ({ request }) => {
lastEventTime,
});
console.log(`Found ${events.length} new events`);
// Send events to client
if (events.length > 0) {
// Update last event time
@@ -52,7 +48,6 @@ export const GET: APIRoute = async ({ request }) => {
// Send each event to the client
for (const event of events) {
console.log(`Sending event: ${JSON.stringify(event.payload)}`);
sendMessage(`data: ${JSON.stringify(event.payload)}\n\n`);
}
}

20
src/tests/setup.bun.ts Normal file
View File

@@ -0,0 +1,20 @@
/**
* Bun test setup file
* This file is automatically loaded before running tests
*/
import { afterEach, beforeEach } from "bun:test";
// Clean up after each test
afterEach(() => {
// Add any cleanup logic here
});
// Setup before each test
beforeEach(() => {
// Add any setup logic here
});
// Add DOM testing support if needed
// import { DOMParser } from "linkedom";
// global.DOMParser = DOMParser;

View File

@@ -18,6 +18,13 @@ export interface ScheduleConfig {
nextRun?: Date;
}
export interface DatabaseCleanupConfig {
enabled: boolean;
retentionDays: number;
lastRun?: Date;
nextRun?: Date;
}
export interface GitHubConfig {
username: string;
token: string;
@@ -34,6 +41,7 @@ export interface SaveConfigApiRequest {
githubConfig: GitHubConfig;
giteaConfig: GiteaConfig;
scheduleConfig: ScheduleConfig;
cleanupConfig: DatabaseCleanupConfig;
}
export interface SaveConfigApiResponse {
@@ -55,6 +63,7 @@ export interface ConfigApiResponse {
githubConfig: GitHubConfig;
giteaConfig: GiteaConfig;
scheduleConfig: ScheduleConfig;
cleanupConfig: DatabaseCleanupConfig;
include: string[];
exclude: string[];
createdAt: Date;