diff --git a/.github/workflows/gitea_backup.yml b/.github/workflows/gitea_backup.yml new file mode 100644 index 0000000000..e81f1f7b42 --- /dev/null +++ b/.github/workflows/gitea_backup.yml @@ -0,0 +1,102 @@ +name: Gitea Offsite Backup + +on: + workflow_dispatch: + inputs: + full_restore_verify: + description: 'Download and validate a complete restored snapshot' + required: true + default: false + type: boolean + maintenance: + description: 'Apply retention and prune unreferenced Restic packs' + required: true + default: false + type: boolean + schedule: + - cron: '23 10 * * *' + - cron: '53 11 * * 0' + +concurrency: + group: gitea-offsite-backup + cancel-in-progress: false + +permissions: + contents: read + +jobs: + backup: + if: (github.event_name == 'workflow_dispatch' && inputs.maintenance == false) || (github.event_name == 'schedule' && github.event.schedule == '23 10 * * *') + runs-on: [self-hosted, macOS, ARM64, halfdan] + timeout-minutes: 240 + + steps: + - name: Prepare non-LFS checkout + env: + GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs + GIT_CONFIG_NOSYSTEM: '1' + run: | + git config --global --unset-all filter.lfs.process || true + git config --global filter.lfs.smudge "cat" + git config --global filter.lfs.clean "cat" + git config --global filter.lfs.required false + if [ -d ".git" ]; then + git config --local --unset-all filter.lfs.process || true + git config --local filter.lfs.smudge "cat" + git config --local filter.lfs.clean "cat" + git config --local filter.lfs.required false + fi + rm -f .git/hooks/post-checkout .git/hooks/post-merge .git/hooks/pre-push + + - uses: actions/checkout@v7 + env: + GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs + GIT_CONFIG_NOSYSTEM: '1' + with: + persist-credentials: false + lfs: false + clean: false + + - name: Back up Gitea + env: + DO_SPACES_ACCESS_KEY: ${{ secrets.GITEA_DO_SPACES_ACCESS_KEY }} + DO_SPACES_ENDPOINT: ${{ secrets.DO_SPACES_ENDPOINT }} + DO_SPACES_SECRET_KEY: ${{ secrets.GITEA_DO_SPACES_SECRET_KEY }} + FULL_RESTORE_VERIFY: ${{ inputs.full_restore_verify }} + GITEA_BACKUP_SSH_KEY: ${{ secrets.GITEA_BACKUP_SSH_KEY }} + RESTIC_PASSWORD: ${{ secrets.GITEA_RESTIC_PASSWORD }} + run: ./ci/github_actions/backup_gitea.sh backup + + maintenance: + if: (github.event_name == 'schedule' && github.event.schedule == '53 11 * * 0') || (github.event_name == 'workflow_dispatch' && inputs.maintenance) + runs-on: [self-hosted, macOS, ARM64, halfdan] + timeout-minutes: 240 + + steps: + - name: Prepare non-LFS checkout + env: + GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs + GIT_CONFIG_NOSYSTEM: '1' + run: | + git config --global --unset-all filter.lfs.process || true + git config --global filter.lfs.smudge "cat" + git config --global filter.lfs.clean "cat" + git config --global filter.lfs.required false + + - uses: actions/checkout@v7 + env: + GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs + GIT_CONFIG_NOSYSTEM: '1' + with: + persist-credentials: false + lfs: false + clean: false + + - name: Apply retention and verify repository + env: + DO_SPACES_ACCESS_KEY: ${{ secrets.GITEA_DO_SPACES_ACCESS_KEY }} + DO_SPACES_ENDPOINT: ${{ secrets.DO_SPACES_ENDPOINT }} + DO_SPACES_SECRET_KEY: ${{ secrets.GITEA_DO_SPACES_SECRET_KEY }} + GITEA_BACKUP_SSH_KEY: ${{ secrets.GITEA_BACKUP_SSH_KEY }} + RESTIC_PASSWORD: ${{ secrets.GITEA_RESTIC_PASSWORD }} + run: ./ci/github_actions/backup_gitea.sh maintenance diff --git a/ci/github_actions/backup_gitea.sh b/ci/github_actions/backup_gitea.sh new file mode 100755 index 0000000000..a08a6fcb1b --- /dev/null +++ b/ci/github_actions/backup_gitea.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env bash + +set -euo pipefail + +BACKUP_MODE=${1:-backup} +GITEA_BACKUP_SSH_HOST=${GITEA_BACKUP_SSH_HOST:-gitea-backup-source} +GITEA_HTTP_URL=${GITEA_HTTP_URL:-http://zog:3000/admin/eagle0.git} +GITEA_VERSION_URL=${GITEA_VERSION_URL:-http://zog:3000/api/v1/version} +GITEA_REMOTE_ROOT=${GITEA_REMOTE_ROOT:-/volume2/docker/gitea} +GITEA_BACKUP_BUCKET=${GITEA_BACKUP_BUCKET:-eagle0-gitea-backups} +GITEA_BACKUP_PREFIX=${GITEA_BACKUP_PREFIX:-restic} +RUN_ID=${GITHUB_RUN_ID:-manual} +GITEA_DATA_DIRECTORIES=( + actions_artifacts + actions_log + attachments + avatars + conf + home + packages + repo-avatars +) + +require_environment_variable() { + local variable_name=$1 + if [ -z "${!variable_name:-}" ]; then + echo "ERROR: Required environment variable '$variable_name' is not set." + exit 1 + fi +} + +require_environment_variable RUNNER_TEMP +require_environment_variable DO_SPACES_ENDPOINT +require_environment_variable DO_SPACES_ACCESS_KEY +require_environment_variable DO_SPACES_SECRET_KEY +require_environment_variable GITEA_BACKUP_SSH_KEY +require_environment_variable RESTIC_PASSWORD + +case "$RUN_ID" in + *[!A-Za-z0-9._-]*) + echo "ERROR: Unsafe run identifier '$RUN_ID'." + exit 1 + ;; +esac + +STAGING_ROOT="$RUNNER_TEMP/eagle0-gitea-backup" +VERIFY_ROOT="$RUNNER_TEMP/eagle0-gitea-restore-verification" +REMOTE_SNAPSHOT_DIR="$GITEA_REMOTE_ROOT/backup-staging" +REMOTE_DATABASE="$GITEA_REMOTE_ROOT/data/gitea/gitea.db" +REMOTE_DATABASE_SNAPSHOT="$REMOTE_SNAPSHOT_DIR/gitea-$RUN_ID.db" +RESTIC_REPOSITORY="s3:${DO_SPACES_ENDPOINT%/}/$GITEA_BACKUP_BUCKET/$GITEA_BACKUP_PREFIX" +SSH_CONFIG="$RUNNER_TEMP/eagle0-gitea-backup-ssh-config" +SSH_PRIVATE_KEY="$RUNNER_TEMP/eagle0-gitea-backup-ssh-key" + +export AWS_ACCESS_KEY_ID="$DO_SPACES_ACCESS_KEY" +export AWS_SECRET_ACCESS_KEY="$DO_SPACES_SECRET_KEY" +export AWS_DEFAULT_REGION=us-east-1 +export RESTIC_REPOSITORY +export RESTIC_PASSWORD +export RESTIC_CACHE_DIR="$RUNNER_TEMP/restic-cache" +export HOMEBREW_NO_AUTO_UPDATE=1 +export HOMEBREW_NO_INSTALL_CLEANUP=1 + +COMMON_PATHS=(/opt/homebrew/bin /usr/local/bin) +for common_path in "${COMMON_PATHS[@]}"; do + if [ -d "$common_path" ]; then + export PATH="$common_path:$PATH" + fi +done + +ensure_tool() { + local command_name=$1 + local brew_package=$2 + + if command -v "$command_name" >/dev/null 2>&1; then + return + fi + + if ! command -v brew >/dev/null 2>&1; then + echo "ERROR: '$command_name' is unavailable and Homebrew is not installed." + exit 1 + fi + + brew install "$brew_package" +} + +safe_reset_directory() { + local directory=$1 + + case "$directory" in + "$RUNNER_TEMP"/eagle0-gitea-*) + rm -rf "$directory" + mkdir -p "$directory" + ;; + *) + echo "ERROR: Refusing to reset unsafe directory '$directory'." + exit 1 + ;; + esac +} + +remove_remote_database_snapshot() { + ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" \ + "rm -f '$REMOTE_DATABASE_SNAPSHOT'" >/dev/null 2>&1 || true +} + +cleanup() { + remove_remote_database_snapshot + + case "$STAGING_ROOT" in + "$RUNNER_TEMP"/eagle0-gitea-*) + rm -rf "$STAGING_ROOT" + ;; + esac + + case "$VERIFY_ROOT" in + "$RUNNER_TEMP"/eagle0-gitea-*) + rm -rf "$VERIFY_ROOT" + ;; + esac + + rm -f "$SSH_CONFIG" "$SSH_PRIVATE_KEY" +} + +trap cleanup EXIT + +umask 077 +printf '%s\n' "$GITEA_BACKUP_SSH_KEY" > "$SSH_PRIVATE_KEY" +printf '%s\n' \ + "Host $GITEA_BACKUP_SSH_HOST" \ + " HostName zog" \ + " User \"Dan Crosby\"" \ + " IdentityFile $SSH_PRIVATE_KEY" \ + " IdentitiesOnly yes" \ + " HostKeyAlias zog" \ + > "$SSH_CONFIG" + +ensure_tool aws awscli +ensure_tool restic restic +ensure_tool sqlite3 sqlite + +require_backup_bucket() { + if ! aws s3api head-bucket \ + --bucket "$GITEA_BACKUP_BUCKET" \ + --endpoint-url "$DO_SPACES_ENDPOINT" >/dev/null 2>&1; then + echo "ERROR: DigitalOcean Space '$GITEA_BACKUP_BUCKET' is unavailable." + exit 1 + fi +} + +initialize_repository_if_needed() { + if restic cat config >/dev/null 2>&1; then + return + fi + + echo "Initializing encrypted Restic repository." + restic init +} + +require_initialized_repository() { + if restic cat config >/dev/null 2>&1; then + return + fi + + echo "ERROR: The encrypted Restic repository is not initialized." + exit 1 +} + +prepare_staging_data() { + safe_reset_directory "$STAGING_ROOT" + mkdir -p "$STAGING_ROOT/gitea" + mkdir -p "$STAGING_ROOT/repositories/admin" + + ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" true + ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" \ + "mkdir -p '$REMOTE_SNAPSHOT_DIR'" + ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" \ + "sqlite3 '$REMOTE_DATABASE' \".backup '$REMOTE_DATABASE_SNAPSHOT'\"" + + scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -pr \ + "$GITEA_BACKUP_SSH_HOST:$GITEA_REMOTE_ROOT/config" \ + "$STAGING_ROOT/" + + for directory in "${GITEA_DATA_DIRECTORIES[@]}"; do + scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -pr \ + "$GITEA_BACKUP_SSH_HOST:$GITEA_REMOTE_ROOT/data/gitea/$directory" \ + "$STAGING_ROOT/gitea/" + done + + scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -p \ + "$GITEA_BACKUP_SSH_HOST:$REMOTE_DATABASE_SNAPSHOT" \ + "$STAGING_ROOT/gitea/gitea.db" + + remove_remote_database_snapshot + + scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -pr \ + "$GITEA_BACKUP_SSH_HOST:$GITEA_REMOTE_ROOT/data/lfs" \ + "$STAGING_ROOT/" + + git clone --mirror \ + "$GITEA_HTTP_URL" \ + "$STAGING_ROOT/repositories/admin/eagle0.git" + + local database_check + database_check=$(sqlite3 "$STAGING_ROOT/gitea/gitea.db" "PRAGMA integrity_check;") + if [ "$database_check" != "ok" ]; then + echo "ERROR: Gitea database snapshot failed SQLite integrity checking." + exit 1 + fi + + git --git-dir="$STAGING_ROOT/repositories/admin/eagle0.git" fsck --full + + local gitea_version + gitea_version=$(curl --fail --silent "$GITEA_VERSION_URL") + local main_commit + main_commit=$(git --git-dir="$STAGING_ROOT/repositories/admin/eagle0.git" rev-parse refs/heads/main) + local captured_at + captured_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + printf '%s\n' \ + "captured_at=$captured_at" \ + "gitea_version=$gitea_version" \ + "main_commit=$main_commit" \ + "source_host=$GITEA_BACKUP_SSH_HOST" \ + "source_root=$GITEA_REMOTE_ROOT" \ + > "$STAGING_ROOT/manifest.txt" +} + +verify_restored_snapshot() { + safe_reset_directory "$VERIFY_ROOT" + restic restore latest --host zog --tag gitea --target "$VERIFY_ROOT" + + local restored_root="$VERIFY_ROOT/${STAGING_ROOT#/}" + local database_check + database_check=$(sqlite3 "$restored_root/gitea/gitea.db" "PRAGMA integrity_check;") + if [ "$database_check" != "ok" ]; then + echo "ERROR: Restored Gitea database failed SQLite integrity checking." + exit 1 + fi + + git --git-dir="$restored_root/repositories/admin/eagle0.git" fsck --full + test -s "$restored_root/manifest.txt" + test -d "$restored_root/lfs" + echo "Full restore verification completed successfully." +} + +run_backup() { + require_backup_bucket + initialize_repository_if_needed + prepare_staging_data + + restic backup "$STAGING_ROOT" --host zog --tag gitea + restic forget \ + --host zog \ + --tag gitea \ + --group-by host,tags \ + --keep-daily 14 \ + --keep-weekly 8 \ + --keep-monthly 12 \ + --keep-yearly 3 + + if [ "${FULL_RESTORE_VERIFY:-false}" = "true" ]; then + restic check --read-data + verify_restored_snapshot + else + restic check --read-data-subset=5% + fi + + restic snapshots --host zog --tag gitea +} + +run_maintenance() { + require_initialized_repository + restic forget \ + --host zog \ + --tag gitea \ + --keep-daily 14 \ + --keep-weekly 8 \ + --keep-monthly 12 \ + --keep-yearly 3 \ + --prune + restic check --read-data-subset=10% +} + +case "$BACKUP_MODE" in + backup) + run_backup + ;; + maintenance) + run_maintenance + ;; + *) + echo "Usage: $0 [backup|maintenance]" + exit 2 + ;; +esac diff --git a/docs/operations/gitea-backups.md b/docs/operations/gitea-backups.md new file mode 100644 index 0000000000..cbc58e3bd3 --- /dev/null +++ b/docs/operations/gitea-backups.md @@ -0,0 +1,87 @@ +# Gitea offsite backups + +Eagle0's Gitea mirror keeps Git and Git LFS data on `zog` for fast local +access. The `Gitea Offsite Backup` GitHub Actions workflow copies the +recoverable state to an encrypted Restic repository in the private +`eagle0-gitea-backups` DigitalOcean Space. + +## Backup contents + +Each snapshot contains: + +- a transactionally consistent SQLite backup of the Gitea database; +- Gitea configuration and persistent application data; +- every locally stored Git LFS object; +- a fresh mirror clone of `admin/eagle0`; +- the Gitea version and captured `main` commit in `manifest.txt`. + +Regenerable indexes, sessions, queues, repository archives, logs, and the +OAuth2 JWT signing key are excluded. Gitea creates a new JWT signing key when +none exists, which invalidates pre-restoration OAuth2 tokens. Container-managed +SSH host keys are not readable by the backup account and must also be +regenerated after a complete NAS loss. + +## Schedule and retention + +The backup runs daily at 10:23 UTC on a `halfdan` self-hosted runner. Each +run validates the SQLite snapshot, runs `git fsck`, uploads an encrypted +snapshot, and reads a rotating 5% sample of the Restic repository. + +Weekly maintenance runs Sunday at 11:53 UTC. It retains 14 daily, 8 weekly, +12 monthly, and 3 yearly snapshots, prunes unreferenced Restic packs, and +reads a rotating 10% sample. + +Use the workflow's `full_restore_verify` option after storage or credential +changes. It downloads the latest snapshot, validates the restored SQLite +database, and runs `git fsck` against the restored repository. + +## Credentials + +GitHub Actions stores: + +- `GITEA_DO_SPACES_ACCESS_KEY` +- `GITEA_DO_SPACES_SECRET_KEY` +- `DO_SPACES_ENDPOINT` +- `GITEA_BACKUP_SSH_KEY` +- `GITEA_RESTIC_PASSWORD` + +`GITEA_BACKUP_SSH_KEY` is a dedicated key accepted by the NAS account used for +backup reads. It is not a copy of a developer's personal SSH key. + +The Restic recovery password also has an offline recovery copy in the +workstation's macOS Keychain under service `eagle0-gitea-restic`. Never put +that password in the repository, workflow logs, or a support ticket. + +## Restore + +Install `restic`, then set the DigitalOcean credentials, Restic repository, +and Restic password in the shell performing the restore. Restore into an +empty directory: + +```bash +restic snapshots --host zog --tag gitea +restic restore latest --host zog --tag gitea --target /path/to/restore +``` + +The snapshot preserves the absolute staging path below the restore target. +Locate `manifest.txt`; its containing directory is the restored backup root. + +Before changing Gitea: + +```bash +sqlite3 /path/to/restored/root/gitea/gitea.db "PRAGMA integrity_check;" +git --git-dir=/path/to/restored/root/repositories/admin/eagle0.git fsck --full +``` + +To recover a complete installation: + +1. Recreate the same Gitea version shown in `manifest.txt`. +2. Stop Gitea. +3. Restore `config`, `gitea`, `lfs`, and the bare repository into the + matching paths under `/volume2/docker/gitea`. +4. Preserve the restored file ownership expected by the container. +5. Allow Gitea to regenerate indexes, sessions, queues, archives, and SSH + host keys. +6. Start Gitea and verify a historical LFS checkout before enabling writes. + +Do not overwrite a running installation during a restore.