Add encrypted offsite backups for Gitea (#8795)

* Add encrypted Gitea offsite backups

* Avoid Homebrew cleanup in Gitea backup

* Use dedicated Spaces credentials for Gitea backups

* Authenticate Gitea backups with a dedicated SSH key

* Use SCP for Synology Gitea backups

* Skip regenerable unreadable Gitea state

* Finalize scheduled Gitea backup workflow

* Group Gitea backup retention across runners

.github/workflows/gitea_backup.yml

@@ -0,0 +1,102 @@
+name: Gitea Offsite Backup
+
+on:
+  workflow_dispatch:
+    inputs:
+      full_restore_verify:
+        description: 'Download and validate a complete restored snapshot'
+        required: true
+        default: false
+        type: boolean
+      maintenance:
+        description: 'Apply retention and prune unreferenced Restic packs'
+        required: true
+        default: false
+        type: boolean
+  schedule:
+    - cron: '23 10 * * *'
+    - cron: '53 11 * * 0'
+
+concurrency:
+  group: gitea-offsite-backup
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  backup:
+    if: (github.event_name == 'workflow_dispatch' && inputs.maintenance == false) || (github.event_name == 'schedule' && github.event.schedule == '23 10 * * *')
+    runs-on: [self-hosted, macOS, ARM64, halfdan]
+    timeout-minutes: 240
+
+    steps:
+      - name: Prepare non-LFS checkout
+        env:
+          GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
+          GIT_CONFIG_NOSYSTEM: '1'
+        run: |
+          git config --global --unset-all filter.lfs.process || true
+          git config --global filter.lfs.smudge "cat"
+          git config --global filter.lfs.clean "cat"
+          git config --global filter.lfs.required false
+          if [ -d ".git" ]; then
+            git config --local --unset-all filter.lfs.process || true
+            git config --local filter.lfs.smudge "cat"
+            git config --local filter.lfs.clean "cat"
+            git config --local filter.lfs.required false
+          fi
+          rm -f .git/hooks/post-checkout .git/hooks/post-merge .git/hooks/pre-push
+
+      - uses: actions/checkout@v7
+        env:
+          GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
+          GIT_CONFIG_NOSYSTEM: '1'
+        with:
+          persist-credentials: false
+          lfs: false
+          clean: false
+
+      - name: Back up Gitea
+        env:
+          DO_SPACES_ACCESS_KEY: ${{ secrets.GITEA_DO_SPACES_ACCESS_KEY }}
+          DO_SPACES_ENDPOINT: ${{ secrets.DO_SPACES_ENDPOINT }}
+          DO_SPACES_SECRET_KEY: ${{ secrets.GITEA_DO_SPACES_SECRET_KEY }}
+          FULL_RESTORE_VERIFY: ${{ inputs.full_restore_verify }}
+          GITEA_BACKUP_SSH_KEY: ${{ secrets.GITEA_BACKUP_SSH_KEY }}
+          RESTIC_PASSWORD: ${{ secrets.GITEA_RESTIC_PASSWORD }}
+        run: ./ci/github_actions/backup_gitea.sh backup
+
+  maintenance:
+    if: (github.event_name == 'schedule' && github.event.schedule == '53 11 * * 0') || (github.event_name == 'workflow_dispatch' && inputs.maintenance)
+    runs-on: [self-hosted, macOS, ARM64, halfdan]
+    timeout-minutes: 240
+
+    steps:
+      - name: Prepare non-LFS checkout
+        env:
+          GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
+          GIT_CONFIG_NOSYSTEM: '1'
+        run: |
+          git config --global --unset-all filter.lfs.process || true
+          git config --global filter.lfs.smudge "cat"
+          git config --global filter.lfs.clean "cat"
+          git config --global filter.lfs.required false
+
+      - uses: actions/checkout@v7
+        env:
+          GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
+          GIT_CONFIG_NOSYSTEM: '1'
+        with:
+          persist-credentials: false
+          lfs: false
+          clean: false
+
+      - name: Apply retention and verify repository
+        env:
+          DO_SPACES_ACCESS_KEY: ${{ secrets.GITEA_DO_SPACES_ACCESS_KEY }}
+          DO_SPACES_ENDPOINT: ${{ secrets.DO_SPACES_ENDPOINT }}
+          DO_SPACES_SECRET_KEY: ${{ secrets.GITEA_DO_SPACES_SECRET_KEY }}
+          GITEA_BACKUP_SSH_KEY: ${{ secrets.GITEA_BACKUP_SSH_KEY }}
+          RESTIC_PASSWORD: ${{ secrets.GITEA_RESTIC_PASSWORD }}
+        run: ./ci/github_actions/backup_gitea.sh maintenance

ci/github_actions/backup_gitea.sh

@@ -0,0 +1,296 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+BACKUP_MODE=${1:-backup}
+GITEA_BACKUP_SSH_HOST=${GITEA_BACKUP_SSH_HOST:-gitea-backup-source}
+GITEA_HTTP_URL=${GITEA_HTTP_URL:-http://zog:3000/admin/eagle0.git}
+GITEA_VERSION_URL=${GITEA_VERSION_URL:-http://zog:3000/api/v1/version}
+GITEA_REMOTE_ROOT=${GITEA_REMOTE_ROOT:-/volume2/docker/gitea}
+GITEA_BACKUP_BUCKET=${GITEA_BACKUP_BUCKET:-eagle0-gitea-backups}
+GITEA_BACKUP_PREFIX=${GITEA_BACKUP_PREFIX:-restic}
+RUN_ID=${GITHUB_RUN_ID:-manual}
+GITEA_DATA_DIRECTORIES=(
+    actions_artifacts
+    actions_log
+    attachments
+    avatars
+    conf
+    home
+    packages
+    repo-avatars
+)
+
+require_environment_variable() {
+    local variable_name=$1
+    if [ -z "${!variable_name:-}" ]; then
+        echo "ERROR: Required environment variable '$variable_name' is not set."
+        exit 1
+    fi
+}
+
+require_environment_variable RUNNER_TEMP
+require_environment_variable DO_SPACES_ENDPOINT
+require_environment_variable DO_SPACES_ACCESS_KEY
+require_environment_variable DO_SPACES_SECRET_KEY
+require_environment_variable GITEA_BACKUP_SSH_KEY
+require_environment_variable RESTIC_PASSWORD
+
+case "$RUN_ID" in
+    *[!A-Za-z0-9._-]*)
+        echo "ERROR: Unsafe run identifier '$RUN_ID'."
+        exit 1
+        ;;
+esac
+
+STAGING_ROOT="$RUNNER_TEMP/eagle0-gitea-backup"
+VERIFY_ROOT="$RUNNER_TEMP/eagle0-gitea-restore-verification"
+REMOTE_SNAPSHOT_DIR="$GITEA_REMOTE_ROOT/backup-staging"
+REMOTE_DATABASE="$GITEA_REMOTE_ROOT/data/gitea/gitea.db"
+REMOTE_DATABASE_SNAPSHOT="$REMOTE_SNAPSHOT_DIR/gitea-$RUN_ID.db"
+RESTIC_REPOSITORY="s3:${DO_SPACES_ENDPOINT%/}/$GITEA_BACKUP_BUCKET/$GITEA_BACKUP_PREFIX"
+SSH_CONFIG="$RUNNER_TEMP/eagle0-gitea-backup-ssh-config"
+SSH_PRIVATE_KEY="$RUNNER_TEMP/eagle0-gitea-backup-ssh-key"
+
+export AWS_ACCESS_KEY_ID="$DO_SPACES_ACCESS_KEY"
+export AWS_SECRET_ACCESS_KEY="$DO_SPACES_SECRET_KEY"
+export AWS_DEFAULT_REGION=us-east-1
+export RESTIC_REPOSITORY
+export RESTIC_PASSWORD
+export RESTIC_CACHE_DIR="$RUNNER_TEMP/restic-cache"
+export HOMEBREW_NO_AUTO_UPDATE=1
+export HOMEBREW_NO_INSTALL_CLEANUP=1
+
+COMMON_PATHS=(/opt/homebrew/bin /usr/local/bin)
+for common_path in "${COMMON_PATHS[@]}"; do
+    if [ -d "$common_path" ]; then
+        export PATH="$common_path:$PATH"
+    fi
+done
+
+ensure_tool() {
+    local command_name=$1
+    local brew_package=$2
+
+    if command -v "$command_name" >/dev/null 2>&1; then
+        return
+    fi
+
+    if ! command -v brew >/dev/null 2>&1; then
+        echo "ERROR: '$command_name' is unavailable and Homebrew is not installed."
+        exit 1
+    fi
+
+    brew install "$brew_package"
+}
+
+safe_reset_directory() {
+    local directory=$1
+
+    case "$directory" in
+        "$RUNNER_TEMP"/eagle0-gitea-*)
+            rm -rf "$directory"
+            mkdir -p "$directory"
+            ;;
+        *)
+            echo "ERROR: Refusing to reset unsafe directory '$directory'."
+            exit 1
+            ;;
+    esac
+}
+
+remove_remote_database_snapshot() {
+    ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" \
+        "rm -f '$REMOTE_DATABASE_SNAPSHOT'" >/dev/null 2>&1 || true
+}
+
+cleanup() {
+    remove_remote_database_snapshot
+
+    case "$STAGING_ROOT" in
+        "$RUNNER_TEMP"/eagle0-gitea-*)
+            rm -rf "$STAGING_ROOT"
+            ;;
+    esac
+
+    case "$VERIFY_ROOT" in
+        "$RUNNER_TEMP"/eagle0-gitea-*)
+            rm -rf "$VERIFY_ROOT"
+            ;;
+    esac
+
+    rm -f "$SSH_CONFIG" "$SSH_PRIVATE_KEY"
+}
+
+trap cleanup EXIT
+
+umask 077
+printf '%s\n' "$GITEA_BACKUP_SSH_KEY" > "$SSH_PRIVATE_KEY"
+printf '%s\n' \
+    "Host $GITEA_BACKUP_SSH_HOST" \
+    "    HostName zog" \
+    "    User \"Dan Crosby\"" \
+    "    IdentityFile $SSH_PRIVATE_KEY" \
+    "    IdentitiesOnly yes" \
+    "    HostKeyAlias zog" \
+    > "$SSH_CONFIG"
+
+ensure_tool aws awscli
+ensure_tool restic restic
+ensure_tool sqlite3 sqlite
+
+require_backup_bucket() {
+    if ! aws s3api head-bucket \
+        --bucket "$GITEA_BACKUP_BUCKET" \
+        --endpoint-url "$DO_SPACES_ENDPOINT" >/dev/null 2>&1; then
+        echo "ERROR: DigitalOcean Space '$GITEA_BACKUP_BUCKET' is unavailable."
+        exit 1
+    fi
+}
+
+initialize_repository_if_needed() {
+    if restic cat config >/dev/null 2>&1; then
+        return
+    fi
+
+    echo "Initializing encrypted Restic repository."
+    restic init
+}
+
+require_initialized_repository() {
+    if restic cat config >/dev/null 2>&1; then
+        return
+    fi
+
+    echo "ERROR: The encrypted Restic repository is not initialized."
+    exit 1
+}
+
+prepare_staging_data() {
+    safe_reset_directory "$STAGING_ROOT"
+    mkdir -p "$STAGING_ROOT/gitea"
+    mkdir -p "$STAGING_ROOT/repositories/admin"
+
+    ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" true
+    ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" \
+        "mkdir -p '$REMOTE_SNAPSHOT_DIR'"
+    ssh -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 "$GITEA_BACKUP_SSH_HOST" \
+        "sqlite3 '$REMOTE_DATABASE' \".backup '$REMOTE_DATABASE_SNAPSHOT'\""
+
+    scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -pr \
+        "$GITEA_BACKUP_SSH_HOST:$GITEA_REMOTE_ROOT/config" \
+        "$STAGING_ROOT/"
+
+    for directory in "${GITEA_DATA_DIRECTORIES[@]}"; do
+        scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -pr \
+            "$GITEA_BACKUP_SSH_HOST:$GITEA_REMOTE_ROOT/data/gitea/$directory" \
+            "$STAGING_ROOT/gitea/"
+    done
+
+    scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -p \
+        "$GITEA_BACKUP_SSH_HOST:$REMOTE_DATABASE_SNAPSHOT" \
+        "$STAGING_ROOT/gitea/gitea.db"
+
+    remove_remote_database_snapshot
+
+    scp -O -F "$SSH_CONFIG" -o BatchMode=yes -o ConnectTimeout=15 -pr \
+        "$GITEA_BACKUP_SSH_HOST:$GITEA_REMOTE_ROOT/data/lfs" \
+        "$STAGING_ROOT/"
+
+    git clone --mirror \
+        "$GITEA_HTTP_URL" \
+        "$STAGING_ROOT/repositories/admin/eagle0.git"
+
+    local database_check
+    database_check=$(sqlite3 "$STAGING_ROOT/gitea/gitea.db" "PRAGMA integrity_check;")
+    if [ "$database_check" != "ok" ]; then
+        echo "ERROR: Gitea database snapshot failed SQLite integrity checking."
+        exit 1
+    fi
+
+    git --git-dir="$STAGING_ROOT/repositories/admin/eagle0.git" fsck --full
+
+    local gitea_version
+    gitea_version=$(curl --fail --silent "$GITEA_VERSION_URL")
+    local main_commit
+    main_commit=$(git --git-dir="$STAGING_ROOT/repositories/admin/eagle0.git" rev-parse refs/heads/main)
+    local captured_at
+    captured_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+    printf '%s\n' \
+        "captured_at=$captured_at" \
+        "gitea_version=$gitea_version" \
+        "main_commit=$main_commit" \
+        "source_host=$GITEA_BACKUP_SSH_HOST" \
+        "source_root=$GITEA_REMOTE_ROOT" \
+        > "$STAGING_ROOT/manifest.txt"
+}
+
+verify_restored_snapshot() {
+    safe_reset_directory "$VERIFY_ROOT"
+    restic restore latest --host zog --tag gitea --target "$VERIFY_ROOT"
+
+    local restored_root="$VERIFY_ROOT/${STAGING_ROOT#/}"
+    local database_check
+    database_check=$(sqlite3 "$restored_root/gitea/gitea.db" "PRAGMA integrity_check;")
+    if [ "$database_check" != "ok" ]; then
+        echo "ERROR: Restored Gitea database failed SQLite integrity checking."
+        exit 1
+    fi
+
+    git --git-dir="$restored_root/repositories/admin/eagle0.git" fsck --full
+    test -s "$restored_root/manifest.txt"
+    test -d "$restored_root/lfs"
+    echo "Full restore verification completed successfully."
+}
+
+run_backup() {
+    require_backup_bucket
+    initialize_repository_if_needed
+    prepare_staging_data
+
+    restic backup "$STAGING_ROOT" --host zog --tag gitea
+    restic forget \
+        --host zog \
+        --tag gitea \
+        --group-by host,tags \
+        --keep-daily 14 \
+        --keep-weekly 8 \
+        --keep-monthly 12 \
+        --keep-yearly 3
+
+    if [ "${FULL_RESTORE_VERIFY:-false}" = "true" ]; then
+        restic check --read-data
+        verify_restored_snapshot
+    else
+        restic check --read-data-subset=5%
+    fi
+
+    restic snapshots --host zog --tag gitea
+}
+
+run_maintenance() {
+    require_initialized_repository
+    restic forget \
+        --host zog \
+        --tag gitea \
+        --keep-daily 14 \
+        --keep-weekly 8 \
+        --keep-monthly 12 \
+        --keep-yearly 3 \
+        --prune
+    restic check --read-data-subset=10%
+}
+
+case "$BACKUP_MODE" in
+    backup)
+        run_backup
+        ;;
+    maintenance)
+        run_maintenance
+        ;;
+    *)
+        echo "Usage: $0 [backup|maintenance]"
+        exit 2
+        ;;
+esac

docs/operations/gitea-backups.md

@@ -0,0 +1,87 @@
+# Gitea offsite backups
+
+Eagle0's Gitea mirror keeps Git and Git LFS data on `zog` for fast local
+access. The `Gitea Offsite Backup` GitHub Actions workflow copies the
+recoverable state to an encrypted Restic repository in the private
+`eagle0-gitea-backups` DigitalOcean Space.
+
+## Backup contents
+
+Each snapshot contains:
+
+- a transactionally consistent SQLite backup of the Gitea database;
+- Gitea configuration and persistent application data;
+- every locally stored Git LFS object;
+- a fresh mirror clone of `admin/eagle0`;
+- the Gitea version and captured `main` commit in `manifest.txt`.
+
+Regenerable indexes, sessions, queues, repository archives, logs, and the
+OAuth2 JWT signing key are excluded. Gitea creates a new JWT signing key when
+none exists, which invalidates pre-restoration OAuth2 tokens. Container-managed
+SSH host keys are not readable by the backup account and must also be
+regenerated after a complete NAS loss.
+
+## Schedule and retention
+
+The backup runs daily at 10:23 UTC on a `halfdan` self-hosted runner. Each
+run validates the SQLite snapshot, runs `git fsck`, uploads an encrypted
+snapshot, and reads a rotating 5% sample of the Restic repository.
+
+Weekly maintenance runs Sunday at 11:53 UTC. It retains 14 daily, 8 weekly,
+12 monthly, and 3 yearly snapshots, prunes unreferenced Restic packs, and
+reads a rotating 10% sample.
+
+Use the workflow's `full_restore_verify` option after storage or credential
+changes. It downloads the latest snapshot, validates the restored SQLite
+database, and runs `git fsck` against the restored repository.
+
+## Credentials
+
+GitHub Actions stores:
+
+- `GITEA_DO_SPACES_ACCESS_KEY`
+- `GITEA_DO_SPACES_SECRET_KEY`
+- `DO_SPACES_ENDPOINT`
+- `GITEA_BACKUP_SSH_KEY`
+- `GITEA_RESTIC_PASSWORD`
+
+`GITEA_BACKUP_SSH_KEY` is a dedicated key accepted by the NAS account used for
+backup reads. It is not a copy of a developer's personal SSH key.
+
+The Restic recovery password also has an offline recovery copy in the
+workstation's macOS Keychain under service `eagle0-gitea-restic`. Never put
+that password in the repository, workflow logs, or a support ticket.
+
+## Restore
+
+Install `restic`, then set the DigitalOcean credentials, Restic repository,
+and Restic password in the shell performing the restore. Restore into an
+empty directory:
+
+```bash
+restic snapshots --host zog --tag gitea
+restic restore latest --host zog --tag gitea --target /path/to/restore
+```
+
+The snapshot preserves the absolute staging path below the restore target.
+Locate `manifest.txt`; its containing directory is the restored backup root.
+
+Before changing Gitea:
+
+```bash
+sqlite3 /path/to/restored/root/gitea/gitea.db "PRAGMA integrity_check;"
+git --git-dir=/path/to/restored/root/repositories/admin/eagle0.git fsck --full
+```
+
+To recover a complete installation:
+
+1. Recreate the same Gitea version shown in `manifest.txt`.
+2. Stop Gitea.
+3. Restore `config`, `gitea`, `lfs`, and the bare repository into the
+   matching paths under `/volume2/docker/gitea`.
+4. Preserve the restored file ownership expected by the container.
+5. Allow Gitea to regenerate indexes, sessions, queues, archives, and SSH
+   host keys.
+6. Start Gitea and verify a historical LFS checkout before enabling writes.
+
+Do not overwrite a running installation during a restore.