eagle0/scripts/deploy-blue-green.sh

#!/bin/bash
#
# Blue-Green Deployment Script for Eagle Server
#
# This script performs a zero-downtime deployment by:
# 1. Starting the new version on a staging port (green)
# 2. Waiting for it to become healthy
# 3. Running warmup traffic to pre-heat the JIT
# 4. Stopping the old version (blue) - which flushes state to storage
# 5. Switching nginx to route traffic to green
# 6. Users reconnect, triggering lazy game loading from fresh storage
# 7. Cleaning up
#
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
#
# Example:
#   ./deploy-blue-green.sh latest
#   ./deploy-blue-green.sh sha-abc123
#

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_DIR="${APP_DIR:-/opt/eagle0}"
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# GitHub raw URL base for fetching config files
GITHUB_RAW_BASE="https://raw.githubusercontent.com/nolen777/eagle0/main"

# Sync config files from GitHub to ensure we have the latest versions
sync_config_files() {
    log_info "Syncing config files from GitHub..."

    # Backup and update docker-compose.prod.yml
    if curl -fsSL "${GITHUB_RAW_BASE}/docker-compose.prod.yml" -o "${COMPOSE_FILE}.new"; then
        cp "${COMPOSE_FILE}" "${COMPOSE_FILE}.bak"
        mv "${COMPOSE_FILE}.new" "${COMPOSE_FILE}"
        log_info "Updated docker-compose.prod.yml"
    else
        log_warn "Failed to fetch docker-compose.prod.yml, using existing"
    fi

    # Backup and update nginx.conf, preserving current active instance
    if curl -fsSL "${GITHUB_RAW_BASE}/nginx/nginx.conf" -o "${NGINX_CONF}.new"; then
        # Preserve the current eagle_backend setting (which instance is active)
        local current_backend
        current_backend=$(grep -o 'default "eagle-[a-z]*:40032"' "${NGINX_CONF}" | head -1 || echo 'default "eagle-blue:40032"')

        cp "${NGINX_CONF}" "${NGINX_CONF}.bak"
        # Update the new config with the current active instance
        sed -i "s/default \"eagle-blue:40032\"/${current_backend}/g" "${NGINX_CONF}.new"
        mv "${NGINX_CONF}.new" "${NGINX_CONF}"
        log_info "Updated nginx.conf (preserved active: ${current_backend})"
    else
        log_warn "Failed to fetch nginx.conf, using existing"
    fi
}

# Determine which instance is currently active
get_active_instance() {
    if grep -q "eagle-blue:40032" "${NGINX_CONF}"; then
        echo "blue"
    elif grep -q "eagle-green:40032" "${NGINX_CONF}"; then
        echo "green"
    else
        log_error "Cannot determine active instance from nginx config"
        exit 1
    fi
}

# Pull image with retry using crane (handles OCI/Docker digest mismatch)
pull_with_retry() {
    local image=$1
    local max_attempts=${2:-3}
    local attempt=1

    # Use crane if available (handles OCI format correctly)
    if [ -x "${APP_DIR}/crane" ]; then
        while [ $attempt -le $max_attempts ]; do
            log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
            if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
                rm -f /tmp/image.tar
                log_info "Image pulled and loaded successfully"
                return 0
            fi
            rm -f /tmp/image.tar
            log_warn "Pull failed, retrying in 5 seconds..."
            sleep 5
            attempt=$((attempt + 1))
        done
    else
        # Fallback to docker pull if crane not available
        log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
        while [ $attempt -le $max_attempts ]; do
            log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
            if docker pull "${image}"; then
                log_info "Image pulled successfully"
                return 0
            fi
            log_warn "Pull failed, retrying in 5 seconds..."
            sleep 5
            attempt=$((attempt + 1))
        done
    fi
    log_error "Failed to pull image after ${max_attempts} attempts"
    return 1
}

# Wait for a container to be healthy
wait_for_healthy() {
    local container=$1
    local max_attempts=${2:-60}
    local attempt=1

    log_info "Waiting for ${container} to become healthy..."
    while [ $attempt -le $max_attempts ]; do
        health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
        if [ "$health" = "healthy" ]; then
            log_info "${container} is healthy"
            return 0
        fi
        echo -n "."
        sleep 2
        attempt=$((attempt + 1))
    done
    echo ""
    log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
    return 1
}

# Main deployment logic
main() {
    local new_tag="${1:-latest}"
    local registry="registry.digitalocean.com/eagle0/eagle-server"
    local new_image="${registry}:${new_tag}"

    log_info "Starting blue-green deployment"
    log_info "New image: ${new_image}"

    cd "${APP_DIR}"

    # Sync config files from GitHub before deployment
    sync_config_files

    # Determine current active instance
    local active=$(get_active_instance)
    local staging
    if [ "$active" = "blue" ]; then
        staging="green"
    else
        staging="blue"
    fi

    log_info "Active instance: eagle-${active}"
    log_info "Staging instance: eagle-${staging}"

    # Pull the new image (with retry for intermittent registry issues)
    if ! pull_with_retry "${new_image}" 3; then
        log_error "Failed to pull new image, aborting deployment"
        exit 1
    fi

    # Start staging instance with new image (and its jfr-sidecar)
    log_info "Starting eagle-${staging} with new image..."
    if [ "$staging" = "green" ]; then
        EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
    else
        EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
    fi

    # Wait for staging to be healthy
    if ! wait_for_healthy "eagle-${staging}" 90; then
        log_error "Staging instance failed health check, aborting deployment"
        docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
        docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${staging}"
        exit 1
    fi

    # Run warmup/smoke test
    local staging_port
    if [ "$staging" = "green" ]; then
        staging_port=40034
    else
        staging_port=40032
    fi

    log_info "Running warmup against eagle-${staging}..."
    if [ -x "${WARMUP_SCRIPT}" ]; then
        if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
            log_error "Warmup/smoke test failed, aborting deployment"
            docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
            docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${staging}"
            exit 1
        fi
    else
        log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
        log_warn "JIT will be cold on first requests"
    fi

    # Backup games.e0es BEFORE stopping the active instance
    # Critical: The old server may wipe games.e0es on shutdown if it doesn't have the merge fix
    local backup_timestamp
    backup_timestamp=$(date +%Y%m%d_%H%M%S)
    log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
    if command -v s3cmd &> /dev/null; then
        if s3cmd put "${APP_DIR}/saves/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
            log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
        else
            log_warn "S3 backup failed (s3cmd put), continuing deployment"
        fi
    else
        log_warn "s3cmd not found, skipping S3 backup"
    fi

    # Stop the active instance (this flushes state to storage)
    log_info "Stopping eagle-${active} (flushing state to storage)..."
    docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"

    # Backup games.e0es to S3 before switching traffic
    # This creates a point-in-time backup we can recover from if something goes wrong
    local backup_timestamp
    backup_timestamp=$(date +%Y%m%d_%H%M%S)
    log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
    if command -v s3cmd &> /dev/null; then
        if s3cmd put "${APP_DIR}/saves/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
            log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
        else
            log_warn "S3 backup failed (s3cmd put), continuing deployment"
        fi
    else
        log_warn "s3cmd not found, skipping S3 backup"
    fi

    # Note: No need to explicitly reload games - Eagle uses lazy loading.
    # Games are loaded on-demand when users reconnect after nginx switches traffic.
    # This ensures games are always loaded from the freshest storage state.

    # Switch nginx upstream
    log_info "Switching nginx upstream to eagle-${staging}..."
    if [ "$staging" = "green" ]; then
        sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
    else
        sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
    fi

    # Reload nginx
    log_info "Reloading nginx..."
    docker compose -f "${COMPOSE_FILE}" exec nginx nginx -s reload

    # Clean up old instance and its jfr-sidecar
    log_info "Removing old eagle-${active} container..."
    docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}"
    if [ "$active" = "green" ]; then
        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true
    else
        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true
    fi

    # Update .env with new EAGLE_ADDR and JFR_SIDECAR_ADDR for admin service
    local env_file="${APP_DIR}/.env"
    if [ "$staging" = "green" ]; then
        log_info "Updating .env for green instance..."
        sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
        echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
        echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
    else
        log_info "Updating .env for blue instance..."
        sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
        echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
        echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
    fi

    # Restart admin to pick up new EAGLE_ADDR and JFR_SIDECAR_ADDR
    log_info "Restarting admin service..."
    docker compose -f "${COMPOSE_FILE}" up -d admin

    log_info "Deployment complete!"
    log_info "Active instance: eagle-${staging}"
}

# Check for required tools
check_requirements() {
    if ! command -v docker &> /dev/null; then
        log_error "docker is required but not installed"
        exit 1
    fi

    if ! command -v sed &> /dev/null; then
        log_error "sed is required but not installed"
        exit 1
    fi

    if [ ! -f "${NGINX_CONF}" ]; then
        log_error "nginx config not found at ${NGINX_CONF}"
        exit 1
    fi

    if [ ! -f "${COMPOSE_FILE}" ]; then
        log_error "docker-compose file not found at ${COMPOSE_FILE}"
        exit 1
    fi
}

# Run
check_requirements
main "$@"