eagle0/scripts/deploy-blue-green.sh

#!/bin/bash
#
# Blue-Green Deployment Script for Eagle Server
#
# This script performs a zero-downtime deployment by:
# 1. Starting the new version on a staging port (green)
# 2. Waiting for it to become healthy
# 3. Running warmup traffic to pre-heat the JIT
# 4. Stopping the old version (blue) - which flushes state to storage
# 5. Switching nginx to route traffic to green
# 6. Users reconnect, triggering lazy game loading from fresh storage
# 7. Cleaning up
#
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
#
# Example:
#   ./deploy-blue-green.sh latest
#   ./deploy-blue-green.sh sha-abc123
#

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_DIR="${APP_DIR:-/opt/eagle0}"
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Determine which instance is currently active
get_active_instance() {
    if grep -q "eagle-blue:40032" "${NGINX_CONF}"; then
        echo "blue"
    elif grep -q "eagle-green:40032" "${NGINX_CONF}"; then
        echo "green"
    else
        log_error "Cannot determine active instance from nginx config"
        exit 1
    fi
}

# Pull image with retry using crane (handles OCI/Docker digest mismatch)
pull_with_retry() {
    local image=$1
    local max_attempts=${2:-3}
    local attempt=1

    # Skip pull if image already exists locally (e.g., CI already pulled it)
    if docker image inspect "${image}" &>/dev/null; then
        log_info "Image ${image} already exists locally, skipping pull"
        return 0
    fi

    # Use crane if available (handles OCI format correctly)
    if [ -x "${APP_DIR}/crane" ]; then
        while [ $attempt -le $max_attempts ]; do
            log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
            if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
                rm -f /tmp/image.tar
                log_info "Image pulled and loaded successfully"
                return 0
            fi
            rm -f /tmp/image.tar
            log_warn "Pull failed, retrying in 5 seconds..."
            sleep 5
            attempt=$((attempt + 1))
        done
    else
        # Fallback to docker pull if crane not available
        log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
        while [ $attempt -le $max_attempts ]; do
            log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
            if docker pull "${image}"; then
                log_info "Image pulled successfully"
                return 0
            fi
            log_warn "Pull failed, retrying in 5 seconds..."
            sleep 5
            attempt=$((attempt + 1))
        done
    fi
    log_error "Failed to pull image after ${max_attempts} attempts"
    return 1
}

# Wait for a container to be healthy
wait_for_healthy() {
    local container=$1
    local max_attempts=${2:-60}
    local attempt=1

    log_info "Waiting for ${container} to become healthy..."
    while [ $attempt -le $max_attempts ]; do
        health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
        if [ "$health" = "healthy" ]; then
            log_info "${container} is healthy"
            return 0
        fi
        echo -n "."
        sleep 2
        attempt=$((attempt + 1))
    done
    echo ""
    log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
    return 1
}

# Main deployment logic
main() {
    local new_tag="${1:-latest}"
    local registry="registry.digitalocean.com/eagle0/eagle-server"
    local new_image="${registry}:${new_tag}"

    log_info "Starting blue-green deployment"
    log_info "New image: ${new_image}"

    cd "${APP_DIR}"

    # Determine current active instance
    local active=$(get_active_instance)
    local staging
    if [ "$active" = "blue" ]; then
        staging="green"
    else
        staging="blue"
    fi

    log_info "Active instance: eagle-${active}"
    log_info "Staging instance: eagle-${staging}"

    # Pull the new image (with retry for intermittent registry issues)
    if ! pull_with_retry "${new_image}" 3; then
        log_error "Failed to pull new image, aborting deployment"
        exit 1
    fi

    # Start staging instance with new image (and its jfr-sidecar)
    log_info "Starting eagle-${staging} with new image..."
    if [ "$staging" = "green" ]; then
        EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
    else
        EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
    fi

    # Wait for staging to be healthy
    if ! wait_for_healthy "eagle-${staging}" 90; then
        log_error "Staging instance failed health check, aborting deployment"
        docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
        docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${staging}"
        exit 1
    fi

    # Run warmup/smoke test
    local staging_port
    if [ "$staging" = "green" ]; then
        staging_port=40034
    else
        staging_port=40032
    fi

    log_info "Running warmup against eagle-${staging}..."
    if [ -x "${WARMUP_SCRIPT}" ]; then
        if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
            log_error "Warmup/smoke test failed, aborting deployment"
            docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
            docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${staging}"
            exit 1
        fi
    else
        log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
        log_warn "JIT will be cold on first requests"
    fi

    # Backup games.e0es BEFORE stopping the active instance
    # Critical: The old server may wipe games.e0es on shutdown if it doesn't have the merge fix
    local backup_timestamp
    backup_timestamp=$(date +%Y%m%d_%H%M%S)
    log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
    if command -v s3cmd &> /dev/null; then
        if s3cmd put "${APP_DIR}/saves/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
            log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
        else
            log_warn "S3 backup failed (s3cmd put), continuing deployment"
        fi
    else
        log_warn "s3cmd not found, skipping S3 backup"
    fi

    # Stop the active instance (flushes state to storage)
    # We must do this BEFORE switching so staging loads fresh data
    log_info "Stopping eagle-${active} (flushing state to storage)..."
    docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"

    # Prepare nginx config and .env BEFORE restarting anything
    log_info "Switching nginx upstream to eagle-${staging}..."
    if [ "$staging" = "green" ]; then
        sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
    else
        sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
    fi

    local env_file="${APP_DIR}/.env"
    if [ "$staging" = "green" ]; then
        log_info "Updating .env for green instance..."
        sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
        echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
        echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
    else
        log_info "Updating .env for blue instance..."
        sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
        echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
        echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
    fi

    # Validate nginx config before reload
    log_info "Validating nginx configuration..."
    if ! docker exec nginx nginx -t 2>&1; then
        log_error "nginx config validation failed, aborting!"
        log_error "Rolling back nginx.conf..."
        mv "${NGINX_CONF}.bak" "${NGINX_CONF}"
        exit 1
    fi

    # Reload nginx (use reload for faster switch, restart if reload fails)
    log_info "Reloading nginx..."
    if ! docker exec nginx nginx -s reload 2>/dev/null; then
        log_warn "Reload failed, restarting nginx..."
        docker compose -f "${COMPOSE_FILE}" restart nginx
    fi

    # Restart admin to pick up new .env
    log_info "Restarting admin service..."
    docker compose -f "${COMPOSE_FILE}" up -d --force-recreate admin

    # Clean up old instance and its jfr-sidecar (in background, not critical path)
    log_info "Cleaning up old eagle-${active} container..."
    docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" &
    if [ "$active" = "green" ]; then
        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null &
    else
        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null &
    fi
    wait

    log_info "Deployment complete!"
    log_info "Active instance: eagle-${staging}"
}

# Ensure s3cmd is installed and configured for S3 backups
ensure_s3cmd() {
    # Install s3cmd if not present
    if ! command -v s3cmd &> /dev/null; then
        log_info "Installing s3cmd for S3 backups..."
        if command -v pip3 &> /dev/null; then
            pip3 install --quiet s3cmd
        elif command -v pip &> /dev/null; then
            pip install --quiet s3cmd
        elif command -v apt-get &> /dev/null; then
            sudo apt-get update -qq && sudo apt-get install -y -qq s3cmd
        else
            log_warn "Cannot install s3cmd (no pip or apt-get), S3 backups will be skipped"
            return 1
        fi

        if ! command -v s3cmd &> /dev/null; then
            log_warn "s3cmd installation failed, S3 backups will be skipped"
            return 1
        fi
        log_info "s3cmd installed successfully"
    fi

    # Configure s3cmd for DigitalOcean Spaces if not already configured
    local s3cfg="${HOME}/.s3cfg"
    if [ ! -f "${s3cfg}" ]; then
        # Load credentials from .env
        local env_file="${APP_DIR}/.env"
        if [ -f "${env_file}" ]; then
            # shellcheck source=/dev/null
            source "${env_file}"
        fi

        if [ -z "${DO_SPACES_ACCESS_KEY:-}" ] || [ -z "${DO_SPACES_SECRET_KEY:-}" ]; then
            log_warn "DO_SPACES_ACCESS_KEY or DO_SPACES_SECRET_KEY not set, S3 backups will be skipped"
            return 1
        fi

        log_info "Configuring s3cmd for DigitalOcean Spaces..."
        cat > "${s3cfg}" << EOF
[default]
access_key = ${DO_SPACES_ACCESS_KEY}
secret_key = ${DO_SPACES_SECRET_KEY}
host_base = sfo3.digitaloceanspaces.com
host_bucket = %(bucket)s.sfo3.digitaloceanspaces.com
use_https = True
EOF
        log_info "s3cmd configured successfully"
    fi

    return 0
}

# Check for required tools
check_requirements() {
    if ! command -v docker &> /dev/null; then
        log_error "docker is required but not installed"
        exit 1
    fi

    if ! command -v sed &> /dev/null; then
        log_error "sed is required but not installed"
        exit 1
    fi

    if [ ! -f "${NGINX_CONF}" ]; then
        log_error "nginx config not found at ${NGINX_CONF}"
        exit 1
    fi

    if [ ! -f "${COMPOSE_FILE}" ]; then
        log_error "docker-compose file not found at ${COMPOSE_FILE}"
        exit 1
    fi

    # Ensure s3cmd is available for backups (non-fatal if it fails)
    ensure_s3cmd || true
}

# Run
check_requirements
main "$@"