eagle0/scripts/deploy-blue-green.sh

#!/bin/bash
#
# Blue-Green Deployment Script for Eagle Server
#
# This script performs a zero-downtime deployment with state consistency:
# 1. Create .deployment_in_progress marker (signals deployment started)
# 2. Start the staging instance (green) with new image
# 3. Run warmup/smoke tests against staging (warms JIT)
# 4. Switch nginx to staging (zero downtime - users immediately route to staging)
# 5. Stop the active instance (blue) - blocks until flush completes
# 6. Create .flush_complete marker (signals disk state is fresh)
#
# The flush marker coordination ensures green never serves stale game data:
# - When users reconnect to green and trigger lazy-load, the code checks for markers
# - If .deployment_in_progress exists, lazy-load WAITS for .flush_complete
# - Once blue's flush completes and marker is created, lazy-load proceeds with fresh data
#
# Key insight: nginx switches to green BEFORE blue stops, achieving zero downtime.
# Users who trigger lazy-load during blue's shutdown will wait for the flush marker.
#
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
#
# Example:
#   ./deploy-blue-green.sh latest
#   ./deploy-blue-green.sh sha-abc123
#

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_DIR="${APP_DIR:-/opt/eagle0}"
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"
SAVES_DIR="${APP_DIR}/saves"
FLUSH_MARKER="${SAVES_DIR}/.flush_complete"
DEPLOYMENT_IN_PROGRESS="${SAVES_DIR}/.deployment_in_progress"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Determine which instance is currently running (not from nginx config)
get_running_instance() {
    local blue_running green_running
    blue_running=$(docker inspect --format='{{.State.Running}}' eagle-blue 2>/dev/null || echo "false")
    green_running=$(docker inspect --format='{{.State.Running}}' eagle-green 2>/dev/null || echo "false")

    if [ "$blue_running" = "true" ] && [ "$green_running" = "true" ]; then
        # Both running - use nginx config to determine primary
        if grep -q "server eagle-blue:40032;" "${NGINX_CONF}" | head -1 | grep -qv backup; then
            echo "blue"
        else
            echo "green"
        fi
    elif [ "$blue_running" = "true" ]; then
        echo "blue"
    elif [ "$green_running" = "true" ]; then
        echo "green"
    else
        # Neither running - default to blue (first deploy or recovery)
        echo "none"
    fi
}

# Pull image with retry using crane (handles OCI/Docker digest mismatch)
pull_with_retry() {
    local image=$1
    local max_attempts=${2:-3}
    local attempt=1

    # Skip pull if image already exists locally (e.g., CI already pulled it)
    if docker image inspect "${image}" &>/dev/null; then
        log_info "Image ${image} already exists locally, skipping pull"
        return 0
    fi

    # Use crane if available (handles OCI format correctly)
    if [ -x "${APP_DIR}/crane" ]; then
        while [ $attempt -le $max_attempts ]; do
            log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
            if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
                rm -f /tmp/image.tar
                log_info "Image pulled and loaded successfully"
                return 0
            fi
            rm -f /tmp/image.tar
            log_warn "Pull failed, retrying in 5 seconds..."
            sleep 5
            attempt=$((attempt + 1))
        done
    else
        # Fallback to docker pull if crane not available
        log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
        while [ $attempt -le $max_attempts ]; do
            log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
            if docker pull "${image}"; then
                log_info "Image pulled successfully"
                return 0
            fi
            log_warn "Pull failed, retrying in 5 seconds..."
            sleep 5
            attempt=$((attempt + 1))
        done
    fi
    log_error "Failed to pull image after ${max_attempts} attempts"
    return 1
}

# Wait for a container to be healthy
wait_for_healthy() {
    local container=$1
    local max_attempts=${2:-60}
    local attempt=1

    log_info "Waiting for ${container} to become healthy..."
    while [ $attempt -le $max_attempts ]; do
        health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
        if [ "$health" = "healthy" ]; then
            log_info "${container} is healthy"
            return 0
        fi
        echo -n "."
        sleep 2
        attempt=$((attempt + 1))
    done
    echo ""
    log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
    return 1
}

# Main deployment logic
main() {
    local new_tag="${1:-latest}"
    local registry="registry.digitalocean.com/eagle0/eagle-server"
    local new_image="${registry}:${new_tag}"

    # Generate deployment ID for log correlation with server logs
    local deploy_id
    deploy_id=$(date +%s)
    local deploy_start_time=$deploy_id

    log_info "========================================="
    log_info "Starting blue-green deployment"
    log_info "Deployment ID: ${deploy_id}"
    log_info "New image: ${new_image}"
    log_info "========================================="

    cd "${APP_DIR}"

    # Step 1: Signal deployment in progress
    log_info "[DEPLOY:${deploy_id}] Step 1: Signaling deployment in progress..."
    rm -f "${FLUSH_MARKER}"
    echo "${deploy_id}" > "${DEPLOYMENT_IN_PROGRESS}"
    log_info "[DEPLOY:${deploy_id}] Deployment marker created"

    # Determine current active instance
    local active=$(get_running_instance)
    local staging
    if [ "$active" = "blue" ] || [ "$active" = "none" ]; then
        staging="green"
        active="blue"  # Normalize "none" to "blue" for first deploy
    else
        staging="blue"
    fi

    log_info "Active instance: eagle-${active}"
    log_info "Staging instance: eagle-${staging}"

    # Pull the new image (with retry for intermittent registry issues)
    if ! pull_with_retry "${new_image}" 3; then
        log_error "Failed to pull new image, aborting deployment"
        # Clean up deployment markers on failure
        rm -f "${DEPLOYMENT_IN_PROGRESS}"
        touch "${FLUSH_MARKER}"
        exit 1
    fi

    # Step 2: Start staging instance with new image
    log_info "Step 2: Starting eagle-${staging} with new image..."
    if [ "$staging" = "green" ]; then
        EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
    else
        EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
    fi

    # Wait for staging to be healthy
    if ! wait_for_healthy "eagle-${staging}" 90; then
        log_error "Staging instance failed health check, aborting deployment"
        docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
        # Clean up deployment markers on failure
        rm -f "${DEPLOYMENT_IN_PROGRESS}"
        touch "${FLUSH_MARKER}"
        exit 1
    fi

    # Step 3: Run warmup/smoke test
    local staging_port
    if [ "$staging" = "green" ]; then
        staging_port=40034
    else
        staging_port=40032
    fi

    log_info "Step 3: Running warmup against eagle-${staging}..."
    if [ -x "${WARMUP_SCRIPT}" ]; then
        if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
            log_error "Warmup/smoke test failed, aborting deployment"
            docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
            # Clean up deployment markers on failure
            rm -f "${DEPLOYMENT_IN_PROGRESS}"
            touch "${FLUSH_MARKER}"
            exit 1
        fi
    else
        log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
        log_warn "JIT will be cold on first requests"
    fi

    # Backup games.e0es BEFORE stopping the active instance
    local backup_timestamp
    backup_timestamp=$(date +%Y%m%d_%H%M%S)
    log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
    if command -v s3cmd &> /dev/null; then
        if s3cmd put "${SAVES_DIR}/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
            log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
        else
            log_warn "S3 backup failed (s3cmd put), continuing deployment"
        fi
    else
        log_warn "s3cmd not found, skipping S3 backup"
    fi

    # Step 4: Switch nginx to staging BEFORE stopping active
    # This achieves zero downtime - users immediately route to staging.
    # Any lazy-loads will wait for the flush marker (created in step 6).
    local nginx_switch_start
    nginx_switch_start=$(date +%s)
    log_info "[DEPLOY:${deploy_id}] Step 4: Switching nginx to eagle-${staging}..."

    # Update nginx config (variable-based routing)
    if [ "$staging" = "green" ]; then
        sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
    else
        sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
    fi

    # Recreate nginx to pick up new config
    docker compose -f "${COMPOSE_FILE}" up -d --force-recreate nginx

    # Verify nginx picked up the correct config
    local nginx_backend
    nginx_backend=$(docker exec nginx grep -o 'eagle-[a-z]*:40032' /etc/nginx/nginx.conf | head -1 || echo "unknown")
    if [ "$nginx_backend" = "eagle-${staging}:40032" ]; then
        log_info "[DEPLOY:${deploy_id}] Verified: nginx routing to eagle-${staging}"
    else
        log_error "[DEPLOY:${deploy_id}] nginx config mismatch! Expected eagle-${staging}:40032, got ${nginx_backend}"
        exit 1
    fi
    log_info "[DEPLOY:${deploy_id}] Traffic switched to eagle-${staging} (lazy-loads will wait for flush marker)"

    # Step 5: Stop active instance (blocks until exit, ensuring flush completes)
    # Users may be lazy-loading on staging during this time - they'll wait for the marker.
    local flush_start
    flush_start=$(date +%s)
    log_info "[DEPLOY:${deploy_id}] Step 5: Stopping eagle-${active} (waiting for flush)..."
    docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"
    local flush_end
    flush_end=$(date +%s)
    local flush_duration=$((flush_end - flush_start))
    log_info "[DEPLOY:${deploy_id}] eagle-${active} stopped, flush completed in ${flush_duration}s"

    # Step 6: Create flush marker - signals that disk state is fresh
    # Any waiting lazy-loads on staging will now proceed with fresh data.
    log_info "[DEPLOY:${deploy_id}] Step 6: Creating flush marker..."
    echo "${deploy_id}" > "${FLUSH_MARKER}"
    rm -f "${DEPLOYMENT_IN_PROGRESS}"
    log_info "[DEPLOY:${deploy_id}] Flush marker created - waiting lazy-loads can now proceed"

    # Update .env for admin service
    local env_file="${APP_DIR}/.env"
    if [ "$staging" = "green" ]; then
        log_info "Updating .env for green instance..."
        sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
        echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
        echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
    else
        log_info "Updating .env for blue instance..."
        sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
        echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
        echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
    fi

    # Restart admin to pick up new .env
    log_info "Restarting admin service..."
    docker compose -f "${COMPOSE_FILE}" up -d --force-recreate admin

    # Clean up old instance
    log_info "Cleaning up old eagle-${active}..."
    docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" 2>/dev/null || true

    # Stop the old jfr-sidecar (it can't attach to removed container anyway)
    if [ "$active" = "green" ]; then
        docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar-green" 2>/dev/null || true
        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true
    else
        docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar" 2>/dev/null || true
        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true
    fi

    local deploy_end_time
    deploy_end_time=$(date +%s)
    local total_duration=$((deploy_end_time - deploy_start_time))
    local user_wait_window=$((flush_end - nginx_switch_start))

    log_info ""
    log_info "========================================="
    log_info "[DEPLOY:${deploy_id}] Deployment complete!"
    log_info "  Active instance: eagle-${staging}"
    log_info "  Total duration: ${total_duration}s"
    log_info "  Flush duration: ${flush_duration}s"
    log_info "  Max user wait window: ${user_wait_window}s"
    log_info "========================================="
}

# Ensure s3cmd is installed and configured for S3 backups
ensure_s3cmd() {
    # Install s3cmd if not present
    if ! command -v s3cmd &> /dev/null; then
        log_info "Installing s3cmd for S3 backups..."
        if command -v pip3 &> /dev/null; then
            pip3 install --quiet s3cmd
        elif command -v pip &> /dev/null; then
            pip install --quiet s3cmd
        elif command -v apt-get &> /dev/null; then
            sudo apt-get update -qq && sudo apt-get install -y -qq s3cmd
        else
            log_warn "Cannot install s3cmd (no pip or apt-get), S3 backups will be skipped"
            return 1
        fi

        if ! command -v s3cmd &> /dev/null; then
            log_warn "s3cmd installation failed, S3 backups will be skipped"
            return 1
        fi
        log_info "s3cmd installed successfully"
    fi

    # Configure s3cmd for DigitalOcean Spaces if not already configured
    local s3cfg="${HOME}/.s3cfg"
    if [ ! -f "${s3cfg}" ]; then
        # Load credentials from .env
        local env_file="${APP_DIR}/.env"
        if [ -f "${env_file}" ]; then
            # shellcheck source=/dev/null
            source "${env_file}"
        fi

        if [ -z "${DO_SPACES_ACCESS_KEY:-}" ] || [ -z "${DO_SPACES_SECRET_KEY:-}" ]; then
            log_warn "DO_SPACES_ACCESS_KEY or DO_SPACES_SECRET_KEY not set, S3 backups will be skipped"
            return 1
        fi

        log_info "Configuring s3cmd for DigitalOcean Spaces..."
        cat > "${s3cfg}" << EOF
[default]
access_key = ${DO_SPACES_ACCESS_KEY}
secret_key = ${DO_SPACES_SECRET_KEY}
host_base = sfo3.digitaloceanspaces.com
host_bucket = %(bucket)s.sfo3.digitaloceanspaces.com
use_https = True
EOF
        log_info "s3cmd configured successfully"
    fi

    return 0
}

# Check for required tools
check_requirements() {
    if ! command -v docker &> /dev/null; then
        log_error "docker is required but not installed"
        exit 1
    fi

    if ! command -v sed &> /dev/null; then
        log_error "sed is required but not installed"
        exit 1
    fi

    if [ ! -f "${NGINX_CONF}" ]; then
        log_error "nginx config not found at ${NGINX_CONF}"
        exit 1
    fi

    if [ ! -f "${COMPOSE_FILE}" ]; then
        log_error "docker-compose file not found at ${COMPOSE_FILE}"
        exit 1
    fi

    # Ensure saves directory exists
    if [ ! -d "${SAVES_DIR}" ]; then
        log_info "Creating saves directory at ${SAVES_DIR}"
        mkdir -p "${SAVES_DIR}"
    fi

    # Ensure s3cmd is available for backups (non-fatal if it fails)
    ensure_s3cmd || true

    # Clean up any stale deployment-in-progress marker from a previous failed deploy
    if [ -f "${DEPLOYMENT_IN_PROGRESS}" ]; then
        log_warn "Found stale deployment-in-progress marker, removing it"
        rm -f "${DEPLOYMENT_IN_PROGRESS}"
    fi
}

# Run
check_requirements
main "$@"