#!/bin/bash # # Blue-Green Deployment Script for Eagle Server # # This script performs a zero-downtime deployment with state consistency: # 1. Create .deployment_in_progress marker (signals deployment started) # 2. Start the staging instance (green) with new image # 3. Run warmup/smoke tests against staging (warms JIT) # 4. Switch nginx to staging (zero downtime - users immediately route to staging) # 5. Stop the active instance (blue) - blocks until flush completes # 6. Create .flush_complete marker (signals disk state is fresh) # # The flush marker coordination ensures green never serves stale game data: # - When users reconnect to green and trigger lazy-load, the code checks for markers # - If .deployment_in_progress exists, lazy-load WAITS for .flush_complete # - Once blue's flush completes and marker is created, lazy-load proceeds with fresh data # # Key insight: nginx switches to green BEFORE blue stops, achieving zero downtime. # Users who trigger lazy-load during blue's shutdown will wait for the flush marker. # # Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG] # # Example: # ./deploy-blue-green.sh latest # ./deploy-blue-green.sh sha-abc123 # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" APP_DIR="${APP_DIR:-/opt/eagle0}" NGINX_CONF="${APP_DIR}/nginx/nginx.conf" COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml" WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh" SAVES_DIR="${APP_DIR}/saves" FLUSH_MARKER="${SAVES_DIR}/.flush_complete" DEPLOYMENT_IN_PROGRESS="${SAVES_DIR}/.deployment_in_progress" ACTIVE_INSTANCE_FILE="${APP_DIR}/.active-instance" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } # Marker file operations use docker exec because saves directory is owned by root (Docker). # We run commands inside a container that has the saves directory mounted. create_deployment_marker() { local deploy_id=$1 local container=$2 # Container to use for file operations docker exec "${container}" rm -f /app/saves/.flush_complete docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.deployment_in_progress" } create_flush_marker() { local deploy_id=$1 local container=$2 # Container to use for file operations docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.flush_complete" docker exec "${container}" rm -f /app/saves/.deployment_in_progress } cleanup_markers_on_failure() { local container=$1 # Container to use for file operations docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true docker exec "${container}" touch /app/saves/.flush_complete 2>/dev/null || true } remove_stale_deployment_marker() { # Try any running eagle container local container container=$(docker ps --filter "name=eagle-" --format "{{.Names}}" | head -1) if [ -n "${container}" ]; then docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true fi } # Determine which instance is currently running (not from nginx config) get_running_instance() { local blue_running green_running blue_running=$(docker inspect --format='{{.State.Running}}' eagle-blue 2>/dev/null || echo "false") green_running=$(docker inspect --format='{{.State.Running}}' eagle-green 2>/dev/null || echo "false") if [ "$blue_running" = "true" ] && [ "$green_running" = "true" ]; then # Both running - use nginx config to determine primary if grep -q "server eagle-blue:40032;" "${NGINX_CONF}" | head -1 | grep -qv backup; then echo "blue" else echo "green" fi elif [ "$blue_running" = "true" ]; then echo "blue" elif [ "$green_running" = "true" ]; then echo "green" else # Neither running - default to blue (first deploy or recovery) echo "none" fi } # Pull image with retry using crane (handles OCI/Docker digest mismatch) pull_with_retry() { local image=$1 local max_attempts=${2:-3} local attempt=1 # Skip pull if image already exists locally (e.g., CI already pulled it) if docker image inspect "${image}" &>/dev/null; then log_info "Image ${image} already exists locally, skipping pull" return 0 fi # Use crane if available (handles OCI format correctly) if [ -x "${APP_DIR}/crane" ]; then while [ $attempt -le $max_attempts ]; do log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..." if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then rm -f /tmp/image.tar log_info "Image pulled and loaded successfully" return 0 fi rm -f /tmp/image.tar log_warn "Pull failed, retrying in 5 seconds..." sleep 5 attempt=$((attempt + 1)) done else # Fallback to docker pull if crane not available log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull" while [ $attempt -le $max_attempts ]; do log_info "Pulling image (attempt ${attempt}/${max_attempts})..." if docker pull "${image}"; then log_info "Image pulled successfully" return 0 fi log_warn "Pull failed, retrying in 5 seconds..." sleep 5 attempt=$((attempt + 1)) done fi log_error "Failed to pull image after ${max_attempts} attempts" return 1 } # Wait for a container to be healthy wait_for_healthy() { local container=$1 local max_attempts=${2:-60} local attempt=1 log_info "Waiting for ${container} to become healthy..." while [ $attempt -le $max_attempts ]; do health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown") if [ "$health" = "healthy" ]; then log_info "${container} is healthy" return 0 fi echo -n "." sleep 2 attempt=$((attempt + 1)) done echo "" log_error "${container} did not become healthy after $((max_attempts * 2)) seconds" return 1 } # Main deployment logic main() { local new_tag="${1:-latest}" local registry="registry.digitalocean.com/eagle0/eagle-server" local new_image="${registry}:${new_tag}" # Generate deployment ID for log correlation with server logs local deploy_id deploy_id=$(date +%s) local deploy_start_time=$deploy_id log_info "=========================================" log_info "Starting blue-green deployment" log_info "Deployment ID: ${deploy_id}" log_info "New image: ${new_image}" log_info "=========================================" cd "${APP_DIR}" # Determine current active instance (need this before creating marker) local active=$(get_running_instance) local staging if [ "$active" = "blue" ] || [ "$active" = "none" ]; then staging="green" active="blue" # Normalize "none" to "blue" for first deploy else staging="blue" fi # Step 1: Signal deployment in progress log_info "[DEPLOY:${deploy_id}] Step 1: Signaling deployment in progress..." # Use active container for marker operations (it's the one currently running) if [ "$active" != "none" ] && docker ps --filter "name=eagle-${active}" --format "{{.Names}}" | grep -q .; then create_deployment_marker "${deploy_id}" "eagle-${active}" log_info "[DEPLOY:${deploy_id}] Deployment marker created via eagle-${active}" else log_warn "[DEPLOY:${deploy_id}] No running container to create marker (first deploy?)" fi # Pull the new image (with retry for intermittent registry issues) if ! pull_with_retry "${new_image}" 3; then log_error "Failed to pull new image, aborting deployment" cleanup_markers_on_failure "eagle-${active}" exit 1 fi # Step 2: Start staging instance with new image log_info "Step 2: Starting eagle-${staging} with new image..." if [ "$staging" = "green" ]; then EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green else EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar fi # Wait for staging to be healthy if ! wait_for_healthy "eagle-${staging}" 90; then log_error "Staging instance failed health check, aborting deployment" docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}" cleanup_markers_on_failure "eagle-${active}" exit 1 fi # Step 3: Run warmup/smoke test local staging_port if [ "$staging" = "green" ]; then staging_port=40034 else staging_port=40032 fi log_info "Step 3: Running warmup against eagle-${staging}..." if [ -x "${WARMUP_SCRIPT}" ]; then if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then log_error "Warmup/smoke test failed, aborting deployment" log_error "Recent eagle-${staging} logs:" docker logs "eagle-${staging}" --tail 200 2>&1 || true docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}" cleanup_markers_on_failure "eagle-${active}" exit 1 fi else log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup" log_warn "JIT will be cold on first requests" fi # Step 4: Switch nginx to staging BEFORE stopping active # This achieves zero downtime - users immediately route to staging. # Any lazy-loads will wait for the flush marker (created in step 6). local nginx_switch_start nginx_switch_start=$(date +%s) log_info "[DEPLOY:${deploy_id}] Step 4: Switching nginx to eagle-${staging}..." # Update nginx config (variable-based routing) if [ "$staging" = "green" ]; then sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}" else sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}" fi # Recreate nginx to pick up new config # Use --no-deps to prevent cascading to auth (which has secrets not available here) docker compose -f "${COMPOSE_FILE}" up -d --force-recreate --no-deps nginx # Verify nginx picked up the correct config local nginx_backend nginx_backend=$(docker exec nginx grep -o 'eagle-[a-z]*:40032' /etc/nginx/nginx.conf | head -1 || echo "unknown") if [ "$nginx_backend" = "eagle-${staging}:40032" ]; then log_info "[DEPLOY:${deploy_id}] Verified: nginx routing to eagle-${staging}" else log_error "[DEPLOY:${deploy_id}] nginx config mismatch! Expected eagle-${staging}:40032, got ${nginx_backend}" exit 1 fi log_info "[DEPLOY:${deploy_id}] Traffic switched to eagle-${staging} (lazy-loads will wait for flush marker)" local nginx_switch_end nginx_switch_end=$(date +%s) # Step 5: Stop active instance (blocks until exit, ensuring flush completes) # Users may be lazy-loading on staging during this time - they'll wait for the marker. local flush_start flush_start=$(date +%s) log_info "[DEPLOY:${deploy_id}] Step 5: Stopping eagle-${active} (waiting for flush)..." docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}" local flush_end flush_end=$(date +%s) local flush_duration=$((flush_end - flush_start)) log_info "[DEPLOY:${deploy_id}] eagle-${active} stopped, flush completed in ${flush_duration}s" # Step 6: Create flush marker - signals that disk state is fresh # Any waiting lazy-loads on staging will now proceed with fresh data. # The Eagle server automatically detects the flush marker update and invalidates any stale cached games. log_info "[DEPLOY:${deploy_id}] Step 6: Creating flush marker..." create_flush_marker "${deploy_id}" "eagle-${staging}" log_info "[DEPLOY:${deploy_id}] Flush marker created - server will auto-invalidate stale cache" # Write active instance file for eagle-exec helper echo "eagle-${staging}" > "${ACTIVE_INSTANCE_FILE}" log_info "[DEPLOY:${deploy_id}] Active instance file updated: eagle-${staging}" # Update .env for admin service local env_file="${APP_DIR}/.env" if [ "$staging" = "green" ]; then log_info "Updating .env for green instance..." sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}" echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}" else log_info "Updating .env for blue instance..." sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}" echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}" fi # Restart admin to pick up new .env and ensure latest image # Use --no-deps to prevent cascading to auth (which has secrets not available here) log_info "Restarting admin service..." docker compose -f "${COMPOSE_FILE}" up -d --force-recreate --no-deps admin # Verify admin container is running sleep 3 if ! docker inspect admin-server --format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then log_error "Admin container failed to start!" log_error "Container logs:" docker logs admin-server --tail 20 2>&1 || true log_error "Container inspect:" docker inspect admin-server 2>&1 | head -50 || true exit 1 fi log_info "Admin service restarted successfully" # Clean up old instance log_info "Cleaning up old eagle-${active}..." docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" 2>/dev/null || true # Stop the old jfr-sidecar (it can't attach to removed container anyway) if [ "$active" = "green" ]; then docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar-green" 2>/dev/null || true docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true else docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar" 2>/dev/null || true docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true fi local deploy_end_time deploy_end_time=$(date +%s) local total_duration=$((deploy_end_time - deploy_start_time)) local user_wait_window=$((flush_end - nginx_switch_end)) log_info "" log_info "=========================================" log_info "[DEPLOY:${deploy_id}] Deployment complete!" log_info " Active instance: eagle-${staging}" log_info " Total duration: ${total_duration}s" log_info " Flush duration: ${flush_duration}s" log_info " Max user wait window: ${user_wait_window}s" log_info "=========================================" } # Check for required tools check_requirements() { if ! command -v docker &> /dev/null; then log_error "docker is required but not installed" exit 1 fi if ! command -v sed &> /dev/null; then log_error "sed is required but not installed" exit 1 fi if [ ! -f "${NGINX_CONF}" ]; then log_error "nginx config not found at ${NGINX_CONF}" exit 1 fi if [ ! -f "${COMPOSE_FILE}" ]; then log_error "docker-compose file not found at ${COMPOSE_FILE}" exit 1 fi # Ensure saves directory exists if [ ! -d "${SAVES_DIR}" ]; then log_info "Creating saves directory at ${SAVES_DIR}" mkdir -p "${SAVES_DIR}" fi # Clean up any stale deployment-in-progress marker from a previous failed deploy if [ -f "${DEPLOYMENT_IN_PROGRESS}" ]; then log_warn "Found stale deployment-in-progress marker, removing it" remove_stale_deployment_marker fi } # Run check_requirements main "$@"