Fix deployment downtime and config sync issues (#5253)

Problems fixed:
1. CI was recreating admin BEFORE blue-green, causing stale .env
2. CI was restarting nginx AFTER blue-green (double restart)
3. Deploy script used slow nginx restart instead of reload
4. Cleanup was blocking the critical path

Changes:
- CI: Only restart shardok before blue-green, let script handle rest
- CI: Remove duplicate nginx restart and fallback path
- Deploy: Use nginx reload (faster) with restart fallback
- Deploy: Update .env before restarting services
- Deploy: Run container cleanup in background

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/docker_build.yml

@@ -319,32 +319,18 @@ jobs:
 
           echo "All images pulled successfully"
 
-          # Recreate non-Eagle services
-          docker compose -f docker-compose.prod.yml up -d --no-deps --force-recreate shardok admin
+          # Recreate shardok (stateless, safe to restart anytime)
+          docker compose -f docker-compose.prod.yml up -d --no-deps --force-recreate shardok
 
-          # Deploy Eagle with blue-green
-          if [ -x "/opt/eagle0/scripts/deploy-blue-green.sh" ]; then
-            echo "Using blue-green deployment for Eagle..."
-            chmod +x /opt/eagle0/scripts/*.sh
-            [ -f "/opt/eagle0/scripts/bin/warmup" ] && chmod +x /opt/eagle0/scripts/bin/warmup
-            GIT_SHA=\$(echo "\${EAGLE_IMAGE}" | sed 's/.*://')
-            /opt/eagle0/scripts/deploy-blue-green.sh "\${GIT_SHA}" || {
-              echo "Blue-green failed, falling back to direct restart"
-              docker compose -f docker-compose.prod.yml up -d --no-deps --force-recreate eagle-blue
-            }
-          else
-            docker compose -f docker-compose.prod.yml up -d --no-deps --force-recreate eagle-blue
-          fi
-
-          # Note: jfr-sidecar is started by deploy-blue-green.sh (either jfr-sidecar or jfr-sidecar-green
-          # depending on which eagle instance is active)
+          # Deploy Eagle with blue-green (handles eagle, nginx, admin, jfr-sidecar)
+          chmod +x /opt/eagle0/scripts/*.sh
+          [ -f "/opt/eagle0/scripts/bin/warmup" ] && chmod +x /opt/eagle0/scripts/bin/warmup
+          GIT_SHA=\$(echo "\${EAGLE_IMAGE}" | sed 's/.*://')
+          /opt/eagle0/scripts/deploy-blue-green.sh "\${GIT_SHA}"
 
           # Ensure auth is running
           docker compose -f docker-compose.prod.yml up -d auth
 
-          # Restart nginx
-          docker compose -f docker-compose.prod.yml up -d --force-recreate nginx
-
           # Verify
           sleep 10
           docker compose -f docker-compose.prod.yml ps

scripts/deploy-blue-green.sh

@@ -224,15 +224,12 @@ main() {
         log_warn "s3cmd not found, skipping S3 backup"
     fi
 
-    # Stop the active instance (this flushes state to storage)
+    # Stop the active instance (flushes state to storage)
+    # We must do this BEFORE switching so staging loads fresh data
     log_info "Stopping eagle-${active} (flushing state to storage)..."
     docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"
 
-    # Note: No need to explicitly reload games - Eagle uses lazy loading.
-    # Games are loaded on-demand when users reconnect after nginx switches traffic.
-    # This ensures games are always loaded from the freshest storage state.
-
-    # Switch nginx upstream
+    # Prepare nginx config and .env BEFORE restarting anything
     log_info "Switching nginx upstream to eagle-${staging}..."
     if [ "$staging" = "green" ]; then
         sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
@@ -240,21 +237,6 @@ main() {
         sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
     fi
 
-    # Restart nginx to ensure fresh config is loaded
-    # Note: We use restart instead of reload to ensure all workers pick up the new config
-    log_info "Restarting nginx..."
-    docker compose -f "${COMPOSE_FILE}" restart nginx
-
-    # Clean up old instance and its jfr-sidecar
-    log_info "Removing old eagle-${active} container..."
-    docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}"
-    if [ "$active" = "green" ]; then
-        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true
-    else
-        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true
-    fi
-
-    # Update .env with new EAGLE_ADDR and JFR_SIDECAR_ADDR for admin service
     local env_file="${APP_DIR}/.env"
     if [ "$staging" = "green" ]; then
         log_info "Updating .env for green instance..."
@@ -268,9 +250,26 @@ main() {
         echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
     fi
 
-    # Restart admin to pick up new EAGLE_ADDR and JFR_SIDECAR_ADDR
+    # Restart nginx (use reload for faster switch, restart if reload fails)
+    log_info "Reloading nginx..."
+    if ! docker exec nginx nginx -s reload 2>/dev/null; then
+        log_warn "Reload failed, restarting nginx..."
+        docker compose -f "${COMPOSE_FILE}" restart nginx
+    fi
+
+    # Restart admin to pick up new .env
     log_info "Restarting admin service..."
-    docker compose -f "${COMPOSE_FILE}" up -d admin
+    docker compose -f "${COMPOSE_FILE}" up -d --force-recreate admin
+
+    # Clean up old instance and its jfr-sidecar (in background, not critical path)
+    log_info "Cleaning up old eagle-${active} container..."
+    docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" &
+    if [ "$active" = "green" ]; then
+        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null &
+    else
+        docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null &
+    fi
+    wait
 
     log_info "Deployment complete!"
     log_info "Active instance: eagle-${staging}"