mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 22:55:41 +00:00
3ae3f6ba16
During warmup, the staging server may cache stale game data that was loaded before the active server flushed. Instead of exposing an RPC for cache invalidation (which leaks internal state), the server now automatically detects when the flush marker is updated and invalidates any cached games. Changes: - GamesManager: Added `invalidateCacheIfFlushMarkerUpdated()` that checks the flush marker's modification time and clears the cache if it's been updated since the last check - Called before checking if a game is in cache, so stale data is cleared before any attempt to use it - Removed the InvalidateGameCache RPC (no longer needed) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
452 lines
17 KiB
Bash
Executable File
452 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Blue-Green Deployment Script for Eagle Server
|
|
#
|
|
# This script performs a zero-downtime deployment with state consistency:
|
|
# 1. Create .deployment_in_progress marker (signals deployment started)
|
|
# 2. Start the staging instance (green) with new image
|
|
# 3. Run warmup/smoke tests against staging (warms JIT)
|
|
# 4. Switch nginx to staging (zero downtime - users immediately route to staging)
|
|
# 5. Stop the active instance (blue) - blocks until flush completes
|
|
# 6. Create .flush_complete marker (signals disk state is fresh)
|
|
#
|
|
# The flush marker coordination ensures green never serves stale game data:
|
|
# - When users reconnect to green and trigger lazy-load, the code checks for markers
|
|
# - If .deployment_in_progress exists, lazy-load WAITS for .flush_complete
|
|
# - Once blue's flush completes and marker is created, lazy-load proceeds with fresh data
|
|
#
|
|
# Key insight: nginx switches to green BEFORE blue stops, achieving zero downtime.
|
|
# Users who trigger lazy-load during blue's shutdown will wait for the flush marker.
|
|
#
|
|
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
|
|
#
|
|
# Example:
|
|
# ./deploy-blue-green.sh latest
|
|
# ./deploy-blue-green.sh sha-abc123
|
|
#
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
APP_DIR="${APP_DIR:-/opt/eagle0}"
|
|
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
|
|
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
|
|
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"
|
|
SAVES_DIR="${APP_DIR}/saves"
|
|
FLUSH_MARKER="${SAVES_DIR}/.flush_complete"
|
|
DEPLOYMENT_IN_PROGRESS="${SAVES_DIR}/.deployment_in_progress"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
|
|
# Marker file operations use docker exec because saves directory is owned by root (Docker).
|
|
# We run commands inside a container that has the saves directory mounted.
|
|
create_deployment_marker() {
|
|
local deploy_id=$1
|
|
local container=$2 # Container to use for file operations
|
|
docker exec "${container}" rm -f /app/saves/.flush_complete
|
|
docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.deployment_in_progress"
|
|
}
|
|
|
|
create_flush_marker() {
|
|
local deploy_id=$1
|
|
local container=$2 # Container to use for file operations
|
|
docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.flush_complete"
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress
|
|
}
|
|
|
|
cleanup_markers_on_failure() {
|
|
local container=$1 # Container to use for file operations
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true
|
|
docker exec "${container}" touch /app/saves/.flush_complete 2>/dev/null || true
|
|
}
|
|
|
|
remove_stale_deployment_marker() {
|
|
# Try any running eagle container
|
|
local container
|
|
container=$(docker ps --filter "name=eagle-" --format "{{.Names}}" | head -1)
|
|
if [ -n "${container}" ]; then
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# Determine which instance is currently running (not from nginx config)
|
|
get_running_instance() {
|
|
local blue_running green_running
|
|
blue_running=$(docker inspect --format='{{.State.Running}}' eagle-blue 2>/dev/null || echo "false")
|
|
green_running=$(docker inspect --format='{{.State.Running}}' eagle-green 2>/dev/null || echo "false")
|
|
|
|
if [ "$blue_running" = "true" ] && [ "$green_running" = "true" ]; then
|
|
# Both running - use nginx config to determine primary
|
|
if grep -q "server eagle-blue:40032;" "${NGINX_CONF}" | head -1 | grep -qv backup; then
|
|
echo "blue"
|
|
else
|
|
echo "green"
|
|
fi
|
|
elif [ "$blue_running" = "true" ]; then
|
|
echo "blue"
|
|
elif [ "$green_running" = "true" ]; then
|
|
echo "green"
|
|
else
|
|
# Neither running - default to blue (first deploy or recovery)
|
|
echo "none"
|
|
fi
|
|
}
|
|
|
|
# Pull image with retry using crane (handles OCI/Docker digest mismatch)
|
|
pull_with_retry() {
|
|
local image=$1
|
|
local max_attempts=${2:-3}
|
|
local attempt=1
|
|
|
|
# Skip pull if image already exists locally (e.g., CI already pulled it)
|
|
if docker image inspect "${image}" &>/dev/null; then
|
|
log_info "Image ${image} already exists locally, skipping pull"
|
|
return 0
|
|
fi
|
|
|
|
# Use crane if available (handles OCI format correctly)
|
|
if [ -x "${APP_DIR}/crane" ]; then
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
|
|
if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
|
|
rm -f /tmp/image.tar
|
|
log_info "Image pulled and loaded successfully"
|
|
return 0
|
|
fi
|
|
rm -f /tmp/image.tar
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
else
|
|
# Fallback to docker pull if crane not available
|
|
log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
|
|
if docker pull "${image}"; then
|
|
log_info "Image pulled successfully"
|
|
return 0
|
|
fi
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
fi
|
|
log_error "Failed to pull image after ${max_attempts} attempts"
|
|
return 1
|
|
}
|
|
|
|
# Wait for a container to be healthy
|
|
wait_for_healthy() {
|
|
local container=$1
|
|
local max_attempts=${2:-60}
|
|
local attempt=1
|
|
|
|
log_info "Waiting for ${container} to become healthy..."
|
|
while [ $attempt -le $max_attempts ]; do
|
|
health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
|
|
if [ "$health" = "healthy" ]; then
|
|
log_info "${container} is healthy"
|
|
return 0
|
|
fi
|
|
echo -n "."
|
|
sleep 2
|
|
attempt=$((attempt + 1))
|
|
done
|
|
echo ""
|
|
log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
|
|
return 1
|
|
}
|
|
|
|
# Main deployment logic
|
|
main() {
|
|
local new_tag="${1:-latest}"
|
|
local registry="registry.digitalocean.com/eagle0/eagle-server"
|
|
local new_image="${registry}:${new_tag}"
|
|
|
|
# Generate deployment ID for log correlation with server logs
|
|
local deploy_id
|
|
deploy_id=$(date +%s)
|
|
local deploy_start_time=$deploy_id
|
|
|
|
log_info "========================================="
|
|
log_info "Starting blue-green deployment"
|
|
log_info "Deployment ID: ${deploy_id}"
|
|
log_info "New image: ${new_image}"
|
|
log_info "========================================="
|
|
|
|
cd "${APP_DIR}"
|
|
|
|
# Determine current active instance (need this before creating marker)
|
|
local active=$(get_running_instance)
|
|
local staging
|
|
if [ "$active" = "blue" ] || [ "$active" = "none" ]; then
|
|
staging="green"
|
|
active="blue" # Normalize "none" to "blue" for first deploy
|
|
else
|
|
staging="blue"
|
|
fi
|
|
|
|
# Step 1: Signal deployment in progress
|
|
log_info "[DEPLOY:${deploy_id}] Step 1: Signaling deployment in progress..."
|
|
# Use active container for marker operations (it's the one currently running)
|
|
if [ "$active" != "none" ] && docker ps --filter "name=eagle-${active}" --format "{{.Names}}" | grep -q .; then
|
|
create_deployment_marker "${deploy_id}" "eagle-${active}"
|
|
log_info "[DEPLOY:${deploy_id}] Deployment marker created via eagle-${active}"
|
|
else
|
|
log_warn "[DEPLOY:${deploy_id}] No running container to create marker (first deploy?)"
|
|
fi
|
|
|
|
# Pull the new image (with retry for intermittent registry issues)
|
|
if ! pull_with_retry "${new_image}" 3; then
|
|
log_error "Failed to pull new image, aborting deployment"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 2: Start staging instance with new image
|
|
log_info "Step 2: Starting eagle-${staging} with new image..."
|
|
if [ "$staging" = "green" ]; then
|
|
EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
|
|
else
|
|
EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
|
|
fi
|
|
|
|
# Wait for staging to be healthy
|
|
if ! wait_for_healthy "eagle-${staging}" 90; then
|
|
log_error "Staging instance failed health check, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 3: Run warmup/smoke test
|
|
local staging_port
|
|
if [ "$staging" = "green" ]; then
|
|
staging_port=40034
|
|
else
|
|
staging_port=40032
|
|
fi
|
|
|
|
log_info "Step 3: Running warmup against eagle-${staging}..."
|
|
if [ -x "${WARMUP_SCRIPT}" ]; then
|
|
if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
|
|
log_error "Warmup/smoke test failed, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
|
|
log_warn "JIT will be cold on first requests"
|
|
fi
|
|
|
|
# Backup games.e0es BEFORE stopping the active instance
|
|
local backup_timestamp
|
|
backup_timestamp=$(date +%Y%m%d_%H%M%S)
|
|
log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
|
|
if command -v s3cmd &> /dev/null; then
|
|
if s3cmd put "${SAVES_DIR}/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
|
|
log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
|
|
else
|
|
log_warn "S3 backup failed (s3cmd put), continuing deployment"
|
|
fi
|
|
else
|
|
log_warn "s3cmd not found, skipping S3 backup"
|
|
fi
|
|
|
|
# Step 4: Switch nginx to staging BEFORE stopping active
|
|
# This achieves zero downtime - users immediately route to staging.
|
|
# Any lazy-loads will wait for the flush marker (created in step 6).
|
|
local nginx_switch_start
|
|
nginx_switch_start=$(date +%s)
|
|
log_info "[DEPLOY:${deploy_id}] Step 4: Switching nginx to eagle-${staging}..."
|
|
|
|
# Update nginx config (variable-based routing)
|
|
if [ "$staging" = "green" ]; then
|
|
sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
|
|
else
|
|
sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
|
|
fi
|
|
|
|
# Recreate nginx to pick up new config
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate nginx
|
|
|
|
# Verify nginx picked up the correct config
|
|
local nginx_backend
|
|
nginx_backend=$(docker exec nginx grep -o 'eagle-[a-z]*:40032' /etc/nginx/nginx.conf | head -1 || echo "unknown")
|
|
if [ "$nginx_backend" = "eagle-${staging}:40032" ]; then
|
|
log_info "[DEPLOY:${deploy_id}] Verified: nginx routing to eagle-${staging}"
|
|
else
|
|
log_error "[DEPLOY:${deploy_id}] nginx config mismatch! Expected eagle-${staging}:40032, got ${nginx_backend}"
|
|
exit 1
|
|
fi
|
|
log_info "[DEPLOY:${deploy_id}] Traffic switched to eagle-${staging} (lazy-loads will wait for flush marker)"
|
|
|
|
# Step 5: Stop active instance (blocks until exit, ensuring flush completes)
|
|
# Users may be lazy-loading on staging during this time - they'll wait for the marker.
|
|
local flush_start
|
|
flush_start=$(date +%s)
|
|
log_info "[DEPLOY:${deploy_id}] Step 5: Stopping eagle-${active} (waiting for flush)..."
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"
|
|
local flush_end
|
|
flush_end=$(date +%s)
|
|
local flush_duration=$((flush_end - flush_start))
|
|
log_info "[DEPLOY:${deploy_id}] eagle-${active} stopped, flush completed in ${flush_duration}s"
|
|
|
|
# Step 6: Create flush marker - signals that disk state is fresh
|
|
# Any waiting lazy-loads on staging will now proceed with fresh data.
|
|
# The Eagle server automatically detects the flush marker update and invalidates any stale cached games.
|
|
log_info "[DEPLOY:${deploy_id}] Step 6: Creating flush marker..."
|
|
create_flush_marker "${deploy_id}" "eagle-${staging}"
|
|
log_info "[DEPLOY:${deploy_id}] Flush marker created - server will auto-invalidate stale cache"
|
|
|
|
# Update .env for admin service
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ "$staging" = "green" ]; then
|
|
log_info "Updating .env for green instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
|
|
else
|
|
log_info "Updating .env for blue instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
|
|
fi
|
|
|
|
# Restart admin to pick up new .env
|
|
log_info "Restarting admin service..."
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate admin
|
|
|
|
# Clean up old instance
|
|
log_info "Cleaning up old eagle-${active}..."
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" 2>/dev/null || true
|
|
|
|
# Stop the old jfr-sidecar (it can't attach to removed container anyway)
|
|
if [ "$active" = "green" ]; then
|
|
docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar-green" 2>/dev/null || true
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true
|
|
else
|
|
docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar" 2>/dev/null || true
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true
|
|
fi
|
|
|
|
local deploy_end_time
|
|
deploy_end_time=$(date +%s)
|
|
local total_duration=$((deploy_end_time - deploy_start_time))
|
|
local user_wait_window=$((flush_end - nginx_switch_start))
|
|
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "[DEPLOY:${deploy_id}] Deployment complete!"
|
|
log_info " Active instance: eagle-${staging}"
|
|
log_info " Total duration: ${total_duration}s"
|
|
log_info " Flush duration: ${flush_duration}s"
|
|
log_info " Max user wait window: ${user_wait_window}s"
|
|
log_info "========================================="
|
|
}
|
|
|
|
# Ensure s3cmd is installed and configured for S3 backups
|
|
ensure_s3cmd() {
|
|
# Install s3cmd if not present
|
|
if ! command -v s3cmd &> /dev/null; then
|
|
log_info "Installing s3cmd for S3 backups..."
|
|
if command -v pip3 &> /dev/null; then
|
|
pip3 install --quiet s3cmd
|
|
elif command -v pip &> /dev/null; then
|
|
pip install --quiet s3cmd
|
|
elif command -v apt-get &> /dev/null; then
|
|
sudo apt-get update -qq && sudo apt-get install -y -qq s3cmd
|
|
else
|
|
log_warn "Cannot install s3cmd (no pip or apt-get), S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
|
|
if ! command -v s3cmd &> /dev/null; then
|
|
log_warn "s3cmd installation failed, S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
log_info "s3cmd installed successfully"
|
|
fi
|
|
|
|
# Configure s3cmd for DigitalOcean Spaces if not already configured
|
|
local s3cfg="${HOME}/.s3cfg"
|
|
if [ ! -f "${s3cfg}" ]; then
|
|
# Load credentials from .env
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ -f "${env_file}" ]; then
|
|
# shellcheck source=/dev/null
|
|
source "${env_file}"
|
|
fi
|
|
|
|
if [ -z "${DO_SPACES_ACCESS_KEY:-}" ] || [ -z "${DO_SPACES_SECRET_KEY:-}" ]; then
|
|
log_warn "DO_SPACES_ACCESS_KEY or DO_SPACES_SECRET_KEY not set, S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Configuring s3cmd for DigitalOcean Spaces..."
|
|
cat > "${s3cfg}" << EOF
|
|
[default]
|
|
access_key = ${DO_SPACES_ACCESS_KEY}
|
|
secret_key = ${DO_SPACES_SECRET_KEY}
|
|
host_base = sfo3.digitaloceanspaces.com
|
|
host_bucket = %(bucket)s.sfo3.digitaloceanspaces.com
|
|
use_https = True
|
|
EOF
|
|
log_info "s3cmd configured successfully"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Check for required tools
|
|
check_requirements() {
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "docker is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v sed &> /dev/null; then
|
|
log_error "sed is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${NGINX_CONF}" ]; then
|
|
log_error "nginx config not found at ${NGINX_CONF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${COMPOSE_FILE}" ]; then
|
|
log_error "docker-compose file not found at ${COMPOSE_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# Ensure saves directory exists
|
|
if [ ! -d "${SAVES_DIR}" ]; then
|
|
log_info "Creating saves directory at ${SAVES_DIR}"
|
|
mkdir -p "${SAVES_DIR}"
|
|
fi
|
|
|
|
# Ensure s3cmd is available for backups (non-fatal if it fails)
|
|
ensure_s3cmd || true
|
|
|
|
# Clean up any stale deployment-in-progress marker from a previous failed deploy
|
|
if [ -f "${DEPLOYMENT_IN_PROGRESS}" ]; then
|
|
log_warn "Found stale deployment-in-progress marker, removing it"
|
|
remove_stale_deployment_marker
|
|
fi
|
|
}
|
|
|
|
# Run
|
|
check_requirements
|
|
main "$@"
|