mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 22:35:42 +00:00
e7745cc9ec
The saves directory is owned by root (created by Docker) but the deploy script runs as the deploy user. Use docker exec to create marker files from inside a running container that has the saves directory mounted. - create_deployment_marker: uses active container (running before staging starts) - create_flush_marker: uses staging container (running after active stops) - cleanup_markers_on_failure: uses active container (still running on failure) - remove_stale_deployment_marker: finds any running eagle container 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
451 lines
17 KiB
Bash
Executable File
451 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Blue-Green Deployment Script for Eagle Server
|
|
#
|
|
# This script performs a zero-downtime deployment with state consistency:
|
|
# 1. Create .deployment_in_progress marker (signals deployment started)
|
|
# 2. Start the staging instance (green) with new image
|
|
# 3. Run warmup/smoke tests against staging (warms JIT)
|
|
# 4. Switch nginx to staging (zero downtime - users immediately route to staging)
|
|
# 5. Stop the active instance (blue) - blocks until flush completes
|
|
# 6. Create .flush_complete marker (signals disk state is fresh)
|
|
#
|
|
# The flush marker coordination ensures green never serves stale game data:
|
|
# - When users reconnect to green and trigger lazy-load, the code checks for markers
|
|
# - If .deployment_in_progress exists, lazy-load WAITS for .flush_complete
|
|
# - Once blue's flush completes and marker is created, lazy-load proceeds with fresh data
|
|
#
|
|
# Key insight: nginx switches to green BEFORE blue stops, achieving zero downtime.
|
|
# Users who trigger lazy-load during blue's shutdown will wait for the flush marker.
|
|
#
|
|
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
|
|
#
|
|
# Example:
|
|
# ./deploy-blue-green.sh latest
|
|
# ./deploy-blue-green.sh sha-abc123
|
|
#
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
APP_DIR="${APP_DIR:-/opt/eagle0}"
|
|
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
|
|
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
|
|
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"
|
|
SAVES_DIR="${APP_DIR}/saves"
|
|
FLUSH_MARKER="${SAVES_DIR}/.flush_complete"
|
|
DEPLOYMENT_IN_PROGRESS="${SAVES_DIR}/.deployment_in_progress"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
|
|
# Marker file operations use docker exec because saves directory is owned by root (Docker).
|
|
# We run commands inside a container that has the saves directory mounted.
|
|
create_deployment_marker() {
|
|
local deploy_id=$1
|
|
local container=$2 # Container to use for file operations
|
|
docker exec "${container}" rm -f /app/saves/.flush_complete
|
|
docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.deployment_in_progress"
|
|
}
|
|
|
|
create_flush_marker() {
|
|
local deploy_id=$1
|
|
local container=$2 # Container to use for file operations
|
|
docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.flush_complete"
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress
|
|
}
|
|
|
|
cleanup_markers_on_failure() {
|
|
local container=$1 # Container to use for file operations
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true
|
|
docker exec "${container}" touch /app/saves/.flush_complete 2>/dev/null || true
|
|
}
|
|
|
|
remove_stale_deployment_marker() {
|
|
# Try any running eagle container
|
|
local container
|
|
container=$(docker ps --filter "name=eagle-" --format "{{.Names}}" | head -1)
|
|
if [ -n "${container}" ]; then
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# Determine which instance is currently running (not from nginx config)
|
|
get_running_instance() {
|
|
local blue_running green_running
|
|
blue_running=$(docker inspect --format='{{.State.Running}}' eagle-blue 2>/dev/null || echo "false")
|
|
green_running=$(docker inspect --format='{{.State.Running}}' eagle-green 2>/dev/null || echo "false")
|
|
|
|
if [ "$blue_running" = "true" ] && [ "$green_running" = "true" ]; then
|
|
# Both running - use nginx config to determine primary
|
|
if grep -q "server eagle-blue:40032;" "${NGINX_CONF}" | head -1 | grep -qv backup; then
|
|
echo "blue"
|
|
else
|
|
echo "green"
|
|
fi
|
|
elif [ "$blue_running" = "true" ]; then
|
|
echo "blue"
|
|
elif [ "$green_running" = "true" ]; then
|
|
echo "green"
|
|
else
|
|
# Neither running - default to blue (first deploy or recovery)
|
|
echo "none"
|
|
fi
|
|
}
|
|
|
|
# Pull image with retry using crane (handles OCI/Docker digest mismatch)
|
|
pull_with_retry() {
|
|
local image=$1
|
|
local max_attempts=${2:-3}
|
|
local attempt=1
|
|
|
|
# Skip pull if image already exists locally (e.g., CI already pulled it)
|
|
if docker image inspect "${image}" &>/dev/null; then
|
|
log_info "Image ${image} already exists locally, skipping pull"
|
|
return 0
|
|
fi
|
|
|
|
# Use crane if available (handles OCI format correctly)
|
|
if [ -x "${APP_DIR}/crane" ]; then
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
|
|
if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
|
|
rm -f /tmp/image.tar
|
|
log_info "Image pulled and loaded successfully"
|
|
return 0
|
|
fi
|
|
rm -f /tmp/image.tar
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
else
|
|
# Fallback to docker pull if crane not available
|
|
log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
|
|
if docker pull "${image}"; then
|
|
log_info "Image pulled successfully"
|
|
return 0
|
|
fi
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
fi
|
|
log_error "Failed to pull image after ${max_attempts} attempts"
|
|
return 1
|
|
}
|
|
|
|
# Wait for a container to be healthy
|
|
wait_for_healthy() {
|
|
local container=$1
|
|
local max_attempts=${2:-60}
|
|
local attempt=1
|
|
|
|
log_info "Waiting for ${container} to become healthy..."
|
|
while [ $attempt -le $max_attempts ]; do
|
|
health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
|
|
if [ "$health" = "healthy" ]; then
|
|
log_info "${container} is healthy"
|
|
return 0
|
|
fi
|
|
echo -n "."
|
|
sleep 2
|
|
attempt=$((attempt + 1))
|
|
done
|
|
echo ""
|
|
log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
|
|
return 1
|
|
}
|
|
|
|
# Main deployment logic
|
|
main() {
|
|
local new_tag="${1:-latest}"
|
|
local registry="registry.digitalocean.com/eagle0/eagle-server"
|
|
local new_image="${registry}:${new_tag}"
|
|
|
|
# Generate deployment ID for log correlation with server logs
|
|
local deploy_id
|
|
deploy_id=$(date +%s)
|
|
local deploy_start_time=$deploy_id
|
|
|
|
log_info "========================================="
|
|
log_info "Starting blue-green deployment"
|
|
log_info "Deployment ID: ${deploy_id}"
|
|
log_info "New image: ${new_image}"
|
|
log_info "========================================="
|
|
|
|
cd "${APP_DIR}"
|
|
|
|
# Determine current active instance (need this before creating marker)
|
|
local active=$(get_running_instance)
|
|
local staging
|
|
if [ "$active" = "blue" ] || [ "$active" = "none" ]; then
|
|
staging="green"
|
|
active="blue" # Normalize "none" to "blue" for first deploy
|
|
else
|
|
staging="blue"
|
|
fi
|
|
|
|
# Step 1: Signal deployment in progress
|
|
log_info "[DEPLOY:${deploy_id}] Step 1: Signaling deployment in progress..."
|
|
# Use active container for marker operations (it's the one currently running)
|
|
if [ "$active" != "none" ] && docker ps --filter "name=eagle-${active}" --format "{{.Names}}" | grep -q .; then
|
|
create_deployment_marker "${deploy_id}" "eagle-${active}"
|
|
log_info "[DEPLOY:${deploy_id}] Deployment marker created via eagle-${active}"
|
|
else
|
|
log_warn "[DEPLOY:${deploy_id}] No running container to create marker (first deploy?)"
|
|
fi
|
|
|
|
# Pull the new image (with retry for intermittent registry issues)
|
|
if ! pull_with_retry "${new_image}" 3; then
|
|
log_error "Failed to pull new image, aborting deployment"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 2: Start staging instance with new image
|
|
log_info "Step 2: Starting eagle-${staging} with new image..."
|
|
if [ "$staging" = "green" ]; then
|
|
EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
|
|
else
|
|
EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
|
|
fi
|
|
|
|
# Wait for staging to be healthy
|
|
if ! wait_for_healthy "eagle-${staging}" 90; then
|
|
log_error "Staging instance failed health check, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 3: Run warmup/smoke test
|
|
local staging_port
|
|
if [ "$staging" = "green" ]; then
|
|
staging_port=40034
|
|
else
|
|
staging_port=40032
|
|
fi
|
|
|
|
log_info "Step 3: Running warmup against eagle-${staging}..."
|
|
if [ -x "${WARMUP_SCRIPT}" ]; then
|
|
if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
|
|
log_error "Warmup/smoke test failed, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
|
|
log_warn "JIT will be cold on first requests"
|
|
fi
|
|
|
|
# Backup games.e0es BEFORE stopping the active instance
|
|
local backup_timestamp
|
|
backup_timestamp=$(date +%Y%m%d_%H%M%S)
|
|
log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
|
|
if command -v s3cmd &> /dev/null; then
|
|
if s3cmd put "${SAVES_DIR}/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
|
|
log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
|
|
else
|
|
log_warn "S3 backup failed (s3cmd put), continuing deployment"
|
|
fi
|
|
else
|
|
log_warn "s3cmd not found, skipping S3 backup"
|
|
fi
|
|
|
|
# Step 4: Switch nginx to staging BEFORE stopping active
|
|
# This achieves zero downtime - users immediately route to staging.
|
|
# Any lazy-loads will wait for the flush marker (created in step 6).
|
|
local nginx_switch_start
|
|
nginx_switch_start=$(date +%s)
|
|
log_info "[DEPLOY:${deploy_id}] Step 4: Switching nginx to eagle-${staging}..."
|
|
|
|
# Update nginx config (variable-based routing)
|
|
if [ "$staging" = "green" ]; then
|
|
sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
|
|
else
|
|
sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
|
|
fi
|
|
|
|
# Recreate nginx to pick up new config
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate nginx
|
|
|
|
# Verify nginx picked up the correct config
|
|
local nginx_backend
|
|
nginx_backend=$(docker exec nginx grep -o 'eagle-[a-z]*:40032' /etc/nginx/nginx.conf | head -1 || echo "unknown")
|
|
if [ "$nginx_backend" = "eagle-${staging}:40032" ]; then
|
|
log_info "[DEPLOY:${deploy_id}] Verified: nginx routing to eagle-${staging}"
|
|
else
|
|
log_error "[DEPLOY:${deploy_id}] nginx config mismatch! Expected eagle-${staging}:40032, got ${nginx_backend}"
|
|
exit 1
|
|
fi
|
|
log_info "[DEPLOY:${deploy_id}] Traffic switched to eagle-${staging} (lazy-loads will wait for flush marker)"
|
|
|
|
# Step 5: Stop active instance (blocks until exit, ensuring flush completes)
|
|
# Users may be lazy-loading on staging during this time - they'll wait for the marker.
|
|
local flush_start
|
|
flush_start=$(date +%s)
|
|
log_info "[DEPLOY:${deploy_id}] Step 5: Stopping eagle-${active} (waiting for flush)..."
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"
|
|
local flush_end
|
|
flush_end=$(date +%s)
|
|
local flush_duration=$((flush_end - flush_start))
|
|
log_info "[DEPLOY:${deploy_id}] eagle-${active} stopped, flush completed in ${flush_duration}s"
|
|
|
|
# Step 6: Create flush marker - signals that disk state is fresh
|
|
# Any waiting lazy-loads on staging will now proceed with fresh data.
|
|
log_info "[DEPLOY:${deploy_id}] Step 6: Creating flush marker..."
|
|
create_flush_marker "${deploy_id}" "eagle-${staging}"
|
|
log_info "[DEPLOY:${deploy_id}] Flush marker created - waiting lazy-loads can now proceed"
|
|
|
|
# Update .env for admin service
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ "$staging" = "green" ]; then
|
|
log_info "Updating .env for green instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
|
|
else
|
|
log_info "Updating .env for blue instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
|
|
fi
|
|
|
|
# Restart admin to pick up new .env
|
|
log_info "Restarting admin service..."
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate admin
|
|
|
|
# Clean up old instance
|
|
log_info "Cleaning up old eagle-${active}..."
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" 2>/dev/null || true
|
|
|
|
# Stop the old jfr-sidecar (it can't attach to removed container anyway)
|
|
if [ "$active" = "green" ]; then
|
|
docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar-green" 2>/dev/null || true
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true
|
|
else
|
|
docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar" 2>/dev/null || true
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true
|
|
fi
|
|
|
|
local deploy_end_time
|
|
deploy_end_time=$(date +%s)
|
|
local total_duration=$((deploy_end_time - deploy_start_time))
|
|
local user_wait_window=$((flush_end - nginx_switch_start))
|
|
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "[DEPLOY:${deploy_id}] Deployment complete!"
|
|
log_info " Active instance: eagle-${staging}"
|
|
log_info " Total duration: ${total_duration}s"
|
|
log_info " Flush duration: ${flush_duration}s"
|
|
log_info " Max user wait window: ${user_wait_window}s"
|
|
log_info "========================================="
|
|
}
|
|
|
|
# Ensure s3cmd is installed and configured for S3 backups
|
|
ensure_s3cmd() {
|
|
# Install s3cmd if not present
|
|
if ! command -v s3cmd &> /dev/null; then
|
|
log_info "Installing s3cmd for S3 backups..."
|
|
if command -v pip3 &> /dev/null; then
|
|
pip3 install --quiet s3cmd
|
|
elif command -v pip &> /dev/null; then
|
|
pip install --quiet s3cmd
|
|
elif command -v apt-get &> /dev/null; then
|
|
sudo apt-get update -qq && sudo apt-get install -y -qq s3cmd
|
|
else
|
|
log_warn "Cannot install s3cmd (no pip or apt-get), S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
|
|
if ! command -v s3cmd &> /dev/null; then
|
|
log_warn "s3cmd installation failed, S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
log_info "s3cmd installed successfully"
|
|
fi
|
|
|
|
# Configure s3cmd for DigitalOcean Spaces if not already configured
|
|
local s3cfg="${HOME}/.s3cfg"
|
|
if [ ! -f "${s3cfg}" ]; then
|
|
# Load credentials from .env
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ -f "${env_file}" ]; then
|
|
# shellcheck source=/dev/null
|
|
source "${env_file}"
|
|
fi
|
|
|
|
if [ -z "${DO_SPACES_ACCESS_KEY:-}" ] || [ -z "${DO_SPACES_SECRET_KEY:-}" ]; then
|
|
log_warn "DO_SPACES_ACCESS_KEY or DO_SPACES_SECRET_KEY not set, S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Configuring s3cmd for DigitalOcean Spaces..."
|
|
cat > "${s3cfg}" << EOF
|
|
[default]
|
|
access_key = ${DO_SPACES_ACCESS_KEY}
|
|
secret_key = ${DO_SPACES_SECRET_KEY}
|
|
host_base = sfo3.digitaloceanspaces.com
|
|
host_bucket = %(bucket)s.sfo3.digitaloceanspaces.com
|
|
use_https = True
|
|
EOF
|
|
log_info "s3cmd configured successfully"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Check for required tools
|
|
check_requirements() {
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "docker is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v sed &> /dev/null; then
|
|
log_error "sed is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${NGINX_CONF}" ]; then
|
|
log_error "nginx config not found at ${NGINX_CONF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${COMPOSE_FILE}" ]; then
|
|
log_error "docker-compose file not found at ${COMPOSE_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# Ensure saves directory exists
|
|
if [ ! -d "${SAVES_DIR}" ]; then
|
|
log_info "Creating saves directory at ${SAVES_DIR}"
|
|
mkdir -p "${SAVES_DIR}"
|
|
fi
|
|
|
|
# Ensure s3cmd is available for backups (non-fatal if it fails)
|
|
ensure_s3cmd || true
|
|
|
|
# Clean up any stale deployment-in-progress marker from a previous failed deploy
|
|
if [ -f "${DEPLOYMENT_IN_PROGRESS}" ]; then
|
|
log_warn "Found stale deployment-in-progress marker, removing it"
|
|
remove_stale_deployment_marker
|
|
fi
|
|
}
|
|
|
|
# Run
|
|
check_requirements
|
|
main "$@"
|