mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 21:35:42 +00:00
405 lines
16 KiB
Bash
Executable File
405 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Blue-Green Deployment Script for Eagle Server
|
|
#
|
|
# This script performs a zero-downtime deployment with state consistency:
|
|
# 1. Create .deployment_in_progress marker (signals deployment started)
|
|
# 2. Start the staging instance (green) with new image
|
|
# 3. Run warmup/smoke tests against staging (warms JIT)
|
|
# 4. Switch nginx to staging (zero downtime - users immediately route to staging)
|
|
# 5. Stop the active instance (blue) - blocks until flush completes
|
|
# 6. Create .flush_complete marker (signals disk state is fresh)
|
|
#
|
|
# The flush marker coordination ensures green never serves stale game data:
|
|
# - When users reconnect to green and trigger lazy-load, the code checks for markers
|
|
# - If .deployment_in_progress exists, lazy-load WAITS for .flush_complete
|
|
# - Once blue's flush completes and marker is created, lazy-load proceeds with fresh data
|
|
#
|
|
# Key insight: nginx switches to green BEFORE blue stops, achieving zero downtime.
|
|
# Users who trigger lazy-load during blue's shutdown will wait for the flush marker.
|
|
#
|
|
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
|
|
#
|
|
# Example:
|
|
# ./deploy-blue-green.sh latest
|
|
# ./deploy-blue-green.sh sha-abc123
|
|
#
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
APP_DIR="${APP_DIR:-/opt/eagle0}"
|
|
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
|
|
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
|
|
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"
|
|
SAVES_DIR="${APP_DIR}/saves"
|
|
FLUSH_MARKER="${SAVES_DIR}/.flush_complete"
|
|
DEPLOYMENT_IN_PROGRESS="${SAVES_DIR}/.deployment_in_progress"
|
|
ACTIVE_INSTANCE_FILE="${APP_DIR}/.active-instance"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
|
|
# Marker file operations use docker exec because saves directory is owned by root (Docker).
|
|
# We run commands inside a container that has the saves directory mounted.
|
|
create_deployment_marker() {
|
|
local deploy_id=$1
|
|
local container=$2 # Container to use for file operations
|
|
docker exec "${container}" rm -f /app/saves/.flush_complete
|
|
docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.deployment_in_progress"
|
|
}
|
|
|
|
create_flush_marker() {
|
|
local deploy_id=$1
|
|
local container=$2 # Container to use for file operations
|
|
docker exec "${container}" sh -c "echo '${deploy_id}' > /app/saves/.flush_complete"
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress
|
|
}
|
|
|
|
cleanup_markers_on_failure() {
|
|
local container=$1 # Container to use for file operations
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true
|
|
docker exec "${container}" touch /app/saves/.flush_complete 2>/dev/null || true
|
|
}
|
|
|
|
remove_stale_deployment_marker() {
|
|
# Try any running eagle container
|
|
local container
|
|
container=$(docker ps --filter "name=eagle-" --format "{{.Names}}" | head -1)
|
|
if [ -n "${container}" ]; then
|
|
docker exec "${container}" rm -f /app/saves/.deployment_in_progress 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# Determine which instance is currently running (not from nginx config)
|
|
get_running_instance() {
|
|
local blue_running green_running
|
|
blue_running=$(docker inspect --format='{{.State.Running}}' eagle-blue 2>/dev/null || echo "false")
|
|
green_running=$(docker inspect --format='{{.State.Running}}' eagle-green 2>/dev/null || echo "false")
|
|
|
|
if [ "$blue_running" = "true" ] && [ "$green_running" = "true" ]; then
|
|
# Both running - use nginx config to determine primary
|
|
if grep -q "server eagle-blue:40032;" "${NGINX_CONF}" | head -1 | grep -qv backup; then
|
|
echo "blue"
|
|
else
|
|
echo "green"
|
|
fi
|
|
elif [ "$blue_running" = "true" ]; then
|
|
echo "blue"
|
|
elif [ "$green_running" = "true" ]; then
|
|
echo "green"
|
|
else
|
|
# Neither running - default to blue (first deploy or recovery)
|
|
echo "none"
|
|
fi
|
|
}
|
|
|
|
# Pull image with retry using crane (handles OCI/Docker digest mismatch)
|
|
pull_with_retry() {
|
|
local image=$1
|
|
local max_attempts=${2:-3}
|
|
local attempt=1
|
|
|
|
# Skip pull if image already exists locally (e.g., CI already pulled it)
|
|
if docker image inspect "${image}" &>/dev/null; then
|
|
log_info "Image ${image} already exists locally, skipping pull"
|
|
return 0
|
|
fi
|
|
|
|
# Use crane if available (handles OCI format correctly)
|
|
if [ -x "${APP_DIR}/crane" ]; then
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
|
|
if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
|
|
rm -f /tmp/image.tar
|
|
log_info "Image pulled and loaded successfully"
|
|
return 0
|
|
fi
|
|
rm -f /tmp/image.tar
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
else
|
|
# Fallback to docker pull if crane not available
|
|
log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
|
|
if docker pull "${image}"; then
|
|
log_info "Image pulled successfully"
|
|
return 0
|
|
fi
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
fi
|
|
log_error "Failed to pull image after ${max_attempts} attempts"
|
|
return 1
|
|
}
|
|
|
|
# Wait for a container to be healthy
|
|
wait_for_healthy() {
|
|
local container=$1
|
|
local max_attempts=${2:-60}
|
|
local attempt=1
|
|
|
|
log_info "Waiting for ${container} to become healthy..."
|
|
while [ $attempt -le $max_attempts ]; do
|
|
health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
|
|
if [ "$health" = "healthy" ]; then
|
|
log_info "${container} is healthy"
|
|
return 0
|
|
fi
|
|
echo -n "."
|
|
sleep 2
|
|
attempt=$((attempt + 1))
|
|
done
|
|
echo ""
|
|
log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
|
|
return 1
|
|
}
|
|
|
|
# Main deployment logic
|
|
main() {
|
|
local new_tag="${1:-latest}"
|
|
local registry="registry.digitalocean.com/eagle0/eagle-server"
|
|
local new_image="${registry}:${new_tag}"
|
|
|
|
# Generate deployment ID for log correlation with server logs
|
|
local deploy_id
|
|
deploy_id=$(date +%s)
|
|
local deploy_start_time=$deploy_id
|
|
|
|
log_info "========================================="
|
|
log_info "Starting blue-green deployment"
|
|
log_info "Deployment ID: ${deploy_id}"
|
|
log_info "New image: ${new_image}"
|
|
log_info "========================================="
|
|
|
|
cd "${APP_DIR}"
|
|
|
|
# Determine current active instance (need this before creating marker)
|
|
local active=$(get_running_instance)
|
|
local staging
|
|
if [ "$active" = "blue" ] || [ "$active" = "none" ]; then
|
|
staging="green"
|
|
active="blue" # Normalize "none" to "blue" for first deploy
|
|
else
|
|
staging="blue"
|
|
fi
|
|
|
|
# Step 1: Signal deployment in progress
|
|
log_info "[DEPLOY:${deploy_id}] Step 1: Signaling deployment in progress..."
|
|
# Use active container for marker operations (it's the one currently running)
|
|
if [ "$active" != "none" ] && docker ps --filter "name=eagle-${active}" --format "{{.Names}}" | grep -q .; then
|
|
create_deployment_marker "${deploy_id}" "eagle-${active}"
|
|
log_info "[DEPLOY:${deploy_id}] Deployment marker created via eagle-${active}"
|
|
else
|
|
log_warn "[DEPLOY:${deploy_id}] No running container to create marker (first deploy?)"
|
|
fi
|
|
|
|
# Pull the new image (with retry for intermittent registry issues)
|
|
if ! pull_with_retry "${new_image}" 3; then
|
|
log_error "Failed to pull new image, aborting deployment"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 2: Start staging instance with new image
|
|
log_info "Step 2: Starting eagle-${staging} with new image..."
|
|
if [ "$staging" = "green" ]; then
|
|
EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
|
|
else
|
|
EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
|
|
fi
|
|
|
|
# Wait for staging to be healthy
|
|
if ! wait_for_healthy "eagle-${staging}" 90; then
|
|
log_error "Staging instance failed health check, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 3: Run warmup/smoke test
|
|
local staging_port
|
|
if [ "$staging" = "green" ]; then
|
|
staging_port=40034
|
|
else
|
|
staging_port=40032
|
|
fi
|
|
|
|
log_info "Step 3: Running warmup against eagle-${staging}..."
|
|
if [ -x "${WARMUP_SCRIPT}" ]; then
|
|
if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
|
|
log_error "Warmup/smoke test failed, aborting deployment"
|
|
log_error "Recent eagle-${staging} logs:"
|
|
docker logs "eagle-${staging}" --tail 200 2>&1 || true
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
cleanup_markers_on_failure "eagle-${active}"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
|
|
log_warn "JIT will be cold on first requests"
|
|
fi
|
|
|
|
# Step 4: Switch nginx to staging BEFORE stopping active
|
|
# This achieves zero downtime - users immediately route to staging.
|
|
# Any lazy-loads will wait for the flush marker (created in step 6).
|
|
local nginx_switch_start
|
|
nginx_switch_start=$(date +%s)
|
|
log_info "[DEPLOY:${deploy_id}] Step 4: Switching nginx to eagle-${staging}..."
|
|
|
|
# Update nginx config (variable-based routing)
|
|
if [ "$staging" = "green" ]; then
|
|
sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
|
|
else
|
|
sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
|
|
fi
|
|
|
|
# Recreate nginx to pick up new config
|
|
# Use --no-deps to prevent cascading to auth (which has secrets not available here)
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate --no-deps nginx
|
|
|
|
# Verify nginx picked up the correct config
|
|
local nginx_backend
|
|
nginx_backend=$(docker exec nginx grep -o 'eagle-[a-z]*:40032' /etc/nginx/nginx.conf | head -1 || echo "unknown")
|
|
if [ "$nginx_backend" = "eagle-${staging}:40032" ]; then
|
|
log_info "[DEPLOY:${deploy_id}] Verified: nginx routing to eagle-${staging}"
|
|
else
|
|
log_error "[DEPLOY:${deploy_id}] nginx config mismatch! Expected eagle-${staging}:40032, got ${nginx_backend}"
|
|
exit 1
|
|
fi
|
|
log_info "[DEPLOY:${deploy_id}] Traffic switched to eagle-${staging} (lazy-loads will wait for flush marker)"
|
|
local nginx_switch_end
|
|
nginx_switch_end=$(date +%s)
|
|
|
|
# Step 5: Stop active instance (blocks until exit, ensuring flush completes)
|
|
# Users may be lazy-loading on staging during this time - they'll wait for the marker.
|
|
local flush_start
|
|
flush_start=$(date +%s)
|
|
log_info "[DEPLOY:${deploy_id}] Step 5: Stopping eagle-${active} (waiting for flush)..."
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"
|
|
local flush_end
|
|
flush_end=$(date +%s)
|
|
local flush_duration=$((flush_end - flush_start))
|
|
log_info "[DEPLOY:${deploy_id}] eagle-${active} stopped, flush completed in ${flush_duration}s"
|
|
|
|
# Step 6: Create flush marker - signals that disk state is fresh
|
|
# Any waiting lazy-loads on staging will now proceed with fresh data.
|
|
# The Eagle server automatically detects the flush marker update and invalidates any stale cached games.
|
|
log_info "[DEPLOY:${deploy_id}] Step 6: Creating flush marker..."
|
|
create_flush_marker "${deploy_id}" "eagle-${staging}"
|
|
log_info "[DEPLOY:${deploy_id}] Flush marker created - server will auto-invalidate stale cache"
|
|
|
|
# Write active instance file for eagle-exec helper
|
|
echo "eagle-${staging}" > "${ACTIVE_INSTANCE_FILE}"
|
|
log_info "[DEPLOY:${deploy_id}] Active instance file updated: eagle-${staging}"
|
|
|
|
# Update .env for admin service
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ "$staging" = "green" ]; then
|
|
log_info "Updating .env for green instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
|
|
else
|
|
log_info "Updating .env for blue instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
|
|
fi
|
|
|
|
# Restart admin to pick up new .env and ensure latest image
|
|
# Use --no-deps to prevent cascading to auth (which has secrets not available here)
|
|
log_info "Restarting admin service..."
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate --no-deps admin
|
|
|
|
# Verify admin container is running
|
|
sleep 3
|
|
if ! docker inspect admin-server --format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
|
|
log_error "Admin container failed to start!"
|
|
log_error "Container logs:"
|
|
docker logs admin-server --tail 20 2>&1 || true
|
|
log_error "Container inspect:"
|
|
docker inspect admin-server 2>&1 | head -50 || true
|
|
exit 1
|
|
fi
|
|
log_info "Admin service restarted successfully"
|
|
|
|
# Clean up old instance
|
|
log_info "Cleaning up old eagle-${active}..."
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" 2>/dev/null || true
|
|
|
|
# Stop the old jfr-sidecar (it can't attach to removed container anyway)
|
|
if [ "$active" = "green" ]; then
|
|
docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar-green" 2>/dev/null || true
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null || true
|
|
else
|
|
docker compose -f "${COMPOSE_FILE}" stop "jfr-sidecar" 2>/dev/null || true
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null || true
|
|
fi
|
|
|
|
local deploy_end_time
|
|
deploy_end_time=$(date +%s)
|
|
local total_duration=$((deploy_end_time - deploy_start_time))
|
|
local user_wait_window=$((flush_end - nginx_switch_end))
|
|
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "[DEPLOY:${deploy_id}] Deployment complete!"
|
|
log_info " Active instance: eagle-${staging}"
|
|
log_info " Total duration: ${total_duration}s"
|
|
log_info " Flush duration: ${flush_duration}s"
|
|
log_info " Max user wait window: ${user_wait_window}s"
|
|
log_info "========================================="
|
|
}
|
|
|
|
# Check for required tools
|
|
check_requirements() {
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "docker is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v sed &> /dev/null; then
|
|
log_error "sed is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${NGINX_CONF}" ]; then
|
|
log_error "nginx config not found at ${NGINX_CONF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${COMPOSE_FILE}" ]; then
|
|
log_error "docker-compose file not found at ${COMPOSE_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# Ensure saves directory exists
|
|
if [ ! -d "${SAVES_DIR}" ]; then
|
|
log_info "Creating saves directory at ${SAVES_DIR}"
|
|
mkdir -p "${SAVES_DIR}"
|
|
fi
|
|
|
|
# Clean up any stale deployment-in-progress marker from a previous failed deploy
|
|
if [ -f "${DEPLOYMENT_IN_PROGRESS}" ]; then
|
|
log_warn "Found stale deployment-in-progress marker, removing it"
|
|
remove_stale_deployment_marker
|
|
fi
|
|
}
|
|
|
|
# Run
|
|
check_requirements
|
|
main "$@"
|