mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 22:55:41 +00:00
6fb14ef9af
Problems fixed: 1. CI was recreating admin BEFORE blue-green, causing stale .env 2. CI was restarting nginx AFTER blue-green (double restart) 3. Deploy script used slow nginx restart instead of reload 4. Cleanup was blocking the critical path Changes: - CI: Only restart shardok before blue-green, let script handle rest - CI: Remove duplicate nginx restart and fallback path - Deploy: Use nginx reload (faster) with restart fallback - Deploy: Update .env before restarting services - Deploy: Run container cleanup in background 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
360 lines
12 KiB
Bash
Executable File
360 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Blue-Green Deployment Script for Eagle Server
|
|
#
|
|
# This script performs a zero-downtime deployment by:
|
|
# 1. Starting the new version on a staging port (green)
|
|
# 2. Waiting for it to become healthy
|
|
# 3. Running warmup traffic to pre-heat the JIT
|
|
# 4. Stopping the old version (blue) - which flushes state to storage
|
|
# 5. Switching nginx to route traffic to green
|
|
# 6. Users reconnect, triggering lazy game loading from fresh storage
|
|
# 7. Cleaning up
|
|
#
|
|
# Usage: ./deploy-blue-green.sh [NEW_IMAGE_TAG]
|
|
#
|
|
# Example:
|
|
# ./deploy-blue-green.sh latest
|
|
# ./deploy-blue-green.sh sha-abc123
|
|
#
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
APP_DIR="${APP_DIR:-/opt/eagle0}"
|
|
NGINX_CONF="${APP_DIR}/nginx/nginx.conf"
|
|
COMPOSE_FILE="${APP_DIR}/docker-compose.prod.yml"
|
|
WARMUP_SCRIPT="${SCRIPT_DIR}/warmup-eagle.sh"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
|
|
# GitHub raw URL base for fetching config files
|
|
GITHUB_RAW_BASE="https://raw.githubusercontent.com/nolen777/eagle0/main"
|
|
|
|
# Sync config files from GitHub to ensure we have the latest versions
|
|
sync_config_files() {
|
|
log_info "Syncing config files from GitHub..."
|
|
|
|
# Backup and update docker-compose.prod.yml
|
|
if curl -fsSL "${GITHUB_RAW_BASE}/docker-compose.prod.yml" -o "${COMPOSE_FILE}.new"; then
|
|
cp "${COMPOSE_FILE}" "${COMPOSE_FILE}.bak"
|
|
mv "${COMPOSE_FILE}.new" "${COMPOSE_FILE}"
|
|
log_info "Updated docker-compose.prod.yml"
|
|
else
|
|
log_warn "Failed to fetch docker-compose.prod.yml, using existing"
|
|
fi
|
|
|
|
# Backup and update nginx.conf, preserving current active instance
|
|
if curl -fsSL "${GITHUB_RAW_BASE}/nginx/nginx.conf" -o "${NGINX_CONF}.new"; then
|
|
# Preserve the current eagle_backend setting (which instance is active)
|
|
local current_backend
|
|
current_backend=$(grep -o 'default "eagle-[a-z]*:40032"' "${NGINX_CONF}" | head -1 || echo 'default "eagle-blue:40032"')
|
|
|
|
cp "${NGINX_CONF}" "${NGINX_CONF}.bak"
|
|
# Update the new config with the current active instance
|
|
sed -i "s/default \"eagle-blue:40032\"/${current_backend}/g" "${NGINX_CONF}.new"
|
|
mv "${NGINX_CONF}.new" "${NGINX_CONF}"
|
|
log_info "Updated nginx.conf (preserved active: ${current_backend})"
|
|
else
|
|
log_warn "Failed to fetch nginx.conf, using existing"
|
|
fi
|
|
}
|
|
|
|
# Determine which instance is currently active
|
|
get_active_instance() {
|
|
if grep -q "eagle-blue:40032" "${NGINX_CONF}"; then
|
|
echo "blue"
|
|
elif grep -q "eagle-green:40032" "${NGINX_CONF}"; then
|
|
echo "green"
|
|
else
|
|
log_error "Cannot determine active instance from nginx config"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Pull image with retry using crane (handles OCI/Docker digest mismatch)
|
|
pull_with_retry() {
|
|
local image=$1
|
|
local max_attempts=${2:-3}
|
|
local attempt=1
|
|
|
|
# Use crane if available (handles OCI format correctly)
|
|
if [ -x "${APP_DIR}/crane" ]; then
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image with crane (attempt ${attempt}/${max_attempts})..."
|
|
if "${APP_DIR}/crane" pull "${image}" /tmp/image.tar && docker load -i /tmp/image.tar; then
|
|
rm -f /tmp/image.tar
|
|
log_info "Image pulled and loaded successfully"
|
|
return 0
|
|
fi
|
|
rm -f /tmp/image.tar
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
else
|
|
# Fallback to docker pull if crane not available
|
|
log_warn "crane not found at ${APP_DIR}/crane, falling back to docker pull"
|
|
while [ $attempt -le $max_attempts ]; do
|
|
log_info "Pulling image (attempt ${attempt}/${max_attempts})..."
|
|
if docker pull "${image}"; then
|
|
log_info "Image pulled successfully"
|
|
return 0
|
|
fi
|
|
log_warn "Pull failed, retrying in 5 seconds..."
|
|
sleep 5
|
|
attempt=$((attempt + 1))
|
|
done
|
|
fi
|
|
log_error "Failed to pull image after ${max_attempts} attempts"
|
|
return 1
|
|
}
|
|
|
|
# Wait for a container to be healthy
|
|
wait_for_healthy() {
|
|
local container=$1
|
|
local max_attempts=${2:-60}
|
|
local attempt=1
|
|
|
|
log_info "Waiting for ${container} to become healthy..."
|
|
while [ $attempt -le $max_attempts ]; do
|
|
health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
|
|
if [ "$health" = "healthy" ]; then
|
|
log_info "${container} is healthy"
|
|
return 0
|
|
fi
|
|
echo -n "."
|
|
sleep 2
|
|
attempt=$((attempt + 1))
|
|
done
|
|
echo ""
|
|
log_error "${container} did not become healthy after $((max_attempts * 2)) seconds"
|
|
return 1
|
|
}
|
|
|
|
# Main deployment logic
|
|
main() {
|
|
local new_tag="${1:-latest}"
|
|
local registry="registry.digitalocean.com/eagle0/eagle-server"
|
|
local new_image="${registry}:${new_tag}"
|
|
|
|
log_info "Starting blue-green deployment"
|
|
log_info "New image: ${new_image}"
|
|
|
|
cd "${APP_DIR}"
|
|
|
|
# Sync config files from GitHub before deployment
|
|
sync_config_files
|
|
|
|
# Determine current active instance
|
|
local active=$(get_active_instance)
|
|
local staging
|
|
if [ "$active" = "blue" ]; then
|
|
staging="green"
|
|
else
|
|
staging="blue"
|
|
fi
|
|
|
|
log_info "Active instance: eagle-${active}"
|
|
log_info "Staging instance: eagle-${staging}"
|
|
|
|
# Pull the new image (with retry for intermittent registry issues)
|
|
if ! pull_with_retry "${new_image}" 3; then
|
|
log_error "Failed to pull new image, aborting deployment"
|
|
exit 1
|
|
fi
|
|
|
|
# Start staging instance with new image (and its jfr-sidecar)
|
|
log_info "Starting eagle-${staging} with new image..."
|
|
if [ "$staging" = "green" ]; then
|
|
EAGLE_IMAGE_NEW="${new_image}" docker compose -f "${COMPOSE_FILE}" --profile blue-green up -d eagle-green jfr-sidecar-green
|
|
else
|
|
EAGLE_IMAGE="${new_image}" docker compose -f "${COMPOSE_FILE}" up -d eagle-blue jfr-sidecar
|
|
fi
|
|
|
|
# Wait for staging to be healthy
|
|
if ! wait_for_healthy "eagle-${staging}" 90; then
|
|
log_error "Staging instance failed health check, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${staging}"
|
|
exit 1
|
|
fi
|
|
|
|
# Run warmup/smoke test
|
|
local staging_port
|
|
if [ "$staging" = "green" ]; then
|
|
staging_port=40034
|
|
else
|
|
staging_port=40032
|
|
fi
|
|
|
|
log_info "Running warmup against eagle-${staging}..."
|
|
if [ -x "${WARMUP_SCRIPT}" ]; then
|
|
if ! "${WARMUP_SCRIPT}" "localhost:${staging_port}"; then
|
|
log_error "Warmup/smoke test failed, aborting deployment"
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${staging}"
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${staging}"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_warn "Warmup script not found at ${WARMUP_SCRIPT}, skipping warmup"
|
|
log_warn "JIT will be cold on first requests"
|
|
fi
|
|
|
|
# Backup games.e0es BEFORE stopping the active instance
|
|
# Critical: The old server may wipe games.e0es on shutdown if it doesn't have the merge fix
|
|
local backup_timestamp
|
|
backup_timestamp=$(date +%Y%m%d_%H%M%S)
|
|
log_info "Creating S3 backup of games.e0es (backup_${backup_timestamp})..."
|
|
if command -v s3cmd &> /dev/null; then
|
|
if s3cmd put "${APP_DIR}/saves/games.e0es" "s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}" 2>/dev/null; then
|
|
log_info "Backup created: s3://eagle0/eagle/save/backups/games.e0es.${backup_timestamp}"
|
|
else
|
|
log_warn "S3 backup failed (s3cmd put), continuing deployment"
|
|
fi
|
|
else
|
|
log_warn "s3cmd not found, skipping S3 backup"
|
|
fi
|
|
|
|
# Stop the active instance (flushes state to storage)
|
|
# We must do this BEFORE switching so staging loads fresh data
|
|
log_info "Stopping eagle-${active} (flushing state to storage)..."
|
|
docker compose -f "${COMPOSE_FILE}" stop "eagle-${active}"
|
|
|
|
# Prepare nginx config and .env BEFORE restarting anything
|
|
log_info "Switching nginx upstream to eagle-${staging}..."
|
|
if [ "$staging" = "green" ]; then
|
|
sed -i.bak 's/eagle-blue:40032/eagle-green:40032/g' "${NGINX_CONF}"
|
|
else
|
|
sed -i.bak 's/eagle-green:40032/eagle-blue:40032/g' "${NGINX_CONF}"
|
|
fi
|
|
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ "$staging" = "green" ]; then
|
|
log_info "Updating .env for green instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-green:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar-green:8081" >> "${env_file}"
|
|
else
|
|
log_info "Updating .env for blue instance..."
|
|
sed -i.bak '/^EAGLE_ADDR=/d; /^JFR_SIDECAR_ADDR=/d' "${env_file}" 2>/dev/null || true
|
|
echo "EAGLE_ADDR=eagle-blue:40032" >> "${env_file}"
|
|
echo "JFR_SIDECAR_ADDR=jfr-sidecar:8081" >> "${env_file}"
|
|
fi
|
|
|
|
# Restart nginx (use reload for faster switch, restart if reload fails)
|
|
log_info "Reloading nginx..."
|
|
if ! docker exec nginx nginx -s reload 2>/dev/null; then
|
|
log_warn "Reload failed, restarting nginx..."
|
|
docker compose -f "${COMPOSE_FILE}" restart nginx
|
|
fi
|
|
|
|
# Restart admin to pick up new .env
|
|
log_info "Restarting admin service..."
|
|
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate admin
|
|
|
|
# Clean up old instance and its jfr-sidecar (in background, not critical path)
|
|
log_info "Cleaning up old eagle-${active} container..."
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "eagle-${active}" &
|
|
if [ "$active" = "green" ]; then
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar-green" 2>/dev/null &
|
|
else
|
|
docker compose -f "${COMPOSE_FILE}" rm -f "jfr-sidecar" 2>/dev/null &
|
|
fi
|
|
wait
|
|
|
|
log_info "Deployment complete!"
|
|
log_info "Active instance: eagle-${staging}"
|
|
}
|
|
|
|
# Ensure s3cmd is installed and configured for S3 backups
|
|
ensure_s3cmd() {
|
|
# Install s3cmd if not present
|
|
if ! command -v s3cmd &> /dev/null; then
|
|
log_info "Installing s3cmd for S3 backups..."
|
|
if command -v pip3 &> /dev/null; then
|
|
pip3 install --quiet s3cmd
|
|
elif command -v pip &> /dev/null; then
|
|
pip install --quiet s3cmd
|
|
elif command -v apt-get &> /dev/null; then
|
|
sudo apt-get update -qq && sudo apt-get install -y -qq s3cmd
|
|
else
|
|
log_warn "Cannot install s3cmd (no pip or apt-get), S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
|
|
if ! command -v s3cmd &> /dev/null; then
|
|
log_warn "s3cmd installation failed, S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
log_info "s3cmd installed successfully"
|
|
fi
|
|
|
|
# Configure s3cmd for DigitalOcean Spaces if not already configured
|
|
local s3cfg="${HOME}/.s3cfg"
|
|
if [ ! -f "${s3cfg}" ]; then
|
|
# Load credentials from .env
|
|
local env_file="${APP_DIR}/.env"
|
|
if [ -f "${env_file}" ]; then
|
|
# shellcheck source=/dev/null
|
|
source "${env_file}"
|
|
fi
|
|
|
|
if [ -z "${DO_SPACES_ACCESS_KEY:-}" ] || [ -z "${DO_SPACES_SECRET_KEY:-}" ]; then
|
|
log_warn "DO_SPACES_ACCESS_KEY or DO_SPACES_SECRET_KEY not set, S3 backups will be skipped"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Configuring s3cmd for DigitalOcean Spaces..."
|
|
cat > "${s3cfg}" << EOF
|
|
[default]
|
|
access_key = ${DO_SPACES_ACCESS_KEY}
|
|
secret_key = ${DO_SPACES_SECRET_KEY}
|
|
host_base = sfo3.digitaloceanspaces.com
|
|
host_bucket = %(bucket)s.sfo3.digitaloceanspaces.com
|
|
use_https = True
|
|
EOF
|
|
log_info "s3cmd configured successfully"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Check for required tools
|
|
check_requirements() {
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "docker is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v sed &> /dev/null; then
|
|
log_error "sed is required but not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${NGINX_CONF}" ]; then
|
|
log_error "nginx config not found at ${NGINX_CONF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${COMPOSE_FILE}" ]; then
|
|
log_error "docker-compose file not found at ${COMPOSE_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
# Ensure s3cmd is available for backups (non-fatal if it fails)
|
|
ensure_s3cmd || true
|
|
}
|
|
|
|
# Run
|
|
check_requirements
|
|
main "$@"
|