Add production deployment pipeline (#4790)

* Next steps for productionization

* Run deploy job on self-hosted runner for secure SSH

* Temporarily disable production environment to debug runner

* Use ubuntu-latest for deploy job

* Add remote_tags to oci_push for latest tag

.github/workflows/docker_build.yml

@@ -82,3 +82,46 @@ jobs:
         # Note: Don't use --platforms here. The image is already built for Linux,
         # but the push script runs on the host (macOS) and needs native tools.
         run: bazel run //ci:shardok_server_push
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [build-eagle, build-shardok]
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_images == 'true')
+    environment: production
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Copy config files to droplet
+        uses: appleboy/scp-action@v0.1.7
+        with:
+          host: ${{ secrets.DO_DROPLET_IP }}
+          username: deploy
+          key: ${{ secrets.DO_SSH_KEY }}
+          source: "docker-compose.prod.yml,nginx/nginx.conf"
+          target: "/opt/eagle0"
+
+      - name: Deploy to production droplet
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.DO_DROPLET_IP }}
+          username: deploy
+          key: ${{ secrets.DO_SSH_KEY }}
+          script: |
+            cd /opt/eagle0
+
+            # Login to registry
+            echo "${{ secrets.DO_REGISTRY_TOKEN }}" | docker login registry.digitalocean.com -u "${{ secrets.DO_REGISTRY_TOKEN }}" --password-stdin
+
+            # Pull latest images
+            docker compose -f docker-compose.prod.yml pull
+
+            # Restart services
+            docker compose -f docker-compose.prod.yml up -d --remove-orphans
+
+            # Wait for health checks
+            sleep 10
+            docker compose -f docker-compose.prod.yml ps
+
+            # Cleanup old images
+            docker image prune -f

ci/BUILD.bazel

@@ -62,6 +62,7 @@ oci_load(
 oci_push(
     name = "eagle_server_push",
     image = ":eagle_server_image",
+    remote_tags = ["latest"],
     repository = "registry.digitalocean.com/eagle0/eagle-server",
 )
 
@@ -124,5 +125,6 @@ oci_load(
 oci_push(
     name = "shardok_server_push",
     image = ":shardok_server_image",
+    remote_tags = ["latest"],
     repository = "registry.digitalocean.com/eagle0/shardok-server",
 )

docker-compose.prod.yml

@@ -1,47 +1,87 @@
-# Docker Compose for local testing of production images
+# Docker Compose for production deployment
-# Build images: bazel run //ci:eagle_server_load && bazel run //ci:shardok_server_load
+#
-# Run: docker compose -f docker-compose.prod.yml up
+# Local testing:
+#   Build images: bazel run //ci:eagle_server_load && bazel run //ci:shardok_server_load
+#   Run: docker compose -f docker-compose.prod.yml up
+#
+# Production deployment:
+#   Run: docker compose -f docker-compose.prod.yml up -d
 
 services:
   eagle:
-    image: eagle0/eagle-server:latest
+    image: ${EAGLE_IMAGE:-registry.digitalocean.com/eagle0/eagle-server:latest}
     container_name: eagle-server
     ports:
       - "40032:40032"
     environment:
-      # Eagle server configuration
       EAGLE_GRPC_PORT: "40032"
       SHARDOK_HOST: "shardok"
       SHARDOK_PORT: "40042"
-      # Resource paths (relative to /app in container)
       EAGLE_RESOURCES_PATH: "/app/resources"
+      OPENAI_API_KEY: "${OPENAI_API_KEY:-}"
+      GPT_MODEL_NAME: "${GPT_MODEL_NAME:-gpt-4o}"
     volumes:
-      # Mount saves directory for persistence
       - ./saves:/app/saves
     depends_on:
       - shardok
     restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "5"
     healthcheck:
-      test: ["CMD", "nc", "-z", "localhost", "40032"]
+      test: ["CMD-SHELL", "nc -z localhost 40032 || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3
       start_period: 30s
 
   shardok:
-    image: eagle0/shardok-server:latest
+    image: ${SHARDOK_IMAGE:-registry.digitalocean.com/eagle0/shardok-server:latest}
     container_name: shardok-server
     ports:
       - "40042:40042"
       - "40052:40052"
     environment:
-      # Shardok server configuration
       SHARDOK_RESOURCES_PATH: "/app/resources"
       SHARDOK_MAPS_PATH: "/app/resources/maps"
     restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "5"
     healthcheck:
-      test: ["CMD", "nc", "-z", "localhost", "40042"]
+      test: ["CMD-SHELL", "nc -z localhost 40042 || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3
       start_period: 10s
+
+  nginx:
+    image: nginx:alpine
+    container_name: nginx
+    ports:
+      - "443:443"
+      - "80:80"
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./certbot/conf:/etc/letsencrypt:ro
+      - ./certbot/www:/var/www/certbot:ro
+    depends_on:
+      - eagle
+    restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "50m"
+        max-file: "3"
+
+  certbot:
+    image: certbot/certbot
+    container_name: certbot
+    volumes:
+      - ./certbot/conf:/etc/letsencrypt
+      - ./certbot/www:/var/www/certbot
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"

nginx/nginx.conf

@@ -0,0 +1,96 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    # Logging
+    log_format grpc_json escape=json '{'
+        '"time":"$time_iso8601",'
+        '"client":"$remote_addr",'
+        '"uri":"$uri",'
+        '"status":$status,'
+        '"grpc_status":"$sent_http_grpc_status",'
+        '"request_time":$request_time,'
+        '"upstream_time":"$upstream_response_time"'
+    '}';
+
+    access_log /var/log/nginx/access.log grpc_json;
+    error_log /var/log/nginx/error.log warn;
+
+    # Rate limiting zone
+    limit_req_zone $binary_remote_addr zone=grpc_limit:10m rate=100r/s;
+
+    # Upstream for Eagle gRPC server
+    upstream eagle_grpc {
+        server eagle:40032;
+        keepalive 100;
+    }
+
+    # HTTP server for Let's Encrypt challenge and redirect
+    server {
+        listen 80;
+        server_name eagle0.net;
+
+        # Let's Encrypt challenge
+        location /.well-known/acme-challenge/ {
+            root /var/www/certbot;
+        }
+
+        # Redirect all other HTTP to HTTPS
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    # HTTPS server for gRPC
+    server {
+        listen 443 ssl;
+        http2 on;
+        server_name eagle0.net;
+
+        # SSL certificates (managed by certbot)
+        ssl_certificate /etc/letsencrypt/live/eagle0.net/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/eagle0.net/privkey.pem;
+
+        # SSL configuration
+        ssl_protocols TLSv1.2 TLSv1.3;
+        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
+        ssl_prefer_server_ciphers off;
+        ssl_session_timeout 1d;
+        ssl_session_cache shared:SSL:10m;
+        ssl_session_tickets off;
+
+        # gRPC proxy for Eagle service
+        location /net.eagle0.eagle.api.Eagle {
+            # Rate limiting
+            limit_req zone=grpc_limit burst=50 nodelay;
+
+            # gRPC proxy
+            grpc_pass grpc://eagle_grpc;
+
+            # Timeouts for long-running streams
+            grpc_read_timeout 1200s;
+            grpc_send_timeout 1200s;
+            grpc_socket_keepalive on;
+
+            # Error handling
+            error_page 502 = /error502grpc;
+        }
+
+        # Health check endpoint
+        location /health {
+            access_log off;
+            return 200 "OK\n";
+            add_header Content-Type text/plain;
+        }
+
+        # gRPC error handling
+        location = /error502grpc {
+            internal;
+            default_type application/grpc;
+            add_header grpc-status 14;
+            add_header grpc-message "unavailable";
+            return 204;
+        }
+    }
+}

scripts/setup_droplet.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+#
+# Setup script for Eagle0 production droplet
+# Run this on a fresh DigitalOcean droplet (Ubuntu 24.04)
+#
+# Usage: curl -sSL https://raw.githubusercontent.com/nolen777/eagle0/main/scripts/setup_droplet.sh | sudo bash
+#
+
+set -euo pipefail
+
+DOMAIN="${DOMAIN:-eagle0.net}"
+DEPLOY_USER="${DEPLOY_USER:-deploy}"
+APP_DIR="/opt/eagle0"
+
+echo "=== Eagle0 Production Server Setup ==="
+echo "Domain: ${DOMAIN}"
+echo "Deploy user: ${DEPLOY_USER}"
+echo ""
+
+# Check if running as root
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root (use sudo)"
+   exit 1
+fi
+
+echo "=== Updating system ==="
+apt-get update
+apt-get upgrade -y
+
+echo "=== Installing Docker ==="
+if ! command -v docker &> /dev/null; then
+    curl -fsSL https://get.docker.com | sh
+    systemctl enable docker
+    systemctl start docker
+else
+    echo "Docker already installed"
+fi
+
+echo "=== Installing Docker Compose plugin ==="
+apt-get install -y docker-compose-plugin
+
+echo "=== Installing additional utilities ==="
+apt-get install -y \
+    curl \
+    wget \
+    git \
+    netcat-openbsd \
+    jq \
+    htop \
+    unattended-upgrades
+
+echo "=== Configuring automatic security updates ==="
+cat > /etc/apt/apt.conf.d/20auto-upgrades << 'EOF'
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Unattended-Upgrade "1";
+APT::Periodic::AutocleanInterval "7";
+EOF
+
+echo "=== Creating deploy user ==="
+if ! id "${DEPLOY_USER}" &>/dev/null; then
+    useradd -m -s /bin/bash -G docker "${DEPLOY_USER}"
+    mkdir -p "/home/${DEPLOY_USER}/.ssh"
+    chmod 700 "/home/${DEPLOY_USER}/.ssh"
+    chown -R "${DEPLOY_USER}:${DEPLOY_USER}" "/home/${DEPLOY_USER}/.ssh"
+    echo ""
+    echo "*** IMPORTANT: Add your SSH public key to /home/${DEPLOY_USER}/.ssh/authorized_keys ***"
+    echo ""
+else
+    echo "User ${DEPLOY_USER} already exists"
+    # Ensure user is in docker group
+    usermod -aG docker "${DEPLOY_USER}"
+fi
+
+echo "=== Creating application directory ==="
+mkdir -p "${APP_DIR}"/{nginx,certbot/conf,certbot/www,saves}
+chown -R "${DEPLOY_USER}:${DEPLOY_USER}" "${APP_DIR}"
+
+echo "=== Configuring Docker registry authentication ==="
+echo ""
+echo "*** IMPORTANT: Run the following command to authenticate with DigitalOcean Container Registry: ***"
+echo "  docker login registry.digitalocean.com"
+echo ""
+
+echo "=== Creating systemd service ==="
+cat > /etc/systemd/system/eagle0.service << EOF
+[Unit]
+Description=Eagle0 Game Servers
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+WorkingDirectory=${APP_DIR}
+ExecStart=/usr/bin/docker compose -f docker-compose.prod.yml up -d
+ExecStop=/usr/bin/docker compose -f docker-compose.prod.yml down
+User=${DEPLOY_USER}
+Group=${DEPLOY_USER}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable eagle0
+
+echo "=== Configuring firewall (UFW) ==="
+if ! command -v ufw &> /dev/null; then
+    apt-get install -y ufw
+fi
+
+ufw default deny incoming
+ufw default allow outgoing
+ufw allow ssh
+ufw allow 80/tcp
+ufw allow 443/tcp
+ufw --force enable
+
+echo "=== Setting up log rotation ==="
+cat > /etc/logrotate.d/eagle0 << EOF
+/var/log/eagle0/*.log {
+    daily
+    missingok
+    rotate 14
+    compress
+    delaycompress
+    notifempty
+    create 0640 ${DEPLOY_USER} ${DEPLOY_USER}
+    sharedscripts
+}
+EOF
+
+mkdir -p /var/log/eagle0
+chown "${DEPLOY_USER}:${DEPLOY_USER}" /var/log/eagle0
+
+echo ""
+echo "=== Setup Complete ==="
+echo ""
+echo "Next steps:"
+echo "1. Add SSH public key to /home/${DEPLOY_USER}/.ssh/authorized_keys"
+echo "2. Copy docker-compose.prod.yml to ${APP_DIR}/"
+echo "3. Copy nginx/nginx.conf to ${APP_DIR}/nginx/"
+echo "4. Create .env file in ${APP_DIR}/ with OPENAI_API_KEY"
+echo "5. Run: docker login registry.digitalocean.com"
+echo "6. Get SSL certificate: (see init_ssl.sh)"
+echo "7. Start services: systemctl start eagle0"
+echo ""
+echo "Server IP: $(curl -s ifconfig.me)"
+echo ""