mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-29 00:35:41 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a314208fdc | ||
|
|
7239a11254 |
@@ -0,0 +1,89 @@
|
||||
# Docker Compose for Shardok-AI server deployment (Hetzner)
|
||||
#
|
||||
# This configuration runs Shardok in AI-only mode, providing AI computation
|
||||
# services for Shardok-Primary instances running on DigitalOcean.
|
||||
#
|
||||
# Architecture:
|
||||
# Unity Client -> Eagle -> Shardok-Primary (DO) -> Shardok-AI (Hetzner)
|
||||
# - Shardok-Primary handles all command processing (low latency for humans)
|
||||
# - Shardok-AI handles AI computation only (high throughput for AI moves)
|
||||
#
|
||||
# Deployment:
|
||||
# 1. Copy this file to the Hetzner server
|
||||
# 2. Set environment variables (see below)
|
||||
# 3. Run: docker compose -f docker-compose.ai.yml up -d
|
||||
#
|
||||
# Required environment variables:
|
||||
# SHARDOK_AI_AUTH_TOKEN_PATH - Path to file containing auth token
|
||||
|
||||
services:
|
||||
shardok-ai:
|
||||
image: ${SHARDOK_IMAGE:-registry.digitalocean.com/eagle0/shardok-server:arm64-latest}
|
||||
container_name: shardok-ai-server
|
||||
mem_limit: 4g # More memory for AI computation
|
||||
memswap_limit: 4g
|
||||
ports:
|
||||
- "40043:40043" # AI service port
|
||||
environment:
|
||||
SHARDOK_RESOURCES_PATH: "/app/resources"
|
||||
SHARDOK_MAPS_PATH: "/app/resources/maps"
|
||||
# Disable Eagle interface - this is AI-only mode
|
||||
SHARDOK_EAGLE_INTERFACE_ADDRESS: ""
|
||||
# Enable AI service
|
||||
SHARDOK_AI_SERVICE_ADDRESS: "0.0.0.0:40043"
|
||||
# Auth token for validating requests from Shardok-Primary
|
||||
SHARDOK_AUTH_TOKEN_PATH: "${SHARDOK_AI_AUTH_TOKEN_PATH:-/etc/eagle0/auth/shardok-ai.token}"
|
||||
# TLS configuration (optional but recommended for production)
|
||||
SHARDOK_SSL_CERT_PATH: "${SHARDOK_SSL_CERT_PATH:-}"
|
||||
SHARDOK_SSL_PRIVATE_KEY_PATH: "${SHARDOK_SSL_PRIVATE_KEY_PATH:-}"
|
||||
volumes:
|
||||
- ./auth:/etc/eagle0/auth:ro # Auth token
|
||||
- ./certbot/conf:/etc/letsencrypt:ro # TLS certificates
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "100m"
|
||||
max-file: "5"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "nc -z localhost 40043 || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
# Optional: nginx for TLS termination and rate limiting
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
container_name: nginx
|
||||
ports:
|
||||
- "443:443"
|
||||
volumes:
|
||||
- ./nginx/nginx-ai.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./certbot/conf:/etc/letsencrypt:ro
|
||||
depends_on:
|
||||
- shardok-ai
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
|
||||
# Optional: certbot for TLS certificate renewal
|
||||
certbot:
|
||||
image: certbot/certbot
|
||||
container_name: certbot
|
||||
volumes:
|
||||
- ./certbot/conf:/etc/letsencrypt
|
||||
- ./certbot/www:/var/www/certbot
|
||||
entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
|
||||
|
||||
networks:
|
||||
default:
|
||||
driver: bridge
|
||||
enable_ipv6: true
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.29.0.0/16
|
||||
- subnet: fd00:dead:cafe::/48
|
||||
@@ -110,6 +110,13 @@ services:
|
||||
SHARDOK_RESOURCES_PATH: "/app/resources"
|
||||
SHARDOK_MAPS_PATH: "/app/resources/maps"
|
||||
SHARDOK_EAGLE_INTERFACE_ADDRESS: "0.0.0.0:40042"
|
||||
# Remote AI offload configuration (optional)
|
||||
# When set, AI computation is offloaded to a remote server for reduced latency
|
||||
SHARDOK_REMOTE_AI_ADDRESS: "${SHARDOK_REMOTE_AI_ADDRESS:-}"
|
||||
SHARDOK_REMOTE_AI_AUTH_TOKEN_PATH: "${SHARDOK_REMOTE_AI_AUTH_TOKEN_PATH:-}"
|
||||
SHARDOK_REMOTE_AI_TIMEOUT_MS: "${SHARDOK_REMOTE_AI_TIMEOUT_MS:-10000}"
|
||||
volumes:
|
||||
- ./auth:/etc/eagle0/auth:ro # Auth token for remote AI
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: "json-file"
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
# Shardok Latency Hiding Strategies
|
||||
|
||||
## Problem Statement
|
||||
|
||||
With Shardok running on Hetzner (Helsinki) and Eagle on DigitalOcean (US), the round-trip latency for human commands is ~200-400ms:
|
||||
|
||||
```
|
||||
Human posts command:
|
||||
Unity → Eagle (DO) → Shardok (Hetzner) → Eagle (DO) → Unity
|
||||
~10ms ~100ms ~100ms ~10ms
|
||||
Total: ~220ms round-trip
|
||||
```
|
||||
|
||||
This latency is acceptable for AI turns (users watch animations anyway), but creates noticeable lag when humans post commands.
|
||||
|
||||
---
|
||||
|
||||
## Strategy 1: Client-Side Animation Masking
|
||||
|
||||
### Concept
|
||||
|
||||
Start animations immediately when the user clicks, before server confirmation arrives. The animation duration masks the network latency.
|
||||
|
||||
### Implementation by Command Type
|
||||
|
||||
**Movement Commands** (deterministic):
|
||||
- Client knows the destination hex and movement path
|
||||
- Start movement animation immediately on click
|
||||
- Server confirms the move (should always match)
|
||||
- If server rejects (invalid state), snap unit back to origin
|
||||
|
||||
**Attack Commands** (RNG-dependent):
|
||||
- Show attack animation immediately (unit swings sword, fires arrow)
|
||||
- Wait for server to return dice roll result
|
||||
- Show damage numbers / hit effects when server responds
|
||||
- Animation typically takes 300-500ms, masking most of the latency
|
||||
|
||||
**End Turn**:
|
||||
- Latency not noticeable (user expects transition delay)
|
||||
|
||||
### Unity Implementation Sketch
|
||||
|
||||
```csharp
|
||||
// In CommandHandler.cs
|
||||
public void OnCommandSelected(Command command) {
|
||||
// Start animation immediately
|
||||
if (command.Type == CommandType.Move) {
|
||||
unitController.StartMoveAnimation(command.TargetHex);
|
||||
} else if (command.Type == CommandType.Attack) {
|
||||
unitController.StartAttackAnimation(command.TargetUnit);
|
||||
}
|
||||
|
||||
// Send to server in parallel
|
||||
connection.SendCommand(command, (response) => {
|
||||
if (response.Success) {
|
||||
// Animation continues, apply result
|
||||
ApplyCommandResult(response);
|
||||
} else {
|
||||
// Rollback animation
|
||||
unitController.CancelAnimation();
|
||||
ShowError(response.ErrorMessage);
|
||||
}
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### Pros
|
||||
- Simple implementation
|
||||
- No server-side changes
|
||||
- Works with existing architecture
|
||||
|
||||
### Cons
|
||||
- Doesn't eliminate latency for attacks with RNG (must wait for dice roll)
|
||||
- Rollback needed if server rejects command (rare but possible)
|
||||
|
||||
### Estimated Improvement
|
||||
- Movement: ~200ms latency hidden (feels instant)
|
||||
- Attacks: ~100-200ms hidden by animation, ~100ms visible wait for dice result
|
||||
|
||||
---
|
||||
|
||||
## Strategy 2: Split Shardok Architecture
|
||||
|
||||
### Concept
|
||||
|
||||
Run two Shardok instances:
|
||||
- **Shardok-Primary (DigitalOcean)**: Handles command processing, source of truth
|
||||
- **Shardok-AI (Hetzner)**: AI computation only
|
||||
|
||||
Human commands go to the nearby Primary for low latency. AI computation uses the powerful Hetzner instance.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
Human commands (low latency ~20ms)
|
||||
Unity ←→ Eagle ←→ Shardok-Primary (DigitalOcean)
|
||||
↓ state sync (when AI turn starts)
|
||||
Shardok-AI (Hetzner)
|
||||
↑ AI command response
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
**Human Turn:**
|
||||
1. Human posts command → Eagle → Shardok-Primary (DO)
|
||||
2. Primary processes command immediately (~10ms local)
|
||||
3. Primary streams result to client via Eagle (~10ms)
|
||||
4. **Total latency: ~20ms** (vs ~220ms current)
|
||||
|
||||
**AI Turn:**
|
||||
1. When AI's turn starts, Primary sends game state snapshot to Hetzner
|
||||
2. Shardok-AI computes best command using full CPU power
|
||||
3. Shardok-AI returns command index to Primary
|
||||
4. Primary executes command locally and streams to client
|
||||
5. Repeat until AI turn ends
|
||||
|
||||
### Protocol Changes
|
||||
|
||||
```protobuf
|
||||
// New service for AI-only computation
|
||||
service ShardokAIService {
|
||||
// Send game state, receive AI's chosen command
|
||||
rpc GetAICommand(AICommandRequest) returns (AICommandResponse);
|
||||
}
|
||||
|
||||
message AICommandRequest {
|
||||
bytes game_state = 1; // Serialized game state
|
||||
int32 player_id = 2; // Which AI player
|
||||
repeated bytes available_commands = 3; // Available command descriptors
|
||||
}
|
||||
|
||||
message AICommandResponse {
|
||||
int32 command_index = 1; // Index into available_commands
|
||||
int32 search_depth = 2; // For debugging
|
||||
double best_score = 3; // For debugging
|
||||
}
|
||||
```
|
||||
|
||||
### Shardok-Primary Requirements
|
||||
|
||||
Shardok-Primary on DigitalOcean needs to:
|
||||
- Process all commands (human and AI)
|
||||
- Maintain authoritative game state
|
||||
- Serialize/deserialize game state for AI requests
|
||||
- Run on minimal CPU (command processing is fast)
|
||||
|
||||
This is essentially the current Shardok, but without running the AI search.
|
||||
|
||||
### Shardok-AI Requirements
|
||||
|
||||
Shardok-AI on Hetzner needs to:
|
||||
- Receive game state snapshots
|
||||
- Run AI evaluation (IterativeDeepeningAI or MCTS)
|
||||
- Return best command index
|
||||
- No persistent state (stateless worker)
|
||||
|
||||
### AI Turn Latency
|
||||
|
||||
Each AI command has ~200ms network latency. This is acceptable because:
|
||||
1. User is watching animations anyway
|
||||
2. Natural pacing lets user observe AI decisions
|
||||
3. AI computation is fast on Hetzner's 16 cores
|
||||
|
||||
For a typical AI turn with 5 commands: 5 × 200ms = 1 second network overhead, plus AI thinking time. With animations, this feels natural.
|
||||
|
||||
### Implementation Phases
|
||||
|
||||
**Phase 1: Add Shardok-Primary (minimal)**
|
||||
- Deploy existing Shardok container to DigitalOcean
|
||||
- Configure Eagle to use local Shardok for all commands
|
||||
- Human latency immediately improves
|
||||
|
||||
**Phase 2: Add AI offload**
|
||||
- Implement `ShardokAIService` RPC
|
||||
- Shardok-Primary calls Hetzner for AI commands
|
||||
- Shardok-AI processes requests statelessly
|
||||
|
||||
**Phase 3: Optimize**
|
||||
- Batch multiple AI actions if possible
|
||||
- Pre-warm Shardok-AI connection
|
||||
- Add fallback if Hetzner unavailable
|
||||
|
||||
### Pros
|
||||
- Human command latency drops from ~220ms to ~20ms
|
||||
- AI still gets Hetzner's CPU power
|
||||
- Clear separation of concerns
|
||||
- Shardok-Primary can fall back to local AI if Hetzner unavailable
|
||||
|
||||
### Cons
|
||||
- Two Shardok instances to maintain
|
||||
- State serialization overhead for AI requests
|
||||
- Each AI action has network round-trip (acceptable with animations)
|
||||
|
||||
---
|
||||
|
||||
## Comparison
|
||||
|
||||
| Approach | Human Latency | AI Throughput | Complexity | Changes Required |
|
||||
|----------|---------------|---------------|------------|------------------|
|
||||
| Current | ~220ms | High | - | - |
|
||||
| Animation masking | ~220ms (perceived ~50ms) | High | Low | Unity only |
|
||||
| Split architecture | ~20ms | High | Medium | New RPC, two deployments |
|
||||
|
||||
---
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Phase 1 (now)**: Implement animation masking in Unity client
|
||||
- Quick win, no server changes
|
||||
- Improves perceived latency significantly for movement
|
||||
- Attacks still show dice animation while waiting
|
||||
|
||||
**Phase 2 (future)**: Split architecture if animation masking insufficient
|
||||
- Only needed if users complain about attack latency
|
||||
- More complex but provides true low latency
|
||||
- Natural evolution of current architecture
|
||||
@@ -1,769 +0,0 @@
|
||||
# Shardok On-Demand Compute Infrastructure Analysis
|
||||
|
||||
## Overview
|
||||
|
||||
This document analyzes options for running Shardok (the tactical combat AI server) on powerful on-demand compute, separate from the low-power droplet that hosts Eagle.
|
||||
|
||||
**Current State**: Shardok runs on the same DigitalOcean droplet as Eagle, limiting AI performance.
|
||||
|
||||
**Goal**: Spin up powerful compute only when battles occur, keeping costs low while improving AI quality.
|
||||
|
||||
---
|
||||
|
||||
## Cloud Provider Pricing Comparison
|
||||
|
||||
### High-Performance CPU Options (8-16 cores)
|
||||
|
||||
| Provider | Instance Type | vCPUs | RAM | Hourly Cost | Monthly Est.* |
|
||||
|----------|--------------|-------|-----|-------------|---------------|
|
||||
| **Hetzner** | CCX23 | 8 | 32GB | €0.078 (~$0.085) | ~$6-8 |
|
||||
| **Hetzner** | CCX33 | 16 | 64GB | €0.155 (~$0.17) | ~$12-16 |
|
||||
| **Vultr** | High Frequency 8-core | 8 | 32GB | $0.238 | ~$20-25 |
|
||||
| **Vultr** | High Frequency 16-core | 16 | 64GB | $0.476 | ~$40-50 |
|
||||
| **DigitalOcean** | Premium CPU 8-core | 8 | 32GB | $0.250 | ~$22-28 |
|
||||
| **DigitalOcean** | Premium CPU 16-core | 16 | 64GB | $0.500 | ~$44-55 |
|
||||
|
||||
*Monthly estimates assume ~4 hours/week of battle usage (~17 hours/month)
|
||||
|
||||
### Key Finding: Hetzner is 3-5x Cheaper
|
||||
|
||||
Hetzner's dedicated vCPU instances offer the best price-performance ratio by a significant margin. For equivalent 16-core compute:
|
||||
- Hetzner: ~$0.17/hr
|
||||
- DigitalOcean: ~$0.50/hr (2.9x more expensive)
|
||||
- Vultr: ~$0.48/hr (2.8x more expensive)
|
||||
|
||||
---
|
||||
|
||||
## Instance Startup Time Analysis
|
||||
|
||||
### Current Startup Times by Provider
|
||||
|
||||
| Provider | Cold Start | Warm Image | Container Registry |
|
||||
|----------|-----------|------------|-------------------|
|
||||
| Hetzner | 30-60s | 20-40s | Native support |
|
||||
| Vultr | 30-45s | 20-30s | Native support |
|
||||
| DigitalOcean | 30-60s | 25-45s | Already in use |
|
||||
|
||||
### 10-Second Max Wait Time Constraint
|
||||
|
||||
A 10-second startup constraint eliminates traditional VM spin-up as an option. Alternatives:
|
||||
|
||||
#### Option 1: Pre-warmed Instance Pool (Recommended)
|
||||
- Keep one small instance running 24/7 as a "hot standby"
|
||||
- Cost: ~$5-10/month for a minimal Hetzner instance
|
||||
- Startup time: **< 1 second** (already running)
|
||||
- Upgrade on demand when battle starts (resize takes 2-3 min, but battle can start immediately on small instance)
|
||||
|
||||
#### Option 2: Predictive Spin-up
|
||||
- Start spinning up when user enters battle setup
|
||||
- Battle setup typically takes 30-60 seconds (map selection, army placement)
|
||||
- Could achieve "instant" availability if spin-up completes during setup
|
||||
- Risk: User might cancel, wasting spin-up cost (minimal at ~$0.01-0.02)
|
||||
|
||||
#### Option 3: Container-Based Quick Start
|
||||
- Use pre-pulled Docker images on a warm container host
|
||||
- Providers like Fly.io or Railway can start containers in 1-5 seconds
|
||||
- Trade-off: Container platforms typically cost more than raw VMs
|
||||
|
||||
#### Option 4: Serverless/Function Compute
|
||||
- Not viable for Shardok due to:
|
||||
- Long-running connections (battles last minutes)
|
||||
- High memory requirements
|
||||
- Stateful game state
|
||||
|
||||
### Recommendation for 10s Constraint
|
||||
|
||||
**Hybrid approach**: Keep a minimal "always-on" instance ($5-10/month) that can handle battles at reduced AI quality. When a battle is anticipated (entering battle setup), start spinning up a powerful instance. If the powerful instance is ready when the battle starts, use it; otherwise, fall back to the always-on instance.
|
||||
|
||||
---
|
||||
|
||||
## ARM Options Outside AWS
|
||||
|
||||
### Available ARM Compute
|
||||
|
||||
| Provider | ARM Offering | vCPUs | RAM | Hourly Cost | Notes |
|
||||
|----------|-------------|-------|-----|-------------|-------|
|
||||
| **Hetzner** | CAX21 | 4 | 8GB | €0.009 | Ampere Altra |
|
||||
| **Hetzner** | CAX31 | 8 | 16GB | €0.018 | Ampere Altra |
|
||||
| **Hetzner** | CAX41 | 16 | 32GB | €0.035 | Ampere Altra |
|
||||
| **Oracle Cloud** | A1 Flex | 4 | 24GB | Free tier | Always Free eligible |
|
||||
| **Oracle Cloud** | A1 Flex | 24 | 144GB | ~$0.10 | Ampere Altra |
|
||||
| **Scaleway** | AMP2-C8 | 8 | 16GB | €0.104 | Ampere Altra |
|
||||
|
||||
### Key Finding: Hetzner ARM is Extremely Cheap
|
||||
|
||||
Hetzner's ARM instances are 4-5x cheaper than their x86 equivalents:
|
||||
- CAX41 (16 ARM cores): €0.035/hr (~$0.04)
|
||||
- CCX33 (16 x86 cores): €0.155/hr (~$0.17)
|
||||
|
||||
### ARM Considerations for Shardok
|
||||
|
||||
**Pros:**
|
||||
- Significant cost savings (4-5x)
|
||||
- Good single-threaded performance on Ampere Altra
|
||||
- Native ARM builds already supported (macOS development)
|
||||
|
||||
**Cons:**
|
||||
- Requires ARM64 Linux build (currently only building x86_64 Linux)
|
||||
- Some performance tuning may be needed
|
||||
- Less mature ecosystem than x86
|
||||
|
||||
**Build Effort**: Medium. Bazel supports cross-compilation, and the codebase is portable C++. Main work is adding ARM64 Linux to the build matrix.
|
||||
|
||||
---
|
||||
|
||||
## Hetzner Geographic Presence
|
||||
|
||||
### Current Locations
|
||||
|
||||
Hetzner operates datacenters in:
|
||||
- **Germany**: Nuremberg (NBG1), Falkenstein (FSN1)
|
||||
- **Finland**: Helsinki (HEL1)
|
||||
- **USA**: Ashburn, Virginia (ASH) - **opened 2023**
|
||||
- **Singapore**: (coming 2024/2025)
|
||||
|
||||
### Latency Implications
|
||||
|
||||
For users primarily in North America:
|
||||
- **Hetzner US (Ashburn)**: 20-40ms to East Coast, 60-80ms to West Coast
|
||||
- **Hetzner EU**: 80-120ms from US East Coast, 140-180ms from US West Coast
|
||||
|
||||
Since Shardok handles AI computation (not real-time player input), latency is less critical. The Eagle-Shardok communication adds some overhead, but AI "thinking time" dominates.
|
||||
|
||||
### Recommendation
|
||||
|
||||
Use **Hetzner US (Ashburn)** for North American users. The US datacenter has the same pricing as EU datacenters and provides better latency.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Options Summary
|
||||
|
||||
### Option A: Simple On-Demand (30-60s startup)
|
||||
- Spin up Hetzner CCX33 when battle starts
|
||||
- Cost: ~$0.17/hr × 4 hrs/week = **~$3/month**
|
||||
- Latency: 30-60s wait at battle start
|
||||
- Complexity: Low
|
||||
|
||||
### Option B: Predictive Spin-up (0-30s startup)
|
||||
- Start spinning up during battle setup phase
|
||||
- Cost: Same as Option A, plus occasional wasted spin-ups (~$1/month)
|
||||
- Latency: 0-30s depending on setup duration
|
||||
- Complexity: Medium
|
||||
|
||||
### Option C: Hybrid Always-On + On-Demand (< 1s startup)
|
||||
- Keep minimal instance always running ($5-10/month)
|
||||
- Spin up powerful instance during battle setup
|
||||
- Use powerful if ready, fall back to minimal
|
||||
- Cost: **~$8-15/month**
|
||||
- Latency: < 1s (always-on fallback)
|
||||
- Complexity: Medium-High
|
||||
|
||||
### Option D: ARM Compute (Cheapest)
|
||||
- Build ARM64 Linux target
|
||||
- Use Hetzner CAX41 (16 ARM cores) at €0.035/hr
|
||||
- Cost: **~$0.60/month** for compute (+ always-on if needed)
|
||||
- Latency: Same as x86 options
|
||||
- Complexity: Medium (build system changes)
|
||||
|
||||
---
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Phase 1 (Quick Win)**: Implement Option B (Predictive Spin-up) with Hetzner CCX33 in Ashburn.
|
||||
- Minimal code changes (add spin-up trigger in Eagle when battle setup begins)
|
||||
- Good user experience (usually ready by battle start)
|
||||
- Low cost (~$3-5/month)
|
||||
|
||||
**Phase 2 (Future)**: Add ARM64 Linux build and switch to ARM instances.
|
||||
- 4-5x cost reduction
|
||||
- Same or better performance
|
||||
- Requires build system work
|
||||
|
||||
**Phase 3 (If Needed)**: Add always-on fallback for guaranteed instant start.
|
||||
- Only if Phase 1 startup times prove problematic
|
||||
- Adds ~$5-10/month fixed cost
|
||||
|
||||
---
|
||||
|
||||
# Implementation Plan
|
||||
|
||||
## Chosen Architecture
|
||||
|
||||
- **Provider**: Hetzner Cloud
|
||||
- **Instance**: CAX41 (16 ARM cores, 32GB RAM)
|
||||
- **Location**: Ashburn, Virginia (ash)
|
||||
- **Cost**: €0.035/hr (~$0.04/hr)
|
||||
- **Strategy**: On-demand with predictive spin-up and idle keep-alive
|
||||
|
||||
---
|
||||
|
||||
## System Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ DigitalOcean │
|
||||
│ ┌─────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Eagle Server │ │
|
||||
│ │ ┌─────────────────┐ ┌─────────────────┐ │ │
|
||||
│ │ │ ShardokManager │ │ ActivityTracker │ │ │
|
||||
│ │ │ - spin up/down │ │ - player active │ │ │
|
||||
│ │ │ - health check │ │ - trigger start │ │ │
|
||||
│ │ └────────┬────────┘ └────────┬────────┘ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ └────────┬───────────┘ │ │
|
||||
│ │ ▼ │ │
|
||||
│ │ ┌────────────────┐ │ │
|
||||
│ │ │ ShardokClient │◄─── TLS + Token Auth │ │
|
||||
│ │ └────────┬───────┘ │ │
|
||||
│ └────────────────────┼─────────────────────────────────────────┘ │
|
||||
└───────────────────────┼──────────────────────────────────────────────┘
|
||||
│ gRPC over TLS (Let's Encrypt)
|
||||
▼
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ Hetzner Cloud (Ashburn) │
|
||||
│ ┌─────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ CAX41 (On-Demand) │ │
|
||||
│ │ ┌─────────────────────────────────────────────────────────┐ │ │
|
||||
│ │ │ Shardok Server │ │ │
|
||||
│ │ │ - 16 ARM cores for AI computation │ │ │
|
||||
│ │ │ - TLS cert from Let's Encrypt (auto-renewed) │ │ │
|
||||
│ │ │ - Validates auth token on each request │ │ │
|
||||
│ │ └─────────────────────────────────────────────────────────┘ │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Instance Lifecycle Management
|
||||
|
||||
### State Machine
|
||||
|
||||
```
|
||||
┌──────────────┐
|
||||
│ STOPPED │◄────────────────────┐
|
||||
└──────┬───────┘ │
|
||||
│ human takes turn │ idle timeout
|
||||
▼ │ (10 minutes)
|
||||
┌──────────────┐ │
|
||||
│ STARTING │ │
|
||||
└──────┬───────┘ │
|
||||
│ health check passes │
|
||||
▼ │
|
||||
┌──────────────┐ │
|
||||
┌─────────►│ READY │─────────────────────┤
|
||||
│ └──────┬───────┘ │
|
||||
│ │ battle starts │
|
||||
│ ▼ │
|
||||
│ ┌──────────────┐ │
|
||||
│ │ IN_USE │ │
|
||||
│ └──────┬───────┘ │
|
||||
│ │ battle ends │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
Note: The IDLE state was removed - we now simply track time since last human turn and shut down after 10 minutes of inactivity.
|
||||
|
||||
### Lifecycle Rules
|
||||
|
||||
1. **Spin-up Trigger**: Human player takes a turn
|
||||
- Any command submitted by a human player to Eagle
|
||||
- Simple and reliable - no prediction needed
|
||||
|
||||
2. **Keep-Alive**: Reset timer on each human turn
|
||||
- Covers thinking time between turns
|
||||
- Natural pauses (snacks, calls) under 10 min don't cause shutdown
|
||||
|
||||
3. **Shutdown**: 10 minutes since last human turn
|
||||
- Hetzner API: `DELETE /servers/{id}` to stop billing
|
||||
- Short timeout keeps costs low during idle periods
|
||||
|
||||
4. **Health Checks**: Every 10 seconds while STARTING
|
||||
- gRPC connectivity check to Shardok
|
||||
- Mark READY only when Shardok responds successfully
|
||||
|
||||
---
|
||||
|
||||
## Spin-up Strategy
|
||||
|
||||
### Simple Approach: Human Turn-Based
|
||||
|
||||
Spin up Shardok whenever a human player takes a turn. Shut down after 10 minutes of inactivity.
|
||||
|
||||
This is simpler than trying to predict battles. Battles happen regularly during active play, so keeping Shardok running while humans are playing ensures it's ready when needed.
|
||||
|
||||
**Spin-up trigger:**
|
||||
- Any human player takes a turn (submits commands to Eagle)
|
||||
|
||||
**Shutdown trigger:**
|
||||
- No human player has taken a turn in the last 10 minutes
|
||||
|
||||
**Why this works:**
|
||||
- Battles are common during active play sessions
|
||||
- 10-minute idle timeout is long enough to cover thinking time between turns
|
||||
- Cost is so low (~$0.04/hr) that running during entire play sessions is fine
|
||||
- No complex prediction logic needed
|
||||
|
||||
### Cost Impact
|
||||
|
||||
| Scenario | Hours/Week | Cost/Month |
|
||||
|----------|------------|------------|
|
||||
| Human turn-based (10 min idle) | ~15-20 hrs | ~$2.50-3.50 |
|
||||
| Always-on 24/7 | 168 hrs | ~$29 |
|
||||
|
||||
Human turn-based is simple and cheap. The 10-minute timeout catches natural breaks (getting a snack, answering a call) while shutting down during longer idle periods.
|
||||
|
||||
### Implementation in Eagle
|
||||
|
||||
```scala
|
||||
class ShardokInstanceManager {
|
||||
private var instanceState: InstanceState = STOPPED
|
||||
private var lastHumanTurnTime: Instant = Instant.MIN
|
||||
private val idleTimeoutMinutes = 10
|
||||
|
||||
// Called when any human player submits a turn
|
||||
def onHumanTurn(): Unit = {
|
||||
lastHumanTurnTime = Instant.now()
|
||||
if (instanceState == STOPPED) {
|
||||
startInstance()
|
||||
}
|
||||
}
|
||||
|
||||
// Run every minute
|
||||
def periodicShutdownCheck(): Unit = {
|
||||
val idleTime = Duration.between(lastHumanTurnTime, Instant.now())
|
||||
if (instanceState != STOPPED && idleTime > Duration.ofMinutes(idleTimeoutMinutes)) {
|
||||
stopInstance()
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Integration Point
|
||||
|
||||
Wire `onHumanTurn()` into Eagle's command processing. The cleanest place is where Eagle receives player commands from the Unity client - likely in `EagleServiceImpl` when handling the gRPC streaming commands.
|
||||
|
||||
---
|
||||
|
||||
## Security: Eagle ↔ Shardok Authentication
|
||||
|
||||
Since Eagle (DigitalOcean) and Shardok (Hetzner) are on different cloud providers, they cannot share a private network. We need secure authentication over the public internet.
|
||||
|
||||
### Approach: TLS + Token Authentication
|
||||
|
||||
This is the same pattern used by Stripe, GitHub, and most cloud APIs.
|
||||
|
||||
**Layer 1: TLS Encryption (Let's Encrypt)**
|
||||
- Server certificate from Let's Encrypt, auto-renewed via certbot
|
||||
- All gRPC traffic encrypted with TLS 1.3
|
||||
- Prevents eavesdropping and MITM attacks
|
||||
|
||||
**Layer 2: Token Authentication**
|
||||
- 256-bit random shared secret in gRPC metadata
|
||||
- Only requests with valid token are processed
|
||||
- Easy to rotate without any certificate changes
|
||||
|
||||
### How Token Auth Works
|
||||
|
||||
**Setup (once):**
|
||||
```bash
|
||||
# Generate a 256-bit random token
|
||||
openssl rand -hex 32
|
||||
# Output: a3f8b2c9d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1
|
||||
|
||||
# Store in GitHub Secrets as SHARDOK_AUTH_TOKEN
|
||||
# Eagle reads from environment, Shardok receives via cloud-init
|
||||
```
|
||||
|
||||
**Every Request:**
|
||||
```
|
||||
Eagle Shardok
|
||||
────── ───────
|
||||
gRPC call with metadata: → Validate token:
|
||||
"authorization: Bearer a3f8b2c9..." if token != expected:
|
||||
return UNAUTHENTICATED
|
||||
else:
|
||||
process request
|
||||
```
|
||||
|
||||
### Server Certificate (Let's Encrypt)
|
||||
|
||||
Shardok needs a domain name for Let's Encrypt. Options:
|
||||
|
||||
1. **Dynamic DNS**: Assign `shardok.prod.eagle0.net` to instance IP on spin-up
|
||||
2. **Elastic IP**: Reserve a Hetzner floating IP, always points to active instance
|
||||
|
||||
Cloud-init runs certbot on first boot:
|
||||
|
||||
```yaml
|
||||
runcmd:
|
||||
# Install certbot
|
||||
- apt-get install -y certbot
|
||||
|
||||
# Get certificate (DNS must already point to this IP)
|
||||
- certbot certonly --standalone -d shardok.prod.eagle0.net --non-interactive --agree-tos -m admin@eagle0.net
|
||||
|
||||
# Certificate auto-renews via systemd timer (certbot installs this)
|
||||
# Certs at: /etc/letsencrypt/live/shardok.prod.eagle0.net/
|
||||
```
|
||||
|
||||
### Implementation
|
||||
|
||||
**Shardok gRPC Server (C++):**
|
||||
```cpp
|
||||
// Load Let's Encrypt certificates
|
||||
std::string cert = ReadFile("/etc/letsencrypt/live/shardok.prod.eagle0.net/fullchain.pem");
|
||||
std::string key = ReadFile("/etc/letsencrypt/live/shardok.prod.eagle0.net/privkey.pem");
|
||||
|
||||
grpc::SslServerCredentialsOptions ssl_opts;
|
||||
ssl_opts.pem_key_cert_pairs.push_back({key, cert});
|
||||
auto creds = grpc::SslServerCredentials(ssl_opts);
|
||||
builder.AddListeningPort("0.0.0.0:50051", creds);
|
||||
|
||||
// Token validation - check on every RPC
|
||||
Status PostCommand(ServerContext* context, const Request* req, Response* resp) {
|
||||
auto auth = context->client_metadata().find("authorization");
|
||||
std::string expected = "Bearer " + auth_token_;
|
||||
if (auth == context->client_metadata().end() ||
|
||||
auth->second != expected) {
|
||||
return Status(grpc::UNAUTHENTICATED, "Invalid token");
|
||||
}
|
||||
// ... handle request
|
||||
}
|
||||
```
|
||||
|
||||
**Eagle gRPC Client (Scala):**
|
||||
```scala
|
||||
// Standard TLS (trusts Let's Encrypt via system CA store)
|
||||
val channel = NettyChannelBuilder
|
||||
.forAddress("shardok.prod.eagle0.net", 50051)
|
||||
.useTransportSecurity() // TLS with system trust store
|
||||
.intercept(new ClientInterceptor {
|
||||
override def interceptCall[Req, Resp](
|
||||
method: MethodDescriptor[Req, Resp],
|
||||
callOptions: CallOptions,
|
||||
next: Channel): ClientCall[Req, Resp] = {
|
||||
new ForwardingClientCall.SimpleForwardingClientCall(
|
||||
next.newCall(method, callOptions)) {
|
||||
override def start(listener: Listener[Resp], headers: Metadata): Unit = {
|
||||
headers.put(
|
||||
Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER),
|
||||
s"Bearer $authToken"
|
||||
)
|
||||
super.start(listener, headers)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.build()
|
||||
```
|
||||
|
||||
### Token Rotation
|
||||
|
||||
When rotating tokens (e.g., if compromised or as routine hygiene):
|
||||
|
||||
1. Generate new token
|
||||
2. Update Shardok to accept **both** old and new tokens temporarily
|
||||
3. Update Eagle to use new token
|
||||
4. Remove old token from Shardok
|
||||
|
||||
No downtime, no certificate regeneration.
|
||||
|
||||
### Firewall Rules
|
||||
|
||||
```bash
|
||||
# Hetzner firewall via cloud-init
|
||||
ufw default deny incoming
|
||||
ufw allow 80/tcp # Let's Encrypt HTTP-01 challenge (certbot)
|
||||
ufw allow 50051/tcp # gRPC (protected by TLS + token)
|
||||
ufw allow from <admin-ip>/32 to any port 22 # SSH for debugging
|
||||
ufw enable
|
||||
```
|
||||
|
||||
### Security Summary
|
||||
|
||||
| Threat | Mitigation |
|
||||
|--------|------------|
|
||||
| Eavesdropping | TLS 1.3 encryption |
|
||||
| MITM attack | Let's Encrypt certificate validates server identity |
|
||||
| Unauthorized access | 256-bit token required on every request |
|
||||
| Token theft | TLS prevents sniffing; rotate if compromised |
|
||||
|
||||
This is the industry-standard approach for API security.
|
||||
|
||||
---
|
||||
|
||||
## Hetzner API Integration
|
||||
|
||||
### API Authentication
|
||||
|
||||
```bash
|
||||
# Store in secrets manager
|
||||
HETZNER_API_TOKEN=<token from Hetzner Cloud Console>
|
||||
```
|
||||
|
||||
### Key API Operations
|
||||
|
||||
**Create Instance:**
|
||||
```bash
|
||||
curl -X POST "https://api.hetzner.cloud/v1/servers" \
|
||||
-H "Authorization: Bearer $HETZNER_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "shardok-prod",
|
||||
"server_type": "cax41",
|
||||
"location": "ash",
|
||||
"image": "docker-ce",
|
||||
"ssh_keys": ["eagle-deploy"],
|
||||
"user_data": "<cloud-init script>",
|
||||
"labels": {"service": "shardok", "env": "prod"}
|
||||
}'
|
||||
```
|
||||
|
||||
**Check Status:**
|
||||
```bash
|
||||
curl "https://api.hetzner.cloud/v1/servers/{id}" \
|
||||
-H "Authorization: Bearer $HETZNER_API_TOKEN"
|
||||
# Response includes: status (running/starting/off), public IP
|
||||
```
|
||||
|
||||
**Delete Instance (Stop Billing):**
|
||||
```bash
|
||||
curl -X DELETE "https://api.hetzner.cloud/v1/servers/{id}" \
|
||||
-H "Authorization: Bearer $HETZNER_API_TOKEN"
|
||||
```
|
||||
|
||||
### Cloud-Init Script
|
||||
|
||||
```yaml
|
||||
#cloud-config
|
||||
package_update: true
|
||||
packages:
|
||||
- docker.io
|
||||
- certbot
|
||||
|
||||
write_files:
|
||||
- path: /etc/shardok/auth_token
|
||||
permissions: '0600'
|
||||
content: |
|
||||
${AUTH_TOKEN}
|
||||
|
||||
runcmd:
|
||||
# Get Let's Encrypt certificate (DNS must point to this IP first)
|
||||
- certbot certonly --standalone -d shardok.prod.eagle0.net --non-interactive --agree-tos -m admin@eagle0.net
|
||||
|
||||
# Pull and run Shardok container
|
||||
- echo ${DO_REGISTRY_TOKEN} | docker login registry.digitalocean.com -u ${DO_REGISTRY_TOKEN} --password-stdin
|
||||
- docker pull registry.digitalocean.com/eagle0/shardok-server:arm64-latest
|
||||
- docker run -d --name shardok \
|
||||
-p 50051:50051 \
|
||||
-v /etc/letsencrypt:/etc/letsencrypt:ro \
|
||||
-v /etc/shardok:/etc/shardok:ro \
|
||||
registry.digitalocean.com/eagle0/shardok-server:arm64-latest \
|
||||
--cert=/etc/letsencrypt/live/shardok.prod.eagle0.net/fullchain.pem \
|
||||
--key=/etc/letsencrypt/live/shardok.prod.eagle0.net/privkey.pem \
|
||||
--token-file=/etc/shardok/auth_token
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Container Registry
|
||||
|
||||
Using DigitalOcean Container Registry for all images (consistent with Eagle and other services).
|
||||
|
||||
Both architectures use the same repository with different tag prefixes:
|
||||
- ARM64 image: `registry.digitalocean.com/eagle0/shardok-server:arm64-latest`
|
||||
- x86 image: `registry.digitalocean.com/eagle0/shardok-server:latest`
|
||||
|
||||
CI automatically builds and pushes ARM64 images when C++ code changes.
|
||||
|
||||
---
|
||||
|
||||
## Fallback Strategy
|
||||
|
||||
### If Shardok Instance Not Ready
|
||||
|
||||
When a battle starts but the Hetzner instance isn't ready:
|
||||
|
||||
1. **Option A: Wait with Progress Indicator**
|
||||
- Show "Preparing battle arena..." with spinner
|
||||
- Most users won't notice 20-30s wait if UI is engaging
|
||||
- Simplest implementation
|
||||
|
||||
2. **Option B: Local Fallback** (Current behavior)
|
||||
- Run Shardok on Eagle's droplet temporarily
|
||||
- Lower AI quality but instant start
|
||||
- Migrate battle to Hetzner when ready (complex)
|
||||
|
||||
3. **Option C: Reduced AI Settings**
|
||||
- Start battle immediately with very short time budgets
|
||||
- Improve AI quality when Hetzner ready
|
||||
- Seamless to user but requires AI hot-swap
|
||||
|
||||
**Recommendation**: Start with Option A. If user feedback indicates wait time is problematic, implement Option B.
|
||||
|
||||
---
|
||||
|
||||
## Monitoring & Observability
|
||||
|
||||
### Metrics to Track
|
||||
|
||||
| Metric | Purpose |
|
||||
|--------|---------|
|
||||
| `shardok_instance_state` | Current lifecycle state |
|
||||
| `shardok_startup_duration_seconds` | Time from spin-up to ready |
|
||||
| `shardok_battles_per_instance` | Efficiency of keep-alive |
|
||||
| `shardok_wasted_spinups` | Spin-ups without battles |
|
||||
| `shardok_hourly_cost` | Running cost tracking |
|
||||
|
||||
### Alerting
|
||||
|
||||
- Alert if startup takes > 90 seconds
|
||||
- Alert if instance stuck in STARTING for > 3 minutes
|
||||
- Alert if Hetzner API errors persist
|
||||
|
||||
---
|
||||
|
||||
## Cost Estimate (Activity-Based, ARM)
|
||||
|
||||
With activity-based spin-up, Shardok runs during active play sessions:
|
||||
|
||||
| Play Pattern | Hours/Week | Compute/Month |
|
||||
|--------------|------------|---------------|
|
||||
| Casual (few sessions) | ~10 hrs | **~$1.75** |
|
||||
| Regular (daily play) | ~20 hrs | **~$3.50** |
|
||||
| Heavy (multiple daily) | ~40 hrs | **~$7.00** |
|
||||
|
||||
Plus fixed costs:
|
||||
- Container registry: $0 (GitHub Container Registry)
|
||||
- Secrets management: $0 (GitHub Secrets)
|
||||
|
||||
**Total estimated cost: $2-7/month** for typical usage.
|
||||
|
||||
Compare to current DigitalOcean droplet upgrade (~$20-40/month for equivalent CPU power).
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Basic On-Demand (1-2 weeks)
|
||||
|
||||
1. ✅ Add ARM64 Linux to CI build matrix (PR #4990, merged 2026-01-02)
|
||||
- Added LLVM toolchain for ARM64 cross-compilation
|
||||
- Created ARM64 sysroot build workflow
|
||||
- Added Ubuntu 24.04 ARM64 base image for containers
|
||||
2. ✅ Push ARM64 container to DigitalOcean Container Registry (PR #4990)
|
||||
- Created shardok_arm64_build.yml workflow
|
||||
- Pushes to registry.digitalocean.com/eagle0/shardok-server:arm64-latest
|
||||
3. ✅ Implement Hetzner API client in Eagle (Scala) (PR #4996, merged 2026-01-02)
|
||||
- HetznerApiClient with create/delete/list/power operations
|
||||
- Async HTTP via OkHttp with Future-based API
|
||||
4. ✅ Add ShardokInstanceManager with spin-up/shutdown logic (PR #4998, merged 2026-01-02)
|
||||
- Activity-based spin-up when players connect
|
||||
- Idle timeout shutdown (default 60 minutes)
|
||||
- State machine: Stopped → Starting → Ready → InUse → Stopping
|
||||
5. ✅ Add TLS + token authentication to Shardok server (PR #5001, merged 2026-01-02)
|
||||
- C++ server: TLS via Let's Encrypt, Bearer token validation
|
||||
- Scala client: TLS and auth token support in ServerSetupHelpers
|
||||
6. 🔄 Test end-to-end with manual triggers ← **CURRENT**
|
||||
- ✅ Hetzner server created (CAX41 ARM64 in Helsinki)
|
||||
- ✅ Floating IPv6 configured and persisted via netplan
|
||||
- ✅ DNS configured (shardok.prod.eagle0.net → floating IP)
|
||||
- ✅ Let's Encrypt certificate obtained
|
||||
- ✅ NAT64 configured for IPv6-only server to reach IPv4 registries
|
||||
- ✅ Fixed runfiles error (missing SHARDOK_RESOURCES_PATH env var in docker run)
|
||||
- 🔄 Re-test container startup on Hetzner
|
||||
|
||||
### Known Issues
|
||||
|
||||
**IPv6-Only Servers Need NAT64**
|
||||
|
||||
Hetzner ARM instances are IPv6-only by default. DigitalOcean Container Registry's blob storage is IPv4-only. Solution: configure NAT64 DNS:
|
||||
|
||||
```bash
|
||||
mkdir -p /etc/systemd/resolved.conf.d
|
||||
cat > /etc/systemd/resolved.conf.d/nat64.conf << 'EOF'
|
||||
[Resolve]
|
||||
DNS=2a00:1098:2b::1 2a00:1098:2c::1
|
||||
EOF
|
||||
systemctl restart systemd-resolved
|
||||
```
|
||||
|
||||
**Floating IP Requires Manual Config**
|
||||
|
||||
Hetzner floating IPs must be added to netplan:
|
||||
|
||||
```yaml
|
||||
# In /etc/netplan/50-cloud-init.yaml, add to addresses list:
|
||||
addresses:
|
||||
- "2a01:4f9:xxxx:xxxx::1/64" # Server's own IP
|
||||
- "2a01:4f9:yyyy:yyyy::1/64" # Floating IP
|
||||
```
|
||||
|
||||
Then run `netplan apply`.
|
||||
|
||||
### Phase 2: Wire into Eagle
|
||||
|
||||
1. Call `onHumanTurn()` when human players submit commands
|
||||
2. Wire ShardokInstanceManager into Eagle's startup
|
||||
3. Add environment variables for Hetzner API token, floating IP
|
||||
4. Add "Preparing battle..." UI state for when Shardok is starting
|
||||
|
||||
### Phase 3: Monitoring & Polish
|
||||
|
||||
1. Add metrics and dashboards
|
||||
2. Implement alerting
|
||||
3. Cost tracking
|
||||
4. Documentation and runbooks
|
||||
|
||||
---
|
||||
|
||||
## Design Decisions
|
||||
|
||||
### Certificate Management
|
||||
|
||||
**Fully automated via Let's Encrypt + certbot.**
|
||||
|
||||
- Certbot runs on instance startup to obtain certificate
|
||||
- Systemd timer auto-renews every 60 days (certs valid 90 days)
|
||||
- No manual intervention required
|
||||
- Token rotation is manual but simple (generate new token, update both sides)
|
||||
|
||||
### Concurrent Battles
|
||||
|
||||
**Decision**: Single shared Shardok instance handles all concurrent battles.
|
||||
|
||||
Shardok already supports multiple simultaneous games. With 16 cores and activity-based spin-up, one instance is sufficient for the foreseeable future.
|
||||
|
||||
---
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
These are not needed now but documented for future consideration:
|
||||
|
||||
### Multi-Region Deployment
|
||||
|
||||
If player base expands globally:
|
||||
- Deploy Shardok instances in multiple Hetzner regions (Ashburn, Helsinki, Singapore)
|
||||
- Route players to nearest region based on their Eagle connection
|
||||
- Each region independent (no cross-region state sharing needed)
|
||||
|
||||
**Trigger**: Consistent complaints about battle latency from non-US players.
|
||||
|
||||
### Dynamic Instance Sizing
|
||||
|
||||
Could adjust instance size based on battle complexity:
|
||||
- Small battles (< 10 units): CAX21 (4 cores)
|
||||
- Medium battles: CAX31 (8 cores)
|
||||
- Large battles: CAX41 (16 cores)
|
||||
|
||||
**Current decision**: Start with CAX41 for all battles. The cost difference is minimal (~$0.02/hr between sizes) and simplicity is valuable.
|
||||
|
||||
### Dedicated Shardok Instances Per Game
|
||||
|
||||
For competitive/tournament play, could spin up dedicated instances:
|
||||
- Guaranteed resources, no contention
|
||||
- Isolated failures
|
||||
- Higher cost
|
||||
|
||||
**Current decision**: Shared instance is fine. Revisit if multiplayer competitive mode launches.
|
||||
@@ -11,7 +11,9 @@ cc_library(
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//src/main/cpp/net/eagle0/shardok/ai:remote_ai_client",
|
||||
"//src/main/cpp/net/eagle0/shardok/library:engine",
|
||||
"//src/main/cpp/net/eagle0/shardok/server:ai_service_grpc_server",
|
||||
"//src/main/cpp/net/eagle0/shardok/server:eagle_interface_grpc_server",
|
||||
"//src/main/cpp/net/eagle0/shardok/server:server_configuration",
|
||||
"//src/main/cpp/net/eagle0/shardok/server:token_auth",
|
||||
|
||||
@@ -393,3 +393,17 @@ cc_library(
|
||||
"@com_google_protobuf//:protobuf",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "remote_ai_client",
|
||||
srcs = ["RemoteAIClient.cpp"],
|
||||
hdrs = ["RemoteAIClient.hpp"],
|
||||
copts = COPTS,
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":shardok_ai_client", # For CommandChoiceResults
|
||||
"//src/main/cpp/net/eagle0/common:protobuf_warning_suppression",
|
||||
"//src/main/protobuf/net/eagle0/common:shardok_internal_interface_cc_grpc",
|
||||
"@grpc//:grpc++",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//
|
||||
// RemoteAIClient.cpp
|
||||
// shardok
|
||||
//
|
||||
// Created by Claude on 2025.
|
||||
// Copyright © 2025 none. All rights reserved.
|
||||
//
|
||||
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/RemoteAIClient.hpp"
|
||||
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace shardok {
|
||||
|
||||
using grpc::Channel;
|
||||
using grpc::ClientContext;
|
||||
using grpc::Status;
|
||||
using net::eagle0::common::ChooseAICommandRequest;
|
||||
using net::eagle0::common::ChooseAICommandResponse;
|
||||
using net::eagle0::common::ShardokAIService;
|
||||
|
||||
RemoteAIClient::RemoteAIClient(
|
||||
std::shared_ptr<Channel> channel,
|
||||
std::string gameId,
|
||||
std::chrono::milliseconds timeout)
|
||||
: stub_(ShardokAIService::NewStub(channel)),
|
||||
timeout_(timeout),
|
||||
gameId_(std::move(gameId)) {}
|
||||
|
||||
auto RemoteAIClient::ChooseCommandIndex(
|
||||
const PlayerId playerId,
|
||||
const bool isDefender,
|
||||
const bool isAllAiBattle,
|
||||
const std::vector<uint8_t>& gameStateBytes,
|
||||
const std::chrono::milliseconds timeBudgetMs) -> CommandChoiceResults {
|
||||
ChooseAICommandRequest request;
|
||||
request.set_game_id(gameId_);
|
||||
request.set_player_id(playerId);
|
||||
request.set_is_defender(isDefender);
|
||||
request.set_is_all_ai_battle(isAllAiBattle);
|
||||
request.set_game_state_bytes(gameStateBytes.data(), gameStateBytes.size());
|
||||
request.set_time_budget_ms(static_cast<int32_t>(timeBudgetMs.count()));
|
||||
|
||||
ChooseAICommandResponse response;
|
||||
ClientContext context;
|
||||
|
||||
// Set deadline for the RPC
|
||||
context.set_deadline(std::chrono::system_clock::now() + timeout_);
|
||||
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
const Status status = stub_->ChooseAICommand(&context, request, &response);
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start);
|
||||
|
||||
if (!status.ok()) {
|
||||
std::cerr << "[RemoteAIClient] RPC failed for game " << gameId_ << " player " << playerId
|
||||
<< ": " << status.error_message() << " (code=" << status.error_code()
|
||||
<< ", elapsed=" << elapsed.count() << "ms)" << std::endl;
|
||||
throw std::runtime_error("Remote AI call failed: " + status.error_message());
|
||||
}
|
||||
|
||||
std::cout << "[RemoteAIClient] Remote AI completed for game " << gameId_ << " player "
|
||||
<< playerId << ": chose index " << response.chosen_index()
|
||||
<< " (depth=" << response.depth_achieved()
|
||||
<< ", evaluated=" << response.commands_evaluated()
|
||||
<< ", remote_time=" << response.elapsed_ms() << "ms, rtt=" << elapsed.count() << "ms)"
|
||||
<< std::endl;
|
||||
|
||||
return CommandChoiceResults{
|
||||
.chosenIndex = static_cast<size_t>(response.chosen_index()),
|
||||
.availableCommandCount = 0, // Not provided by remote
|
||||
.depthAchieved = response.depth_achieved(),
|
||||
.commandCountEvaluated = static_cast<size_t>(response.commands_evaluated()),
|
||||
.completionReason = EvaluationCompletionReason::RAN_OUT_OF_TIME // Assumed
|
||||
};
|
||||
}
|
||||
|
||||
// RemoteAIClientFactory implementation
|
||||
|
||||
namespace {
|
||||
// Custom call credentials that add bearer token
|
||||
class BearerTokenAuthenticator : public grpc::MetadataCredentialsPlugin {
|
||||
public:
|
||||
explicit BearerTokenAuthenticator(std::string token) : token_(std::move(token)) {}
|
||||
|
||||
auto GetMetadata(
|
||||
grpc::string_ref /*service_url*/,
|
||||
grpc::string_ref /*method_name*/,
|
||||
const grpc::AuthContext& /*channel_auth_context*/,
|
||||
std::multimap<grpc::string, grpc::string>* metadata) -> grpc::Status override {
|
||||
metadata->insert(std::make_pair("authorization", "Bearer " + token_));
|
||||
return grpc::Status::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string token_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
RemoteAIClientFactory::RemoteAIClientFactory(
|
||||
const std::string& address,
|
||||
const std::string& authToken,
|
||||
std::chrono::milliseconds timeout)
|
||||
: timeout_(timeout) {
|
||||
std::cout << "[RemoteAIClientFactory] Creating channel to " << address << std::endl;
|
||||
|
||||
if (authToken.empty()) {
|
||||
// Insecure channel for local testing
|
||||
channel_ = grpc::CreateChannel(address, grpc::InsecureChannelCredentials());
|
||||
} else {
|
||||
// SSL channel with bearer token authentication
|
||||
grpc::SslCredentialsOptions sslOpts;
|
||||
auto channelCreds = grpc::SslCredentials(sslOpts);
|
||||
|
||||
auto callCreds = grpc::MetadataCredentialsFromPlugin(
|
||||
std::make_unique<BearerTokenAuthenticator>(authToken));
|
||||
|
||||
auto compositeCreds = grpc::CompositeChannelCredentials(channelCreds, callCreds);
|
||||
channel_ = grpc::CreateChannel(address, compositeCreds);
|
||||
}
|
||||
}
|
||||
|
||||
auto RemoteAIClientFactory::CreateClient(const std::string& gameId)
|
||||
-> std::unique_ptr<RemoteAIClient> {
|
||||
return std::make_unique<RemoteAIClient>(channel_, gameId, timeout_);
|
||||
}
|
||||
|
||||
auto RemoteAIClientFactory::IsConnected() const -> bool {
|
||||
if (!channel_) { return false; }
|
||||
const auto state = channel_->GetState(false);
|
||||
return state == GRPC_CHANNEL_READY || state == GRPC_CHANNEL_IDLE;
|
||||
}
|
||||
|
||||
} // namespace shardok
|
||||
@@ -0,0 +1,109 @@
|
||||
//
|
||||
// RemoteAIClient.hpp
|
||||
// shardok
|
||||
//
|
||||
// Created by Claude on 2025.
|
||||
// Copyright © 2025 none. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef RemoteAIClient_hpp
|
||||
#define RemoteAIClient_hpp
|
||||
|
||||
#include <chrono>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "src/main/cpp/net/eagle0/common/ProtobufWarningSuppression.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/ShardokAIClient.hpp"
|
||||
#pragma GCC diagnostic push
|
||||
SUPPRESS_PROTOBUF_WARNINGS
|
||||
#include <grpcpp/grpcpp.h>
|
||||
|
||||
#include "src/main/protobuf/net/eagle0/common/shardok_internal_interface.grpc.pb.h"
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
namespace shardok {
|
||||
|
||||
/**
|
||||
* Client for offloading AI computation to a remote Shardok-AI server.
|
||||
*
|
||||
* Used by Shardok-Primary on DigitalOcean to call Shardok-AI on Hetzner
|
||||
* for AI command selection while keeping command processing local for
|
||||
* low latency.
|
||||
*/
|
||||
class RemoteAIClient {
|
||||
private:
|
||||
std::unique_ptr<net::eagle0::common::ShardokAIService::Stub> stub_;
|
||||
std::chrono::milliseconds timeout_;
|
||||
std::string gameId_;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Create a remote AI client.
|
||||
*
|
||||
* @param channel Shared gRPC channel to the remote AI server
|
||||
* @param gameId Game identifier for logging/debugging
|
||||
* @param timeout Timeout for remote calls (should be > AI time budget + network latency)
|
||||
*/
|
||||
RemoteAIClient(
|
||||
std::shared_ptr<grpc::Channel> channel,
|
||||
std::string gameId,
|
||||
std::chrono::milliseconds timeout = std::chrono::milliseconds(10000));
|
||||
|
||||
/**
|
||||
* Choose a command using the remote AI server.
|
||||
*
|
||||
* @param playerId AI player ID
|
||||
* @param isDefender Whether this AI is defending
|
||||
* @param isAllAiBattle Whether this is an AI-only battle (affects time budget)
|
||||
* @param gameStateBytes Serialized game state (FlatBuffers)
|
||||
* @param timeBudgetMs Time budget for AI thinking
|
||||
* @return CommandChoiceResults with chosen index and performance metrics
|
||||
* @throws std::runtime_error if remote call fails
|
||||
*/
|
||||
[[nodiscard]] auto ChooseCommandIndex(
|
||||
PlayerId playerId,
|
||||
bool isDefender,
|
||||
bool isAllAiBattle,
|
||||
const std::vector<uint8_t>& gameStateBytes,
|
||||
std::chrono::milliseconds timeBudgetMs) -> CommandChoiceResults;
|
||||
};
|
||||
|
||||
/**
|
||||
* Factory for creating RemoteAIClient instances with shared channel.
|
||||
*
|
||||
* Maintains a persistent gRPC channel to the remote AI server that can be
|
||||
* shared across multiple games.
|
||||
*/
|
||||
class RemoteAIClientFactory {
|
||||
private:
|
||||
std::shared_ptr<grpc::Channel> channel_;
|
||||
std::chrono::milliseconds timeout_;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Create a factory with connection to remote AI server.
|
||||
*
|
||||
* @param address Remote AI server address (e.g., "ai.eagle0.net:40043")
|
||||
* @param authToken Optional bearer token for authentication
|
||||
* @param timeout Timeout for remote calls
|
||||
*/
|
||||
RemoteAIClientFactory(
|
||||
const std::string& address,
|
||||
const std::string& authToken = "",
|
||||
std::chrono::milliseconds timeout = std::chrono::milliseconds(10000));
|
||||
|
||||
/**
|
||||
* Create a RemoteAIClient for a specific game.
|
||||
*/
|
||||
[[nodiscard]] auto CreateClient(const std::string& gameId) -> std::unique_ptr<RemoteAIClient>;
|
||||
|
||||
/**
|
||||
* Check if the remote AI server is reachable.
|
||||
*/
|
||||
[[nodiscard]] auto IsConnected() const -> bool;
|
||||
};
|
||||
|
||||
} // namespace shardok
|
||||
|
||||
#endif /* RemoteAIClient_hpp */
|
||||
@@ -12,6 +12,7 @@ cc_library(
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":game_update_receiver",
|
||||
"//src/main/cpp/net/eagle0/shardok/ai:remote_ai_client",
|
||||
"//src/main/cpp/net/eagle0/shardok/ai:shardok_ai_client",
|
||||
"//src/main/cpp/net/eagle0/shardok/library:engine",
|
||||
"//src/main/cpp/net/eagle0/shardok/library:shardok_c_types",
|
||||
|
||||
@@ -154,7 +154,48 @@ void ShardokGameController::DoAIThread() {
|
||||
}
|
||||
|
||||
// Phase 2: AI thinks (NO LOCK - this is the slow part)
|
||||
const auto results = aiClient->ChooseCommandIndex(settings, gsv, availableCommands);
|
||||
CommandChoiceResults results;
|
||||
|
||||
// Try remote AI first if configured and not in fallback mode
|
||||
if (remoteAIClient && !useLocalAIFallback) {
|
||||
try {
|
||||
// Get game state bytes and AI client info for remote call
|
||||
const auto gameStateBytes = engine->GetCurrentGameStateBytes();
|
||||
const bool isDefender =
|
||||
aiClient->GetPlayerId() == playerId &&
|
||||
std::ranges::any_of(engine->GetPlayerInfos(), [playerId](const auto &pi) {
|
||||
return pi.player_id() == playerId && pi.is_defender();
|
||||
});
|
||||
|
||||
// Calculate time budget (simplified - remote will recalculate if needed)
|
||||
const auto timeBudget = std::chrono::milliseconds(2000);
|
||||
|
||||
// Check if all players are AI
|
||||
const bool isAllAiBattle = std::ranges::all_of(
|
||||
engine->GetPlayerInfos(),
|
||||
[](const auto &pi) { return pi.is_ai(); });
|
||||
|
||||
results = remoteAIClient->ChooseCommandIndex(
|
||||
playerId,
|
||||
isDefender,
|
||||
isAllAiBattle,
|
||||
gameStateBytes,
|
||||
timeBudget);
|
||||
|
||||
printf("[DoAIThread] Remote AI chose command %zu for player %d\n",
|
||||
results.chosenIndex,
|
||||
playerId);
|
||||
} catch (const std::exception &e) {
|
||||
printf("[DoAIThread] Remote AI failed: %s, falling back to local AI\n", e.what());
|
||||
useLocalAIFallback = true;
|
||||
// Fall through to local AI below
|
||||
}
|
||||
}
|
||||
|
||||
// Use local AI if remote is not configured, failed, or in fallback mode
|
||||
if (!remoteAIClient || useLocalAIFallback) {
|
||||
results = aiClient->ChooseCommandIndex(settings, gsv, availableCommands);
|
||||
}
|
||||
|
||||
// Phase 3: Post the command (brief lock)
|
||||
{
|
||||
@@ -189,6 +230,12 @@ ShardokGameController::~ShardokGameController() {
|
||||
aiThread.join();
|
||||
}
|
||||
|
||||
void ShardokGameController::SetRemoteAIClient(std::unique_ptr<RemoteAIClient> client) {
|
||||
remoteAIClient = std::move(client);
|
||||
useLocalAIFallback = false; // Reset fallback when setting new client
|
||||
printf("[ShardokGameController] Remote AI client set for game %s\n", cachedGameId.c_str());
|
||||
}
|
||||
|
||||
void CheckFactionId(
|
||||
const unique_ptr<ShardokEngine> &engine,
|
||||
const PlayerId shardokPlayerId,
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#ifndef ShardokGameController_hpp
|
||||
#define ShardokGameController_hpp
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
@@ -16,6 +17,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "GameUpdateReceiver.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/RemoteAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/ShardokAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/ShardokEngine.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/UnitPlacementInfo.hpp"
|
||||
@@ -103,6 +105,10 @@ private:
|
||||
vector<shared_ptr<ShardokAIClient>> aiClients;
|
||||
std::thread aiThread;
|
||||
|
||||
// Remote AI offload support
|
||||
std::unique_ptr<RemoteAIClient> remoteAIClient;
|
||||
std::atomic<bool> useLocalAIFallback{false};
|
||||
|
||||
static auto MakeLogFilePath() -> string {
|
||||
const time_t timer = time(nullptr);
|
||||
char buf[255];
|
||||
@@ -169,6 +175,11 @@ public:
|
||||
|
||||
[[nodiscard]] auto GetLogFilePath() const -> string { return logFilePath; }
|
||||
|
||||
/// Set a remote AI client for offloading AI computation to a remote server.
|
||||
/// When set, AI decisions will be made by the remote server instead of locally.
|
||||
/// If the remote call fails, falls back to local AI for the remainder of the game.
|
||||
void SetRemoteAIClient(std::unique_ptr<RemoteAIClient> client);
|
||||
|
||||
/// Register a subscriber to receive streaming updates for this game.
|
||||
/// The subscriber will receive updates until it becomes inactive or is unregistered.
|
||||
void RegisterSubscriber(std::shared_ptr<StreamSubscriber> subscriber);
|
||||
|
||||
@@ -12,6 +12,7 @@ cc_library(
|
||||
deps = [
|
||||
":server_configuration",
|
||||
"//src/main/cpp/net/eagle0/common:tsv_parser",
|
||||
"//src/main/cpp/net/eagle0/shardok/ai:remote_ai_client",
|
||||
"//src/main/cpp/net/eagle0/shardok/ai:shardok_ai_client",
|
||||
"//src/main/cpp/net/eagle0/shardok/controller",
|
||||
"//src/main/cpp/net/eagle0/shardok/library:engine",
|
||||
@@ -60,3 +61,20 @@ cc_library(
|
||||
"@grpc//:grpc++",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "ai_service_grpc_server",
|
||||
srcs = ["ShardokAIServiceImpl.cpp"],
|
||||
hdrs = ["ShardokAIServiceImpl.hpp"],
|
||||
copts = COPTS,
|
||||
visibility = ["//src/main/cpp/net/eagle0/shardok:__subpackages__"],
|
||||
deps = [
|
||||
":token_auth",
|
||||
"//src/main/cpp/net/eagle0/common:protobuf_warning_suppression",
|
||||
"//src/main/cpp/net/eagle0/shardok/ai:shardok_ai_client",
|
||||
"//src/main/cpp/net/eagle0/shardok/library:engine",
|
||||
"//src/main/cpp/net/eagle0/shardok/library/settings:game_settings",
|
||||
"//src/main/protobuf/net/eagle0/common:shardok_internal_interface_cc_grpc",
|
||||
"@grpc//:grpc++",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -25,6 +25,11 @@ const string ServerConfiguration::kEagleGrpcAddress = "eagleGrpcAddress";
|
||||
const string ServerConfiguration::kSslCertPath = "sslCertPath";
|
||||
const string ServerConfiguration::kSslPrivateKeyPath = "sslPrivateKeyPath";
|
||||
const string ServerConfiguration::kAuthTokenPath = "authTokenPath";
|
||||
// Remote AI offload configuration
|
||||
const string ServerConfiguration::kRemoteAIAddress = "remoteAIAddress";
|
||||
const string ServerConfiguration::kRemoteAIAuthTokenPath = "remoteAIAuthTokenPath";
|
||||
const string ServerConfiguration::kRemoteAITimeoutMs = "remoteAITimeoutMs";
|
||||
const string ServerConfiguration::kAIServiceAddress = "aiServiceAddress";
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
using std::unordered_map;
|
||||
@@ -37,11 +42,28 @@ unordered_map<string, string> DefaultValues() {
|
||||
// to listen on all interfaces for container networking
|
||||
const char* eagleInterfaceAddr = getenv("SHARDOK_EAGLE_INTERFACE_ADDRESS");
|
||||
const char* shardokAddr = getenv("SHARDOK_GRPC_ADDRESS");
|
||||
// Remote AI offload configuration
|
||||
const char* remoteAIAddr = getenv("SHARDOK_REMOTE_AI_ADDRESS");
|
||||
const char* remoteAIAuthTokenPath = getenv("SHARDOK_REMOTE_AI_AUTH_TOKEN_PATH");
|
||||
const char* remoteAITimeoutMs = getenv("SHARDOK_REMOTE_AI_TIMEOUT_MS");
|
||||
const char* aiServiceAddr = getenv("SHARDOK_AI_SERVICE_ADDRESS");
|
||||
|
||||
return unordered_map<string, string>{
|
||||
unordered_map<string, string> defaults{
|
||||
{ServerConfiguration::kShardokGrpcAddress, shardokAddr ? shardokAddr : "localhost"},
|
||||
{ServerConfiguration::kEagleInterfaceGrpcAddress,
|
||||
eagleInterfaceAddr ? eagleInterfaceAddr : "localhost:40042"}};
|
||||
|
||||
// Only add remote AI config if environment variables are set
|
||||
if (remoteAIAddr) { defaults[ServerConfiguration::kRemoteAIAddress] = remoteAIAddr; }
|
||||
if (remoteAIAuthTokenPath) {
|
||||
defaults[ServerConfiguration::kRemoteAIAuthTokenPath] = remoteAIAuthTokenPath;
|
||||
}
|
||||
if (remoteAITimeoutMs) {
|
||||
defaults[ServerConfiguration::kRemoteAITimeoutMs] = remoteAITimeoutMs;
|
||||
}
|
||||
if (aiServiceAddr) { defaults[ServerConfiguration::kAIServiceAddress] = aiServiceAddr; }
|
||||
|
||||
return defaults;
|
||||
}
|
||||
|
||||
ServerConfiguration::ServerConfiguration(const string& filePath) {
|
||||
|
||||
@@ -30,6 +30,12 @@ public:
|
||||
const static string kSslCertPath;
|
||||
const static string kSslPrivateKeyPath;
|
||||
const static string kAuthTokenPath;
|
||||
|
||||
// Remote AI offload configuration
|
||||
const static string kRemoteAIAddress;
|
||||
const static string kRemoteAIAuthTokenPath;
|
||||
const static string kRemoteAITimeoutMs;
|
||||
const static string kAIServiceAddress;
|
||||
};
|
||||
|
||||
#endif /* ServerConfiguration_hpp */
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//
|
||||
// ShardokAIServiceImpl.cpp
|
||||
// shardok
|
||||
//
|
||||
// Created by Claude on 2025.
|
||||
// Copyright © 2025 none. All rights reserved.
|
||||
//
|
||||
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/ShardokAIServiceImpl.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
|
||||
#include "src/main/cpp/net/eagle0/common/mcts/abstract/MCTSTypes.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/ShardokAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/ShardokEngine.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/fb_helpers/GameStateHelpers.hpp"
|
||||
|
||||
namespace shardok {
|
||||
|
||||
using grpc::ServerContext;
|
||||
using grpc::Status;
|
||||
using grpc::StatusCode;
|
||||
using net::eagle0::common::ChooseAICommandRequest;
|
||||
using net::eagle0::common::ChooseAICommandResponse;
|
||||
|
||||
ShardokAIServiceImpl::ShardokAIServiceImpl(GameSettingsSPtr gameSettings, std::string authToken)
|
||||
: tokenValidator_(std::move(authToken)),
|
||||
gameSettings_(std::move(gameSettings)) {
|
||||
std::cout << "[ShardokAIService] Initialized with auth "
|
||||
<< (tokenValidator_.IsEnabled() ? "enabled" : "disabled") << std::endl;
|
||||
}
|
||||
|
||||
auto ShardokAIServiceImpl::ChooseAICommand(
|
||||
ServerContext* context,
|
||||
const ChooseAICommandRequest* request,
|
||||
ChooseAICommandResponse* response) -> Status {
|
||||
// Validate auth token
|
||||
if (auto authStatus = tokenValidator_.ValidateOrStatus(context)) { return *authStatus; }
|
||||
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
|
||||
std::cout << "[ShardokAIService] ChooseAICommand for game " << request->game_id() << " player "
|
||||
<< request->player_id() << " (defender=" << request->is_defender()
|
||||
<< ", allAI=" << request->is_all_ai_battle()
|
||||
<< ", budget=" << request->time_budget_ms() << "ms"
|
||||
<< ", stateSize=" << request->game_state_bytes().size() << ")" << std::endl;
|
||||
|
||||
try {
|
||||
// Reconstruct game state from bytes
|
||||
const auto& stateBytes = request->game_state_bytes();
|
||||
byte_vector gameStateVec(stateBytes.begin(), stateBytes.end());
|
||||
|
||||
auto gameState = GameStateW::FromByteVector(std::move(gameStateVec));
|
||||
if (!gameState) {
|
||||
std::cerr << "[ShardokAIService] Failed to deserialize game state" << std::endl;
|
||||
return Status(StatusCode::INVALID_ARGUMENT, "Failed to deserialize game state");
|
||||
}
|
||||
|
||||
// Create engine from game state
|
||||
auto engine = std::make_unique<ShardokEngine>(gameSettings_, gameState);
|
||||
|
||||
// MCTS configuration (matches what's used in MakeAIClients)
|
||||
mcts::MCTSConfig mctsConfig;
|
||||
mctsConfig.backpropagationPolicy = mcts::MCTSBackpropagationPolicy::MINIMAX;
|
||||
mctsConfig.maxPlayerFlips = 1;
|
||||
mctsConfig.maxSimulationFlips = 1;
|
||||
mctsConfig.simulationPolicy = mcts::MCTSSimulationPolicy::WEIGHTED_HEURISTIC;
|
||||
|
||||
// Create AI client
|
||||
auto aiClient = std::make_unique<ShardokAIClient>(
|
||||
static_cast<PlayerId>(request->player_id()),
|
||||
request->is_defender(),
|
||||
request->is_all_ai_battle(),
|
||||
engine->GetCurrentGameState()->hex_map(),
|
||||
gameSettings_->GetGetter(),
|
||||
AIAlgorithmType::ITERATIVE_DEEPENING,
|
||||
ScoringCalculatorType::MCTS_OPTIMIZED,
|
||||
mctsConfig);
|
||||
|
||||
// Get available commands
|
||||
const auto availableCommands = engine->GetAvailableCommandsForAIPlayer(
|
||||
static_cast<PlayerId>(request->player_id()));
|
||||
|
||||
if (!availableCommands || availableCommands->empty()) {
|
||||
std::cerr << "[ShardokAIService] No commands available for player "
|
||||
<< request->player_id() << std::endl;
|
||||
return Status(StatusCode::FAILED_PRECONDITION, "No commands available for player");
|
||||
}
|
||||
|
||||
// Get game state view for AI
|
||||
const auto gsv = engine->GetGameStateView(static_cast<PlayerId>(request->player_id()));
|
||||
|
||||
// Run AI evaluation
|
||||
const auto results = aiClient->ChooseCommandIndex(gameSettings_, gsv, availableCommands);
|
||||
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start);
|
||||
|
||||
std::cout << "[ShardokAIService] AI chose command " << results.chosenIndex << " for player "
|
||||
<< request->player_id() << " (depth=" << results.depthAchieved
|
||||
<< ", evaluated=" << results.commandCountEvaluated
|
||||
<< ", elapsed=" << elapsed.count() << "ms)" << std::endl;
|
||||
|
||||
// Populate response
|
||||
response->set_chosen_index(static_cast<int32_t>(results.chosenIndex));
|
||||
response->set_depth_achieved(results.depthAchieved);
|
||||
response->set_commands_evaluated(static_cast<int32_t>(results.commandCountEvaluated));
|
||||
response->set_elapsed_ms(static_cast<int32_t>(elapsed.count()));
|
||||
|
||||
return Status::OK;
|
||||
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "[ShardokAIService] Exception during AI computation: " << e.what()
|
||||
<< std::endl;
|
||||
return Status(StatusCode::INTERNAL, std::string("AI computation failed: ") + e.what());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace shardok
|
||||
@@ -0,0 +1,64 @@
|
||||
//
|
||||
// ShardokAIServiceImpl.hpp
|
||||
// shardok
|
||||
//
|
||||
// Created by Claude on 2025.
|
||||
// Copyright © 2025 none. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef ShardokAIServiceImpl_hpp
|
||||
#define ShardokAIServiceImpl_hpp
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "src/main/cpp/net/eagle0/common/ProtobufWarningSuppression.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/settings/GameSettings.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/TokenAuthInterceptor.hpp"
|
||||
#pragma GCC diagnostic push
|
||||
SUPPRESS_PROTOBUF_WARNINGS
|
||||
#include <grpcpp/grpcpp.h>
|
||||
|
||||
#include "src/main/protobuf/net/eagle0/common/shardok_internal_interface.grpc.pb.h"
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
namespace shardok {
|
||||
|
||||
/**
|
||||
* gRPC service implementation for remote AI computation.
|
||||
*
|
||||
* This service runs on Shardok-AI (Hetzner) and handles AI command selection
|
||||
* requests from Shardok-Primary (DigitalOcean).
|
||||
*
|
||||
* Each request is stateless: the full game state is sent with the request,
|
||||
* and only the chosen command index is returned.
|
||||
*/
|
||||
class ShardokAIServiceImpl final : public net::eagle0::common::ShardokAIService::Service {
|
||||
private:
|
||||
TokenValidator tokenValidator_;
|
||||
GameSettingsSPtr gameSettings_;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Create the AI service.
|
||||
*
|
||||
* @param gameSettings Shared game settings (immutable, used for all requests)
|
||||
* @param authToken Optional auth token. If non-empty, all requests must include
|
||||
* "authorization: Bearer <token>" metadata.
|
||||
*/
|
||||
explicit ShardokAIServiceImpl(GameSettingsSPtr gameSettings, std::string authToken = "");
|
||||
|
||||
/**
|
||||
* Handle an AI command selection request.
|
||||
*
|
||||
* Reconstructs game state from bytes, runs AI evaluation, returns chosen index.
|
||||
*/
|
||||
auto ChooseAICommand(
|
||||
grpc::ServerContext* context,
|
||||
const net::eagle0::common::ChooseAICommandRequest* request,
|
||||
net::eagle0::common::ChooseAICommandResponse* response) -> grpc::Status override;
|
||||
};
|
||||
|
||||
} // namespace shardok
|
||||
|
||||
#endif /* ShardokAIServiceImpl_hpp */
|
||||
@@ -13,6 +13,7 @@
|
||||
#include <utility>
|
||||
|
||||
#include "ServerConfiguration.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/RemoteAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/ShardokAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/controller/ShardokGameController.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/settings_loader/SettingsLoader.hpp"
|
||||
@@ -30,7 +31,9 @@ auto SetUpController(
|
||||
int month,
|
||||
const vector<Unit> &units,
|
||||
const vector<PlayerInfoProto> &playerSetupInfos,
|
||||
const string &serializedRequest) -> shared_ptr<ShardokGameController>;
|
||||
const string &serializedRequest,
|
||||
const std::shared_ptr<RemoteAIClientFactory> &remoteAIClientFactory)
|
||||
-> shared_ptr<ShardokGameController>;
|
||||
|
||||
ShardokGamesManager::ShardokGamesManager(const std::vector<std::string> &extraSettings) {
|
||||
gameSettings = SettingsLoader::LoadSettings();
|
||||
@@ -46,6 +49,12 @@ ShardokGamesManager::ShardokGamesManager(const std::vector<std::string> &extraSe
|
||||
}
|
||||
}
|
||||
|
||||
void ShardokGamesManager::SetRemoteAIClientFactory(std::shared_ptr<RemoteAIClientFactory> factory) {
|
||||
remoteAIClientFactory = std::move(factory);
|
||||
std::cout << "[ShardokGamesManager] Remote AI client factory "
|
||||
<< (remoteAIClientFactory ? "enabled" : "disabled") << std::endl;
|
||||
}
|
||||
|
||||
auto ShardokGamesManager::GetController(const GameId &gameId)
|
||||
-> std::shared_ptr<ShardokGameController> {
|
||||
std::shared_lock<std::shared_mutex> lk(runningControllersLock);
|
||||
@@ -111,7 +120,8 @@ auto ShardokGamesManager::UnlockedCreateSpecifiedGame(
|
||||
month,
|
||||
units,
|
||||
playerInfos,
|
||||
serializedRequest);
|
||||
serializedRequest,
|
||||
remoteAIClientFactory);
|
||||
|
||||
return LockedCreateGame(controller, playerSetupInfos, serializedRequest);
|
||||
}
|
||||
@@ -137,7 +147,9 @@ auto SetUpController(
|
||||
const int month,
|
||||
const vector<Unit> &units,
|
||||
const vector<PlayerInfoProto> &playerInfos,
|
||||
const string &serializedRequest) -> shared_ptr<ShardokGameController> {
|
||||
const string &serializedRequest,
|
||||
const std::shared_ptr<RemoteAIClientFactory> &remoteAIClientFactory)
|
||||
-> shared_ptr<ShardokGameController> {
|
||||
auto unplacedUnits = std::vector<Unit>();
|
||||
|
||||
UnitId nextUnitId = 0;
|
||||
@@ -155,7 +167,16 @@ auto SetUpController(
|
||||
gameId,
|
||||
playerInfos);
|
||||
|
||||
return std::make_shared<ShardokGameController>(std::move(engine), mapName, serializedRequest);
|
||||
auto controller =
|
||||
std::make_shared<ShardokGameController>(std::move(engine), mapName, serializedRequest);
|
||||
|
||||
// Set up remote AI client if factory is available
|
||||
if (remoteAIClientFactory) {
|
||||
auto remoteClient = remoteAIClientFactory->CreateClient(gameId);
|
||||
if (remoteClient) { controller->SetRemoteAIClient(std::move(remoteClient)); }
|
||||
}
|
||||
|
||||
return controller;
|
||||
}
|
||||
|
||||
} // namespace shardok
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/RemoteAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/ShardokAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/library/settings/GameSettings.hpp"
|
||||
#include "src/main/protobuf/net/eagle0/shardok/common/game_status.pb.h"
|
||||
@@ -81,6 +82,7 @@ public:
|
||||
class ShardokGamesManager {
|
||||
private:
|
||||
GameSettingsSPtr gameSettings;
|
||||
std::shared_ptr<RemoteAIClientFactory> remoteAIClientFactory;
|
||||
|
||||
std::shared_mutex runningControllersLock;
|
||||
gtl::flat_hash_map<GameId, std::shared_ptr<ShardokGameController>> runningControllers;
|
||||
@@ -93,6 +95,17 @@ private:
|
||||
public:
|
||||
explicit ShardokGamesManager(const std::vector<std::string> &extraSettings);
|
||||
|
||||
/**
|
||||
* Set the factory for creating remote AI clients.
|
||||
* When set, game controllers will use remote AI for AI computation.
|
||||
*/
|
||||
void SetRemoteAIClientFactory(std::shared_ptr<RemoteAIClientFactory> factory);
|
||||
|
||||
/**
|
||||
* Get the game settings used by this manager.
|
||||
*/
|
||||
[[nodiscard]] auto GetGameSettings() const -> GameSettingsSPtr { return gameSettings; }
|
||||
|
||||
auto GetController(const GameId &gameId) -> std::shared_ptr<ShardokGameController>;
|
||||
|
||||
auto UnlockedCreateSpecifiedGame(
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include <cstddef>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
@@ -25,8 +26,10 @@ SUPPRESS_PROTOBUF_WARNINGS
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
#include "src/main/cpp/net/eagle0/shardok/ai/RemoteAIClient.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/EagleInterfaceGrpcServer.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/ServerConfiguration.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/ShardokAIServiceImpl.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/ShardokGamesManager.hpp"
|
||||
#include "src/main/cpp/net/eagle0/shardok/server/TokenAuthInterceptor.hpp"
|
||||
|
||||
@@ -52,6 +55,13 @@ auto CreateEagleInterfaceService(
|
||||
const std::shared_ptr<ServerConfiguration> &config,
|
||||
std::shared_ptr<ShardokGamesManager> shardokGamesManager) -> ServerThreadInfo;
|
||||
|
||||
auto CreateAIService(
|
||||
const std::shared_ptr<ServerConfiguration> &config,
|
||||
shardok::GameSettingsSPtr gameSettings) -> std::optional<ServerThreadInfo>;
|
||||
|
||||
auto SetupRemoteAIClient(const std::shared_ptr<ServerConfiguration> &config)
|
||||
-> std::shared_ptr<shardok::RemoteAIClientFactory>;
|
||||
|
||||
auto ReadFileContents(const string &path) -> string;
|
||||
|
||||
void handler(const int sig) {
|
||||
@@ -84,9 +94,27 @@ auto main(const int argc, char **argv) -> int {
|
||||
|
||||
const auto shardokGamesManager = std::make_shared<ShardokGamesManager>(extraSettings);
|
||||
|
||||
ServerThreadInfo eagleInterfaceInfo = CreateEagleInterfaceService(config, shardokGamesManager);
|
||||
// Set up remote AI client if configured (for Shardok-Primary mode)
|
||||
auto remoteAIFactory = SetupRemoteAIClient(config);
|
||||
if (remoteAIFactory) { shardokGamesManager->SetRemoteAIClientFactory(remoteAIFactory); }
|
||||
|
||||
eagleInterfaceInfo.thread.join();
|
||||
// Start AI service if configured (for Shardok-AI mode)
|
||||
std::optional<ServerThreadInfo> aiServiceInfo =
|
||||
CreateAIService(config, shardokGamesManager->GetGameSettings());
|
||||
|
||||
// Start Eagle interface service (may be disabled in AI-only mode)
|
||||
const string eagleInterfaceAddress =
|
||||
config->stringForKey(ServerConfiguration::kEagleInterfaceGrpcAddress);
|
||||
std::optional<ServerThreadInfo> eagleInterfaceInfo;
|
||||
if (!eagleInterfaceAddress.empty()) {
|
||||
eagleInterfaceInfo = CreateEagleInterfaceService(config, shardokGamesManager);
|
||||
} else {
|
||||
std::cout << "Eagle interface disabled (no address configured)" << std::endl;
|
||||
}
|
||||
|
||||
// Wait for services to complete
|
||||
if (eagleInterfaceInfo) { eagleInterfaceInfo->thread.join(); }
|
||||
if (aiServiceInfo) { aiServiceInfo->thread.join(); }
|
||||
}
|
||||
|
||||
auto ReadFileContents(const string &path) -> string {
|
||||
@@ -187,3 +215,61 @@ auto StartInThread(grpc::ServerBuilder *serverBuilder) -> ServerThreadInfo {
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
auto SetupRemoteAIClient(const std::shared_ptr<ServerConfiguration> &config)
|
||||
-> std::shared_ptr<shardok::RemoteAIClientFactory> {
|
||||
const string remoteAIAddress = config->stringForKey(ServerConfiguration::kRemoteAIAddress);
|
||||
|
||||
if (remoteAIAddress.empty()) {
|
||||
std::cout << "Remote AI disabled (no address configured)" << std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Read auth token from file
|
||||
const string authTokenPath = config->stringForKey(ServerConfiguration::kRemoteAIAuthTokenPath);
|
||||
const string authToken = shardok::ReadAuthTokenFromFile(authTokenPath);
|
||||
|
||||
// Parse timeout (default 10 seconds)
|
||||
const string timeoutStr = config->stringForKey(ServerConfiguration::kRemoteAITimeoutMs);
|
||||
auto timeout = std::chrono::milliseconds(10000);
|
||||
if (!timeoutStr.empty()) {
|
||||
try {
|
||||
timeout = std::chrono::milliseconds(std::stoi(timeoutStr));
|
||||
} catch (...) {
|
||||
std::cerr << "Invalid remote AI timeout value: " << timeoutStr << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Remote AI enabled: " << remoteAIAddress << " (timeout=" << timeout.count() << "ms"
|
||||
<< ", auth=" << (authToken.empty() ? "disabled" : "enabled") << ")" << std::endl;
|
||||
|
||||
return std::make_shared<shardok::RemoteAIClientFactory>(remoteAIAddress, authToken, timeout);
|
||||
}
|
||||
|
||||
auto CreateAIService(
|
||||
const std::shared_ptr<ServerConfiguration> &config,
|
||||
shardok::GameSettingsSPtr gameSettings) -> std::optional<ServerThreadInfo> {
|
||||
const string aiServiceAddress = config->stringForKey(ServerConfiguration::kAIServiceAddress);
|
||||
|
||||
if (aiServiceAddress.empty()) { return std::nullopt; }
|
||||
|
||||
// TLS configuration
|
||||
const string certPath = config->stringForKey(ServerConfiguration::kSslCertPath);
|
||||
const string keyPath = config->stringForKey(ServerConfiguration::kSslPrivateKeyPath);
|
||||
|
||||
auto *serverBuilder = CreateServerBuilder(aiServiceAddress, certPath, keyPath);
|
||||
|
||||
// Auth token configuration (same as Eagle interface)
|
||||
const string authTokenPath = config->stringForKey(ServerConfiguration::kAuthTokenPath);
|
||||
const string authToken = shardok::ReadAuthTokenFromFile(authTokenPath);
|
||||
|
||||
auto aiService = std::make_shared<shardok::ShardokAIServiceImpl>(gameSettings, authToken);
|
||||
serverBuilder->RegisterService(aiService.get());
|
||||
|
||||
ServerThreadInfo info = StartInThread(serverBuilder);
|
||||
info.service = aiService;
|
||||
|
||||
std::cout << "AI service listening on " << aiServiceAddress << std::endl;
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
@@ -95,3 +95,31 @@ message HexMapRequest {
|
||||
message HexMapResponse {
|
||||
.net.eagle0.shardok.common.HexMap map = 1;
|
||||
}
|
||||
|
||||
// Service for offloading AI computation to a remote server.
|
||||
// Used by Shardok-Primary (DigitalOcean) to call Shardok-AI (Hetzner).
|
||||
service ShardokAIService {
|
||||
// Stateless AI command selection - receives game state, returns chosen command index
|
||||
rpc ChooseAICommand(ChooseAICommandRequest) returns (ChooseAICommandResponse) {}
|
||||
}
|
||||
|
||||
message ChooseAICommandRequest {
|
||||
string game_id = 1;
|
||||
int32 player_id = 2;
|
||||
bool is_defender = 3;
|
||||
bool is_all_ai_battle = 4;
|
||||
// Serialized game state (FlatBuffers GameState)
|
||||
bytes game_state_bytes = 5;
|
||||
// Available commands for the AI player
|
||||
repeated .net.eagle0.shardok.api.CommandDescriptor available_commands = 6;
|
||||
// Time budget hint in milliseconds
|
||||
int32 time_budget_ms = 7;
|
||||
}
|
||||
|
||||
message ChooseAICommandResponse {
|
||||
int32 chosen_index = 1;
|
||||
// Performance metrics for monitoring
|
||||
int32 depth_achieved = 2;
|
||||
int32 commands_evaluated = 3;
|
||||
int32 elapsed_ms = 4;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user