eagle0/.github/workflows/shardok_arm64_build.yml

name: Shardok ARM64 Build and Push

on:
  push:
    branches: [ "main" ]
    paths:
      - 'src/main/cpp/**'
      - 'src/main/protobuf/net/eagle0/shardok/**'
      - 'src/main/protobuf/net/eagle0/common/**'
      - 'src/main/resources/net/eagle0/shardok/**'
      - 'ci/BUILD.bazel'
      - 'MODULE.bazel'
      - '.bazelrc'
      - '.github/workflows/shardok_arm64_build.yml'
  workflow_dispatch:
    inputs:
      push_images:
        description: 'Push images to container registry'
        required: true
        default: 'true'
        type: boolean

concurrency:
  group: shardok-arm64-deploy
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  build-shardok-arm64:
    runs-on: [self-hosted, bazel]
    outputs:
      image_tag: ${{ steps.push-shardok.outputs.image_tag }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Build Shardok ARM64 binary (cross-compile for Linux ARM64)
        run: |
          set -ex

          echo "=== Building shardok-server binary for linux-aarch64 ==="
          bazel build \
            --platforms=//:linux_arm64 \
            --extra_toolchains=@llvm_toolchain_linux_arm64//:all \
            //src/main/cpp/net/eagle0/shardok:shardok-server

          LINUX_BIN="bazel-bin/src/main/cpp/net/eagle0/shardok/shardok-server"
          echo "=== Checking binary at: $LINUX_BIN ==="

          if [ ! -f "$LINUX_BIN" ]; then
            echo "ERROR: Binary not found at $LINUX_BIN"
            exit 1
          fi

          # Verify it's ELF (Linux) not Mach-O (macOS)
          echo "=== Verifying binary format ==="
          MAGIC=$(head -c 4 "$LINUX_BIN" | xxd -p)
          echo "Binary magic bytes: $MAGIC"

          if [ "$MAGIC" = "7f454c46" ]; then
            echo "SUCCESS: Binary is ELF format (Linux)"
            # Check if it's ARM64 (e_machine = 0xB7 = 183 for aarch64)
            # od -tx2 reads as a 16-bit value in host byte order (little-endian on macOS),
            # so the little-endian ELF bytes b7 00 are displayed as 00b7.
            E_MACHINE=$(od -An -j18 -N2 -tx2 "$LINUX_BIN" | tr -d ' ')
            echo "ELF e_machine: $E_MACHINE"
            if [ "$E_MACHINE" = "00b7" ]; then
              echo "SUCCESS: Binary is ARM64 (aarch64)"
            else
              echo "WARNING: Binary e_machine is $E_MACHINE (expected 00b7 for aarch64)"
            fi
          elif [ "$MAGIC" = "cfaeedfe" ] || [ "$MAGIC" = "cffaedfe" ]; then
            echo "ERROR: Binary is Mach-O format (macOS) - cross-compilation failed!"
            exit 1
          else
            echo "WARNING: Unknown binary format: $MAGIC"
            file "$LINUX_BIN" || true
          fi

      - name: Build Shardok ARM64 Docker image
        id: build-shardok
        run: |
          set -ex

          bazel build \
            --platforms=//:linux_arm64 \
            --extra_toolchains=@llvm_toolchain_linux_arm64//:all \
            //ci:shardok_server_image_arm64

          IMAGE_PATH=$(readlink -f bazel-bin/ci/shardok_server_image_arm64)
          echo "Image path: $IMAGE_PATH"
          echo "image_path=$IMAGE_PATH" >> $GITHUB_OUTPUT

          # Verify the binary inside the tar layer is ARM64 ELF
          echo "=== Verifying binary in image tar ==="
          BINARY_TAR="bazel-bin/ci/shardok_binary_layer_arm64.tar"
          if [ -f "$BINARY_TAR" ]; then
            echo "Checking binary in $BINARY_TAR"
            MAGIC=$(tar -xOf "$BINARY_TAR" app/shardok-server 2>/dev/null | head -c 4 | xxd -p)
            echo "Binary magic in tar: $MAGIC"
            if [ "$MAGIC" = "7f454c46" ]; then
              echo "SUCCESS: Binary in tar is ELF format (Linux)"
            else
              echo "ERROR: Binary in tar is NOT ELF format!"
              exit 1
            fi
          else
            echo "WARNING: Could not find $BINARY_TAR"
          fi

      - name: Login to DigitalOcean Container Registry
        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_images == 'true')
        env:
          DO_REGISTRY_TOKEN: ${{ secrets.DO_REGISTRY_TOKEN }}
        run: |
          AUTH=$(echo -n "${DO_REGISTRY_TOKEN}:${DO_REGISTRY_TOKEN}" | base64)
          echo "{\"auths\":{\"registry.digitalocean.com\":{\"auth\":\"${AUTH}\"}}}" > ~/.docker/config.json

      - name: Push Shardok ARM64 image to DigitalOcean
        id: push-shardok
        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_images == 'true')
        run: |
          set -ex

          CROSS_IMAGE="${{ steps.build-shardok.outputs.image_path }}"
          echo "Using cross-compiled image: $CROSS_IMAGE"

          if [ -z "$CROSS_IMAGE" ] || [ ! -d "$CROSS_IMAGE" ]; then
            echo "ERROR: Cross-compiled image not found at: $CROSS_IMAGE"
            exit 1
          fi

          # Get crane (cached across runs to avoid rebuilding bazel target,
          # which would discard the ARM64 analysis cache due to platform change)
          CRANE_VERSION="v0.20.2"
          CRANE_DIR="${HOME}/.local/bin"
          CRANE="${CRANE_DIR}/crane"
          if [ ! -x "$CRANE" ] || ! "$CRANE" version 2>/dev/null | grep -q "0.20.2"; then
            echo "Installing crane ${CRANE_VERSION}..."
            mkdir -p "$CRANE_DIR"
            curl -sL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Darwin_arm64.tar.gz" | tar xzf - -C "$CRANE_DIR" crane
            chmod +x "$CRANE"
          else
            echo "Using cached crane at $CRANE"
          fi

          # Push with arm64-prefixed SHA tag (same repo as x86, different tag)
          GIT_SHA=$(git rev-parse --short=8 HEAD)
          IMAGE_TAG="registry.digitalocean.com/eagle0/shardok-server:arm64-${GIT_SHA}"
          echo "Pushing shardok ARM64 image: $IMAGE_TAG"
          $CRANE push "$CROSS_IMAGE" "$IMAGE_TAG"

          echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT

          # Also update :arm64-latest tag for convenience
          echo "Copying to :arm64-latest tag"
          $CRANE copy "$IMAGE_TAG" "registry.digitalocean.com/eagle0/shardok-server:arm64-latest"

          echo "=== Push complete ==="
          echo "Image: $IMAGE_TAG"
          echo "Also tagged as: registry.digitalocean.com/eagle0/shardok-server:arm64-latest"

  deploy-hetzner:
    runs-on: [self-hosted, bazel]
    needs: [build-shardok-arm64]
    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_images == 'true')
    environment: production
    env:
      SHARDOK_IMAGE: ${{ needs.build-shardok-arm64.outputs.image_tag }}
    steps:
      - name: Setup SSH key
        run: |
          mkdir -p ~/.ssh
          echo "${{ secrets.HETZNER_SSH_KEY }}" > ~/.ssh/hetzner_deploy
          chmod 600 ~/.ssh/hetzner_deploy
          # Add host key to known_hosts to avoid prompt
          ssh-keyscan -H ${{ secrets.HETZNER_IP }} >> ~/.ssh/known_hosts 2>/dev/null || true

      - name: Deploy to Hetzner
        run: |
          ssh -i ~/.ssh/hetzner_deploy -o StrictHostKeyChecking=accept-new deploy@${{ secrets.HETZNER_IP }} << 'ENDSSH'
          set -ex
          cd /opt/eagle0

          # Login to registry
          echo "${{ secrets.DO_REGISTRY_TOKEN }}" | docker login registry.digitalocean.com -u "${{ secrets.DO_REGISTRY_TOKEN }}" --password-stdin

          echo "Deploying Shardok ARM64: ${{ needs.build-shardok-arm64.outputs.image_tag }}"

          # Pre-warm NAT64 path to DigitalOcean registry (IPv6-only Hetzner → IPv4 registry)
          curl -sf --max-time 10 https://registry.digitalocean.com/v2/ > /dev/null 2>&1 || true

          # Pull the new image (retry up to 3 times for NAT64 connectivity flakiness)
          for attempt in 1 2 3; do
            echo "Pull attempt $attempt..."
            if docker pull "${{ needs.build-shardok-arm64.outputs.image_tag }}"; then
              break
            fi
            if [ "$attempt" -eq 3 ]; then
              echo "ERROR: docker pull failed after 3 attempts"
              exit 1
            fi
            echo "Pull failed, retrying in 10s..."
            sleep 10
          done

          # Stop and remove any container using port 40042 or named shardok*
          docker ps -q --filter "publish=40042" | xargs -r docker stop
          docker ps -aq --filter "name=shardok" | xargs -r docker rm -f
          docker ps -aq --filter "publish=40042" | xargs -r docker rm -f

          # Run new container
          docker run -d \
            --name shardok-ai \
            --restart unless-stopped \
            -p 40042:40042 \
            -v /opt/eagle0/data:/data \
            -v /etc/shardok:/etc/shardok:ro \
            -v /etc/letsencrypt:/etc/letsencrypt:ro \
            -v /usr/local/share/eagle0:/usr/local/share/eagle0:ro \
            -e SHARDOK_RESOURCES_PATH=/app/resources \
            -e SHARDOK_MAPS_PATH=/app/resources/maps \
            "${{ needs.build-shardok-arm64.outputs.image_tag }}"

          # Wait and verify
          sleep 5
          docker ps | grep shardok-ai

          # Cleanup old images
          docker image prune -f

          echo "=== Hetzner deployment complete ==="
          ENDSSH

      - name: Cleanup SSH key
        if: always()
        run: rm -f ~/.ssh/hetzner_deploy