Add Shardok production profiling smoke workflow

.github/workflows/shardok_profile.yml

@@ -0,0 +1,114 @@
+name: Shardok Production Profile
+
+on:
+  workflow_dispatch:
+    inputs:
+      duration_seconds:
+        description: Profile duration
+        required: true
+        default: "30"
+        type: choice
+        options:
+          - "30"
+          - "60"
+          - "120"
+      sample_frequency:
+        description: CPU samples per second
+        required: true
+        default: "99"
+        type: choice
+        options:
+          - "49"
+          - "99"
+
+concurrency:
+  group: shardok-production-profile
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  profile-shardok:
+    name: Audit and profile Hetzner Shardok
+    runs-on: [self-hosted, macOS, ARM64]
+    environment: production
+    timeout-minutes: 10
+    env:
+      PROFILE_DURATION: ${{ inputs.duration_seconds }}
+      SAMPLE_FREQUENCY: ${{ inputs.sample_frequency }}
+      SSH_KEY_PATH: ${{ runner.temp }}/hetzner-profile-key
+      KNOWN_HOSTS_PATH: ${{ runner.temp }}/hetzner-profile-known-hosts
+      REMOTE_PROFILE_DIR: /tmp/eagle0-shardok-profile-${{ github.run_id }}-${{ github.run_attempt }}
+      REMOTE_PROFILE_ARCHIVE: /tmp/eagle0-shardok-profile-${{ github.run_id }}-${{ github.run_attempt }}.tar.gz
+      LOCAL_PROFILE_ARCHIVE: ${{ runner.temp }}/eagle0-shardok-profile-${{ github.run_id }}-${{ github.run_attempt }}.tar.gz
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+          lfs: false
+
+      - name: Configure Hetzner SSH access
+        env:
+          HETZNER_SSH_KEY: ${{ secrets.HETZNER_SSH_KEY }}
+          HETZNER_IP: ${{ secrets.HETZNER_IP }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$HETZNER_SSH_KEY" > "$SSH_KEY_PATH"
+          chmod 600 "$SSH_KEY_PATH"
+          ssh-keyscan -H "$HETZNER_IP" > "$KNOWN_HOSTS_PATH" 2>/dev/null
+
+      - name: Audit host and capture profile
+        env:
+          HETZNER_IP: ${{ secrets.HETZNER_IP }}
+        run: |
+          set -euo pipefail
+          ssh \
+            -i "$SSH_KEY_PATH" \
+            -o StrictHostKeyChecking=yes \
+            -o UserKnownHostsFile="$KNOWN_HOSTS_PATH" \
+            "deploy@$HETZNER_IP" \
+            bash -s -- \
+            "$PROFILE_DURATION" \
+            "$SAMPLE_FREQUENCY" \
+            "$REMOTE_PROFILE_DIR" \
+            < scripts/capture_shardok_profile.sh
+
+      - name: Copy profile artifact from Hetzner
+        if: always()
+        env:
+          HETZNER_IP: ${{ secrets.HETZNER_IP }}
+        run: |
+          set -euo pipefail
+          scp \
+            -i "$SSH_KEY_PATH" \
+            -o StrictHostKeyChecking=yes \
+            -o UserKnownHostsFile="$KNOWN_HOSTS_PATH" \
+            "deploy@$HETZNER_IP:$REMOTE_PROFILE_ARCHIVE" \
+            "$LOCAL_PROFILE_ARCHIVE"
+
+      - name: Upload profile artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: shardok-profile-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.LOCAL_PROFILE_ARCHIVE }}
+          if-no-files-found: warn
+          retention-days: 14
+          compression-level: 0
+
+      - name: Clean up profile files and SSH key
+        if: always()
+        env:
+          HETZNER_IP: ${{ secrets.HETZNER_IP }}
+        run: |
+          if [ -f "$SSH_KEY_PATH" ] && [ -f "$KNOWN_HOSTS_PATH" ]; then
+            ssh \
+              -i "$SSH_KEY_PATH" \
+              -o StrictHostKeyChecking=yes \
+              -o UserKnownHostsFile="$KNOWN_HOSTS_PATH" \
+              "deploy@$HETZNER_IP" \
+              rm -rf -- "$REMOTE_PROFILE_DIR" "$REMOTE_PROFILE_ARCHIVE" || true
+          fi
+          rm -f "$SSH_KEY_PATH" "$KNOWN_HOSTS_PATH" "$LOCAL_PROFILE_ARCHIVE"

scripts/capture_shardok_profile.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly duration_seconds="${1:-}"
+readonly sample_frequency="${2:-}"
+readonly profile_dir="${3:-}"
+readonly profile_archive="${profile_dir}.tar.gz"
+
+case "$duration_seconds" in
+    30|60|120) ;;
+    *) echo "Invalid profile duration: $duration_seconds" >&2; exit 2 ;;
+esac
+case "$sample_frequency" in
+    49|99) ;;
+    *) echo "Invalid sample frequency: $sample_frequency" >&2; exit 2 ;;
+esac
+if [[ ! "$profile_dir" =~ ^/tmp/eagle0-shardok-profile-[0-9]+-[0-9]+$ ]]; then
+    echo "Invalid profile directory: $profile_dir" >&2
+    exit 2
+fi
+
+mkdir -m 700 "$profile_dir"
+
+finalize() {
+    local exit_code=$?
+    trap - EXIT
+    printf '%s\n' "$exit_code" > "$profile_dir/capture-exit-code.txt"
+    printf 'capture_finished_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+        >> "$profile_dir/metadata.txt"
+    if ! tar -C /tmp -czf "$profile_archive" "$(basename "$profile_dir")"; then
+        echo "Failed to create profile archive" >&2
+        exit 90
+    fi
+    exit "$exit_code"
+}
+trap finalize EXIT
+
+{
+    printf 'capture_started_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    printf 'duration_seconds=%s\n' "$duration_seconds"
+    printf 'sample_frequency=%s\n' "$sample_frequency"
+    printf 'user=%s\n' "$(id)"
+    printf '\n=== uname ===\n'
+    uname -a
+    printf '\n=== os-release ===\n'
+    cat /etc/os-release 2>/dev/null || true
+    printf '\n=== perf controls ===\n'
+    printf 'perf_event_paranoid='
+    cat /proc/sys/kernel/perf_event_paranoid 2>/dev/null || true
+    printf 'kptr_restrict='
+    cat /proc/sys/kernel/kptr_restrict 2>/dev/null || true
+    printf 'ptrace_scope='
+    cat /proc/sys/kernel/yama/ptrace_scope 2>/dev/null || true
+    if sudo -n true 2>/dev/null; then
+        printf 'noninteractive_sudo=true\n'
+    else
+        printf 'noninteractive_sudo=false\n'
+    fi
+} > "$profile_dir/metadata.txt"
+
+if ! command -v docker > /dev/null; then
+    printf 'docker_not_installed\n' > "$profile_dir/capture-status.txt"
+    exit 10
+fi
+if ! docker inspect shardok-ai > /dev/null 2>&1; then
+    printf 'shardok_container_not_running\n' > "$profile_dir/capture-status.txt"
+    exit 11
+fi
+
+host_pid="$(docker inspect --format '{{.State.Pid}}' shardok-ai)"
+case "$host_pid" in
+    ''|*[!0-9]*)
+        printf 'invalid_shardok_host_pid=%s\n' "$host_pid" > "$profile_dir/capture-status.txt"
+        exit 12
+        ;;
+esac
+if [[ "$host_pid" -le 1 || ! -d "/proc/$host_pid" ]]; then
+    printf 'shardok_host_pid_not_alive=%s\n' "$host_pid" > "$profile_dir/capture-status.txt"
+    exit 12
+fi
+readonly host_pid
+
+{
+    printf 'container_id=%s\n' "$(docker inspect --format '{{.Id}}' shardok-ai)"
+    printf 'container_image=%s\n' "$(docker inspect --format '{{.Config.Image}}' shardok-ai)"
+    printf 'image_id=%s\n' "$(docker inspect --format '{{.Image}}' shardok-ai)"
+    printf 'container_started_at=%s\n' "$(docker inspect --format '{{.State.StartedAt}}' shardok-ai)"
+    printf 'host_pid=%s\n' "$host_pid"
+    printf 'binary_sha256=%s\n' \
+        "$(docker exec shardok-ai sha256sum /app/shardok-server | awk '{print $1}')"
+} > "$profile_dir/shardok-container.txt"
+
+ps -L -p "$host_pid" -o pid,tid,psr,pcpu,pmem,stat,comm \
+    > "$profile_dir/shardok-threads-before.txt" 2>&1 || true
+cat "/proc/$host_pid/status" > "$profile_dir/shardok-process-status.txt" 2>&1 || true
+cat "/proc/$host_pid/limits" > "$profile_dir/shardok-process-limits.txt" 2>&1 || true
+cat "/proc/$host_pid/cgroup" > "$profile_dir/shardok-process-cgroup.txt" 2>&1 || true
+docker stats --no-stream shardok-ai > "$profile_dir/docker-stats-before.txt" 2>&1 || true
+
+perf_bin="$(command -v perf || true)"
+if [[ -z "$perf_bin" ]]; then
+    printf 'perf_not_installed\n' > "$profile_dir/capture-status.txt"
+    exit 20
+fi
+readonly perf_bin
+"$perf_bin" version > "$profile_dir/perf-version.txt" 2>&1 || true
+if command -v getcap > /dev/null; then
+    getcap "$perf_bin" > "$profile_dir/perf-capabilities.txt" 2>&1 || true
+fi
+
+capture_start="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+vmstat_pid=""
+if command -v vmstat > /dev/null; then
+    vmstat 1 "$((duration_seconds + 1))" > "$profile_dir/vmstat.txt" 2>&1 &
+    vmstat_pid=$!
+fi
+
+set +e
+"$perf_bin" record \
+    --freq "$sample_frequency" \
+    --event cpu-clock \
+    --call-graph fp \
+    --pid "$host_pid" \
+    --output "$profile_dir/perf.data" \
+    -- sleep "$duration_seconds" \
+    > "$profile_dir/perf-record.stdout.txt" \
+    2> "$profile_dir/perf-record.stderr.txt"
+perf_exit_code=$?
+set -e
+
+if [[ -n "$vmstat_pid" ]]; then
+    if [[ "$perf_exit_code" -ne 0 ]]; then
+        kill "$vmstat_pid" 2>/dev/null || true
+    fi
+    wait "$vmstat_pid" 2>/dev/null || true
+fi
+capture_end="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+docker logs \
+    --timestamps \
+    --since "$capture_start" \
+    --until "$capture_end" \
+    shardok-ai > "$profile_dir/shardok-container.log" 2>&1 || true
+ps -L -p "$host_pid" -o pid,tid,psr,pcpu,pmem,stat,comm \
+    > "$profile_dir/shardok-threads-after.txt" 2>&1 || true
+docker stats --no-stream shardok-ai > "$profile_dir/docker-stats-after.txt" 2>&1 || true
+
+printf 'perf_exit_code=%s\n' "$perf_exit_code" >> "$profile_dir/metadata.txt"
+if [[ "$perf_exit_code" -ne 0 || ! -s "$profile_dir/perf.data" ]]; then
+    printf 'perf_record_failed\n' > "$profile_dir/capture-status.txt"
+    exit 21
+fi
+
+"$perf_bin" report \
+    --input "$profile_dir/perf.data" \
+    --stdio \
+    --sort comm,dso,symbol \
+    > "$profile_dir/perf-report.txt" 2> "$profile_dir/perf-report.stderr.txt" || true
+"$perf_bin" script \
+    --input "$profile_dir/perf.data" \
+    > "$profile_dir/perf-script.txt" 2> "$profile_dir/perf-script.stderr.txt" || true
+"$perf_bin" buildid-list \
+    --input "$profile_dir/perf.data" \
+    > "$profile_dir/perf-buildids.txt" 2>&1 || true
+
+if grep -q 'shardok-server' "$profile_dir/perf-report.txt"; then
+    printf 'profile_captured_with_shardok_samples\n' > "$profile_dir/capture-status.txt"
+else
+    printf 'profile_captured_without_shardok_samples\n' > "$profile_dir/capture-status.txt"
+fi