Add Shardok profiling admin backend (#8754)

docker-compose.prod.yml

@@ -234,6 +234,10 @@ services:
       - "auth:40033"
       - "--jfr-sidecar-addr"
       - "${JFR_SIDECAR_ADDR:-jfr-sidecar:8081}"  # Can be switched for blue-green
+      - "--shardok-profile-addr"
+      - "${SHARDOK_ADDRESS}"
+      - "--shardok-profile-tls"
+      - "true"
       - "--http-port"
       - "8080"
       - "--maps-dir"
@@ -247,6 +251,8 @@ services:
       DO_SPACES_SECRET_KEY: "${ADMIN_S3_SECRET_KEY:-}"
       # GitHub PAT for map editor PR creation (scoped to contents:write + pull_requests:write)
       GITHUB_TOKEN: "${GITHUB_TOKEN:-}"
+      # Reuse Eagle's Shardok endpoint and auth token for admin-only CPU profiling.
+      SHARDOK_AUTH_TOKEN: "${SHARDOK_AUTH_TOKEN:-}"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     # No external port - accessed via nginx at admin.eagle0.net

src/main/go/net/eagle0/admin_server/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "admin_server_lib",
@@ -6,6 +6,7 @@ go_library(
         "admin_server.go",
         "github_pr.go",
         "map_editor.go",
+        "shardok_profile.go",
     ],
     embedsrcs = glob([
         "static/**",
@@ -15,6 +16,7 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//src/main/go/net/eagle0/util/aws",
+        "//src/main/protobuf/net/eagle0/common:shardok_internal_interface_go_grpc",
         "//src/main/protobuf/net/eagle0/eagle/admin:game_admin_go_proto",
         "//src/main/protobuf/net/eagle0/eagle/api:api_go_proto",  # keep
         "//src/main/protobuf/net/eagle0/eagle/api/admin:admin_go_proto",
@@ -49,3 +51,10 @@ go_binary(
         "main.gitCommit": "{STABLE_GIT_COMMIT}",
     },
 )
+
+go_test(
+    name = "admin_server_test",
+    srcs = ["shardok_profile_test.go"],
+    embed = [":admin_server_lib"],
+    deps = ["//src/main/protobuf/net/eagle0/common:shardok_internal_interface_go_grpc"],
+)

src/main/go/net/eagle0/admin_server/admin_server.go

@@ -234,6 +234,15 @@ func main() {
 		log.Printf("Connected to Auth server at %s", *authAddr)
 	}
 
+	shardokProfileConn, err := initializeShardokProfileClient()
+	if err != nil {
+		log.Fatalf("Failed to configure Shardok CPU profiling: %v", err)
+	}
+	if shardokProfileConn != nil {
+		defer shardokProfileConn.Close()
+		log.Printf("Shardok CPU profiling configured through %s", *shardokProfileAddr)
+	}
+
 	// Serve static files
 	staticContent, err := fs.Sub(staticFS, "static")
 	if err != nil {
@@ -279,6 +288,10 @@ func main() {
 	http.HandleFunc("/jfr/start", requireAuth(handleJfrStart))
 	http.HandleFunc("/jfr/stop", requireAuth(handleJfrStop))
 	http.HandleFunc("/jfr/download", requireAuth(handleJfrDownload))
+	http.HandleFunc("/shardok-profile/status", requireAuth(handleShardokProfileStatus))
+	http.HandleFunc("/shardok-profile/start", requireAuth(handleShardokProfileStart))
+	http.HandleFunc("/shardok-profile/stop", requireAuth(handleShardokProfileStop))
+	http.HandleFunc("/shardok-profile/download", requireAuth(handleShardokProfileDownload))
 	http.HandleFunc("/postgres-benchmark", requireAuth(handlePostgresBenchmark))
 	http.HandleFunc("/postgres-history-smoke", requireAuth(handlePostgresHistorySmoke))
 	http.HandleFunc("/server/restart", requireAuth(handleEagleRestart))

src/main/go/net/eagle0/admin_server/shardok_profile.go

@@ -0,0 +1,261 @@
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	shardokpb "github.com/nolen777/eagle0/src/main/protobuf/net/eagle0/common/shardok_internal_interface"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+)
+
+var (
+	shardokProfileAddr   = flag.String("shardok-profile-addr", "", "Shardok gRPC address for CPU profiling")
+	shardokProfileTLS    = flag.Bool("shardok-profile-tls", true, "Use TLS for Shardok CPU profiling")
+	shardokProfileClient shardokpb.ShardokInternalInterfaceClient
+)
+
+type shardokProfileStatusResponse struct {
+	Recording         bool   `json:"recording"`
+	ProfileAvailable  bool   `json:"profile_available"`
+	DurationSeconds   int32  `json:"duration_seconds,omitempty"`
+	StartedAtEpochMs  int64  `json:"started_at_epoch_millis,omitempty"`
+	ProfileSizeBytes  int64  `json:"profile_size_bytes,omitempty"`
+	SampleFrequencyHz int32  `json:"sample_frequency_hz,omitempty"`
+	Status            string `json:"status,omitempty"`
+	Error             string `json:"error,omitempty"`
+}
+
+func initializeShardokProfileClient() (*grpc.ClientConn, error) {
+	if *shardokProfileAddr == "" {
+		return nil, nil
+	}
+
+	var transportCredentials credentials.TransportCredentials
+	if *shardokProfileTLS {
+		transportCredentials = credentials.NewClientTLSFromCert(nil, "")
+	} else {
+		transportCredentials = insecure.NewCredentials()
+	}
+
+	conn, err := grpc.NewClient(
+		*shardokProfileAddr,
+		grpc.WithTransportCredentials(transportCredentials),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("connect to Shardok profiler: %w", err)
+	}
+	shardokProfileClient = shardokpb.NewShardokInternalInterfaceClient(conn)
+	return conn, nil
+}
+
+func shardokProfileDisabled() bool {
+	return shardokProfileClient == nil
+}
+
+func shardokProfileContext(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
+	ctx, cancel := context.WithTimeout(parent, timeout)
+	if token := os.Getenv("SHARDOK_AUTH_TOKEN"); token != "" {
+		ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token)
+	}
+	return ctx, cancel
+}
+
+func writeShardokProfileJSON(w http.ResponseWriter, status int, response shardokProfileStatusResponse) {
+	w.Header().Set("Cache-Control", "no-store")
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		log.Printf("Failed to write Shardok profile response: %v", err)
+	}
+}
+
+func profileStatusResponse(status *shardokpb.CpuProfileStatusResponse) shardokProfileStatusResponse {
+	return shardokProfileStatusResponse{
+		Recording:         status.GetRecording(),
+		ProfileAvailable:  status.GetProfileAvailable(),
+		DurationSeconds:   status.GetDurationSeconds(),
+		StartedAtEpochMs:  status.GetStartedAtEpochMillis(),
+		ProfileSizeBytes:  status.GetProfileSizeBytes(),
+		SampleFrequencyHz: status.GetSampleFrequencyHz(),
+	}
+}
+
+func handleShardokProfileStatus(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if shardokProfileDisabled() {
+		writeShardokProfileJSON(w, http.StatusOK, shardokProfileStatusResponse{
+			Status: "disabled",
+		})
+		return
+	}
+
+	ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second)
+	defer cancel()
+	status, err := shardokProfileClient.GetCpuProfileStatus(ctx, &shardokpb.CpuProfileStatusRequest{})
+	if err != nil {
+		writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()})
+		return
+	}
+	writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status))
+}
+
+func handleShardokProfileStart(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if shardokProfileDisabled() {
+		http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable)
+		return
+	}
+
+	duration, err := strconv.Atoi(r.URL.Query().Get("duration"))
+	if err != nil || (duration != 30 && duration != 60 && duration != 120) {
+		writeShardokProfileJSON(w, http.StatusBadRequest, shardokProfileStatusResponse{
+			Error: "Profile duration must be 30, 60, or 120 seconds",
+		})
+		return
+	}
+
+	ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second)
+	defer cancel()
+	status, err := shardokProfileClient.StartCpuProfile(ctx, &shardokpb.StartCpuProfileRequest{
+		DurationSeconds: int32(duration),
+	})
+	if err != nil {
+		writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()})
+		return
+	}
+	writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status))
+}
+
+func handleShardokProfileStop(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if shardokProfileDisabled() {
+		http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable)
+		return
+	}
+
+	ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second)
+	defer cancel()
+	status, err := shardokProfileClient.StopCpuProfile(ctx, &shardokpb.StopCpuProfileRequest{})
+	if err != nil {
+		writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()})
+		return
+	}
+	writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status))
+}
+
+func copyShardokProfileArtifact(
+	ctx context.Context,
+	zipWriter *zip.Writer,
+	filename string,
+	artifactType shardokpb.CpuProfileArtifactType,
+) error {
+	header := &zip.FileHeader{Name: filename, Method: zip.Store}
+	header.SetModTime(time.Now())
+	entry, err := zipWriter.CreateHeader(header)
+	if err != nil {
+		return fmt.Errorf("create %s zip entry: %w", filename, err)
+	}
+
+	stream, err := shardokProfileClient.DownloadCpuProfile(ctx, &shardokpb.CpuProfileDownloadRequest{
+		ArtifactType: artifactType,
+	})
+	if err != nil {
+		return fmt.Errorf("start %s download: %w", filename, err)
+	}
+	for {
+		chunk, receiveErr := stream.Recv()
+		if receiveErr == io.EOF {
+			return nil
+		}
+		if receiveErr != nil {
+			return fmt.Errorf("download %s: %w", filename, receiveErr)
+		}
+		if _, err := entry.Write(chunk.GetData()); err != nil {
+			return fmt.Errorf("write %s: %w", filename, err)
+		}
+	}
+}
+
+func handleShardokProfileDownload(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if shardokProfileDisabled() {
+		http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable)
+		return
+	}
+
+	ctx, cancel := shardokProfileContext(r.Context(), 5*time.Minute)
+	defer cancel()
+	status, err := shardokProfileClient.GetCpuProfileStatus(ctx, &shardokpb.CpuProfileStatusRequest{})
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Failed to get Shardok profile status: %v", err), http.StatusBadGateway)
+		return
+	}
+	if !status.GetProfileAvailable() {
+		http.Error(w, "No Shardok CPU profile is available", http.StatusConflict)
+		return
+	}
+
+	timestamp := time.Now().Format("2006-01-02T15-04-05")
+	w.Header().Set("Content-Type", "application/zip")
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=shardok-profile-%s.zip", timestamp))
+	w.Header().Set("Cache-Control", "no-store")
+
+	zipWriter := zip.NewWriter(w)
+	defer func() {
+		if err := zipWriter.Close(); err != nil {
+			log.Printf("Failed to finish Shardok profile zip: %v", err)
+		}
+	}()
+
+	if err := copyShardokProfileArtifact(
+		ctx,
+		zipWriter,
+		"shardok.prof",
+		shardokpb.CpuProfileArtifactType_CPU_PROFILE_ARTIFACT_TYPE_PROFILE,
+	); err != nil {
+		log.Printf("Failed to stream Shardok CPU profile: %v", err)
+		return
+	}
+	if err := copyShardokProfileArtifact(
+		ctx,
+		zipWriter,
+		"shardok-server",
+		shardokpb.CpuProfileArtifactType_CPU_PROFILE_ARTIFACT_TYPE_EXECUTABLE,
+	); err != nil {
+		log.Printf("Failed to stream Shardok executable: %v", err)
+		return
+	}
+
+	readme, err := zipWriter.Create("README.txt")
+	if err != nil {
+		log.Printf("Failed to add Shardok profile README: %v", err)
+		return
+	}
+	if _, err := fmt.Fprintln(readme, "Analyze with: go tool pprof shardok-server shardok.prof"); err != nil {
+		log.Printf("Failed to write Shardok profile README: %v", err)
+	}
+}

src/main/go/net/eagle0/admin_server/shardok_profile_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	shardokpb "github.com/nolen777/eagle0/src/main/protobuf/net/eagle0/common/shardok_internal_interface"
+)
+
+func TestShardokProfileStatusResponse(t *testing.T) {
+	response := profileStatusResponse(&shardokpb.CpuProfileStatusResponse{
+		Recording:            true,
+		ProfileAvailable:     true,
+		DurationSeconds:      60,
+		StartedAtEpochMillis: 1234,
+		ProfileSizeBytes:     5678,
+		SampleFrequencyHz:    100,
+	})
+
+	if !response.Recording || !response.ProfileAvailable {
+		t.Fatalf("unexpected status flags: %+v", response)
+	}
+	if response.DurationSeconds != 60 || response.SampleFrequencyHz != 100 {
+		t.Fatalf("unexpected profile settings: %+v", response)
+	}
+	if response.StartedAtEpochMs != 1234 || response.ProfileSizeBytes != 5678 {
+		t.Fatalf("unexpected profile metadata: %+v", response)
+	}
+}
+
+func TestShardokProfileStatusDisabled(t *testing.T) {
+	previousClient := shardokProfileClient
+	shardokProfileClient = nil
+	defer func() { shardokProfileClient = previousClient }()
+
+	request := httptest.NewRequest(http.MethodGet, "/shardok-profile/status", nil)
+	recorder := httptest.NewRecorder()
+	handleShardokProfileStatus(recorder, request)
+
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("unexpected HTTP status: %d", recorder.Code)
+	}
+	if recorder.Header().Get("Cache-Control") != "no-store" {
+		t.Fatalf("unexpected cache policy: %q", recorder.Header().Get("Cache-Control"))
+	}
+
+	var response shardokProfileStatusResponse
+	if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+	if response.Status != "disabled" {
+		t.Fatalf("unexpected response: %+v", response)
+	}
+}
+
+func TestShardokProfileEndpointsRequireExpectedMethod(t *testing.T) {
+	tests := []struct {
+		handler http.HandlerFunc
+		method  string
+	}{
+		{handler: handleShardokProfileStatus, method: http.MethodPost},
+		{handler: handleShardokProfileStart, method: http.MethodGet},
+		{handler: handleShardokProfileStop, method: http.MethodGet},
+		{handler: handleShardokProfileDownload, method: http.MethodPost},
+	}
+	for _, test := range tests {
+		request := httptest.NewRequest(test.method, "/shardok-profile", nil)
+		recorder := httptest.NewRecorder()
+		test.handler(recorder, request)
+
+		if recorder.Code != http.StatusMethodNotAllowed {
+			t.Fatalf("unexpected HTTP status: %d", recorder.Code)
+		}
+	}
+}