From 46400dd6e1091078737e57c27d2feb83a04331fb Mon Sep 17 00:00:00 2001 From: Dan Crosby Date: Thu, 23 Jul 2026 07:15:34 -0700 Subject: [PATCH] Add Shardok profiling admin backend (#8754) --- docker-compose.prod.yml | 6 + .../go/net/eagle0/admin_server/BUILD.bazel | 11 +- .../net/eagle0/admin_server/admin_server.go | 13 + .../eagle0/admin_server/shardok_profile.go | 261 ++++++++++++++++++ .../admin_server/shardok_profile_test.go | 77 ++++++ 5 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 src/main/go/net/eagle0/admin_server/shardok_profile.go create mode 100644 src/main/go/net/eagle0/admin_server/shardok_profile_test.go diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 965081bbf8..368c51060f 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -234,6 +234,10 @@ services: - "auth:40033" - "--jfr-sidecar-addr" - "${JFR_SIDECAR_ADDR:-jfr-sidecar:8081}" # Can be switched for blue-green + - "--shardok-profile-addr" + - "${SHARDOK_ADDRESS}" + - "--shardok-profile-tls" + - "true" - "--http-port" - "8080" - "--maps-dir" @@ -247,6 +251,8 @@ services: DO_SPACES_SECRET_KEY: "${ADMIN_S3_SECRET_KEY:-}" # GitHub PAT for map editor PR creation (scoped to contents:write + pull_requests:write) GITHUB_TOKEN: "${GITHUB_TOKEN:-}" + # Reuse Eagle's Shardok endpoint and auth token for admin-only CPU profiling. + SHARDOK_AUTH_TOKEN: "${SHARDOK_AUTH_TOKEN:-}" volumes: - /var/run/docker.sock:/var/run/docker.sock # No external port - accessed via nginx at admin.eagle0.net diff --git a/src/main/go/net/eagle0/admin_server/BUILD.bazel b/src/main/go/net/eagle0/admin_server/BUILD.bazel index 2632c0f15c..594875454f 100644 --- a/src/main/go/net/eagle0/admin_server/BUILD.bazel +++ b/src/main/go/net/eagle0/admin_server/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") go_library( name = "admin_server_lib", @@ -6,6 +6,7 @@ go_library( "admin_server.go", "github_pr.go", "map_editor.go", + "shardok_profile.go", ], embedsrcs = glob([ "static/**", @@ -15,6 +16,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//src/main/go/net/eagle0/util/aws", + "//src/main/protobuf/net/eagle0/common:shardok_internal_interface_go_grpc", "//src/main/protobuf/net/eagle0/eagle/admin:game_admin_go_proto", "//src/main/protobuf/net/eagle0/eagle/api:api_go_proto", # keep "//src/main/protobuf/net/eagle0/eagle/api/admin:admin_go_proto", @@ -49,3 +51,10 @@ go_binary( "main.gitCommit": "{STABLE_GIT_COMMIT}", }, ) + +go_test( + name = "admin_server_test", + srcs = ["shardok_profile_test.go"], + embed = [":admin_server_lib"], + deps = ["//src/main/protobuf/net/eagle0/common:shardok_internal_interface_go_grpc"], +) diff --git a/src/main/go/net/eagle0/admin_server/admin_server.go b/src/main/go/net/eagle0/admin_server/admin_server.go index 65350c3002..08fba7321b 100644 --- a/src/main/go/net/eagle0/admin_server/admin_server.go +++ b/src/main/go/net/eagle0/admin_server/admin_server.go @@ -234,6 +234,15 @@ func main() { log.Printf("Connected to Auth server at %s", *authAddr) } + shardokProfileConn, err := initializeShardokProfileClient() + if err != nil { + log.Fatalf("Failed to configure Shardok CPU profiling: %v", err) + } + if shardokProfileConn != nil { + defer shardokProfileConn.Close() + log.Printf("Shardok CPU profiling configured through %s", *shardokProfileAddr) + } + // Serve static files staticContent, err := fs.Sub(staticFS, "static") if err != nil { @@ -279,6 +288,10 @@ func main() { http.HandleFunc("/jfr/start", requireAuth(handleJfrStart)) http.HandleFunc("/jfr/stop", requireAuth(handleJfrStop)) http.HandleFunc("/jfr/download", requireAuth(handleJfrDownload)) + http.HandleFunc("/shardok-profile/status", requireAuth(handleShardokProfileStatus)) + http.HandleFunc("/shardok-profile/start", requireAuth(handleShardokProfileStart)) + http.HandleFunc("/shardok-profile/stop", requireAuth(handleShardokProfileStop)) + http.HandleFunc("/shardok-profile/download", requireAuth(handleShardokProfileDownload)) http.HandleFunc("/postgres-benchmark", requireAuth(handlePostgresBenchmark)) http.HandleFunc("/postgres-history-smoke", requireAuth(handlePostgresHistorySmoke)) http.HandleFunc("/server/restart", requireAuth(handleEagleRestart)) diff --git a/src/main/go/net/eagle0/admin_server/shardok_profile.go b/src/main/go/net/eagle0/admin_server/shardok_profile.go new file mode 100644 index 0000000000..383d0c3bd8 --- /dev/null +++ b/src/main/go/net/eagle0/admin_server/shardok_profile.go @@ -0,0 +1,261 @@ +package main + +import ( + "archive/zip" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "time" + + shardokpb "github.com/nolen777/eagle0/src/main/protobuf/net/eagle0/common/shardok_internal_interface" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +var ( + shardokProfileAddr = flag.String("shardok-profile-addr", "", "Shardok gRPC address for CPU profiling") + shardokProfileTLS = flag.Bool("shardok-profile-tls", true, "Use TLS for Shardok CPU profiling") + shardokProfileClient shardokpb.ShardokInternalInterfaceClient +) + +type shardokProfileStatusResponse struct { + Recording bool `json:"recording"` + ProfileAvailable bool `json:"profile_available"` + DurationSeconds int32 `json:"duration_seconds,omitempty"` + StartedAtEpochMs int64 `json:"started_at_epoch_millis,omitempty"` + ProfileSizeBytes int64 `json:"profile_size_bytes,omitempty"` + SampleFrequencyHz int32 `json:"sample_frequency_hz,omitempty"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` +} + +func initializeShardokProfileClient() (*grpc.ClientConn, error) { + if *shardokProfileAddr == "" { + return nil, nil + } + + var transportCredentials credentials.TransportCredentials + if *shardokProfileTLS { + transportCredentials = credentials.NewClientTLSFromCert(nil, "") + } else { + transportCredentials = insecure.NewCredentials() + } + + conn, err := grpc.NewClient( + *shardokProfileAddr, + grpc.WithTransportCredentials(transportCredentials), + ) + if err != nil { + return nil, fmt.Errorf("connect to Shardok profiler: %w", err) + } + shardokProfileClient = shardokpb.NewShardokInternalInterfaceClient(conn) + return conn, nil +} + +func shardokProfileDisabled() bool { + return shardokProfileClient == nil +} + +func shardokProfileContext(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithTimeout(parent, timeout) + if token := os.Getenv("SHARDOK_AUTH_TOKEN"); token != "" { + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token) + } + return ctx, cancel +} + +func writeShardokProfileJSON(w http.ResponseWriter, status int, response shardokProfileStatusResponse) { + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(response); err != nil { + log.Printf("Failed to write Shardok profile response: %v", err) + } +} + +func profileStatusResponse(status *shardokpb.CpuProfileStatusResponse) shardokProfileStatusResponse { + return shardokProfileStatusResponse{ + Recording: status.GetRecording(), + ProfileAvailable: status.GetProfileAvailable(), + DurationSeconds: status.GetDurationSeconds(), + StartedAtEpochMs: status.GetStartedAtEpochMillis(), + ProfileSizeBytes: status.GetProfileSizeBytes(), + SampleFrequencyHz: status.GetSampleFrequencyHz(), + } +} + +func handleShardokProfileStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + if shardokProfileDisabled() { + writeShardokProfileJSON(w, http.StatusOK, shardokProfileStatusResponse{ + Status: "disabled", + }) + return + } + + ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second) + defer cancel() + status, err := shardokProfileClient.GetCpuProfileStatus(ctx, &shardokpb.CpuProfileStatusRequest{}) + if err != nil { + writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()}) + return + } + writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status)) +} + +func handleShardokProfileStart(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + if shardokProfileDisabled() { + http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable) + return + } + + duration, err := strconv.Atoi(r.URL.Query().Get("duration")) + if err != nil || (duration != 30 && duration != 60 && duration != 120) { + writeShardokProfileJSON(w, http.StatusBadRequest, shardokProfileStatusResponse{ + Error: "Profile duration must be 30, 60, or 120 seconds", + }) + return + } + + ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second) + defer cancel() + status, err := shardokProfileClient.StartCpuProfile(ctx, &shardokpb.StartCpuProfileRequest{ + DurationSeconds: int32(duration), + }) + if err != nil { + writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()}) + return + } + writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status)) +} + +func handleShardokProfileStop(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + if shardokProfileDisabled() { + http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable) + return + } + + ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second) + defer cancel() + status, err := shardokProfileClient.StopCpuProfile(ctx, &shardokpb.StopCpuProfileRequest{}) + if err != nil { + writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()}) + return + } + writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status)) +} + +func copyShardokProfileArtifact( + ctx context.Context, + zipWriter *zip.Writer, + filename string, + artifactType shardokpb.CpuProfileArtifactType, +) error { + header := &zip.FileHeader{Name: filename, Method: zip.Store} + header.SetModTime(time.Now()) + entry, err := zipWriter.CreateHeader(header) + if err != nil { + return fmt.Errorf("create %s zip entry: %w", filename, err) + } + + stream, err := shardokProfileClient.DownloadCpuProfile(ctx, &shardokpb.CpuProfileDownloadRequest{ + ArtifactType: artifactType, + }) + if err != nil { + return fmt.Errorf("start %s download: %w", filename, err) + } + for { + chunk, receiveErr := stream.Recv() + if receiveErr == io.EOF { + return nil + } + if receiveErr != nil { + return fmt.Errorf("download %s: %w", filename, receiveErr) + } + if _, err := entry.Write(chunk.GetData()); err != nil { + return fmt.Errorf("write %s: %w", filename, err) + } + } +} + +func handleShardokProfileDownload(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + if shardokProfileDisabled() { + http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable) + return + } + + ctx, cancel := shardokProfileContext(r.Context(), 5*time.Minute) + defer cancel() + status, err := shardokProfileClient.GetCpuProfileStatus(ctx, &shardokpb.CpuProfileStatusRequest{}) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to get Shardok profile status: %v", err), http.StatusBadGateway) + return + } + if !status.GetProfileAvailable() { + http.Error(w, "No Shardok CPU profile is available", http.StatusConflict) + return + } + + timestamp := time.Now().Format("2006-01-02T15-04-05") + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=shardok-profile-%s.zip", timestamp)) + w.Header().Set("Cache-Control", "no-store") + + zipWriter := zip.NewWriter(w) + defer func() { + if err := zipWriter.Close(); err != nil { + log.Printf("Failed to finish Shardok profile zip: %v", err) + } + }() + + if err := copyShardokProfileArtifact( + ctx, + zipWriter, + "shardok.prof", + shardokpb.CpuProfileArtifactType_CPU_PROFILE_ARTIFACT_TYPE_PROFILE, + ); err != nil { + log.Printf("Failed to stream Shardok CPU profile: %v", err) + return + } + if err := copyShardokProfileArtifact( + ctx, + zipWriter, + "shardok-server", + shardokpb.CpuProfileArtifactType_CPU_PROFILE_ARTIFACT_TYPE_EXECUTABLE, + ); err != nil { + log.Printf("Failed to stream Shardok executable: %v", err) + return + } + + readme, err := zipWriter.Create("README.txt") + if err != nil { + log.Printf("Failed to add Shardok profile README: %v", err) + return + } + if _, err := fmt.Fprintln(readme, "Analyze with: go tool pprof shardok-server shardok.prof"); err != nil { + log.Printf("Failed to write Shardok profile README: %v", err) + } +} diff --git a/src/main/go/net/eagle0/admin_server/shardok_profile_test.go b/src/main/go/net/eagle0/admin_server/shardok_profile_test.go new file mode 100644 index 0000000000..b02da24f06 --- /dev/null +++ b/src/main/go/net/eagle0/admin_server/shardok_profile_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + shardokpb "github.com/nolen777/eagle0/src/main/protobuf/net/eagle0/common/shardok_internal_interface" +) + +func TestShardokProfileStatusResponse(t *testing.T) { + response := profileStatusResponse(&shardokpb.CpuProfileStatusResponse{ + Recording: true, + ProfileAvailable: true, + DurationSeconds: 60, + StartedAtEpochMillis: 1234, + ProfileSizeBytes: 5678, + SampleFrequencyHz: 100, + }) + + if !response.Recording || !response.ProfileAvailable { + t.Fatalf("unexpected status flags: %+v", response) + } + if response.DurationSeconds != 60 || response.SampleFrequencyHz != 100 { + t.Fatalf("unexpected profile settings: %+v", response) + } + if response.StartedAtEpochMs != 1234 || response.ProfileSizeBytes != 5678 { + t.Fatalf("unexpected profile metadata: %+v", response) + } +} + +func TestShardokProfileStatusDisabled(t *testing.T) { + previousClient := shardokProfileClient + shardokProfileClient = nil + defer func() { shardokProfileClient = previousClient }() + + request := httptest.NewRequest(http.MethodGet, "/shardok-profile/status", nil) + recorder := httptest.NewRecorder() + handleShardokProfileStatus(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("unexpected HTTP status: %d", recorder.Code) + } + if recorder.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("unexpected cache policy: %q", recorder.Header().Get("Cache-Control")) + } + + var response shardokProfileStatusResponse + if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil { + t.Fatalf("decode response: %v", err) + } + if response.Status != "disabled" { + t.Fatalf("unexpected response: %+v", response) + } +} + +func TestShardokProfileEndpointsRequireExpectedMethod(t *testing.T) { + tests := []struct { + handler http.HandlerFunc + method string + }{ + {handler: handleShardokProfileStatus, method: http.MethodPost}, + {handler: handleShardokProfileStart, method: http.MethodGet}, + {handler: handleShardokProfileStop, method: http.MethodGet}, + {handler: handleShardokProfileDownload, method: http.MethodPost}, + } + for _, test := range tests { + request := httptest.NewRequest(test.method, "/shardok-profile", nil) + recorder := httptest.NewRecorder() + test.handler(recorder, request) + + if recorder.Code != http.StatusMethodNotAllowed { + t.Fatalf("unexpected HTTP status: %d", recorder.Code) + } + } +}