mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 20:55:44 +00:00
Add Shardok profiling admin backend (#8754)
This commit is contained in:
@@ -234,6 +234,10 @@ services:
|
||||
- "auth:40033"
|
||||
- "--jfr-sidecar-addr"
|
||||
- "${JFR_SIDECAR_ADDR:-jfr-sidecar:8081}" # Can be switched for blue-green
|
||||
- "--shardok-profile-addr"
|
||||
- "${SHARDOK_ADDRESS}"
|
||||
- "--shardok-profile-tls"
|
||||
- "true"
|
||||
- "--http-port"
|
||||
- "8080"
|
||||
- "--maps-dir"
|
||||
@@ -247,6 +251,8 @@ services:
|
||||
DO_SPACES_SECRET_KEY: "${ADMIN_S3_SECRET_KEY:-}"
|
||||
# GitHub PAT for map editor PR creation (scoped to contents:write + pull_requests:write)
|
||||
GITHUB_TOKEN: "${GITHUB_TOKEN:-}"
|
||||
# Reuse Eagle's Shardok endpoint and auth token for admin-only CPU profiling.
|
||||
SHARDOK_AUTH_TOKEN: "${SHARDOK_AUTH_TOKEN:-}"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# No external port - accessed via nginx at admin.eagle0.net
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "admin_server_lib",
|
||||
@@ -6,6 +6,7 @@ go_library(
|
||||
"admin_server.go",
|
||||
"github_pr.go",
|
||||
"map_editor.go",
|
||||
"shardok_profile.go",
|
||||
],
|
||||
embedsrcs = glob([
|
||||
"static/**",
|
||||
@@ -15,6 +16,7 @@ go_library(
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
"//src/main/go/net/eagle0/util/aws",
|
||||
"//src/main/protobuf/net/eagle0/common:shardok_internal_interface_go_grpc",
|
||||
"//src/main/protobuf/net/eagle0/eagle/admin:game_admin_go_proto",
|
||||
"//src/main/protobuf/net/eagle0/eagle/api:api_go_proto", # keep
|
||||
"//src/main/protobuf/net/eagle0/eagle/api/admin:admin_go_proto",
|
||||
@@ -49,3 +51,10 @@ go_binary(
|
||||
"main.gitCommit": "{STABLE_GIT_COMMIT}",
|
||||
},
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "admin_server_test",
|
||||
srcs = ["shardok_profile_test.go"],
|
||||
embed = [":admin_server_lib"],
|
||||
deps = ["//src/main/protobuf/net/eagle0/common:shardok_internal_interface_go_grpc"],
|
||||
)
|
||||
|
||||
@@ -234,6 +234,15 @@ func main() {
|
||||
log.Printf("Connected to Auth server at %s", *authAddr)
|
||||
}
|
||||
|
||||
shardokProfileConn, err := initializeShardokProfileClient()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure Shardok CPU profiling: %v", err)
|
||||
}
|
||||
if shardokProfileConn != nil {
|
||||
defer shardokProfileConn.Close()
|
||||
log.Printf("Shardok CPU profiling configured through %s", *shardokProfileAddr)
|
||||
}
|
||||
|
||||
// Serve static files
|
||||
staticContent, err := fs.Sub(staticFS, "static")
|
||||
if err != nil {
|
||||
@@ -279,6 +288,10 @@ func main() {
|
||||
http.HandleFunc("/jfr/start", requireAuth(handleJfrStart))
|
||||
http.HandleFunc("/jfr/stop", requireAuth(handleJfrStop))
|
||||
http.HandleFunc("/jfr/download", requireAuth(handleJfrDownload))
|
||||
http.HandleFunc("/shardok-profile/status", requireAuth(handleShardokProfileStatus))
|
||||
http.HandleFunc("/shardok-profile/start", requireAuth(handleShardokProfileStart))
|
||||
http.HandleFunc("/shardok-profile/stop", requireAuth(handleShardokProfileStop))
|
||||
http.HandleFunc("/shardok-profile/download", requireAuth(handleShardokProfileDownload))
|
||||
http.HandleFunc("/postgres-benchmark", requireAuth(handlePostgresBenchmark))
|
||||
http.HandleFunc("/postgres-history-smoke", requireAuth(handlePostgresHistorySmoke))
|
||||
http.HandleFunc("/server/restart", requireAuth(handleEagleRestart))
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
shardokpb "github.com/nolen777/eagle0/src/main/protobuf/net/eagle0/common/shardok_internal_interface"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
var (
|
||||
shardokProfileAddr = flag.String("shardok-profile-addr", "", "Shardok gRPC address for CPU profiling")
|
||||
shardokProfileTLS = flag.Bool("shardok-profile-tls", true, "Use TLS for Shardok CPU profiling")
|
||||
shardokProfileClient shardokpb.ShardokInternalInterfaceClient
|
||||
)
|
||||
|
||||
type shardokProfileStatusResponse struct {
|
||||
Recording bool `json:"recording"`
|
||||
ProfileAvailable bool `json:"profile_available"`
|
||||
DurationSeconds int32 `json:"duration_seconds,omitempty"`
|
||||
StartedAtEpochMs int64 `json:"started_at_epoch_millis,omitempty"`
|
||||
ProfileSizeBytes int64 `json:"profile_size_bytes,omitempty"`
|
||||
SampleFrequencyHz int32 `json:"sample_frequency_hz,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func initializeShardokProfileClient() (*grpc.ClientConn, error) {
|
||||
if *shardokProfileAddr == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var transportCredentials credentials.TransportCredentials
|
||||
if *shardokProfileTLS {
|
||||
transportCredentials = credentials.NewClientTLSFromCert(nil, "")
|
||||
} else {
|
||||
transportCredentials = insecure.NewCredentials()
|
||||
}
|
||||
|
||||
conn, err := grpc.NewClient(
|
||||
*shardokProfileAddr,
|
||||
grpc.WithTransportCredentials(transportCredentials),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to Shardok profiler: %w", err)
|
||||
}
|
||||
shardokProfileClient = shardokpb.NewShardokInternalInterfaceClient(conn)
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func shardokProfileDisabled() bool {
|
||||
return shardokProfileClient == nil
|
||||
}
|
||||
|
||||
func shardokProfileContext(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
if token := os.Getenv("SHARDOK_AUTH_TOKEN"); token != "" {
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token)
|
||||
}
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
func writeShardokProfileJSON(w http.ResponseWriter, status int, response shardokProfileStatusResponse) {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
log.Printf("Failed to write Shardok profile response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func profileStatusResponse(status *shardokpb.CpuProfileStatusResponse) shardokProfileStatusResponse {
|
||||
return shardokProfileStatusResponse{
|
||||
Recording: status.GetRecording(),
|
||||
ProfileAvailable: status.GetProfileAvailable(),
|
||||
DurationSeconds: status.GetDurationSeconds(),
|
||||
StartedAtEpochMs: status.GetStartedAtEpochMillis(),
|
||||
ProfileSizeBytes: status.GetProfileSizeBytes(),
|
||||
SampleFrequencyHz: status.GetSampleFrequencyHz(),
|
||||
}
|
||||
}
|
||||
|
||||
func handleShardokProfileStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if shardokProfileDisabled() {
|
||||
writeShardokProfileJSON(w, http.StatusOK, shardokProfileStatusResponse{
|
||||
Status: "disabled",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
status, err := shardokProfileClient.GetCpuProfileStatus(ctx, &shardokpb.CpuProfileStatusRequest{})
|
||||
if err != nil {
|
||||
writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()})
|
||||
return
|
||||
}
|
||||
writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status))
|
||||
}
|
||||
|
||||
func handleShardokProfileStart(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if shardokProfileDisabled() {
|
||||
http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
duration, err := strconv.Atoi(r.URL.Query().Get("duration"))
|
||||
if err != nil || (duration != 30 && duration != 60 && duration != 120) {
|
||||
writeShardokProfileJSON(w, http.StatusBadRequest, shardokProfileStatusResponse{
|
||||
Error: "Profile duration must be 30, 60, or 120 seconds",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
status, err := shardokProfileClient.StartCpuProfile(ctx, &shardokpb.StartCpuProfileRequest{
|
||||
DurationSeconds: int32(duration),
|
||||
})
|
||||
if err != nil {
|
||||
writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()})
|
||||
return
|
||||
}
|
||||
writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status))
|
||||
}
|
||||
|
||||
func handleShardokProfileStop(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if shardokProfileDisabled() {
|
||||
http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := shardokProfileContext(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
status, err := shardokProfileClient.StopCpuProfile(ctx, &shardokpb.StopCpuProfileRequest{})
|
||||
if err != nil {
|
||||
writeShardokProfileJSON(w, http.StatusBadGateway, shardokProfileStatusResponse{Error: err.Error()})
|
||||
return
|
||||
}
|
||||
writeShardokProfileJSON(w, http.StatusOK, profileStatusResponse(status))
|
||||
}
|
||||
|
||||
func copyShardokProfileArtifact(
|
||||
ctx context.Context,
|
||||
zipWriter *zip.Writer,
|
||||
filename string,
|
||||
artifactType shardokpb.CpuProfileArtifactType,
|
||||
) error {
|
||||
header := &zip.FileHeader{Name: filename, Method: zip.Store}
|
||||
header.SetModTime(time.Now())
|
||||
entry, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s zip entry: %w", filename, err)
|
||||
}
|
||||
|
||||
stream, err := shardokProfileClient.DownloadCpuProfile(ctx, &shardokpb.CpuProfileDownloadRequest{
|
||||
ArtifactType: artifactType,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("start %s download: %w", filename, err)
|
||||
}
|
||||
for {
|
||||
chunk, receiveErr := stream.Recv()
|
||||
if receiveErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if receiveErr != nil {
|
||||
return fmt.Errorf("download %s: %w", filename, receiveErr)
|
||||
}
|
||||
if _, err := entry.Write(chunk.GetData()); err != nil {
|
||||
return fmt.Errorf("write %s: %w", filename, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleShardokProfileDownload(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if shardokProfileDisabled() {
|
||||
http.Error(w, "Shardok CPU profiling is not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := shardokProfileContext(r.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
status, err := shardokProfileClient.GetCpuProfileStatus(ctx, &shardokpb.CpuProfileStatusRequest{})
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Failed to get Shardok profile status: %v", err), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if !status.GetProfileAvailable() {
|
||||
http.Error(w, "No Shardok CPU profile is available", http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
timestamp := time.Now().Format("2006-01-02T15-04-05")
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=shardok-profile-%s.zip", timestamp))
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
|
||||
zipWriter := zip.NewWriter(w)
|
||||
defer func() {
|
||||
if err := zipWriter.Close(); err != nil {
|
||||
log.Printf("Failed to finish Shardok profile zip: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := copyShardokProfileArtifact(
|
||||
ctx,
|
||||
zipWriter,
|
||||
"shardok.prof",
|
||||
shardokpb.CpuProfileArtifactType_CPU_PROFILE_ARTIFACT_TYPE_PROFILE,
|
||||
); err != nil {
|
||||
log.Printf("Failed to stream Shardok CPU profile: %v", err)
|
||||
return
|
||||
}
|
||||
if err := copyShardokProfileArtifact(
|
||||
ctx,
|
||||
zipWriter,
|
||||
"shardok-server",
|
||||
shardokpb.CpuProfileArtifactType_CPU_PROFILE_ARTIFACT_TYPE_EXECUTABLE,
|
||||
); err != nil {
|
||||
log.Printf("Failed to stream Shardok executable: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
readme, err := zipWriter.Create("README.txt")
|
||||
if err != nil {
|
||||
log.Printf("Failed to add Shardok profile README: %v", err)
|
||||
return
|
||||
}
|
||||
if _, err := fmt.Fprintln(readme, "Analyze with: go tool pprof shardok-server shardok.prof"); err != nil {
|
||||
log.Printf("Failed to write Shardok profile README: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
shardokpb "github.com/nolen777/eagle0/src/main/protobuf/net/eagle0/common/shardok_internal_interface"
|
||||
)
|
||||
|
||||
func TestShardokProfileStatusResponse(t *testing.T) {
|
||||
response := profileStatusResponse(&shardokpb.CpuProfileStatusResponse{
|
||||
Recording: true,
|
||||
ProfileAvailable: true,
|
||||
DurationSeconds: 60,
|
||||
StartedAtEpochMillis: 1234,
|
||||
ProfileSizeBytes: 5678,
|
||||
SampleFrequencyHz: 100,
|
||||
})
|
||||
|
||||
if !response.Recording || !response.ProfileAvailable {
|
||||
t.Fatalf("unexpected status flags: %+v", response)
|
||||
}
|
||||
if response.DurationSeconds != 60 || response.SampleFrequencyHz != 100 {
|
||||
t.Fatalf("unexpected profile settings: %+v", response)
|
||||
}
|
||||
if response.StartedAtEpochMs != 1234 || response.ProfileSizeBytes != 5678 {
|
||||
t.Fatalf("unexpected profile metadata: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardokProfileStatusDisabled(t *testing.T) {
|
||||
previousClient := shardokProfileClient
|
||||
shardokProfileClient = nil
|
||||
defer func() { shardokProfileClient = previousClient }()
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/shardok-profile/status", nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
handleShardokProfileStatus(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected HTTP status: %d", recorder.Code)
|
||||
}
|
||||
if recorder.Header().Get("Cache-Control") != "no-store" {
|
||||
t.Fatalf("unexpected cache policy: %q", recorder.Header().Get("Cache-Control"))
|
||||
}
|
||||
|
||||
var response shardokProfileStatusResponse
|
||||
if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if response.Status != "disabled" {
|
||||
t.Fatalf("unexpected response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardokProfileEndpointsRequireExpectedMethod(t *testing.T) {
|
||||
tests := []struct {
|
||||
handler http.HandlerFunc
|
||||
method string
|
||||
}{
|
||||
{handler: handleShardokProfileStatus, method: http.MethodPost},
|
||||
{handler: handleShardokProfileStart, method: http.MethodGet},
|
||||
{handler: handleShardokProfileStop, method: http.MethodGet},
|
||||
{handler: handleShardokProfileDownload, method: http.MethodPost},
|
||||
}
|
||||
for _, test := range tests {
|
||||
request := httptest.NewRequest(test.method, "/shardok-profile", nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
test.handler(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("unexpected HTTP status: %d", recorder.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user