Add authenticated Shardok CPU profiling RPCs (#8753)

* Add admin-controlled Shardok CPU profiling

* Skip gperftools sampling under ASan

* Limit profiling PR to Shardok service

MODULE.bazel

@@ -70,6 +70,7 @@ scala_deps.scala_proto()
 # Language Support - C++
 #
 
+bazel_dep(name = "gperftools", version = "2.18.1")
 bazel_dep(name = "toolchains_llvm", version = "1.8.0")
 
 llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")

MODULE.bazel.lock

@@ -200,6 +200,8 @@
     "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/MODULE.bazel": "827f54f492a3ce549c940106d73de332c2b30cebd0c20c0bc5d786aba7f116cb",
     "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/source.json": "3664514073a819992320ffbce5825e4238459df344d8b01748af2208f8d2e1eb",
     "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46",
+    "https://bcr.bazel.build/modules/gperftools/2.18.1/MODULE.bazel": "2bfde64989ae531d9fd9da9c2ccfff1212eafcfcb26b0594b59667d0190b6252",
+    "https://bcr.bazel.build/modules/gperftools/2.18.1/source.json": "3425124f51a743ab03b3145f55bff4c0d255b5311b4ca54886db79ea461ebedb",
     "https://bcr.bazel.build/modules/grpc-java/1.62.2/MODULE.bazel": "99b8771e8c7cacb130170fed2a10c9e8fed26334a93e73b42d2953250885a158",
     "https://bcr.bazel.build/modules/grpc-java/1.66.0/MODULE.bazel": "86ff26209fac846adb89db11f3714b3dc0090fb2fb81575673cc74880cda4e7e",
     "https://bcr.bazel.build/modules/grpc-java/1.69.0/MODULE.bazel": "53887af6a00b3b406d70175d3d07e84ea9362016ff55ea90b9185f0227bfaf98",

src/main/cpp/net/eagle0/shardok/server/BUILD.bazel

@@ -1,6 +1,18 @@
 load("@rules_cc//cc:defs.bzl", "cc_library")
 load("//tools:copts.bzl", "COPTS")
 
+cc_library(
+    name = "cpu_profiler",
+    srcs = ["CpuProfiler.cpp"],
+    hdrs = ["CpuProfiler.hpp"],
+    copts = COPTS,
+    visibility = [
+        "//src/main/cpp/net/eagle0/shardok:__subpackages__",
+        "//src/test/cpp/net/eagle0/shardok:__subpackages__",
+    ],
+    deps = ["@gperftools//:cpu_profiler"],
+)
+
 cc_library(
     name = "games_manager",
     srcs = ["ShardokGamesManager.cpp"],
@@ -75,6 +87,7 @@ cc_library(
     copts = COPTS,
     visibility = ["//src/main/cpp/net/eagle0/shardok:__subpackages__"],
     deps = [
+        ":cpu_profiler",
         ":game_over_response_populator",
         ":games_manager",
         ":token_auth",

src/main/cpp/net/eagle0/shardok/server/CpuProfiler.cpp

@@ -0,0 +1,102 @@
+#include "CpuProfiler.hpp"
+
+#include <gperftools/profiler.h>
+
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <system_error>
+#include <thread>
+#include <utility>
+
+namespace shardok {
+namespace {
+
+auto EpochMillisNow() -> int64_t {
+    return std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::system_clock::now().time_since_epoch())
+            .count();
+}
+
+auto FileSize(const std::string& path) -> int64_t {
+    std::error_code error;
+    const auto size = std::filesystem::file_size(path, error);
+    if (error) { return 0; }
+    return static_cast<int64_t>(size);
+}
+
+}  // namespace
+
+CpuProfiler::CpuProfiler(std::string profilePath, std::string executablePath)
+    : state_(std::make_shared<State>()),
+      executablePath_(std::move(executablePath)) {
+    state_->profilePath = std::move(profilePath);
+    setenv("CPUPROFILE_FREQUENCY", "100", 1);
+}
+
+CpuProfiler::~CpuProfiler() { Stop(); }
+
+auto CpuProfiler::Start(const int32_t durationSeconds) -> std::optional<std::string> {
+    auto state = state_;
+    std::lock_guard lock(state->mutex);
+    if (state->recording) { return "A Shardok CPU profile is already recording"; }
+
+    std::error_code removeError;
+    std::filesystem::remove(state->profilePath, removeError);
+    if (removeError) {
+        return "Unable to remove the previous Shardok CPU profile: " + removeError.message();
+    }
+
+    if (ProfilerStart(state->profilePath.c_str()) == 0) {
+        return "gperftools failed to start the Shardok CPU profile";
+    }
+
+    state->recording = true;
+    state->durationSeconds = durationSeconds;
+    state->startedAtEpochMillis = EpochMillisNow();
+    const uint64_t generation = ++state->generation;
+
+    std::thread([state = std::move(state), generation, durationSeconds]() {
+        std::this_thread::sleep_for(std::chrono::seconds(durationSeconds));
+        std::lock_guard autoStopLock(state->mutex);
+        if (state->recording && state->generation == generation) {
+            CpuProfiler::StopLocked(*state);
+        }
+    }).detach();
+
+    return std::nullopt;
+}
+
+void CpuProfiler::StopLocked(State& state) {
+    if (!state.recording) { return; }
+    ProfilerStop();
+    state.recording = false;
+}
+
+void CpuProfiler::Stop() {
+    std::lock_guard lock(state_->mutex);
+    StopLocked(*state_);
+}
+
+void CpuProfiler::Flush() {
+    std::lock_guard lock(state_->mutex);
+    if (state_->recording) { ProfilerFlush(); }
+}
+
+auto CpuProfiler::Snapshot() const -> CpuProfileSnapshot {
+    std::lock_guard lock(state_->mutex);
+    const int64_t profileSize = FileSize(state_->profilePath);
+    return CpuProfileSnapshot{
+            .recording = state_->recording,
+            .profileAvailable = profileSize > 0,
+            .durationSeconds = state_->durationSeconds,
+            .startedAtEpochMillis = state_->startedAtEpochMillis,
+            .profileSizeBytes = profileSize,
+    };
+}
+
+auto CpuProfiler::ProfilePath() const -> const std::string& { return state_->profilePath; }
+
+auto CpuProfiler::ExecutablePath() const -> const std::string& { return executablePath_; }
+
+}  // namespace shardok

src/main/cpp/net/eagle0/shardok/server/CpuProfiler.hpp

@@ -0,0 +1,57 @@
+#ifndef EAGLE0_SHARDOK_SERVER_CPU_PROFILER_HPP
+#define EAGLE0_SHARDOK_SERVER_CPU_PROFILER_HPP
+
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+
+namespace shardok {
+
+struct CpuProfileSnapshot {
+    bool recording;
+    bool profileAvailable;
+    int32_t durationSeconds;
+    int64_t startedAtEpochMillis;
+    int64_t profileSizeBytes;
+};
+
+class CpuProfiler {
+private:
+    struct State {
+        std::mutex mutex;
+        bool recording = false;
+        int32_t durationSeconds = 0;
+        int64_t startedAtEpochMillis = 0;
+        uint64_t generation = 0;
+        std::string profilePath;
+    };
+
+    std::shared_ptr<State> state_;
+    std::string executablePath_;
+
+    static void StopLocked(State& state);
+
+public:
+    static constexpr int32_t kSampleFrequencyHz = 100;
+
+    explicit CpuProfiler(
+            std::string profilePath = "/tmp/shardok-admin.prof",
+            std::string executablePath = "/proc/self/exe");
+    ~CpuProfiler();
+
+    CpuProfiler(const CpuProfiler&) = delete;
+    auto operator=(const CpuProfiler&) -> CpuProfiler& = delete;
+
+    auto Start(int32_t durationSeconds) -> std::optional<std::string>;
+    void Stop();
+    void Flush();
+    [[nodiscard]] auto Snapshot() const -> CpuProfileSnapshot;
+    [[nodiscard]] auto ProfilePath() const -> const std::string&;
+    [[nodiscard]] auto ExecutablePath() const -> const std::string&;
+};
+
+}  // namespace shardok
+
+#endif  // EAGLE0_SHARDOK_SERVER_CPU_PROFILER_HPP

src/main/cpp/net/eagle0/shardok/server/EagleInterfaceGrpcServer.cpp

@@ -9,6 +9,8 @@
 #include "EagleInterfaceGrpcServer.hpp"
 
 #include <algorithm>
+#include <array>
+#include <fstream>
 #include <iostream>
 #include <iterator>
 #include <memory>
@@ -29,6 +31,8 @@
 namespace shardok {
 using grpc::Status;
 using grpc::StatusCode;
+using net::eagle0::common::CPU_PROFILE_ARTIFACT_TYPE_EXECUTABLE;
+using net::eagle0::common::CPU_PROFILE_ARTIFACT_TYPE_PROFILE;
 using net::eagle0::common::GameSetupInfo;
 using net::eagle0::common::ScopedShardokLatencyTrace;
 using net::eagle0::common::ShardokLatencyFields;
@@ -46,6 +50,17 @@ public:
     auto GetStatus() const -> Status { return status; }
 };
 
+void PopulateCpuProfileStatus(
+        const CpuProfileSnapshot &snapshot,
+        CpuProfileStatusResponse *response) {
+    response->set_recording(snapshot.recording);
+    response->set_profile_available(snapshot.profileAvailable);
+    response->set_duration_seconds(snapshot.durationSeconds);
+    response->set_started_at_epoch_millis(snapshot.startedAtEpochMillis);
+    response->set_profile_size_bytes(snapshot.profileSizeBytes);
+    response->set_sample_frequency_hz(CpuProfiler::kSampleFrequencyHz);
+}
+
 auto EagleInterfaceImpl::ControllerForGame(const GameId &gameId, const GameSetupInfo &setupInfo)
         -> std::shared_ptr<ShardokGameController> {
     ScopedShardokLatencyTrace trace(
@@ -96,6 +111,89 @@ EagleInterfaceImpl::EagleInterfaceImpl(
     }
 }
 
+auto EagleInterfaceImpl::GetCpuProfileStatus(
+        ServerContext *context,
+        const CpuProfileStatusRequest * /*request*/,
+        CpuProfileStatusResponse *response) -> Status {
+    if (auto authStatus = tokenValidator_.ValidateOrStatus(context)) { return *authStatus; }
+    PopulateCpuProfileStatus(cpuProfiler_.Snapshot(), response);
+    return Status::OK;
+}
+
+auto EagleInterfaceImpl::StartCpuProfile(
+        ServerContext *context,
+        const StartCpuProfileRequest *request,
+        CpuProfileStatusResponse *response) -> Status {
+    if (auto authStatus = tokenValidator_.ValidateOrStatus(context)) { return *authStatus; }
+
+    const int32_t durationSeconds = request->duration_seconds();
+    if (durationSeconds != 30 && durationSeconds != 60 && durationSeconds != 120) {
+        return Status(
+                StatusCode::INVALID_ARGUMENT,
+                "Profile duration must be 30, 60, or 120 seconds");
+    }
+    if (const auto error = cpuProfiler_.Start(durationSeconds)) {
+        return Status(StatusCode::FAILED_PRECONDITION, *error);
+    }
+
+    PopulateCpuProfileStatus(cpuProfiler_.Snapshot(), response);
+    return Status::OK;
+}
+
+auto EagleInterfaceImpl::StopCpuProfile(
+        ServerContext *context,
+        const StopCpuProfileRequest * /*request*/,
+        CpuProfileStatusResponse *response) -> Status {
+    if (auto authStatus = tokenValidator_.ValidateOrStatus(context)) { return *authStatus; }
+    cpuProfiler_.Stop();
+    PopulateCpuProfileStatus(cpuProfiler_.Snapshot(), response);
+    return Status::OK;
+}
+
+auto EagleInterfaceImpl::DownloadCpuProfile(
+        ServerContext *context,
+        const CpuProfileDownloadRequest *request,
+        grpc::ServerWriter<CpuProfileChunk> *writer) -> Status {
+    if (auto authStatus = tokenValidator_.ValidateOrStatus(context)) { return *authStatus; }
+
+    std::string path;
+    switch (request->artifact_type()) {
+        case CPU_PROFILE_ARTIFACT_TYPE_PROFILE:
+            cpuProfiler_.Flush();
+            if (!cpuProfiler_.Snapshot().profileAvailable) {
+                return Status(
+                        StatusCode::FAILED_PRECONDITION,
+                        "No Shardok CPU profile is available");
+            }
+            path = cpuProfiler_.ProfilePath();
+            break;
+        case CPU_PROFILE_ARTIFACT_TYPE_EXECUTABLE: path = cpuProfiler_.ExecutablePath(); break;
+        default: return Status(StatusCode::INVALID_ARGUMENT, "Unknown CPU profile artifact type");
+    }
+
+    std::ifstream input(path, std::ios::binary);
+    if (!input.is_open()) {
+        return Status(StatusCode::NOT_FOUND, "Unable to open CPU profile artifact");
+    }
+
+    std::array<char, 64 * 1024> buffer{};
+    while (input.good()) {
+        input.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
+        const std::streamsize count = input.gcount();
+        if (count <= 0) { break; }
+        CpuProfileChunk chunk;
+        chunk.set_data(buffer.data(), static_cast<size_t>(count));
+        if (!writer->Write(chunk)) {
+            return Status(StatusCode::CANCELLED, "Profile download cancelled");
+        }
+    }
+    if (input.bad()) {
+        return Status(StatusCode::INTERNAL, "Failed while reading CPU profile artifact");
+    }
+
+    return Status::OK;
+}
+
 auto ConvertPlayerInfo(
         const google::protobuf::RepeatedPtrField<net::eagle0::common::PlayerSetupInfo> &allPis,
         PlayerId shardokPid) -> PlayerInfoWithUnits {

src/main/cpp/net/eagle0/shardok/server/EagleInterfaceGrpcServer.hpp

@@ -14,6 +14,7 @@
 #include <thread>
 
 #include "src/main/cpp/net/eagle0/common/ProtobufWarningSuppression.hpp"
+#include "src/main/cpp/net/eagle0/shardok/server/CpuProfiler.hpp"
 #include "src/main/cpp/net/eagle0/shardok/server/ShardokGamesManager.hpp"
 #include "src/main/cpp/net/eagle0/shardok/server/TokenAuthInterceptor.hpp"
 #pragma GCC diagnostic push
@@ -31,6 +32,10 @@ SUPPRESS_PROTOBUF_WARNINGS
 namespace shardok {
 using grpc::ServerContext;
 using grpc::Status;
+using net::eagle0::common::CpuProfileChunk;
+using net::eagle0::common::CpuProfileDownloadRequest;
+using net::eagle0::common::CpuProfileStatusRequest;
+using net::eagle0::common::CpuProfileStatusResponse;
 using net::eagle0::common::GameSetupInfo;
 using net::eagle0::common::GameStatusResponse;
 using net::eagle0::common::GameSubscriptionRequest;
@@ -42,11 +47,14 @@ using net::eagle0::common::NewGameRequest;
 using net::eagle0::common::PlacementCommandsRequest;
 using net::eagle0::common::PostCommandRequest;
 using net::eagle0::common::ShardokInternalInterface;
+using net::eagle0::common::StartCpuProfileRequest;
+using net::eagle0::common::StopCpuProfileRequest;
 
 class EagleInterfaceImpl final : public ShardokInternalInterface::Service {
 private:
     std::shared_ptr<ShardokGamesManager> gamesManager;
     TokenValidator tokenValidator_;
+    CpuProfiler cpuProfiler_;
 
     auto ControllerForGame(const GameId& gameId, const GameSetupInfo& setupInfo)
             -> std::shared_ptr<ShardokGameController>;
@@ -87,6 +95,23 @@ public:
             ServerContext* context,
             const GameSubscriptionRequest* request,
             grpc::ServerWriter<GameStatusResponse>* writer) -> Status override;
+
+    auto GetCpuProfileStatus(
+            ServerContext* context,
+            const CpuProfileStatusRequest* request,
+            CpuProfileStatusResponse* response) -> Status override;
+    auto StartCpuProfile(
+            ServerContext* context,
+            const StartCpuProfileRequest* request,
+            CpuProfileStatusResponse* response) -> Status override;
+    auto StopCpuProfile(
+            ServerContext* context,
+            const StopCpuProfileRequest* request,
+            CpuProfileStatusResponse* response) -> Status override;
+    auto DownloadCpuProfile(
+            ServerContext* context,
+            const CpuProfileDownloadRequest* request,
+            grpc::ServerWriter<CpuProfileChunk>* writer) -> Status override;
 };
 }  // namespace shardok
 

src/main/protobuf/net/eagle0/common/shardok_internal_interface.proto

@@ -29,6 +29,43 @@ service ShardokInternalInterface {
 
     rpc GetHexMap(HexMapRequest) returns (HexMapResponse) {}
     rpc GetHexMapNames(HexMapNamesRequest) returns (HexMapNamesResponse) {}
+
+    // Authenticated, on-demand CPU profiling for production diagnostics.
+    rpc GetCpuProfileStatus(CpuProfileStatusRequest) returns (CpuProfileStatusResponse) {}
+    rpc StartCpuProfile(StartCpuProfileRequest) returns (CpuProfileStatusResponse) {}
+    rpc StopCpuProfile(StopCpuProfileRequest) returns (CpuProfileStatusResponse) {}
+    rpc DownloadCpuProfile(CpuProfileDownloadRequest) returns (stream CpuProfileChunk) {}
+}
+
+message CpuProfileStatusRequest {}
+
+message StartCpuProfileRequest {
+    int32 duration_seconds = 1;
+}
+
+message StopCpuProfileRequest {}
+
+message CpuProfileStatusResponse {
+    bool recording = 1;
+    bool profile_available = 2;
+    int32 duration_seconds = 3;
+    int64 started_at_epoch_millis = 4;
+    int64 profile_size_bytes = 5;
+    int32 sample_frequency_hz = 6;
+}
+
+enum CpuProfileArtifactType {
+    CPU_PROFILE_ARTIFACT_TYPE_UNSPECIFIED = 0;
+    CPU_PROFILE_ARTIFACT_TYPE_PROFILE = 1;
+    CPU_PROFILE_ARTIFACT_TYPE_EXECUTABLE = 2;
+}
+
+message CpuProfileDownloadRequest {
+    CpuProfileArtifactType artifact_type = 1;
+}
+
+message CpuProfileChunk {
+    bytes data = 1;
 }
 
 message GameSubscriptionRequest {

src/test/cpp/net/eagle0/shardok/server/BUILD.bazel

@@ -1,6 +1,18 @@
 load("@rules_cc//cc:defs.bzl", "cc_test")
 load("//tools:copts.bzl", "TEST_COPTS")
 
+cc_test(
+    name = "cpu_profiler_test",
+    srcs = ["CpuProfilerTest.cpp"],
+    copts = TEST_COPTS,
+    linkstatic = True,
+    deps = [
+        "//src/main/cpp/net/eagle0/shardok/server:cpu_profiler",
+        "@googletest//:gtest",
+        "@googletest//:gtest_main",
+    ],
+)
+
 cc_test(
     name = "game_over_response_populator_test",
     srcs = ["GameOverResponsePopulatorTest.cpp"],

src/test/cpp/net/eagle0/shardok/server/CpuProfilerTest.cpp

@@ -0,0 +1,54 @@
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include <atomic>
+#include <chrono>
+#include <filesystem>
+#include <string>
+
+#include "src/main/cpp/net/eagle0/shardok/server/CpuProfiler.hpp"
+
+namespace shardok {
+namespace {
+
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+#define EAGLE0_ADDRESS_SANITIZER_ENABLED 1
+#endif
+#endif
+#if defined(__SANITIZE_ADDRESS__)
+#define EAGLE0_ADDRESS_SANITIZER_ENABLED 1
+#endif
+
+TEST(CpuProfilerTest, RecordsBoundedProfileAndRejectsConcurrentStart) {
+#if defined(EAGLE0_ADDRESS_SANITIZER_ENABLED)
+    GTEST_SKIP() << "gperftools stack sampling is incompatible with ASan stack redzones";
+#endif
+
+    const std::string profilePath =
+            (std::filesystem::temp_directory_path() /
+             ("shardok-cpu-profiler-test-" + std::to_string(getpid()) + ".prof"))
+                    .string();
+    std::filesystem::remove(profilePath);
+
+    CpuProfiler profiler(profilePath, "/proc/self/exe");
+    ASSERT_FALSE(profiler.Start(30).has_value());
+    EXPECT_TRUE(profiler.Snapshot().recording);
+    EXPECT_TRUE(profiler.Start(30).has_value());
+
+    std::atomic<uint64_t> work = 0;
+    const auto workUntil = std::chrono::steady_clock::now() + std::chrono::milliseconds(250);
+    while (std::chrono::steady_clock::now() < workUntil) { work.fetch_add(1); }
+
+    profiler.Stop();
+    const CpuProfileSnapshot snapshot = profiler.Snapshot();
+    EXPECT_FALSE(snapshot.recording);
+    EXPECT_TRUE(snapshot.profileAvailable);
+    EXPECT_GT(snapshot.profileSizeBytes, 0);
+    EXPECT_GT(work.load(), 0);
+
+    std::filesystem::remove(profilePath);
+}
+
+}  // namespace
+}  // namespace shardok