Server verifies sync status in heartbeat and reports mismatches

- handleHeartbeat now checks client's reported counts against server's
- Compares Eagle unfiltered_result_count and Shardok filtered counts
- Returns GameSyncResult/ShardokSyncResult only for mismatched games
- Logs detected mismatches for debugging

Backwards compatible: old client sends HeartbeatRequest without
GameSyncStatuses, server handles empty list (no sync checks).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/PersistentClientConnection.cs

@@ -46,6 +46,8 @@ namespace eagle {
         private readonly Logger _timingsLogger = Logger.GetLogger("ConnectionLogger");
         private volatile bool _isConnecting = false;
         private Timer _idleCheckTimer = null;
+        private Timer _heartbeatTimer = null;
+        private const double HeartbeatIntervalSeconds = 10.0;
 
         // Connection metrics for diagnostics
         private DateTime? _lastConnectAttempt = null;
@@ -363,6 +365,9 @@ namespace eagle {
                     // Start monitoring for idle timeout (relies on HTTP/2 keepalive)
                     StartIdleCheckTimer();
 
+                    // Start sending heartbeats with sync status
+                    StartHeartbeatTimer();
+
                     await TryPendingCommands();
                 } catch (Exception e) {
                     _circuitBreaker.RecordFailure();
@@ -482,6 +487,7 @@ namespace eagle {
                 _lastDisconnect = DateTime.UtcNow;
                 LogConnectionEvent("disconnect_explicit");
                 StopIdleCheckTimer();
+                StopHeartbeatTimer();
                 _streamingCall?.Dispose();
                 _streamingCall = null;
             }
@@ -491,6 +497,7 @@ namespace eagle {
             lock (this) {
                 // Dispose timers first to stop any pending callbacks
                 StopIdleCheckTimer();
+                StopHeartbeatTimer();
 
                 if (_retryTimer != null) {
                     _retryTimer.Enabled = false;
@@ -743,7 +750,7 @@ namespace eagle {
 
                         switch (current.ResponseDetailsCase) {
                             case UpdateStreamResponse.ResponseDetailsOneofCase.HeartbeatResponse:
-                                _remoteEagleClientLogger.LogLine("Got a heartbeat response!");
+                                HandleHeartbeatResponse(current.HeartbeatResponse);
                                 break;
 
                             case UpdateStreamResponse.ResponseDetailsOneofCase.GameUpdate:
@@ -978,5 +985,105 @@ namespace eagle {
                 }
             }
         }
+
+        private void StartHeartbeatTimer() {
+            StopHeartbeatTimer();
+            _heartbeatTimer =
+                    new Timer { AutoReset = true, Interval = HeartbeatIntervalSeconds * 1000 };
+            _heartbeatTimer.Elapsed += (sender, args) => Task.Run(() => SendHeartbeat());
+            _heartbeatTimer.Enabled = true;
+        }
+
+        private void StopHeartbeatTimer() {
+            if (_heartbeatTimer != null) {
+                _heartbeatTimer.Enabled = false;
+                _heartbeatTimer.Dispose();
+                _heartbeatTimer = null;
+            }
+        }
+
+        private async Task SendHeartbeat() {
+            if (_cancellationToken.IsCancellationRequested) { return; }
+            if (_currentState != ConnectionState.Connected) { return; }
+
+            // Build sync status for all subscribed games
+            var heartbeatRequest = new HeartbeatRequest {
+                ClientTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
+            };
+
+            List<IClientConnectionSubscriber> subscribers;
+            lock (this) { subscribers = _subscribers.Values.ToList(); }
+
+            foreach (var subscriber in subscribers) {
+                var gameSyncStatus = new GameSyncStatus {
+                    GameId = subscriber.GameId,
+                    UnfilteredResultCount = subscriber.LastUnfilteredResultCount
+                };
+
+                foreach (var shardokStatus in subscriber.ShardokViewStatuses) {
+                    gameSyncStatus.ShardokSyncStatuses.Add(new ShardokSyncStatus {
+                        ShardokGameId = shardokStatus.shardokGameId,
+                        FilteredResultCount = shardokStatus.filteredResultCount
+                    });
+                }
+
+                heartbeatRequest.GameSyncStatuses.Add(gameSyncStatus);
+            }
+
+            var request = new UpdateStreamRequest { HeartbeatRequest = heartbeatRequest };
+
+            var sent = await SendUpdateStreamRequestAsync(request);
+            if (sent) {
+                _remoteEagleClientLogger.LogLine(
+                        $"[HEARTBEAT] Sent heartbeat with {heartbeatRequest.GameSyncStatuses.Count} games");
+            }
+        }
+
+        private void HandleHeartbeatResponse(HeartbeatResponse response) {
+            _remoteEagleClientLogger.LogLine(
+                    $"[HEARTBEAT] Got response, server_timestamp={response.ServerTimestamp}");
+
+            // Check for sync mismatches reported by server
+            if (response.GameSyncResults.Count > 0) {
+                foreach (var syncResult in response.GameSyncResults) {
+                    if (!syncResult.EagleInSync) {
+                        _remoteEagleClientLogger.LogLine(
+                                $"[SYNC_MISMATCH] Game {syncResult.GameId}: Eagle out of sync, " +
+                                $"server has {syncResult.ServerUnfilteredResultCount} results");
+                        LogConnectionEvent(
+                                "sync_mismatch_eagle",
+                                $"game={syncResult.GameId}, server_count={syncResult.ServerUnfilteredResultCount}");
+                    }
+
+                    foreach (var shardokResult in syncResult.ShardokSyncResults) {
+                        if (!shardokResult.InSync) {
+                            _remoteEagleClientLogger.LogLine(
+                                    $"[SYNC_MISMATCH] Game {syncResult.GameId}, Shardok {shardokResult.ShardokGameId}: " +
+                                    $"out of sync, server has {shardokResult.ServerFilteredResultCount} results");
+                            LogConnectionEvent(
+                                    "sync_mismatch_shardok",
+                                    $"game={syncResult.GameId}, shardok={shardokResult.ShardokGameId}, " +
+                                            $"server_count={shardokResult.ServerFilteredResultCount}");
+                        }
+                    }
+                }
+
+                // Trigger resync by reconnecting - this will re-subscribe with current counts
+                // and the server will send missing updates
+                _remoteEagleClientLogger.LogLine(
+                        "[SYNC_MISMATCH] Detected sync mismatch, triggering reconnect to resync");
+                LogConnectionEvent("sync_mismatch_reconnect", "Triggering reconnect to resync");
+
+                // Schedule reconnect to resync
+                lock (this) {
+                    _streamingCall?.Dispose();
+                    _streamingCall = null;
+                    _threadCancellationTokenSource?.Cancel();
+                }
+                MarkAllShardokGamesForResync();
+                CancelAllPendingSubscriptionAcks();
+                ScheduleReconnect("SyncMismatch");
+            }
+        }
     }
 }

src/main/protobuf/net/eagle0/eagle/api/eagle.proto

@@ -171,10 +171,43 @@ message GameUpdate {
 
 message HeartbeatRequest {
     int64 client_timestamp = 1;
+    // Sync verification: client reports its known action counts per game
+    repeated GameSyncStatus game_sync_statuses = 2;
+}
+
+// Client's known sync state for a single game
+message GameSyncStatus {
+    int64 game_id = 1;
+    // Eagle action count (matches ActionResultResponse.unfiltered_result_count_after)
+    int32 unfiltered_result_count = 2;
+    // Shardok action counts per tactical battle
+    repeated ShardokSyncStatus shardok_sync_statuses = 3;
+}
+
+message ShardokSyncStatus {
+    string shardok_game_id = 1;
+    // Matches ShardokActionResultResponse.filtered_result_count_after
+    int32 filtered_result_count = 2;
 }
 
 message HeartbeatResponse {
     int64 server_timestamp = 1;
+    // Sync verification results - only included if there are mismatches
+    repeated GameSyncResult game_sync_results = 2;
+}
+
+// Server's sync verification result for a single game
+message GameSyncResult {
+    int64 game_id = 1;
+    bool eagle_in_sync = 2;
+    int32 server_unfiltered_result_count = 3;  // Server's count for comparison
+    repeated ShardokSyncResult shardok_sync_results = 4;
+}
+
+message ShardokSyncResult {
+    string shardok_game_id = 1;
+    bool in_sync = 2;
+    int32 server_filtered_result_count = 3;  // Server's count for comparison
 }
 
 message NewGameOptions {

src/main/scala/net/eagle0/eagle/service/EagleServiceImpl.scala

@@ -176,7 +176,7 @@ class EagleServiceImpl(
             ) =>
           streamOneUpdate(streamGameRequest, responseObserver)
         case UpdateStreamRequest.RequestDetails.HeartbeatRequest(value) =>
-          handleHeartbeat(value.clientTimestamp, responseObserver)
+          handleHeartbeat(value, responseObserver)
         case UpdateStreamRequest.RequestDetails.EnterLobbyRequest(_)    =>
           lockAndDoWithUserName { userName =>
             lobbyUsers = lobbyUsers + (userName -> responseObserver)
@@ -355,18 +355,61 @@ class EagleServiceImpl(
   )
 
   private def handleHeartbeat(
-      clientTimestamp: Long,
+      request: HeartbeatRequest,
       responseObserver: SyncResponseObserver
   ): Unit = {
     lockAndDoWithUserName { userName =>
       println(
-        s"got heartbeat from $userName. Client timestamp $clientTimestamp, server timestamp ${System.currentTimeMillis()}"
+        s"got heartbeat from $userName. Client timestamp ${request.clientTimestamp}, server timestamp ${System.currentTimeMillis()}"
       )
+
+      // Verify sync status for each game the client reports
+      val gameSyncResults = request.gameSyncStatuses.flatMap { clientGameStatus =>
+        gamesManager.gameControllerInfos.get(clientGameStatus.gameId).flatMap { controllerInfo =>
+          val controller = controllerInfo.controller
+          controller.userNameToFactionId.get(userName).map { factionId =>
+            val serverUnfilteredCount = controller.engine.history.count
+
+            // Check Shardok sync status
+            val shardokSyncResults = clientGameStatus.shardokSyncStatuses.map { clientShardokStatus =>
+              val serverShardokCount = controller.engine.history.shardokPlayerCount(
+                clientShardokStatus.shardokGameId,
+                factionId
+              )
+              ShardokSyncResult(
+                shardokGameId = clientShardokStatus.shardokGameId,
+                inSync = clientShardokStatus.filteredResultCount == serverShardokCount,
+                serverFilteredResultCount = serverShardokCount
+              )
+            }
+
+            GameSyncResult(
+              gameId = clientGameStatus.gameId,
+              eagleInSync = clientGameStatus.unfilteredResultCount == serverUnfilteredCount,
+              serverUnfilteredResultCount = serverUnfilteredCount,
+              shardokSyncResults = shardokSyncResults
+            )
+          }
+        }
+      }
+
+      // Only include sync results if there are mismatches (to reduce message size)
+      val mismatchedResults = gameSyncResults.filter { result =>
+        !result.eagleInSync || result.shardokSyncResults.exists(!_.inSync)
+      }
+
+      if mismatchedResults.nonEmpty then {
+        println(
+          s"[HEARTBEAT] Detected sync mismatches for $userName: ${mismatchedResults.map(r => s"game ${r.gameId}: eagle=${r.eagleInSync}").mkString(", ")}"
+        )
+      }
+
       val _ = responseObserver.onNext(
         UpdateStreamResponse(
           responseDetails = UpdateStreamResponse.ResponseDetails.HeartbeatResponse(
             HeartbeatResponse(
-              serverTimestamp = System.currentTimeMillis()
+              serverTimestamp = System.currentTimeMillis(),
+              gameSyncResults = mismatchedResults
             )
           )
         )