Retry empty generated text responses (#8693)

src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextDiskCache.cs

@@ -41,7 +41,7 @@ namespace eagle {
         }
 
         public static void SaveCompletedText(long gameId, string llmId, string text) {
-            if (String.IsNullOrEmpty(llmId)) { return; }
+            if (String.IsNullOrEmpty(llmId) || String.IsNullOrEmpty(text)) { return; }
 
             lock (Lock) {
                 if (_cacheDirectory == null) {
@@ -97,8 +97,8 @@ namespace eagle {
                 if (cache?.entries == null) { return; }
 
                 foreach (var entry in cache.entries) {
-                    if (!String.IsNullOrEmpty(entry.llmId)) {
-                        gameCache.entries[entry.llmId] = entry.text ?? "";
+                    if (!String.IsNullOrEmpty(entry.llmId) && !String.IsNullOrEmpty(entry.text)) {
+                        gameCache.entries[entry.llmId] = entry.text;
                     }
                 }
             } catch (Exception e) {

src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextProvider.cs

@@ -63,8 +63,11 @@ namespace eagle {
                 _pendingUpdates.Clear();
 
                 foreach (var (llmId, text) in completedTexts) {
-                    if (String.IsNullOrEmpty(llmId)) { continue; }
-                    _streamingTexts[llmId] = new TextEntry(text ?? "", true);
+                    // A completed entry with no content was produced by an invalid empty LLM
+                    // stream. Do not tell the server that this legacy cache entry is complete;
+                    // omitting it lets the repaired server request regenerate and resend it.
+                    if (String.IsNullOrEmpty(llmId) || String.IsNullOrEmpty(text)) { continue; }
+                    _streamingTexts[llmId] = new TextEntry(text, true);
                     _pendingUpdates.Add(llmId);
                 }
             }

src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Editor/ClientTextProviderTests.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using NUnit.Framework;
 
 namespace eagle0.Tests {
@@ -23,5 +24,20 @@ namespace eagle0.Tests {
             Assert.AreEqual("New replacement text", updated);
             Assert.AreEqual("retry-attempt", provider.GetTextEntry("chronicle").StreamAttemptId);
         }
+
+        [Test]
+        public void CachedCompletedTextsIgnoreEntriesWithoutContent() {
+            var provider = new eagle.ClientTextProvider();
+
+            provider.ReplaceWithCachedCompletedTexts(new Dictionary<string, string> {
+                { "empty-backstory", "" },
+                { "complete-backstory", "A completed backstory." }
+            });
+
+            Assert.IsFalse(provider.All().ContainsKey("empty-backstory"));
+            Assert.AreEqual(
+                    "A completed backstory.",
+                    provider.GetTextEntry("complete-backstory").Text);
+        }
     }
 }

src/main/scala/net/eagle0/eagle/client_text/ClientTextStore.scala

@@ -87,6 +87,12 @@ trait ClientTextStore {
 
   def withMovedBackToUnrequested(id: ClientTextId): ClientTextStore
 
+  /** Repairs an invalid completed text that contains no content by restoring its original generation request. */
+  def withRequeuedEmptyCompletedText(
+      id: ClientTextId,
+      llmRequest: GeneratedTextRequestT
+  ): ClientTextStore
+
   /**
    * Removes generated text rows created after a rewind target. Anything past the retained action count belongs to
    * history that no longer exists, and deterministic request IDs may be reused if the game re-advances.

src/main/scala/net/eagle0/eagle/client_text/ClientTextStoreImpl.scala

@@ -219,6 +219,26 @@ case class ClientTextStoreImpl(
         this
       }
 
+  override def withRequeuedEmptyCompletedText(
+      id: ClientTextId,
+      llmRequest: GeneratedTextRequestT
+  ): ClientTextStore =
+    completeTexts.get(id) match {
+      case Some(complete) if complete.text.isEmpty =>
+        copy(
+          completeTexts = completeTexts - id,
+          unrequestedTexts = unrequestedTexts + (id -> UnrequestedClientText(
+            id = id,
+            requestedAfterHistoryCount = complete.requestedAfterHistoryCount,
+            llmRequest = llmRequest
+          )),
+          // Force the legacy complete-text file to be rewritten without the empty entry.
+          savedCompleteCount = -1,
+          incompleteTextsAreSaved = false
+        )
+      case _                                       => this
+    }
+
   override def withoutTextsAfter(maxHistoryCount: Int): ClientTextStore = {
     val removedIds =
       completeTexts.values.filter(_.requestedAfterHistoryCount > maxHistoryCount).map(_.id).toSet ++

src/main/scala/net/eagle0/eagle/client_text/PostgresClientTextStore.scala

@@ -594,6 +594,26 @@ class PostgresClientTextStore private[client_text] (
     }
   }
 
+  override def withRequeuedEmptyCompletedText(
+      id: ClientTextId,
+      llmRequest: GeneratedTextRequestT
+  ): ClientTextStore = withConnection {
+    val sql  =
+      """UPDATE client_texts
+        |SET status = 'unrequested', text = '', llm_request = ?, requested_at_millis = NULL, last_update_at_millis = NULL
+        |WHERE id = ? AND status = 'complete' AND text = ''""".stripMargin
+    val stmt = connection.prepareStatement(sql)
+    JfrEvents.postgresQuery(databaseName, "PostgresClientTextStore.withRequeuedEmptyCompletedText", sql) { event =>
+      try {
+        stmt.setBytes(1, GeneratedTextRequestConverter.toProto(llmRequest).toByteArray)
+        stmt.setString(2, id)
+        event.rows = stmt.executeUpdate()
+      } finally stmt.close()
+    }
+    val _    = completeTextCache.remove(id)
+    this
+  }
+
   override def withoutTextsAfter(maxHistoryCount: Int): ClientTextStore = withConnection {
     val selectSql  =
       """SELECT id FROM client_texts

src/main/scala/net/eagle0/eagle/client_text/SqliteClientTextStore.scala

@@ -517,6 +517,24 @@ class SqliteClientTextStore private[client_text] (
     }
   }
 
+  override def withRequeuedEmptyCompletedText(
+      id: ClientTextId,
+      llmRequest: GeneratedTextRequestT
+  ): ClientTextStore = {
+    val stmt = connection.prepareStatement(
+      """UPDATE texts
+        |SET status = 'unrequested', text = '', llm_request = ?, requested_at_millis = NULL, last_update_at_millis = NULL
+        |WHERE id = ? AND status = 'complete' AND text = ''""".stripMargin
+    )
+    try {
+      stmt.setBytes(1, GeneratedTextRequestConverter.toProto(llmRequest).toByteArray)
+      stmt.setString(2, id)
+      stmt.executeUpdate()
+    } finally stmt.close()
+
+    this
+  }
+
   override def withoutTextsAfter(maxHistoryCount: Int): ClientTextStore = {
     val idsStmt = connection.prepareStatement(
       """SELECT id FROM texts

src/main/scala/net/eagle0/eagle/service/GamesManager.scala

@@ -40,7 +40,7 @@ import net.eagle0.eagle.library.util.hero_generator.HeroGenerator
 import net.eagle0.eagle.library.util.view_filters.GameStateViewFilter
 import net.eagle0.eagle.library.util.EagleRequire
 import net.eagle0.eagle.library.EagleInternalException
-import net.eagle0.eagle.model.action_result.generated_text_request.GeneratedTextRequestT
+import net.eagle0.eagle.model.action_result.generated_text_request.{GeneratedTextRequestT, LlmRequestT}
 import net.eagle0.eagle.model.action_result.ActionResultT
 import net.eagle0.eagle.model.proto_converters.hero.ProfessionConverter
 import net.eagle0.eagle.model.proto_converters.shardok_battle.ShardokBattleConverter
@@ -297,6 +297,35 @@ object GamesManager {
     end if
   }
 
+  private[service] def recoverEmptyCompletedLlmTexts(
+      clientTextStore: ClientTextStore,
+      history: FullGameHistory
+  ): ClientTextStore = {
+    val emptyCompletedIds = clientTextStore.completeTexts.values
+      .filter(_.text.isEmpty)
+      .map(_.id)
+      .toSet
+
+    if emptyCompletedIds.isEmpty then clientTextStore
+    else {
+      val requestsById = history.all.flatMap { resultWithState =>
+        resultWithState.actionResult.newGeneratedTextRequests.collect {
+          case request: LlmRequestT if emptyCompletedIds.contains(request.requestId) =>
+            request.requestId -> request
+        }
+      }.toMap
+
+      requestsById.toVector.sortBy(_._1).foldLeft(clientTextStore) {
+        case (store, (textId, request)) =>
+          SimpleTimedLogger.printLogger.logLine(
+            s"Recovering completed LLM text with no content: $textId"
+          )
+          store.withRequeuedEmptyCompletedText(textId, request)
+      }
+    }
+    end if
+  }
+
   private[service] def pregeneratedTextPairs(
       pregenerated: PregeneratedClientTextStore,
       heroes: Iterable[HeroT]
@@ -497,8 +526,12 @@ object GamesManager {
         )
       else withPregeneratedTexts
 
-    val withRecoveredChronicleTexts = recoverMissingChronicleTexts(
+    val withRecoveredEmptyTexts     = recoverEmptyCompletedLlmTexts(
       clientTextStore = withRequiredTexts,
+      history = history
+    )
+    val withRecoveredChronicleTexts = recoverMissingChronicleTexts(
+      clientTextStore = withRecoveredEmptyTexts,
       history = history,
       factionIds = factionIds
     )

src/main/scala/net/eagle0/eagle/service/LlmResolver.scala

@@ -135,6 +135,12 @@ object LlmResolver {
       }
       .headOption
 
+  private[service] def completedWithoutText(
+      streamedChars: Int,
+      partialCompletion: Option[String]
+  ): Boolean =
+    streamedChars == 0 && partialCompletion.forall(_.isEmpty)
+
   case class LlmRequestWithGameState(
       llmRequest: GeneratedTextRequestT,
       gameState: GameState,
@@ -556,11 +562,15 @@ class LlmResolver(updateReceiver: LlmUpdateQueuingProxy, isGameValid: GameId =>
                         )
                     }
 
-                    val streamingConsumer = createStreamingConsumer(providerName, streamAttemptId).andThen {
-                      streamingResults =>
-                        val now = System.currentTimeMillis()
+                    val forwardingConsumer = createStreamingConsumer(providerName, streamAttemptId)
+                    val streamingConsumer  = new Consumer[StreamingTextResults] {
+                      override def accept(streamingResults: StreamingTextResults): Unit = {
+                        val now                  = System.currentTimeMillis()
+                        val totalStreamedChars   = streamChars.addAndGet(streamingResults.value.length)
+                        val emptyCompletedStream = streamingResults.completed &&
+                          LlmResolver.completedWithoutText(totalStreamedChars, partialCompletion)
+
                         streamChunkCount.incrementAndGet()
-                        streamChars.addAndGet(streamingResults.value.length)
                         streamId.set(streamingResults.streamId)
                         streamLastUpdateAtMillis.set(now)
 
@@ -569,6 +579,11 @@ class LlmResolver(updateReceiver: LlmUpdateQueuingProxy, isGameValid: GameId =>
                           streamTerminal.set(true)
                           val _ = diagnosticTask.cancel(false)
                         }
+
+                        // Do not let an empty terminal callback poison the server/client caches as a completed text.
+                        // The failed future below will use the normal provider failover and retry path instead.
+                        if !emptyCompletedStream then forwardingConsumer.accept(streamingResults)
+                      }
                     }
 
                     caller
@@ -579,7 +594,20 @@ class LlmResolver(updateReceiver: LlmUpdateQueuingProxy, isGameValid: GameId =>
                       )
                       .transform {
                         case Success(result)                                     =>
-                          if streamCompleted.get() then {
+                          if streamCompleted.get() &&
+                            LlmResolver.completedWithoutText(streamChars.get(), partialCompletion)
+                          then {
+                            logProviderFutureFinished("completed-without-text")
+                            recordProviderFailure(providerName)
+                            LlmUsageTracker.recordFailure(providerName, llmRequest.getClass.getSimpleName)
+                            Failure(
+                              ExternalTextGenerationError.ServerError(
+                                code = 502,
+                                msg =
+                                  s"Provider $providerName completed without text for ${LlmResolver.diagnosticSummary(llmRequest)}"
+                              )
+                            )
+                          } else if streamCompleted.get() then {
                             logProviderFutureFinished("succeeded")
                             recordProviderSuccess(providerName)
                             Success(result)

src/test/scala/net/eagle0/eagle/client_text/SqliteClientTextStoreTest.scala

@@ -447,6 +447,26 @@ class SqliteClientTextStoreTest extends AnyFlatSpec with Matchers with BeforeAnd
     unrequested.llmRequest shouldBe genericLlmRequest
   }
 
+  "withRequeuedEmptyCompletedText" should "restore an empty completed text as unrequested" in {
+    val store = createStore()
+      .withAddedTextRequest(
+        id = "id",
+        accessibleTo = Vector(5),
+        requestedAfterHistoryCount = requestedAfterHistoryCount,
+        llmRequest = genericLlmRequest
+      )
+      .withMarkedRequested("id")
+      .withAppendedText("id", "", complete = true)
+      .clientTextStore
+
+    val recovered = store.withRequeuedEmptyCompletedText("id", genericLlmRequest)
+
+    recovered.completeTexts shouldBe empty
+    recovered.incompleteTexts shouldBe empty
+    recovered.unrequestedTexts("id").llmRequest shouldBe genericLlmRequest
+    recovered.accessibleTo("id") shouldBe Vector(5)
+  }
+
   it should "not throw for non-existent id" in {
     val store = createStore()
     noException shouldBe thrownBy {

src/test/scala/net/eagle0/eagle/service/GamesManagerTest.scala

@@ -22,6 +22,7 @@ import net.eagle0.eagle.api.eagle.JoinGameResult.*
 import net.eagle0.eagle.auth.NoOpUserService
 import net.eagle0.eagle.client_text.{
   ClientTextStore,
+  CompleteClientText,
   PregeneratedClientTextStore,
   SqliteClientTextStore,
   TextGenerationDependencyUnknown
@@ -421,6 +422,37 @@ class GamesManagerTest extends AnyFlatSpec with MockFactory with Matchers with B
     GamesManager.recoverMissingChronicleTexts(textStore, history, Vector(3)) shouldBe recoveredStore
   }
 
+  "recoverEmptyCompletedLlmTexts" should "requeue a completed request that produced no content" in {
+    val textId         = "hero_183_initial_backstory_58"
+    val request        = LlmRequestT.HeroInitialBackstoryRequest(
+      requestId = textId,
+      eagleGameId = gameId,
+      heroId = 183,
+      factionId = None,
+      personalityWords = Vector.empty
+    )
+    val actionResult   = ActionResultC(
+      actionResultType = ActionResultType.HeroAppears,
+      newGeneratedTextRequests = Vector(request)
+    )
+    val result         = ActionResultWithResultingState(actionResult, emptyGameState(Vector.empty))
+    val history        = mock[FullGameHistory]
+    val textStore      = mock[ClientTextStore]
+    val recoveredStore = mock[ClientTextStore]
+
+    (() => textStore.completeTexts)
+      .expects()
+      .returning(Map(textId -> CompleteClientText(textId, "", 58)))
+      .once(): Unit
+    (() => history.all).expects().returning(Vector(result)).once(): Unit
+    textStore.withRequeuedEmptyCompletedText
+      .expects(textId, request)
+      .returning(recoveredStore)
+      .once(): Unit
+
+    GamesManager.recoverEmptyCompletedLlmTexts(textStore, history) shouldBe recoveredStore
+  }
+
   private def defaultGamesManager: GamesManager =
     defaultGamesManager(command => command.run())
 

src/test/scala/net/eagle0/eagle/service/LlmResolverTest.scala

@@ -191,6 +191,15 @@ class LlmResolverTest extends AnyFlatSpec with MockFactory {
     selected.map(_._1).shouldBe(Some("gemini/gemini-2.5-flash-lite"))
   }
 
+  it should "reject a completed stream that produced no text" in
+    LlmResolver.completedWithoutText(streamedChars = 0, partialCompletion = None).shouldBe(true)
+
+  it should "accept a completed stream that retained a partial completion" in
+    LlmResolver.completedWithoutText(streamedChars = 0, partialCompletion = Some("partial")).shouldBe(false)
+
+  it should "accept a completed stream that produced text" in
+    LlmResolver.completedWithoutText(streamedChars = 1, partialCompletion = None).shouldBe(false)
+
   it should "sort success results that are dependencies first" in {
     val sortedRequests =
       llmResolver.sortedLlmRequestsWithPrompts(unsortedRequests)