diff --git a/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextDiskCache.cs b/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextDiskCache.cs index 17ecd1404d..c9d111752c 100644 --- a/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextDiskCache.cs +++ b/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextDiskCache.cs @@ -41,7 +41,7 @@ namespace eagle { } public static void SaveCompletedText(long gameId, string llmId, string text) { - if (String.IsNullOrEmpty(llmId)) { return; } + if (String.IsNullOrEmpty(llmId) || String.IsNullOrEmpty(text)) { return; } lock (Lock) { if (_cacheDirectory == null) { @@ -97,8 +97,8 @@ namespace eagle { if (cache?.entries == null) { return; } foreach (var entry in cache.entries) { - if (!String.IsNullOrEmpty(entry.llmId)) { - gameCache.entries[entry.llmId] = entry.text ?? ""; + if (!String.IsNullOrEmpty(entry.llmId) && !String.IsNullOrEmpty(entry.text)) { + gameCache.entries[entry.llmId] = entry.text; } } } catch (Exception e) { diff --git a/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextProvider.cs b/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextProvider.cs index ac7f8e7f44..357ea750f4 100644 --- a/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextProvider.cs +++ b/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Eagle/ClientTextProvider.cs @@ -63,8 +63,11 @@ namespace eagle { _pendingUpdates.Clear(); foreach (var (llmId, text) in completedTexts) { - if (String.IsNullOrEmpty(llmId)) { continue; } - _streamingTexts[llmId] = new TextEntry(text ?? "", true); + // A completed entry with no content was produced by an invalid empty LLM + // stream. Do not tell the server that this legacy cache entry is complete; + // omitting it lets the repaired server request regenerate and resend it. + if (String.IsNullOrEmpty(llmId) || String.IsNullOrEmpty(text)) { continue; } + _streamingTexts[llmId] = new TextEntry(text, true); _pendingUpdates.Add(llmId); } } diff --git a/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Editor/ClientTextProviderTests.cs b/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Editor/ClientTextProviderTests.cs index 0cd317018b..aaad761670 100644 --- a/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Editor/ClientTextProviderTests.cs +++ b/src/main/csharp/net/eagle0/clients/unity/eagle0/Assets/Editor/ClientTextProviderTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using NUnit.Framework; namespace eagle0.Tests { @@ -23,5 +24,20 @@ namespace eagle0.Tests { Assert.AreEqual("New replacement text", updated); Assert.AreEqual("retry-attempt", provider.GetTextEntry("chronicle").StreamAttemptId); } + + [Test] + public void CachedCompletedTextsIgnoreEntriesWithoutContent() { + var provider = new eagle.ClientTextProvider(); + + provider.ReplaceWithCachedCompletedTexts(new Dictionary { + { "empty-backstory", "" }, + { "complete-backstory", "A completed backstory." } + }); + + Assert.IsFalse(provider.All().ContainsKey("empty-backstory")); + Assert.AreEqual( + "A completed backstory.", + provider.GetTextEntry("complete-backstory").Text); + } } } diff --git a/src/main/scala/net/eagle0/eagle/client_text/ClientTextStore.scala b/src/main/scala/net/eagle0/eagle/client_text/ClientTextStore.scala index 1608b1f5b5..a48259a5f5 100644 --- a/src/main/scala/net/eagle0/eagle/client_text/ClientTextStore.scala +++ b/src/main/scala/net/eagle0/eagle/client_text/ClientTextStore.scala @@ -87,6 +87,12 @@ trait ClientTextStore { def withMovedBackToUnrequested(id: ClientTextId): ClientTextStore + /** Repairs an invalid completed text that contains no content by restoring its original generation request. */ + def withRequeuedEmptyCompletedText( + id: ClientTextId, + llmRequest: GeneratedTextRequestT + ): ClientTextStore + /** * Removes generated text rows created after a rewind target. Anything past the retained action count belongs to * history that no longer exists, and deterministic request IDs may be reused if the game re-advances. diff --git a/src/main/scala/net/eagle0/eagle/client_text/ClientTextStoreImpl.scala b/src/main/scala/net/eagle0/eagle/client_text/ClientTextStoreImpl.scala index fc54e585ac..b2c27aa526 100644 --- a/src/main/scala/net/eagle0/eagle/client_text/ClientTextStoreImpl.scala +++ b/src/main/scala/net/eagle0/eagle/client_text/ClientTextStoreImpl.scala @@ -219,6 +219,26 @@ case class ClientTextStoreImpl( this } + override def withRequeuedEmptyCompletedText( + id: ClientTextId, + llmRequest: GeneratedTextRequestT + ): ClientTextStore = + completeTexts.get(id) match { + case Some(complete) if complete.text.isEmpty => + copy( + completeTexts = completeTexts - id, + unrequestedTexts = unrequestedTexts + (id -> UnrequestedClientText( + id = id, + requestedAfterHistoryCount = complete.requestedAfterHistoryCount, + llmRequest = llmRequest + )), + // Force the legacy complete-text file to be rewritten without the empty entry. + savedCompleteCount = -1, + incompleteTextsAreSaved = false + ) + case _ => this + } + override def withoutTextsAfter(maxHistoryCount: Int): ClientTextStore = { val removedIds = completeTexts.values.filter(_.requestedAfterHistoryCount > maxHistoryCount).map(_.id).toSet ++ diff --git a/src/main/scala/net/eagle0/eagle/client_text/PostgresClientTextStore.scala b/src/main/scala/net/eagle0/eagle/client_text/PostgresClientTextStore.scala index 6ecd4e8431..258f22ad97 100644 --- a/src/main/scala/net/eagle0/eagle/client_text/PostgresClientTextStore.scala +++ b/src/main/scala/net/eagle0/eagle/client_text/PostgresClientTextStore.scala @@ -594,6 +594,26 @@ class PostgresClientTextStore private[client_text] ( } } + override def withRequeuedEmptyCompletedText( + id: ClientTextId, + llmRequest: GeneratedTextRequestT + ): ClientTextStore = withConnection { + val sql = + """UPDATE client_texts + |SET status = 'unrequested', text = '', llm_request = ?, requested_at_millis = NULL, last_update_at_millis = NULL + |WHERE id = ? AND status = 'complete' AND text = ''""".stripMargin + val stmt = connection.prepareStatement(sql) + JfrEvents.postgresQuery(databaseName, "PostgresClientTextStore.withRequeuedEmptyCompletedText", sql) { event => + try { + stmt.setBytes(1, GeneratedTextRequestConverter.toProto(llmRequest).toByteArray) + stmt.setString(2, id) + event.rows = stmt.executeUpdate() + } finally stmt.close() + } + val _ = completeTextCache.remove(id) + this + } + override def withoutTextsAfter(maxHistoryCount: Int): ClientTextStore = withConnection { val selectSql = """SELECT id FROM client_texts diff --git a/src/main/scala/net/eagle0/eagle/client_text/SqliteClientTextStore.scala b/src/main/scala/net/eagle0/eagle/client_text/SqliteClientTextStore.scala index dc04316eb4..6091ba3aa3 100644 --- a/src/main/scala/net/eagle0/eagle/client_text/SqliteClientTextStore.scala +++ b/src/main/scala/net/eagle0/eagle/client_text/SqliteClientTextStore.scala @@ -517,6 +517,24 @@ class SqliteClientTextStore private[client_text] ( } } + override def withRequeuedEmptyCompletedText( + id: ClientTextId, + llmRequest: GeneratedTextRequestT + ): ClientTextStore = { + val stmt = connection.prepareStatement( + """UPDATE texts + |SET status = 'unrequested', text = '', llm_request = ?, requested_at_millis = NULL, last_update_at_millis = NULL + |WHERE id = ? AND status = 'complete' AND text = ''""".stripMargin + ) + try { + stmt.setBytes(1, GeneratedTextRequestConverter.toProto(llmRequest).toByteArray) + stmt.setString(2, id) + stmt.executeUpdate() + } finally stmt.close() + + this + } + override def withoutTextsAfter(maxHistoryCount: Int): ClientTextStore = { val idsStmt = connection.prepareStatement( """SELECT id FROM texts diff --git a/src/main/scala/net/eagle0/eagle/service/GamesManager.scala b/src/main/scala/net/eagle0/eagle/service/GamesManager.scala index db874c00f5..f91cdd476b 100644 --- a/src/main/scala/net/eagle0/eagle/service/GamesManager.scala +++ b/src/main/scala/net/eagle0/eagle/service/GamesManager.scala @@ -40,7 +40,7 @@ import net.eagle0.eagle.library.util.hero_generator.HeroGenerator import net.eagle0.eagle.library.util.view_filters.GameStateViewFilter import net.eagle0.eagle.library.util.EagleRequire import net.eagle0.eagle.library.EagleInternalException -import net.eagle0.eagle.model.action_result.generated_text_request.GeneratedTextRequestT +import net.eagle0.eagle.model.action_result.generated_text_request.{GeneratedTextRequestT, LlmRequestT} import net.eagle0.eagle.model.action_result.ActionResultT import net.eagle0.eagle.model.proto_converters.hero.ProfessionConverter import net.eagle0.eagle.model.proto_converters.shardok_battle.ShardokBattleConverter @@ -297,6 +297,35 @@ object GamesManager { end if } + private[service] def recoverEmptyCompletedLlmTexts( + clientTextStore: ClientTextStore, + history: FullGameHistory + ): ClientTextStore = { + val emptyCompletedIds = clientTextStore.completeTexts.values + .filter(_.text.isEmpty) + .map(_.id) + .toSet + + if emptyCompletedIds.isEmpty then clientTextStore + else { + val requestsById = history.all.flatMap { resultWithState => + resultWithState.actionResult.newGeneratedTextRequests.collect { + case request: LlmRequestT if emptyCompletedIds.contains(request.requestId) => + request.requestId -> request + } + }.toMap + + requestsById.toVector.sortBy(_._1).foldLeft(clientTextStore) { + case (store, (textId, request)) => + SimpleTimedLogger.printLogger.logLine( + s"Recovering completed LLM text with no content: $textId" + ) + store.withRequeuedEmptyCompletedText(textId, request) + } + } + end if + } + private[service] def pregeneratedTextPairs( pregenerated: PregeneratedClientTextStore, heroes: Iterable[HeroT] @@ -497,8 +526,12 @@ object GamesManager { ) else withPregeneratedTexts - val withRecoveredChronicleTexts = recoverMissingChronicleTexts( + val withRecoveredEmptyTexts = recoverEmptyCompletedLlmTexts( clientTextStore = withRequiredTexts, + history = history + ) + val withRecoveredChronicleTexts = recoverMissingChronicleTexts( + clientTextStore = withRecoveredEmptyTexts, history = history, factionIds = factionIds ) diff --git a/src/main/scala/net/eagle0/eagle/service/LlmResolver.scala b/src/main/scala/net/eagle0/eagle/service/LlmResolver.scala index b6e746c04a..0cad505bf9 100644 --- a/src/main/scala/net/eagle0/eagle/service/LlmResolver.scala +++ b/src/main/scala/net/eagle0/eagle/service/LlmResolver.scala @@ -135,6 +135,12 @@ object LlmResolver { } .headOption + private[service] def completedWithoutText( + streamedChars: Int, + partialCompletion: Option[String] + ): Boolean = + streamedChars == 0 && partialCompletion.forall(_.isEmpty) + case class LlmRequestWithGameState( llmRequest: GeneratedTextRequestT, gameState: GameState, @@ -556,11 +562,15 @@ class LlmResolver(updateReceiver: LlmUpdateQueuingProxy, isGameValid: GameId => ) } - val streamingConsumer = createStreamingConsumer(providerName, streamAttemptId).andThen { - streamingResults => - val now = System.currentTimeMillis() + val forwardingConsumer = createStreamingConsumer(providerName, streamAttemptId) + val streamingConsumer = new Consumer[StreamingTextResults] { + override def accept(streamingResults: StreamingTextResults): Unit = { + val now = System.currentTimeMillis() + val totalStreamedChars = streamChars.addAndGet(streamingResults.value.length) + val emptyCompletedStream = streamingResults.completed && + LlmResolver.completedWithoutText(totalStreamedChars, partialCompletion) + streamChunkCount.incrementAndGet() - streamChars.addAndGet(streamingResults.value.length) streamId.set(streamingResults.streamId) streamLastUpdateAtMillis.set(now) @@ -569,6 +579,11 @@ class LlmResolver(updateReceiver: LlmUpdateQueuingProxy, isGameValid: GameId => streamTerminal.set(true) val _ = diagnosticTask.cancel(false) } + + // Do not let an empty terminal callback poison the server/client caches as a completed text. + // The failed future below will use the normal provider failover and retry path instead. + if !emptyCompletedStream then forwardingConsumer.accept(streamingResults) + } } caller @@ -579,7 +594,20 @@ class LlmResolver(updateReceiver: LlmUpdateQueuingProxy, isGameValid: GameId => ) .transform { case Success(result) => - if streamCompleted.get() then { + if streamCompleted.get() && + LlmResolver.completedWithoutText(streamChars.get(), partialCompletion) + then { + logProviderFutureFinished("completed-without-text") + recordProviderFailure(providerName) + LlmUsageTracker.recordFailure(providerName, llmRequest.getClass.getSimpleName) + Failure( + ExternalTextGenerationError.ServerError( + code = 502, + msg = + s"Provider $providerName completed without text for ${LlmResolver.diagnosticSummary(llmRequest)}" + ) + ) + } else if streamCompleted.get() then { logProviderFutureFinished("succeeded") recordProviderSuccess(providerName) Success(result) diff --git a/src/test/scala/net/eagle0/eagle/client_text/SqliteClientTextStoreTest.scala b/src/test/scala/net/eagle0/eagle/client_text/SqliteClientTextStoreTest.scala index 7d193d3a88..ad4c0c1c8a 100644 --- a/src/test/scala/net/eagle0/eagle/client_text/SqliteClientTextStoreTest.scala +++ b/src/test/scala/net/eagle0/eagle/client_text/SqliteClientTextStoreTest.scala @@ -447,6 +447,26 @@ class SqliteClientTextStoreTest extends AnyFlatSpec with Matchers with BeforeAnd unrequested.llmRequest shouldBe genericLlmRequest } + "withRequeuedEmptyCompletedText" should "restore an empty completed text as unrequested" in { + val store = createStore() + .withAddedTextRequest( + id = "id", + accessibleTo = Vector(5), + requestedAfterHistoryCount = requestedAfterHistoryCount, + llmRequest = genericLlmRequest + ) + .withMarkedRequested("id") + .withAppendedText("id", "", complete = true) + .clientTextStore + + val recovered = store.withRequeuedEmptyCompletedText("id", genericLlmRequest) + + recovered.completeTexts shouldBe empty + recovered.incompleteTexts shouldBe empty + recovered.unrequestedTexts("id").llmRequest shouldBe genericLlmRequest + recovered.accessibleTo("id") shouldBe Vector(5) + } + it should "not throw for non-existent id" in { val store = createStore() noException shouldBe thrownBy { diff --git a/src/test/scala/net/eagle0/eagle/service/GamesManagerTest.scala b/src/test/scala/net/eagle0/eagle/service/GamesManagerTest.scala index cd3953b47a..ba986a920f 100644 --- a/src/test/scala/net/eagle0/eagle/service/GamesManagerTest.scala +++ b/src/test/scala/net/eagle0/eagle/service/GamesManagerTest.scala @@ -22,6 +22,7 @@ import net.eagle0.eagle.api.eagle.JoinGameResult.* import net.eagle0.eagle.auth.NoOpUserService import net.eagle0.eagle.client_text.{ ClientTextStore, + CompleteClientText, PregeneratedClientTextStore, SqliteClientTextStore, TextGenerationDependencyUnknown @@ -421,6 +422,37 @@ class GamesManagerTest extends AnyFlatSpec with MockFactory with Matchers with B GamesManager.recoverMissingChronicleTexts(textStore, history, Vector(3)) shouldBe recoveredStore } + "recoverEmptyCompletedLlmTexts" should "requeue a completed request that produced no content" in { + val textId = "hero_183_initial_backstory_58" + val request = LlmRequestT.HeroInitialBackstoryRequest( + requestId = textId, + eagleGameId = gameId, + heroId = 183, + factionId = None, + personalityWords = Vector.empty + ) + val actionResult = ActionResultC( + actionResultType = ActionResultType.HeroAppears, + newGeneratedTextRequests = Vector(request) + ) + val result = ActionResultWithResultingState(actionResult, emptyGameState(Vector.empty)) + val history = mock[FullGameHistory] + val textStore = mock[ClientTextStore] + val recoveredStore = mock[ClientTextStore] + + (() => textStore.completeTexts) + .expects() + .returning(Map(textId -> CompleteClientText(textId, "", 58))) + .once(): Unit + (() => history.all).expects().returning(Vector(result)).once(): Unit + textStore.withRequeuedEmptyCompletedText + .expects(textId, request) + .returning(recoveredStore) + .once(): Unit + + GamesManager.recoverEmptyCompletedLlmTexts(textStore, history) shouldBe recoveredStore + } + private def defaultGamesManager: GamesManager = defaultGamesManager(command => command.run()) diff --git a/src/test/scala/net/eagle0/eagle/service/LlmResolverTest.scala b/src/test/scala/net/eagle0/eagle/service/LlmResolverTest.scala index c60f05b823..574a1356e6 100644 --- a/src/test/scala/net/eagle0/eagle/service/LlmResolverTest.scala +++ b/src/test/scala/net/eagle0/eagle/service/LlmResolverTest.scala @@ -191,6 +191,15 @@ class LlmResolverTest extends AnyFlatSpec with MockFactory { selected.map(_._1).shouldBe(Some("gemini/gemini-2.5-flash-lite")) } + it should "reject a completed stream that produced no text" in + LlmResolver.completedWithoutText(streamedChars = 0, partialCompletion = None).shouldBe(true) + + it should "accept a completed stream that retained a partial completion" in + LlmResolver.completedWithoutText(streamedChars = 0, partialCompletion = Some("partial")).shouldBe(false) + + it should "accept a completed stream that produced text" in + LlmResolver.completedWithoutText(streamedChars = 1, partialCompletion = None).shouldBe(false) + it should "sort success results that are dependencies first" in { val sortedRequests = llmResolver.sortedLlmRequestsWithPrompts(unsortedRequests)