Unload idle games to reduce Postgres pressure (#8369)

This commit is contained in:
2026-07-09 09:57:06 -07:00
committed by GitHub
parent 043166c35b
commit 29c1d4a232
6 changed files with 315 additions and 17 deletions
@@ -1,6 +1,7 @@
package net.eagle0.eagle.client_text
import java.sql.{Connection, DriverManager, ResultSet}
import java.util.concurrent.atomic.AtomicInteger
import scala.util.Try
@@ -20,6 +21,7 @@ class PostgresClientTextStore private[client_text] (
private val databaseName = s"postgres-text:${gameId.toHexString}"
private var currentConnection = initialConnection
private var closed = false
private[client_text] def connection: Connection =
this.synchronized {
@@ -27,7 +29,9 @@ class PostgresClientTextStore private[client_text] (
SimpleTimedLogger.printLogger.logLine(
s"PostgresClientTextStore: reopening closed connection for game ${gameId.toHexString}"
)
if !closed then PostgresClientTextStore.recordClosed(s"already closed game ${gameId.toHexString}")
currentConnection = openReplacementConnection()
closed = false
}
currentConnection
}
@@ -198,7 +202,10 @@ class PostgresClientTextStore private[client_text] (
override def close(): Unit =
this.synchronized {
if !currentConnection.isClosed then currentConnection.close()
if !closed then {
closed = true
PostgresClientTextStore.closeOpenedConnection(currentConnection, s"game ${gameId.toHexString}")
}
}
override def saved: ClientTextStore = this
@@ -926,7 +933,7 @@ object PostgresClientTextStore {
connection
} catch {
case t: Throwable =>
connection.close()
closeOpenedConnection(connection, s"failed open for game ${gameId.toHexString}")
throw t
}
}
@@ -938,13 +945,30 @@ object PostgresClientTextStore {
loaded(gameId, pregenerated).get
private def openConnection(): Connection = {
val cfg = config.getOrElse(
val cfg = config.getOrElse(
throw new EagleInternalException(
"PostgresClientTextStore requires EAGLE_POSTGRES_HOST, DATABASE, USER, and PASSWORD"
)
)
Class.forName("org.postgresql.Driver")
DriverManager.getConnection(cfg.jdbcUrl, cfg.user, cfg.password)
val connection = DriverManager.getConnection(cfg.jdbcUrl, cfg.user, cfg.password)
val count = openConnections.incrementAndGet()
SimpleTimedLogger.printLogger.logLine(s"PostgresClientTextStore opened connection; openTextConnections=$count")
connection
}
private val openConnections = new AtomicInteger(0)
private def closeOpenedConnection(connection: Connection, description: String): Unit =
try
if !connection.isClosed then connection.close()
finally recordClosed(description)
private def recordClosed(description: String): Unit = {
val count = openConnections.decrementAndGet()
SimpleTimedLogger.printLogger.logLine(
s"PostgresClientTextStore closed connection for $description; openTextConnections=$count"
)
}
private def config: Option[PostgresConfig] =
@@ -12,7 +12,8 @@ case class ControllerInfo(
isAiGame: Boolean,
maxPlayers: Int,
lastPlayedByUserId: Map[UserId, Long] = Map.empty,
createdTimestampMillis: Long = 0L
createdTimestampMillis: Long = 0L,
lastAccessMillis: Long = System.currentTimeMillis()
)
final case class GameControllerRegistry(
@@ -123,4 +124,11 @@ final case class GameControllerRegistry(
lastPlayedByUserId = controllers(gameId).lastPlayedByUserId.updated(userId, millis)
)
)
def withTouchedLastAccess(gameId: GameId, millis: Long): GameControllerRegistry =
controllers.get(gameId) match {
case None => this
case Some(controllerInfo) =>
put(gameId, controllerInfo.copy(lastAccessMillis = millis))
}
}
@@ -82,6 +82,20 @@ object GamesManager {
private val DroppedGameCleanupIntervalMillis: Long = TimeUnit.HOURS.toMillis(1)
private val InitialStreamingLlmFailureRetryDelayMillis = TimeUnit.SECONDS.toMillis(15)
private val MaxStreamingLlmFailureRetryDelayMillis = TimeUnit.MINUTES.toMillis(5)
private val LoadedGameCleanupIntervalMillis: Long = TimeUnit.MINUTES.toMillis(1)
private val LoadedGameIdleTimeoutMillis: Long = envLong(
name = "EAGLE_LOADED_GAME_IDLE_TIMEOUT_MILLIS",
defaultValue = TimeUnit.MINUTES.toMillis(
envLong(
name = "EAGLE_LOADED_GAME_IDLE_TIMEOUT_MINUTES",
defaultValue = 30L
)
)
)
private val MaxLoadedGames: Int = envInt(
name = "EAGLE_MAX_LOADED_GAMES",
defaultValue = 25
)
// Track when this instance started, used to determine if we should wait for flush marker
// during blue-green deployments. Only new instances (started after deployment began) should wait.
@@ -115,6 +129,12 @@ object GamesManager {
(TutorialGameCreation.tutorialChronicleGeneratedTextId -> TutorialGameCreation.tutorialChronicleText)
)
private def envLong(name: String, defaultValue: Long): Long =
sys.env.get(name).flatMap(_.toLongOption).getOrElse(defaultValue)
private def envInt(name: String, defaultValue: Int): Int =
sys.env.get(name).flatMap(_.toIntOption).getOrElse(defaultValue)
private[service] def rawRewindBattlePersistenceError(
targetActionCount: Int,
targetState: GameState,
@@ -491,6 +511,9 @@ class GamesManager(
private def updateLastPlayed(gameId: GameId, userId: UserId): Unit =
updateState(_.withUpdatedLastPlayed(gameId, userId, System.currentTimeMillis()))
private def touchLoadedGameAccess(gameId: GameId): Unit =
updateState(_.withTouchedControllerAccess(gameId, System.currentTimeMillis()))
private def streamingLlmFailureRetryDelayMillis(failureCount: Int): Long = {
val multiplier = 1L << math.min(failureCount - 1, 8)
math.min(
@@ -527,10 +550,14 @@ class GamesManager(
}
private def controllerInfo(gameId: GameId): Option[ControllerInfo] =
readState(_.controllerInfo(gameId))
transitionState(_.withTouchedControllerAccessIfPresent(gameId, System.currentTimeMillis()))
private def requiredController(gameId: GameId): GameController =
readState(_.controller(gameId))
controllerInfo(gameId)
.getOrElse(
throw new EagleInternalException(s"No loaded controller for game ${gameId.toHexString}")
)
.controller
private def hasControllerInfo(gameId: GameId): Boolean =
readState(_.hasControllerInfo(gameId))
@@ -558,6 +585,47 @@ class GamesManager(
try info.controller.clientTextStore.close()
finally GamesManager.extractFullHistory(info.controller.engine).close()
private def hasLiveClients(info: ControllerInfo): Boolean =
info.controller.humanClients.exists(client => !client.grpcObserver.isCancelled)
private def cleanupInactiveLoadedGames(excludedGameIds: Set[GameId], force: Boolean): Unit = {
val now = System.currentTimeMillis()
val toClose = transitionState { state =>
val (updatedWithCheck, shouldCheck) = state.withLastLoadedGameCleanupCheckIfDue(
nowMillis = now,
intervalMillis = GamesManager.LoadedGameCleanupIntervalMillis
)
if !force && !shouldCheck then updatedWithCheck -> Vector.empty[(GameId, ControllerInfo)]
else {
val (updated, removed) = updatedWithCheck.withoutInactiveControllerInfos(
nowMillis = now,
idleTimeoutMillis = GamesManager.LoadedGameIdleTimeoutMillis,
maxLoadedGames = GamesManager.MaxLoadedGames,
excludedGameIds = excludedGameIds,
hasLiveClients = hasLiveClients
)
updated -> removed.toVector
}
}
if toClose.nonEmpty then {
val removedGameIds = toClose.map(_._1).toSet
this.synchronized {
streamingLlmFailureRetryStates = streamingLlmFailureRetryStates.filterNot {
case ((failedGameId, _), _) => removedGameIds.contains(failedGameId)
}
}
SimpleTimedLogger.printLogger.logLine(
s"Unloading ${toClose.size} inactive loaded game(s); " +
s"loadedGamesAfterUnload=${gameControllerInfos.size}; " +
s"idleTimeoutMillis=${GamesManager.LoadedGameIdleTimeoutMillis}; " +
s"maxLoadedGames=${GamesManager.MaxLoadedGames}; " +
s"gameIds=${toClose.map(_._1.toHexString).mkString(",")}"
)
toClose.foreach { case (_, info) => closeControllerInfo(info) }
}
}
private def unloadGame(gameId: GameId): Option[ControllerInfo] =
val existing = transitionState(_.withoutControllerInfo(gameId))
streamingLlmFailureRetryStates = streamingLlmFailureRetryStates.filterNot {
@@ -852,10 +920,12 @@ class GamesManager(
// Check if flush marker was updated - if so, invalidate any stale cached games
// This handles the case where games were cached during warmup before the active server flushed
invalidateCacheIfFlushMarkerUpdated()
cleanupInactiveLoadedGames(excludedGameIds = Set(gameId), force = false)
// Quick check under main lock. A controller published for load-time LLM processing is not ready
// for client/game operations until the per-game loading lock has finished initialization.
if readState(_.isControllerReady(gameId)) then
touchLoadedGameAccess(gameId)
ensureOutstandingBattlesSubmitted(gameId)
true
else {
@@ -867,6 +937,7 @@ class GamesManager(
lock.synchronized {
// Double-check after acquiring per-game lock — another thread may have loaded it
if readState(_.isControllerReady(gameId)) then {
touchLoadedGameAccess(gameId)
ensureOutstandingBattlesSubmitted(gameId)
true
} else {
@@ -990,6 +1061,7 @@ class GamesManager(
}
ensureOutstandingBattlesSubmitted(afterLoadContinuation.gameController.gameId)
cleanupInactiveLoadedGames(excludedGameIds = Set(gameId), force = true)
SimpleTimedLogger.printLogger.logLine(
s"Successfully loaded game ${gameId.toHexString}"
@@ -100,6 +100,7 @@ case class GamesManagerState(
tutorialBattleConfigs: Map[GameId, TutorialBattleConfig] = Map.empty,
lastFlushMarkerCheck: Long = 0L,
flushMarkerModTime: Long = 0L,
lastLoadedGameCleanupCheckMillis: Long = 0L,
cachedRunningGames: Option[Vector[RunningGame]] = None
) {
def controllerInfosSnapshot: Map[GameId, ControllerInfo] =
@@ -147,6 +148,14 @@ case class GamesManagerState(
if shouldCheckDroppedGameCleanup(nowMillis, intervalMillis) then withLastDroppedGameCleanupCheck(nowMillis) -> true
else this -> false
def withLastLoadedGameCleanupCheckIfDue(
nowMillis: Long,
intervalMillis: Long
): (GamesManagerState, Boolean) =
if nowMillis - lastLoadedGameCleanupCheckMillis >= intervalMillis then
copy(lastLoadedGameCleanupCheckMillis = nowMillis) -> true
else this -> false
def unloadedRunningGames(existingGames: Vector[RunningGame]): Vector[RunningGame] =
existingGames.filterNot(runningGame =>
controllerRegistry.contains(runningGame.gameId) || removedGameIds.contains(runningGame.gameId)
@@ -190,6 +199,55 @@ case class GamesManagerState(
def withUpdatedLastPlayed(gameId: GameId, userId: UserId, millis: Long): GamesManagerState =
withControllerRegistry(controllerRegistry.withUpdatedLastPlayed(gameId, userId, millis))
def withTouchedControllerAccess(gameId: GameId, millis: Long): GamesManagerState =
withControllerRegistry(controllerRegistry.withTouchedLastAccess(gameId, millis))
def withTouchedControllerAccessIfPresent(
gameId: GameId,
millis: Long
): (GamesManagerState, Option[ControllerInfo]) = {
val updated = withTouchedControllerAccess(gameId, millis)
updated -> updated.controllerInfo(gameId)
}
def withoutInactiveControllerInfos(
nowMillis: Long,
idleTimeoutMillis: Long,
maxLoadedGames: Int,
excludedGameIds: Set[GameId],
hasLiveClients: ControllerInfo => Boolean
): (GamesManagerState, Iterable[(GameId, ControllerInfo)]) = {
val idleCandidates =
if idleTimeoutMillis <= 0 then Vector.empty
else
controllerRegistry.snapshot.toVector.filter {
case (gameId, controllerInfo) =>
!excludedGameIds.contains(gameId) &&
!loadingGameIds.contains(gameId) &&
!hasLiveClients(controllerInfo) &&
nowMillis - controllerInfo.lastAccessMillis >= idleTimeoutMillis
}
val overLimitCandidates =
if maxLoadedGames <= 0 || controllerRegistry.size <= maxLoadedGames then Vector.empty
else {
val removable = controllerRegistry.snapshot.toVector.filter {
case (gameId, controllerInfo) =>
!excludedGameIds.contains(gameId) &&
!loadingGameIds.contains(gameId) &&
!hasLiveClients(controllerInfo)
}.sortBy { case (_, controllerInfo) => controllerInfo.lastAccessMillis }
removable.take(controllerRegistry.size - maxLoadedGames)
}
val toRemove = (idleCandidates ++ overLimitCandidates).groupBy(_._1).view.mapValues(_.head._2).toMap
val updatedRegistry = toRemove.keys.foldLeft(controllerRegistry) {
case (registry, gameId) => registry.remove(gameId)._1
}
withControllerRegistry(updatedRegistry) -> toRemove
}
def withController(
controller: GameController,
isAiGame: Boolean,
@@ -1,10 +1,11 @@
package net.eagle0.eagle.service
import java.sql.{Connection, DriverManager, Types}
import java.util.concurrent.atomic.AtomicInteger
import scala.collection.mutable
import net.eagle0.common.JfrEvents
import net.eagle0.common.{JfrEvents, SimpleTimedLogger}
import net.eagle0.eagle.{FactionId, GameId, RoundId, ShardokGameId}
import net.eagle0.eagle.internal.game_state.GameState as GameStateProto
import net.eagle0.eagle.library.actions.applier.ActionResultWithResultingState
@@ -27,6 +28,7 @@ class PostgresHistory private[service] (
private val dbLock = new Object
private val actionCache = mutable.Map.empty[Int, ActionResultT]
private val stateCache = mutable.Map.empty[Int, GameState]
private var closed = false
private def withDbLock[A](f: => A): A = dbLock.synchronized(f)
@@ -40,7 +42,12 @@ class PostgresHistory private[service] (
override def saveNow: PostgresHistory = this
override def close(): Unit = connection.close()
override def close(): Unit = withDbLock {
if !closed then {
closed = true
PostgresHistory.closeOpenedConnection(connection, s"game ${gameId.toHexString}")
}
}
private[service] def createSqliteSaveExport(
dbFile: java.io.File,
@@ -635,6 +642,7 @@ object PostgresHistory {
private val SnapshotInterval = 25
private val SchemaVersion = 1
private val applier = SqliteHistory.applier
private val openConnections = new AtomicInteger(0)
val ActionResultInsertSql: String =
"""INSERT INTO action_results
@@ -662,7 +670,7 @@ object PostgresHistory {
history.withNewResults(startingResults)
} catch {
case t: Throwable =>
connection.close()
closeOpenedConnection(connection, s"failed create for game ${gameId.toHexString}")
throw t
}
}
@@ -676,7 +684,7 @@ object PostgresHistory {
println(
s"PostgresHistory: no schema $schemaName for game ${gameId.toHexString}"
)
connection.close()
closeOpenedConnection(connection, s"missing schema for game ${gameId.toHexString}")
None
} else {
useSchema(connection, schemaName)
@@ -685,13 +693,13 @@ object PostgresHistory {
println(
s"PostgresHistory: schema $schemaName for game ${gameId.toHexString} has no action results"
)
connection.close()
closeOpenedConnection(connection, s"empty schema for game ${gameId.toHexString}")
None
} else Some(new PostgresHistory(gameId, connection))
}
} catch {
case t: Throwable =>
connection.close()
closeOpenedConnection(connection, s"failed load for game ${gameId.toHexString}")
throw t
}
}.getOrElse(None)
@@ -719,13 +727,27 @@ object PostgresHistory {
)
private def openConnection(): Connection = {
val cfg = config.getOrElse(
val cfg = config.getOrElse(
throw new EagleInternalException(
"EAGLE_HISTORY_BACKEND=postgres requires EAGLE_POSTGRES_HOST, DATABASE, USER, and PASSWORD"
)
)
Class.forName("org.postgresql.Driver")
DriverManager.getConnection(cfg.jdbcUrl, cfg.user, cfg.password)
val connection = DriverManager.getConnection(cfg.jdbcUrl, cfg.user, cfg.password)
val count = openConnections.incrementAndGet()
SimpleTimedLogger.printLogger.logLine(s"PostgresHistory opened connection; openHistoryConnections=$count")
connection
}
private def closeOpenedConnection(connection: Connection, description: String): Unit =
try connection.close()
finally recordClosed(description)
private def recordClosed(description: String): Unit = {
val count = openConnections.decrementAndGet()
SimpleTimedLogger.printLogger.logLine(
s"PostgresHistory closed connection for $description; openHistoryConnections=$count"
)
}
private def envNonEmpty(name: String): Option[String] =
@@ -746,14 +768,14 @@ object PostgresHistory {
val stmt = connection.createStatement()
try stmt.execute(s"DROP SCHEMA IF EXISTS $schemaName CASCADE"): Unit
finally stmt.close()
} finally connection.close()
} finally closeOpenedConnection(connection, s"drop schema for game ${gameId.toHexString}")
}
private[service] def exists(gameId: GameId): Boolean =
config.exists { _ =>
val connection = openConnection()
try schemaExists(connection, schemaNameForGame(gameId))
finally connection.close()
finally closeOpenedConnection(connection, s"exists check for game ${gameId.toHexString}")
}
private def createSchema(connection: Connection, schemaName: String): Unit = {
@@ -32,6 +32,14 @@ class GamesManagerStateTest extends AnyFlatSpec with Matchers {
greatPeople = Vector.empty
)
private def controllerInfo(lastAccessMillis: Long): ControllerInfo =
ControllerInfo(
controller = null,
isAiGame = false,
maxPlayers = 2,
lastAccessMillis = lastAccessMillis
)
"withSavedRunningGames" should "refresh the running-games cache and clear removed game markers" in {
val runningGames = Vector(
RunningGame(gameId = 101L, ongoing = true),
@@ -96,6 +104,112 @@ class GamesManagerStateTest extends AnyFlatSpec with Matchers {
updated.lastFlushMarkerCheck.shouldBe(200L)
}
"withLastLoadedGameCleanupCheckIfDue" should "advance the timestamp and request a check when due" in {
val state = GamesManagerState(
controllerRegistry = GameControllerRegistry(Map.empty),
gamesAwaitingPlayers = Vector.empty,
lastLoadedGameCleanupCheckMillis = 100L
)
val (updated, shouldCheck) = state.withLastLoadedGameCleanupCheckIfDue(
nowMillis = 250L,
intervalMillis = 100L
)
shouldCheck.shouldBe(true)
updated.lastLoadedGameCleanupCheckMillis.shouldBe(250L)
}
it should "leave the timestamp unchanged and skip the check when not due" in {
val state = GamesManagerState(
controllerRegistry = GameControllerRegistry(Map.empty),
gamesAwaitingPlayers = Vector.empty,
lastLoadedGameCleanupCheckMillis = 200L
)
val (updated, shouldCheck) = state.withLastLoadedGameCleanupCheckIfDue(
nowMillis = 250L,
intervalMillis = 100L
)
shouldCheck.shouldBe(false)
updated.lastLoadedGameCleanupCheckMillis.shouldBe(200L)
}
"withoutInactiveControllerInfos" should "remove idle controllers without live clients" in {
val state = GamesManagerState(
controllerRegistry = GameControllerRegistry(
Map(
101L -> controllerInfo(lastAccessMillis = 100L),
202L -> controllerInfo(lastAccessMillis = 950L)
)
),
gamesAwaitingPlayers = Vector.empty
)
val (updated, removed) = state.withoutInactiveControllerInfos(
nowMillis = 1000L,
idleTimeoutMillis = 500L,
maxLoadedGames = 0,
excludedGameIds = Set.empty,
hasLiveClients = _ => false
)
removed.map(_._1).toSet.shouldBe(Set(101L))
updated.hasControllerInfo(101L).shouldBe(false)
updated.hasControllerInfo(202L).shouldBe(true)
}
it should "keep excluded and live-client controllers" in {
val state = GamesManagerState(
controllerRegistry = GameControllerRegistry(
Map(
101L -> controllerInfo(lastAccessMillis = 100L),
202L -> controllerInfo(lastAccessMillis = 100L),
303L -> controllerInfo(lastAccessMillis = 100L)
)
),
gamesAwaitingPlayers = Vector.empty
)
val (updated, removed) = state.withoutInactiveControllerInfos(
nowMillis = 1000L,
idleTimeoutMillis = 500L,
maxLoadedGames = 0,
excludedGameIds = Set(202L),
hasLiveClients = info => info eq state.controllerInfo(303L).get
)
removed.map(_._1).toSet.shouldBe(Set(101L))
updated.hasControllerInfo(101L).shouldBe(false)
updated.hasControllerInfo(202L).shouldBe(true)
updated.hasControllerInfo(303L).shouldBe(true)
}
it should "remove oldest idle controllers when loaded game count exceeds the cap" in {
val state = GamesManagerState(
controllerRegistry = GameControllerRegistry(
Map(
101L -> controllerInfo(lastAccessMillis = 100L),
202L -> controllerInfo(lastAccessMillis = 200L),
303L -> controllerInfo(lastAccessMillis = 300L)
)
),
gamesAwaitingPlayers = Vector.empty
)
val (updated, removed) = state.withoutInactiveControllerInfos(
nowMillis = 1000L,
idleTimeoutMillis = 0L,
maxLoadedGames = 2,
excludedGameIds = Set.empty,
hasLiveClients = _ => false
)
removed.map(_._1).toSet.shouldBe(Set(101L))
updated.controllerInfosSnapshot.keySet.shouldBe(Set(202L, 303L))
}
"withFlushMarkerModTimeIfNew" should "advance the mod time and report a new marker" in {
val state = GamesManagerState(
controllerRegistry = GameControllerRegistry(Map.empty),