agents: rolling restart + session auto-resume across Pod restarts

services/agent-gateway/src/main/kotlin/com/jorisjonkers/personalstack/agentgateway/tmux/AgentSessionManager.kt

@@ -67,6 +67,7 @@ class AgentSessionManager(
     fun spawn(
         kind: AgentKind,
         workspacePath: String? = null,
+        resumeCliSessionId: String? = null,
     ): AgentSession {
         val id = UUID.randomUUID().toString().substring(0, 8)
         val tmuxSession = "agent-$id"
@@ -76,7 +77,7 @@ class AgentSessionManager(
         Files.deleteIfExists(logFile)
         Files.createFile(logFile)
 
-        val (command, cliSessionId) = commandAndSessionIdFor(kind)
+        val (command, cliSessionId) = commandAndSessionIdFor(kind, resumeCliSessionId)
         tmux.newSession(tmuxSession, command, cwd)
         tmux.startPipeToFile(tmuxSession, logFile)
 
@@ -138,23 +139,59 @@ class AgentSessionManager(
     }
 
     /**
-     * Build the CLI command and return the native session id alongside
-     * it. For Claude the id is generated here and passed as
-     * `--session-id <uuid>`; the same UUID is stored in the session
-     * and returned to assistant-api for persist+resume. For Codex
-     * no deterministic create-time flag exists; async discovery from
-     * `$CODEX_HOME/sessions` is a follow-up. Shell has no session id.
+     * Build the CLI command and return the native session id alongside it.
+     *
+     * For Claude: when [resumeCliSessionId] is a UUID the existing session is
+     * continued via `--resume <id>` and that id is echoed back. Otherwise a
+     * fresh UUID is generated and passed as `--session-id <uuid>` so the
+     * conversation can be resumed in a future Pod restart.
+     *
+     * For Codex: session files live in `$CODEX_HOME/sessions/` and survive Pod
+     * restarts (codex-credentials PVC). When [resumeCliSessionId] is the
+     * sentinel [CODEX_RESUME_LAST], `codex resume --last` picks up the most
+     * recent session whose CWD matches the workspace — no stored ID required.
+     * When it is a specific UUID, `codex resume <id>` targets that session.
+     *
+     * Shell has no session id.
      */
-    private fun commandAndSessionIdFor(kind: AgentKind): Pair<List<String>, String?> =
+    private fun commandAndSessionIdFor(
+        kind: AgentKind,
+        resumeCliSessionId: String?,
+    ): Pair<List<String>, String?> =
         when (kind) {
             AgentKind.CLAUDE -> {
-                val cliSessionId = UUID.randomUUID().toString()
-                val cmd =
-                    listOf(props.cli.claude) + props.cli.claudeArgs +
-                        listOf("--session-id", cliSessionId)
-                cmd to cliSessionId
+                if (resumeCliSessionId != null) {
+                    val cmd =
+                        listOf(props.cli.claude) + props.cli.claudeArgs +
+                            listOf("--resume", resumeCliSessionId)
+                    cmd to resumeCliSessionId
+                } else {
+                    val cliSessionId = UUID.randomUUID().toString()
+                    val cmd =
+                        listOf(props.cli.claude) + props.cli.claudeArgs +
+                            listOf("--session-id", cliSessionId)
+                    cmd to cliSessionId
+                }
+            }
+            AgentKind.CODEX -> {
+                when (resumeCliSessionId) {
+                    CODEX_RESUME_LAST -> {
+                        val cmd = listOf(props.cli.codex, "resume", "--last") + props.cli.codexArgs
+                        cmd to null
+                    }
+                    null -> (listOf(props.cli.codex) + props.cli.codexArgs) to null
+                    else -> {
+                        // Specific session UUID — direct resume
+                        val cmd = listOf(props.cli.codex, "resume", resumeCliSessionId) + props.cli.codexArgs
+                        cmd to resumeCliSessionId
+                    }
+                }
             }
-            AgentKind.CODEX -> (listOf(props.cli.codex) + props.cli.codexArgs) to null
             AgentKind.SHELL -> listOf("/bin/bash", "-l") to null
         }
+
+    companion object {
+        /** Sentinel value for [resumeCliSessionId] that tells the gateway to run `codex resume --last`. */
+        const val CODEX_RESUME_LAST = "LATEST"
+    }
 }

services/agent-gateway/src/main/kotlin/com/jorisjonkers/personalstack/agentgateway/web/AgentController.kt

@@ -27,7 +27,7 @@ class AgentController(
     fun spawn(
         @RequestBody req: SpawnAgentRequest,
     ): ResponseEntity<AgentResponse> {
-        val session = sessions.spawn(req.kind, req.workspacePath)
+        val session = sessions.spawn(req.kind, req.workspacePath, req.resumeCliSessionId)
         return ResponseEntity.status(HttpStatus.CREATED).body(toResponse(session))
     }
 

services/agent-gateway/src/main/kotlin/com/jorisjonkers/personalstack/agentgateway/web/dto/Dtos.kt

@@ -5,6 +5,7 @@ import com.jorisjonkers.personalstack.agentgateway.tmux.AgentKind
 data class SpawnAgentRequest(
     val kind: AgentKind,
     val workspacePath: String? = null,
+    val resumeCliSessionId: String? = null,
 )
 
 data class SendInputRequest(

services/agent-gateway/src/test/kotlin/com/jorisjonkers/personalstack/agentgateway/tmux/AgentSessionManagerTest.kt

@@ -81,6 +81,64 @@ class AgentSessionManagerTest {
         }
     }
 
+    @Test
+    fun `spawn resumes claude session when resumeCliSessionId is provided`(
+        @TempDir tmp: Path,
+    ) {
+        val mgr = manager(tmp)
+        val previousId = "prev-session-uuid"
+        val s = mgr.spawn(AgentKind.CLAUDE, resumeCliSessionId = previousId)
+        assertThat(s.cliSessionId).isEqualTo(previousId)
+        verify {
+            tmux.newSession(
+                s.tmuxSession,
+                match { cmd ->
+                    cmd.containsAll(listOf("/usr/local/bin/claude", "--dangerously-skip-permissions")) &&
+                        cmd.contains("--resume") &&
+                        cmd[cmd.indexOf("--resume") + 1] == previousId &&
+                        !cmd.contains("--session-id")
+                },
+                "/workspace",
+            )
+        }
+    }
+
+    @Test
+    fun `spawn resumes last codex session when LATEST sentinel is provided`(
+        @TempDir tmp: Path,
+    ) {
+        val mgr = manager(tmp)
+        val s = mgr.spawn(AgentKind.CODEX, resumeCliSessionId = "LATEST")
+        assertThat(s.cliSessionId).isNull()
+        verify {
+            tmux.newSession(
+                s.tmuxSession,
+                match { cmd -> cmd.containsAll(listOf("/usr/local/bin/codex", "resume", "--last")) },
+                "/workspace",
+            )
+        }
+    }
+
+    @Test
+    fun `spawn resumes specific codex session when a UUID is provided`(
+        @TempDir tmp: Path,
+    ) {
+        val mgr = manager(tmp)
+        val sessionId = "prev-codex-uuid"
+        val s = mgr.spawn(AgentKind.CODEX, resumeCliSessionId = sessionId)
+        assertThat(s.cliSessionId).isEqualTo(sessionId)
+        verify {
+            tmux.newSession(
+                s.tmuxSession,
+                match { cmd ->
+                    cmd.containsAll(listOf("/usr/local/bin/codex", "resume", sessionId)) &&
+                        !cmd.contains("--last")
+                },
+                "/workspace",
+            )
+        }
+    }
+
     @Test
     fun `spawn launches codex with approval+sandbox bypass flag`(
         @TempDir tmp: Path,

services/assistant-api/src/main/kotlin/com/jorisjonkers/personalstack/assistant/application/command/StartAgentSessionCommandHandler.kt

@@ -2,8 +2,10 @@ package com.jorisjonkers.personalstack.assistant.application.command
 
 import com.jorisjonkers.personalstack.assistant.application.exception.AgentRunnerUnavailableException
 import com.jorisjonkers.personalstack.assistant.domain.model.Workspace
+import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceAgentKind
 import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceAgentSession
 import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceAgentSessionStatus
+import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceId
 import com.jorisjonkers.personalstack.assistant.domain.port.AgentGatewayClient
 import com.jorisjonkers.personalstack.assistant.domain.port.AgentRunnerOrchestrator
 import com.jorisjonkers.personalstack.assistant.domain.port.WorkspaceAgentSessionRepository
@@ -80,7 +82,8 @@ class StartAgentSessionCommandHandler(
             )
         sessions.save(session)
 
-        val gatewayAgent = spawnAgentWithRetry(healthy, command)
+        val resumeId = previousSessionResumeId(command.workspaceId, command.kind)
+        val gatewayAgent = spawnAgentWithRetry(healthy, command, resumeId)
         sessions.save(session.bindGatewayAgent(gatewayAgent.id, gatewayAgent.cliSessionId))
     }
 
@@ -165,6 +168,37 @@ class StartAgentSessionCommandHandler(
         )
     }
 
+    /**
+     * Return the resume hint to pass to the gateway for the given [workspaceId]
+     * and [kind]:
+     *
+     * - CLAUDE: the most recent non-failed session's `cliSessionId` (a UUID),
+     *   so `--resume <id>` continues the exact conversation.
+     * - CODEX: the sentinel "LATEST" when any prior non-failed session exists,
+     *   causing `codex resume --last` to pick up the most recent CWD-matched
+     *   session from the codex-credentials PVC.
+     * - SHELL: null (no resume concept).
+     */
+    private fun previousSessionResumeId(
+        workspaceId: WorkspaceId,
+        kind: WorkspaceAgentKind,
+    ): String? {
+        val prior =
+            sessions
+                .findAllByWorkspaceId(workspaceId)
+                .filter { it.status != WorkspaceAgentSessionStatus.FAILED && it.kind == kind }
+        return when (kind) {
+            WorkspaceAgentKind.CLAUDE ->
+                prior
+                    .filter { it.cliSessionId != null }
+                    .maxByOrNull { it.createdAt }
+                    ?.cliSessionId
+            WorkspaceAgentKind.CODEX ->
+                if (prior.isNotEmpty()) CODEX_RESUME_LATEST else null
+            WorkspaceAgentKind.SHELL -> null
+        }
+    }
+
     /**
      * Retry the spawn on `ResourceAccessException` (the Spring
      * RestClient wrapping of any transport-level failure: socket
@@ -175,11 +209,12 @@ class StartAgentSessionCommandHandler(
     private fun spawnAgentWithRetry(
         workspace: Workspace,
         command: StartAgentSessionCommand,
+        resumeCliSessionId: String? = null,
     ): AgentGatewayClient.GatewayAgent {
         var lastFailure: ResourceAccessException? = null
         repeat(MAX_SPAWN_ATTEMPTS) { attempt ->
             try {
-                return gateway.spawnAgent(workspace, command.kind)
+                return gateway.spawnAgent(workspace, command.kind, resumeCliSessionId = resumeCliSessionId)
             } catch (ex: ResourceAccessException) {
                 lastFailure = ex
                 val sleepMs = backoffInitialMs * (attempt + 1)
@@ -204,5 +239,10 @@ class StartAgentSessionCommandHandler(
     companion object {
         const val MAX_SPAWN_ATTEMPTS: Int = 3
         const val BACKOFF_INITIAL_MS: Long = 1_000
+
+        // Sentinel forwarded to the gateway when a prior Codex session exists.
+        // The gateway translates this to `codex resume --last`, which picks up
+        // the most recently used session for the workspace CWD.
+        const val CODEX_RESUME_LATEST: String = "LATEST"
     }
 }

services/assistant-api/src/main/kotlin/com/jorisjonkers/personalstack/assistant/application/idle/ConnectedClientTracker.kt

@@ -0,0 +1,34 @@
+package com.jorisjonkers.personalstack.assistant.application.idle
+
+import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceId
+import org.springframework.stereotype.Component
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicInteger
+
+/**
+ * Counts open browser WebSocket connections per workspace. The idle
+ * sweep consults this tracker to guarantee a workspace with a live
+ * client is never scaled to zero while the user is watching — even if
+ * they are not actively typing (no [WorkspaceActivityTracker.touch]
+ * from keystrokes).
+ *
+ * Attach/detach are called by [com.jorisjonkers.personalstack.assistant.infrastructure.ws.SessionAttachHandler]
+ * on WebSocket open and close respectively. Thread-safe via atomic
+ * reference counting; the count never goes below zero.
+ */
+@Component
+class ConnectedClientTracker {
+    private val counts = ConcurrentHashMap<WorkspaceId, AtomicInteger>()
+
+    fun attach(workspaceId: WorkspaceId) {
+        counts.computeIfAbsent(workspaceId) { AtomicInteger(0) }.incrementAndGet()
+    }
+
+    fun detach(workspaceId: WorkspaceId) {
+        counts.computeIfPresent(workspaceId) { _, counter ->
+            if (counter.decrementAndGet() <= 0) null else counter
+        }
+    }
+
+    fun isConnected(workspaceId: WorkspaceId): Boolean = (counts[workspaceId]?.get() ?: 0) > 0
+}

services/assistant-api/src/main/kotlin/com/jorisjonkers/personalstack/assistant/application/idle/IdleScaleDownScheduler.kt

@@ -1,8 +1,10 @@
 package com.jorisjonkers.personalstack.assistant.application.idle
 
 import com.jorisjonkers.personalstack.assistant.domain.model.Workspace
+import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceAgentSessionStatus
 import com.jorisjonkers.personalstack.assistant.domain.model.WorkspaceStatus
 import com.jorisjonkers.personalstack.assistant.domain.port.AgentRunnerOrchestrator
+import com.jorisjonkers.personalstack.assistant.domain.port.WorkspaceAgentSessionRepository
 import com.jorisjonkers.personalstack.assistant.domain.port.WorkspaceRepository
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Value
@@ -13,37 +15,58 @@ import java.time.Duration
 import java.time.Instant
 
 /**
- * Periodic sweep: any READY workspace whose last activity is older
- * than `agent-runtime.idle-after` gets its Pod + Service torn down
- * via [AgentRunnerOrchestrator.scaleDown]. The workspace PVC and
- * per-workspace deploy-key Secret are preserved so a follow-up wake
- * can re-provision the Pod against the same disk without a re-clone.
+ * Periodic sweep: any READY workspace that is both idle long enough
+ * and has no connected browser clients gets its Pod + Service torn
+ * down via [AgentRunnerOrchestrator.scaleDown].
  *
- * Defaults: idle threshold 30 min, sweep every 5 min. Both
- * overridable via env so a noisier or quieter cluster can tune.
+ * Scale-down is suppressed when:
+ *  - A browser WebSocket is open ([ConnectedClientTracker.isConnected]).
+ *  - The workspace has RUNNING agent sessions and their last activity
+ *    is within `agent-runtime.agent-idle-after-seconds` (default 4 h).
+ *    This protects long-running autonomous agent tasks after the user
+ *    disconnects — the idle timer resets while the AI streams output.
+ *    Once the AI goes quiet the longer grace period begins.
+ *
+ * Defaults: 30 min for sessions-idle, 4 h for agent-running, sweep
+ * every 5 min. All overridable via env.
  */
 @Component
 class IdleScaleDownScheduler(
     private val workspaces: WorkspaceRepository,
+    private val agentSessions: WorkspaceAgentSessionRepository,
     private val orchestrator: AgentRunnerOrchestrator,
     private val tracker: WorkspaceActivityTracker,
+    private val connected: ConnectedClientTracker,
     private val clock: Clock = Clock.systemUTC(),
     @param:Value("\${agent-runtime.idle-after-seconds:1800}")
     private val idleAfterSeconds: Long,
+    @param:Value("\${agent-runtime.agent-idle-after-seconds:14400}")
+    private val agentIdleAfterSeconds: Long,
 ) {
     private val log = LoggerFactory.getLogger(IdleScaleDownScheduler::class.java)
 
     @Scheduled(fixedDelayString = "\${agent-runtime.idle-sweep-period-ms:300000}")
     fun sweep() {
-        val threshold = clock.instant().minus(Duration.ofSeconds(idleAfterSeconds))
         val candidates =
             workspaces
                 .findAllByStatusNot(WorkspaceStatus.DESTROYED)
-                .filter { it.status == WorkspaceStatus.READY && !effectiveLastSeen(it).isAfter(threshold) }
+                .filter { it.status == WorkspaceStatus.READY && isEligibleForScaleDown(it) }
         candidates.forEach { scaleDown(it) }
         if (candidates.isNotEmpty()) log.info("idle-sweep scaled down {} workspace(s)", candidates.size)
     }
 
+    private fun isEligibleForScaleDown(workspace: Workspace): Boolean {
+        if (connected.isConnected(workspace.id)) return false
+        val lastSeen = effectiveLastSeen(workspace)
+        val hasRunning =
+            agentSessions
+                .findAllByWorkspaceId(workspace.id)
+                .any { it.status == WorkspaceAgentSessionStatus.RUNNING }
+        val threshold =
+            Duration.ofSeconds(if (hasRunning) agentIdleAfterSeconds else idleAfterSeconds)
+        return !lastSeen.isAfter(clock.instant().minus(threshold))
+    }
+
     private fun effectiveLastSeen(workspace: Workspace): Instant = tracker.lastSeen(workspace.id) ?: workspace.updatedAt
 
     private fun scaleDown(workspace: Workspace) {