From 685a1de5fc8b8b4c8aa21c90b9c07444f0c65c5d Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:29:02 +0800 Subject: [PATCH 1/5] fix(codex-app): make turn ownership and recovery durable --- src/adapters/backend/herdr-backend.ts | 31 +- src/adapters/backend/pty-backend.ts | 6 +- src/adapters/backend/sandbox.ts | 20 +- src/adapters/backend/tmux-backend.ts | 6 +- src/adapters/backend/tmux-pipe-backend.ts | 4 +- src/adapters/backend/zellij-backend.ts | 16 +- .../backend/zellij-observe-backend.ts | 24 +- src/adapters/cli/codex-app.ts | 8 + src/adapters/cli/runner-input.ts | 79 +- src/adapters/cli/types.ts | 28 +- src/bot-registry.ts | 11 + src/cli.ts | 660 +++- src/cli/session-list-liveness.ts | 2 + src/codex-app-runner.ts | 835 ++++- src/core/command-handler.ts | 707 ++-- src/core/dashboard-ipc-server.ts | 354 +- src/core/idle-worker-sweeper.ts | 32 + src/core/inflight-input-tracker.ts | 11 + src/core/pending-repo-journal.ts | 88 + src/core/raw-command-writer.ts | 75 + src/core/reply-target.ts | 40 +- src/core/resource-monitor/runtime.ts | 5 +- src/core/session-manager.ts | 1039 ++++-- src/core/session-mutation-guard.ts | 55 + src/core/trigger-session.ts | 407 ++- src/core/types.ts | 37 + src/core/worker-pool.ts | 1892 +++++++++- src/daemon.ts | 1609 ++++++++- src/dashboard/session-card-model.ts | 14 +- src/dashboard/web/i18n.ts | 8 +- src/dashboard/web/kanban-model.ts | 2 +- src/dashboard/web/sessions-kanban.tsx | 1 + src/dashboard/web/sessions-page.tsx | 1 + src/dashboard/web/sessions.ts | 3 +- src/dashboard/web/style.css | 2 + src/dashboard/web/ui.ts | 1 + src/i18n/en.ts | 4 + src/i18n/zh.ts | 4 + src/im/lark/card-builder.ts | 3 +- src/im/lark/card-handler.ts | 643 ++-- src/im/lark/event-dispatcher.ts | 116 +- src/im/lark/overview-card.ts | 3 +- src/services/session-store.ts | 199 +- src/types.ts | 156 +- src/utils/bot-routing.ts | 4 +- src/utils/child-env.ts | 3 + src/utils/codex-app-control.ts | 3100 +++++++++++++++++ src/utils/codex-app-dispatch-ledger.ts | 207 ++ src/utils/codex-app-turn-dispatch.ts | 196 ++ src/utils/codex-app-turn-liveness.ts | 364 ++ src/utils/pending-input-queue.ts | 10 +- src/worker.ts | 2161 +++++++++++- test/anchor-serializer.test.ts | 36 +- test/bot-registry.test.ts | 6 + test/bot-routing.test.ts | 7 + test/bridge-final-output-retry.test.ts | 249 ++ test/card-builder.test.ts | 13 + test/card-handler-repo-select.test.ts | 361 +- test/card-handler-voice.test.ts | 51 +- test/card-integration.test.ts | 51 +- test/cli-send-hook-context.test.ts | 225 +- test/codex-app-control.test.ts | 1722 +++++++++ test/codex-app-dispatch-ledger.test.ts | 201 ++ test/codex-app-runner.integration.test.ts | 1154 +++++- test/codex-app-turn-dispatch.test.ts | 116 + test/codex-app-turn-liveness.test.ts | 513 +++ test/command-handler.test.ts | 65 +- test/crash-loop-diagnostic.test.ts | 3 +- test/daemon-codex-app-workflow-wiring.test.ts | 12 +- test/daemon-refork-substitute-wiring.test.ts | 6 +- test/daemon-rename-route.test.ts | 1173 ++++++- test/dashboard-attention-signals.test.ts | 9 +- test/dashboard-create-session.test.ts | 758 +++- test/dashboard-ipc.test.ts | 344 +- test/dashboard-sessions-ui.test.ts | 9 + test/event-dispatcher.test.ts | 252 +- test/fixtures/card-action-events.ts | 16 +- test/fixtures/fake-codex-app-server.mjs | 234 +- test/herdr-backend.test.ts | 27 +- test/idle-worker-sweeper.test.ts | 98 +- test/kill-worker-orphaned-backend.test.ts | 11 +- test/overview-card.test.ts | 6 + test/pending-repo-journal.test.ts | 163 + test/raw-input-followup-atomicity.test.ts | 89 +- test/reply-target-fallback.test.ts | 37 +- test/resource-monitor-runtime.test.ts | 1 + test/restore-zombie-close.test.ts | 145 +- test/runner-input.test.ts | 56 +- test/scheduler-silent-execute.test.ts | 7 +- test/session-adopt.test.ts | 38 +- test/session-card-model.test.ts | 8 + test/session-kanban.test.ts | 1 + test/session-lifecycle-start.test.ts | 873 ++++- test/session-list-liveness.test.ts | 7 + test/session-manager-auto-recover.test.ts | 37 +- test/session-mutation-guard.test.ts | 66 + test/session-ready-cli.test.ts | 1 + test/session-reply-thread-anchor.test.ts | 12 + test/session-resume.test.ts | 92 + test/session-store.test.ts | 34 + test/tmux-backend-env.test.ts | 13 +- test/tmux-reattach-backend.test.ts | 10 + test/transfer-session.test.ts | 235 ++ test/trigger-session-root-message.test.ts | 248 +- test/vc-meeting-daemon-session.test.ts | 26 +- ...eeting-receiver-recovery-lifecycle.test.ts | 34 + .../vc-meeting-runtime-lease-recovery.test.ts | 28 +- test/worker-app-runner-control-wiring.test.ts | 224 +- ...codex-app-turn-routing.integration.test.ts | 598 ++++ test/worker-durable-expiry-order.test.ts | 34 +- test/worker-pipe-initial-screen-order.test.ts | 227 ++ test/worker-ready-display-mode.test.ts | 72 + test/worker-suspend.test.ts | 24 + test/zellij-observe-backend.test.ts | 9 + 114 files changed, 24245 insertions(+), 2218 deletions(-) create mode 100644 src/core/pending-repo-journal.ts create mode 100644 src/core/raw-command-writer.ts create mode 100644 src/core/session-mutation-guard.ts create mode 100644 src/utils/codex-app-control.ts create mode 100644 src/utils/codex-app-dispatch-ledger.ts create mode 100644 src/utils/codex-app-turn-dispatch.ts create mode 100644 src/utils/codex-app-turn-liveness.ts create mode 100644 test/codex-app-control.test.ts create mode 100644 test/codex-app-dispatch-ledger.test.ts create mode 100644 test/codex-app-turn-dispatch.test.ts create mode 100644 test/codex-app-turn-liveness.test.ts create mode 100644 test/pending-repo-journal.test.ts create mode 100644 test/session-mutation-guard.test.ts create mode 100644 test/worker-codex-app-turn-routing.integration.test.ts diff --git a/src/adapters/backend/herdr-backend.ts b/src/adapters/backend/herdr-backend.ts index 2c1283b5d..42ebedbc1 100644 --- a/src/adapters/backend/herdr-backend.ts +++ b/src/adapters/backend/herdr-backend.ts @@ -353,6 +353,7 @@ export class HerdrBackend implements SessionBackend { private lastText = ''; private exited = false; private started = false; + private actuallyReattached = false; private cols = 200; private rows = 50; private agentProbeFailures = 0; @@ -467,7 +468,7 @@ export class HerdrBackend implements SessionBackend { } get isReattach(): boolean { - return this.opts.isReattach ?? false; + return this.actuallyReattached; } spawn(bin: string, args: string[], opts: SpawnOpts): void { @@ -497,6 +498,7 @@ export class HerdrBackend implements SessionBackend { const external = this.opts.externalTarget; if (external) { + this.actuallyReattached = false; this.paneId = external.paneId ?? external.target; } else { // Reuse an existing `botmux` agent ONLY when we're genuinely re-attaching @@ -507,8 +509,9 @@ export class HerdrBackend implements SessionBackend { // row from persisted metadata, and reuse would skip `agent start` so the // new command never ran. killSession() now deletes that metadata, but we // also gate reuse on isReattach so a stale row can never be adopted. - const existing = this.isReattach ? this.getAgent() : undefined; + const existing = this.opts.isReattach ? this.getAgent() : undefined; if (existing) { + this.actuallyReattached = true; this.paneId = existing.pane_id; } else if (herdrUsesPaneAgentStart()) { this.paneId = this.startPaneAgent(bin, args, opts); @@ -540,25 +543,31 @@ export class HerdrBackend implements SessionBackend { // - Re-attach / external adopt: snapshot the current screen so we only // stream new deltas. Worker.ts explicitly seeds the initial screen // via captureCurrentScreen() in those paths. - this.lastText = (this.isReattach || this.opts.externalTarget) ? this.readRecentAnsi() : ''; + this.lastText = (this.actuallyReattached || this.opts.externalTarget) ? this.readRecentAnsi() : ''; this.startPolling(); this.startStatusWatcher(); } - write(data: string): void { - if (this.exited) return; + write(data: string): boolean { + if (this.exited) return false; const target = this.paneId ?? this.agentName; - runHerdr(herdrSessionArgs(this.sessionName, ['pane', 'send-text', target, data]), { timeout: 5000 }); + return runHerdr( + herdrSessionArgs(this.sessionName, ['pane', 'send-text', target, data]), + { timeout: 5000 }, + ); } - sendText(text: string): void { - this.write(text); + sendText(text: string): boolean { + return this.write(text); } - sendSpecialKeys(...keys: string[]): void { - if (this.exited) return; + sendSpecialKeys(...keys: string[]): boolean { + if (this.exited) return false; const target = this.paneId ?? this.agentName; - runHerdr(herdrSessionArgs(this.sessionName, ['pane', 'send-keys', target, ...keys]), { timeout: 5000 }); + return runHerdr( + herdrSessionArgs(this.sessionName, ['pane', 'send-keys', target, ...keys]), + { timeout: 5000 }, + ); } pasteText(text: string): void { diff --git a/src/adapters/backend/pty-backend.ts b/src/adapters/backend/pty-backend.ts index be38c7569..a5ee86b35 100644 --- a/src/adapters/backend/pty-backend.ts +++ b/src/adapters/backend/pty-backend.ts @@ -45,8 +45,10 @@ export class PtyBackend implements SessionBackend { logger.debug(`[pty] spawned pid=${this.process.pid}`); } - write(data: string): void { - this.process?.write(data); + write(data: string): boolean { + if (!this.process) return false; + this.process.write(data); + return true; } resize(cols: number, rows: number): void { diff --git a/src/adapters/backend/sandbox.ts b/src/adapters/backend/sandbox.ts index e333145f3..a93155e3c 100644 --- a/src/adapters/backend/sandbox.ts +++ b/src/adapters/backend/sandbox.ts @@ -883,6 +883,7 @@ export function buildRelayHostEnv( const env: NodeJS.ProcessEnv = { ...baseEnv }; delete env.BOTMUX_SEND_RELAY; delete env.BOTMUX_CARD_PREPARED_CONTENT_FILE; + delete env.BOTMUX_HOST_RELAY_REQUIRES_CODEX_APP_LEDGER; if (preparedContentFile) { env.BOTMUX_CARD_LOCAL_LINK_MODE = 'disabled'; env.BOTMUX_CARD_PREPARED_CONTENT_FILE = preparedContentFile; @@ -904,7 +905,19 @@ export function startOutboxWatcher( * absent the relay still runs, but carries NO durable origin — a missing * hook must never let the sandbox promote its own origin fields. */ authorize?: (claim: { capability?: string }) => - | { ok: true; origin: { turnId?: string; dispatchAttempt?: number } } + | { + ok: true; + origin: { + turnId?: string; + dispatchAttempt?: number; + /** The worker matched an unsettled Codex App ledger entry. The + * host child must still find that exact entry before any provider + * side effect; terminal settlement/revocation between authorize + * and re-exec therefore fails closed instead of degrading to an + * ordinary mutable-session send. */ + requiresCodexAppLedger?: boolean; + }; + } | { ok: false; error: string }; cliPath?: string; } = {}, @@ -1028,6 +1041,11 @@ export function startOutboxWatcher( if (trustedOrigin?.dispatchAttempt !== undefined) { requestEnv.BOTMUX_DISPATCH_ATTEMPT = String(trustedOrigin.dispatchAttempt); } + if (trustedOrigin?.requiresCodexAppLedger) { + requestEnv.BOTMUX_HOST_RELAY_REQUIRES_CODEX_APP_LEDGER = '1'; + } else { + delete requestEnv.BOTMUX_HOST_RELAY_REQUIRES_CODEX_APP_LEDGER; + } const child = spawn(process.execPath, [cli, 'send', ...hostArgs], { env: requestEnv }); let out = '', err = ''; child.stdout.on('data', d => { out += d; }); diff --git a/src/adapters/backend/tmux-backend.ts b/src/adapters/backend/tmux-backend.ts index 1580e23d6..0b158615d 100644 --- a/src/adapters/backend/tmux-backend.ts +++ b/src/adapters/backend/tmux-backend.ts @@ -370,8 +370,10 @@ export class TmuxBackend implements SessionBackend { * file's cwd field so a recycled PID can't mislead the resolver. */ cliCwd?: string; - write(data: string): void { - this.process?.write(data); + write(data: string): boolean { + if (!this.process) return false; + this.process.write(data); + return true; } /** diff --git a/src/adapters/backend/tmux-pipe-backend.ts b/src/adapters/backend/tmux-pipe-backend.ts index d4e697062..30f35d310 100644 --- a/src/adapters/backend/tmux-pipe-backend.ts +++ b/src/adapters/backend/tmux-pipe-backend.ts @@ -247,9 +247,9 @@ export class TmuxPipeBackend implements SessionBackend { } } - write(data: string): void { + write(data: string): boolean { // No PTY to write to — interpret as a literal send-keys. - this.sendText(data); + return this.sendText(data); } sendText(text: string): boolean { diff --git a/src/adapters/backend/zellij-backend.ts b/src/adapters/backend/zellij-backend.ts index b8ca9f272..899781dd2 100644 --- a/src/adapters/backend/zellij-backend.ts +++ b/src/adapters/backend/zellij-backend.ts @@ -201,20 +201,24 @@ export class ZellijBackend implements SessionBackend { // are forwarded verbatim to the focused pane — so every input path collapses // to pty.write(), exactly like TmuxBackend.write(). - write(data: string): void { - this.process?.write(data); + write(data: string): boolean { + if (!this.process) return false; + this.process.write(data); + return true; } /** Literal text, no Enter. */ - sendText(text: string): void { - this.process?.write(text); + sendText(text: string): boolean { + return this.write(text); } /** Special keys by tmux-style name (Enter, Escape, C-c, M-Enter, …). */ - sendSpecialKeys(...keys: string[]): void { + sendSpecialKeys(...keys: string[]): boolean { + if (!this.process) return false; for (const key of keys) { - this.process?.write(tmuxKeyToBytes(key)); + this.process.write(tmuxKeyToBytes(key)); } + return true; } /** Bracketed paste: wrap with \e[200~ … \e[201~ so TUIs (CoCo/Ink/Codex) diff --git a/src/adapters/backend/zellij-observe-backend.ts b/src/adapters/backend/zellij-observe-backend.ts index f4eb8e304..2eba19cc5 100644 --- a/src/adapters/backend/zellij-observe-backend.ts +++ b/src/adapters/backend/zellij-observe-backend.ts @@ -182,19 +182,22 @@ export class ZellijObserveBackend implements ObserveBackend { // ── Input: targeted `action` calls (focus-neutral, non-invasive) ── - write(data: string): void { - this.writeBytes(data); + write(data: string): boolean { + return this.writeBytes(data); } /** Literal text via write-chars (preserves UTF-8). */ - sendText(text: string): void { - if (!text) return; - this.action(['write-chars', '--pane-id', this.paneId, '--', text]); + sendText(text: string): boolean { + if (!text) return true; + return this.action(['write-chars', '--pane-id', this.paneId, '--', text]) !== null; } /** Special keys by tmux-style name → raw bytes → `action write`. */ - sendSpecialKeys(...keys: string[]): void { - for (const key of keys) this.writeBytes(tmuxKeyToBytes(key)); + sendSpecialKeys(...keys: string[]): boolean { + for (const key of keys) { + if (!this.writeBytes(tmuxKeyToBytes(key))) return false; + } + return true; } /** Bracketed paste — wrap so TUIs detect the boundary (mirrors paste-buffer -p). */ @@ -207,14 +210,15 @@ export class ZellijObserveBackend implements ObserveBackend { /** Write arbitrary bytes via `action write …` (handles control/escape). * Chunked so a large web-terminal paste can't blow the argv limit; zellij * serialises the writes in arrival order. */ - private writeBytes(data: string): void { - if (!data) return; + private writeBytes(data: string): boolean { + if (!data) return true; const buf = Buffer.from(data, 'utf-8'); const CHUNK = 512; for (let i = 0; i < buf.length; i += CHUNK) { const bytes = Array.from(buf.subarray(i, i + CHUNK), b => String(b)); - this.action(['write', '--pane-id', this.paneId, ...bytes]); + if (this.action(['write', '--pane-id', this.paneId, ...bytes]) === null) return false; } + return true; } /** Resize is a NO-OP in observe mode — the pane size is the user's, and diff --git a/src/adapters/cli/codex-app.ts b/src/adapters/cli/codex-app.ts index dead8db3c..b198e173d 100644 --- a/src/adapters/cli/codex-app.ts +++ b/src/adapters/cli/codex-app.ts @@ -6,6 +6,14 @@ import type { CliAdapter, PtyHandle } from './types.js'; import { writeRunnerInput } from './runner-input.js'; function runnerPath(): string { + // Source-level worker integration tests execute through tsx and need the + // matching source runner rather than a possibly absent/stale ignored dist + // tree. Keep the override strictly test-scoped so production launch + // resolution remains canonical and cannot be redirected by ambient env. + const testOverride = process.env.NODE_ENV === 'test' + ? process.env.BOTMUX_TEST_CODEX_APP_RUNNER_PATH + : undefined; + if (testOverride) return resolve(testOverride); const here = dirname(fileURLToPath(import.meta.url)); const compiledSibling = resolve(here, '..', '..', 'codex-app-runner.js'); if (existsSync(compiledSibling)) return compiledSibling; diff --git a/src/adapters/cli/runner-input.ts b/src/adapters/cli/runner-input.ts index 37b03520e..1b06185bf 100644 --- a/src/adapters/cli/runner-input.ts +++ b/src/adapters/cli/runner-input.ts @@ -1,4 +1,4 @@ -import type { PtyHandle } from './types.js'; +import type { PtyHandle, RunnerSubmissionDisposition } from './types.js'; import type { CodexAppTurnInput } from '../../types.js'; import { delay } from '../../utils/timing.js'; @@ -17,9 +17,9 @@ import { delay } from '../../utils/timing.js'; * webhook whose full MR JSON is embedded — ~16-21KB after base64) that single * injection overruns the pane pty's input buffer (N_TTY's ~4KB read buffer): * tmux's write blocks until the reader drains, which takes longer than - * execFileSync's 5s timeout, so the send-keys is killed and the keystroke is - * silently dropped — yet the old writeInput still reported `submitted: true`, - * wedging the session "busy" forever. (Compare claude-code, which throttles + * execFileSync's 5s timeout, so Botmux can no longer prove whether the + * keystroke landed — yet the old writeInput still reported `submitted: true`, + * potentially wedging the session "busy" forever. (Compare claude-code, which throttles * its send-keys for exactly this reason; codex-app/mira were the only naive * single-shot writers.) * @@ -61,11 +61,10 @@ export function chunkAscii(line: string, maxBytes: number): string[] { /** * Write one control line to a runner adapter's stdin, chunked + throttled. * - * Returns `{ submitted: false }` when any chunk (or the final Enter) fails to - * write — the tmux backend's send methods return `false` on a dropped-but-pane- - * alive keystroke, so a genuine drop is now surfaced to the worker (which raises - * a submit-failure notice + recheck so the user can retry) instead of being - * swallowed as a false success. + * Returns `{ submitted: false }` when any chunk (or the final Enter) cannot be + * confirmed. A tmux timeout is ambiguous: bytes may still have reached the + * pane. `submissionDisposition` therefore tells the worker whether the new + * frame was provably untouched or the runner generation must be fenced. * * Buffer-hygiene contract (the runner only clears its stdin buffer on a newline, * see handleInput in codex-app-runner.ts / mira-runner.ts — a half-written @@ -74,9 +73,10 @@ export function chunkAscii(line: string, maxBytes: number): string[] { * - Pre-flush: emit one Enter before writing, terminating any partial line a * prior failed write may have left behind (runner discards the fragment as * bad input; an empty buffer just ignores the blank line). - * - On a dropped chunk: emit a flush Enter so the partial we just wrote can't - * merge with the next message, then report non-submission (submit-failure). - * - Submit Enter is retried — a single dropped Enter would otherwise leave a + * - On an unconfirmed chunk: attempt a flush Enter so a partial frame is less + * likely to merge with the next message, then report an ambiguous dirty + * generation; the flush cannot make delivery proof retroactive. + * - Submit Enter is retried — a single unconfirmed Enter could otherwise leave a * COMPLETE but unsubmitted line in the buffer. */ export async function writeRunnerInput( @@ -84,18 +84,24 @@ export async function writeRunnerInput( markerPrefix: string, content: string, codexAppInput?: CodexAppTurnInput, -): Promise<{ submitted: boolean }> { +): Promise<{ submitted: boolean; submissionDisposition: RunnerSubmissionDisposition }> { const line = `${markerPrefix}${encodeRunnerInput(content, codexAppInput)}`; // Non-tmux fallback (raw PTY): a single write is fine — there's no send-keys // process to time out, and the PTY write isn't bounded the same way. if (!pty.sendText || !pty.sendSpecialKeys) { try { - pty.write(line + '\r'); + if (pty.write(line + '\r') === false) { + // SessionBackend.write(false) is a rejection, but its contract does not + // prove whether a lower layer accepted a prefix before reporting it. + return { submitted: false, submissionDisposition: 'dirty_unknown' }; + } } catch { - return { submitted: false }; + // A throwing PTY write does not prove whether the kernel accepted a + // prefix (or the complete line) before surfacing the error. + return { submitted: false, submissionDisposition: 'dirty_unknown' }; } - return { submitted: true }; + return { submitted: true, submissionDisposition: 'submitted' }; } const sendText = pty.sendText.bind(pty); @@ -117,24 +123,51 @@ export async function writeRunnerInput( // the user can retry); we never touch the buffer with a half write. (Idempotent // on the happy path: the previous message's // submit Enter already emptied the buffer, so this enqueues an ignored blank.) - if (!sendEnterWithRetry()) return { submitted: false }; + try { + if (!sendEnterWithRetry()) { + return { submitted: false, submissionDisposition: 'untouched' }; + } + } catch { + // The backend threw while attempting the pre-flush. No new frame bytes + // were intentionally written, but the Enter itself may have landed, so the + // runner generation is not proven clean enough for same-generation reuse. + return { submitted: false, submissionDisposition: 'dirty_unknown' }; + } const chunks = chunkAscii(line, RUNNER_INPUT_CHUNK_BYTES); for (let i = 0; i < chunks.length; i++) { - if (sendText(chunks[i]) === false) { + let chunkWritten: void | boolean; + try { + chunkWritten = sendText(chunks[i]); + } catch { + return { submitted: false, submissionDisposition: 'dirty_unknown' }; + } + if (chunkWritten === false) { // The chunks already written are a partial control line with no // terminating newline. Flush it (with retry) so it's less likely to // linger; even if every retry drops, the NEXT call's pre-flush gate above // refuses to write onto the dirty buffer, so no corruption-as-success can // slip through. - sendEnterWithRetry(); - return { submitted: false }; + try { sendEnterWithRetry(); } catch { /* disposition remains unknown */ } + // A tmux send-keys timeout reports false but cannot prove that the pane + // received zero bytes. In particular, a last-chunk timeout followed by a + // successful cleanup Enter may have submitted the complete valid frame. + // Never cancel attribution on this ambiguous boundary. + return { submitted: false, submissionDisposition: 'dirty_unknown' }; } if (i < chunks.length - 1) await delay(RUNNER_INPUT_THROTTLE_MS); } - // Submit (with retry — a single dropped Enter would leave a complete but + // Submit (with retry — a single unconfirmed Enter could leave a complete but // unsubmitted line in the buffer). - if (!sendEnterWithRetry()) return { submitted: false }; - return { submitted: true }; + try { + if (!sendEnterWithRetry()) { + // The complete, valid line may still be buffered. Retrying a successor + // in this generation could submit it against the wrong FIFO head. + return { submitted: false, submissionDisposition: 'dirty_unknown' }; + } + } catch { + return { submitted: false, submissionDisposition: 'dirty_unknown' }; + } + return { submitted: true, submissionDisposition: 'submitted' }; } diff --git a/src/adapters/cli/types.ts b/src/adapters/cli/types.ts index b25637081..0e0e4aa69 100644 --- a/src/adapters/cli/types.ts +++ b/src/adapters/cli/types.ts @@ -1,14 +1,17 @@ import type { CodexAppTurnInput } from '../../types.js'; export interface PtyHandle { - write(data: string): void; + /** `false` means the backend rejected the write before it could confirm + * delivery. Callers must not silently promote that result to success. */ + write(data: string): void | boolean; /** Send text literally via tmux send-keys -l (tmux mode only). - * Returns `false` when the write was dropped (e.g. send-keys failed while the - * pane is still alive) so callers can surface a non-submission; `void`/`true` - * means the write was issued. Backends that can't tell return void. */ + * Returns `false` when the backend could not confirm the write (for example, + * send-keys timed out while the pane stayed alive). Delivery may still have + * occurred, so callers must treat false as ambiguous rather than proof that + * zero bytes landed. `void`/`true` means the write was issued. */ sendText?(text: string): void | boolean; /** Send special keys via tmux send-keys, e.g. 'Enter', 'Escape', 'C-c' (tmux mode only). - * Returns `false` on a dropped write (see sendText). */ + * Returns `false` on an unconfirmed write (see sendText). */ sendSpecialKeys?(...keys: string[]): void | boolean; /** Paste text via tmux load-buffer + paste-buffer (auto-brackets if terminal supports it). */ pasteText?(text: string): void; @@ -30,6 +33,19 @@ export type SubmitRecheckResult = boolean | { cliSessionId?: string; }; +/** What the adapter can prove about a failed runner-protocol write. + * + * Runner adapters write a framed line and then a newline. `submitted:false` + * alone is insufficient for recovery: the line may be untouched, may have + * been flushed as an invalid fragment, or may still be a complete valid frame + * waiting in the runner's stdin buffer. Only the first two dispositions are + * safe to cancel/retry in the same generation. */ +export type RunnerSubmissionDisposition = + | 'submitted' + | 'untouched' + | 'flushed_invalid' + | 'dirty_unknown'; + /** A session discovered on disk that botmux can resume (import) into a topic — * surfaced by `/adopt`'s second filter. Unlike an AdoptableSession (a live * tmux/zellij pane botmux *observes*), this is a stored transcript botmux @@ -193,6 +209,7 @@ export interface CliAdapter { ): Promise SubmitRecheckResult | Promise; }>; diff --git a/src/bot-registry.ts b/src/bot-registry.ts index 2f015284e..f5e065a39 100644 --- a/src/bot-registry.ts +++ b/src/bot-registry.ts @@ -31,6 +31,16 @@ export type { VcMeetingConsumerProfileConfig, } from './types.js'; +/** Bound every official-SDK HTTP call so one stalled provider request cannot + * hold a bot-turn admission or maintenance mutation indefinitely. */ +export const LARK_REQUEST_TIMEOUT_MS = 15_000; + +export function configureLarkClientHttpTimeout(client: unknown): void { + const defaults = (client as { httpInstance?: { defaults?: { timeout?: number } } } | null) + ?.httpInstance?.defaults; + if (defaults) defaults.timeout = LARK_REQUEST_TIMEOUT_MS; +} + export type ChatReplyMode = 'chat' | 'new-topic' | 'shared' | 'chat-topic'; export type ContentTriggerScope = 'topic' | 'regularGroup' | 'both'; export type ContentTriggerMatchType = 'keyword' | 'regex'; @@ -1406,6 +1416,7 @@ export function registerBot(cfg: BotConfig): BotState { domain: sdkDomain(normalizeBrand(cfg.brand)), logger: larkLogger, }); + configureLarkClientHttpTimeout(client); const state: BotState = { config: cfg, client, diff --git a/src/cli.ts b/src/cli.ts index 7e830dfcc..138a37c9c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -76,7 +76,10 @@ import { import { interactiveSelect, pickChoice, pickCliSelection } from './setup/interactive-select.js'; import { buildPreset, serializePreset, presetFilename } from './setup/agent-preset.js'; import type { CliId } from './adapters/cli/types.js'; +import type { CodexAppDispatchLedgerEntry } from './types.js'; +import { hasProtectedSessionMutationOwnership } from './core/session-mutation-guard.js'; import { logger } from './utils/logger.js'; +import { withFileLock, withFileLockSync } from './utils/file-lock.js'; import { scrubSessionCliHomeEnv } from './utils/child-env.js'; import { scheduleTimeZone } from './utils/timezone.js'; import { expandHomePath, invalidWorkingDirs } from './utils/working-dir.js'; @@ -88,7 +91,6 @@ import { resolveDaemonEnv } from './cli/daemon-lifecycle-env.js'; import { buildPm2SpawnCommand } from './cli/pm2-command.js'; import { callDashboard, type DashboardEndpoint, type DashboardResult } from './cli/dashboard-endpoint.js'; import { globalInstallUpdateLockTargetIn, installLatestBotmuxSync } from './core/maintenance.js'; -import { withFileLockSync } from './utils/file-lock.js'; import { formatGlobalInstallCommand, resolveGlobalInstallPlan, @@ -3072,8 +3074,13 @@ interface SessionData { /** Chat-scope quote chain — see Session.quoteTargetId in types.ts. */ quoteTargetId?: string; currentReplyTarget?: { rootMessageId: string; turnId: string; updatedAt: string; quoteOnly?: boolean; substitute?: boolean }; - /** Per-turn reply targets（见 Session.replyTargets in types.ts）——排队/并发轮次各自的回复锚点。 */ replyTargets?: Record; + codexAppDispatchLedger?: CodexAppDispatchLedgerEntry[]; + codexAppGenerationCommits?: unknown; + queued?: boolean; + queuedActivationPending?: boolean; + queuedActivationTail?: import('./types.js').QueuedActivationTailEntry[]; + pendingRepoSetup?: import('./types.js').PendingRepoSetup; /** Current persisted worker lifetime and its exact input-queue receipts. */ workerGeneration?: number; dispatchInputReceipts?: Record { if (process.env.BOTMUX_LARK_APP_ID) loadPerBot(process.env.BOTMUX_LARK_APP_ID); } - // Migrate: remove sessions from legacy file if they have larkAppId (belong in per-bot files) - let legacyDirty = false; - for (const [k, v] of Object.entries(legacyData)) { - const s = v as SessionData; - if (s.larkAppId) { - delete legacyData[k]; - legacyDirty = true; - // Ensure the session exists in its per-bot file - const perBotFp = join(dataDir, `sessions-${s.larkAppId}.json`); - let perBotData: Record = {}; - if (existsSync(perBotFp)) { - try { perBotData = JSON.parse(readFileSync(perBotFp, 'utf-8')); } catch { /* */ } - } - // Only write if per-bot file doesn't already have this session - if (!perBotData[k]) { - perBotData[k] = s; - const tmpFp = perBotFp + '.tmp'; - writeFileSync(tmpFp, JSON.stringify(perBotData, null, 2), 'utf-8'); - renameSync(tmpFp, perBotFp); - } - } - } - if (legacyDirty) { - const tmpFp = legacyFp + '.tmp'; - writeFileSync(tmpFp, JSON.stringify(legacyData, null, 2), 'utf-8'); - renameSync(tmpFp, legacyFp); - } + // Deliberately read-only. Older CLIs opportunistically migrated legacy rows + // here, which made even `botmux list` a whole-file writer capable of racing a + // daemon ledger save. Durable migration belongs to daemon startup under the + // session-store lock; this generic discovery path only composes snapshots. return sessions; } -/** Save a single session back to its appropriate file based on larkAppId. */ -function loadSessionFresh(session: SessionData): SessionData | undefined { - return loadSessions().get(session.sessionId); -} - -function saveSession(session: SessionData): void { +/** Offline-only narrow session mutation. Callers must prefer the owning daemon + * while it is available; rereading the exact row here prevents stale CLI + * snapshots from copying FIFO fields during offline maintenance. */ +function mutateSessionOffline( + session: SessionData, + mutate: (current: SessionData) => boolean, +): SessionData | undefined { const dataDir = resolveDataDir(); const fileName = session.larkAppId ? `sessions-${session.larkAppId}.json` : 'sessions.json'; const fp = join(dataDir, fileName); + return withFileLockSync(fp, () => { + // The owning daemon uses the same session-file lock for every save. Check + // absence while holding it so two CLIs cannot race each other and an + // already-published daemon cannot be bypassed by the offline fallback. + if (session.larkAppId) { + try { if (findDaemon(session.larkAppId)) return undefined; } catch { /* offline */ } + } + let data: Record = {}; + if (existsSync(fp)) { + try { data = JSON.parse(readFileSync(fp, 'utf-8')); } catch { /* start fresh */ } + } + const current = data[session.sessionId]; + if (!current || !mutate(current)) return current; + data[session.sessionId] = current; + + // Clean up entries where file key doesn't match the entry's sessionId + // (data corruption), and strip removed placeholder-card fields. + for (const [key, val] of Object.entries(data)) { + if (val && typeof val === 'object' && 'sessionId' in val && (val as SessionData).sessionId !== key) { + delete data[key]; + continue; + } + if (val && typeof val === 'object') stripLegacyPendingCardFields(val as unknown as Record); + } - // Read current file, update session, write back - let data: Record = {}; - if (existsSync(fp)) { - try { data = JSON.parse(readFileSync(fp, 'utf-8')); } catch { /* start fresh */ } - } - data[session.sessionId] = session; - - // Clean up entries where file key doesn't match the entry's sessionId (data - // corruption), and strip legacy placeholder-card fields so the file converges - // to clean (see stripLegacyPendingCardFields in services/session-store). - for (const [key, val] of Object.entries(data)) { - if (val && typeof val === 'object' && 'sessionId' in val && (val as SessionData).sessionId !== key) { - delete data[key]; - continue; + // Recheck immediately before publication. A daemon that appears during + // the read/decision phase becomes authoritative; leave the file untouched. + if (session.larkAppId) { + try { if (findDaemon(session.larkAppId)) return undefined; } catch { /* offline */ } } - if (val && typeof val === 'object') stripLegacyPendingCardFields(val as unknown as Record); - } + const tmpFp = `${fp}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`; + writeFileSync(tmpFp, JSON.stringify(data, null, 2), 'utf-8'); + renameSync(tmpFp, fp); + return current; + }); +} + +type OfflineAbandonResult = + | { ok: true; current: SessionData; cleanedBacking?: string } + | { ok: false; error: string }; - const tmpFp = fp + '.tmp'; - writeFileSync(tmpFp, JSON.stringify(data, null, 2), 'utf-8'); - renameSync(tmpFp, fp); +/** Offline explicit abandon stops the exact Botmux-owned worker, then closes + * the newest durable row under the shared session-file lock. Provider-specific + * backing cleanup remains owned by the dedicated backend lifecycle changes. */ +async function abandonSessionOffline(session: SessionData): Promise { + let current = mutateSessionOffline(session, () => false); + if (!current) return { ok: false, error: 'owning_daemon_became_available' }; - // Remove duplicate from legacy file if session moved to per-bot file (or vice versa) - const otherFile = session.larkAppId ? 'sessions.json' : null; - if (otherFile) { - const otherFp = join(dataDir, otherFile); - if (existsSync(otherFp)) { + const originalPid = adoptedCliPid(current); + const ownedWorkerPid = current.pid && current.pid !== originalPid ? current.pid : undefined; + if (ownedWorkerPid) { + // Narrow the unavoidable descriptor race: do not signal a worker after an + // owning daemon has become visible. The locked write below repeats this. + if (current.larkAppId) { try { - const otherData: Record = JSON.parse(readFileSync(otherFp, 'utf-8')); - if (otherData[session.sessionId]) { - delete otherData[session.sessionId]; - const otherTmp = otherFp + '.tmp'; - writeFileSync(otherTmp, JSON.stringify(otherData, null, 2), 'utf-8'); - renameSync(otherTmp, otherFp); + if (findDaemon(current.larkAppId)) { + return { ok: false, error: 'owning_daemon_became_available' }; } - } catch { /* ignore */ } + } catch { /* still offline */ } + } + if (isProcessAlive(ownedWorkerPid)) { + const signalled = killProcess(ownedWorkerPid); + if ((!signalled && isProcessAlive(ownedWorkerPid)) + || !(await waitForProcessExit(ownedWorkerPid))) { + return { ok: false, error: `worker_pid_kill_failed:${ownedWorkerPid}` }; + } + } + + // Persist worker-less state without touching FIFO authority. If a new + // daemon/generation changed the row while SIGTERM settled, fail closed. + let workerCleared = false; + const afterStop = mutateSessionOffline(current, latest => { + if (latest.pid !== ownedWorkerPid + || isAdoptedSession(latest) !== isAdoptedSession(current!)) return false; + delete latest.pid; + workerCleared = true; + return true; + }); + if (!afterStop || !workerCleared) { + return { ok: false, error: 'session_changed_while_stopping_worker' }; } + current = afterStop; + } else { + // Even without a Botmux worker pid, re-read after the first authority check + // so the cleanup inputs are the newest locked backend/task lineage. + const refreshed = mutateSessionOffline(current, () => false); + if (!refreshed) return { ok: false, error: 'owning_daemon_became_available' }; + current = refreshed; + } + + let applied = false; + const published = mutateSessionOffline(current, latest => { + if (isAdoptedSession(latest) !== isAdoptedSession(current)) return false; + if (latest.status === 'closed') { + applied = true; + return false; + } + latest.status = 'closed'; + latest.closedAt = new Date().toISOString(); + delete latest.codexAppDispatchLedger; + delete latest.codexAppGenerationCommits; + delete latest.queuedActivationPending; + delete latest.queuedActivationTail; + delete latest.pendingRepoSetup; + applied = true; + return true; + }); + if (!published || !applied) { + return { ok: false, error: 'session_changed_during_offline_cleanup' }; + } + + return { ok: true, current: published }; +} + +function pruneSessionOfflineIfLedgerEmpty(session: SessionData): boolean { + let pruned = false; + mutateSessionOffline(session, current => { + if (hasProtectedSessionMutationOwnership(current)) return false; + current.status = 'closed'; + current.closedAt = new Date().toISOString(); + delete current.codexAppDispatchLedger; + delete current.codexAppGenerationCommits; + pruned = true; + return true; + }); + return pruned; +} + +function patchSessionWhiteboardOffline(session: SessionData, whiteboardId: string): boolean { + return !!mutateSessionOffline(session, current => { + current.whiteboardId = whiteboardId; + return true; + }); +} + +async function postOwningDaemonSessionMutation( + session: SessionData, + suffix: 'close' | 'prune' | 'whiteboard', + body?: Record, +): Promise<'applied' | 'refused' | 'unavailable'> { + if (!session.larkAppId) return 'unavailable'; + let daemon: ReturnType; + try { daemon = findDaemon(session.larkAppId); } catch { return 'unavailable'; } + if (!daemon) return 'unavailable'; + let secret: string; + try { secret = loadDaemonIpcSecret(); } catch { return 'unavailable'; } + const res = await fetchDaemonIpc( + daemon.ipcPort, + `/api/sessions/${encodeURIComponent(session.sessionId)}/${suffix}`, + { + method: 'POST', + ...(body + ? { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + } + : {}), + }, + secret, + ); + if (suffix === 'prune' && res.status === 409) return 'refused'; + if (!res.ok) { + throw new Error(`owning daemon ${suffix} mutation failed: HTTP ${res.status}`); } + return 'applied'; +} + +type AuthoritativeAbandonResult = + | { ok: true; mode: 'daemon' } + | { ok: true; mode: 'offline'; current: SessionData; cleanedBacking?: string } + | { ok: false; error?: string }; + +async function abandonSessionAuthoritatively(session: SessionData): Promise { + const result = await postOwningDaemonSessionMutation(session, 'close'); + if (result === 'applied') return { ok: true, mode: 'daemon' }; + if (result === 'refused') return { ok: false }; + const offline = await abandonSessionOffline(session); + return offline.ok + ? { + ok: true, + mode: 'offline', + current: offline.current, + ...(offline.cleanedBacking ? { cleanedBacking: offline.cleanedBacking } : {}), + } + : { ok: false, error: offline.error }; +} + +async function pruneSessionAuthoritatively(session: SessionData): Promise { + const result = await postOwningDaemonSessionMutation(session, 'prune'); + if (result === 'applied') return true; + if (result === 'refused') return false; + return pruneSessionOfflineIfLedgerEmpty(session); +} + +async function patchSessionWhiteboardAuthoritatively( + session: SessionData, + whiteboardId: string, +): Promise { + const result = await postOwningDaemonSessionMutation(session, 'whiteboard', { whiteboardId }); + if (result === 'applied') return true; + if (result === 'refused') return false; + return patchSessionWhiteboardOffline(session, whiteboardId); } function isProcessAlive(pid: number): boolean { @@ -3253,6 +3401,15 @@ function killProcess(pid: number): boolean { } } +async function waitForProcessExit(pid: number, timeoutMs = 2_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isProcessAlive(pid)) return true; + await new Promise(resolve => setTimeout(resolve, 50)); + } + return !isProcessAlive(pid); +} + function formatDuration(ms: number): string { const seconds = Math.floor(ms / 1000); if (seconds < 60) return `${seconds}s`; @@ -3628,26 +3785,21 @@ function interactiveSessionPicker(active: SessionData[]): Promise { process.stdout.write('\x1b[?1049l'); // leave alt screen } - function deleteSession(idx: number): void { + async function deleteSession(idx: number): Promise { const r = rows[idx]; const s = r.session; - // Kill botmux's worker process. For adopted sessions, never kill the - // user's original CLI pid if an old record stored it in `pid`. - const originalPid = adoptedCliPid(s); - if (s.pid && s.pid !== originalPid && isProcessAlive(s.pid)) { - killProcess(s.pid); - } - - // Kill only botmux-owned tmux sessions. Adopted panes belong to the user. - if (!r.isAdopt && r.hasTmux) { - try { execSync(`tmux kill-session -t '${r.tmuxName}' 2>/dev/null`, { stdio: 'ignore', env: tmuxEnv() }); } catch { /* */ } + // Explicit delete is the abandon boundary. Prefer the owning daemon so + // its current in-memory FIFO is cleared atomically with close; offline + // fallback rereads the exact row and clears the latest ledger/commits. + const result = await abandonSessionAuthoritatively(s); + if (!result.ok) { + flashMsg = `\x1b[31m删除失败 ${s.sessionId.substring(0, 8)}${result.error ? `：${result.error}` : ''}；会话保持活跃，可重试\x1b[0m`; + return; } - - // Mark closed & persist - s.status = 'closed'; - s.closedAt = new Date().toISOString(); - saveSession(s); + // A live daemon owns process/pane teardown and has re-resolved the latest + // generation. Only the locked offline path may clean up resources, using + // the freshly-read row rather than this TUI's stale snapshot. // Remove from active list and TUI rows const activeIdx = active.indexOf(s); @@ -3658,12 +3810,12 @@ function interactiveSessionPicker(active: SessionData[]): Promise { flashMsg = `\x1b[32m✓ 已删除 ${s.sessionId.substring(0, 8)}\x1b[0m`; } - process.stdin.on('data', (key: string) => { + process.stdin.on('data', async (key: string) => { // Delete confirmation mode if (confirmDelete) { confirmDelete = false; if (key === 'y' || key === 'Y') { - deleteSession(cursor); + await deleteSession(cursor); } else { flashMsg = '\x1b[2m取消删除\x1b[0m'; } @@ -3769,18 +3921,26 @@ async function cmdList(): Promise { else if (disposition === 'prune_scratch') prunedScratch.push(s); else live.push(s); } - const closeNow = (arr: SessionData[]) => { + const closeNow = async (arr: SessionData[]): Promise => { + let closed = 0; for (const s of arr) { - s.status = 'closed'; - s.closedAt = new Date().toISOString(); - saveSession(s); + if (await pruneSessionAuthoritatively(s)) { + closed++; + } else { + // The owning daemon observed a newly unsettled FIFO after this CLI's + // liveness snapshot. Keep the owner visible instead of abandoning it. + live.push(s); + } } + return closed; }; // Scratches: close silently — they were placeholders, not dead sessions. - closeNow(prunedScratch); + await closeNow(prunedScratch); if (pruned.length > 0) { - closeNow(pruned); - console.log(`已自动清理 ${pruned.length} 个不可恢复的会话（进程已退出或无可恢复后端）`); + const prunedCount = await closeNow(pruned); + if (prunedCount > 0) { + console.log(`已自动清理 ${prunedCount} 个不可恢复的会话（进程已退出或无可恢复后端）`); + } } // Sort by creation time, newest first @@ -3801,7 +3961,7 @@ async function cmdList(): Promise { await interactiveSessionPicker(live); } -function cmdDelete(): void { +async function cmdDelete(): Promise { const target = process.argv[3]; if (!target) { console.error('用法: botmux delete '); @@ -3852,29 +4012,13 @@ function cmdDelete(): void { } for (const s of toDelete) { - const originalPid = adoptedCliPid(s); - - // Kill botmux's worker process if running. For adopted sessions, never - // kill the user's original CLI pid. - if (s.pid && s.pid !== originalPid && isProcessAlive(s.pid)) { - killProcess(s.pid); - console.log(` killed pid ${s.pid}`); + const result = await abandonSessionAuthoritatively(s); + if (!result.ok) { + throw new Error(`failed to close session ${s.sessionId}${result.error ? `: ${result.error}` : ''}`); } - - // Kill associated botmux-owned tmux session if it exists. Adopted panes - // belong to the user and must be left untouched. - const tmuxName = `bmx-${s.sessionId.substring(0, 8)}`; - if (!isAdoptedSession(s)) { - try { - execSync(`tmux kill-session -t '${tmuxName}' 2>/dev/null`, { stdio: 'ignore', env: tmuxEnv() }); - console.log(` killed tmux ${tmuxName}`); - } catch { /* no tmux session */ } + if (result.mode === 'offline') { + if (result.cleanedBacking) console.log(` killed ${result.cleanedBacking}`); } - - // Mark session as closed - s.status = 'closed'; - s.closedAt = new Date().toISOString(); - saveSession(s); console.log(`✓ ${s.sessionId.substring(0, 8)} ${s.title}`); } console.log(`\n已关闭 ${toDelete.length} 个会话`); @@ -5079,7 +5223,10 @@ Context flags: --session-id, --lark-app-id, --chat-id, --working-dir/--repo`); let meta = ctx.session?.whiteboardId ? getWhiteboard(ctx.session.whiteboardId) : undefined; if (!meta && argFlag(rest, '--create')) { meta = ensureDefaultWhiteboard({ larkAppId: ctx.larkAppId, chatId: ctx.chatId, workingDir: ctx.workingDir, sessionId: ctx.sessionId }); - if (ctx.session) { ctx.session.whiteboardId = meta.id; saveSession(ctx.session); } + if (ctx.session) { + await patchSessionWhiteboardAuthoritatively(ctx.session, meta.id); + ctx.session.whiteboardId = meta.id; + } } if (!meta) { console.log(JSON.stringify({ enabled: true, current: null, hint: 'Run `botmux whiteboard current --create` to ensure the default board.' }, null, 2)); @@ -5093,7 +5240,10 @@ Context flags: --session-id, --lark-app-id, --chat-id, --working-dir/--repo`); requireWhiteboardEnabled(); const ctx = currentWhiteboardContext(rest); const meta = createWhiteboard({ id: argValue(rest, '--id'), title: argValue(rest, '--title'), larkAppId: ctx.larkAppId, chatId: ctx.chatId, workingDir: ctx.workingDir, sessionId: ctx.sessionId }); - if (ctx.session && !ctx.session.whiteboardId) { ctx.session.whiteboardId = meta.id; saveSession(ctx.session); } + if (ctx.session && !ctx.session.whiteboardId) { + await patchSessionWhiteboardAuthoritatively(ctx.session, meta.id); + ctx.session.whiteboardId = meta.id; + } console.log(JSON.stringify({ board: meta, path: whiteboardPath(meta.id) }, null, 2)); return; } @@ -5114,7 +5264,10 @@ Context flags: --session-id, --lark-app-id, --chat-id, --working-dir/--repo`); if (!id && whiteboardEnabled() && action === 'update') { const meta = ensureDefaultWhiteboard({ larkAppId: ctx.larkAppId, chatId: ctx.chatId, workingDir: ctx.workingDir, sessionId: ctx.sessionId }); id = meta.id; - if (ctx.session) { ctx.session.whiteboardId = id; saveSession(ctx.session); } + if (ctx.session) { + await patchSessionWhiteboardAuthoritatively(ctx.session, id); + ctx.session.whiteboardId = id; + } } if (!id) { console.error('No whiteboard id. Pass --id or run `botmux whiteboard current --create`.'); process.exit(1); } @@ -6034,10 +6187,40 @@ async function cmdSend(rest: string[]): Promise { ) ? trustedRelayCandidate : undefined; + // Keep origin authority independent from the requested destination. In + // particular, `--session-id ` may select where an ordinary Lark turn + // is sent, but it must never replace the session whose durable delivery sink + // governs this process. Session identity may fall back to the inherited env + // for detached commands; the turn tuple may not. `ancestorCtx` deliberately + // adds spawn-time BOTMUX_TURN_ID/ATTEMPT compatibility fallbacks, which can be + // stale after a later turn, so only a parent-bound host relay or a fresh + // marker/protected-capability tuple can authorize a durable dispatch. const originSessionId = trustedRelayCtx?.sessionId - || liveMarkerCtx?.sessionId - || ancestorCtx?.sessionId - || sessionIdArg; + ?? liveMarkerCtx?.sessionId + ?? ancestorCtx?.sessionId + ?? process.env.BOTMUX_SESSION_ID; + const authoritativeOriginTurnCtx = trustedRelayCtx + ? (trustedRelayCtx.turnId ? trustedRelayCtx : undefined) + : (liveMarkerCtx?.turnId ? liveMarkerCtx : undefined); + const originTurnId = authoritativeOriginTurnCtx?.turnId; + const originDispatchAttempt = authoritativeOriginTurnCtx?.dispatchAttempt; + const originSession = originSessionId + ? sessionsForOrigin.get(originSessionId) + : undefined; + const hostRelayRequiresCodexAppLedger = !!trustedRelayCtx + && process.env.BOTMUX_HOST_RELAY_REQUIRES_CODEX_APP_LEDGER === '1'; + // The live worker authorizes a Codex App relay only while an exact durable + // dispatch remains unsettled. Recheck that prerequisite before any other + // managed-output policy or provider setup: terminal settlement can win the + // race between watcher authorization and this host child loading the store, + // and must never downgrade the send into an ordinary Lark message. + if (hostRelayRequiresCodexAppLedger + && (originSession?.codexAppDispatchLedger?.length ?? 0) === 0) { + console.error( + `botmux send refused: authorized Codex App origin ${originSessionId}/${originTurnId} is no longer unsettled`, + ); + process.exit(2); + } let explicitVcMeetingImOrigin: ReturnType; let vcMeetingListenerOutputOwner: { listenerAppId: string; meetingId: string } | undefined; let vcMeetingManagedSendOrigin: VcMeetingManagedSendOrigin | undefined; @@ -6047,14 +6230,12 @@ async function cmdSend(rest: string[]): Promise { dispatchAttempt: number; } | undefined; if (originSessionId) { - const originSession = sessionsForOrigin.get(originSessionId); - const originTurnId = trustedRelayCtx?.turnId ?? liveMarkerCtx?.turnId; const imOrigin = resolveVcMeetingImTurnOrigin(originSession, originTurnId); const managedOrigin: VcMeetingManagedSendOrigin = { receiverSessionId: originSessionId, receiverSession: !!originSession?.vcMeetingReceiver, turnId: originTurnId, - dispatchAttempt: trustedRelayCtx?.dispatchAttempt ?? liveMarkerCtx?.dispatchAttempt, + dispatchAttempt: originDispatchAttempt, currentImTurnOrigin: imOrigin, }; const decision = evaluateVcMeetingManagedSend(resolveDataDir(), managedOrigin); @@ -6076,7 +6257,7 @@ async function cmdSend(rest: string[]): Promise { } } if (decision.kind === 'listener_thread' - && (trustedRelayCtx?.dispatchAttempt ?? liveMarkerCtx?.dispatchAttempt) === undefined) { + && originDispatchAttempt === undefined) { explicitVcMeetingImOrigin = imOrigin; } } @@ -6238,7 +6419,7 @@ async function cmdSend(rest: string[]): Promise { } const sessions = loadSessions(); - const currentTurnId = ancestorCtx?.turnId ?? process.env.BOTMUX_TURN_ID; + const currentTurnId = originTurnId; let s = sessions.get(sid); // Riff (remote backend) sandbox: no local daemon/sessions.json/bots.json. @@ -6269,10 +6450,103 @@ async function cmdSend(rest: string[]): Promise { let deferredMaterializedByThisCommand = false; let deferredTopicRootMessageIdForOutput: string | undefined; - // 本轮的回复锚点：优先按 turnId 精确取 per-turn 落点（排队/并发轮次不再被 - // 后到轮次覆盖 currentReplyTarget 而塌成群顶层 plain），无记录再退回单槽。 + // Prefer the exact per-turn reply anchor; the latest single slot is only a + // compatibility fallback for sessions persisted before replyTargets. const turnReplyTarget = pickTurnReplyTarget(s, currentTurnId); + const exactOriginDispatch = (() => { + const unsettledOriginDispatches = originSession?.codexAppDispatchLedger ?? []; + // The live worker authorized this host re-exec against an exact unsettled + // Codex App entry. If terminal settlement/revocation won the race before + // this child reloaded the store, never degrade into an ordinary Lark send + // against mutable session state. + if (unsettledOriginDispatches.length === 0) { + return undefined; + } + // A detached/backgrounded command can still inherit the correct session id + // but only a spawn-time (and now stale) turn env. If this session has any + // unsettled durable output, absence of a fresh marker/capability tuple must + // fail closed rather than silently falling through to ordinary Lark IM. + if (!originTurnId) { + console.error( + `botmux send refused: origin session ${originSessionId} has unsettled durable output but no fresh authoritative dispatch identity`, + ); + process.exit(2); + } + const turnMatches = unsettledOriginDispatches + .filter(entry => entry.turnId === originTurnId); + const exactMatches = originDispatchAttempt === undefined + ? turnMatches + : turnMatches.filter(entry => entry.dispatchAttempt === originDispatchAttempt); + // Once a durable turn exists, a supplied attempt must match it exactly. + // Falling back to another attempt would let a stale/replayed process select + // the wrong sink. With no attempt, multiple entries are equally unsafe. + if (exactMatches.length !== 1) { + const detail = originDispatchAttempt === undefined + ? `${turnMatches.length} ledger entries share the turn id` + : `${exactMatches.length} ledger entries match attempt ${originDispatchAttempt}`; + console.error( + `botmux send refused: origin dispatch ${originSessionId}/${originTurnId} is ambiguous (${detail})`, + ); + process.exit(2); + } + return exactMatches[0]; + })(); + + // Frozen reply routing applies only when the selected destination is the + // trusted origin session. A cross-session publish must not accidentally + // reuse a coincidentally-equal turn id from the destination ledger. + const frozenTurnDispatch = originSessionId === sid + ? exactOriginDispatch + : undefined; + const frozenTurnReplyTarget = frozenTurnDispatch?.replyTarget; + if (exactOriginDispatch?.deliverySink === 'http_wait' + || exactOriginDispatch?.deliverySink === 'http_async' + || exactOriginDispatch?.deliverySink === 'suppressed') { + console.error( + `botmux send refused: origin turn ${originTurnId} is bound to the ` + + `${exactOriginDispatch.deliverySink} host sink`, + ); + process.exit(2); + } + + // A document-comment turn has exactly one supported observable effect: a + // plain text reply to its frozen origin target. Validate the complete shape + // before reading stdin/content/card files and before any TTS, upload, or Lark + // provider call. In particular, an explicit destination session is not an + // escape hatch from the origin sink. + const docTarget = originTurnId + ? originSession?.docCommentTargets?.[originTurnId] + : undefined; + // Codex App turns are governed exclusively by their exact dispatch ledger: + // a settled/missing ledger entry must never fall back to mutable session + // state. Other CLI adapters predate that ledger, so their frozen per-turn + // docCommentTargets entry remains the authoritative origin sink. + const isOriginDocCommentTurn = exactOriginDispatch?.deliverySink === 'doc_comment' + || (!exactOriginDispatch && originSession?.cliId !== 'codex-app' && !!docTarget); + if (isOriginDocCommentTurn) { + if (!docTarget || !originSession?.larkAppId) { + console.error('botmux send refused: this turn is bound to a document comment, but its exact origin target is no longer available'); + process.exit(2); + } + if (sid !== originSessionId + || sendTopLevel + || !!overrideChatId + || !!sendInto + || asVoice + || images.length > 0 + || files.length > 0 + || videoAttachments.length > 0 + || videoCovers.length > 0 + || customCardRequested + || attention.requested + || explicitQuote !== undefined + || noQuote) { + console.error('botmux send refused: a document-comment turn supports only its exact plain-text comment reply'); + process.exit(2); + } + } + // Read content from: --content-file > positional arg > stdin let content = ''; let customCard: Record | undefined; @@ -6422,17 +6696,19 @@ async function cmdSend(rest: string[]): Promise { deferredTopicRootMessageIdForOutput = deferred.rootMessageId; messageId = deferred.messageId; } else { - const canonicalTarget = resolveSendTarget({ - into: sendInto, - topLevel: sendTopLevel, - chatScope: s.scope === 'chat', - chatId: canonicalOutput.targetChatId, - rootMessageId: s.rootMessageId, - replyTargetRootId: turnReplyTarget?.rootMessageId, - replyTargetTurnId: turnReplyTarget?.turnId, - replyTargetQuoteOnly: turnReplyTarget?.quoteOnly, - currentTurnId, - }); + const canonicalTarget = !sendInto && !sendTopLevel && !overrideChatId && frozenTurnReplyTarget + ? frozenTurnReplyTarget + : resolveSendTarget({ + into: sendInto, + topLevel: sendTopLevel, + chatScope: s.scope === 'chat', + chatId: canonicalOutput.targetChatId, + rootMessageId: s.rootMessageId, + replyTargetRootId: turnReplyTarget?.rootMessageId, + replyTargetTurnId: turnReplyTarget?.turnId, + replyTargetQuoteOnly: turnReplyTarget?.quoteOnly, + currentTurnId, + }); messageId = canonicalTarget.mode === 'plain' ? await sendMessage( appId, @@ -6491,28 +6767,22 @@ async function cmdSend(rest: string[]): Promise { } // ── 文档评论入口分流（/watch-comment / /subscribe-lark-doc）───────────────── - // 本轮若由飞书文档评论触发（daemon 已把落点写进 session.docCommentTargets[turnId]）， - // 把用户可见回复发表为飞书文档评论，而非发回飞书会话。绕过 @ 硬门（评论不 @ 飞书 - // 用户）。显式改路由（--top-level / --chat-id / --into）时不分流，让模型仍能主动 - // 用 BOTMUX_TURN_ID 从 per-turn map 取本轮落点；非文档轮的 turnId 不会命中 map， - // 天然不会误投。per-turn map 设计：并发评论之间互不覆盖，不会串线。 - const docTarget = currentTurnId ? s.docCommentTargets?.[currentTurnId] : undefined; - if (docTarget && !sendTopLevel && !overrideChatId && !sendInto) { - if (customCardRequested) { - console.error('botmux send: 文档评论回复不支持 --card-file/--card-json；请改用普通文本，或显式 --top-level/--chat-id 发到飞书群'); - process.exit(2); - } + // The authority and payload-shape checks ran before content/provider work. + // Use only the trusted origin session here; `--session-id` is a destination + // selector for ordinary Lark turns and cannot retarget a document turn. + if (isOriginDocCommentTurn) { + const exactDocTarget = docTarget!; + const exactDocSession = originSession!; const { registerBot, loadBotConfigs } = await import('./bot-registry.js'); try { for (const cfg of loadBotConfigs()) registerBot(cfg); } catch { /* */ } if (envPinnedRiffBot) { try { registerBot(envPinnedRiffBot); } catch { /* */ } } const { replyToDocComment, chunkCommentText, removeCommentReaction } = await import('./im/lark/doc-comment.js'); - const appId = s.larkAppId!; - const loc = localeForBot(appId); + const appId = exactDocSession.larkAppId!; try { // @ 落点：--mention-back → 回 @ 原评论人；--mention → @ 指定人； // 否则（--no-mention / 无）不 @。文档评论里靠 person 元素渲染 @，仅首块加。 let docMentionOpenId: string | undefined; - if (mentionBack) docMentionOpenId = docTarget.replyToOpenId; + if (mentionBack) docMentionOpenId = exactDocTarget.replyToOpenId; else if (mentionArgs.length > 0) { const first = mentionArgs[0]; const idx = first.indexOf(':'); @@ -6521,27 +6791,31 @@ async function cmdSend(rest: string[]): Promise { // 嵌套回复到用户那条评论 thread（已挂其下，无需 ↪ 前缀）。 const chunks = chunkCommentText(content); for (let i = 0; i < chunks.length; i++) { - await replyToDocComment(appId, { fileToken: docTarget.fileToken, fileType: docTarget.fileType }, docTarget.commentId, chunks[i], i === 0 ? docMentionOpenId : undefined); + await replyToDocComment( + appId, + { fileToken: exactDocTarget.fileToken, fileType: exactDocTarget.fileType }, + exactDocTarget.commentId, + chunks[i], + i === 0 ? docMentionOpenId : undefined, + ); } // 清理 "Typing" reaction（bot 已回复完毕）。 - if (docTarget.reactionId && docTarget.replyId) { + if (exactDocTarget.reactionId && exactDocTarget.replyId) { await removeCommentReaction(appId, - { fileToken: docTarget.fileToken, fileType: docTarget.fileType }, - docTarget.commentId, docTarget.replyId, docTarget.reactionId); + { fileToken: exactDocTarget.fileToken, fileType: exactDocTarget.fileType }, + exactDocTarget.commentId, exactDocTarget.replyId, exactDocTarget.reactionId); } // 写 bridge send marker → 抑制 worker 的 final_output 兜底（否则会再补一条评论）。 try { const markerDir = join(resolveDataDir(), 'turn-sends'); if (!existsSync(markerDir)) mkdirSync(markerDir, { recursive: true }); - appendFileSync(join(markerDir, `${sid}.jsonl`), JSON.stringify({ sentAtMs: Date.now(), messageId: `doc:${docTarget.commentId}`, contentLength: content.length }) + '\n'); + appendFileSync(join(markerDir, `${originSessionId}.jsonl`), JSON.stringify({ sentAtMs: Date.now(), messageId: `doc:${exactDocTarget.commentId}`, contentLength: content.length }) + '\n'); } catch { /* best-effort：漏记只多一条兜底 */ } - // 清理已消费的 per-turn 落点，避免 session 文件无限堆积。 - if (s.docCommentTargets && currentTurnId && s.docCommentTargets[currentTurnId]) { - delete s.docCommentTargets[currentTurnId]; - try { saveSession(s); } catch { /* best-effort */ } - } - console.error(`✓ 已回复文档评论 ${docTarget.commentId.slice(0, 12)}（${chunks.length} 条）`); - console.log(JSON.stringify({ success: true, commentId: docTarget.commentId, sessionId: sid, kind: 'doc-comment', chunks: chunks.length })); + // Do not write this startup snapshot back after the async provider calls: + // the daemon may have advanced the dispatch ledger or accepted another + // turn in the meantime. Daemon settlement owns exact target retirement. + console.error(`✓ 已回复文档评论 ${exactDocTarget.commentId.slice(0, 12)}（${chunks.length} 条）`); + console.log(JSON.stringify({ success: true, commentId: exactDocTarget.commentId, sessionId: originSessionId, kind: 'doc-comment', chunks: chunks.length })); } catch (e: any) { console.error(`文档评论发送失败：${e?.message ?? e}`); process.exit(1); @@ -6563,6 +6837,7 @@ async function cmdSend(rest: string[]): Promise { } } const replyTargetSenderOpenId = explicitVcMeetingImOrigin?.replyTargetSenderOpenId + ?? frozenTurnDispatch?.replyTargetSenderOpenId ?? s.quoteTargetSenderOpenId; // @ hard-gate (config.send.requireMentionDecision, default on): force the @@ -6663,7 +6938,9 @@ async function cmdSend(rest: string[]): Promise { }; // Dispatch helper: top-level / chat-scope send vs reply-in-thread, single // decision point. Used for file attachments (always plain in chat scope). - const sendTarget = resolveSendTarget({ into: sendInto, topLevel: sendTopLevel, chatScope: isChatScope, chatId: targetChatId, rootMessageId: s.rootMessageId, replyTargetRootId: turnReplyTarget?.rootMessageId, replyTargetTurnId: turnReplyTarget?.turnId, replyTargetQuoteOnly: turnReplyTarget?.quoteOnly, currentTurnId }); + const sendTarget = !sendInto && !sendTopLevel && !overrideChatId && frozenTurnReplyTarget + ? frozenTurnReplyTarget + : resolveSendTarget({ into: sendInto, topLevel: sendTopLevel, chatScope: isChatScope, chatId: targetChatId, rootMessageId: s.rootMessageId, replyTargetRootId: turnReplyTarget?.rootMessageId, replyTargetTurnId: turnReplyTarget?.turnId, replyTargetQuoteOnly: turnReplyTarget?.quoteOnly, currentTurnId }); const dispatch = async ( content: string, msgType: string, @@ -6684,28 +6961,28 @@ async function cmdSend(rest: string[]): Promise { msgType, uuid, sendRoot: (body, type, rootUuid) => sendMessage( - appId, - targetChatId, - body, - type, - rootUuid, - hookContext, - suppressHook ? { suppressHook: true } : undefined, - ), + appId, + targetChatId, + body, + type, + rootUuid, + hookContext, + suppressHook ? { suppressHook: true } : undefined, + ), // The optional title is the root seed and the actual alert follows as // its first reply. Do not emit a user outbound hook for presentation- // only seed text; the alert itself still goes through the hook path. sendTitleSeed: (title, rootUuid) => sendMessage(appId, targetChatId, title, 'text', rootUuid), replyRoot: (root, body, type, replyUuid) => replyMessage( - appId, - root, - body, - type, - true, - replyUuid, - hookContext, - suppressHook ? { suppressHook: true } : undefined, - ), + appId, + root, + body, + type, + true, + replyUuid, + hookContext, + suppressHook ? { suppressHook: true } : undefined, + ), }); if (deferred.handled && deferred.messageId) { deferredMaterializedByThisCommand ||= deferred.materializedNow === true; @@ -6757,7 +7034,9 @@ async function cmdSend(rest: string[]): Promise { // belong to another queued IM turn and is not part of the delivery action. sessionQuoteTargetId: vcMeetingDeliveryReplyOrigin ? undefined - : explicitVcMeetingImOrigin?.larkMessageId ?? s.quoteTargetId, + : explicitVcMeetingImOrigin?.larkMessageId + ?? frozenTurnDispatch?.quoteTargetId + ?? s.quoteTargetId, }); let primaryQuotedId: string | null = null; let vcMeetingListenerReplyReplay = false; @@ -6814,7 +7093,7 @@ async function cmdSend(rest: string[]): Promise { // retains the normal chat-scope dispatch semantics. On a mismatch the // frozen canonical target remains authoritative. dispatch: prepared?.outputMismatch - ? (body, type, uuid, suppressHook) => { + ? async (body, type, uuid, suppressHook) => { revalidateVcMeetingManagedSend(); return sendMessage( appId, @@ -6980,11 +7259,18 @@ async function cmdSend(rest: string[]): Promise { // --no-mention 显式不 @ 任何人 → 连 footer 的"发送给/cc"寻址 也清空， // 否则 footer 仍会 @ 人，与 --no-mention 语义和"未@任何人"输出自相矛盾 // （Codex review P2）。--top-level 同样无特定收件人。 + const frozenFooterAddressingSource = explicitVcMeetingImOrigin?.replyTargetSenderOpenId + ? { ...s, lastCallerOpenId: explicitVcMeetingImOrigin.replyTargetSenderOpenId } + : frozenTurnDispatch?.replyTargetSenderOpenId + ? { + ...s, + lastCallerOpenId: frozenTurnDispatch.replyTargetSenderOpenId, + lastCallerIsBot: frozenTurnDispatch.replyTargetSenderIsBot, + } + : s; const footerAddressing = (sendTopLevel || noMention) ? { sendTo: undefined as string | undefined, cc: [] as string[] } - : buildFooterAddressing(explicitVcMeetingImOrigin?.replyTargetSenderOpenId - ? { ...s, lastCallerOpenId: explicitVcMeetingImOrigin.replyTargetSenderOpenId } - : s, { + : buildFooterAddressing(frozenFooterAddressingSource, { isOncall: !!oncallEntry, isSubstitute: turnReplyTarget?.turnId === currentTurnId && turnReplyTarget?.substitute === true, hasExplicitBotMention: explicitKnownBotMention, @@ -9748,7 +10034,7 @@ switch (command) { case 'ls': await cmdList(); break; case 'delete': case 'del': - case 'rm': cmdDelete(); break; + case 'rm': await cmdDelete(); break; case 'resume': await cmdResume(); break; case 'suspend': await cmdSuspend(); break; case 'slash': await cmdSlash(); break; diff --git a/src/cli/session-list-liveness.ts b/src/cli/session-list-liveness.ts index c212b365d..810cadd63 100644 --- a/src/cli/session-list-liveness.ts +++ b/src/cli/session-list-liveness.ts @@ -3,6 +3,7 @@ export interface SessionListMarkers { cliId?: unknown; lastCliInput?: unknown; adoptedFrom?: unknown; + codexAppDispatchLedger?: readonly unknown[]; } export type SessionListDisposition = 'keep' | 'prune_real' | 'prune_scratch'; @@ -22,6 +23,7 @@ export function sessionListDisposition( ): SessionListDisposition { if (runtime.hasPid || runtime.hasBackingSession) return 'keep'; if (session.suspendedColdResume === true) return 'keep'; + if ((session.codexAppDispatchLedger?.length ?? 0) > 0) return 'keep'; return session.cliId || session.lastCliInput || session.adoptedFrom ? 'prune_real' : 'prune_scratch'; diff --git a/src/codex-app-runner.ts b/src/codex-app-runner.ts index b801c87e8..eeb7d811f 100644 --- a/src/codex-app-runner.ts +++ b/src/codex-app-runner.ts @@ -1,6 +1,8 @@ #!/usr/bin/env node import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process'; import { Buffer } from 'node:buffer'; +import { createConnection, type Socket } from 'node:net'; +import type { KeyObject } from 'node:crypto'; import type { CodexAppTurnInput } from './types.js'; import { buildCodexAppTurnStartParams, @@ -10,6 +12,21 @@ import { type CodexVersion, } from './adapters/cli/codex-app-turn.js'; import { RunnerControlWriter } from './adapters/cli/runner-control-channel.js'; +import { + CODEX_APP_CONTROL_BOOTSTRAP_ENV, + CODEX_APP_CONTROL_FINAL_CHUNK_BYTES, + CODEX_APP_CONTROL_FINAL_MAX_BYTES, + CodexAppControlEndpointTracker, + CodexAppControlLineDecoder, + CodexAppControlRunnerHandshake, + armCodexAppControlHandshakeTimeout, + armCodexAppControlStartupTimeout, + consumeCodexAppControlBootstrap, + encodeCodexAppControlAuth, + encodeCodexAppSignedControlMarker, + parseCodexAppControlWireRecord, + takeCodexAppControlLocatorEndpoint, +} from './utils/codex-app-control.js'; type JsonObject = Record; @@ -17,6 +34,10 @@ interface Args { sessionId: string; codexBin: string; cwd: string; + controlGeneration: string; + controlPrivateKey: KeyObject; + controlSocketPath?: string; + controlLocatorPath?: string; threadId?: string; botName?: string; botOpenId?: string; @@ -27,6 +48,7 @@ interface PendingRequest { resolve: (value: any) => void; reject: (error: Error) => void; method: string; + timer?: NodeJS.Timeout; } interface ActiveTurn { @@ -34,8 +56,21 @@ interface ActiveTurn { * notifications from the server; botmux routing uses the stable client * message id carried alongside the queued input. */ nativeTurnId?: string; + /** Immutable Botmux/Lark identity sent through app-server. Mismatched native + * completions may be adopted only when their full items contain this exact + * client id. */ + clientUserMessageId?: string; + epoch: number; + reconciliation?: Promise; + identityConflictReported: boolean; + completed: boolean; + requestKind: 'start' | 'steer'; + requestAccepted: boolean; + pendingCompletions: JsonObject[]; + pendingNotifications: JsonObject[]; serverStarted: boolean; startedAtMs: number; + lastActivityMarkerAtMs: number; finalText: string; allAgentText: string; itemText: Map; @@ -49,16 +84,46 @@ interface QueuedInput { } const output = new RunnerControlWriter(); +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; +const RECONCILIATION_TIMEOUT_MS = 5_000; +const RECONCILIATION_PAGE_LIMIT = 3; +const RECONCILIATION_PAGE_SIZE = 50; + +class AppServerRpcError extends Error { + constructor( + readonly method: string, + readonly code: number | undefined, + readonly data: unknown, + message: string, + ) { + super(`${method}: ${message}`); + this.name = 'AppServerRpcError'; + } +} + +class AppServerRequestTimeoutError extends Error { + constructor(readonly method: string, readonly timeoutMs: number) { + super(`${method}: timed out after ${timeoutMs}ms; request acceptance is unknown`); + this.name = 'AppServerRequestTimeoutError'; + } +} function asError(value: unknown): Error { return value instanceof Error ? value : new Error(String(value)); } function parseArgs(argv: string[]): Args { + const controlBootstrapPath = process.env[CODEX_APP_CONTROL_BOOTSTRAP_ENV]; + // app-server and every model-launched tool inherit process.env. Remove even + // the non-secret bootstrap path before either can start; private key material + // was never present in env/argv/layout. + delete process.env[CODEX_APP_CONTROL_BOOTSTRAP_ENV]; const out: Args = { sessionId: '', codexBin: 'codex', cwd: process.cwd(), + controlGeneration: '', + controlPrivateKey: undefined as unknown as KeyObject, }; for (let i = 0; i < argv.length; i++) { const key = argv[i]; @@ -72,13 +137,15 @@ function parseArgs(argv: string[]): Args { else if (key === '--locale' && val !== undefined) { out.locale = val; i++; } } if (!out.sessionId) throw new Error('--session-id is required'); + if (!controlBootstrapPath) throw new Error(`${CODEX_APP_CONTROL_BOOTSTRAP_ENV} is required`); + const control = consumeCodexAppControlBootstrap(controlBootstrapPath, out.sessionId); + out.controlGeneration = control.generation; + out.controlPrivateKey = control.privateKey; + out.controlSocketPath = control.socketPath; + out.controlLocatorPath = control.locatorPath; return out; } -function emitMarker(kind: string, payload: unknown): void { - output.marker(kind, payload); -} - function writeLine(text = ''): void { output.line(text); } @@ -158,22 +225,40 @@ class AppServerClient { this.requestHandlers.push(handler); } - async initialize(): Promise { + async initialize(timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS): Promise { await this.request('initialize', { clientInfo: { name: 'botmux-codex-app', version: '0.0.0' }, capabilities: { experimentalApi: true }, - }); + }, { timeoutMs }); this.notify('initialized'); } - request(method: string, params: unknown): Promise { + request( + method: string, + params: unknown, + options: { timeoutMs?: number } = {}, + ): Promise { const id = this.nextId++; return new Promise((resolve, reject) => { - this.pending.set(id, { resolve, reject, method }); + const timeoutMs = options.timeoutMs; + const pending: PendingRequest = { resolve, reject, method }; + if (timeoutMs !== undefined) { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + reject(new AppServerRequestTimeoutError(method, Math.max(0, timeoutMs))); + return; + } + pending.timer = setTimeout(() => { + if (!this.pending.delete(id)) return; + reject(new AppServerRequestTimeoutError(method, timeoutMs)); + }, timeoutMs); + pending.timer.unref?.(); + } + this.pending.set(id, pending); try { this.write({ jsonrpc: '2.0', id, method, params }); } catch (err) { this.pending.delete(id); + if (pending.timer) clearTimeout(pending.timer); reject(asError(err)); } }); @@ -201,7 +286,10 @@ class AppServerClient { private failAll(err: Error): void { this.fatalError = this.fatalError ?? err; const fatal = this.fatalError; - for (const pending of this.pending.values()) pending.reject(fatal); + for (const pending of this.pending.values()) { + if (pending.timer) clearTimeout(pending.timer); + pending.reject(fatal); + } this.pending.clear(); } @@ -228,7 +316,15 @@ class AppServerClient { const pending = this.pending.get(msg.id); if (!pending) return; this.pending.delete(msg.id); - if (msg.error) pending.reject(new Error(`${pending.method}: ${JSON.stringify(msg.error)}`)); + if (pending.timer) clearTimeout(pending.timer); + if (msg.error) { + pending.reject(new AppServerRpcError( + pending.method, + typeof msg.error.code === 'number' ? msg.error.code : undefined, + msg.error.data, + typeof msg.error.message === 'string' ? msg.error.message : JSON.stringify(msg.error), + )); + } else pending.resolve(msg.result); return; } @@ -255,18 +351,217 @@ try { process.exit(2); } -const client = new AppServerClient(args.codexBin, args.cwd); +let controlSeq = 0; +let controlAckedSeq = 0; +let controlSentThrough = 0; +const controlQueue: Array<{ seq: number; kind: string; payload: JsonObject }> = []; +let controlFatal = false; +let controlSocket: Socket | undefined; +let controlChallenge: string | undefined; +let controlAccepted = false; +let controlAcceptanceCount = 0; +let controlReconnectTimer: NodeJS.Timeout | undefined; +let resolveControlReady!: () => void; +const controlReady = new Promise(resolve => { resolveControlReady = resolve; }); +const CONTROL_QUEUE_MAX_RECORDS = 2_048; +const controlEndpoints = new CodexAppControlEndpointTracker(); + +function scheduleControlReconnect(): void { + if (controlFatal || controlReconnectTimer) return; + controlReconnectTimer = setTimeout(() => { + controlReconnectTimer = undefined; + connectControlSocket(); + }, 250); +} + +function flushControlQueue(): void { + const socket = controlSocket; + const challenge = controlChallenge; + if (controlFatal || !socket || socket.destroyed || !controlAccepted || !challenge) return; + try { + for (const marker of controlQueue) { + if (marker.seq <= controlSentThrough) continue; + socket.write(`${encodeCodexAppSignedControlMarker( + args.controlPrivateKey, + args.sessionId, + args.controlGeneration, + challenge, + marker.seq, + marker.kind, + marker.payload, + )}\n`); + controlSentThrough = marker.seq; + } + } catch (err: any) { + controlFatal = true; + console.error(`Codex App control channel failed closed: ${err?.message ?? err}`); + process.exit(2); + } +} + +function nextControlEndpoint(): { endpoint: string; epoch?: string } | undefined { + if (args.controlSocketPath) return { endpoint: args.controlSocketPath }; + if (!args.controlLocatorPath) return undefined; + return takeCodexAppControlLocatorEndpoint({ + locatorPath: args.controlLocatorPath, + sessionId: args.sessionId, + tracker: controlEndpoints, + }); +} + +function connectControlSocket(): void { + if (controlFatal || (controlSocket && !controlSocket.destroyed)) return; + const target = nextControlEndpoint(); + if (!target) { + scheduleControlReconnect(); + return; + } + // A never-accepted locator endpoint may retry with the existing 250ms + // reconnect backoff; the protected 256-bit locator epoch is still required + // before acceptance. Once accepted, its endpoint is permanently burned and + // only a newly published locator can be used. + const socket = createConnection(target.endpoint); + const handshakeTimer = armCodexAppControlHandshakeTimeout(() => { + socket.destroy(new Error('Codex App control endpoint handshake timed out')); + }); + handshakeTimer.unref?.(); + const decoder = new CodexAppControlLineDecoder(); + const handshake = new CodexAppControlRunnerHandshake( + args.sessionId, + args.controlGeneration, + target.epoch, + ); + controlSocket = socket; + controlChallenge = undefined; + controlAccepted = false; + controlSentThrough = controlAckedSeq; + socket.setNoDelay(true); + socket.on('data', chunk => { + const decoded = decoder.push(chunk); + if (decoded.droppedMalformed) { + socket.destroy(new Error('oversized Codex App control response')); + return; + } + for (const line of decoded.lines) { + if (controlSocket !== socket) { + socket.destroy(new Error('unexpected Codex App control response')); + return; + } + const action = handshake.handle(parseCodexAppControlWireRecord(line), controlSentThrough); + if (action.type === 'authenticate') { + controlChallenge = action.challenge; + socket.write(`${encodeCodexAppControlAuth( + args.controlPrivateKey, + args.sessionId, + args.controlGeneration, + action.challenge, + )}\n`); + } else if (action.type === 'accepted') { + // `accepted` is intentionally unsigned. Its authority is the protected + // locator's independent epoch plus the already-bound random endpoint. + // Ed25519 still authenticates every runner marker to the worker. + controlAccepted = true; + clearTimeout(handshakeTimer); + if (args.controlLocatorPath) controlEndpoints.noteAccepted(target.endpoint); + controlAcceptanceCount++; + resolveControlReady(); + // The first acceptance happens before app-server initialization; it is + // not a ready boundary. Re-authentication can publish the live state + // only after initialization has completed. + if (controlAcceptanceCount > 1 && runnerReady) emitRunnerState(); + flushControlQueue(); + } else if (action.type === 'ack' && controlAccepted) { + if (action.seq > controlAckedSeq) controlAckedSeq = action.seq; + while (controlQueue[0] && controlQueue[0].seq <= controlAckedSeq) controlQueue.shift(); + } else { + socket.destroy(new Error('out-of-order Codex App control response')); + return; + } + } + }); + socket.on('error', () => { /* close schedules a retry */ }); + socket.on('close', () => { + clearTimeout(handshakeTimer); + if (controlSocket === socket) { + controlSocket = undefined; + controlChallenge = undefined; + controlAccepted = false; + } + scheduleControlReconnect(); + }); +} + +function emitMarker(kind: string, payload: JsonObject): void { + if (controlQueue.length >= CONTROL_QUEUE_MAX_RECORDS) { + controlFatal = true; + console.error('Codex App control queue exceeded its fail-closed bound'); + process.exit(2); + return; + } + controlQueue.push({ seq: ++controlSeq, kind, payload }); + flushControlQueue(); +} + +function emitFinalMarker(payload: JsonObject): void { + const original = Buffer.from(String(payload.content ?? ''), 'utf8'); + const truncated = original.length > CODEX_APP_CONTROL_FINAL_MAX_BYTES; + const content = truncated + ? Buffer.concat([ + original.subarray(0, CODEX_APP_CONTROL_FINAL_MAX_BYTES - 64), + Buffer.from('\n\n[botmux: final output truncated at control limit]', 'utf8'), + ]) + : original; + const id = `${String(payload.turnId ?? 'turn')}:${String(payload.completedAtMs ?? Date.now())}`; + const total = Math.ceil(content.length / CODEX_APP_CONTROL_FINAL_CHUNK_BYTES); + const { content: _content, ...metadata } = payload; + emitMarker('final-start', { id, total, truncated, ...metadata }); + for (let index = 0; index < total; index++) { + const start = index * CODEX_APP_CONTROL_FINAL_CHUNK_BYTES; + emitMarker('final-chunk', { + id, + index, + data: content.subarray(start, start + CODEX_APP_CONTROL_FINAL_CHUNK_BYTES).toString('base64'), + }); + } + emitMarker('final-end', { id, total }); +} + +connectControlSocket(); + +let client!: AppServerClient; let threadId = args.threadId; let threadReady = false; let activeTurn: ActiveTurn | null = null; +let activeTurnEpoch = 0; +/** App-server may start a Goal continuation without a Botmux input. Keep that + * native lifecycle separate from `activeTurn`; otherwise the next Lark input + * is incorrectly sent with turn/start and its completion can be discarded as + * belonging to an "unexpected" native turn. */ +let nativeActiveTurnId: string | undefined; const queue: QueuedInput[] = []; let inputBuffer = ''; let processing = false; +let runnerReady = false; let cleanInputUnsupported = false; let codexVersionChecked = false; let codexVersion: CodexVersion | undefined; let cleanVersionWarningShown = false; +function emitRunnerState( + busy = processing || queue.length > 0 || nativeActiveTurnId !== undefined, + tracksTurn = activeTurn !== null, +): void { + emitMarker('state', { + busy, + atMs: Date.now(), + // Input is accepted only after the runner has initialized and emitted this + // signed state. The worker uses this field as a runtime type-ahead gate; + // authentication alone never releases a prompt. + acceptingInput: runnerReady, + ...(busy && !tracksTurn ? { tracksTurn: false } : {}), + }); +} + function detectedCodexVersion(): CodexVersion | undefined { if (codexVersionChecked) return codexVersion; codexVersionChecked = true; @@ -284,11 +579,20 @@ function detectedCodexVersion(): CodexVersion | undefined { return codexVersion; } -function makeTurn(): ActiveTurn { +function makeTurn(clientUserMessageId: string | undefined, requestKind: 'start' | 'steer'): ActiveTurn { let resolveDone!: () => void; const done = new Promise(resolve => { resolveDone = resolve; }); return { + ...(clientUserMessageId ? { clientUserMessageId } : {}), + epoch: ++activeTurnEpoch, + identityConflictReported: false, + completed: false, + requestKind, + requestAccepted: false, + pendingCompletions: [], + pendingNotifications: [], startedAtMs: Date.now(), + lastActivityMarkerAtMs: 0, serverStarted: false, finalText: '', allAgentText: '', @@ -298,6 +602,25 @@ function makeTurn(): ActiveTurn { }; } +const TURN_ACTIVITY_MARKER_MIN_INTERVAL_MS = 5_000; + +/** + * Expose app-server lifecycle activity to the parent worker without polluting + * the visible terminal. Progress markers are throttled because token-delta + * notifications can arrive many times per second; submitted/completed edges + * are always emitted. + */ +function emitTurnActivity(turn: ActiveTurn, phase: 'submitted' | 'progress' | 'completed', force = false): void { + const atMs = Date.now(); + if (!force && atMs - turn.lastActivityMarkerAtMs < TURN_ACTIVITY_MARKER_MIN_INTERVAL_MS) return; + turn.lastActivityMarkerAtMs = atMs; + emitMarker('activity', { + phase, + atMs, + ...(turn.nativeTurnId ? { turnId: turn.nativeTurnId } : {}), + }); +} + function handleServerRequest(msg: JsonObject): boolean { const method = msg.method; if (method === 'item/commandExecution/requestApproval') { @@ -331,16 +654,236 @@ function handleServerRequest(msg: JsonObject): boolean { return false; } -function handleNotification(msg: JsonObject): void { +function exactClientItemIndexes(turn: JsonObject, clientUserMessageId: string): number[] { + if (turn?.itemsView !== 'full' || !Array.isArray(turn?.items)) return []; + const indexes: number[] = []; + for (let index = 0; index < turn.items.length; index++) { + const item = turn.items[index]; + if (item?.type === 'userMessage' && item.clientId === clientUserMessageId) indexes.push(index); + } + return indexes; +} + +function isTerminalNativeTurn(turn: JsonObject): boolean { + return turn?.status === undefined + || turn.status === 'completed' + || turn.status === 'failed' + || turn.status === 'interrupted'; +} + +/** Rebuild only from content causally after the exact user item. Never reuse + * streamed text from a different native turn during identity reconciliation. */ +function rebuildReconciledFinal(turn: JsonObject, userItemIndex: number): string { + const following = Array.isArray(turn.items) ? turn.items.slice(userItemIndex + 1) : []; + const finalAnswers = following.filter( + (item: JsonObject) => item?.type === 'agentMessage' && item.phase === 'final_answer', + ); + if (finalAnswers.length > 0) return String(finalAnswers.at(-1)?.text ?? ''); + if (turn?.error?.message) return `Codex App turn failed: ${String(turn.error.message)}`; + return ''; +} + +function completeActiveTurnFromNative(turn: ActiveTurn, nativeTurn: JsonObject, exactIndex?: number): void { + if (activeTurn !== turn || turn.completed || !isTerminalNativeTurn(nativeTurn)) return; + if (exactIndex !== undefined) { + turn.finalText = rebuildReconciledFinal(nativeTurn, exactIndex); + turn.allAgentText = ''; + } else if (nativeTurn?.error?.message && !turn.finalText) { + turn.finalText = `Codex App turn failed: ${String(nativeTurn.error.message)}`; + } + if (typeof nativeTurn?.id === 'string') { + turn.nativeTurnId = nativeTurn.id; + if (nativeActiveTurnId === nativeTurn.id) nativeActiveTurnId = undefined; + } + turn.completed = true; + emitTurnActivity(turn, 'completed', true); + turn.resolveDone(); +} + +function reportIdentityConflict(turn: ActiveTurn, observedNativeTurnId?: string, reason = 'no exact client id match'): void { + if (activeTurn !== turn || turn.identityConflictReported) return; + turn.identityConflictReported = true; + const stableTurnId = turn.clientUserMessageId; + const message = `Codex App native turn identity conflict (${reason}); refusing to attribute a completion without an exact clientUserMessageId match`; + writeLine(`[codex-app] ${message}`); + emitMarker('diagnostic', { + code: 'native_turn_identity_conflict', + message, + ...(stableTurnId ? { turnId: stableTurnId } : {}), + ...(turn.nativeTurnId ? { expectedNativeTurnId: turn.nativeTurnId } : {}), + ...(observedNativeTurnId ? { observedNativeTurnId } : {}), + atMs: Date.now(), + }); + // Attribution failed closed, but the logical Botmux turn must still settle. + // Leaving turn.done unresolved permanently poisons the runner FIFO and keeps + // the session "busy" forever. Emit one explicit error final under the stable + // client turn id; never reuse untrusted native text. + turn.finalText = `Codex App turn failed: ${message}`; + turn.allAgentText = ''; + if (turn.nativeTurnId && nativeActiveTurnId === turn.nativeTurnId) { + nativeActiveTurnId = undefined; + } + turn.completed = true; + emitTurnActivity(turn, 'completed', true); + turn.resolveDone(); +} + +async function reconcileCompletedTurn(turn: ActiveTurn, observedNativeTurnId?: string): Promise { + if (turn.reconciliation || turn.completed) return turn.reconciliation; + const clientUserMessageId = turn.clientUserMessageId; + if (!clientUserMessageId || !threadId) { + reportIdentityConflict(turn, observedNativeTurnId, 'legacy input has no clientUserMessageId'); + return; + } + const epoch = turn.epoch; + const deadlineAtMs = Date.now() + RECONCILIATION_TIMEOUT_MS; + turn.reconciliation = (async () => { + const matches: Array<{ turn: JsonObject; itemIndex: number }> = []; + let cursor: string | null | undefined; + for (let page = 0; page < RECONCILIATION_PAGE_LIMIT; page++) { + const remaining = deadlineAtMs - Date.now(); + if (remaining <= 0) break; + const result = await client.request('thread/turns/list', { + threadId, + ...(cursor ? { cursor } : {}), + limit: RECONCILIATION_PAGE_SIZE, + sortDirection: 'desc', + itemsView: 'full', + }, { timeoutMs: remaining }); + for (const candidate of Array.isArray(result?.data) ? result.data : []) { + if (!isTerminalNativeTurn(candidate)) continue; + const indexes = exactClientItemIndexes(candidate, clientUserMessageId); + if (indexes.length === 1) matches.push({ turn: candidate, itemIndex: indexes[0] }); + else if (indexes.length > 1) { + reportIdentityConflict(turn, observedNativeTurnId, 'client id appears more than once in one turn'); + return; + } + } + cursor = typeof result?.nextCursor === 'string' ? result.nextCursor : null; + if (!cursor) break; + } + if (activeTurn !== turn || turn.epoch !== epoch || turn.completed) return; + if (matches.length === 1) { + completeActiveTurnFromNative(turn, matches[0].turn, matches[0].itemIndex); + return; + } + reportIdentityConflict( + turn, + observedNativeTurnId, + matches.length === 0 ? 'bounded history lookup found no match' : 'bounded history lookup found multiple matches', + ); + })().catch(err => { + if (activeTurn === turn && !turn.completed) { + reportIdentityConflict(turn, observedNativeTurnId, `bounded history lookup failed: ${asError(err).message}`); + } + }); + await turn.reconciliation; +} + +function handleNotification(msg: JsonObject, replayedAfterResponse = false): void { const params = msg.params ?? {}; - if (!activeTurn || params.threadId !== threadId) return; - if (activeTurn.nativeTurnId && params.turnId && params.turnId !== activeTurn.nativeTurnId) return; + if (params.threadId !== threadId) return; + const notificationTurnId = params.turnId ?? params.turn?.id; if (msg.method === 'turn/started') { - activeTurn.serverStarted = true; - activeTurn.nativeTurnId = params.turn?.id ?? params.turnId ?? activeTurn.nativeTurnId; + const startedId = typeof notificationTurnId === 'string' ? notificationTurnId : undefined; + if (startedId && (!replayedAfterResponse + || nativeActiveTurnId === undefined + || nativeActiveTurnId === startedId)) nativeActiveTurnId = startedId; + const turn = activeTurn; + if (turn && startedId) { + const exact = turn.clientUserMessageId + ? exactClientItemIndexes(params.turn, turn.clientUserMessageId) + : []; + if (!turn.nativeTurnId && exact.length === 1) turn.nativeTurnId = startedId; + if (turn.nativeTurnId === startedId) { + turn.serverStarted = true; + emitTurnActivity(turn, 'progress', true); + return; + } + // app-server is allowed to publish turn/started before replying to + // turn/start. Without the response we do not yet know whether this is + // our native turn, but dropping it also loses the first (and sometimes + // only) progress edge. Replay it after the RPC binds nativeTurnId. + if (!turn.requestAccepted && turn.requestKind === 'start') { + const alreadyBufferedStart = turn.pendingNotifications.some( + notification => notification.method === 'turn/started' + && (notification.params?.turnId ?? notification.params?.turn?.id) === startedId, + ); + if (!alreadyBufferedStart) turn.pendingNotifications.push(msg); + return; + } + } + // A Goal continuation is native work, not a Botmux turn. Keep the worker + // busy while explicitly advertising that the initialized runner can accept + // a Lark follow-up through turn/steer. + if (runnerReady) emitRunnerState(true, false); + return; + } + + if (msg.method === 'turn/completed') { + const nativeTurn = params.turn ?? {}; + const completedId = typeof notificationTurnId === 'string' ? notificationTurnId : undefined; + if (completedId && nativeActiveTurnId === completedId) nativeActiveTurnId = undefined; + const turn = activeTurn; + if (!turn) { + if (runnerReady) emitRunnerState(); + return; + } + if (turn.nativeTurnId && completedId === turn.nativeTurnId) { + const exact = turn.clientUserMessageId + ? exactClientItemIndexes(nativeTurn, turn.clientUserMessageId) + : []; + if (exact.length === 1) { + completeActiveTurnFromNative(turn, nativeTurn, exact[0]); + return; + } + // A captured Goal turn can finish while turn/steer is still in flight. + // Until the RPC succeeds, native-id equality proves only that the old + // autonomous work completed; it does not prove this Lark input landed. + if (!turn.requestAccepted) { + turn.pendingCompletions.push(nativeTurn); + return; + } + if (turn.requestKind === 'steer') { + void reconcileCompletedTurn(turn, completedId); + return; + } + completeActiveTurnFromNative(turn, nativeTurn); + return; + } + const exact = turn.clientUserMessageId + ? exactClientItemIndexes(nativeTurn, turn.clientUserMessageId) + : []; + if (exact.length === 1) { + completeActiveTurnFromNative(turn, nativeTurn, exact[0]); + return; + } + if (!turn.requestAccepted) { + turn.pendingCompletions.push(nativeTurn); + return; + } + void reconcileCompletedTurn(turn, completedId); + return; + } + + const turn = activeTurn; + if (!turn) return; + if (!turn.requestAccepted) { + // turn/start notifications may beat their response. Buffer only that + // request's candidate native events and replay after the response chooses + // the authoritative id. For turn/steer, pre-response events can be old + // autonomous output and are deliberately never promoted into the final. + if (turn.requestKind === 'start' && typeof notificationTurnId === 'string') { + turn.pendingNotifications.push(msg); + } return; } + if (turn.nativeTurnId && notificationTurnId && notificationTurnId !== turn.nativeTurnId) return; + + // Every notification for the active app-server turn is evidence of forward + // progress, including reasoning/status events that do not render text. + emitTurnActivity(turn, 'progress'); if (msg.method === 'item/started') { const item = params.item; @@ -355,8 +898,8 @@ function handleNotification(msg: JsonObject): void { if (msg.method === 'item/agentMessage/delta') { const delta = String(params.delta ?? ''); const itemId = String(params.itemId ?? ''); - activeTurn.itemText.set(itemId, (activeTurn.itemText.get(itemId) ?? '') + delta); - activeTurn.allAgentText += delta; + turn.itemText.set(itemId, (turn.itemText.get(itemId) ?? '') + delta); + turn.allAgentText += delta; output.display(delta); return; } @@ -369,25 +912,35 @@ function handleNotification(msg: JsonObject): void { if (msg.method === 'item/completed') { const item = params.item; if (item?.type === 'agentMessage') { - if (item.phase === 'final_answer') activeTurn.finalText = String(item.text ?? ''); - else if (!activeTurn.itemText.has(item.id) && item.text) { - activeTurn.allAgentText += String(item.text); + if (item.phase === 'final_answer') turn.finalText = String(item.text ?? ''); + else if (!turn.itemText.has(item.id) && item.text) { + turn.allAgentText += String(item.text); } } return; } +} - if (msg.method === 'turn/completed') { - const turn = params.turn; - if (turn?.id && activeTurn.nativeTurnId && turn.id !== activeTurn.nativeTurnId) return; - if (turn?.error?.message && !activeTurn.finalText) { - activeTurn.finalText = `Codex App turn failed: ${turn.error.message}`; - } - activeTurn.resolveDone(); - } +function startupRequestTimeout(deadlineAtMs: number | undefined, method: string): number { + if (deadlineAtMs === undefined) return DEFAULT_REQUEST_TIMEOUT_MS; + const remaining = deadlineAtMs - Date.now(); + if (remaining <= 0) throw new AppServerRequestTimeoutError(method, 0); + return remaining; +} + +function isExplicitMissingThread(error: unknown): boolean { + if (!(error instanceof AppServerRpcError)) return false; + return /(thread|rollout|conversation).*(not found|does not exist|missing|unknown)|not found.*(thread|rollout|conversation)/i + .test(error.message); } -async function ensureThread(): Promise { +function isExplicitExpectedTurnInactive(error: unknown): boolean { + return error instanceof AppServerRpcError + && /(expected|active).*(turn).*(not active|no longer active|mismatch|does not match)|(turn).*(not active|no longer active).*(expected)/i + .test(error.message); +} + +async function ensureThread(startupDeadlineAtMs?: number): Promise { if (threadReady && threadId) return threadId; if (threadId) { @@ -403,13 +956,17 @@ async function ensureThread(): Promise { // Keep Codex App's rich history in sync with turns created by this // external runner so the desktop UI can render follow-up messages. persistExtendedHistory: true, - }); + }, { timeoutMs: startupRequestTimeout(startupDeadlineAtMs, 'thread/resume') }); const resumedThreadId = String(resumed.thread.id); threadId = resumedThreadId; threadReady = true; emitMarker('thread', { threadId: resumedThreadId }); return resumedThreadId; } catch (err: any) { + // A transport error or timeout is an ambiguous acceptance boundary. It + // must never fork history by silently creating a fresh thread. Only an + // explicit app-server "missing thread" rejection permits fallback. + if (!isExplicitMissingThread(err)) throw err; writeLine(`[codex-app] resume failed, starting a fresh thread: ${err?.message ?? err}`); threadId = undefined; threadReady = false; @@ -428,24 +985,31 @@ async function ensureThread(): Promise { // Keep Codex App's rich history in sync with turns created by this // external runner so the desktop UI can render follow-up messages. persistExtendedHistory: true, - }); + }, { timeoutMs: startupRequestTimeout(startupDeadlineAtMs, 'thread/start') }); const startedThreadId = String(started.thread.id); threadId = startedThreadId; threadReady = true; emitMarker('thread', { threadId: startedThreadId }); - try { - await client.request('thread/name/set', { + void client.request('thread/name/set', { threadId: startedThreadId, name: `botmux ${args.sessionId.slice(0, 8)}`, - }); - } catch { /* naming is cosmetic */ } + }, { timeoutMs: 2_000 }).catch(() => { /* naming is cosmetic */ }); return startedThreadId; } async function runTurn(message: QueuedInput): Promise { const tid = await ensureThread(); - const turn = makeTurn(); + const stableTurnId = message.codexAppInput?.clientUserMessageId; + let expectedSteerTurnId = nativeActiveTurnId; + const turn = makeTurn(stableTurnId, expectedSteerTurnId ? 'steer' : 'start'); + if (expectedSteerTurnId) { + turn.nativeTurnId = expectedSteerTurnId; + turn.serverStarted = true; + } activeTurn = turn; + // This edge proves the runner decoded and dequeued Botmux's control line, + // even if app-server stalls before acknowledging turn/start. + emitTurnActivity(turn, 'submitted', true); const version = message.codexAppInput ? detectedCodexVersion() : undefined; let built = buildCodexAppTurnStartParams({ threadId: tid, @@ -468,46 +1032,124 @@ async function runTurn(message: QueuedInput): Promise { writeLine(built.structured && message.codexAppInput ? message.codexAppInput.text : message.content); writeLine(); - let result; - try { - result = await client.request('turn/start', built.params); - } catch (err) { - if (!built.structured || turn.serverStarted || !isCleanInputCapabilityError(err)) throw err; - // The app-server explicitly rejected the experimental field before a turn - // started. Disable structured input for this runner lifetime and retry the - // preserved legacy prompt exactly once. - cleanInputUnsupported = true; - writeLine('[codex-app] clean input unsupported by app-server; retrying this turn with the legacy prompt'); - built = buildCodexAppTurnStartParams({ - threadId: tid, - cwd: args.cwd, - legacyContent: message.content, - codexAppInput: message.codexAppInput, - codexVersion: version, - structuredDisabled: true, + const requestBuiltTurn = (candidate: typeof built): Promise => { + if (!expectedSteerTurnId) return client.request('turn/start', candidate.params); + const { threadId, input, clientUserMessageId, additionalContext } = candidate.params; + return client.request('turn/steer', { + threadId, + input, + expectedTurnId: expectedSteerTurnId, + ...(clientUserMessageId ? { clientUserMessageId } : {}), + ...(additionalContext ? { additionalContext } : {}), }); - result = await client.request('turn/start', built.params); + }; + + let result: any; + let capabilityRetried = false; + let inactiveSteerFallback = false; + for (;;) { + try { + result = await requestBuiltTurn(built); + break; + } catch (err) { + if (expectedSteerTurnId + && !inactiveSteerFallback + && nativeActiveTurnId === undefined + && isExplicitExpectedTurnInactive(err)) { + // turn/steer was explicitly rejected before acceptance and the signed + // native completion already proved the captured Goal turn is gone. + // Starting the same client-id input is safe; timeout/transport/generic + // errors never enter this branch. + inactiveSteerFallback = true; + expectedSteerTurnId = undefined; + turn.requestKind = 'start'; + turn.nativeTurnId = undefined; + turn.serverStarted = false; + turn.pendingCompletions.length = 0; + turn.pendingNotifications.length = 0; + turn.finalText = ''; + turn.allAgentText = ''; + turn.itemText.clear(); + writeLine('[codex-app] captured native turn completed before steer acceptance; starting the same client-id input as a new turn'); + continue; + } + if (capabilityRetried + || !built.structured + || (!expectedSteerTurnId && turn.serverStarted) + || !isCleanInputCapabilityError(err)) throw err; + // The app-server explicitly rejected the experimental field before a turn + // started. Disable structured input for this runner lifetime and retry the + // preserved legacy prompt exactly once. + capabilityRetried = true; + cleanInputUnsupported = true; + writeLine('[codex-app] clean input unsupported by app-server; retrying this turn with the legacy prompt'); + built = buildCodexAppTurnStartParams({ + threadId: tid, + cwd: args.cwd, + legacyContent: message.content, + codexAppInput: message.codexAppInput, + codexVersion: version, + structuredDisabled: true, + }); + } + } + turn.nativeTurnId = result.turn?.id ?? result.turnId ?? turn.nativeTurnId; + turn.requestAccepted = true; + // A response may arrive after its turn completed or after a Goal + // continuation B already started. Never resurrect the completed lifecycle, + // and never let late response A overwrite the newer global lifecycle. If A + // is still pending and no newer turn exists, temporarily restoring A is safe + // because the buffered A completion below clears it immediately. + if (!turn.completed + && turn.nativeTurnId + && (nativeActiveTurnId === undefined || nativeActiveTurnId === turn.nativeTurnId)) { + nativeActiveTurnId = turn.nativeTurnId; + } + const pendingNotifications = turn.pendingNotifications.splice(0); + for (const notification of pendingNotifications) { + const notificationTurnId = notification.params?.turnId ?? notification.params?.turn?.id; + if (notificationTurnId === turn.nativeTurnId) handleNotification(notification, true); + } + const pendingCompletions = turn.pendingCompletions.splice(0); + if (pendingCompletions.length > 0 && !turn.completed) { + const exactMatches = stableTurnId + ? pendingCompletions.flatMap(completion => { + const indexes = exactClientItemIndexes(completion, stableTurnId); + return indexes.length === 1 ? [{ completion, itemIndex: indexes[0] }] : []; + }) + : []; + const nativeMatches = pendingCompletions.filter( + completion => completion?.id === turn.nativeTurnId, + ); + if (exactMatches.length === 1) { + completeActiveTurnFromNative(turn, exactMatches[0].completion, exactMatches[0].itemIndex); + } else if (exactMatches.length > 1 || nativeMatches.length > 1) { + reportIdentityConflict(turn, turn.nativeTurnId, 'multiple pre-response completions matched one request'); + } else if (turn.requestKind === 'steer' && nativeMatches.length === 1) { + void reconcileCompletedTurn(turn, turn.nativeTurnId); + } else if (turn.requestKind === 'start' && nativeMatches.length === 1) { + completeActiveTurnFromNative(turn, nativeMatches[0]); + } } - turn.nativeTurnId = result.turn?.id ?? turn.nativeTurnId; await turn.done; const finalText = (turn.finalText || turn.allAgentText).trim(); const completedAtMs = Date.now(); - if (finalText) { - // clientUserMessageId is the daemon-frozen botmux/Lark turn identity. The - // app-server generates a different id for the same logical turn; exposing - // that native id as `turnId` breaks daemon wait maps, VC suppression and - // reply routing. When no structured sidecar exists, omit turnId so the - // worker deliberately falls back to its current botmux turn attribution. - const stableTurnId = message.codexAppInput?.clientUserMessageId; - emitMarker('final', { - ...(stableTurnId ? { turnId: stableTurnId } : {}), - ...(turn.nativeTurnId ? { nativeTurnId: turn.nativeTurnId } : {}), - content: finalText, - startedAtMs: turn.startedAtMs, - completedAtMs, - }); - } + // Every dequeued runner input gets one complete final transaction, including + // an empty model answer. Without that zero-chunk boundary the worker cannot + // advance its attribution FIFO safely before the next queued turn finishes. + // clientUserMessageId is the daemon-frozen botmux/Lark turn identity. The + // app-server generates a different id for the same logical turn; exposing + // that native id as `turnId` breaks daemon wait maps, VC suppression and + // reply routing. When no structured sidecar exists, omit turnId so the + // worker resolves it from its own FIFO head. + emitFinalMarker({ + ...(stableTurnId ? { turnId: stableTurnId } : {}), + ...(turn.nativeTurnId ? { nativeTurnId: turn.nativeTurnId } : {}), + content: finalText, + startedAtMs: turn.startedAtMs, + completedAtMs, + }); writeLine(); activeTurn = null; } @@ -526,7 +1168,7 @@ async function drainQueue(): Promise { const stableTurnId = next.codexAppInput?.clientUserMessageId; const nativeTurnId = activeTurn?.nativeTurnId; writeLine(message); - emitMarker('final', { + emitFinalMarker({ ...(stableTurnId ? { turnId: stableTurnId } : {}), ...(nativeTurnId ? { nativeTurnId } : {}), content: message, @@ -535,7 +1177,16 @@ async function drainQueue(): Promise { }); activeTurn = null; } - prompt(); + // Do not publish a transient idle boundary between inputs already queued + // in the serial runner. Once the queue is truly empty, append signed + // busy:false only AFTER completed + every final fragment. The worker can + // therefore become ready even if the terminal prompt is lost, while its + // IPC order remains final_output before prompt_ready. + if (queue.length === 0) { + const nativeBusy = nativeActiveTurnId !== undefined; + emitRunnerState(nativeBusy, !nativeBusy); + if (!nativeBusy) prompt(); + } } } finally { processing = false; @@ -586,11 +1237,31 @@ function handleInput(data: Buffer): void { } async function main(): Promise { + const testTimeout = process.env.NODE_ENV === 'test' + ? Number(process.env.BOTMUX_TEST_CODEX_APP_STARTUP_TIMEOUT_MS) + : Number.NaN; + const startupTimeoutMs = Number.isFinite(testTimeout) && testTimeout > 0 + ? testTimeout + : 90_000; + const startupDeadlineAtMs = Date.now() + startupTimeoutMs; + const authTimeout = armCodexAppControlStartupTimeout(() => { + console.error('Codex App startup timed out before the first signed runner state'); + process.exit(2); + }, startupTimeoutMs); + await controlReady; + client = new AppServerClient(args.codexBin, args.cwd); client.onRequest(handleServerRequest); client.onNotification(handleNotification); - await client.initialize(); - await ensureThread(); + await client.initialize(startupRequestTimeout(startupDeadlineAtMs, 'initialize')); + await ensureThread(startupDeadlineAtMs); writeLine('Codex App connected.'); + runnerReady = true; + // Initial readiness is signed too; never rely on terminal rendering as the + // only path that releases the worker's first-prompt gate. + emitRunnerState(false); + // Authentication is deliberately insufficient. Keep the absolute startup + // timer armed through initialize + resume/start and the first signed state. + clearTimeout(authTimeout); if (process.stdin.isTTY) process.stdin.setRawMode(true); process.stdin.resume(); process.stdin.on('data', handleInput); @@ -598,16 +1269,24 @@ async function main(): Promise { } process.on('SIGTERM', () => { - client.close(); + if (controlReconnectTimer) clearTimeout(controlReconnectTimer); + controlSocket?.destroy(); + client?.close(); process.exit(0); }); process.on('SIGINT', () => { - client.close(); + if (controlReconnectTimer) clearTimeout(controlReconnectTimer); + controlSocket?.destroy(); + client?.close(); process.exit(130); }); main().catch(err => { + if (!runnerReady && err instanceof AppServerRequestTimeoutError) { + output.error('Codex App startup timed out before the first signed runner state\n'); + process.exit(2); + } output.error(`${err?.stack ?? err?.message ?? err}\n`); process.exit(1); }); diff --git a/src/core/command-handler.ts b/src/core/command-handler.ts index 4d8545875..789ed99fe 100644 --- a/src/core/command-handler.ts +++ b/src/core/command-handler.ts @@ -24,7 +24,7 @@ import { chatAppLink, normalizeBrand } from '../im/lark/lark-hosts.js'; import { claimPairing } from '../services/pairing-store.js'; import { logger } from '../utils/logger.js'; import { scheduleTimeZone } from '../utils/timezone.js'; -import { killWorker, suspendWorker, forkWorker, forkAdoptWorker, adoptSandboxBlocked, getCurrentCliVersion, postFreshStreamingCard, postPrivateSnapshotCard, resolvePrivateCardAudience, deliverEphemeralOrReply, deliverWritableTerminalCardTo } from './worker-pool.js'; +import { killWorker, suspendWorker, forkWorker, forkAdoptWorker, adoptSandboxBlocked, getCurrentCliVersion, postFreshStreamingCard, postPrivateSnapshotCard, resolvePrivateCardAudience, deliverEphemeralOrReply, deliverWritableTerminalCardTo, closeSession as closeWorkerPoolSession, withActiveSessionKeyLock } from './worker-pool.js'; import { expandHome, getSessionWorkingDir, @@ -84,14 +84,16 @@ import { readRoleProfileEntry, writeRoleProfileEntry, } from '../services/role-profile-store.js'; -import type { LarkMessage, DaemonToWorker } from '../types.js'; -import { sessionKey, sessionAnchorId, markRepoCardConsumed, claimCurrentRepoCard } from './types.js'; +import type { LarkMessage, DaemonToWorker, CodexAppTurnInput } from '../types.js'; +import { activeSessionKey, sessionKey, sessionAnchorId, markRepoCardConsumed } from './types.js'; import type { DaemonSession } from './types.js'; import { t, localeForBot, type Locale } from '../i18n/index.js'; import { runSkillsImCommand } from './skills/im-command.js'; import { fetchDaemonIpc } from './daemon-ipc-auth.js'; import { updateSessionTitle } from './session-title.js'; import { requestAgentSessionRename } from './session-rename.js'; +import { hasProtectedSessionMutationOwnership } from './session-mutation-guard.js'; +import { withBotTurnMutation } from './bot-turn-mutation-gate.js'; // ─── Exported constants ────────────────────────────────────────────────────── @@ -1254,21 +1256,33 @@ export async function handleCommand( switch (cmd) { case '/close': { if (ds) { - // Capture the closed-session card BEFORE killWorker/closeSession — - // it reads the live session's identity off `ds`. - const card = buildClosedSessionCard(ds, loc); - killWorker(ds); - sessionStore.closeSession(ds.session.sessionId); - activeSessions.delete(sessionKey(rootId, larkAppId!)); + const targetSessionId = ds.session.sessionId; + const closed = await withBotTurnMutation(ds.larkAppId, async () => { + // Re-resolve the exact session after all peer admissions drain. A + // relay may have moved it to another key while this command was + // admitted; closeSession removes its current identity, never the + // stale root key from this message. + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId, + ); + if (!current) return undefined; + const card = buildClosedSessionCard(current, localeForBot(current.larkAppId)); + await closeWorkerPoolSession(targetSessionId); + return { status: 'closed' as const, current, card }; + }); + if (!closed) { + await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); + break; + } // 「会话已关闭」卡片优先「仅自己可见」：普通群里走 ephemeral 只发给执行 // /close 的本人；话题群不支持 ephemeral(18053) 时回退为正常的群内可见回复 // ——与流式卡片上「关闭会话」按钮的送达方式保持一致。 await deliverEphemeralOrReply( - ds, + closed.current, message.senderId, - card, + closed.card, 'interactive', - () => sessionReply(rootId, card, 'interactive'), + () => sessionReply(rootId, closed.card, 'interactive'), ); logger.info(`[${logTag}] Session closed by /close command`); } else { @@ -1315,9 +1329,18 @@ export async function handleCommand( break; } const closedSessionId = ds.session.sessionId; - killWorker(ds); - sessionStore.closeSession(closedSessionId); - activeSessions.delete(sessionKey(rootId, larkAppId!)); + const detached = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === closedSessionId, + ); + if (!current || !current.adoptedFrom) return false; + await closeWorkerPoolSession(closedSessionId); + return true; + }); + if (!detached) { + await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); + break; + } await sessionReply(rootId, t('cmd.detach.success', undefined, loc)); logger.info(`[${logTag}] Detached (adopt) by ${cmd} command`); break; @@ -1325,15 +1348,41 @@ export async function handleCommand( case '/restart': { if (ds) { - if (ds.worker && !ds.worker.killed) { - ds.worker.send({ type: 'restart' } as DaemonToWorker); - const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); - await sessionReply(rootId, t('cmd.restart.in_progress', { cliName }, loc)); - } else { - killWorker(ds); - const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); - await sessionReply(rootId, t('cmd.restart.terminated', { cliName }, loc)); + const targetSessionId = ds.session.sessionId; + const restart = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current) return { status: 'gone' as const }; + if (hasProtectedSessionMutationOwnership(current)) { + return { status: 'pending' as const }; + } + const cliName = getCliDisplayName(getBot(current.larkAppId).config.cliId); + if (current.worker && !current.worker.killed) { + current.worker.send({ type: 'restart', reason: 'operator' } as DaemonToWorker); + return { status: 'restarting' as const, cliName }; + } + killWorker(current); + return { status: 'terminated' as const, cliName }; + }); + if (restart.status === 'pending') { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能重启；请等待本轮完成或关闭会话。', + ); + break; + } + if (restart.status === 'gone') { + await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); + break; } + await sessionReply( + rootId, + restart.status === 'restarting' + ? t('cmd.restart.in_progress', { cliName: restart.cliName }, loc) + : t('cmd.restart.terminated', { cliName: restart.cliName }, loc), + ); logger.info(`[${logTag}] Restart by /restart command`); } else { await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); @@ -1351,21 +1400,53 @@ export async function handleCommand( await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); break; } + // Cheap preflight avoids creating a requested directory when the + // current FIFO already makes the switch impossible. The mutation + // below repeats this check after draining peer admissions. + if (hasProtectedSessionMutationOwnership(ds)) { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能切换工作目录；请等待本轮完成或关闭会话。', + ); + break; + } const validation = validateWorkingDir(targetPath, loc, { autoCreate: true }); if (!validation.ok) { await sessionReply(rootId, validation.error); break; } const resolvedPath = validation.resolvedPath; - // Persistent backends can cold-resume without treating a cwd switch as - // a real session close. This also preserves a sandbox upper long enough - // for the next worker to relocate Claude's cwd-scoped transcript. - // Adopted sessions are observers of a user-owned CLI and keep the old - // detach/kill behavior; PTY falls back to killWorker, while its normal - // on-disk transcript remains available for the resume-sync preflight. - const suspended = !ds.adoptedFrom && suspendWorker(ds, 'working_dir_changed'); - if (!suspended) killWorker(ds); - repinSessionWorkingDir(ds, resolvedPath); + const targetSessionId = ds.session.sessionId; + const switched = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current) return 'gone' as const; + if (hasProtectedSessionMutationOwnership(current)) { + return 'pending' as const; + } + const suspended = !current.adoptedFrom + && suspendWorker(current, 'working_dir_changed'); + if (!suspended) killWorker(current); + repinSessionWorkingDir(current, resolvedPath); + // cwd 变了，riff 多仓 stamp（选择卡多选时写入）随之失效——保留会让下次 + // refork 仍按旧仓库组合推导、无视新目录。 + current.session.riffRepoDirs = undefined; + sessionStore.updateSession(current.session); + return 'switched' as const; + }); + if (switched === 'pending') { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能切换工作目录；请等待本轮完成或关闭会话。', + ); + break; + } + if (switched === 'gone') { + await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); + break; + } if (validation.created) { await sessionReply(rootId, t('cmd.cd.created_switched', { path: resolvedPath }, loc)); } else { @@ -1408,141 +1489,150 @@ export async function handleCommand( case '/repo': { const repoArg = message.content.replace(/^\/repo\s*/, '').trim(); + if (ds && !ds.pendingRepo + && hasProtectedSessionMutationOwnership(ds)) { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能切换仓库；请等待本轮完成或关闭会话。', + ); + break; + } // First-spawn fork: consume the buffered prompt/attachments and start the // CLI in whatever workingDir is currently set on the session. Shared by // `commitRepoSelection` (a repo was named) and the bare-`/repo` launch // (use the default workingDir) — both only run while `pendingRepo`. - const forkPendingCli = async (replyText: string, claimAlready = false): Promise => { - if (!claimAlready) { - if (ds!.pendingRepoCommitInFlight) { - await sessionReply(rootId, t('cmd.repo.worktree_in_progress', undefined, loc)); - return false; + const forkPendingCli = async ( + replyText: string, + selection?: { path: string; riffRepoDirs?: string[] }, + ) => { + const targetSessionId = ds!.session.sessionId; + const started = await withBotTurnMutation(ds!.larkAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current || current !== ds || !current.pendingRepo) return false; + if (selection) { + current.workingDir = selection.path; + current.session.workingDir = selection.path; + current.session.riffRepoDirs = selection.riffRepoDirs; + sessionStore.updateSession(current.session); } - ds!.pendingRepoCommitInFlight = true; - } - const selfBot = getBot(ds!.larkAppId); - const botCfg = selfBot.config; - const commitGenSessionId = ds!.session.sessionId; - let committed = false; - try { - // Keep pendingRepo=true while prompt context is awaited. Incoming - // messages stay in the same buffer and card selections see the - // shared in-flight claim instead of treating this as a repo switch. - const initiallyBuffered = - (ds!.pendingPrompt?.trim().length ?? 0) > 0 || - (ds!.pendingAttachments?.length ?? 0) > 0 || - (ds!.pendingFollowUps?.length ?? 0) > 0; - const availableBots = initiallyBuffered - ? await getAvailableBots(ds!.larkAppId, ds!.chatId) - : []; - const stillActive = activeSessions.get(sessionKey(rootId, larkAppId!)) === ds; - if (!stillActive || ds!.session.sessionId !== commitGenSessionId || - !ds!.pendingRepo || (ds!.worker && !ds!.worker.killed)) { - logger.warn(`[${logTag}] Session changed while preparing the pending CLI (${commitGenSessionId} → ${ds!.session.sessionId}, active=${stillActive}, pending=${!!ds!.pendingRepo}, worker=${!!ds!.worker})`); - return false; + const selfBot = getBot(current.larkAppId); + const botCfg = selfBot.config; + const pendingPrompt = current.pendingPrompt ?? ''; + const pendingRawInput = current.pendingRawInput; + const hasBufferedInput = pendingPrompt.trim().length > 0 + || current.pendingCodexAppText !== undefined + || (current.pendingAttachments?.length ?? 0) > 0 + || (current.pendingFollowUps?.length ?? 0) > 0; + let wrappedInput: { content: string; codexAppInput?: CodexAppTurnInput } | undefined; + if (hasBufferedInput) { + const { buildNewTopicCliInput: buildInput, ensureSessionWhiteboard, getAvailableBots } = await import('./session-manager.js'); + ensureSessionWhiteboard(current); + const availableBots = await getAvailableBots(current.larkAppId, current.chatId); + // Detached lifecycle work can still close/replace while the + // roster lookup awaits. Never fork the captured generation. + if (current.session.status !== 'active' + || [...activeSessions.values()].find(candidate => candidate.session.sessionId === targetSessionId) !== current + || !current.pendingRepo) return false; + wrappedInput = buildInput( + pendingPrompt, + current.session.sessionId, + current.session.cliId ?? botCfg.cliId, + current.session.cliPathOverride ?? botCfg.cliPathOverride, + current.pendingAttachments, + current.pendingMentions, + availableBots, + current.pendingFollowUps, + { name: selfBot.botName, openId: selfBot.botOpenId }, + loc, + current.pendingSender, + { + larkAppId, + chatId: current.chatId, + whiteboardId: current.session.whiteboardId, + substituteTrigger: current.pendingSubstituteTrigger, + codexAppText: current.pendingCodexAppText, + codexAppApplicationContext: current.pendingCodexAppApplicationContext, + codexAppMessageContext: current.pendingCodexAppMessageContext, + codexAppFollowUps: current.pendingCodexAppFollowUps, + codexAppFollowUpContexts: current.pendingCodexAppFollowUpContexts, + }, + ); } - - const pendingPrompt = ds!.pendingPrompt ?? ''; - const pendingRawInput = ds!.pendingRawInput; - const hasBufferedInput = - pendingPrompt.trim().length > 0 || - (ds!.pendingAttachments?.length ?? 0) > 0 || - (ds!.pendingFollowUps?.length ?? 0) > 0; - if (hasBufferedInput) ensureSessionWhiteboard(ds!); - const wrappedInput = hasBufferedInput - ? buildNewTopicCliInput( - pendingPrompt, - ds!.session.sessionId, - ds!.session.cliId ?? botCfg.cliId, - ds!.session.cliPathOverride ?? botCfg.cliPathOverride, - ds!.pendingAttachments, - ds!.pendingMentions, - availableBots, - ds!.pendingFollowUps, - { name: selfBot.botName, openId: selfBot.botOpenId }, - loc, - ds!.pendingSender, - { - larkAppId, - chatId: ds!.chatId, - whiteboardId: ds!.session.whiteboardId, - substituteTrigger: ds!.pendingSubstituteTrigger, - codexAppText: ds!.pendingCodexAppText, - codexAppApplicationContext: ds!.pendingCodexAppApplicationContext, - codexAppMessageContext: ds!.pendingCodexAppMessageContext, - codexAppFollowUps: ds!.pendingCodexAppFollowUps, - codexAppFollowUpContexts: ds!.pendingCodexAppFollowUpContexts, - }, - ) - : { content: '' }; - const pendingTurnId = ds!.pendingTurnId; - const previousPendingFollowUpInput = ds!.pendingFollowUpInput; - if (pendingRawInput && hasBufferedInput) { - ds!.pendingFollowUpInput = { - userPrompt: ds!.pendingCodexAppText !== undefined || ds!.pendingCodexAppFollowUps - ? [ds!.pendingCodexAppText ?? '', ...(ds!.pendingCodexAppFollowUps ?? [])].filter(Boolean).join('\n\n') - : pendingPrompt || ds!.pendingFollowUps?.join('\n\n') || '', + if (pendingRawInput && hasBufferedInput && wrappedInput) { + current.pendingFollowUpInput = { + userPrompt: current.pendingCodexAppText !== undefined || current.pendingCodexAppFollowUps + ? [current.pendingCodexAppText ?? '', ...(current.pendingCodexAppFollowUps ?? [])].filter(Boolean).join('\n\n') + : pendingPrompt || current.pendingFollowUps?.join('\n\n') || '', cliInput: wrappedInput.content, - ...(ds!.pendingFollowUpTurnId ? { turnId: ds!.pendingFollowUpTurnId } : {}), - ...((ds!.session.cliId ?? botCfg.cliId) === 'codex-app' && botCfg.codexAppCleanInput === true && wrappedInput.codexAppInput + ...((current.pendingFollowUpTurnIds?.at(-1) ?? current.pendingFollowUpTurnId) + ? { turnId: current.pendingFollowUpTurnIds?.at(-1) ?? current.pendingFollowUpTurnId } + : {}), + ...((current.session.cliId ?? botCfg.cliId) === 'codex-app' && botCfg.codexAppCleanInput === true && wrappedInput.codexAppInput ? { codexAppInput: wrappedInput.codexAppInput } : {}), codexAppInputGateFrozen: true, }; } - - ds!.pendingRepo = false; - publishAttentionPatch(ds!); - try { - if (pendingRawInput) forkWorker(ds!, '', false); - else if (pendingTurnId && hasBufferedInput) forkWorker(ds!, wrappedInput, { turnId: pendingTurnId }); - else if (hasBufferedInput) forkWorker(ds!, wrappedInput); - else forkWorker(ds!, '', false); - } catch (e) { - ds!.pendingRepo = true; - ds!.pendingFollowUpInput = previousPendingFollowUpInput; - publishAttentionPatch(ds!); - throw e; - } - if (pendingRawInput || hasBufferedInput) { - rememberLastCliInput(ds!, pendingRawInput ?? pendingPrompt, pendingRawInput ?? wrappedInput); - } - ds!.pendingPrompt = undefined; - ds!.pendingCodexAppText = undefined; - ds!.pendingCodexAppApplicationContext = undefined; - ds!.pendingCodexAppMessageContext = undefined; - ds!.pendingAttachments = undefined; - ds!.pendingMentions = undefined; - ds!.pendingSubstituteTrigger = undefined; - ds!.pendingSender = undefined; - ds!.pendingFollowUps = undefined; - ds!.pendingCodexAppFollowUps = undefined; - ds!.pendingCodexAppFollowUpContexts = undefined; - ds!.pendingFollowUpTurnId = undefined; - ds!.pendingTurnId = undefined; - committed = true; - - // Local one-shot consume BEFORE network awaits. Feishu withdraw is - // best-effort; stale card clicks must be rejected via this mark even - // if sessionReply throws or deleteMessage lags/fails. - const cardToWithdraw = ds!.repoCardMessageId; - markRepoCardConsumed(ds!, cardToWithdraw); - ds!.repoCardMessageId = undefined; - - // Hold the claim through confirmation + best-effort card withdraw. + if (pendingRawInput) rememberLastCliInput(current, pendingRawInput, pendingRawInput); + else if (hasBufferedInput && wrappedInput) rememberLastCliInput(current, pendingPrompt, wrappedInput); + + // forkWorker performs the synchronous pre-accept/write-ahead work. + // Keep the opening reservation and every buffered field intact if + // that step throws, so a failed launch cannot silently consume the + // first user turn or expose this worker:null owner as scratch. + const pendingTurnId = current.pendingTurnId + ?? current.session.pendingRepoSetup?.turnId; + forkWorker( + current, + pendingRawInput ? '' : (wrappedInput ?? ''), + !pendingRawInput && pendingTurnId ? { turnId: pendingTurnId } : false, + ); + current.pendingRepo = false; + current.pendingRepoCommitInFlight = true; + // Queued activation ownership lasts through adapter submission. + // These source buffers were folded into opening N; clear them but + // keep the gate so later inbounds enter the exact post-ACK FIFO. + current.initialStartPending = current.session.queuedActivationPending === true; + publishAttentionPatch(current); + current.pendingPrompt = undefined; + current.pendingCodexAppText = undefined; + current.pendingCodexAppApplicationContext = undefined; + current.pendingCodexAppMessageContext = undefined; + current.pendingAttachments = undefined; + current.pendingMentions = undefined; + current.pendingSubstituteTrigger = undefined; + current.pendingSender = undefined; + current.pendingFollowUps = undefined; + current.pendingFollowUpTurnId = undefined; + current.pendingFollowUpTurnIds = undefined; + current.pendingCodexAppFollowUps = undefined; + current.pendingCodexAppFollowUpContexts = undefined; + current.pendingCodexAppFollowUpGateAccepted = undefined; + current.pendingTurnId = undefined; + const cardToWithdraw = current.repoCardMessageId; + markRepoCardConsumed(current, cardToWithdraw); + current.repoCardMessageId = undefined; + return { current, cardToWithdraw }; + }); + if (!started) return false; + try { try { await sessionReply(rootId, replyText); } catch (e) { logger.warn(`[${logTag}] Confirm reply after pending repo commit failed: ${e instanceof Error ? e.message : e}`); } - if (cardToWithdraw) { - try { await deleteMessage(ds!.larkAppId, cardToWithdraw); } catch { /* best-effort */ } + if (started.cardToWithdraw) { + try { await deleteMessage(started.current.larkAppId, started.cardToWithdraw); } + catch { /* best-effort */ } } } finally { - ds!.pendingRepoCommitInFlight = false; + started.current.pendingRepoCommitInFlight = false; } - return committed; + return true; }; // Shared commit path for an already-resolved repo: update the session's @@ -1551,26 +1641,13 @@ export async function handleCommand( // numeric `/repo ` form and the `/repo ` form. const commitRepoSelection = async (selectedPath: string, displayName: string, how: string): Promise => { if (ds!.pendingRepo) { - if (ds!.pendingRepoCommitInFlight) { - await sessionReply(rootId, t('cmd.repo.worktree_in_progress', undefined, loc)); - return false; - } - ds!.pendingRepoCommitInFlight = true; - try { - // Claim before changing cwd so a card selection cannot overwrite - // this choice while the pending prompt is prepared. - ds!.workingDir = selectedPath; - ds!.session.workingDir = selectedPath; - ds!.session.riffRepoDirs = undefined; - sessionStore.updateSession(ds!.session); - // forkPendingCli owns claim release + confirm reply + card withdraw. - if (!await forkPendingCli(t('cmd.repo.selected_in_pending', { name: displayName }, loc), true)) { - return false; - } - } catch (e) { - ds!.pendingRepoCommitInFlight = false; - throw e; - } + // First spawn: the cwd pin and fork are one exclusive commit. Two + // simultaneous selections cannot make A reply while forking B's cwd. + const started = await forkPendingCli( + t('cmd.repo.selected_in_pending', { name: displayName }, loc), + { path: selectedPath, riffRepoDirs: undefined }, + ); + if (!started) return false; } else { // Safety net: a mid-session `/repo` switch closes the running // session and spawns a fresh one on the SAME anchor. Without a @@ -1591,53 +1668,96 @@ export async function handleCommand( // card), so `claude --resume` later would reopen the old context in // the new repo's cwd. The new repo is pinned onto the fresh session // below instead. - const claimedCard = claimCurrentRepoCard(ds!, undefined); - const closedCard = buildClosedSessionCard(ds!, loc); - killWorker(ds!); - sessionStore.closeSession(ds!.session.sessionId); + const targetSessionId = ds!.session.sessionId; + const switched = await withBotTurnMutation(ds!.larkAppId, async () => { + const candidate = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId, + ); + if (!candidate || candidate !== ds || candidate.session.status !== 'active') { + return { ok: false as const, error: 'session_replaced' as const }; + } + const key = activeSessionKey(candidate); + return withActiveSessionKeyLock(activeSessions, key, async () => { + // Resume/scheduler/dashboard creators use this same key lock + // without joining the bot admission gate. Re-resolve after the + // lock wait, then keep it across close -> replacement publish. + const current = [...activeSessions.values()].find( + owner => owner.session.sessionId === targetSessionId, + ); + if (!current || current !== candidate + || activeSessions.get(key) !== current + || current.session.status !== 'active') { + return { ok: false as const, error: 'session_replaced' as const }; + } + if (hasProtectedSessionMutationOwnership(current)) { + return { ok: false as const, error: 'dispatch_pending' as const }; + } + const closedCard = buildClosedSessionCard(current, loc); + const oldSession = current.session; + await closeWorkerPoolSession(targetSessionId); + // The key lock excludes every sanctioned creator. A direct + // lifecycle callback may still have published unexpectedly; + // fail closed instead of overwriting that first owner. + if (activeSessions.get(key) === current) activeSessions.delete(key); + if (activeSessions.has(key)) { + return { ok: false as const, error: 'session_replaced' as const }; + } + const cardToWithdraw = current.repoCardMessageId; + markRepoCardConsumed(current, cardToWithdraw); + current.repoCardMessageId = undefined; + + const session = sessionStore.createSession( + current.chatId, + current.scope === 'chat' ? oldSession.rootMessageId : rootId, + displayName, + current.chatType, + current.scope, + ); + current.session = session; + current.lastUserPrompt = undefined; + current.lastCliInput = undefined; + current.workingDir = selectedPath; + session.workingDir = selectedPath; + session.larkAppId = current.larkAppId; + session.chatDisplayName = oldSession.chatDisplayName; + session.ownerOpenId = oldSession.ownerOpenId; + session.creatorOpenId = oldSession.creatorOpenId; + session.lastCallerOpenId = oldSession.lastCallerOpenId; + sessionStore.updateSession(session); + current.hasHistory = false; + activeSessions.set(key, current); + forkWorker(current, '', false); + return { ok: true as const, current, closedCard, cardToWithdraw }; + }); + }); + if (!switched.ok) { + if (switched.error === 'dispatch_pending') { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能切换仓库；请等待本轮完成或关闭会话。', + ); + } else { + logger.warn(`[${logTag}] Repo switch aborted because the session was replaced`); + } + return false; + } await deliverEphemeralOrReply( - ds!, + switched.current, message.senderId, - closedCard, + switched.closedCard, 'interactive', - () => sessionReply(rootId, closedCard, 'interactive'), - ); - - const oldSession = ds!.session; - // `rootId` is the routing anchor. For chat-scope sessions it is the - // `oc_...` chat id, not the traceable `om_...` message root stored on - // Session. Preserve the old identity and explicitly persist scope so - // repo switches cannot turn a chat session into a legacy scope-less - // record that the CLI later mistakes for a thread. - const session = sessionStore.createSession( - ds!.chatId, - ds!.scope === 'chat' ? oldSession.rootMessageId : rootId, - displayName, - ds!.chatType, - ds!.scope, + () => sessionReply(rootId, switched.closedCard, 'interactive'), ); - ds!.session = session; - ds!.lastUserPrompt = undefined; - ds!.lastCliInput = undefined; - ds!.workingDir = selectedPath; - ds!.session.workingDir = selectedPath; - ds!.session.larkAppId = ds!.larkAppId; - ds!.session.chatDisplayName = oldSession.chatDisplayName; - ds!.session.ownerOpenId = oldSession.ownerOpenId; - ds!.session.creatorOpenId = oldSession.creatorOpenId; - ds!.session.lastCallerOpenId = oldSession.lastCallerOpenId; - sessionStore.updateSession(ds!.session); - ds!.hasHistory = false; - forkWorker(ds!, '', false); - try { - await sessionReply(rootId, t('cmd.repo.switched_to', { name: displayName }, loc)); - } catch (e) { - logger.warn(`[${logTag}] Confirm reply after mid-session repo switch failed: ${e instanceof Error ? e.message : e}`); - } - if (claimedCard) { - try { await deleteMessage(ds!.larkAppId, claimedCard); } catch { /* best-effort */ } + await sessionReply(rootId, t('cmd.repo.switched_to', { name: displayName }, loc)); + if (switched.cardToWithdraw) { + try { await deleteMessage(switched.current.larkAppId, switched.cardToWithdraw); } + catch { /* best-effort */ } } } + if (ds!.repoCardMessageId) { + deleteMessage(ds!.larkAppId, ds!.repoCardMessageId); + ds!.repoCardMessageId = undefined; + } logger.info(`[${logTag}] Repo selected via ${how}: ${selectedPath}`); return true; }; @@ -3403,23 +3523,43 @@ export async function startCodexAppThreadSession( deps.sessionReply(rid, content, msgType, larkAppId); const loc: Locale = localeForBot(ds.larkAppId ?? larkAppId); const title = codexAppThreadTitle(thread); - - ds.adoptedFrom = undefined; - ds.workingDir = thread.cwd; - ds.hasHistory = true; - ds.currentTurnTitle = undefined; - ds.lastScreenContent = undefined; - ds.lastScreenStatus = undefined; - - ds.session.workingDir = thread.cwd; - ds.session.title = `Codex App: ${title}`; - ds.session.cliId = 'codex-app'; - ds.session.cliSessionId = thread.threadId; - ds.session.adoptedFrom = undefined; - sessionStore.updateSession(ds.session); - - forkWorker(ds, '', true); - await sessionReply(sessionAnchorId(ds), t('cmd.codex_app_adopt.success', { title }, loc)); + const targetSessionId = ds.session.sessionId; + const switched = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...deps.activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current || current !== ds) return { status: 'gone' as const }; + if (hasProtectedSessionMutationOwnership(current)) { + return { status: 'pending' as const, anchor: sessionAnchorId(current) }; + } + current.adoptedFrom = undefined; + current.workingDir = thread.cwd; + current.hasHistory = true; + current.currentTurnTitle = undefined; + current.lastScreenContent = undefined; + current.lastScreenStatus = undefined; + current.session.workingDir = thread.cwd; + current.session.title = `Codex App: ${title}`; + current.session.cliId = 'codex-app'; + current.session.cliSessionId = thread.threadId; + current.session.adoptedFrom = undefined; + sessionStore.updateSession(current.session); + forkWorker(current, '', true); + return { status: 'switched' as const, anchor: sessionAnchorId(current) }; + }); + if (switched.status === 'gone') { + await sessionReply(sessionAnchorId(ds), t('cmd.no_active_session', undefined, loc)); + return; + } + if (switched.status === 'pending') { + await sessionReply( + switched.anchor, + '当前 Codex App 仍有未结算消息，不能切换原生 thread；请等待本轮完成或先关闭会话。', + ); + return; + } + await sessionReply(switched.anchor, t('cmd.codex_app_adopt.success', { title }, loc)); } export async function startAdoptSession( @@ -3470,34 +3610,55 @@ export async function startAdoptSession( const project = target.cwd.split('/').pop() || target.cwd; const pane = zellij ? `${target.zellijSession}/${target.zellijPaneId}` : adoptTargetLabel(target); - - ds.workingDir = target.cwd; - ds.session.workingDir = target.cwd; - ds.session.title = `Adopt: ${project}`; - ds.adoptedFrom = { - source: zellij ? 'zellij' : target.source, - tmuxTarget: zellij ? undefined : target.tmuxTarget, - zellijSession: zellij ? target.zellijSession : undefined, - zellijPaneId: zellij ? target.zellijPaneId : undefined, - herdrSessionName: zellij ? undefined : target.herdrSessionName, - herdrTarget: zellij ? undefined : target.herdrTarget, - herdrPaneId: zellij ? undefined : target.herdrPaneId, - herdrAgentName: zellij ? undefined : target.herdrAgentName, - herdrTerminalId: zellij ? undefined : target.herdrTerminalId, - originalCliPid: target.cliPid, - sessionId: target.sessionId, - cliId: target.cliId, - cwd: target.cwd, - paneCols: target.paneCols, - paneRows: target.paneRows, - }; - ds.session.adoptedFrom = { ...ds.adoptedFrom }; - sessionStore.updateSession(ds.session); - - forkAdoptWorker(ds); + const targetSessionId = ds.session.sessionId; + const adopted = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...deps.activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current || current !== ds) return { status: 'gone' as const }; + if (hasProtectedSessionMutationOwnership(current)) { + return { status: 'pending' as const, anchor: sessionAnchorId(current) }; + } + current.workingDir = target.cwd; + current.session.workingDir = target.cwd; + current.session.title = `Adopt: ${project}`; + current.adoptedFrom = { + source: zellij ? 'zellij' : target.source, + tmuxTarget: zellij ? undefined : target.tmuxTarget, + zellijSession: zellij ? target.zellijSession : undefined, + zellijPaneId: zellij ? target.zellijPaneId : undefined, + herdrSessionName: zellij ? undefined : target.herdrSessionName, + herdrTarget: zellij ? undefined : target.herdrTarget, + herdrPaneId: zellij ? undefined : target.herdrPaneId, + herdrAgentName: zellij ? undefined : target.herdrAgentName, + herdrTerminalId: zellij ? undefined : target.herdrTerminalId, + originalCliPid: target.cliPid, + sessionId: target.sessionId, + cliId: target.cliId, + cwd: target.cwd, + paneCols: target.paneCols, + paneRows: target.paneRows, + }; + current.session.adoptedFrom = { ...current.adoptedFrom }; + sessionStore.updateSession(current.session); + forkAdoptWorker(current); + return { status: 'adopted' as const, anchor: sessionAnchorId(current) }; + }); + if (adopted.status === 'gone') { + await sessionReply(sessionAnchorId(ds), t('cmd.no_active_session', undefined, loc)); + return; + } + if (adopted.status === 'pending') { + await sessionReply( + adopted.anchor, + '当前 Codex App 仍有未结算消息，不能切换到外部会话；请等待本轮完成或先关闭会话。', + ); + return; + } const cliName = getCliDisplayName(target.cliId); - await sessionReply(sessionAnchorId(ds), t('cmd.adopt.success', { cliName, project, pane }, loc)); + await sessionReply(adopted.anchor, t('cmd.adopt.success', { cliName, project, pane }, loc)); } /** Discover the sessions resumable from disk for `cliId`, excluding any whose @@ -3547,19 +3708,39 @@ export async function startResumeImportSession( deps.sessionReply(rid, content, msgType, larkAppId); const loc: Locale = localeForBot(ds.larkAppId ?? larkAppId); const project = target.cwd.split('/').pop() || target.cwd; - - ds.workingDir = target.cwd; - ds.session.workingDir = target.cwd; - ds.session.cliSessionId = target.cliSessionId; - ds.session.title = target.title || `Import: ${project}`; - // Resume sandbox decision is left to forkWorker (resume=true → not sandboxed, - // matching restore semantics). Mark history so the session is treated as a - // resume, not a fresh spawn. - ds.hasHistory = true; - sessionStore.updateSession(ds.session); - - forkWorker(ds, '', true); + const targetSessionId = ds.session.sessionId; + const resumed = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...deps.activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current || current !== ds) return { status: 'gone' as const }; + if (hasProtectedSessionMutationOwnership(current)) { + return { status: 'pending' as const, anchor: sessionAnchorId(current) }; + } + current.workingDir = target.cwd; + current.session.workingDir = target.cwd; + current.session.cliSessionId = target.cliSessionId; + current.session.title = target.title || `Import: ${project}`; + // Resume sandbox decision is left to forkWorker (resume=true → not + // sandboxed, matching restore semantics). Mark history so this is a resume. + current.hasHistory = true; + sessionStore.updateSession(current.session); + forkWorker(current, '', true); + return { status: 'resumed' as const, anchor: sessionAnchorId(current) }; + }); + if (resumed.status === 'gone') { + await sessionReply(sessionAnchorId(ds), t('cmd.no_active_session', undefined, loc)); + return; + } + if (resumed.status === 'pending') { + await sessionReply( + resumed.anchor, + '当前 Codex App 仍有未结算消息，不能导入外部会话；请等待本轮完成或先关闭会话。', + ); + return; + } const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); - await sessionReply(sessionAnchorId(ds), t('cmd.adopt.resume_success', { cliName, project, title: target.title || target.cliSessionId.slice(0, 8) }, loc)); + await sessionReply(resumed.anchor, t('cmd.adopt.resume_success', { cliName, project, title: target.title || target.cliSessionId.slice(0, 8) }, loc)); } diff --git a/src/core/dashboard-ipc-server.ts b/src/core/dashboard-ipc-server.ts index 80df951e3..20e9a353d 100644 --- a/src/core/dashboard-ipc-server.ts +++ b/src/core/dashboard-ipc-server.ts @@ -92,6 +92,13 @@ import { validateTriggerRequest, type TriggerResponse } from '../services/trigge import { resolveCliSelection, selectionKeyForBot } from '../setup/cli-selection.js'; import { checkCliAvailability } from '../setup/cli-availability.js'; import { enrichHistorySenders, type HistoryBotInfo } from '../dashboard/history-senders.js'; +import { + validateCodexAppManagedSendOrigin, +} from '../utils/codex-app-dispatch-ledger.js'; +import { withBotTurnAdmission, withBotTurnMutation } from './bot-turn-mutation-gate.js'; +import { + protectedSessionMutationReasons, +} from './session-mutation-guard.js'; let exactChatGrantHandler: typeof applyExactChatGrantRequest = applyExactChatGrantRequest; /** Test seam: replace the exact-grant service without touching live Feishu/config state. */ @@ -192,6 +199,43 @@ export function jsonRes(res: ServerResponse, status: number, body: unknown): voi res.end(JSON.stringify(body)); } +function rejectProtectedSessionMutation( + res: ServerResponse, + values: readonly (DaemonSession | Session)[], +): boolean { + const bySessionId = new Map; + }>(); + for (const value of values) { + const session = 'session' in value ? value.session : value; + const reasons = protectedSessionMutationReasons(value); + if (reasons.length === 0) continue; + const existing = bySessionId.get(session.sessionId); + if (existing) { + existing.reasons = [...new Set([...existing.reasons, ...reasons])]; + continue; + } + bySessionId.set(session.sessionId, { + sessionId: session.sessionId, + ...(session.cliId ? { cliId: session.cliId } : {}), + reasons, + }); + } + const blockingSessions = [...bySessionId.values()]; + if (blockingSessions.length === 0) return false; + const codexDispatchOnly = blockingSessions.every(blocker => + blocker.reasons.every(reason => reason === 'codex_app_dispatch')); + jsonRes(res, 409, { + ok: false, + error: codexDispatchOnly + ? 'codex_app_dispatch_pending' + : 'session_mutation_pending', + blockingSessions, + }); + return true; +} export class JsonBodyTooLargeError extends Error { constructor(readonly maxBytes: number) { super(`JSON request body exceeds ${maxBytes} bytes`); @@ -332,6 +376,86 @@ export async function readJsonBody( return JSON.parse(Buffer.concat(chunks).toString('utf8')); } +class IpcBodyTooLargeError extends Error {} +class IpcBodyTimeoutError extends Error {} + +/** Strict reader for the one unauthenticated capability challenge route. The + * generic IPC reader intentionally has no cap for trusted-host endpoints; a + * loopback-confined process must not be able to buffer arbitrary chunked input + * before its capability is checked. */ +async function readBoundedJsonBody( + req: IncomingMessage, + maxBytes: number, + timeoutMs: number, +): Promise { + const contentLength = req.headers['content-length']; + if (typeof contentLength === 'string') { + if (!/^\d+$/.test(contentLength) || Number(contentLength) > maxBytes) { + throw new IpcBodyTooLargeError('request body too large'); + } + } + return await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + const cleanup = () => { + clearTimeout(timer); + req.off('data', onData); + req.off('end', onEnd); + req.off('error', onError); + req.off('aborted', onAborted); + }; + const fail = (err: Error) => { + if (settled) return; + settled = true; + req.pause(); + cleanup(); + reject(err); + }; + const onData = (raw: Buffer | string) => { + const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(raw); + total += chunk.length; + if (total > maxBytes) { + fail(new IpcBodyTooLargeError('request body too large')); + return; + } + chunks.push(chunk); + }; + const onEnd = () => { + if (settled) return; + settled = true; + cleanup(); + try { + resolve(chunks.length === 0 + ? {} as T + : JSON.parse(Buffer.concat(chunks, total).toString('utf8')) as T); + } catch (err) { + reject(err); + } + }; + const onError = (err: Error) => fail(err); + const onAborted = () => fail(new Error('request aborted')); + const timer = setTimeout( + () => fail(new IpcBodyTimeoutError('request body timed out')), + timeoutMs, + ); + timer.unref?.(); + req.on('data', onData); + req.once('end', onEnd); + req.once('error', onError); + req.once('aborted', onAborted); + }); +} + +function closeUntrustedRequestAfterResponse(req: IncomingMessage, res: ServerResponse): void { + res.setHeader('connection', 'close'); + res.once('finish', () => req.destroy()); + // Drain whatever is already buffered until the response has been flushed; + // the finish hook then closes a partial/slow body instead of reusing it as a + // keep-alive request. + req.resume(); +} + // ─── Trusted-host auth (loopback + route-bound HMAC) ──────────────────────── // // Production start enables a server-wide gate: loopback is connectivity, not @@ -501,8 +625,38 @@ ipcRoute('GET', '/api/sessions/:sessionId', (_req, res, params) => { }); ipcRoute('POST', '/api/sessions/:sessionId/close', async (_req, res, params) => { - const r = await closeSession(params.sessionId); - jsonRes(res, 200, r); + const initial = findSessionRecord(params.sessionId); + if (!initial) return jsonRes(res, 200, { ok: true, alreadyClosed: true }); + const larkAppId = initial.larkAppId || cachedLarkAppId; + if (!larkAppId) return jsonRes(res, 503, { ok: false, error: 'bot_not_found' }); + return withBotTurnMutation(larkAppId, async () => { + // Re-resolve only after every earlier admitted turn has drained. Explicit + // close is the abandon boundary and may clear accepted FIFO, but it must + // never clear a stale pre-drain snapshot while a turn is still preparing. + const current = findSessionRecord(params.sessionId); + if (!current || current.status === 'closed') { + return jsonRes(res, 200, { ok: true, alreadyClosed: true }); + } + const r = await closeSession(params.sessionId); + jsonRes(res, r.ok ? 200 : 502, r); + }); +}); + +// `botmux list` zombie pruning is maintenance, not explicit abandon. Serialize +// against inbound admission and refuse if any backend-neutral durable owner +// became unsettled after the CLI took its liveness snapshot. +ipcRoute('POST', '/api/sessions/:sessionId/prune', async (_req, res, params) => { + const initial = findSessionRecord(params.sessionId); + if (!initial) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + const larkAppId = initial.larkAppId || cachedLarkAppId; + if (!larkAppId) return jsonRes(res, 503, { ok: false, error: 'bot_not_found' }); + return withBotTurnMutation(larkAppId, async () => { + const current = findSessionRecord(params.sessionId); + if (!current) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + if (rejectProtectedSessionMutation(res, [current])) return; + const r = await closeSession(params.sessionId); + jsonRes(res, r.ok ? 200 : 502, r); + }); }); /** Post a scope-aware "restarting" notice into the session's Lark thread/chat, @@ -531,7 +685,10 @@ function postRestartNotice(ds: DaemonSession, fresh: boolean): void { } } -ipcRoute('POST', '/api/sessions/:sessionId/restart', (_req, res, params) => { +ipcRoute('POST', '/api/sessions/:sessionId/restart', async (_req, res, params) => { + const initial = findActiveBySessionId(params.sessionId); + if (!initial) return jsonRes(res, 404, { ok: false, error: 'session_not_active' }); + return withBotTurnMutation(initial.larkAppId, () => { const ds = findActiveBySessionId(params.sessionId); if (!ds) return jsonRes(res, 404, { ok: false, error: 'session_not_active' }); // Adopt/observed sessions: botmux never owned the CLI — restarting would kill @@ -539,11 +696,12 @@ ipcRoute('POST', '/api/sessions/:sessionId/restart', (_req, res, params) => { if (ds.adoptedFrom || ds.initConfig?.adoptMode) { return jsonRes(res, 409, { ok: false, error: 'adopt_restart_unsupported' }); } + if (rejectProtectedSessionMutation(res, [ds])) return; const cliId = ds.session.cliId ?? 'unknown'; if (ds.worker && !ds.worker.killed) { // Live worker → in-place CLI restart (kills the CLI, respawns with --resume). try { - ds.worker.send({ type: 'restart' } as DaemonToWorker); + ds.worker.send({ type: 'restart', reason: 'operator' } as DaemonToWorker); } catch (err) { return jsonRes(res, 502, { ok: false, error: String(err) }); } @@ -558,6 +716,7 @@ ipcRoute('POST', '/api/sessions/:sessionId/restart', (_req, res, params) => { forkWorker(ds, '', ds.hasHistory); postRestartNotice(ds, true); jsonRes(res, 200, { ok: true, sessionId: params.sessionId, cliId, revived: true }); + }); }); /** Manually suspend one active session: kill the worker + CLI/pane, session @@ -565,7 +724,10 @@ ipcRoute('POST', '/api/sessions/:sessionId/restart', (_req, res, params) => { * the same semantics the idle-worker sweeper applies over the live cap. * Primary use: `botmux suspend --isolated` after a credential rotation, so * isolated bots' next cold spawn re-provisions the freshest creds. */ -ipcRoute('POST', '/api/sessions/:sessionId/suspend', (_req, res, params) => { +ipcRoute('POST', '/api/sessions/:sessionId/suspend', async (_req, res, params) => { + const initial = findActiveBySessionId(params.sessionId); + if (!initial) return jsonRes(res, 404, { ok: false, error: 'session_not_active' }); + return withBotTurnMutation(initial.larkAppId, () => { const ds = findActiveBySessionId(params.sessionId); if (!ds) return jsonRes(res, 404, { ok: false, error: 'session_not_active' }); // Adopt/observed sessions: botmux never owned the CLI — suspending would kill @@ -573,6 +735,7 @@ ipcRoute('POST', '/api/sessions/:sessionId/suspend', (_req, res, params) => { if (ds.adoptedFrom || ds.initConfig?.adoptMode) { return jsonRes(res, 409, { ok: false, error: 'adopt_suspend_unsupported' }); } + if (rejectProtectedSessionMutation(res, [ds])) return; if (!ds.worker || ds.worker.killed) { // Worker already gone (idle-suspended / crash-stopped) — the goal state is // already reached, so report idempotent success without a live kill. @@ -584,6 +747,7 @@ ipcRoute('POST', '/api/sessions/:sessionId/suspend', (_req, res, params) => { return jsonRes(res, 409, { ok: false, error: 'backend_not_suspendable' }); } jsonRes(res, 200, { ok: true, sessionId: params.sessionId, suspended: true }); + }); }); /** 会话级 CLI IPC（slash/cd）的调用方证明：trusted-host（.dashboard-secret HMAC， @@ -759,41 +923,89 @@ ipcRoute('POST', '/api/sessions/:sessionId/board', async (req, res, params) => { if (!column && position === null) return jsonRes(res, 400, { ok: false, error: 'bad_request' }); const session = findSessionRecord(params.sessionId); if (!session) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + const larkAppId = session.larkAppId || cachedLarkAppId; + if (!larkAppId) return jsonRes(res, 503, { ok: false, error: 'bot_not_found' }); + return withBotTurnAdmission(larkAppId, async () => { + const currentSession = findSessionRecord(params.sessionId); + if (!currentSession) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); // 待办池(queued)会话被拖到「进行中」= 激活：把暂存内容当首轮发给 CLI 开跑。 // activateQueuedSession 内部会清 queued + 把列设成 in_progress + forkWorker。 const activeDs = findActiveBySessionId(params.sessionId); + let activationTransferred = false; if (column === 'in_progress' && activeDs?.session.queued) { - await activateQueuedSession(activeDs); + const activated = await activateQueuedSession(activeDs); + if (!activated.ok) return jsonRes(res, 500, activated); + activationTransferred = true; } else if (column) { - session.kanbanColumn = column; + currentSession.kanbanColumn = column; + } + if (position !== null) currentSession.kanbanPosition = position; + try { + sessionStore.updateSession(currentSession); + } catch (err) { + if (!activationTransferred) throw err; + logger.error( + `[dashboard] board metadata persistence failed after queued activation ownership transferred: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); } - if (position !== null) session.kanbanPosition = position; - sessionStore.updateSession(session); dashboardEventBus.publish({ type: 'session.update', body: { sessionId: params.sessionId, // queued 一并下发：激活后 session.queued 已为 false，前端浅合并若不带这个字段 // 会残留 queued=true（卡片仍显示「开始」、再点 409）。!!session.queued 始终反映现态。 - patch: { kanbanColumn: session.kanbanColumn, kanbanPosition: session.kanbanPosition, queued: !!session.queued }, + patch: { kanbanColumn: currentSession.kanbanColumn, kanbanPosition: currentSession.kanbanPosition, queued: !!currentSession.queued }, }, }); jsonRes(res, 200, { ok: true }); + }); +}); + +// Narrow CLI whiteboard binding mutation. Keeping this daemon-side avoids a +// short-lived `botmux whiteboard` process rewriting a stale whole Session row +// over a concurrent Codex App FIFO transition. +ipcRoute('POST', '/api/sessions/:sessionId/whiteboard', async (req, res, params) => { + let body: { whiteboardId?: unknown }; + try { body = await readJsonBody(req); } + catch { return jsonRes(res, 400, { ok: false, error: 'bad_json' }); } + if (typeof body.whiteboardId !== 'string' + || body.whiteboardId.length === 0 + || body.whiteboardId.length > 256) { + return jsonRes(res, 400, { ok: false, error: 'bad_whiteboard_id' }); + } + const session = findSessionRecord(params.sessionId); + if (!session) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + const larkAppId = session.larkAppId || cachedLarkAppId; + if (!larkAppId) return jsonRes(res, 503, { ok: false, error: 'bot_not_found' }); + return withBotTurnAdmission(larkAppId, async () => { + const current = findSessionRecord(params.sessionId); + if (!current) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + current.whiteboardId = body.whiteboardId as string; + sessionStore.updateSession(current); + jsonRes(res, 200, { ok: true, whiteboardId: current.whiteboardId }); + }); }); // 待办池会话「开始」：把 parked 会话激活（发首轮、起 CLI），与拖到「进行中」同义。 ipcRoute('POST', '/api/sessions/:sessionId/start', async (_req, res, params) => { + const initial = findActiveBySessionId(params.sessionId); + if (!initial) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + return withBotTurnAdmission(initial.larkAppId, async () => { const ds = findActiveBySessionId(params.sessionId); if (!ds) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); if (!ds.session.queued) return jsonRes(res, 409, { ok: false, error: 'not_queued' }); const r = await activateQueuedSession(ds); if (!r.ok) return jsonRes(res, 500, r); - sessionStore.updateSession(ds.session); dashboardEventBus.publish({ type: 'session.update', - body: { sessionId: params.sessionId, patch: { kanbanColumn: ds.session.kanbanColumn, queued: false } }, + body: { + sessionId: params.sessionId, + patch: { kanbanColumn: ds.session.kanbanColumn, queued: !!ds.session.queued }, + }, }); jsonRes(res, 200, { ok: true }); + }); }); // Dashboard「创建会话」spawn：在新建的群里为本 daemon 的 bot 拉起/暂存一条 chat-scope @@ -807,6 +1019,7 @@ ipcRoute('POST', '/api/sessions/spawn', async (req, res) => { const parsed = parseSpawnRequest(body); if (!parsed.ok) return jsonRes(res, 400, { ok: false, error: parsed.error }); const postBanner = !!(body as any).postBanner; + return withBotTurnAdmission(cachedLarkAppId, async () => { let attachments; try { attachments = materializeDashboardImages(cachedLarkAppId, parsed.value.images); @@ -832,6 +1045,7 @@ ipcRoute('POST', '/api/sessions/spawn', async (req, res) => { return jsonRes(res, r.error === 'session_exists' ? 409 : 500, r); } jsonRes(res, 200, r); + }); }); ipcRoute('POST', '/api/chat-reply-mode', async (req, res) => { @@ -1172,6 +1386,13 @@ function workingDirForSession(sessionId: string): string | undefined { */ ipcRoute('POST', '/api/sessions/:sessionId/resume', async (req, res, params) => { const sessionId = params.sessionId; + const sourceSession = findSessionRecord(sessionId); + if (!sourceSession) return jsonRes(res, 404, { ok: false, error: 'not_found' }); + // Legacy persisted sessions may carry an empty larkAppId and are hydrated + // with this daemon's identity by resumeSession. Use the same fallback for + // admission instead of rejecting an otherwise valid recovery record. + const larkAppId = sourceSession.larkAppId || cachedLarkAppId || '__legacy_unbound__'; + return withBotTurnAdmission(larkAppId, async () => { const reg = getActiveSessionsRegistry(); if (!reg) return jsonRes(res, 503, { ok: false, error: 'registry_unavailable' }); const result = await resumeSession(sessionId, reg); @@ -1238,6 +1459,7 @@ ipcRoute('POST', '/api/sessions/:sessionId/resume', async (req, res, params) => workingDir: ds.session.workingDir, cliId, }); + }); }); /** @@ -2521,6 +2743,7 @@ ipcRoute('PUT', '/api/bot-avatar', async (req, res) => { // path; this covers the hot-switch path). ipcRoute('PUT', '/api/bot-agent', async (req, res) => { if (!cachedLarkAppId) return jsonRes(res, 503, { error: 'larkAppId_not_set' }); + const larkAppId = cachedLarkAppId; let body: { cliId?: unknown; model?: unknown }; try { body = await readJsonBody<{ cliId?: unknown; model?: unknown }>(req); } catch { return jsonRes(res, 400, { ok: false, error: 'bad_json' }); } @@ -2534,7 +2757,7 @@ ipcRoute('PUT', '/api/bot-agent', async (req, res) => { return jsonRes(res, 400, { ok: false, error: 'invalid_cli', message: err?.message ?? String(err) }); } const model = typeof body.model === 'string' ? body.model.trim() : ''; - const currentBotConfig = getBot(cachedLarkAppId).config; + const currentBotConfig = getBot(larkAppId).config; const availability = checkCliAvailability({ cliId: selected.cliId, wrapperCli: selected.wrapperCli, @@ -2547,12 +2770,28 @@ ipcRoute('PUT', '/api/bot-agent', async (req, res) => { ? undefined : `配置已保存，但所选 Agent 当前无法启动：${availability.reason ?? '本地启动依赖不可用'}。请先在 daemon 所在机器安装或修正 PATH / CLI 路径。`; - // If the new CLI/wrapper can no longer enforce a currently-on read isolation, - // auto-clear the flag here so the next session doesn't fail-close on it. (The - // read-isolation toggle validates at enable time; changing the agent afterwards - // is the other way a bot could end up configured-but-unenforceable.) - let readIsolationCleared = false; - const r = await rmwBotEntry(cachedLarkAppId, (entry) => { + return withBotTurnMutation(larkAppId, async () => { + // Agent selection can replace every live worker generation and may also + // auto-clear readIsolation. Close admission and drain in-flight acceptance + // before inspecting both the registry and restart source of truth. A + // settings mutation is not an explicit abandon boundary: an unsettled + // Codex App FIFO must survive unchanged for recovery. + const activeBotSessions = listActiveSessions().filter(ds => ds.larkAppId === larkAppId); + const persistedActiveBotSessions = sessionStore.listSessions().filter(session => + session.status === 'active' + && (session.larkAppId === larkAppId || !session.larkAppId), + ); + if (rejectProtectedSessionMutation(res, [ + ...activeBotSessions, + ...persistedActiveBotSessions, + ])) return; + + // If the new CLI/wrapper can no longer enforce a currently-on read isolation, + // auto-clear the flag here so the next session doesn't fail-close on it. (The + // read-isolation toggle validates at enable time; changing the agent afterwards + // is the other way a bot could end up configured-but-unenforceable.) + let readIsolationCleared = false; + const r = await rmwBotEntry(larkAppId, (entry) => { entry.cliId = selected.cliId; if (selected.wrapperCli) entry.wrapperCli = selected.wrapperCli; else delete entry.wrapperCli; @@ -2573,42 +2812,43 @@ ipcRoute('PUT', '/api/bot-agent', async (req, res) => { delete entry.backendType; } return { write: true, result: null }; - }); - if (!r.ok) return jsonRes(res, 400, { ok: false, error: r.reason }); - - const bot = getBot(cachedLarkAppId); - bot.config.cliId = selected.cliId; - if (selected.wrapperCli) bot.config.wrapperCli = selected.wrapperCli; - else bot.config.wrapperCli = undefined; - bot.config.model = model || undefined; - if (readIsolationCleared) bot.config.readIsolation = false; - if (selected.cliId === 'riff') { - bot.config.backendType = 'riff'; - } else if (bot.config.backendType === 'riff') { - bot.config.backendType = undefined; - } + }); + if (!r.ok) return jsonRes(res, 400, { ok: false, error: r.reason }); + + const bot = getBot(larkAppId); + bot.config.cliId = selected.cliId; + if (selected.wrapperCli) bot.config.wrapperCli = selected.wrapperCli; + else bot.config.wrapperCli = undefined; + bot.config.model = model || undefined; + if (readIsolationCleared) bot.config.readIsolation = false; + if (selected.cliId === 'riff') { + bot.config.backendType = 'riff'; + } else if (bot.config.backendType === 'riff') { + bot.config.backendType = undefined; + } - // 热切后立刻清掉本 bot 名下失配的存量会话——否则它们冻结的旧 CLI 会被下一条 - // 消息 lazy resume 复活，要等下次 daemon 重启才被 restore 守卫清理。 - const closedMismatchedSessions = await closeCliMismatchedSessionsForBot(cachedLarkAppId); + // 热切后立刻清掉本 bot 名下失配的存量会话——否则它们冻结的旧 CLI 会被下一条 + // 消息 lazy resume 复活，要等下次 daemon 重启才被 restore 守卫清理。 + const closedMismatchedSessions = await closeCliMismatchedSessionsForBot(larkAppId); - const selectionKey = selectionKeyForBot(selected.cliId, selected.wrapperCli); - jsonRes(res, 200, { - ok: true, - cliId: selected.cliId, - wrapperCli: selected.wrapperCli ?? null, - model: model || null, - selectionKey, - closedMismatchedSessions, - // Report the (possibly auto-cleared) read-isolation state + whether the new - // agent can still enforce it, so the dashboard updates its toggle immediately - // instead of showing a stale enabled/supported state until a full refetch. - readIsolation: bot.config.readIsolation === true, - readIsolationSupported: readIsolationEnforceableFor(bot.config), - readIsolationCleared, - agentAvailable: availability.available, - availabilityWarning, - requiredCommand: availability.command, + const selectionKey = selectionKeyForBot(selected.cliId, selected.wrapperCli); + jsonRes(res, 200, { + ok: true, + cliId: selected.cliId, + wrapperCli: selected.wrapperCli ?? null, + model: model || null, + selectionKey, + closedMismatchedSessions, + // Report the (possibly auto-cleared) read-isolation state + whether the new + // agent can still enforce it, so the dashboard updates its toggle immediately + // instead of showing a stale enabled/supported state until a full refetch. + readIsolation: bot.config.readIsolation === true, + readIsolationSupported: readIsolationEnforceableFor(bot.config), + readIsolationCleared, + agentAvailable: availability.available, + availabilityWarning, + requiredCommand: availability.command, + }); }); }); @@ -2864,6 +3104,9 @@ ipcRoute('PUT', '/api/bot-sandbox', async (req, res) => { let body: { enabled?: unknown }; try { body = await readJsonBody<{ enabled?: unknown }>(req); } catch { return jsonRes(res, 400, { ok: false, error: 'bad_json' }); } + // File-sandbox policy is frozen onto each Session at creation and reused on + // restore; this toggle is intentionally next-session-only and cannot mutate + // a live pane's profile. const r = await sandboxStore.updateBotSandbox(cachedLarkAppId, body.enabled === true); if (!r.ok) return jsonRes(res, 400, { ok: false, error: r.reason }); jsonRes(res, 200, { ok: true, sandbox: r.sandbox }); @@ -3260,6 +3503,10 @@ export function startIpcServer(opts: { * deliberate secret repair cannot strand a daemon on a stale cached key. * Tests that omit this option retain the lightweight in-process server. */ authRequired?: boolean; + /** Daemon restore barrier. The socket/health route may come up early so its + * descriptor is discoverable, but every state-bearing route waits until all + * durable session owners have been registered. */ + ready?: Promise; }): Promise { let boundPort = opts.port; const server: Server = createServer(async (req, res) => { @@ -3279,6 +3526,7 @@ export function startIpcServer(opts: { return jsonRes(res, 401, { ok: false, error: 'unauthorized', reason: auth.reason }); } } + if (!publicRoute && opts.ready) await opts.ready; for (const r of routes) { if (r.method !== req.method) continue; const m = r.pattern.exec(url.pathname); diff --git a/src/core/idle-worker-sweeper.ts b/src/core/idle-worker-sweeper.ts index bbb44e679..43dac96de 100644 --- a/src/core/idle-worker-sweeper.ts +++ b/src/core/idle-worker-sweeper.ts @@ -1,6 +1,7 @@ import type { DaemonSession } from './types.js'; import { suspendWorker } from './worker-pool.js'; import { isSuspendableBackendType } from './persistent-backend.js'; +import { tryWithBotTurnMutation } from './bot-turn-mutation-gate.js'; /** * Default per-bot live-session cap applied when a bot has no explicit @@ -11,6 +12,7 @@ import { isSuspendableBackendType } from './persistent-backend.js'; * ('botDefaults.maxLiveWorkers*' i18n) — keep them in sync. */ export const DEFAULT_MAX_LIVE_WORKERS = 30; +export const IDLE_WORKER_SWEEP_MUTATION_ACQUIRE_TIMEOUT_MS = 1_000; export interface IdleWorkerSweepOptions { /** @@ -20,6 +22,11 @@ export interface IdleWorkerSweepOptions { * never suspend). */ maxLiveWorkers?: number; + /** + * Bound how long a detached sweep may wait for already-admitted turns. + * A wedged admission must not hold the bot-wide mutation gate forever. + */ + mutationAcquireTimeoutMs?: number; } export interface IdleWorkerSweepResult { @@ -80,3 +87,28 @@ export function sweepIdleWorkers( } return suspended; } + +/** + * Run a cap sweep only after every already-admitted inbound turn has either + * durably accepted its input or finished. A worker spawn can put the bot over + * cap while another handler is paused in sender/reaction setup, before that + * handler has changed its screen status or dispatch ledger. A synchronous + * sweep could otherwise mistake that handler's worker for idle, suspend it, + * and make the subsequent send fail before acceptance. + * + * Spawn/idle callbacks commonly run inside an admission, so the mutation gate + * upgrades the current lease when possible. Otherwise acquisition is bounded: + * a wedged admission skips this sweep instead of freezing every later turn for + * the bot. The next idle/spawn callback retries. + */ +export function sweepIdleWorkersAfterTurnDrain( + larkAppId: string, + activeSessions: Map, + opts: IdleWorkerSweepOptions = {}, +): Promise { + return tryWithBotTurnMutation( + larkAppId, + opts.mutationAcquireTimeoutMs ?? IDLE_WORKER_SWEEP_MUTATION_ACQUIRE_TIMEOUT_MS, + () => sweepIdleWorkers(activeSessions, opts), + ).then(result => result.acquired ? result.value : []); +} diff --git a/src/core/inflight-input-tracker.ts b/src/core/inflight-input-tracker.ts index 8eaa611c5..b578c3e56 100644 --- a/src/core/inflight-input-tracker.ts +++ b/src/core/inflight-input-tracker.ts @@ -26,7 +26,10 @@ export type InflightItem = { content: string; logicalContent?: string; turnId?: string; + replyTurnId?: string; dispatchAttempt?: number; + codexAppDispatchId?: string; + queuedActivationToken?: string; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; codexAppInput?: CodexAppTurnInput; }; @@ -40,6 +43,14 @@ export class InflightInputTracker { this.unacked.push(item); } + /** Remove one exact item after a definitive local write failure before the + * caller re-queues it. Covers both the live in-flight set and a synchronous + * backend-exit handoff that may already have moved it to carryOver. */ + forget(item: InflightItem): void { + this.unacked = this.unacked.filter(candidate => candidate !== item); + this.carryOver = this.carryOver.filter(candidate => candidate !== item); + } + /** CLI is back at its idle prompt — everything written has been consumed * (answered, steered into the active turn, or drained from the TUI's own * type-ahead queue). Nothing is in flight anymore. */ diff --git a/src/core/pending-repo-journal.ts b/src/core/pending-repo-journal.ts new file mode 100644 index 000000000..740f210c2 --- /dev/null +++ b/src/core/pending-repo-journal.ts @@ -0,0 +1,88 @@ +import type { DaemonSession } from './types.js'; +import type { PendingRepoSetup } from '../types.js'; +import * as sessionStore from '../services/session-store.js'; + +export function stagePendingRepoSetup( + ds: DaemonSession, + args: Pick & Partial>, +): void { + const prior = { + queued: ds.session.queued, + queuedPrompt: ds.session.queuedPrompt, + queuedCodexAppText: ds.session.queuedCodexAppText, + queuedCodexAppMessageContext: ds.session.queuedCodexAppMessageContext, + pendingRepoSetup: ds.session.pendingRepoSetup, + }; + const setup: PendingRepoSetup = { + mode: args.mode, + prompt: ds.pendingPrompt ?? '', + ...(ds.pendingRawInput ? { rawInput: ds.pendingRawInput } : {}), + ...(args.turnId ? { turnId: args.turnId } : {}), + ...(args.baseDir ? { baseDir: args.baseDir } : {}), + ...(ds.pendingCodexAppText !== undefined ? { codexAppText: ds.pendingCodexAppText } : {}), + ...(ds.pendingCodexAppApplicationContext + ? { codexAppApplicationContext: ds.pendingCodexAppApplicationContext } + : {}), + ...(ds.pendingCodexAppMessageContext + ? { codexAppMessageContext: ds.pendingCodexAppMessageContext } + : {}), + ...(ds.pendingAttachments?.length + ? { attachments: ds.pendingAttachments.map(attachment => ({ ...attachment })) } + : {}), + ...(ds.pendingMentions?.length + ? { mentions: ds.pendingMentions.map(mention => ({ ...mention })) } + : {}), + ...(ds.pendingSubstituteTrigger + ? { substituteTrigger: structuredClone(ds.pendingSubstituteTrigger) } + : {}), + ...(ds.pendingSender ? { sender: { ...ds.pendingSender } } : {}), + }; + ds.session.queued = true; + ds.session.queuedPrompt = setup.prompt; + ds.session.queuedCodexAppText = setup.codexAppText; + ds.session.queuedCodexAppMessageContext = setup.codexAppMessageContext; + ds.session.pendingRepoSetup = setup; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.queued = prior.queued; + ds.session.queuedPrompt = prior.queuedPrompt; + ds.session.queuedCodexAppText = prior.queuedCodexAppText; + ds.session.queuedCodexAppMessageContext = prior.queuedCodexAppMessageContext; + ds.session.pendingRepoSetup = prior.pendingRepoSetup; + throw err; + } +} + +export function persistPendingRepoCardMessageId(ds: DaemonSession, messageId: string): void { + const setup = ds.session.pendingRepoSetup; + if (!setup) return; + const prior = setup.repoCardMessageId; + setup.repoCardMessageId = messageId; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + setup.repoCardMessageId = prior; + throw err; + } +} + +export function restorePendingRepoRuntime(ds: DaemonSession): boolean { + const setup = ds.session.pendingRepoSetup; + if (!setup || ds.session.queuedActivationPending) return false; + ds.pendingRepo = true; + ds.pendingPrompt = setup.prompt; + ds.pendingRawInput = setup.rawInput; + ds.pendingCodexAppText = setup.codexAppText; + ds.pendingCodexAppApplicationContext = setup.codexAppApplicationContext; + ds.pendingCodexAppMessageContext = setup.codexAppMessageContext; + ds.pendingAttachments = setup.attachments?.map(attachment => ({ ...attachment })); + ds.pendingMentions = setup.mentions?.map(mention => ({ ...mention })); + ds.pendingSubstituteTrigger = setup.substituteTrigger + ? structuredClone(setup.substituteTrigger) + : undefined; + ds.pendingSender = setup.sender ? { ...setup.sender } : undefined; + ds.repoCardMessageId = setup.repoCardMessageId; + ds.initialStartPending = false; + return true; +} diff --git a/src/core/raw-command-writer.ts b/src/core/raw-command-writer.ts new file mode 100644 index 000000000..a48c95e38 --- /dev/null +++ b/src/core/raw-command-writer.ts @@ -0,0 +1,75 @@ +/** Minimal write surface shared by PTY/session backends for literal slash + * commands. Historical backends return void on success; guarded backends + * return false when the target pane/process is already unavailable. */ +export interface RawCommandWriter { + write(data: string): void | boolean; + sendText?: (text: string) => void | boolean; + sendSpecialKeys?: (...keys: string[]) => void | boolean; +} + +export interface RawCommandWriteOptions { + coco?: boolean; + cocoThrottleMs?: number; + submitBeatMs?: number; + delay?: (ms: number) => Promise; +} + +/** Type one literal input line and report whether both the text and submit + * writes were accepted by the backend. `undefined` remains legacy success; + * explicit false is a proven drop and must never produce an activation ACK. */ +export async function writeRawCommandLine( + backend: RawCommandWriter, + content: string, + opts: RawCommandWriteOptions = {}, +): Promise { + const delay = opts.delay ?? (ms => new Promise(resolve => setTimeout(resolve, ms))); + const beatMs = opts.submitBeatMs ?? 200; + const sendText = backend.sendText?.bind(backend); + const sendSpecialKeys = backend.sendSpecialKeys?.bind(backend); + + if (sendText && sendSpecialKeys) { + if (opts.coco) { + const cmd = content.trim(); + const typed = cmd.includes(' ') ? cmd : `${cmd} `; + for (const ch of typed) { + if (sendText(ch) === false) return false; + await delay(opts.cocoThrottleMs ?? 40); + } + await delay(beatMs); + return sendSpecialKeys('Enter') !== false; + } + if (sendText(content) === false) return false; + await delay(beatMs); + return sendSpecialKeys('Enter') !== false; + } + + if (backend.write(content) === false) return false; + await delay(beatMs); + return backend.write('\r') !== false; +} + +export interface RawCommandDeliveryFinalizer { + accepted: boolean; + durableActivation: boolean; + acknowledgeActivation: boolean; + hasFollowUp: boolean; + onAccepted: () => void; + onFollowUp: () => void; + onActivationAck: () => void; + onDurableFailure: () => void; +} + +/** Apply the raw-input side effects at the acceptance boundary. A rejected + * text/Enter write can neither enqueue the follower nor ACK the durable head; + * it must retire the worker generation so daemon exit recovery keeps the exact + * journal routable. */ +export function finalizeRawCommandDelivery(args: RawCommandDeliveryFinalizer): boolean { + if (!args.accepted) { + if (args.durableActivation) args.onDurableFailure(); + return false; + } + args.onAccepted(); + if (args.hasFollowUp) args.onFollowUp(); + if (args.acknowledgeActivation) args.onActivationAck(); + return true; +} diff --git a/src/core/reply-target.ts b/src/core/reply-target.ts index c7051263f..2535d1d11 100644 --- a/src/core/reply-target.ts +++ b/src/core/reply-target.ts @@ -1,5 +1,5 @@ import type { DaemonSession } from './types.js'; -import type { Session } from '../types.js'; +import type { FrozenSessionReplyContext, Session } from '../types.js'; export type SessionReplyTarget = | { mode: 'plain'; chatId: string } @@ -114,6 +114,34 @@ export function beginReplyTargetTurn( nowIso = new Date().toISOString(), opts?: { quoteOnly?: boolean; substitute?: boolean }, ): void { + const exactTarget: SessionReplyTarget = ds.scope === 'chat' + ? replyRootId + ? opts?.quoteOnly + ? { mode: 'quote', rootMessageId: replyRootId } + : { mode: 'thread', rootMessageId: replyRootId } + : { mode: 'plain', chatId: ds.chatId } + : { mode: 'thread', rootMessageId: ds.session.rootMessageId }; + const exactContexts = { ...(ds.session.turnReplyContexts ?? {}) }; + // Re-insertion keeps the newest turn at the end for deterministic bounding. + delete exactContexts[turnId]; + exactContexts[turnId] = { + target: exactTarget, + ...(ds.session.quoteTargetId ? { quoteTargetId: ds.session.quoteTargetId } : {}), + ...(ds.session.quoteTargetSenderOpenId + ? { replyTargetSenderOpenId: ds.session.quoteTargetSenderOpenId } + : {}), + ...(ds.session.quoteTargetSenderIsBot !== undefined + ? { replyTargetSenderIsBot: ds.session.quoteTargetSenderIsBot } + : {}), + }; + const overflow = Object.keys(exactContexts).length - 256; + if (overflow > 0) { + for (const staleTurnId of Object.keys(exactContexts).slice(0, overflow)) { + delete exactContexts[staleTurnId]; + } + } + ds.session.turnReplyContexts = exactContexts; + if (ds.scope !== 'chat') return; if (replyRootId) { const aliases = { ...(ds.replyThreadAliases ?? ds.session.replyThreadAliases ?? {}) }; @@ -146,6 +174,16 @@ export function beginReplyTargetTurn( ds.session.currentReplyTarget = undefined; } +/** Resolve a turn's immutable inbound destination, falling back only for + * legacy/non-Lark turns that predate the bounded registry. */ +export function frozenReplyContextForTurn( + ds: Pick, + turnId?: string, +): FrozenSessionReplyContext { + const frozen = turnId ? ds.session.turnReplyContexts?.[turnId] : undefined; + return frozen ?? { target: resolveSessionReplyTarget(ds, turnId) }; +} + /** * Effective turnId for a daemon-side message. Callers that know their turn * (worker final_output, placeholder cards) pass it explicitly and the diff --git a/src/core/resource-monitor/runtime.ts b/src/core/resource-monitor/runtime.ts index b13bf68a2..6bb62a23a 100644 --- a/src/core/resource-monitor/runtime.ts +++ b/src/core/resource-monitor/runtime.ts @@ -23,7 +23,10 @@ export interface RuntimeBotInput { daemonStatus?: RuntimeDaemonStatus; } -const WORKING_STATUSES = new Set(['working', 'analyzing', 'active']); +// `stalled` means no observable progress, not completion: the CLI/model/tool +// may still be executing. Keep it in active runtime pressure rather than +// misclassifying it as idle, human-waiting, or unknown. +const WORKING_STATUSES = new Set(['working', 'analyzing', 'active', 'stalled']); const STARTING_STATUSES = new Set(['starting', 'queued']); const IDLE_STATUSES = new Set(['idle', 'dormant']); diff --git a/src/core/session-manager.ts b/src/core/session-manager.ts index 29d4b4deb..6b9104eda 100644 --- a/src/core/session-manager.ts +++ b/src/core/session-manager.ts @@ -12,7 +12,7 @@ import * as sessionStore from '../services/session-store.js'; import * as messageQueue from '../services/message-queue.js'; import { downloadMessageResource, listChatBotMembers, UserTokenMissingError } from '../im/lark/client.js'; import { logger } from '../utils/logger.js'; -import { forkWorker, sendWorkerInput, forkAdoptWorker, adoptSandboxBlocked, killStalePids, getCurrentCliVersion, restoreUsageLimitRuntimeState, setActiveSessionSafe, isRelayableRealSession, closeSession, getActiveSessionsRegistry, suspendWorker } from './worker-pool.js'; +import { forkWorker, sendWorkerInput, promoteQueuedActivationTail, forkAdoptWorker, adoptSandboxBlocked, killStalePids, getCurrentCliVersion, restoreUsageLimitRuntimeState, setActiveSessionSafe, isRelayableRealSession, closeSession, getActiveSessionsRegistry, suspendWorker, withActiveSessionKeyLock } from './worker-pool.js'; import { createCliAdapterSync } from '../adapters/cli/registry.js'; import { buildBotmuxShellHints } from '../adapters/cli/shared-hints.js'; import { @@ -46,10 +46,18 @@ import { validateZellijAdoptTarget } from './zellij-adopt-discovery.js'; import type { BackendType } from '../adapters/backend/types.js'; import type { CliTurnPayload, CodexAppAdditionalContextEntry, CodexAppTurnInput, LarkAttachment, LarkMention, ScheduledTask, SubstituteTrigger } from '../types.js'; import { addCodexAppContext } from '../utils/codex-app-context.js'; +import { hasUnsettledCodexAppDispatch } from '../utils/codex-app-dispatch-ledger.js'; +import { hasProtectedSessionMutationOwnership } from './session-mutation-guard.js'; import type { MessageResource } from '../im/lark/message-parser.js'; import type { ResolvedSender } from '../im/lark/identity-cache.js'; -import { activeSessionKey, sessionKey, sessionAnchorId, storedSessionAnchorId } from './types.js'; +import { + activeSessionKey, + sessionKey, + sessionAnchorId, + storedSessionAnchorId, +} from './types.js'; import type { DaemonSession } from './types.js'; +import { stagePendingRepoSetup, persistPendingRepoCardMessageId, restorePendingRepoRuntime } from './pending-repo-journal.js'; import { announceSessionRow, markSessionActivity, announcePendingRepoSession } from './session-activity.js'; import { scanMultipleProjects } from '../services/project-scanner.js'; import { buildRepoSelectCard } from '../im/lark/card-builder.js'; @@ -76,6 +84,114 @@ function sessionLastMessageAtMs(session: { createdAt?: string; lastMessageAt?: s return session.lastMessageAt ? (Date.parse(session.lastMessageAt) || sessionCreatedAtMs(session)) : sessionCreatedAtMs(session); } +async function resumeRestoredPendingRepoSetup( + ds: DaemonSession, + activeSessions: Map, +): Promise { + const setup = ds.session.pendingRepoSetup; + if (!setup || ds.session.queuedActivationPending || !ds.pendingRepo) return; + const anchor = sessionAnchorId(ds); + const notify = async (content: string): Promise => { + const { sendMessage, replyMessage } = await import('../im/lark/client.js'); + return ds.scope === 'thread' + ? replyMessage(ds.larkAppId, anchor, content, 'text', true) + : sendMessage(ds.larkAppId, ds.chatId, content, 'text'); + }; + + if (setup.mode === 'auto_worktree' && setup.baseDir) { + const { runAutoWorktreeCommit } = await import('../im/lark/card-handler.js'); + void runAutoWorktreeCommit({ + ds, + anchor, + larkAppId: ds.larkAppId, + baseDir: setup.baseDir, + title: ds.session.title, + prompt: setup.prompt, + operatorOpenId: ds.session.ownerOpenId, + activeSessions, + notify, + }).catch((err) => { + // Git/worktree recovery is deliberately detached. A failed publish or + // build may not reject daemon startup or erase this durable setup owner. + restorePendingRepoRuntime(ds); + logger.error( + `[${ds.session.sessionId.substring(0, 8)}] Pending auto-worktree recovery failed; ` + + `durable setup retained for retry: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + logger.info(`[${ds.session.sessionId.substring(0, 8)}] Restarted durable pending auto-worktree setup`); + return; + } + + // Always publish a fresh picker after restart: a persisted message id proves + // identity, not that the Lark message still exists. Persist the replacement + // before best-effort withdrawal so exactly one card remains authoritative. + const oldCardMessageId = setup.repoCardMessageId; + const scanDirs = getProjectScanDirs(ds).filter(dir => existsSync(dir)); + const projects = scanDirs.length > 0 + ? scanMultipleProjects(scanDirs, 3, repoPickerScanOptions()) + : []; + if (projects.length > 0) { + const bot = getBot(ds.larkAppId); + const card = buildRepoSelectCard( + projects, + getSessionWorkingDir(ds), + anchor, + localeForBot(ds.larkAppId), + bot.config.worktreeMultiPicker, + ); + const { sendMessage, replyMessage, deleteMessage } = await import('../im/lark/client.js'); + const messageId = ds.scope === 'thread' + ? await replyMessage(ds.larkAppId, anchor, card, 'interactive', true) + : await sendMessage(ds.larkAppId, ds.chatId, card, 'interactive'); + ds.repoCardMessageId = messageId; + persistPendingRepoCardMessageId(ds, messageId); + if (oldCardMessageId && oldCardMessageId !== messageId) { + void deleteMessage(ds.larkAppId, oldCardMessageId).catch(() => {}); + } + announcePendingRepoSession(ds); + logger.info(`[${ds.session.sessionId.substring(0, 8)}] Re-posted durable pending repo picker`); + return; + } + + // The original pre-card scan may have crashed before learning there was no + // selectable repo. Resume the same no-project fallback without replacing N. + ds.pendingRepo = false; + ensureSessionWhiteboard(ds); + if (setup.rawInput) { + rememberLastCliInput(ds, setup.rawInput, setup.rawInput); + forkWorker(ds, '', { resume: false, turnId: setup.turnId }); + } else { + const bot = getBot(ds.larkAppId); + const availableBots = await getAvailableBots(ds.larkAppId, ds.chatId); + const input = buildNewTopicCliInput( + setup.prompt, + ds.session.sessionId, + ds.session.cliId ?? bot.config.cliId, + ds.session.cliPathOverride ?? bot.config.cliPathOverride, + ds.pendingAttachments, + ds.pendingMentions, + availableBots, + undefined, + { name: bot.botName, openId: bot.botOpenId }, + localeForBot(ds.larkAppId), + ds.pendingSender, + { + larkAppId: ds.larkAppId, + chatId: ds.chatId, + whiteboardId: ds.session.whiteboardId, + substituteTrigger: ds.pendingSubstituteTrigger, + codexAppText: ds.pendingCodexAppText, + codexAppApplicationContext: ds.pendingCodexAppApplicationContext, + codexAppMessageContext: ds.pendingCodexAppMessageContext, + }, + ); + rememberLastCliInput(ds, setup.prompt, input); + forkWorker(ds, input, { resume: false, turnId: setup.turnId }); + } + logger.info(`[${ds.session.sessionId.substring(0, 8)}] Resumed durable pending repo opening without selectable projects`); +} + function sameUsageLimit(a: DaemonSession['usageLimit'], b: DaemonSession['usageLimit']): boolean { if (!a && !b) return true; if (!a || !b) return false; @@ -108,6 +224,23 @@ async function closeActiveSessionIfCliMismatch(ds: DaemonSession): Promise void, batchSize: number = RECOVERY_FORK_BATCH_SIZE, delayMs: number = RECOVERY_FORK_DELAY_MS, + stillOwned: (ds: DaemonSession) => boolean = ds => ds.session.status === 'active', ): Promise { let spawnedInBatch = 0; for (const ds of sessions) { - if (ds.worker) continue; // already woken by a real message — don't clobber it - fork(ds); + // The batch delay is a lifecycle boundary: close/replace can remove this + // exact object while we sleep. Never resurrect a closed or orphaned ds. + if (ds.worker || ds.session.status !== 'active' || !stillOwned(ds)) continue; + try { + fork(ds); + } catch (err) { + // One malformed/stale pane or synchronous init-IPC failure must not + // abort recovery for every later durable owner. forkWorker compensates + // its own pre-init journal mutations; isolate this row and continue. + logger.error( + `[${ds.session.sessionId.substring(0, 8)}] Recovery fork failed; retained for later retry: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + continue; + } if (++spawnedInBatch >= batchSize) { spawnedInBatch = 0; if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs)); @@ -1204,6 +1359,7 @@ export async function restoreActiveSessions(activeSessions: Map 0, workingDir: session.workingDir, ownerOpenId: session.ownerOpenId, // Restore persisted streaming-card state — next screen_update will PATCH @@ -1404,15 +1650,50 @@ export async function restoreActiveSessions(activeSessions: Map 0 + && !promoteQueuedActivationTail(ds, { send: false })) { + throw new Error(`failed to promote durable activation tail for ${session.sessionId}`); + } const anchor = sessionAnchorId(ds); messageQueue.ensureQueue(anchor); if (ds.usageLimit) restoreUsageLimitRuntimeState(ds); // Same-key collision guard — see adopt-branch comment above. - await setActiveSessionSafe(activeSessions, activeSessionKey(ds), ds); + const registration = await setActiveSessionSafe(activeSessions, activeSessionKey(ds), ds); + if (!registration.accepted) { + if (registration.reason === 'both_pending' || registration.reason === 'cleanup_failed') { + logger.error( + `[${session.sessionId.substring(0, 8)}] Isolated active restore collision; ` + + `durable row/pane retained without aborting daemon startup: ${registration.reason === 'both_pending' + ? `two protected owners at ${activeSessionKey(ds)}` + : `cleanup failed for ${registration.cleanupSessionId}: ${registration.error}`}`, + ); + continue; + } + logger.warn(`[${session.sessionId.substring(0, 8)}] restore collision lost to unsettled session ${registration.keptSessionId.substring(0, 8)}`); + continue; + } announceSessionRow(ds); logger.debug(`Registered session ${session.sessionId} (scope: ${scope}, anchor: ${anchor})`); + } catch (err) { + // Restore is a per-row reconciliation job. A malformed legacy hybrid or + // one transient persistence failure must retain that row for inspection + // and retry without preventing every later healthy session from loading. + logger.error( + `[${session.sessionId.substring(0, 8)}] Isolated session restore failure; ` + + `durable row retained and daemon startup continuing: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + continue; + } } // Persistent backends: auto-fork workers for sessions whose backing session @@ -1440,7 +1721,13 @@ export async function restoreActiveSessions(activeSessions: Map forkWorker(ds, '', true)); + await staggeredRecoveryFork( + toReattach, + (ds) => { + const recoverExactNonCodex = ds.session.queuedActivationPending + && ds.session.cliId !== 'codex-app' + && ds.session.queuedActivationInput; + forkWorker( + ds, + recoverExactNonCodex || '', + recoverExactNonCodex + ? { + resume: ds.session.queuedActivationResume ?? ds.hasHistory, + turnId: ds.session.queuedActivationTurnId, + dispatchAttempt: ds.session.queuedActivationDispatchAttempt, + } + : true, + ); + }, + RECOVERY_FORK_BATCH_SIZE, + RECOVERY_FORK_DELAY_MS, + ds => activeSessions.get(activeSessionKey(ds)) === ds, + ); const hasPersistentBackend = [...activeSessions.values()].some(ds => !!getSessionPersistentBackendType(ds)); logger.info(`Restored ${active.length} session(s)${hasPersistentBackend ? '' : ', waiting for messages to resume'}`); @@ -1587,7 +1913,7 @@ export async function resumeSession( activeSessions: Map, ): Promise<{ ok: true; ds: DaemonSession } | { ok: false; error: 'not_found' | 'not_closed' | 'anchor_occupied' | 'adopt_unsupported' | 'vc_receiver_managed' | 'deferred_unmaterialized'; activeSessionId?: string }> { - const session = sessionStore.getSession(sessionId); + let session = sessionStore.getSession(sessionId); if (!session) return { ok: false, error: 'not_found' }; if (session.status !== 'closed') return { ok: false, error: 'not_closed' }; @@ -1624,6 +1950,22 @@ export async function resumeSession( const anchor = storedSessionAnchorId(session); const key = sessionKey(anchor, larkAppId); + return withActiveSessionKeyLock(activeSessions, key, async () => { + // The resume request may have waited behind a fresh creator. Re-read the + // durable row and keep the already-active creator as first owner. + const latest = sessionStore.getSession(sessionId); + if (!latest) return { ok: false as const, error: 'not_found' as const }; + if (latest.status !== 'closed') return { ok: false as const, error: 'not_closed' as const }; + if (latest.vcMeetingReceiver) return { ok: false as const, error: 'vc_receiver_managed' as const }; + if (latest.deferredScheduleRun + && !readDeferredTopicBinding(config.session.dataDir, latest.sessionId)) { + return { ok: false as const, error: 'deferred_unmaterialized' as const }; + } + if (latest.title?.startsWith('Adopt:') || latest.adoptedFrom) { + return { ok: false as const, error: 'adopt_unsupported' as const }; + } + session = latest; + // In-memory occupant check. A daemon-command scratch (e.g. an unconfirmed // `/relay` picker, a bare `/help`) parks a worker:null placeholder at this // anchor; daemon.ts creates one for ANY DAEMON_COMMAND in a session-less @@ -1639,7 +1981,11 @@ export async function resumeSession( // throwaway command container, so clobbering it would lose real intent. const existing = activeSessions.get(key); if (existing) { - if (isRelayableRealSession(existing) || existing.pendingRepo) { + if (hasProtectedSessionMutationOwnership(existing) + || isRelayableRealSession(existing) + || existing.pendingRepo + || existing.initialStartPending + || existing.session.queued) { return { ok: false, error: 'anchor_occupied', activeSessionId: existing.session.sessionId }; } await closeSession(existing.session.sessionId); @@ -1655,9 +2001,10 @@ export async function resumeSession( // store-level conflict invisible to the Map check above. // // Same scratch carve-out applies on disk: a persisted scratch has neither - // `cliId` nor `lastCliInput` (those are only written once a real CLI ran). - // A real conflict (either marker present) still blocks; scratch-only - // conflicts get closed so they stop occupying the anchor on disk. + // `cliId` / `lastCliInput` (those are only written once a real backend ran). + // A real conflict (any marker present) still + // blocks; scratch-only conflicts get closed so they stop occupying the + // anchor on disk. // // CAVEAT — this path canNOT honor the pendingRepo carve-out the in-memory // branch above applies: `pendingRepo` is a runtime DaemonSession flag that @@ -1673,11 +2020,18 @@ export async function resumeSession( const conflicts = sessionStore.listSessions().filter(s => s.sessionId !== sessionId && s.status === 'active' - && (s.larkAppId ?? '') === larkAppId + // Legacy active rows predate per-bot stamping. Conservatively attribute an + // unscoped row to this daemon, matching transfer/restore collision rules; + // otherwise a disk-only legacy anchor can be ghosted by the resumed row. + && (!s.larkAppId || s.larkAppId === larkAppId) && (s.scope === 'chat' ? 'chat' : 'thread') === scope && storedSessionAnchorId(s) === anchor, ); - const realConflict = conflicts.find(s => !!s.cliId || !!s.lastCliInput); + const realConflict = conflicts.find(s => + hasProtectedSessionMutationOwnership(s) + || !!s.cliId + || !!s.lastCliInput, + ); if (realConflict) { return { ok: false, error: 'anchor_occupied', activeSessionId: realConflict.sessionId }; } @@ -1685,12 +2039,25 @@ export async function resumeSession( await closeSession(scratch.sessionId); } - // Reactivate in store — clear closedAt so dashboard rows don't keep showing - // the stale close timestamp on the now-active session. - session.status = 'active'; - session.closedAt = undefined; - session.lastMessageAt = new Date().toISOString(); - sessionStore.updateSession(session); + // Scratch cleanup and disk scans can await. A creator that does not yet use + // the shared key lock still wins if it published meanwhile; never reactivate + // the closed row and then replace that fresh owner. + const lateExisting = activeSessions.get(key); + if (lateExisting) { + return { + ok: false, + error: 'anchor_occupied', + activeSessionId: lateExisting.session.sessionId, + }; + } + + // Reactivate and abandon any queued/setup ownership in one durable replace. + // Closed rows created by older releases may still contain a prepared head, + // tail, or repo picker; generic resume starts a new lifecycle and must never + // replay those abandoned inputs. + const reactivated = sessionStore.reactivateClosedSession(sessionId); + if (!reactivated.ok) return reactivated; + session = reactivated.session; const now = Date.now(); const ds: DaemonSession = { @@ -1722,12 +2089,13 @@ export async function resumeSession( }; messageQueue.ensureQueue(anchor); - // setActiveSessionSafe over a bare Map.set: the scratch-eviction above - // should already have freed `key`, but if any occupant remains it closes - // it rather than silently orphaning it (consistent with restore/transfer). - await setActiveSessionSafe(activeSessions, key, ds); + // Live creation is first-owner-wins. Restore-time setActiveSessionSafe may + // replace disposable historical collisions, but a user resume must never + // close a newly-created worker/reservation at this anchor. + activeSessions.set(key, ds); logger.info(`Resumed session ${sessionId.substring(0, 8)} (scope: ${scope}, anchor: ${anchor.substring(0, 12)})`); return { ok: true, ds }; + }); } // ─── Scheduled task execution ──────────────────────────────────────────────── @@ -1978,102 +2346,156 @@ export async function executeScheduledTask( refreshCliVersion(bot.config.cliId, bot.config.cliPathOverride); - // Silent fires flip the model's default from "post progress via botmux send" - // to "say nothing unless the alert condition is met". const firePrompt = silent ? `${buildSilentScheduleHint(task.name, localeForBot(larkAppId))}\n\n${task.prompt}` : task.prompt; - // Inject into a live session if one already exists at this anchor. - const existing = activeSessions.get(sessionKey(anchor, larkAppId)); - if (isContinuation && existing?.worker && !existing.worker.killed) { - markSessionActivity(existing); - try { - ensureSessionWhiteboard(existing); - if (sharedTopicRootId) { - beginReplyTargetTurn(existing, sharedTopicRootId, scheduledTurnId); - sessionStore.updateSession(existing.session); + const key = sessionKey(anchor, larkAppId); + return withActiveSessionKeyLock(activeSessions, key, async () => { + // Reuse the canonical owner only when it is an actual conversation. A + // worker:null entry is also used for three deliberately non-runnable + // states: first-turn setup, repo selection, and dashboard backlog. Waking + // any of those here would let the scheduled prompt overtake (or replace) + // the opening prompt that owns the reservation. + const existing = activeSessions.get(key); + if (existing) { + const reservedState = existing.pendingRepo + ? 'pending_repo' + : existing.initialStartPending + ? 'initial_start_pending' + : existing.worktreeCreating + ? 'worktree_creating' + : existing.session.queued + ? 'queued_backlog' + : undefined; + if (reservedState) { + throw new Error( + `Scheduled task ${task.id} found owner ${existing.session.sessionId} ` + + `in ${reservedState}; preserving its opening prompt`, + ); } - const input = buildFollowUpCliInput(firePrompt, existing.session.sessionId, { - isAdoptMode: false, - cliId: existing.session.cliId ?? bot.config.cliId, - cliPathOverride: existing.session.cliPathOverride ?? bot.config.cliPathOverride, - locale: localeForBot(larkAppId), - larkAppId, - chatId: task.chatId, - whiteboardId: existing.session.whiteboardId, - }); - rememberLastCliInput(existing, task.prompt, input); - if (silent) armSilentScheduledTurn(existing, scheduledTurnId); - if (!sendWorkerInput(existing, input, scheduledTurnId)) { - if (silent) disarmSilentScheduledTurn(existing, scheduledTurnId); - throw new Error('worker unavailable'); + if (hasProtectedSessionMutationOwnership(existing)) { + throw new Error( + `Scheduled task ${task.id} found durable activation owner ` + + `${existing.session.sessionId}; preserving it for recovery`, + ); + } + + if (isRelayableRealSession(existing)) { + markSessionActivity(existing); + ensureSessionWhiteboard(existing); + if (sharedTopicRootId) { + beginReplyTargetTurn(existing, sharedTopicRootId, scheduledTurnId); + sessionStore.updateSession(existing.session); + } + const input = buildFollowUpCliInput(firePrompt, existing.session.sessionId, { + isAdoptMode: false, + cliId: existing.session.cliId ?? bot.config.cliId, + cliPathOverride: existing.session.cliPathOverride ?? bot.config.cliPathOverride, + locale: localeForBot(larkAppId), + larkAppId, + chatId: task.chatId, + whiteboardId: existing.session.whiteboardId, + }); + if (existing.worker && !existing.worker.killed) { + if (silent) armSilentScheduledTurn(existing, scheduledTurnId); + if (!sendWorkerInput(existing, input, scheduledTurnId)) { + if (silent) disarmSilentScheduledTurn(existing, scheduledTurnId); + throw new Error( + `Scheduled task ${task.id} worker refused input before acceptance`, + ); + } + rememberLastCliInput(existing, task.prompt, input); + } else { + rememberLastCliInput(existing, task.prompt, input); + if (silent) armSilentScheduledTurn(existing, scheduledTurnId); + try { + forkWorker(existing, input, { resume: existing.hasHistory, turnId: scheduledTurnId }); + } catch (err) { + if (silent) disarmSilentScheduledTurn(existing, scheduledTurnId); + throw err; + } + } + logger.info(`[scheduler] Task "${task.name}" routed to existing session ${existing.session.sessionId}`); + return; + } + + // Daemon-command scratches have no CLI history and no pending user + // intent. Retire the exact scratch before creating the scheduled + // conversation; never fork the scratch as if it were resumable. + await closeSession(existing.session.sessionId); + if (activeSessions.get(key) === existing) activeSessions.delete(key); + const successor = activeSessions.get(key); + if (successor) { + throw new Error( + `Scheduled task ${task.id} lost anchor ${key} to ${successor.session.sessionId} ` + + 'while retiring a scratch owner', + ); } - logger.info(`[scheduler] Task "${task.name}" injected into live session ${existing.session.sessionId}${silent ? ' (silent)' : ''}`); - return; - } catch (err: any) { - if (silent) disarmSilentScheduledTurn(existing, scheduledTurnId); - logger.warn(`[scheduler] Failed to inject into live session (${err.message}); spawning fresh worker`); } - } - // Spawn a fresh session bound to the chosen anchor. + // Spawn a fresh session bound to the chosen anchor. // Thread-scope: rootMessageId = anchor. Chat-scope: rootMessageId stores the - // chatId-as-seed for audit (sessionAnchorId() returns chatId via scope). If a - // formerly chat-scope task was redirected into a converted topic chat, promote - // the runtime session to thread-scope so follow-up replies stay in-thread. - const deferredFreshTopic = executionPosition === 'new-topic' && silent; - const runtimeScope: 'thread' | 'chat' = deferredFreshTopic - ? 'chat' - : scope === 'chat' && anchor !== task.chatId ? 'thread' : scope; - const session = sessionStore.createSession(task.chatId, anchor, `${t('schedule.title_prefix', undefined, localeForBot(larkAppId))} ${task.name}`, task.chatType === 'p2p' ? 'p2p' : 'group'); - const now = Date.now(); - session.larkAppId = larkAppId; - session.scope = runtimeScope; - if (deferredFreshTopic) { - session.deferredScheduleRun = { - taskId: task.id, - turnId: scheduledTurnId, - routingAnchor: anchor, - ...(task.topicTitle?.trim() ? { topicTitle: task.topicTitle.trim() } : {}), - createdAt: new Date(now).toISOString(), - }; - } - session.lastMessageAt = new Date(now).toISOString(); - sessionStore.updateSession(session); - messageQueue.ensureQueue(anchor); + // chatId-as-seed for audit (sessionAnchorId() returns chatId via scope). If a + // formerly chat-scope task was redirected into a converted topic chat, promote + // the runtime session to thread-scope so follow-up replies stay in-thread. + const deferredFreshTopic = executionPosition === 'new-topic' && silent; + const runtimeScope: 'thread' | 'chat' = deferredFreshTopic + ? 'chat' + : scope === 'chat' && anchor !== task.chatId ? 'thread' : scope; + const session = sessionStore.createSession(task.chatId, anchor, `${t('schedule.title_prefix', undefined, localeForBot(larkAppId))} ${task.name}`, task.chatType === 'p2p' ? 'p2p' : 'group'); + const now = Date.now(); + session.larkAppId = larkAppId; + session.scope = runtimeScope; + if (deferredFreshTopic) { + session.deferredScheduleRun = { + taskId: task.id, + turnId: scheduledTurnId, + routingAnchor: anchor, + ...(task.topicTitle?.trim() ? { topicTitle: task.topicTitle.trim() } : {}), + createdAt: new Date(now).toISOString(), + }; + } + session.lastMessageAt = new Date(now).toISOString(); + sessionStore.updateSession(session); + messageQueue.ensureQueue(anchor); - const ds: DaemonSession = { - session, - worker: null, - workerPort: null, - workerToken: null, - larkAppId, - chatId: task.chatId, - chatType: task.chatType === 'p2p' ? 'p2p' : 'group', - scope: runtimeScope, - spawnedAt: sessionCreatedAtMs(session), - cliVersion: getCurrentCliVersion(), - lastMessageAt: now, - hasHistory: isContinuation, - workingDir: task.workingDir, - }; - if (sharedTopicRootId) { - beginReplyTargetTurn(ds, sharedTopicRootId, scheduledTurnId); - sessionStore.updateSession(ds.session); - } - ensureSessionWhiteboard(ds); - const prompt = buildNewTopicCliInput(firePrompt, session.sessionId, bot.config.cliId, bot.config.cliPathOverride, undefined, undefined, undefined, undefined, { name: bot.botName, openId: bot.botOpenId }, localeForBot(larkAppId), undefined, { larkAppId, chatId: task.chatId, whiteboardId: ds.session.whiteboardId }); - activeSessions.set(sessionKey(anchor, larkAppId), ds); - rememberLastCliInput(ds, task.prompt, prompt); - if (silent) armSilentScheduledTurn(ds, scheduledTurnId); - try { - forkWorker(ds, prompt, scheduledTurnId); - } catch (err) { - if (silent) disarmSilentScheduledTurn(ds, scheduledTurnId); - throw err; - } + const ds: DaemonSession = { + session, + worker: null, + workerPort: null, + workerToken: null, + larkAppId, + chatId: task.chatId, + chatType: task.chatType === 'p2p' ? 'p2p' : 'group', + scope: runtimeScope, + spawnedAt: sessionCreatedAtMs(session), + cliVersion: getCurrentCliVersion(), + lastMessageAt: now, + hasHistory: isContinuation, + workingDir: task.workingDir, + initialStartPending: true, + pendingPrompt: firePrompt, + }; + if (sharedTopicRootId) { + beginReplyTargetTurn(ds, sharedTopicRootId, scheduledTurnId); + sessionStore.updateSession(ds.session); + } + ensureSessionWhiteboard(ds); + const prompt = buildNewTopicCliInput(firePrompt, session.sessionId, bot.config.cliId, bot.config.cliPathOverride, undefined, undefined, undefined, undefined, { name: bot.botName, openId: bot.botOpenId }, localeForBot(larkAppId), undefined, { larkAppId, chatId: task.chatId, whiteboardId: ds.session.whiteboardId }); + activeSessions.set(key, ds); + rememberLastCliInput(ds, task.prompt, prompt); + if (silent) armSilentScheduledTurn(ds, scheduledTurnId); + try { + forkWorker(ds, prompt, scheduledTurnId); + } catch (err) { + if (silent) disarmSilentScheduledTurn(ds, scheduledTurnId); + throw err; + } + ds.initialStartPending = false; + ds.pendingPrompt = undefined; - logger.info(`[scheduler] Task "${task.name}" spawned (session: ${session.sessionId}, scope: ${runtimeScope}, anchor: ${anchor}, continuation: ${isContinuation}${silent ? ', silent' : ''})`); + logger.info(`[scheduler] Task "${task.name}" spawned (session: ${session.sessionId}, scope: ${runtimeScope}, anchor: ${anchor}, continuation: ${isContinuation}${silent ? ', silent' : ''})`); + }); } // ─── Dashboard「创建会话」spawn / activate ─────────────────────────────────── @@ -2101,7 +2523,10 @@ function resolveDashboardSpawnWorkingDir(larkAppId: string, chatId: string): str * buildRepoSelectCard（含 worktree）。用户点卡片由 card-handler 的 pendingRepo 分支起 CLI。 * - 没钉也没项目 → 回退用 bot 默认 cwd 直接起。 * userContent 是已按角色包装好的首轮内容（lead 前言等），不论哪条路都原样带过去。 */ -async function forkOrShowRepoCard(ds: DaemonSession, userContent: string): Promise { +async function forkOrShowRepoCard( + ds: DaemonSession, + userContent: string, +): Promise<'forked' | 'pending_repo'> { const larkAppId = ds.larkAppId; const bot = getBot(larkAppId); const locale = localeForBot(larkAppId); @@ -2119,6 +2544,14 @@ async function forkOrShowRepoCard(ds: DaemonSession, userContent: string): Promi const baseDir = ds.workingDir; ds.pendingRepo = true; // router buffers concurrent msgs; commit clears it ds.pendingPrompt = userContent; // folded into the first turn by commitRepoSelection + stagePendingRepoSetup(ds, { + mode: 'auto_worktree', + baseDir, + turnId: ds.currentReplyTarget?.turnId ?? ds.session.rootMessageId, + }); + // Route visibility moves atomically from initial-start reservation to + // pendingRepo before the dynamic import can yield. + ds.initialStartPending = false; // (The pending dashboard row is announced inside runAutoWorktreeCommit so all // three spawn callers get it from one place — no publish needed here.) const { runAutoWorktreeCommit } = await import('../im/lark/card-handler.js'); @@ -2130,14 +2563,14 @@ async function forkOrShowRepoCard(ds: DaemonSession, userContent: string): Promi notify: (m) => sendMessage(larkAppId, ds.chatId, m), }); logger.info(`[createSession] session ${ds.session.sessionId.substring(0, 8)} → pending, building worktree off ${baseDir}`); - return; + return 'pending_repo'; } } const buildPrompt = () => buildNewTopicCliInput( userContent, ds.session.sessionId, bot.config.cliId, bot.config.cliPathOverride, - ds.pendingAttachments, undefined, undefined, undefined, - { name: bot.botName, openId: bot.botOpenId }, locale, undefined, + ds.pendingAttachments, ds.pendingMentions, undefined, ds.pendingFollowUps, + { name: bot.botName, openId: bot.botOpenId }, locale, ds.pendingSender, { larkAppId, chatId: ds.chatId, @@ -2145,6 +2578,8 @@ async function forkOrShowRepoCard(ds: DaemonSession, userContent: string): Promi codexAppText: ds.pendingCodexAppText, codexAppApplicationContext: ds.pendingCodexAppApplicationContext, codexAppMessageContext: ds.pendingCodexAppMessageContext, + codexAppFollowUps: ds.pendingCodexAppFollowUps, + codexAppFollowUpContexts: ds.pendingCodexAppFollowUpContexts, }, ); @@ -2153,24 +2588,56 @@ async function forkOrShowRepoCard(ds: DaemonSession, userContent: string): Promi const scanDirs = getProjectScanDirs(ds).filter(d => existsSync(d)); const projects = scanDirs.length > 0 ? scanMultipleProjects(scanDirs, 3, repoPickerScanOptions()) : []; if (projects.length > 0) { + const card = buildRepoSelectCard(projects, getSessionWorkingDir(ds), ds.chatId, locale, bot.config.worktreeMultiPicker); + const { sendMessage } = await import('../im/lark/client.js'); + ds.pendingRepo = true; + ds.pendingPrompt = userContent; try { - const card = buildRepoSelectCard(projects, getSessionWorkingDir(ds), ds.chatId, locale, bot.config.worktreeMultiPicker); - const { sendMessage } = await import('../im/lark/client.js'); - ds.pendingRepo = true; - ds.pendingPrompt = userContent; - ds.repoCardMessageId = await sendMessage(larkAppId, ds.chatId, card, 'interactive'); - announcePendingRepoSession(ds); - // 弹卡片这条路不经 forkWorker，session.spawned 不会自动发——手动 upsert 一条， - // 让 dashboard 显示这条「待选仓库」会话（in_progress 首次 spawn 走这里才会出现）。 - dashboardEventBus.publish({ type: 'session.spawned', body: { session: composeRowFromActive(ds) } }); - logger.info(`[createSession] repo select card posted for session ${ds.session.sessionId.substring(0, 8)} (${projects.length} projects)`); - return; + stagePendingRepoSetup(ds, { + mode: 'picker', + turnId: ds.currentReplyTarget?.turnId ?? ds.session.rootMessageId, + }); } catch (err) { - // 发卡失败：退回直接起，别把会话卡死在 pendingRepo。 + // Nothing external was published. The journal helper rolled its own + // mutation back, so direct fork remains a safe fallback. ds.pendingRepo = false; ds.pendingPrompt = undefined; ds.repoCardMessageId = undefined; - logger.warn(`[createSession] repo card failed (${(err as Error)?.message ?? err}); forking with default cwd`); + logger.warn(`[createSession] repo setup stage failed (${(err as Error)?.message ?? err}); forking with default cwd`); + } + if (ds.pendingRepo) { + let publishedCardId: string | undefined; + try { + publishedCardId = await sendMessage(larkAppId, ds.chatId, card, 'interactive'); + } catch (err) { + // No picker exists, so the already-durable opening may safely move + // into forkWorker's tokened activation journal. + ds.pendingRepo = false; + ds.repoCardMessageId = undefined; + logger.warn(`[createSession] repo card publish failed (${(err as Error)?.message ?? err}); forking with default cwd`); + } + if (publishedCardId) { + ds.repoCardMessageId = publishedCardId; + try { + persistPendingRepoCardMessageId(ds, publishedCardId); + } catch (err) { + // The card is already visible. Keep its runtime identity and the + // durable setup owner fail-closed; restart will publish and persist + // a fresh picker instead of starting the CLI behind this card. + logger.error( + `[createSession] repo card id persistence failed for ${ds.session.sessionId.substring(0, 8)}; ` + + `retaining pending picker ${publishedCardId}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + ds.initialStartPending = false; + announcePendingRepoSession(ds); + // 弹卡片这条路不经 forkWorker，session.spawned 不会自动发——手动 upsert 一条， + // 让 dashboard 显示这条「待选仓库」会话（in_progress 首次 spawn 走这里才会出现）。 + dashboardEventBus.publish({ type: 'session.spawned', body: { session: composeRowFromActive(ds) } }); + logger.info(`[createSession] repo select card posted for session ${ds.session.sessionId.substring(0, 8)} (${projects.length} projects)`); + return 'pending_repo'; + } } } } @@ -2179,10 +2646,24 @@ async function forkOrShowRepoCard(ds: DaemonSession, userContent: string): Promi const prompt = buildPrompt(); rememberLastCliInput(ds, userContent, prompt); forkWorker(ds, prompt); - ds.pendingCodexAppText = undefined; - ds.pendingCodexAppApplicationContext = undefined; - ds.pendingCodexAppMessageContext = undefined; - ds.pendingAttachments = undefined; + // forkWorker pre-accept is synchronous. Keep the reservation and all input + // buffers intact if it throws; only expose the normal worker state after it + // has accepted the first turn. + if (!ds.session.queuedActivationPending) { + ds.initialStartPending = false; + ds.pendingPrompt = undefined; + ds.pendingCodexAppText = undefined; + ds.pendingCodexAppApplicationContext = undefined; + ds.pendingCodexAppMessageContext = undefined; + ds.pendingAttachments = undefined; + ds.pendingMentions = undefined; + ds.pendingSender = undefined; + ds.pendingFollowUps = undefined; + ds.pendingFollowUpTurnIds = undefined; + ds.pendingCodexAppFollowUps = undefined; + ds.pendingCodexAppFollowUpContexts = undefined; + } + return 'forked'; } export interface SpawnDashboardSessionArgs { @@ -2227,8 +2708,11 @@ export async function spawnDashboardSession( // chat-scope：锚点就是 chatId。先挡掉「同群同 bot 已有真会话」的撞键（会被 // Map.set 覆盖而泄漏 worker）。queued 占位 / 纯 scratch 不算冲突。 const anchor = chatId; - const existing = activeSessions.get(sessionKey(anchor, larkAppId)); - if (existing && (existing.worker || existing.session.queued || isRelayableRealSession(existing))) { + const key = sessionKey(anchor, larkAppId); + const existing = activeSessions.get(key); + if (existing && (existing.worker || existing.session.queued || existing.pendingRepo + || existing.initialStartPending || existing.worktreeCreating || isRelayableRealSession(existing) + || hasProtectedSessionMutationOwnership(existing))) { return { ok: false, error: 'session_exists' }; } @@ -2264,65 +2748,132 @@ export async function spawnDashboardSession( const userContent = composeSpawnUserContent({ content, role, coworkers: args.coworkers, locale }); const codexAppMessageContext = composeSpawnCodexAppContext({ role, coworkers: args.coworkers, locale }); - const resolvedTitle = args.title || deriveSessionTitleFromContent(content); - const session = sessionStore.createSession(chatId, bannerMessageId ?? chatId, resolvedTitle, 'group'); - const now = Date.now(); - session.larkAppId = larkAppId; - session.scope = 'chat'; - session.ownerOpenId = args.ownerOpenId ?? getOwnerOpenId(larkAppId); - session.creatorOpenId = session.ownerOpenId; - if (args.ownerUnionId) session.ownerUnionId = args.ownerUnionId; - session.lastMessageAt = new Date(now).toISOString(); - if (args.attachments?.length) session.dashboardAttachments = args.attachments; - if (column === 'backlog') { - session.queued = true; - session.queuedPrompt = userContent; - session.queuedCodexAppText = content; - session.queuedCodexAppMessageContext = codexAppMessageContext; - session.queuedAttachments = args.attachments; - session.kanbanColumn = 'backlog'; - } + const registered = await withActiveSessionKeyLock(activeSessions, key, async () => { + const current = activeSessions.get(key); + if (current) { + const protectedOwner = current.worker || current.session.queued || current.pendingRepo + || current.initialStartPending || current.worktreeCreating + || isRelayableRealSession(current) + || hasProtectedSessionMutationOwnership(current); + if (protectedOwner) return undefined; + await closeSession(current.session.sessionId); + } - // 钉 workingDir：oncall 绑定（弹框填了目录）/ bot 默认。都没有 → undefined，激活/开跑时 - // 会弹 /repo 卡片让用户在群里选仓库（复用普通新话题逻辑）。 - const workingDir = resolveDashboardSpawnWorkingDir(larkAppId, chatId); - if (workingDir) session.workingDir = workingDir; - sessionStore.updateSession(session); - messageQueue.ensureQueue(anchor); + const resolvedTitle = args.title || deriveSessionTitleFromContent(content); + const session = sessionStore.createSession(chatId, bannerMessageId ?? chatId, resolvedTitle, 'group'); + const now = Date.now(); + session.larkAppId = larkAppId; + session.scope = 'chat'; + session.ownerOpenId = args.ownerOpenId ?? getOwnerOpenId(larkAppId); + session.creatorOpenId = session.ownerOpenId; + if (args.ownerUnionId) session.ownerUnionId = args.ownerUnionId; + session.lastMessageAt = new Date(now).toISOString(); + if (args.attachments?.length) session.dashboardAttachments = args.attachments; + if (column === 'backlog') { + session.queued = true; + session.queuedPrompt = userContent; + session.queuedCodexAppText = content; + session.queuedCodexAppMessageContext = codexAppMessageContext; + if (args.attachments?.length) session.queuedAttachments = args.attachments; + session.kanbanColumn = 'backlog'; + } - const ds: DaemonSession = { - session, - worker: null, - workerPort: null, - workerToken: null, - larkAppId, - chatId, - chatType: 'group', - scope: 'chat', - spawnedAt: sessionCreatedAtMs(session), - cliVersion: getCurrentCliVersion(), - lastMessageAt: now, - hasHistory: false, - workingDir, - ownerOpenId: session.ownerOpenId, - currentTurnTitle: resolvedTitle, - pendingCodexAppText: content, - pendingCodexAppMessageContext: codexAppMessageContext, - pendingAttachments: args.attachments, - }; - activeSessions.set(sessionKey(anchor, larkAppId), ds); + const workingDir = resolveDashboardSpawnWorkingDir(larkAppId, chatId); + if (workingDir) session.workingDir = workingDir; + sessionStore.updateSession(session); + messageQueue.ensureQueue(anchor); + + const ds: DaemonSession = { + session, + worker: null, + workerPort: null, + workerToken: null, + larkAppId, + chatId, + chatType: 'group', + scope: 'chat', + spawnedAt: sessionCreatedAtMs(session), + cliVersion: getCurrentCliVersion(), + lastMessageAt: now, + hasHistory: false, + workingDir, + ownerOpenId: session.ownerOpenId, + currentTurnTitle: resolvedTitle, + pendingCodexAppText: content, + pendingCodexAppMessageContext: codexAppMessageContext, + pendingAttachments: args.attachments, + }; + if (column !== 'backlog') { + ds.initialStartPending = true; + ds.pendingPrompt = userContent; + } + activeSessions.set(key, ds); + if (column === 'backlog') { + ds.pendingPrompt = userContent; + dashboardEventBus.publish({ type: 'session.spawned', body: { session: composeRowFromActive(ds) } }); + } else { + // Keep the key reservation through initial worker/repo-picker + // installation. Otherwise a concurrent creator can classify this brief + // worker:null interval as disposable scratch, replace it, and let this + // caller resume by forking an orphan. + try { + await forkOrShowRepoCard(ds, userContent); + } catch (err) { + const durableOwner = hasProtectedSessionMutationOwnership(ds.session); + const liveWorkerOwner = !!ds.worker && !ds.worker.killed; + if (durableOwner || liveWorkerOwner) { + // `forkOrShowRepoCard` can fail after the repo/setup journal was + // durably staged (for example picker publish fallback followed by a + // fork rejection). That journal is now the exact opening owner; never + // erase it in the generic spawn rollback. Rebuild volatile picker + // buffers where possible and leave this exact map row active/retryable. + if (!ds.session.queuedActivationPending && ds.session.pendingRepoSetup) { + restorePendingRepoRuntime(ds); + } + ds.initialStartPending = ds.session.queuedActivationPending === true + || (ds.session.queuedActivationTail?.length ?? 0) > 0; + // The create IPC reports failure, but this durable row is still the + // authoritative retry owner. Upsert it so dashboard clients do not + // see an invisible occupied chat until the next full hydrate. + announceSessionRow(ds); + logger.error( + `[createSession] opening failed after durable ownership transfer for ${session.sessionId}; ` + + `retaining exact active owner: ${err instanceof Error ? err.message : String(err)}`, + ); + return { error: err instanceof Error ? err.message : String(err) }; + } + // No durable or live owner accepted the opening. Roll back only our + // exact runtime claim and close only our row; a concurrently published + // successor must survive. + if (activeSessions.get(key) === ds) activeSessions.delete(key); + try { sessionStore.closeSession(session.sessionId); } + catch (closeErr) { + logger.error( + `[createSession] failed to close unaccepted session ${session.sessionId}: ` + + `${closeErr instanceof Error ? closeErr.message : String(closeErr)}`, + ); + } + return { + error: err instanceof Error ? err.message : String(err), + }; + } + } + return { ds, session }; + }); + if (!registered) return { ok: false, error: 'session_exists' }; + if ('error' in registered && typeof registered.error === 'string') { + return { ok: false, error: registered.error }; + } + const { ds, session } = registered; if (column === 'backlog') { // Parked：不起 CLI。手动广播 session.spawned，让 dashboard 立刻显示待办池卡片 // （forkWorker 才会自动发这个事件，parked 路径要自己发）。 - ds.pendingPrompt = userContent; - dashboardEventBus.publish({ type: 'session.spawned', body: { session: composeRowFromActive(ds) } }); logger.info(`[createSession] queued session ${session.sessionId.substring(0, 8)} (bot=${larkAppId}, chat=${chatId}, role=${role})`); return { ok: true, sessionId: session.sessionId }; } // in_progress：立即开跑或弹 /repo 卡片（没钉目录时）。userContent 已按角色包装好。 - await forkOrShowRepoCard(ds, userContent); logger.info(`[createSession] spawned session ${session.sessionId.substring(0, 8)} (bot=${larkAppId}, chat=${chatId}, role=${role}, pendingRepo=${!!ds.pendingRepo})`); return { ok: true, sessionId: session.sessionId }; } @@ -2335,33 +2886,103 @@ export async function activateQueuedSession(ds: DaemonSession): Promise<{ ok: bo return (ds.worker && !ds.worker.killed) ? { ok: true } : { ok: false, error: 'not_queued' }; } if (ds.worker && !ds.worker.killed) { + if (ds.session.queuedActivationPending + || (ds.session.queuedActivationTail?.length ?? 0) > 0 + || ds.session.pendingRepoSetup) { + // A contradictory live+queued snapshot can occur around recovery or a + // stale dashboard action. Never reinterpret the live child as proof that + // its durable activation/setup owner crossed the adapter ACK boundary. + logger.warn( + `[${ds.session.sessionId.substring(0, 8)}] Ignored queued activation cleanup while durable ownership remains`, + ); + return { ok: true }; + } // 不该发生（queued 一定 worker:null），但保险：清标记即可。 ds.session.queued = false; ds.session.queuedPrompt = undefined; ds.session.queuedCodexAppText = undefined; ds.session.queuedCodexAppMessageContext = undefined; ds.session.queuedAttachments = undefined; + ds.session.queuedActivationPending = undefined; + ds.session.queuedActivationToken = undefined; + ds.session.queuedActivationInput = undefined; + ds.session.queuedActivationTurnId = undefined; + ds.session.queuedActivationDispatchAttempt = undefined; + ds.session.queuedActivationResume = undefined; + ds.session.pendingRepoSetup = undefined; sessionStore.updateSession(ds.session); return { ok: true }; } - const content = ds.session.queuedPrompt ?? ds.pendingPrompt ?? ''; - // A parked dashboard task may have crossed a daemon restart. Restore the - // persisted clean-input sidecar before clearing the durable backlog fields; - // forkOrShowRepoCard will carry it through either immediate fork or /repo. - ds.pendingCodexAppText ??= ds.session.queuedCodexAppText; - ds.pendingCodexAppMessageContext ??= ds.session.queuedCodexAppMessageContext; - ds.pendingAttachments ??= ds.session.queuedAttachments; - ds.session.queued = false; - ds.session.queuedPrompt = undefined; - ds.session.queuedCodexAppText = undefined; - ds.session.queuedCodexAppMessageContext = undefined; - ds.session.queuedAttachments = undefined; - ds.pendingPrompt = undefined; - // 激活即视为开始：从待办池挪到进行中，让卡片归位。 - if (ds.session.kanbanColumn === 'backlog') ds.session.kanbanColumn = 'in_progress'; - sessionStore.updateSession(ds.session); - // 起会话或弹 /repo 卡片（没钉目录时）。content 已是包装好的首轮内容。 - await forkOrShowRepoCard(ds, content); - logger.info(`[createSession] activated queued session ${ds.session.sessionId.substring(0, 8)} (bot=${ds.larkAppId}, pendingRepo=${!!ds.pendingRepo})`); - return { ok: true }; + // Repo selection / auto-worktree already owns this activation attempt. Keep + // the durable queued payload as its crash-recovery journal and make repeated + // dashboard starts idempotent instead of posting a second picker/build. + if (ds.pendingRepo) return { ok: true }; + const activate = async (): Promise<{ ok: boolean; error?: string }> => { + if (!ds.session.queued) { + return (ds.worker && !ds.worker.killed) ? { ok: true } : { ok: false, error: 'not_queued' }; + } + const content = ds.session.queuedPrompt ?? ds.pendingPrompt ?? ''; + // Preserve the durable queued payload until fork or pendingRepo setup has + // succeeded. Ordinary inbound routing does not take the key lock, so the + // runtime reservation must also be visible throughout every await. + ds.initialStartPending = true; + ds.pendingPrompt = content; + ds.pendingCodexAppText ??= ds.session.queuedCodexAppText; + ds.pendingCodexAppMessageContext ??= ds.session.queuedCodexAppMessageContext; + ds.pendingAttachments ??= ds.session.queuedAttachments; + let outcome: 'forked' | 'pending_repo'; + try { + const exactRetry = ds.session.queuedActivationInput; + if (exactRetry) { + forkWorker(ds, exactRetry, { + resume: ds.session.queuedActivationResume ?? ds.hasHistory, + turnId: ds.session.queuedActivationTurnId, + dispatchAttempt: ds.session.queuedActivationDispatchAttempt, + }); + outcome = 'forked'; + } else { + outcome = await forkOrShowRepoCard(ds, content); + } + } catch (err) { + // queued remains authoritative and retryable. Undo only transient route + // state installed by this failed attempt; never discard the opening + // prompt/sidecar before a worker or repo picker accepted it. + ds.initialStartPending = false; + ds.pendingRepo = false; + ds.repoCardMessageId = undefined; + ds.pendingPrompt = content; + return { ok: false, error: (err as Error)?.message ?? 'start_failed' }; + } + + // Ownership has transferred to a worker or pending-repo flow. Nothing + // below this point may advertise a retryable activation failure. + if (outcome === 'forked') { + ds.session.queued = false; + ds.session.queuedAttachments = undefined; + } + // pending_repo deliberately retains queued + queuedPrompt: pendingPrompt + // is runtime-only, so clearing the durable copy here would lose the first + // turn if the daemon dies before repo selection/worktree commit forks. + if (ds.session.kanbanColumn === 'backlog') ds.session.kanbanColumn = 'in_progress'; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + logger.error( + `[createSession] queued activation metadata persistence failed after ownership transfer: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + logger.info(`[createSession] activated queued session ${ds.session.sessionId.substring(0, 8)} (bot=${ds.larkAppId}, pendingRepo=${!!ds.pendingRepo})`); + return { ok: true }; + }; + + const registry = getActiveSessionsRegistry(); + if (!registry) return activate(); + const key = activeSessionKey(ds); + return withActiveSessionKeyLock(registry, key, async () => { + if (registry.get(key) !== ds || ds.session.status !== 'active') { + return { ok: false, error: 'session_not_active' }; + } + return activate(); + }); } diff --git a/src/core/session-mutation-guard.ts b/src/core/session-mutation-guard.ts new file mode 100644 index 000000000..b5fa203eb --- /dev/null +++ b/src/core/session-mutation-guard.ts @@ -0,0 +1,55 @@ +import type { Session } from '../types.js'; +import { hasUnsettledCodexAppDispatch } from '../utils/codex-app-dispatch-ledger.js'; + +type ProtectedSessionMutationState = Pick< + Session, + | 'codexAppDispatchLedger' + | 'queued' + | 'queuedActivationPending' + | 'queuedActivationTail' + | 'pendingRepoSetup' +>; + +type ProtectedRuntimeMutationState = { + session: ProtectedSessionMutationState; + initialStartPending?: boolean; +}; + +export type ProtectedSessionMutationReason = + | 'codex_app_dispatch' + | 'queued_todo' + | 'activation_head' + | 'activation_tail' + | 'repository_setup' + | 'initial_start'; + +export function protectedSessionMutationReasons( + value: ProtectedRuntimeMutationState | ProtectedSessionMutationState, +): ProtectedSessionMutationReason[] { + const ds: ProtectedRuntimeMutationState | undefined = 'session' in value + ? value + : undefined; + const session: ProtectedSessionMutationState = ds + ? ds.session + : value as ProtectedSessionMutationState; + const reasons: ProtectedSessionMutationReason[] = []; + if (hasUnsettledCodexAppDispatch(session.codexAppDispatchLedger)) reasons.push('codex_app_dispatch'); + if (session.queued === true) reasons.push('queued_todo'); + if (session.queuedActivationPending === true) reasons.push('activation_head'); + if ((session.queuedActivationTail?.length ?? 0) > 0) reasons.push('activation_tail'); + if (session.pendingRepoSetup !== undefined) reasons.push('repository_setup'); + if (ds?.initialStartPending === true) reasons.push('initial_start'); + return reasons; +} + +/** + * True while a session owns work that an ordinary config/restart/switch + * mutation is not authorized to abandon. Explicit close remains the escape + * hatch. Keep this backend-neutral: the activation journal protects opening + * submissions just as the Codex App ledger protects accepted dispatches. + */ +export function hasProtectedSessionMutationOwnership( + value: ProtectedRuntimeMutationState | ProtectedSessionMutationState, +): boolean { + return protectedSessionMutationReasons(value).length > 0; +} diff --git a/src/core/trigger-session.ts b/src/core/trigger-session.ts index 2da8d1f9a..03059b53d 100644 --- a/src/core/trigger-session.ts +++ b/src/core/trigger-session.ts @@ -9,13 +9,22 @@ import { localeForBot, t } from '../i18n/index.js'; import { validateWorkingDir } from './working-dir.js'; import { buildFollowUpCliInput, buildNewTopicCliInput, ensureSessionWhiteboard, getAvailableBots, rememberLastCliInput } from './session-manager.js'; import { markSessionActivity } from './session-activity.js'; -import { forkWorker, getCurrentCliVersion, sendWorkerInput } from './worker-pool.js'; +import { + forkWorker, + getCurrentCliVersion, + hasQueuedActivationAdmissionGate, + sendWorkerInput, + withActiveSessionKeyLock, +} from './worker-pool.js'; import { botAutoWorktreeEnabled } from '../services/default-worktree.js'; import * as messageQueue from '../services/message-queue.js'; import type { DaemonSession } from './types.js'; -import { sessionKey } from './types.js'; +import { activeSessionKey, sessionKey } from './types.js'; import type { TriggerRequest, TriggerResponse } from '../services/trigger-types.js'; import type { CliTurnPayload } from '../types.js'; +import { withBotTurnAdmission } from './bot-turn-mutation-gate.js'; +import { stagePendingRepoSetup } from './pending-repo-journal.js'; +import { hasProtectedSessionMutationOwnership } from './session-mutation-guard.js'; export interface TriggerSessionDeps { larkAppId: string; @@ -185,7 +194,18 @@ function waitForSessionFinalOutput( resolve({ ok: false, triggerId, errorCode: 'trigger_failed', error: err.message }); }, }); - dispatchTurn(); + try { + dispatchTurn(); + } catch (err) { + clearTimeout(timer); + ds.pendingWaitPromises?.delete(triggerId); + resolve({ + ok: false, + triggerId, + errorCode: 'trigger_failed', + error: err instanceof Error ? err.message : String(err), + }); + } }); } @@ -262,7 +282,7 @@ function buildExistingSessionContent( }); } -export async function triggerSessionTurn( +async function triggerSessionTurnAdmitted( req: TriggerRequest, deps: TriggerSessionDeps, internal?: TriggerSessionInternalOptions, @@ -377,94 +397,147 @@ export async function triggerSessionTurn( }; } - if (ds?.worker && !ds.worker.killed) { + const deliverToExisting = async (target: DaemonSession): Promise => { + const targetKey = activeSessionKey(target); + if (deps.activeSessions.get(targetKey) !== target || target.session.status !== 'active') { + return { + ok: false, + triggerId, + errorCode: 'session_not_found', + error: `active session ownership changed before dispatch: ${target.session.sessionId}`, + }; + } + const workerIsLive = !!target.worker && !target.worker.killed; + if (!workerIsLive && (target.pendingRepo || target.initialStartPending + || target.worktreeCreating || target.session.queued + || hasProtectedSessionMutationOwnership(target))) { + const state = target.pendingRepo + ? 'pending_repo' + : target.initialStartPending + ? 'initial_start_pending' + : target.worktreeCreating + ? 'worktree_creating' + : target.session.queued + ? 'queued_backlog' + : 'durable_owner'; + return { + ok: false, + triggerId, + errorCode: 'trigger_failed', + error: `target session ${target.session.sessionId} is not runnable (${state}); preserving its opening prompt`, + }; + } const content = buildExistingSessionContent( - ds, prompt, larkAppId, chatId, codexAppText, codexAppApplicationContext, codexAppMessageContext, + target, prompt, larkAppId, chatId, codexAppText, codexAppApplicationContext, codexAppMessageContext, ); - markSessionActivity(ds); - rememberInput(ds, prompt, content); + const queuedBehindActivation = workerIsLive + && hasQueuedActivationAdmissionGate(target); + const recordAcceptedInput = (): void => { + markSessionActivity(target); + rememberInput(target, prompt, content); + }; - if (req.options?.waitForFinalOutput) { - return waitForSessionFinalOutput( - ds, - triggerId, - req.options?.timeoutMs ?? 120_000, - (text) => ({ - ok: true, + if (workerIsLive) { + if (req.options?.waitForFinalOutput) { + return waitForSessionFinalOutput( + target, triggerId, - action: 'completed', - target: { kind: 'turn', sessionId: ds!.session.sessionId, chatId }, - output: { content: text }, - message: 'delivered to existing session and completed', - }), - () => { - const dispatchAttempt = prepareStableDispatch(ds!, false); - armFinalOutputSuppression(ds!, dispatchAttempt); - sendWorkerInput(ds!, content, triggerId, { - ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), - }); - }, - ); - } - - if (req.options?.asyncReturnSessionId) { - beginAsyncTrigger(ds, triggerId); - const dispatchAttempt = prepareStableDispatch(ds, false); - armFinalOutputSuppression(ds, dispatchAttempt); - sendWorkerInput(ds, content, triggerId, { + req.options?.timeoutMs ?? 120_000, + (text) => ({ + ok: true, + triggerId, + action: 'completed', + target: { kind: 'turn', sessionId: target.session.sessionId, chatId }, + output: { content: text }, + message: 'delivered to existing session and completed', + }), + () => { + const dispatchAttempt = prepareStableDispatch(target, false); + armFinalOutputSuppression(target, dispatchAttempt); + const accepted = sendWorkerInput(target, content, triggerId, { + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + }); + if (!accepted) throw new Error('worker refused trigger input before acceptance'); + recordAcceptedInput(); + }, + ); + } + + if (req.options?.asyncReturnSessionId) { + beginAsyncTrigger(target, triggerId); + const dispatchAttempt = prepareStableDispatch(target, false); + armFinalOutputSuppression(target, dispatchAttempt); + const accepted = sendWorkerInput(target, content, triggerId, { + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + }); + if (!accepted) { + target.asyncTriggerResults?.delete(triggerId); + if (target.latestAsyncTriggerId === triggerId) target.latestAsyncTriggerId = undefined; + return { + ok: false, + triggerId, + errorCode: 'trigger_failed', + error: 'worker refused async trigger input before acceptance', + }; + } + recordAcceptedInput(); + return buildAsyncQueuedResponse( + triggerId, + target.session.sessionId, + chatId, + 'delivered to existing session; poll by sessionId or triggerId for final output', + ); + } + + const dispatchAttempt = prepareStableDispatch(target, false); + armFinalOutputSuppression(target, dispatchAttempt); + const accepted = sendWorkerInput(target, content, stableTurnId ? triggerId : undefined, { ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), }); - return buildAsyncQueuedResponse( + if (!accepted) { + return { + ok: false, + triggerId, + errorCode: 'trigger_failed', + error: 'worker refused trigger input before acceptance', + }; + } + recordAcceptedInput(); + return { + ok: true, triggerId, - ds.session.sessionId, - chatId, - 'delivered to existing session; poll by sessionId or triggerId for final output', - ); + action: queuedBehindActivation ? 'queued' : 'delivered', + target: { kind: 'turn', sessionId: target.session.sessionId, chatId }, + message: queuedBehindActivation + ? 'durably queued behind the existing activation' + : 'delivered to existing session', + }; } - const dispatchAttempt = prepareStableDispatch(ds, false); - armFinalOutputSuppression(ds, dispatchAttempt); - sendWorkerInput(ds, content, stableTurnId ? triggerId : undefined, { - ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), - }); - return { - ok: true, - triggerId, - action: 'delivered', - target: { kind: 'turn', sessionId: ds.session.sessionId, chatId }, - message: 'delivered to existing session', - }; - } - - // An explicit session target stays bound to that session even while its - // worker is dormant. The old rootMessageId-only condition accidentally fell - // through to createSession for chat-scope sessions, which is unsafe for a - // durable meeting receiver whose projection pins one receiverSessionId. - if (ds) { - const content = buildExistingSessionContent( - ds, prompt, larkAppId, chatId, codexAppText, codexAppApplicationContext, codexAppMessageContext, - ); - markSessionActivity(ds); - rememberInput(ds, prompt, content); + recordAcceptedInput(); + // An explicit session target stays bound to that session even while its + // worker is dormant. The old rootMessageId-only condition accidentally + // fell through to createSession for chat-scope sessions, which is unsafe + // for a durable meeting receiver whose projection pins one receiver id. if (req.options?.waitForFinalOutput) { return waitForSessionFinalOutput( - ds, + target, triggerId, req.options?.timeoutMs ?? 120_000, (text) => ({ ok: true, triggerId, action: 'completed', - target: { kind: 'turn', sessionId: ds!.session.sessionId, chatId }, + target: { kind: 'turn', sessionId: target.session.sessionId, chatId }, output: { content: text }, message: 'delivered to existing session and completed', }), () => { - const dispatchAttempt = prepareStableDispatch(ds!, true); - armFinalOutputSuppression(ds!, dispatchAttempt); - forkWorker(ds!, content, { - resume: ds!.hasHistory, + const dispatchAttempt = prepareStableDispatch(target, true); + armFinalOutputSuppression(target, dispatchAttempt); + forkWorker(target, content, { + resume: target.hasHistory, turnId: triggerId, ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), }); @@ -473,26 +546,26 @@ export async function triggerSessionTurn( } if (req.options?.asyncReturnSessionId) { - beginAsyncTrigger(ds, triggerId); - const dispatchAttempt = prepareStableDispatch(ds, true); - armFinalOutputSuppression(ds, dispatchAttempt); - forkWorker(ds, content, { - resume: ds.hasHistory, + beginAsyncTrigger(target, triggerId); + const dispatchAttempt = prepareStableDispatch(target, true); + armFinalOutputSuppression(target, dispatchAttempt); + forkWorker(target, content, { + resume: target.hasHistory, turnId: triggerId, ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), }); return buildAsyncQueuedResponse( triggerId, - ds.session.sessionId, + target.session.sessionId, chatId, 'delivered to existing session; poll by sessionId or triggerId for final output', ); } - const dispatchAttempt = prepareStableDispatch(ds, true); - armFinalOutputSuppression(ds, dispatchAttempt); - forkWorker(ds, content, { - resume: ds.hasHistory, + const dispatchAttempt = prepareStableDispatch(target, true); + armFinalOutputSuppression(target, dispatchAttempt); + forkWorker(target, content, { + resume: target.hasHistory, turnId: triggerId, ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), }); @@ -500,10 +573,12 @@ export async function triggerSessionTurn( ok: true, triggerId, action: 'queued', - target: { kind: 'turn', sessionId: ds.session.sessionId, chatId }, + target: { kind: 'turn', sessionId: target.session.sessionId, chatId }, message: 'queued existing session turn', }; - } + }; + + if (ds) return deliverToExisting(ds); const wd = resolveWorkingDir(larkAppId, chatId); if (!wd.ok) { @@ -524,34 +599,6 @@ export async function triggerSessionTurn( scope = 'thread'; } - const session = sessionStore.createSession(chatId, anchor, triggerTitle(req), 'group'); - const now = Date.now(); - session.larkAppId = larkAppId; - session.scope = scope; - if (shouldOpenOwnTopic && topicMessage === null) session.externalTriggerTopicless = true; - session.lastMessageAt = new Date(now).toISOString(); - session.workingDir = wd.workingDir; - session.cliId = bot.config.cliId; - sessionStore.updateSession(session); - - messageQueue.ensureQueue(anchor); - - const newDs: DaemonSession = { - session, - worker: null, - workerPort: null, - workerToken: null, - larkAppId, - chatId, - chatType: 'group', - scope, - spawnedAt: Date.parse(session.createdAt) || now, - cliVersion: getCurrentCliVersion(), - lastMessageAt: now, - hasHistory: false, - workingDir: wd.workingDir, - }; - // 仅默认目录 + auto-worktree：chat 驱动的 webhook 开新会话且落在本 bot 自己的默认目录时，走 // pendingRepo 挂起 + 异步提交（登记挂起→关键路径外建 worktree→commitRepoSelection 提交+fork）， // detach 后立即返回 queued。规则：**仅普通 webhook 适用**——HTTP 应答模式（waitForFinalOutput / @@ -564,15 +611,81 @@ export async function triggerSessionTurn( && !stableTurnId && wd.fromBotDefault && botAutoWorktreeEnabled(larkAppId); - if (useAutoWt) { - // Register BEFORE the detached commit so its guard + the router's pendingRepo - // buffering both see the session. pendingRepo=true → no force-fork in the window. - newDs.pendingRepo = true; // router buffers concurrent events; commit clears it - newDs.pendingPrompt = prompt; // folded into the first turn by commitRepoSelection + + // New trigger sessions participate in the same first-owner lock as resume, + // dashboard creation, and scheduled creation. The earlier routing lookup + // necessarily precedes chat-membership/mode awaits, so it is only a hint; + // the owner must be re-read at the commit point. Publish a reservation + // before any post-registration await so resume can never pass its final + // check and then have this trigger overwrite it. + const key = sessionKey(anchor, larkAppId); + const claim = await withActiveSessionKeyLock(deps.activeSessions, key, () => { + const current = deps.activeSessions.get(key); + if (current) return { kind: 'existing' as const, ds: current }; + + const session = sessionStore.createSession(chatId, anchor, triggerTitle(req), 'group'); + const now = Date.now(); + session.larkAppId = larkAppId; + session.scope = scope; + if (shouldOpenOwnTopic && topicMessage === null) session.externalTriggerTopicless = true; + session.lastMessageAt = new Date(now).toISOString(); + session.workingDir = wd.workingDir; + session.cliId = bot.config.cliId; + sessionStore.updateSession(session); + messageQueue.ensureQueue(anchor); + + const newDs: DaemonSession = { + session, + worker: null, + workerPort: null, + workerToken: null, + larkAppId, + chatId, + chatType: 'group', + scope, + spawnedAt: Date.parse(session.createdAt) || now, + cliVersion: getCurrentCliVersion(), + lastMessageAt: now, + hasHistory: false, + workingDir: wd.workingDir, + }; + // Retain the complete opening input until a worker or repo workflow has + // synchronously accepted it. This is both the route reservation and the + // retry payload if a write-ahead hook/fork throws before acceptance. + newDs.pendingPrompt = prompt; newDs.pendingCodexAppText = codexAppText; newDs.pendingCodexAppApplicationContext = codexAppApplicationContext || undefined; newDs.pendingCodexAppMessageContext = codexAppMessageContext; - deps.activeSessions.set(sessionKey(anchor, larkAppId), newDs); + if (useAutoWt) { + newDs.pendingRepo = true; + try { + stagePendingRepoSetup(newDs, { + mode: 'auto_worktree', + baseDir: wd.workingDir, + turnId: triggerId, + }); + } catch (err) { + // The route was never published, but createSession already persisted + // an active row. Close only this unaccepted row so a staging fault + // cannot reappear as an unregistered owner after restart. + try { sessionStore.closeSession(session.sessionId); } + catch { /* keep the original admission error */ } + throw err; + } + } else { + newDs.initialStartPending = true; + } + deps.activeSessions.set(key, newDs); + return { kind: 'created' as const, ds: newDs }; + }); + + if (claim.kind === 'existing') return deliverToExisting(claim.ds); + const newDs = claim.ds; + const session = newDs.session; + + if (useAutoWt) { + // The key-lock claim registered pendingRepo before this dynamic import; + // repo commit and inbound routing therefore see the same reservation. const { runAutoWorktreeCommit } = await import('../im/lark/card-handler.js'); void runAutoWorktreeCommit({ ds: newDs, anchor, larkAppId, baseDir: wd.workingDir, title: triggerTitle(req), @@ -591,6 +704,20 @@ export async function triggerSessionTurn( } ensureSessionWhiteboard(newDs); + let availableBots: Awaited>; + try { + availableBots = await getAvailableBots(larkAppId, chatId); + } catch (err) { + // Prompt construction failed before any worker existed. Retire only the + // still-owned reservation so a retry is not blocked by a ghost active row. + await withActiveSessionKeyLock(deps.activeSessions, key, () => { + if (deps.activeSessions.get(key) === newDs && newDs.initialStartPending) { + deps.activeSessions.delete(key); + sessionStore.closeSession(session.sessionId); + } + }); + throw err; + } const promptInput = buildNewTopicCliInput( prompt, session.sessionId, @@ -598,7 +725,7 @@ export async function triggerSessionTurn( bot.config.cliPathOverride, undefined, undefined, - await getAvailableBots(larkAppId, chatId), + availableBots, undefined, { name: bot.botName, openId: bot.botOpenId }, localeForBot(larkAppId), @@ -612,12 +739,29 @@ export async function triggerSessionTurn( codexAppMessageContext, }, ); - // Register right before the fork branches (no await between here and forkWorker) - // so a concurrent inbound message can't observe this session worker-less and - // race a duplicate re-fork — the set-and-fork atomicity the original path had. - deps.activeSessions.set(sessionKey(anchor, larkAppId), newDs); + // No await from the ownership check through forkWorker. The reservation and + // opening buffers are released only after synchronous pre-accept succeeds. + if (deps.activeSessions.get(key) !== newDs + || newDs.session.status !== 'active' + || !newDs.initialStartPending) { + if (newDs.session.status !== 'closed') sessionStore.closeSession(session.sessionId); + return { + ok: false, + triggerId, + errorCode: 'trigger_failed', + error: 'new trigger session lost its first-owner reservation before startup', + }; + } rememberInput(newDs, prompt, promptInput); + const releaseInitialReservation = (): void => { + newDs.initialStartPending = false; + newDs.pendingPrompt = undefined; + newDs.pendingCodexAppText = undefined; + newDs.pendingCodexAppApplicationContext = undefined; + newDs.pendingCodexAppMessageContext = undefined; + }; + if (req.options?.waitForFinalOutput) { return waitForSessionFinalOutput( newDs, @@ -637,17 +781,19 @@ export async function triggerSessionTurn( forkWorker(newDs, promptInput, dispatchAttempt === undefined ? triggerId : { turnId: triggerId, dispatchAttempt }); + releaseInitialReservation(); }, ); } if (req.options?.asyncReturnSessionId) { - beginAsyncTrigger(newDs, triggerId); const dispatchAttempt = prepareStableDispatch(newDs, true); armFinalOutputSuppression(newDs, dispatchAttempt); forkWorker(newDs, promptInput, dispatchAttempt === undefined ? triggerId : { turnId: triggerId, dispatchAttempt }); + releaseInitialReservation(); + beginAsyncTrigger(newDs, triggerId); return buildAsyncQueuedResponse( triggerId, session.sessionId, @@ -662,8 +808,12 @@ export async function triggerSessionTurn( forkWorker(newDs, promptInput, dispatchAttempt === undefined ? triggerId : { turnId: triggerId, dispatchAttempt }); + releaseInitialReservation(); + } + else { + forkWorker(newDs, promptInput); + releaseInitialReservation(); } - else forkWorker(newDs, promptInput); return { ok: true, @@ -673,3 +823,14 @@ export async function triggerSessionTurn( message: 'queued new session turn', }; } + +export async function triggerSessionTurn( + req: TriggerRequest, + deps: TriggerSessionDeps, + internal?: TriggerSessionInternalOptions, +): Promise { + return withBotTurnAdmission( + deps.larkAppId, + () => triggerSessionTurnAdmitted(req, deps, internal), + ); +} diff --git a/src/core/types.ts b/src/core/types.ts index 9fcabbcf8..794ad5a95 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -1,6 +1,7 @@ import type { ChildProcess } from 'node:child_process'; import type { CodexAppTurnInput, + CliTurnPayload, Session, DaemonToWorker, LarkAttachment, @@ -83,6 +84,25 @@ export interface DaemonSession { * separate from worktreeCreating because plain select, skip, and /repo can * also await prompt context before the fork. */ pendingRepoCommitInFlight?: boolean; + /** A fresh live-route owner has been published but its opening input has not + * reached the first fork yet. Same-anchor turns must buffer into the opening + * input instead of reforking worker:null and overtaking it. In-memory only. */ + initialStartPending?: boolean; + /** Generation token for the handler that atomically reserved a worker:null + * refork. Later same-anchor handlers may prepare concurrently, but only this + * owner may cross the fork boundary; followers buffer behind its gate. */ + initialStartClaimToken?: string; + /** Number of activation-tail arrivals that reserved FIFO order before an + * asynchronous prompt/sender build and have not yet durably admitted or + * failed. An opening ACK must not clear the route while this is non-zero. */ + queuedActivationTailAdmissionsOutstanding?: number; + /** An opening ACK (or ordinary cold-start handoff) observed while an + * asynchronous tail admission was outstanding. The final settler replays + * this release so a late durable successor cannot be stranded. */ + queuedActivationTailReleasePending?: { acknowledgedToken?: string }; + /** Retry timer for an ordinary cold-start handoff whose durable promotion + * failed after all asynchronous admissions had settled. */ + queuedActivationTailReleaseRetryTimer?: ReturnType; repoCardMessageId?: string; // message_id of the repo selection card — for withdrawal /** * Repo-select card message ids already consumed by a successful pending→worker @@ -144,8 +164,25 @@ export interface DaemonSession { /** Exact turn for a same-caller pendingRawInput follow-up batch. Cleared on * mixed callers so the combined prompt fails closed instead of borrowing. */ pendingFollowUpTurnId?: string; + pendingFollowUpTurnIds?: string[]; // matching inbound ids; last id attributes a folded buffered batch pendingCodexAppFollowUps?: string[]; // matching raw user texts for clean Codex App materialization pendingCodexAppFollowUpContexts?: string[]; // matching metadata-only context; never duplicates the raw follow-up text + /** Arrival-time clean-input decisions matching pendingCodexAppFollowUps. + * Used only when a literal raw cold start must fold followers onto the same + * text→Enter IPC boundary. */ + pendingCodexAppFollowUpGateAccepted?: boolean[]; + /** Exact turns that arrived while a previously attempted queued activation + * was re-parked. They remain separate FIFO items behind the retained opening + * payload and advance only after worker acceptance. In-memory only. */ + pendingQueuedActivationFollowUps?: Array<{ + userPrompt: string; + cliInput: CliTurnPayload; + turnId: string; + dispatchAttempt?: number; + /** Legacy volatile entries already crossed the clean-input gate when they + * were staged. Migration must preserve that exact sidecar decision. */ + codexAppInputGateFrozen?: true; + }>; ownerOpenId?: string; // topic creator's open_id — receives write-enabled terminal link via DM streamCardId?: string; // message_id of the streaming card in group (PATCHed with live output) streamCardNonce?: string; // unique nonce for the current streaming card — embedded in button values to distinguish old vs current card diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 133c0878e..b270f14f6 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -18,7 +18,7 @@ import { config } from '../config.js'; import { readGlobalConfig } from '../global-config.js'; import * as sessionStore from '../services/session-store.js'; import { persistStreamCardState, rememberLastCliInput } from './session-manager.js'; -import { fallbackTurnId, isSubstituteTurn } from './reply-target.js'; +import { fallbackTurnId, frozenReplyContextForTurn, isSubstituteTurn } from './reply-target.js'; import { updateMessage, deleteMessage, sendEphemeralCard, sendUserMessage, addReaction, removeReaction, getMessageChatId, MessageWithdrawnError } from '../im/lark/client.js'; import { buildStreamingCard, buildPrivateSnapshotCard, buildSessionCard, buildTuiPromptCard, buildTuiPromptResolvedCard, buildRelayedFrozenCard, getCliDisplayName } from '../im/lark/card-builder.js'; import { loadFrozenCards, saveFrozenCards } from '../services/frozen-card-store.js'; @@ -37,6 +37,7 @@ import { TmuxBackend } from '../adapters/backend/tmux-backend.js'; import { HerdrBackend } from '../adapters/backend/herdr-backend.js'; import { sandboxEnabled } from '../adapters/backend/sandbox.js'; import { isSuspendableBackendType, getSessionPersistentBackendType, persistentBackendTargetForSession, killPersistentBackendTarget, managedTargetsForCliChange, resolvePairedSpawnBackendType, resolvePersistentBackendTarget } from './persistent-backend.js'; +import { withBotTurnMutation } from './bot-turn-mutation-gate.js'; import { getBot, getAllBots, loadBotConfigs, resolveBrandLabel } from '../bot-registry.js'; /** A random id minted once per daemon process (this lifetime). Stamped onto @@ -74,8 +75,37 @@ import type { CliId } from '../adapters/cli/types.js'; import { isStructuredBridgeAdoptCli } from '../services/structured-bridge-clis.js'; import { resolveEffectivePluginIds } from './plugins/effective.js'; import { ensureGatewayEntry } from './plugins/mcp/gateway-installer.js'; -import type { CliTurnPayload, CodexAppTurnInput, DaemonToWorker, WorkerToDaemon, Session, DisplayMode } from '../types.js'; -import { activeSessionKey, sessionKey, sessionAnchorId, isDocNativeSession, type DaemonSession } from './types.js'; +import type { + CliTurnPayload, + CodexAppDeliverySink, + CodexAppDispatchLedgerEntry, + CodexAppGenerationCommit, + CodexAppTurnInput, + FrozenSessionReplyTarget, + DaemonToWorker, + WorkerToDaemon, + Session, + DisplayMode, + QueuedActivationTailEntry, +} from '../types.js'; +import { + appendAcceptedCodexAppDispatch, + cancelCodexAppDispatch, + committedCodexAppSequence, + hasUnsettledCodexAppDispatch, + prepareCodexAppDispatch, + retryPreparedCodexAppDispatch, + retainFreshCodexAppGeneration, + settleCodexAppDispatch, +} from '../utils/codex-app-dispatch-ledger.js'; +import { + activeSessionKey, + sessionKey, + sessionAnchorId, + isDocNativeSession, + type DaemonSession, +} from './types.js'; +import { hasProtectedSessionMutationOwnership } from './session-mutation-guard.js'; import { DONE_REACTION_EMOJI_TYPE } from './pending-response.js'; import { buildTerminalUrl } from './terminal-url.js'; import { prependBotmuxBin } from './botmux-wrapper.js'; @@ -141,6 +171,8 @@ export interface WorkerSessionReplyOptions { * share a visible chat anchor with ordinary sessions, so the anchor alone * cannot identify the transcript/lifecycle owner. */ sourceSessionId?: string; + /** Exact daemon-frozen destination for a durable turn. */ + replyTarget?: FrozenSessionReplyTarget; } export interface WorkerPoolCallbacks { @@ -154,8 +186,11 @@ export interface WorkerPoolCallbacks { ) => Promise; getSessionWorkingDir: (ds?: DaemonSession) => string; getActiveCount: () => number; - /** Close a stale session (message withdrawn, etc.) */ - closeSession: (ds: DaemonSession) => void; + /** Close a stale session (message withdrawn, etc.). `false` means the + * authoritative close failed and the active owner must remain retryable. + * `void` is retained for older embedders/tests that implement a synchronous + * best-effort close; the production daemon always returns an exact boolean. */ + closeSession: (ds: DaemonSession) => boolean | void | Promise; /** Re-check the per-bot resident-session cap after a process starts or an * over-cap busy session becomes idle. Optional for unit-test callers. */ enforceLiveSessionCap?: () => void; @@ -204,6 +239,19 @@ export interface WorkerPoolCallbacks { disposition: 'queued_removed' | 'cli_fenced'; }, ) => void; + /** Called only after a durable ledger mutation was persisted and its worker + * ACK was attempted. Runtime CLI-mismatch cleanup may now close the old + * generation without abandoning a FIFO entry. */ + onCodexAppLedgerDrained?: (ds: DaemonSession) => void | Promise; + /** The exact queued opening crossed the adapter submission boundary. The + * daemon may now release its runtime route reservation and flush follow-ups. */ + /** Return false only when a buffered follow-up was not accepted and should + * be retried. Once accepted, later presentation persistence is best-effort + * and must not request a delivery retry. */ + onQueuedActivationSubmitted?: ( + ds: DaemonSession, + activationToken: string, + ) => boolean | void | Promise; } let callbacks: WorkerPoolCallbacks | undefined; @@ -1142,6 +1190,36 @@ function flushCardPatch(ds: DaemonSession): void { }); } +/** Root withdrawal is not an explicit abandon boundary. Preserve a Codex App + * owner (including its live worker) while any durable FIFO entry is unsettled; + * only ledger-empty sessions retain the historical stale-root auto-close. */ +async function closeWithdrawnSessionIfLedgerEmpty( + ds: DaemonSession, + context: string, +): Promise { + if (hasProtectedSessionMutationOwnership(ds)) { + logger.warn(`[${tag(ds)}] ${context}; preserving session because Codex App dispatch is unsettled`); + return false; + } + logger.warn(`[${tag(ds)}] ${context}; closing ledger-empty stale session`); + // The callback routes through the authoritative async closeSession helper; + // never pre-kill a worker before that helper settles its backend boundary. + try { + const closed = await requireCallbacks().closeSession(ds); + if (closed === false) { + logger.warn(`[${tag(ds)}] ${context}; authoritative close failed, session retained for retry`); + return false; + } + return true; + } catch (err) { + logger.warn( + `[${tag(ds)}] ${context}; authoritative close threw, session retained for retry: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } +} + // ─── Restart rate-limiting ────────────────────────────────────────────────── export const restartCounts = new Map(); @@ -1403,6 +1481,10 @@ function reclaimParkedCrashDiagnostic(ds: DaemonSession): void { } export function suspendWorker(ds: DaemonSession, reason = 'suspended_idle'): boolean { + if (hasProtectedSessionMutationOwnership(ds)) { + logger.warn(`[${tag(ds)}] Refused worker suspend (${reason}) while Codex App dispatch ownership is non-empty`); + return false; + } if (!ds.worker || ds.worker.killed) { // There is no live generation that can still own this capability. ds.managedTurnOrigin = undefined; @@ -1578,20 +1660,189 @@ export async function closeSession( * * Setting the same `ds` at its own key is a no-op (no close). */ +export type SetActiveSessionResult = + | { accepted: true; closedSessionId?: string } + | { + accepted: false; + reason: 'kept_pending_owner'; + keptSessionId: string; + closedIncomingSessionId: string; + } + | { + accepted: false; + reason: 'both_pending'; + keptSessionId: string; + preservedIncomingSessionId: string; + } + | { + accepted: false; + reason: 'cleanup_failed'; + keptSessionId: string; + preservedIncomingSessionId: string; + cleanupSessionId: string; + error: string; + }; + +const activeSessionKeyLocks = new WeakMap< + Map, + Map> +>(); + +/** Serialize asynchronous ownership decisions for one registry key. Ordinary + * bot turn admissions are intentionally concurrent, so a read/await/set helper + * is not a CAS unless every contender shares this lock. */ +export async function withActiveSessionKeyLock( + map: Map, + key: string, + action: () => Promise | T, +): Promise { + let locks = activeSessionKeyLocks.get(map); + if (!locks) { + locks = new Map(); + activeSessionKeyLocks.set(map, locks); + } + const previous = locks.get(key) ?? Promise.resolve(); + let release!: () => void; + const hold = new Promise(resolve => { release = resolve; }); + const tail = previous.catch(() => { /* predecessor errors do not poison the lock */ }).then(() => hold); + locks.set(key, tail); + await previous.catch(() => { /* predecessor already reported its own error */ }); + try { + return await action(); + } finally { + release(); + if (locks.get(key) === tail) locks.delete(key); + } +} + +async function closeUnregisteredCollisionLoser( + map: Map, + loser: DaemonSession, +): Promise<{ ok: true } | { ok: false; error: string }> { + // Operate on exact object identity in the caller's registry. Looking up by + // session id through the daemon-global registry can conflate a non-global + // restore map (or a stale duplicate object) with an unrelated canonical + // owner and close the wrong generation. + const sameIdDifferentOwner = [...map.values()].some(candidate => + candidate !== loser && candidate.session.sessionId === loser.session.sessionId) + || (!!activeSessionsRegistry && activeSessionsRegistry !== map + && [...activeSessionsRegistry.values()].some(candidate => + candidate !== loser && candidate.session.sessionId === loser.session.sessionId)); + if (sameIdDifferentOwner) { + logger.error( + `[setActiveSessionSafe] refusing persistence close for ambiguous session id ` + + `${loser.session.sessionId}; preserving every owner`, + ); + return { ok: false, error: 'ambiguous_session_id' }; + } + + const stored = sessionStore.getSession(loser.session.sessionId); + try { + if (stored?.status !== 'closed') { + sessionStore.closeSession(loser.session.sessionId); + } + } catch (err) { + logger.error( + `[setActiveSessionSafe] durable loser close failed for ${loser.session.sessionId}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return { ok: false, error: 'durable_close_failed' }; + } + + killWorker(loser); + for (const [registeredKey, candidate] of map) { + if (candidate === loser) map.delete(registeredKey); + } + return { ok: true }; +} + export async function setActiveSessionSafe( map: Map, key: string, ds: DaemonSession, -): Promise { - const prev = map.get(key); - if (prev && prev !== ds) { - logger.warn( - `[setActiveSessionSafe] key already occupied by ${prev.session.sessionId.substring(0, 8)} ` + - `(worker=${prev.worker ? 'live' : 'null'}); closing it before set`, +): Promise { + const canonicalKey = activeSessionKey(ds); + if (canonicalKey !== key) { + throw new Error( + `refusing noncanonical active-session registration: requested=${key} canonical=${canonicalKey}`, ); - await closeSession(prev.session.sessionId); } - map.set(key, ds); + return withActiveSessionKeyLock(map, key, async () => { + const prev = map.get(key); + if (prev && prev !== ds) { + if (prev.session.sessionId === ds.session.sessionId) { + logger.error( + `[setActiveSessionSafe] refusing collision between distinct owners of session id ` + + `${prev.session.sessionId}; preserving both objects and the durable row`, + ); + return { + accepted: false, + reason: 'cleanup_failed', + keptSessionId: prev.session.sessionId, + preservedIncomingSessionId: ds.session.sessionId, + cleanupSessionId: prev.session.sessionId, + error: 'ambiguous_session_id', + }; + } + const prevPending = hasProtectedSessionMutationOwnership(prev); + const incomingPending = hasProtectedSessionMutationOwnership(ds); + if (prevPending && incomingPending) { + logger.error( + `[setActiveSessionSafe] refusing collision between two unsettled Codex App owners ` + + `${prev.session.sessionId} and ${ds.session.sessionId}; preserving both rows/panes`, + ); + return { + accepted: false, + reason: 'both_pending', + keptSessionId: prev.session.sessionId, + preservedIncomingSessionId: ds.session.sessionId, + }; + } + if (prevPending) { + logger.warn( + `[setActiveSessionSafe] keeping unsettled owner ${prev.session.sessionId.substring(0, 8)}; ` + + `closing ledger-empty incoming ${ds.session.sessionId.substring(0, 8)}`, + ); + const cleanup = await closeUnregisteredCollisionLoser(map, ds); + if (!cleanup.ok) { + return { + accepted: false, + reason: 'cleanup_failed', + keptSessionId: prev.session.sessionId, + preservedIncomingSessionId: ds.session.sessionId, + cleanupSessionId: ds.session.sessionId, + error: cleanup.error, + }; + } + return { + accepted: false, + reason: 'kept_pending_owner', + keptSessionId: prev.session.sessionId, + closedIncomingSessionId: ds.session.sessionId, + }; + } + logger.warn( + `[setActiveSessionSafe] key already occupied by ${prev.session.sessionId.substring(0, 8)} ` + + `(worker=${prev.worker ? 'live' : 'null'}); closing it before set`, + ); + const cleanup = await closeUnregisteredCollisionLoser(map, prev); + if (!cleanup.ok) { + return { + accepted: false, + reason: 'cleanup_failed', + keptSessionId: prev.session.sessionId, + preservedIncomingSessionId: ds.session.sessionId, + cleanupSessionId: prev.session.sessionId, + error: cleanup.error, + }; + } + } + map.set(key, ds); + return { + accepted: true, + ...(prev && prev !== ds ? { closedSessionId: prev.session.sessionId } : {}), + }; + }); } // ─── Session transfer (cross-chat relay) ──────────────────────────────────── @@ -1664,6 +1915,8 @@ export async function transferSession( forkWorkerImpl?: typeof forkWorker; /** @internal Override for tests — mirror of forkWorkerImpl for killWorker. */ killWorkerImpl?: typeof killWorker; + /** @internal Recursive marker: the bot-wide transfer mutation is held. */ + mutationHeld?: boolean; }, ): Promise<{ ok: true } | { ok: false; error: string }> { // Depth defense — unreachable per TS narrowing above, but guards against @@ -1671,9 +1924,28 @@ export async function transferSession( if ((targetChatType as string) !== 'group' && (targetChatType as string) !== 'p2p') { return { ok: false, error: 'target_chat_type_unsupported' }; } + const initial = findActiveBySessionId(sessionId); + if (!initial) return { ok: false, error: 'session_not_active' }; + if (!opts?.mutationHeld) { + // Transfer rewrites both the source and target routing identities and may + // await scratch cleanup. Drain every admitted turn for this bot first so + // no source prompt can become durable mid-transfer and no target creator + // can slip into the cleanup window. + return withBotTurnMutation(initial.larkAppId, () => transferSession( + sessionId, + targetChatId, + targetRootMessageId, + targetChatType, + targetScope, + { ...opts, mutationHeld: true }, + )); + } const ds = findActiveBySessionId(sessionId); - if (!ds) return { ok: false, error: 'session_not_active' }; + if (!ds || ds.session.status !== 'active') return { ok: false, error: 'session_not_active' }; if (ds.session.vcMeetingReceiver) return { ok: false, error: 'vc_receiver_not_relayable' }; + if (hasProtectedSessionMutationOwnership(ds)) { + return { ok: false, error: 'codex_app_dispatch_pending' }; + } // Anchor-based identity. A thread-scope session in the SAME chat (different // root) is a legitimate cross-topic move, so we refuse only when the target // anchor equals the source anchor (relaying a session onto itself). Replaces @@ -1730,22 +2002,56 @@ export async function transferSession( // Anchor-based: chat-scope anchors on chatId, thread-scope on rootMessageId. // Only a session at the target anchor collides — same-chat other-topic // sessions have a different anchor and are fine (enables同群话题间搬运). + const targetKey = sessionKey(targetAnchor, ds.larkAppId); + const sourceKey = activeSessionKey(ds); const scratchesToClose: string[] = []; if (activeSessionsRegistry) { for (const existing of activeSessionsRegistry.values()) { if (existing === ds) continue; if (existing.larkAppId !== ds.larkAppId) continue; - if (sessionAnchorId(existing) !== targetAnchor) continue; - if (!existing.worker) { - scratchesToClose.push(existing.session.sessionId); - continue; + // VC receivers intentionally share visible output routes while occupying + // an immutable `vc-receiver:` registry key. Only an owner of + // the ordinary target key can collide with this transfer. + if (activeSessionKey(existing) !== targetKey) continue; + if (isRelayableRealSession(existing) + || existing.pendingRepo + || existing.session.queued + || hasProtectedSessionMutationOwnership(existing)) { + return { ok: false, error: 'target_chat_has_session' }; } - return { ok: false, error: 'target_chat_has_session' }; + // The only disposable occupant is a proven command scratch: no worker, + // no persisted CLI markers, no pending picker/backlog, and no ledger. + scratchesToClose.push(existing.session.sessionId); } } for (const sid of scratchesToClose) { await closeSession(sid); } + // A partially hydrated daemon can have an active target row absent from the + // runtime map. Durable ownership makes that row a real conflict even when it + // has worker:null and no legacy CLI markers; only proven ledger-empty + // scratch rows may be retired. + const runtimeIds = new Set(activeSessionsRegistry + ? [...activeSessionsRegistry.values()].map(item => item.session.sessionId) + : []); + const persistedTargetConflicts = sessionStore.listSessions().filter(item => + item.sessionId !== ds.session.sessionId + && item.status === 'active' + && (!item.larkAppId || item.larkAppId === ds.larkAppId) + && !item.vcMeetingReceiver + && (item.scope === 'chat' ? item.chatId : item.rootMessageId) === targetAnchor + && !runtimeIds.has(item.sessionId), + ); + if (persistedTargetConflicts.some(item => + hasProtectedSessionMutationOwnership(item) + || !!item.cliId + || !!item.lastCliInput + || item.queued === true)) { + return { ok: false, error: 'target_chat_has_session' }; + } + for (const scratch of persistedTargetConflicts) { + await closeSession(scratch.sessionId); + } const fkw = opts?.forkWorkerImpl ?? forkWorker; const kw = opts?.killWorkerImpl ?? killWorker; @@ -1753,36 +2059,40 @@ export async function transferSession( const tagPrefix = sessionId.substring(0, 8); const oldAnchor = sessionAnchorId(ds); const oldChatId = ds.chatId; - - // Freeze the source-chat streaming card BEFORE we kill the worker (and - // before we clear streamCardId below). The live card's action buttons - // (close / toggle / get write link) carry `session_id` in their value, so - // clicks AFTER relay still reach the now-relocated session — closing it, - // toggling its display mode, etc. — with feedback landing on the NEW - // card in the target chat. PATCH the source-chat card to an inert - // snapshot so the user sees clearly it's historical, and so the buttons - // are gone. Best-effort: on PATCH failure (card withdrawn, expired) we - // log and continue; the relay itself must not depend on this. - if (ds.streamCardId && ds.streamCardId !== CARD_POSTING_SENTINEL) { - try { - const cliId = (ds.session.cliId as CliId | undefined) - ?? (() => { try { return getBot(ds.larkAppId).config.cliId; } catch { return undefined; } })(); - const frozenJson = buildRelayedFrozenCard( - ds.currentTurnTitle || ds.session.title || '', - cliId, - ds.currentImageKey, - localeForBot(ds.larkAppId), - ); - await updateMessage(ds.larkAppId, ds.streamCardId, frozenJson); - } catch (err) { - logger.warn(`[${tagPrefix}] freeze source-chat card failed: ${err instanceof Error ? err.message : err}`); - } + const oldStreamCardId = ds.streamCardId; + const oldCurrentImageKey = ds.currentImageKey; + + // Scratch/store cleanup above awaited. Re-resolve BOTH identities before + // the final synchronous move. A close/replace from a detached lifecycle + // path must win without letting this stale transfer kill/fork its successor. + if (ds.session.status !== 'active' + || findActiveBySessionId(sessionId) !== ds + || activeSessionsRegistry?.get(sourceKey) !== ds) { + return { ok: false, error: 'session_not_active' }; + } + // No await is allowed between this final reservation check and the map set + // below. A target session that arrived during the earlier scratch cleanup + // wins; the source is still live and untouched at this point. + const lateTarget = activeSessionsRegistry?.get(targetKey); + if (lateTarget && lateTarget !== ds) { + return { ok: false, error: 'target_chat_has_session' }; + } + // Scratch/store cleanup above awaited. A fresh source turn may have been + // admitted during that window; recheck in the same synchronous section as + // the final target claim so transfer never kills newly durable work. + if (hasProtectedSessionMutationOwnership(ds)) { + return { ok: false, error: 'codex_app_dispatch_pending' }; + } + if (ds.worker && !ds.worker.killed + && ds.lastScreenStatus !== 'idle' + && ds.lastScreenStatus !== 'limited') { + return { ok: false, error: 'worker_busy' }; } // Detach worker — TmuxBackend.kill() does NOT destroy the tmux session, so // the CLI process and its rolling jsonl continue running. kw(ds); - activeSessionsRegistry?.delete(sessionKey(oldAnchor, ds.larkAppId)); + activeSessionsRegistry?.delete(sourceKey); // Rewrite routing fields per the requested target scope. // chat-scope: routes by chatId; `targetRootMessageId` (e.g. an M1 id) is @@ -1815,7 +2125,36 @@ export async function transferSession( const newAnchor = sessionAnchorId(ds); if (activeSessionsRegistry) { - await setActiveSessionSafe(activeSessionsRegistry, sessionKey(newAnchor, ds.larkAppId), ds); + // targetKey was checked immediately before the synchronous kill/rewrite; + // direct publication avoids an awaited registration/rollback window where + // a stale transfer could overwrite a newly-created source successor. + activeSessionsRegistry.set(sessionKey(newAnchor, ds.larkAppId), ds); + } + + // Re-attach while the routing move is still one synchronous ownership + // commit. The source-card PATCH below awaits Lark and a detached close may + // legitimately win during that wait; deferring this fork until afterward + // would resurrect the stale transferred object after its route was removed. + fkw(ds, '', /*resume*/true); + + // Routing/store ownership is now committed. Only now freeze the old card: + // if a target arrived during an earlier await we returned with the source + // fully untouched, rather than irreversibly disabling its live controls. + // The old identifiers were captured before the target fields were cleared. + if (oldStreamCardId && oldStreamCardId !== CARD_POSTING_SENTINEL) { + try { + const cliId = (ds.session.cliId as CliId | undefined) + ?? (() => { try { return getBot(ds.larkAppId).config.cliId; } catch { return undefined; } })(); + const frozenJson = buildRelayedFrozenCard( + ds.currentTurnTitle || ds.session.title || '', + cliId, + oldCurrentImageKey, + localeForBot(ds.larkAppId), + ); + await updateMessage(ds.larkAppId, oldStreamCardId, frozenJson); + } catch (err) { + logger.warn(`[${tagPrefix}] freeze source-chat card failed: ${err instanceof Error ? err.message : err}`); + } } dashboardEventBus.publish({ @@ -1831,10 +2170,6 @@ export async function transferSession( }, }); - // forkWorker with resume=true — TmuxBackend.spawn detects the surviving - // `bmx-` session and re-attaches instead of creating a new one. - fkw(ds, '', /*resume*/true); - logger.info( `[${tagPrefix}] transferred ${oldChatId} → ${targetChatId} ` + `(anchor ${oldAnchor.substring(0, 8)} → ${newAnchor.substring(0, 8)})`, @@ -1853,18 +2188,372 @@ function resolvesToHome(p: string): boolean { catch { return p === homedir(); } } +export function codexAppCleanInputAcceptedForSession(ds: DaemonSession): boolean { + try { + const botCfg = getBot(ds.larkAppId).config; + const effectiveCliId = ds.session.cliId ?? botCfg.cliId; + return effectiveCliId === 'codex-app' + && botCfg.codexAppCleanInput === true + && !ds.adoptedFrom; + } catch { + // Admission metadata is optional for every other CLI. If bot config is + // temporarily unavailable, fail closed instead of leaking a clean sidecar. + return false; + } +} + +function cloneFrozenCodexAppInput( + input: CodexAppTurnInput | undefined, + turnId?: string, +): CodexAppTurnInput | undefined { + if (!input) return undefined; + const cloned = structuredClone(input); + if (turnId && !cloned.clientUserMessageId) cloned.clientUserMessageId = turnId; + return cloned; +} + function codexAppInputForSession( ds: DaemonSession, input: CodexAppTurnInput | undefined, turnId?: string, ): CodexAppTurnInput | undefined { if (!input) return undefined; + if (!codexAppCleanInputAcceptedForSession(ds)) return undefined; + return cloneFrozenCodexAppInput(input, turnId); +} + +function codexAppDeliverySinkForTurn( + ds: DaemonSession, + turnId: string, + dispatchAttempt: number | undefined, +): CodexAppDeliverySink { + const armedThrough = ds.suppressedFinalOutputTurns?.get(turnId); + if (dispatchAttempt !== undefined + && armedThrough !== undefined + && dispatchAttempt <= armedThrough) return 'suppressed'; + if (ds.session.docCommentTargets?.[turnId] || ds.docCommentTurns?.has(turnId)) { + return 'doc_comment'; + } + if (ds.pendingWaitPromises?.has(turnId)) return 'http_wait'; + if (ds.asyncTriggerResults?.has(turnId)) return 'http_async'; + return 'lark'; +} + +/** A recovered transient/non-IM sink has no safe provider to replay into. + * Treat it as consumed so the runner can advance, but never fall through to a + * Lark card. Doc-comment replay also fails closed: chunk posting has no durable + * per-chunk effect journal yet, so blind crash replay could duplicate comments. */ +function codexAppDeliveryMustFailClosed( + ds: DaemonSession, + entry: CodexAppDispatchLedgerEntry, +): boolean { + const sink = entry.deliverySink + ?? (ds.session.docCommentTargets?.[entry.turnId] + ? 'doc_comment' + : ds.chatId.startsWith('http_wait_') + ? 'http_wait' + : ds.chatId.startsWith('http_async_') + ? 'http_async' + : 'lark'); + if (sink === 'suppressed') return true; + if (sink === 'doc_comment') return !ds.docCommentTurns?.has(entry.turnId); + if (sink === 'http_wait') return !ds.pendingWaitPromises?.has(entry.turnId); + if (sink === 'http_async') return !ds.asyncTriggerResults?.has(entry.turnId); + return false; +} + +function acceptCodexAppDispatch( + ds: DaemonSession, + payload: { + content: string; + codexAppInput?: CodexAppTurnInput; + replyTurnId?: string; + replyTarget?: FrozenSessionReplyTarget; + quoteTargetId?: string; + replyTargetSenderOpenId?: string; + replyTargetSenderIsBot?: boolean; + queuedActivationToken?: string; + }, + turnId: string | undefined, + dispatchAttempt: number | undefined, + vcMeetingImTurnOrigin: ReturnType, + opts: { persist?: boolean } = {}, +): string | undefined { const botCfg = getBot(ds.larkAppId).config; const effectiveCliId = ds.session.cliId ?? botCfg.cliId; - if (effectiveCliId !== 'codex-app' || botCfg.codexAppCleanInput !== true || ds.adoptedFrom) return undefined; - return turnId && !input.clientUserMessageId - ? { ...input, clientUserMessageId: turnId } - : input; + if (effectiveCliId !== 'codex-app' || !turnId) return undefined; + const dispatchId = randomUUID(); + const priorLedger = ds.session.codexAppDispatchLedger; + ds.session.codexAppDispatchLedger = appendAcceptedCodexAppDispatch( + ds.session.codexAppDispatchLedger ?? [], + { + dispatchId, + turnId, + ...(payload.queuedActivationToken + ? { queuedActivationToken: payload.queuedActivationToken } + : {}), + ...(payload.replyTurnId ? { replyTurnId: payload.replyTurnId } : {}), + ...(payload.replyTarget ? { replyTarget: payload.replyTarget } : {}), + ...(payload.quoteTargetId ? { quoteTargetId: payload.quoteTargetId } : {}), + ...(payload.replyTargetSenderOpenId + ? { replyTargetSenderOpenId: payload.replyTargetSenderOpenId } + : {}), + ...(payload.replyTargetSenderIsBot !== undefined + ? { replyTargetSenderIsBot: payload.replyTargetSenderIsBot } + : {}), + deliverySink: codexAppDeliverySinkForTurn(ds, turnId, dispatchAttempt), + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + content: payload.content, + ...(payload.codexAppInput ? { codexAppInput: payload.codexAppInput } : {}), + ...(vcMeetingImTurnOrigin ? { vcMeetingImTurnOrigin } : {}), + }, + ); + if (opts.persist !== false) { + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.codexAppDispatchLedger = priorLedger; + throw err; + } + } + return dispatchId; +} + +function rollbackAcceptedCodexAppDispatch( + ds: DaemonSession, + dispatchId: string | undefined, + turnId: string | undefined, + dispatchAttempt: number | undefined, +): void { + if (!dispatchId || !turnId) return; + const cancelled = cancelCodexAppDispatch(ds.session.codexAppDispatchLedger ?? [], { + dispatchId, + turnId, + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + }); + if (!cancelled.ok) return; + const priorLedger = ds.session.codexAppDispatchLedger; + ds.session.codexAppDispatchLedger = cancelled.ledger; + try { + sessionStore.updateSession(ds.session); + if (!hasUnsettledCodexAppDispatch(ds.session.codexAppDispatchLedger)) { + void Promise.resolve(callbacks?.onCodexAppLedgerDrained?.(ds)).catch(err => { + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] post-rollback ledger-drain cleanup failed: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + }); + } + } catch { + ds.session.codexAppDispatchLedger = priorLedger; + } +} + +type QueuedWorkerForkSnapshot = { + queued: true; + queuedPrompt: Session['queuedPrompt']; + queuedCodexAppText: Session['queuedCodexAppText']; + queuedCodexAppMessageContext: Session['queuedCodexAppMessageContext']; + queuedActivationPending: Session['queuedActivationPending']; + queuedActivationToken: Session['queuedActivationToken']; + queuedActivationInput: Session['queuedActivationInput']; + queuedActivationTurnId: Session['queuedActivationTurnId']; + queuedActivationDispatchAttempt: Session['queuedActivationDispatchAttempt']; + queuedActivationResume: Session['queuedActivationResume']; + initialStartPending: DaemonSession['initialStartPending']; +}; + +type QueuedActivationJournalSnapshot = Pick< + Session, + | 'queuedActivationPending' + | 'queuedActivationToken' + | 'queuedActivationInput' + | 'queuedActivationTurnId' + | 'queuedActivationDispatchAttempt' + | 'queuedActivationResume' + | 'queuedPrompt' + | 'queuedCodexAppText' + | 'queuedCodexAppMessageContext' + | 'pendingRepoSetup' +>; + +function snapshotQueuedActivationJournal(session: Session): QueuedActivationJournalSnapshot { + return { + queuedActivationPending: session.queuedActivationPending, + queuedActivationToken: session.queuedActivationToken, + queuedActivationInput: session.queuedActivationInput, + queuedActivationTurnId: session.queuedActivationTurnId, + queuedActivationDispatchAttempt: session.queuedActivationDispatchAttempt, + queuedActivationResume: session.queuedActivationResume, + queuedPrompt: session.queuedPrompt, + queuedCodexAppText: session.queuedCodexAppText, + queuedCodexAppMessageContext: session.queuedCodexAppMessageContext, + pendingRepoSetup: session.pendingRepoSetup + ? structuredClone(session.pendingRepoSetup) + : undefined, + }; +} + +function restoreQueuedActivationJournal( + session: Session, + snapshot: QueuedActivationJournalSnapshot, +): void { + Object.assign(session, snapshot); +} + +function clearQueuedActivationJournal(session: Session): void { + session.queuedActivationPending = undefined; + session.queuedActivationToken = undefined; + session.queuedActivationInput = undefined; + session.queuedActivationTurnId = undefined; + session.queuedActivationDispatchAttempt = undefined; + session.queuedActivationResume = undefined; + session.queuedPrompt = undefined; + session.queuedCodexAppText = undefined; + session.queuedCodexAppMessageContext = undefined; + session.pendingRepoSetup = undefined; +} + +/** A worker disappeared before the exact activation ACK. Re-park non-Codex + * work from the retained exact input; ACK loss may duplicate, but N can never + * be silently replaced by a later inbound turn in the same daemon lifetime. */ +function reparkUnsubmittedQueuedActivation(ds: DaemonSession, reason: string): boolean { + if (!ds.session.queuedActivationPending + || !ds.session.queuedActivationToken + || ds.session.cliId === 'codex-app') return false; + ds.session.queued = true; + ds.session.queuedActivationPending = undefined; + ds.session.queuedActivationToken = undefined; + ds.initialStartPending = false; + ds.initialStartClaimToken = undefined; + ds.pendingPrompt ??= ds.session.queuedPrompt; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + // Memory remains explicitly parked while the old durable marker still + // provides restart recovery on disk. + logger.error( + `[${tag(ds)}] Failed to persist queued activation re-park after ${reason}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + logger.warn(`[${tag(ds)}] Re-parked unacknowledged queued activation after ${reason}`); + return true; +} + +/** The opening activation was ACKed, but one of the turns held behind its + * runtime reservation was not accepted before that worker exited. Promote the + * exact FIFO head into a new durable queued activation so the next inbound or + * Dashboard activation reforks it before every remaining tail item. */ +function reparkQueuedActivationFollowUpTail(ds: DaemonSession, reason: string): boolean { + if (!ds.initialStartPending || ds.session.queuedActivationPending) return false; + const next = ds.pendingQueuedActivationFollowUps?.[0]; + if (!next) return false; + const matchingCodexEntries = ds.session.cliId === 'codex-app' + ? (ds.session.codexAppDispatchLedger ?? []).filter(entry => + (entry.state === 'accepted' || entry.state === 'prepared') + && entry.turnId === next.turnId + && entry.dispatchAttempt === next.dispatchAttempt) + : []; + if (matchingCodexEntries.length > 1) { + logger.error( + `[${tag(ds)}] Cannot re-park queued follow-up after ${reason}: ` + + `${matchingCodexEntries.length} Codex entries match turn ${next.turnId}`, + ); + return false; + } + const retainedCodexEntry = matchingCodexEntries[0]; + const retainedCodexToken = retainedCodexEntry + ? (retainedCodexEntry.queuedActivationToken ?? randomUUID()) + : undefined; + // A failed daemon→worker IPC normally rolls its newly accepted Codex entry + // back. If that rollback persistence itself failed, the durable FIFO remains + // authoritative: recover it through a tokened ACK journal instead of + // creating an invalid queued+unsettled hybrid. + ds.session.queued = !retainedCodexEntry; + ds.session.queuedPrompt = next.cliInput.content; + ds.session.queuedCodexAppText = next.cliInput.codexAppInput?.text; + ds.session.queuedCodexAppMessageContext = undefined; + ds.session.queuedActivationInput = next.cliInput; + ds.session.queuedActivationTurnId = next.turnId; + ds.session.queuedActivationDispatchAttempt = next.dispatchAttempt; + ds.session.queuedActivationPending = retainedCodexEntry ? true : undefined; + ds.session.queuedActivationToken = retainedCodexToken; + if (retainedCodexEntry && retainedCodexToken) { + retainedCodexEntry.queuedActivationToken = retainedCodexToken; + } + ds.pendingQueuedActivationFollowUps!.shift(); + if (ds.pendingQueuedActivationFollowUps!.length === 0) { + ds.pendingQueuedActivationFollowUps = undefined; + } + ds.pendingPrompt = next.cliInput.content; + ds.initialStartPending = false; + ds.initialStartClaimToken = undefined; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + // Keep the exact head parked in memory. The caller has already fenced the + // dead worker, so a later inbound can safely retry this owner even if the + // durable projection is temporarily unavailable. + logger.error( + `[${tag(ds)}] Failed to persist queued follow-up re-park after ${reason}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + logger.warn(`[${tag(ds)}] Re-parked unaccepted queued activation follow-up after ${reason}`); + return true; +} + +export const __testOnly_reparkQueuedActivationFollowUpTail = reparkQueuedActivationFollowUpTail; + +type AcceptedWorkerForkDispatch = { + dispatchId: string; + turnId: string; + dispatchAttempt?: number; +}; + +/** Compensate only the durable mutations made before a worker accepts its init + * IPC. This is one synchronous call stack, so removing the exact dispatch that + * this fork appended cannot race a later worker transition or damage the FIFO + * that existed before the attempt. Persist the queued payload and ledger in one + * write so a daemon crash cannot observe only half of the compensation. */ +function rollbackWorkerForkPreInit( + ds: DaemonSession, + queuedSnapshot: QueuedWorkerForkSnapshot | undefined, + acceptedDispatch: AcceptedWorkerForkDispatch | undefined, +): void { + const exactActivationInput = ds.session.queuedActivationInput; + const exactActivationTurnId = ds.session.queuedActivationTurnId; + const exactActivationDispatchAttempt = ds.session.queuedActivationDispatchAttempt; + const exactActivationResume = ds.session.queuedActivationResume; + let changed = false; + if (acceptedDispatch) { + const cancelled = cancelCodexAppDispatch( + ds.session.codexAppDispatchLedger ?? [], + acceptedDispatch, + ); + if (!cancelled.ok) { + throw new Error(`failed to cancel pre-init Codex App dispatch: ${cancelled.error}`); + } + ds.session.codexAppDispatchLedger = cancelled.ledger; + changed = true; + } + if (queuedSnapshot) { + ds.session.queued = queuedSnapshot.queued; + restoreQueuedActivationJournal(ds.session, queuedSnapshot); + // The activation may have folded a triggering group reply into the final + // init payload. Retain that exact retry body even though the in-flight + // marker/token are rolled back with the fenced child. + ds.session.queuedActivationPending = undefined; + ds.session.queuedActivationToken = undefined; + ds.session.queuedActivationInput = exactActivationInput; + ds.session.queuedActivationTurnId = exactActivationTurnId; + ds.session.queuedActivationDispatchAttempt = exactActivationDispatchAttempt; + ds.session.queuedActivationResume = exactActivationResume; + ds.initialStartPending = queuedSnapshot.initialStartPending; + changed = true; + } + if (changed) sessionStore.updateSession(ds.session); } /** Send one normal (non-raw) worker turn while applying the per-bot Codex App @@ -1880,12 +2569,18 @@ export function sendWorkerInput( ): boolean { if (!ds.worker || ds.worker.killed) return false; const normalized = typeof payload === 'string' ? { content: payload } : payload; - const codexAppInput = codexAppInputForSession(ds, normalized.codexAppInput, turnId); + const effectiveCliId = ds.session.cliId ?? getBot(ds.larkAppId).config.cliId; + const effectiveTurnId = turnId ?? (effectiveCliId === 'codex-app' + ? `codex-app-dispatch-${randomUUID()}` + : undefined); + const replyTurnId = turnId ? undefined : ds.currentReplyTarget?.turnId; + const routingTurnId = turnId ?? replyTurnId; + const replyContext = frozenReplyContextForTurn(ds, routingTurnId); + const vcMeetingImTurnOrigin = resolveVcMeetingImTurnOrigin(ds.session, routingTurnId); let nativeSessionTitlePrompt: string | undefined; let nativeSessionTitle: string | undefined; if (ds.session.nativeSessionTitleAwaitingContent && !ds.session.nativeSessionTitleUserDefined && !ds.adoptedFrom) { const bot = getBot(ds.larkAppId); - const effectiveCliId = ds.session.cliId ?? bot.config.cliId; if (effectiveCliId === 'codex') { nativeSessionTitlePrompt = extractBotmuxLarkNativeSessionTitlePrompt( normalized.codexAppInput?.text ?? normalized.content, @@ -1903,22 +2598,283 @@ export function sendWorkerInput( } } } - const vcMeetingImTurnOrigin = resolveVcMeetingImTurnOrigin(ds.session, turnId); - ds.worker.send({ - type: 'message', - content: normalized.content, + + if (hasQueuedActivationAdmissionGate(ds)) { + const queuedTurnId = effectiveTurnId + ?? routingTurnId + ?? `queued-activation-followup-${randomUUID()}`; + try { + admitQueuedActivationTail(ds, { + userPrompt: normalized.content, + cliInput: { + content: normalized.content, + ...(normalized.codexAppInput + ? { codexAppInput: normalized.codexAppInput } + : {}), + }, + turnId: queuedTurnId, + ...(opts.dispatchAttempt !== undefined + ? { dispatchAttempt: opts.dispatchAttempt } + : {}), + }); + logger.info( + `[${tag(ds)}] Staged turn ${queuedTurnId} behind queued activation ACK`, + ); + return true; + } catch (err) { + logger.error( + `[${tag(ds)}] Failed to durably stage turn ${queuedTurnId} behind activation: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + const codexAppInput = codexAppInputForSession( + ds, + normalized.codexAppInput, + effectiveTurnId, + ); + const codexAppDispatchId = acceptCodexAppDispatch( + ds, + { + content: normalized.content, + ...(codexAppInput ? { codexAppInput } : {}), + ...(replyTurnId ? { replyTurnId } : {}), + replyTarget: replyContext.target, + ...(replyContext.quoteTargetId ? { quoteTargetId: replyContext.quoteTargetId } : {}), + ...(replyContext.replyTargetSenderOpenId + ? { replyTargetSenderOpenId: replyContext.replyTargetSenderOpenId } + : {}), + ...(replyContext.replyTargetSenderIsBot !== undefined + ? { replyTargetSenderIsBot: replyContext.replyTargetSenderIsBot } + : {}), + }, + effectiveTurnId, + opts.dispatchAttempt, + vcMeetingImTurnOrigin, + ); + try { + ds.worker.send({ + type: 'message', + content: normalized.content, + ...(codexAppInput ? { codexAppInput } : {}), + ...(nativeSessionTitle ? { nativeSessionTitle } : {}), + ...(nativeSessionTitlePrompt ? { nativeSessionTitlePrompt } : {}), + ...(effectiveTurnId ? { turnId: effectiveTurnId } : {}), + ...(replyTurnId ? { replyTurnId } : {}), + ...(opts.dispatchAttempt !== undefined ? { dispatchAttempt: opts.dispatchAttempt } : {}), + ...(codexAppDispatchId ? { codexAppDispatchId } : {}), + ...(vcMeetingImTurnOrigin + ? { vcMeetingImTurnOrigin } + : {}), + } as DaemonToWorker); + } catch { + rollbackAcceptedCodexAppDispatch( + ds, + codexAppDispatchId, + effectiveTurnId, + opts.dispatchAttempt, + ); + return false; + } + return true; +} + +/** Promote the oldest durable activation successor into a fresh tokened + * journal and (for Codex App) its single accepted-ledger owner in one store + * update, then hand it to the current worker. The tail is never shifted merely + * because ChildProcess.send returned: the promoted journal survives until the + * adapter ACK carrying this token arrives. */ +export function promoteQueuedActivationTail( + ds: DaemonSession, + opts: { send?: boolean } = {}, +): boolean { + if (ds.session.queuedActivationPending) return true; + if (opts.send !== false + && (!ds.worker || ds.worker.killed)) return false; + const ordered = [...(ds.session.queuedActivationTail ?? [])] + .sort((a, b) => a.order - b.order || a.id.localeCompare(b.id)); + const head = ordered[0]; + if (!head) return false; + + const priorJournal = snapshotQueuedActivationJournal(ds.session); + const priorTail = ds.session.queuedActivationTail?.map(entry => ({ + ...entry, + cliInput: { + ...entry.cliInput, + ...(entry.cliInput.codexAppInput + ? { codexAppInput: structuredClone(entry.cliInput.codexAppInput) } + : {}), + }, + })); + const priorLedger = ds.session.codexAppDispatchLedger?.map(entry => ({ ...entry })); + const priorPendingPrompt = ds.pendingPrompt; + const token = randomUUID(); + const replyContext = frozenReplyContextForTurn(ds, head.turnId); + // The tail entry crossed its admission boundary with the clean-input gate + // already frozen. Do not re-run that immediate bot-config gate at promotion: + // a true→false toggle between N+1 admission and N's ACK would otherwise + // discard N+1's exact sidecar/hidden context. Only fill the deterministic id + // for legacy entries persisted before admission began stamping it. + const codexAppInput = cloneFrozenCodexAppInput( + head.cliInput.codexAppInput, + head.turnId, + ); + const exactInput: CliTurnPayload = { + content: head.cliInput.content, ...(codexAppInput ? { codexAppInput } : {}), - ...(nativeSessionTitle ? { nativeSessionTitle } : {}), - ...(nativeSessionTitlePrompt ? { nativeSessionTitlePrompt } : {}), - ...(turnId ? { turnId } : {}), - ...(opts.dispatchAttempt !== undefined ? { dispatchAttempt: opts.dispatchAttempt } : {}), - ...(vcMeetingImTurnOrigin - ? { vcMeetingImTurnOrigin } - : {}), - } as DaemonToWorker); + }; + const vcMeetingImTurnOrigin = resolveVcMeetingImTurnOrigin(ds.session, head.turnId); + + ds.session.queued = false; + ds.session.queuedActivationPending = true; + ds.session.queuedActivationToken = token; + ds.session.queuedActivationInput = exactInput; + ds.session.queuedActivationTurnId = head.turnId; + ds.session.queuedActivationDispatchAttempt = head.dispatchAttempt; + ds.session.queuedActivationResume = ds.hasHistory; + ds.session.queuedActivationTail = ordered.slice(1); + if (ds.session.queuedActivationTail.length === 0) { + ds.session.queuedActivationTail = undefined; + } + ds.pendingPrompt = exactInput.content; + ds.initialStartPending = true; + + let codexAppDispatchId: string | undefined; + try { + codexAppDispatchId = acceptCodexAppDispatch( + ds, + { + content: exactInput.content, + ...(codexAppInput ? { codexAppInput } : {}), + queuedActivationToken: token, + replyTarget: replyContext.target, + ...(replyContext.quoteTargetId ? { quoteTargetId: replyContext.quoteTargetId } : {}), + ...(replyContext.replyTargetSenderOpenId + ? { replyTargetSenderOpenId: replyContext.replyTargetSenderOpenId } + : {}), + ...(replyContext.replyTargetSenderIsBot !== undefined + ? { replyTargetSenderIsBot: replyContext.replyTargetSenderIsBot } + : {}), + }, + head.turnId, + head.dispatchAttempt, + vcMeetingImTurnOrigin, + { persist: false }, + ); + sessionStore.updateSession(ds.session); + } catch (err) { + restoreQueuedActivationJournal(ds.session, priorJournal); + ds.session.queuedActivationTail = priorTail; + ds.session.codexAppDispatchLedger = priorLedger; + ds.pendingPrompt = priorPendingPrompt; + logger.error( + `[${tag(ds)}] Failed to atomically promote queued activation tail ${head.id}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + + if (opts.send === false) return true; + try { + ds.worker!.send({ + type: 'message', + content: exactInput.content, + ...(codexAppInput ? { codexAppInput } : {}), + turnId: head.turnId, + ...(head.dispatchAttempt !== undefined + ? { dispatchAttempt: head.dispatchAttempt } + : {}), + ...(codexAppDispatchId ? { codexAppDispatchId } : {}), + queuedActivationToken: token, + ...(vcMeetingImTurnOrigin ? { vcMeetingImTurnOrigin } : {}), + } as DaemonToWorker); + } catch (err) { + // Durable ownership already moved to the journal (and Codex ledger). Never + // append another owner on retry; fence this IPC generation and let recovery + // replay the exact tokened head. + logger.error( + `[${tag(ds)}] Worker IPC rejected promoted activation ${head.id}; retaining journal: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + try { ds.worker!.kill(); } catch { /* exit/error path will fence runtime */ } + } return true; } +export type QueuedActivationTailReservation = Pick & { + /** Clean-input decision captured synchronously with FIFO order, before any + * caller-specific async prompt construction. This field is runtime-only. */ + codexAppInputAccepted?: boolean; +}; + +/** Reserve arrival order synchronously, before any caller-specific prompt + * rendering may await. Gaps are harmless; reusing an order after a failed + * durable admission would not be. */ +export function reserveQueuedActivationTailAdmission( + ds: DaemonSession, +): QueuedActivationTailReservation { + const order = (ds.session.queuedActivationTailNextOrder ?? 0) + 1; + ds.session.queuedActivationTailNextOrder = order; + return { + id: randomUUID(), + order, + codexAppInputAccepted: codexAppCleanInputAcceptedForSession(ds), + }; +} + +/** Admit one exact successor behind a queued activation. The response boundary + * is the session-store write: callers must not report acceptance or use live + * worker IPC unless this returns. `reservation` lets routes reserve FIFO order + * before asynchronous prompt construction without duplicating persistence + * logic. */ +export function admitQueuedActivationTail( + ds: DaemonSession, + entry: Omit, + reservation: QueuedActivationTailReservation = reserveQueuedActivationTailAdmission(ds), + opts: { codexAppInputGateFrozen?: boolean } = {}, +): QueuedActivationTailEntry { + const acceptedCodexAppInput = opts.codexAppInputGateFrozen === true + ? cloneFrozenCodexAppInput(entry.cliInput.codexAppInput, entry.turnId) + : reservation.codexAppInputAccepted === true + ? cloneFrozenCodexAppInput(entry.cliInput.codexAppInput, entry.turnId) + : undefined; + const admitted: QueuedActivationTailEntry = { + id: reservation.id, + order: reservation.order, + ...entry, + cliInput: { + content: entry.cliInput.content, + ...(acceptedCodexAppInput ? { codexAppInput: acceptedCodexAppInput } : {}), + }, + }; + const priorTail = ds.session.queuedActivationTail; + const next = [...(priorTail ?? [])]; + if (!next.some(candidate => candidate.id === admitted.id)) next.push(admitted); + next.sort((a, b) => a.order - b.order || a.id.localeCompare(b.id)); + ds.session.queuedActivationTail = next; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.queuedActivationTail = priorTail; + throw err; + } + return admitted; +} + +/** True while a live worker's opening activation still owns submission order. + * Every ingress that sees this state must use admitQueuedActivationTail rather + * than ordinary worker IPC. */ +export function hasQueuedActivationAdmissionGate(ds: DaemonSession): boolean { + return ds.session.queuedActivationPending === true + || (ds.session.queuedActivationTail?.length ?? 0) > 0 + || (ds.queuedActivationTailAdmissionsOutstanding ?? 0) > 0 + || ds.queuedActivationTailReleasePending !== undefined + || (ds.initialStartPending === true + && ds.session.queuedActivationInput !== undefined); +} + export function forkWorker( ds: DaemonSession, promptInput: string | CliTurnPayload, @@ -1926,6 +2882,9 @@ export function forkWorker( resume?: boolean; turnId?: string; dispatchAttempt?: number; + /** The payload is an exact retained activation journal. Do not re-read the + * live clean-input feature flag on retry/replay. */ + codexAppInputGateFrozen?: boolean; } = false, ): void { // Device enrollment briefly freezes every daemon before the one-way host @@ -1941,45 +2900,132 @@ export function forkWorker( logger.info(`[${tag(ds)}] worker spawn deferred during device credential activation`); return; } - const cb = requireCallbacks(); - const bot = getBot(ds.larkAppId); - const botCfg = bot.config; const promptPayload = typeof promptInput === 'string' ? { content: promptInput } : promptInput; const prompt = promptPayload.content; - // 不变式：一旦真正起 CLI，会话就不再是「待办池(queued)」parked 态。无论由哪条 - // 路径触发（激活按钮 / 拖到进行中 / 群里来消息抢先起会话），都在此清掉 queued - // 标记并落盘——否则重启后会被当 parked 恢复成 hasHistory:false 而丢掉真历史。 - if (ds.session.queued) { - ds.session.queued = false; - ds.session.queuedPrompt = undefined; - ds.session.queuedCodexAppText = undefined; - ds.session.queuedCodexAppMessageContext = undefined; - sessionStore.updateSession(ds.session); - } - // worker.js lives in the same directory as daemon.js (src/) - const workerPath = join(__dirname, '..', 'worker.js'); - const t = tag(ds); - ds.localProcessAttestation = undefined; - let resume = false; let initTurnId: string | undefined; let initDispatchAttempt: number | undefined; + let initCodexAppInputGateFrozen = promptInput === ds.session.queuedActivationInput; if (typeof resumeOrTurnId === 'string') { initTurnId = resumeOrTurnId; } else if (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null) { resume = resumeOrTurnId.resume === true; initTurnId = resumeOrTurnId.turnId; initDispatchAttempt = resumeOrTurnId.dispatchAttempt; + initCodexAppInputGateFrozen ||= resumeOrTurnId.codexAppInputGateFrozen === true; } else { resume = resumeOrTurnId; } + if (ds.session.queuedActivationPending && ds.session.queuedActivationResume !== undefined) { + resume = ds.session.queuedActivationResume; + } + + // Central double-fork guard. A live worker with durable ownership must never + // be killed and replaced: empty reforks are rejected, while a real follow-up + // is appended through the existing worker's ordinary durable FIFO. Keep this + // before queued/cwd/sandbox mutations so a rejected refork is side-effect free. + if (ds.worker && !ds.worker.killed + && hasProtectedSessionMutationOwnership(ds)) { + if (prompt.length === 0) { + logger.warn(`[${tag(ds)}] Refused empty worker refork while durable activation ownership is non-empty`); + return; + } + if (hasQueuedActivationAdmissionGate(ds)) { + const turnId = initTurnId + ?? ds.currentReplyTarget?.turnId + ?? `queued-activation-followup-${randomUUID()}`; + admitQueuedActivationTail(ds, { + userPrompt: prompt, + cliInput: { + content: prompt, + ...(promptPayload.codexAppInput + ? { codexAppInput: promptPayload.codexAppInput } + : {}), + }, + turnId, + ...(initDispatchAttempt !== undefined + ? { dispatchAttempt: initDispatchAttempt } + : {}), + }); + logger.info( + `[${tag(ds)}] Staged double-fork prompt behind queued activation ACK ` + + `(turn=${turnId})`, + ); + return; + } + const routed = sendWorkerInput(ds, promptPayload, initTurnId, { + ...(initDispatchAttempt !== undefined ? { dispatchAttempt: initDispatchAttempt } : {}), + }); + logger[routed ? 'info' : 'warn']( + `[${tag(ds)}] ${routed ? 'Routed' : 'Failed to route'} double-fork prompt through existing durable owner`, + ); + return; + } + + const cb = requireCallbacks(); + const bot = getBot(ds.larkAppId); + const botCfg = bot.config; + // A bare /repo placeholder (and a non-Codex empty group-join setup) owns no + // model turn. Starting its CLI with an empty prompt must not mint a queued + // activation token: the worker has nothing to submit and could never ACK it. + // Real raw openings and clean Codex sidecars remain tokened. + if (ds.session.queued + && ds.session.pendingRepoSetup + && prompt.length === 0 + && !promptPayload.codexAppInput + && !ds.pendingRawInput + && (ds.session.queuedActivationTail?.length ?? 0) === 0) { + const priorJournal = snapshotQueuedActivationJournal(ds.session); + ds.session.queued = false; + clearQueuedActivationJournal(ds.session); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.queued = true; + restoreQueuedActivationJournal(ds.session, priorJournal); + throw err; + } + } + const queuedForkSnapshot: QueuedWorkerForkSnapshot | undefined = ds.session.queued + ? { + queued: true, + queuedPrompt: ds.session.queuedPrompt, + queuedCodexAppText: ds.session.queuedCodexAppText, + queuedCodexAppMessageContext: ds.session.queuedCodexAppMessageContext, + queuedActivationPending: ds.session.queuedActivationPending, + queuedActivationToken: ds.session.queuedActivationToken, + queuedActivationInput: ds.session.queuedActivationInput, + queuedActivationTurnId: ds.session.queuedActivationTurnId, + queuedActivationDispatchAttempt: ds.session.queuedActivationDispatchAttempt, + queuedActivationResume: ds.session.queuedActivationResume, + initialStartPending: ds.initialStartPending, + } + : undefined; + let acceptedForkDispatch: AcceptedWorkerForkDispatch | undefined; + let spawnedWorker: ChildProcess | undefined; + let worker!: ChildProcess; + let startupState!: WorkerStartupState; + let initMsg!: DaemonToWorker; + let agentCfg!: ReturnType; + const t = tag(ds); + ds.localProcessAttestation = undefined; + try { + // worker.js lives in the same directory as daemon.js (src/) + const workerPath = join(__dirname, '..', 'worker.js'); // Per-turn authority is never inferred from mutable session state. Human // message routes pass their accepted Lark message id explicitly; restore, // scheduler, card retry and other system starts stay unattributed. Falling // back to currentReplyTarget here would let a later system prompt reuse an // older human turn after a worker replacement. - const initAttributionTurnId = initTurnId; + let initAttributionTurnId = initTurnId + ?? (queuedForkSnapshot ? ds.session.pendingRepoSetup?.turnId : undefined); + // Reply routing is frozen only from the same explicitly accepted turn. A + // system prompt must not borrow an older mutable currentReplyTarget. + const initReplyTurnId = initTurnId; + const initReplyContext = prompt.length > 0 + ? frozenReplyContextForTurn(ds, initReplyTurnId) + : undefined; // A fork() whose cwd no longer exists emits an unhandled 'error' (spawn // ENOENT) that crashes the WHOLE daemon (→ pm2 crash-loop). Fall back to @@ -2046,7 +3092,9 @@ export function forkWorker( // Guard against double-fork: if a worker is already running, kill it first if (ds.worker && !ds.worker.killed) { logger.warn(`[${t}] Worker already running (pid: ${ds.worker.pid}), killing before re-fork`); - try { ds.worker.send({ type: 'close' } as DaemonToWorker); } catch { /* ignore */ } + try { + ds.worker.send({ type: 'close' } as DaemonToWorker); + } catch { /* ignore */ } try { ds.worker.kill(); } catch { /* ignore */ } ds.worker = null; ds.workerPort = null; @@ -2069,7 +3117,10 @@ export function forkWorker( // real bmx-; without this, bmx-diag- + its .ansi file would leak. if (!ds.initConfig?.adoptMode && !ds.adoptedFrom) reclaimParkedCrashDiagnostic(ds); - const agentCfg = sessionAgentConfig(ds, botCfg); + agentCfg = sessionAgentConfig(ds, botCfg); + if (!initTurnId && prompt.length > 0 && agentCfg.cliId === 'codex-app') { + initAttributionTurnId = `codex-app-dispatch-${randomUUID()}`; + } ensureCliEnv(agentCfg.cliId, agentCfg.cliPathOverride); let nativeSessionTitle: string | undefined; let nativeSessionTitlePrompt: string | undefined; @@ -2121,6 +3172,17 @@ export function forkWorker( // (`~/.claude.json` for claude, `.claude-runtime/.claude.json` for seed). const familyAdapter = createCliAdapterSync(agentCfg.cliId, agentCfg.cliPathOverride); if (familyAdapter.claudeStateJsonPath) ensureClaudeFolderTrust(cwd, familyAdapter.claudeStateJsonPath); + const resolvedBackendType = resolvePairedSpawnBackendType( + agentCfg.cliId, + ds.session.backendType, + botCfg.backendType, + config.daemon.backendType, + ); + if (ds.session.cliId !== agentCfg.cliId || ds.session.backendType !== resolvedBackendType) { + ds.session.cliId = agentCfg.cliId; + ds.session.backendType = resolvedBackendType; + sessionStore.updateSession(ds.session); + } // Prepend ~/.botmux/bin to PATH so CLIs can call `botmux send` etc. // The wrapper script there is written by the daemon at startup. @@ -2128,7 +3190,103 @@ export function forkWorker( const pathWithBotmux = prependBotmuxBin(botmuxBinDir, process.env.PATH); const forkEnv = workerForkEnv(process.env); - const worker = fork(workerPath, [], { + // Dequeue only after every earlier launch-preparation write has finished. + // The exact input remains journaled until the worker confirms its adapter + // submission boundary; daemon send/ready are intentionally too early. + const recoveredActivationLedgerEntry = ds.session.queuedActivationPending + ? [...(ds.session.codexAppDispatchLedger ?? [])] + .reverse() + .find(entry => entry.state === 'accepted' || entry.state === 'prepared') + : undefined; + const queuedActivationToken = queuedForkSnapshot + ? randomUUID() + : ds.session.queuedActivationPending + ? (ds.session.queuedActivationToken + ?? recoveredActivationLedgerEntry?.queuedActivationToken + ?? randomUUID()) + : undefined; + if (queuedForkSnapshot) { + ds.session.queued = false; + ds.session.queuedActivationPending = true; + ds.session.queuedActivationToken = queuedActivationToken; + ds.session.queuedActivationResume = resume; + ds.initialStartPending = true; + } else if (ds.session.queuedActivationPending && queuedActivationToken) { + // Migrate journals written before the token/ACK protocol. The queued + // activation is the newest unsettled FIFO entry; stamp the same token onto + // both journal and entry before the recovery worker can submit it. + let migrated = ds.session.queuedActivationToken !== queuedActivationToken; + ds.session.queuedActivationToken = queuedActivationToken; + ds.initialStartPending = true; + if (recoveredActivationLedgerEntry + && recoveredActivationLedgerEntry.queuedActivationToken !== queuedActivationToken) { + recoveredActivationLedgerEntry.queuedActivationToken = queuedActivationToken; + migrated = true; + } + if (migrated) sessionStore.updateSession(ds.session); + } + + // Snapshot the prior durable FIFO before accepting this fork's new prompt. + // A pure reattach receives the full snapshot; a refork carrying N+1 restores + // old N first and reserves N+1 through the normal worker write path. + const codexAppRecoveredDispatches = agentCfg.cliId === 'codex-app' + ? (ds.session.codexAppDispatchLedger ?? []).map(entry => ({ ...entry })) + : []; + const promptCodexAppInput = initCodexAppInputGateFrozen + ? cloneFrozenCodexAppInput(promptPayload.codexAppInput, initAttributionTurnId) + : codexAppInputForSession(ds, promptPayload.codexAppInput, initAttributionTurnId); + if (queuedForkSnapshot) { + ds.session.queuedActivationInput = { + content: prompt, + ...(promptCodexAppInput ? { codexAppInput: promptCodexAppInput } : {}), + }; + ds.session.queuedActivationTurnId = initAttributionTurnId; + ds.session.queuedActivationDispatchAttempt = initDispatchAttempt; + } + const initVcMeetingImTurnOrigin = resolveVcMeetingImTurnOrigin( + ds.session, + initTurnId ?? initReplyTurnId, + ); + const codexAppDispatchId = prompt.length > 0 + ? acceptCodexAppDispatch( + ds, + { + content: prompt, + ...(promptCodexAppInput ? { codexAppInput: promptCodexAppInput } : {}), + ...(queuedActivationToken ? { queuedActivationToken } : {}), + ...(initReplyTurnId ? { replyTurnId: initReplyTurnId } : {}), + ...(initReplyContext ? { + replyTarget: initReplyContext.target, + ...(initReplyContext.quoteTargetId + ? { quoteTargetId: initReplyContext.quoteTargetId } + : {}), + ...(initReplyContext.replyTargetSenderOpenId + ? { replyTargetSenderOpenId: initReplyContext.replyTargetSenderOpenId } + : {}), + ...(initReplyContext.replyTargetSenderIsBot !== undefined + ? { replyTargetSenderIsBot: initReplyContext.replyTargetSenderIsBot } + : {}), + } : {}), + }, + initAttributionTurnId, + initDispatchAttempt, + initVcMeetingImTurnOrigin, + ) + : undefined; + if (codexAppDispatchId && initAttributionTurnId) { + acceptedForkDispatch = { + dispatchId: codexAppDispatchId, + turnId: initAttributionTurnId, + ...(initDispatchAttempt !== undefined + ? { dispatchAttempt: initDispatchAttempt } + : {}), + }; + } + if (queuedForkSnapshot && !codexAppDispatchId) { + sessionStore.updateSession(ds.session); + } + + worker = fork(workerPath, [], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe', 'ipc'], cwd, @@ -2143,7 +3301,8 @@ export function forkWorker( LARK_APP_SECRET: botCfg.larkAppSecret, }, } as WindowsForkOptions); - const startupState: WorkerStartupState = { + spawnedWorker = worker; + startupState = { ready: false, failureNotified: false, initTurnId: initAttributionTurnId, initDispatchAttempt, @@ -2155,6 +3314,23 @@ export function forkWorker( worker.on('error', (err) => { const reason = (err as Error)?.message ?? String(err); logger.error(`[${t}] Worker fork error: ${reason}`); + if (ds.worker === worker) { + if (ds.session.queuedActivationPending) { + // The durable opening journal remains authoritative until the adapter + // proves submission. A failed child must release only its runtime + // claim so the next generation can replay that exact head. + ds.initialStartPending = false; + ds.initialStartClaimToken = undefined; + } else { + reparkQueuedActivationFollowUpTail(ds, 'worker error during activation follow-up handoff'); + } + ds.worker = null; + ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; + ds.managedTurnOrigin = undefined; + try { worker.kill(); } catch { /* best-effort failed-child fence */ } + } if (startupState.failureNotified) return; startupState.failureNotified = true; emitSessionLifecycleHook(ds, 'session.requires_attention', { @@ -2204,12 +3380,7 @@ export function forkWorker( }); // Send init config — use per-bot settings - const promptCodexAppInput = codexAppInputForSession( - ds, - promptPayload.codexAppInput, - initAttributionTurnId, - ); - const initMsg: DaemonToWorker = { + initMsg = { type: 'init', sessionId: ds.session.sessionId, chatId: ds.chatId, @@ -2265,6 +3436,7 @@ export function forkWorker( ...(nativeSessionTitlePrompt ? { nativeSessionTitlePrompt } : {}), prompt, ...(promptCodexAppInput ? { promptCodexAppInput } : {}), + ...(queuedActivationToken ? { queuedActivationToken } : {}), resume, cliSessionId: ds.session.cliSessionId, ownerOpenId: ds.ownerOpenId, @@ -2276,11 +3448,16 @@ export function forkWorker( botOpenId: bot.botOpenId, locale: botLocale(botCfg), turnId: initAttributionTurnId, + ...(initReplyTurnId ? { replyTurnId: initReplyTurnId } : {}), dispatchAttempt: initDispatchAttempt, - vcMeetingImTurnOrigin: resolveVcMeetingImTurnOrigin( - ds.session, - initAttributionTurnId, - ), + ...(codexAppDispatchId ? { codexAppDispatchId } : {}), + ...(codexAppRecoveredDispatches.length > 0 + ? { codexAppRecoveredDispatches } + : {}), + ...((ds.session.codexAppGenerationCommits?.length ?? 0) > 0 + ? { codexAppGenerationCommits: ds.session.codexAppGenerationCommits } + : {}), + vcMeetingImTurnOrigin: initVcMeetingImTurnOrigin, pluginBindings: botCfg.plugins, skillPolicy: botCfg.skills, }; @@ -2314,29 +3491,84 @@ export function forkWorker( ds.worker = worker; ds.spawnedAt = Date.now(); ds.cliVersion = currentCliVersion; - sessionStore.updateSessionPid(ds.session.sessionId, worker.pid ?? null); + worker.send(initMsg); + // A later child 'error' event is ambiguous: init may already be executing, + // so only synchronous failures before this point are safe to compensate. + spawnedWorker = undefined; + } catch (err) { + if (ds.worker === spawnedWorker) { + ds.worker = null; + ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; + } + if (spawnedWorker) { + try { spawnedWorker.kill(); } catch { /* best-effort pre-init child fence */ } + } + try { + rollbackWorkerForkPreInit(ds, queuedForkSnapshot, acceptedForkDispatch); + } catch (rollbackErr) { + logger.error( + `[${tag(ds)}] Worker pre-init failure could not durably restore queued/FIFO state: ` + + `${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}`, + ); + throw new AggregateError( + [err, rollbackErr], + `Worker pre-init failed and durable rollback failed for ${ds.session.sessionId}`, + ); + } + throw err; + } + ds.initConfig = initMsg; + try { + sessionStore.updateSessionPid(ds.session.sessionId, worker.pid ?? null); + } catch (err) { + // Init may already be executing. PID bookkeeping failure must not turn an + // accepted activation into a retryable error or orphan the attached child. + logger.error(`[${t}] Failed to persist attached worker pid: ${err instanceof Error ? err.message : String(err)}`); + } logger.info(`[${t}] Worker forked (pid: ${worker.pid}, active: ${cb.getActiveCount()})`); // Reset the exit-emit flag for the freshly spawned worker so a subsequent // exit publishes again (the previous lifecycle's flag would otherwise mask it). ds.exitEventEmitted = false; // Notify dashboard SSE subscribers a new session is live. - dashboardEventBus.publish({ - type: 'session.spawned', - body: { session: composeRowFromActive(ds) }, - }); - cb.enforceLiveSessionCap?.(); - emitSessionLifecycleHook(ds, 'session.start', { - reason: resume ? 'resume' : 'worker_spawn', - pid: worker.pid ?? null, - }); + try { + dashboardEventBus.publish({ + type: 'session.spawned', + body: { session: composeRowFromActive(ds) }, + }); + } catch (err) { + logger.error(`[${t}] Failed to publish attached worker state: ${err instanceof Error ? err.message : String(err)}`); + } + try { + cb.enforceLiveSessionCap?.(); + } catch (err) { + logger.error(`[${t}] Failed to enforce live-session cap after attach: ${err instanceof Error ? err.message : String(err)}`); + } + try { + emitSessionLifecycleHook(ds, 'session.start', { + reason: resume ? 'resume' : 'worker_spawn', + pid: worker.pid ?? null, + }); + } catch (err) { + logger.error(`[${t}] Failed to emit attached worker lifecycle hook: ${err instanceof Error ? err.message : String(err)}`); + } // Usage ledger: fresh spawns anchor the baseline so pre-existing transcript // history is never billed. Restores reconcile instead — an in-flight turn // may have completed inside tmux while the daemon was down, and that work // was submitted by botmux (anchoring would swallow it). - if (resume) reconcileUsageForDaemonSession(ds); - else anchorUsageForDaemonSession(ds); - recordOwnershipForDaemonSession(ds); + try { + if (resume) reconcileUsageForDaemonSession(ds); + else anchorUsageForDaemonSession(ds); + } catch (err) { + logger.error(`[${t}] Failed to initialize attached worker usage state: ${err instanceof Error ? err.message : String(err)}`); + } + try { + recordOwnershipForDaemonSession(ds); + } catch (err) { + logger.error(`[${t}] Failed to record attached worker ownership: ${err instanceof Error ? err.message : String(err)}`); + } } // ─── Shared worker IPC handler ────────────────────────────────────────────── @@ -2563,6 +3795,69 @@ function setupWorkerHandlers( }; break; } + case 'queued_activation_submitted': { + if (ds.worker !== worker || msg.sessionId !== ds.session.sessionId) break; + if (!ds.session.queuedActivationPending + || !ds.session.queuedActivationToken + || ds.session.queuedActivationToken !== msg.activationToken) { + logger.warn(`[${t}] Ignored stale queued activation ACK ${msg.activationToken.substring(0, 8)}`); + break; + } + const releaseReservation = async (attempt: number): Promise => { + if (ds.worker !== worker || !ds.initialStartPending) return; + try { + if (cb.onQueuedActivationSubmitted) { + const released = await cb.onQueuedActivationSubmitted(ds, msg.activationToken); + if (released === false && ds.worker === worker && ds.initialStartPending) { + const delayMs = Math.min(100 * (2 ** Math.min(attempt, 6)), 5_000); + logger.warn( + `[${t}] Queued activation follow-up was not accepted (attempt ${attempt + 1}); ` + + `retrying in ${delayMs}ms`, + ); + const timer = setTimeout(() => { void releaseReservation(attempt + 1); }, delayMs); + timer.unref?.(); + } + } else { + ds.initialStartPending = false; + ds.initialStartClaimToken = undefined; + } + } catch (err) { + logger.error( + `[${t}] Queued activation follow-up release failed unexpectedly; ` + + `delivery was not retried because acceptance is unknown: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + }; + + const journal = snapshotQueuedActivationJournal(ds.session); + const persistActivationAck = async (attempt: number): Promise => { + if (ds.worker !== worker + || !ds.session.queuedActivationPending + || ds.session.queuedActivationToken !== msg.activationToken) return; + clearQueuedActivationJournal(ds.session); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + restoreQueuedActivationJournal(ds.session, journal); + const delayMs = Math.min(100 * (2 ** Math.min(attempt, 6)), 5_000); + logger.error( + `[${t}] Failed to persist queued activation ACK; retained journal and ` + + `will retry in ${delayMs}ms: ${err instanceof Error ? err.message : String(err)}`, + ); + const timer = setTimeout(() => { void persistActivationAck(attempt + 1); }, delayMs); + timer.unref?.(); + return; + } + // Crossing the adapter boundary means this session now owns CLI + // history even if the worker dies before it can publish a session id. + ds.hasHistory = true; + await releaseReservation(0); + }; + await persistActivationAck(0); + break; + } + case 'ready': { startupState.ready = true; ds.workerPort = msg.port; @@ -2765,9 +4060,7 @@ function setupWorkerHandlers( flushPendingRiffUrlPatch(ds); } catch (err) { if (err instanceof MessageWithdrawnError) { - logger.warn(`[${t}] Root message withdrawn, closing stale session`); - killWorker(ds); - cb.closeSession(ds); + await closeWithdrawnSessionIfLedgerEmpty(ds, 'Root message withdrawn while creating worker-ready card'); break; } logger.warn(`[${t}] Failed to send streaming card, falling back to static card: ${err}`); @@ -2811,9 +4104,7 @@ function setupWorkerHandlers( } } catch (fallbackErr) { if (fallbackErr instanceof MessageWithdrawnError) { - logger.warn(`[${t}] Root message withdrawn, closing stale session`); - killWorker(ds); - cb.closeSession(ds); + await closeWithdrawnSessionIfLedgerEmpty(ds, 'Root message withdrawn while creating fallback worker-ready card'); break; } throw fallbackErr; @@ -2855,10 +4146,15 @@ function setupWorkerHandlers( ds.worker.send({ type: 'raw_input', content: rawInput, - ...(rawTurnId ? { turnId: rawTurnId } : {}), + ...((ds.session.queuedActivationTurnId ?? rawTurnId) + ? { turnId: ds.session.queuedActivationTurnId ?? rawTurnId } + : {}), followUpContent: followUp?.cliInput, ...(followUp?.turnId ? { followUpTurnId: followUp.turnId } : {}), ...(followUpCodexAppInput ? { followUpCodexAppInput } : {}), + ...(ds.session.queuedActivationToken + ? { queuedActivationToken: ds.session.queuedActivationToken } + : {}), } as DaemonToWorker); logger.info(`[${t}] Sent pending raw input after prompt_ready: ${rawInput.substring(0, 80)}${followUp ? ` (+follow-up ${followUp.cliInput.length} chars)` : ''}`); if (followUp) rememberLastCliInput(ds, followUp.userPrompt, { @@ -3024,11 +4320,9 @@ function setupWorkerHandlers( flushPendingLocalCliOpenReadinessPatch(ds); flushPendingRiffUrlPatch(ds); }) - .catch(err => { + .catch(async err => { if (err instanceof MessageWithdrawnError) { - logger.warn(`[${t}] Root message withdrawn, closing stale session`); - killWorker(ds); - cb.closeSession(ds); + await closeWithdrawnSessionIfLedgerEmpty(ds, 'Root message withdrawn while creating streaming card'); return; } logger.debug(`[${t}] Failed to create streaming card: ${err}`); @@ -3436,8 +4730,7 @@ function setupWorkerHandlers( await scopedReply(parts.join('\n\n'), 'text', undefined); } catch (replyErr) { if (replyErr instanceof MessageWithdrawnError) { - logger.warn(`[${t}] Root message withdrawn, closing stale session`); - cb.closeSession(ds); + await closeWithdrawnSessionIfLedgerEmpty(ds, 'Root message withdrawn while sending crash diagnostic'); } } } @@ -3447,7 +4740,7 @@ function setupWorkerHandlers( // Auto-restart CLI within the same worker if (ds.worker && !ds.worker.killed) { logger.info(`[${t}] Auto-restarting ${getCliDisplayName(effectiveCliId)}...`); - ds.worker.send({ type: 'restart' } as DaemonToWorker); + ds.worker.send({ type: 'restart', reason: 'cli_crash' } as DaemonToWorker); } break; } @@ -3679,7 +4972,235 @@ function setupWorkerHandlers( break; } + case 'codex_app_dispatch_transition': { + const acknowledge = (ok: boolean, error?: string): void => { + try { + worker.send({ + type: 'codex_app_dispatch_persisted', + requestId: msg.requestId, + ok, + ...(error ? { error } : {}), + } as DaemonToWorker); + } catch { /* worker exit makes the prepared/final state replayable */ } + }; + if (ds.worker !== worker) { + acknowledge(false, 'stale_worker_generation'); + break; + } + if (msg.sessionId !== ds.session.sessionId) { + acknowledge(false, 'session_mismatch'); + break; + } + const ledger = ds.session.codexAppDispatchLedger ?? []; + let next: { ok: true; ledger: CodexAppDispatchLedgerEntry[] } | { ok: false; error: string }; + if (msg.operation === 'submit') { + if (msg.entries.length !== 1) { + acknowledge(false, 'submit_requires_one_entry'); + break; + } + next = prepareCodexAppDispatch(ledger, msg.entries[0]); + } else if (msg.operation === 'retry') { + if (msg.entries.length !== 1) { + acknowledge(false, 'retry_requires_one_entry'); + break; + } + next = retryPreparedCodexAppDispatch(ledger, msg.entries[0]); + } else { + if (msg.entries.length !== 1) { + acknowledge(false, 'cancel_requires_one_entry'); + break; + } + next = cancelCodexAppDispatch(ledger, msg.entries[0]); + } + if (!next.ok) { + acknowledge(false, next.error); + break; + } + const priorLedger = ds.session.codexAppDispatchLedger; + ds.session.codexAppDispatchLedger = next.ledger; + let persisted = false; + try { + sessionStore.updateSession(ds.session); + persisted = true; + acknowledge(true); + } catch (err: any) { + ds.session.codexAppDispatchLedger = priorLedger; + acknowledge(false, err?.message ?? 'session_store_write_failed'); + } + if (persisted && !hasUnsettledCodexAppDispatch(ds.session.codexAppDispatchLedger)) { + try { await cb.onCodexAppLedgerDrained?.(ds); } + catch (err) { logger.error(`[${t}] post-drain cleanup failed: ${err instanceof Error ? err.message : String(err)}`); } + } + break; + } + + case 'codex_app_generation_active': { + if (ds.worker !== worker || msg.sessionId !== ds.session.sessionId) break; + if (!msg.fresh) break; + const priorCommits = ds.session.codexAppGenerationCommits; + ds.session.codexAppGenerationCommits = retainFreshCodexAppGeneration( + ds.session.codexAppGenerationCommits ?? [], + msg.generation, + ); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.codexAppGenerationCommits = priorCommits; + logger.error(`[${t}] Failed to persist fresh Codex App generation: ${err instanceof Error ? err.message : String(err)}`); + } + break; + } + case 'final_output': { + if (msg.codexAppSettlement) { + const settlement = msg.codexAppSettlement; + const acknowledge = (ok: boolean, error?: string): void => { + try { + worker.send({ + type: 'codex_app_dispatch_persisted', + requestId: settlement.requestId, + ok, + ...(error ? { error } : {}), + } as DaemonToWorker); + } catch { /* runner keeps the final unacknowledged for replacement */ } + }; + if (ds.worker !== worker) { + acknowledge(false, 'stale_worker_generation'); + break; + } + if (msg.sessionId !== ds.session.sessionId) { + acknowledge(false, 'session_mismatch'); + break; + } + if (committedCodexAppSequence( + ds.session.codexAppGenerationCommits ?? [], + settlement.generation, + settlement.seq, + )) { + acknowledge(true); + if (!hasUnsettledCodexAppDispatch(ds.session.codexAppDispatchLedger)) { + try { await cb.onCodexAppLedgerDrained?.(ds); } + catch (err) { logger.error(`[${t}] post-drain cleanup failed: ${err instanceof Error ? err.message : String(err)}`); } + } + break; + } + const identity = { + dispatchId: settlement.dispatchId, + turnId: msg.turnId, + ...(msg.dispatchAttempt !== undefined + ? { dispatchAttempt: msg.dispatchAttempt } + : {}), + }; + const preview = settleCodexAppDispatch( + ds.session.codexAppDispatchLedger ?? [], + ds.session.codexAppGenerationCommits ?? [], + identity, + settlement.generation, + settlement.seq, + ); + if (!preview.ok) { + acknowledge(false, preview.error); + break; + } + const key = `${ds.session.sessionId}:${settlement.generation}:${settlement.seq}`; + let inFlight = codexAppFinalSettlementInFlight.get(key); + if (!inFlight) { + inFlight = (async () => { + const unavailableSinkFailClosed = codexAppDeliveryMustFailClosed( + ds, + preview.settledEntry, + ); + const deliverySuppressed = msg.suppressDelivery === true + || managedFinalOutputSuppressed(msg.turnId, msg.dispatchAttempt) + || unavailableSinkFailClosed; + if (unavailableSinkFailClosed + && preview.settledEntry.deliverySink !== 'suppressed' + && msg.suppressDelivery !== true) { + logger.warn( + `[${t}] Codex App recovery suppressed unavailable ` + + `${preview.settledEntry.deliverySink ?? 'legacy non-Lark'} sink ` + + `(turn ${msg.turnId.substring(0, 8)})`, + ); + } + const alreadyDelivered = ds.lastBridgeEmittedUuid === finalOutputDedupeKey(ds, msg); + const owned = deliverySuppressed || !msg.content.trim() || alreadyDelivered + ? true + : await new Promise(resolve => { + deliverFinalOutput( + ds, + msg, + t, + 0, + resolve, + () => ds.worker === worker + && ds.session.sessionId === msg.sessionId, + preview.settledEntry.replyTarget, + ); + }); + if (!owned) return false; + + // Re-read after the asynchronous external delivery. A concurrent + // replacement may already have committed this signed sequence; + // that is idempotent success, never a second FIFO pop. + if (committedCodexAppSequence( + ds.session.codexAppGenerationCommits ?? [], + settlement.generation, + settlement.seq, + )) return true; + const committed = settleCodexAppDispatch( + ds.session.codexAppDispatchLedger ?? [], + ds.session.codexAppGenerationCommits ?? [], + identity, + settlement.generation, + settlement.seq, + ); + if (!committed.ok) return false; + if (msg.dispatchAttempt !== undefined) { + try { + // Durable receivers must not depend on the worker surviving + // the daemon ACK long enough to emit a later terminal IPC. + // Persist the exact completed attempt first; the worker's + // ordered duplicate terminal remains idempotent. + await cb.onTurnTerminal?.(ds, { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: msg.turnId, + dispatchAttempt: msg.dispatchAttempt, + status: 'completed', + }, { workerGeneration }); + } catch (err) { + logger.error(`[${t}] Failed to persist Codex App settlement terminal: ${err instanceof Error ? err.message : String(err)}`); + return false; + } + } + const priorLedger = ds.session.codexAppDispatchLedger; + const priorCommits = ds.session.codexAppGenerationCommits; + ds.session.codexAppDispatchLedger = committed.ledger; + ds.session.codexAppGenerationCommits = committed.commits; + try { + // One atomic sessions-file replacement owns both the exact FIFO + // pop and cumulative runner ACK boundary. Only after this write + // may the worker acknowledge final-end to the runner. + sessionStore.updateSession(ds.session); + return true; + } catch (err) { + ds.session.codexAppDispatchLedger = priorLedger; + ds.session.codexAppGenerationCommits = priorCommits; + logger.error(`[${t}] Failed to persist Codex App final settlement: ${err instanceof Error ? err.message : String(err)}`); + return false; + } + })().finally(() => codexAppFinalSettlementInFlight.delete(key)); + codexAppFinalSettlementInFlight.set(key, inFlight); + } + const persisted = await inFlight; + acknowledge(persisted, persisted ? undefined : 'final_settlement_failed'); + if (persisted && !hasUnsettledCodexAppDispatch(ds.session.codexAppDispatchLedger)) { + try { await cb.onCodexAppLedgerDrained?.(ds); } + catch (err) { logger.error(`[${t}] post-drain cleanup failed: ${err instanceof Error ? err.message : String(err)}`); } + } + break; + } + // Adopt-bridge: worker harvested the assistant turn from Claude Code's // transcript JSONL and forwarded it to us. Dedup with a session-scoped // key so a re-drain can't re-send the same answer or cross-suppress @@ -3702,7 +5223,15 @@ function setupWorkerHandlers( // Worker pops the turn off its queue right after emit, so it will // NOT re-send this payload on its own. Daemon owns retry on // transient Lark failures. - deliverFinalOutput(ds, msg, t, 0); + deliverFinalOutput( + ds, + msg, + t, + 0, + undefined, + () => ds.worker === worker + && (!msg.sessionId || ds.session.sessionId === msg.sessionId), + ); break; } @@ -3758,6 +5287,14 @@ function setupWorkerHandlers( // A stale takeover worker never clears the replacement — during takeover the // old worker's exit fires AFTER the new worker has been assigned. if (ds.worker === worker) { + if (ds.session.queuedActivationPending) { + // Journal ownership is backend-independent. The next worker replays + // this exact head with queuedActivationResume before durable tail N+1. + ds.initialStartPending = false; + ds.initialStartClaimToken = undefined; + } else { + reparkQueuedActivationFollowUpTail(ds, 'worker exit during activation follow-up handoff'); + } ds.worker = null; ds.workerPort = null; ds.managedTurnOrigin = undefined; @@ -3815,6 +5352,7 @@ function setupWorkerHandlers( // ─── Bridge final-output delivery (with retry) ────────────────────────────── const FINAL_OUTPUT_RETRY_BACKOFF_MS = [0, 5000, 15000]; // immediate, +5s, +15s +const codexAppFinalSettlementInFlight = new Map>(); function finalOutputDedupeKey(ds: DaemonSession, msg: Extract): string { return `${msg.sessionId ?? ds.session.sessionId}:${msg.lastUuid || msg.turnId}`; @@ -3898,9 +5436,9 @@ async function finishTurnReactions(ds: DaemonSession): Promise { } } -/** Deliver a bridge `final_output` to Lark. The worker emits each turn - * exactly once (it pops the turn off its queue at emit time), so the - * daemon owns retries on transient failures. After 3 attempts we log +/** Deliver a bridge `final_output` to Lark. The current worker generation pops + * the turn at emit time, while replacement recovery may replay it with the + * same provider key; the daemon owns bounded transient retries. After 3 attempts we log * and give up — the user's answer is lost; better than leaking memory * via an unbounded retry loop. */ function deliverFinalOutput( @@ -3908,7 +5446,14 @@ function deliverFinalOutput( msg: Extract, t: string, attempt: number, + onComplete?: (owned: boolean) => void, + isStillOwned: () => boolean = () => true, + frozenReplyTarget?: FrozenSessionReplyTarget, ): void { + if (!isStillOwned()) { + onComplete?.(false); + return; + } const managedReceiver = !!ds.session.vcMeetingReceiver; // Wait Mode / HTTP Sync Override: // If this turn is being waited for by an HTTP webhook request, intercept the @@ -3920,6 +5465,7 @@ function deliverFinalOutput( waitPromise.resolve(msg.content); ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); logger.info(`[${t}] Intercepted final_output for Wait Mode HTTP request (turn ${msg.turnId.substring(0, 8)})`); + onComplete?.(true); return; } @@ -3930,6 +5476,7 @@ function deliverFinalOutput( asyncResult.completedAt = Date.now(); ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); logger.info(`[${t}] Captured final_output for Async HTTP request (turn ${msg.turnId.substring(0, 8)})`); + onComplete?.(true); return; } const cb = requireCallbacks(); @@ -3950,11 +5497,17 @@ function deliverFinalOutput( : opts, ); setTimeout(async () => { + if (!isStillOwned()) { + logger.info(`[${t}] Bridge final_output abandoned — worker/session ownership changed`); + onComplete?.(false); + return; + } // Guard: if the user closed the session (or it was torn down for any // other reason) between attempts, don't post a stale final answer to // a closed thread. if (ds.session.status === 'closed') { logger.info(`[${t}] Bridge final_output abandoned — session closed (turn ${msg.turnId.substring(0, 8)})`); + onComplete?.(true); return; } try { @@ -3982,6 +5535,7 @@ function deliverFinalOutput( } ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); logger.info(`[${t}] doc-comment final_output → posted ${chunks.length} comment(s) on file=${docTurn.fileToken.slice(0, 12)} (turn ${msg.turnId.substring(0, 8)})`); + onComplete?.(true); return; } @@ -4015,6 +5569,7 @@ function deliverFinalOutput( `[${t}] VC final_output lost current membership authority ` + `(${managedDecision.errorCode}) turn=${msg.turnId.substring(0, 8)}`, ); + onComplete?.(true); return; } const revalidateManagedSend = (): void => { @@ -4096,12 +5651,25 @@ function deliverFinalOutput( }, proposedOutput) : undefined; const preparedListenerReply = preparedImReply ?? preparedDeliveryReply; + // Codex App final settlement is delivery-before-commit. If the daemon + // dies after Lark accepts the reply but before the FIFO/sequence commit, + // the replacement must retry with the same provider key. Keep the key + // dispatch-stable (not generation/seq-stable), so the user-visible Lark + // message is idempotent across crash reconciliation. The existing + // outbound-hook contract remains unchanged; it is a separate best-effort + // side effect and is not covered by the provider UUID. + const codexAppSettlementReply = msg.codexAppSettlement + ? { + uuid: `ca_${msg.codexAppSettlement.dispatchId}`.slice(0, 50), + } + : undefined; if (preparedListenerReply?.kind === 'conflict') { ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); logger.error( `[${t}] VC listener fallback suppressed (${preparedListenerReply.reason}) ` + `turn=${msg.turnId.substring(0, 8)}: ${preparedListenerReply.detail}`, ); + onComplete?.(true); return; } const canonicalOutput = preparedListenerReply?.canonicalOutput ?? proposedOutput; @@ -4140,6 +5708,7 @@ function deliverFinalOutput( `[${t}] VC listener fallback replayed existing provider result ` + `(turn ${msg.turnId.substring(0, 8)})`, ); + onComplete?.(true); return; } @@ -4147,21 +5716,28 @@ function deliverFinalOutput( // place. message.patch is silent (no Feishu notification / unread), which // used to swallow the answer; a brand-new message always pings. revalidateManagedSend(); + if (!isStillOwned()) { + onComplete?.(false); + return; + } + const deliveryReplyOptions = preparedListenerReply + ? { + uuid: preparedListenerReply.providerKey, + quoteMessageId: canonicalOutput.quoteTargetId, + beforeQuoteFallback: revalidateManagedSend, + // Managed output has one audited external effect (the Lark + // provider call). Never fan meeting content out to user hooks, + // including the first attempt and crash reconciliation replay. + suppressHook: true, + } + : codexAppSettlementReply; const messageId = await scopedReply( canonicalOutput.content, canonicalOutput.msgType, - msg.turnId, - preparedListenerReply - ? { - uuid: preparedListenerReply.providerKey, - quoteMessageId: canonicalOutput.quoteTargetId, - beforeQuoteFallback: revalidateManagedSend, - // Managed output has one audited external effect (the Lark - // provider call). Never fan meeting content out to user hooks, - // including the first attempt and crash reconciliation replay. - suppressHook: true, - } - : undefined, + msg.replyTurnId ?? msg.turnId, + frozenReplyTarget && !managedReceiver + ? { ...deliveryReplyOptions, replyTarget: frozenReplyTarget } + : deliveryReplyOptions, ); recordPrimaryOutput(messageId); if (preparedListenerReply?.kind === 'send' || preparedListenerReply?.kind === 'succeeded') { @@ -4169,13 +5745,21 @@ function deliverFinalOutput( } ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); logger.info(`[${t}] Bridge final_output forwarded (turn ${msg.turnId.substring(0, 8)}, ${msg.content.length} chars, kind=${msg.kind ?? 'bridge'}, attempt ${attempt + 1})`); + onComplete?.(true); } catch (err: any) { if (err instanceof MessageWithdrawnError) { - // Root message gone — no point retrying. Mark as emitted so any - // duplicate IPC is correctly deduped, and tear the session down. - ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); - logger.warn(`[${t}] Root message withdrawn while forwarding final_output, closing session`); - cb.closeSession(ds); + // Withdrawal is permanent for this target, but it is not explicit + // abandon. Keep an unsettled FIFO owner recoverable; only ledger-empty + // sessions may commit the dedupe marker and auto-close. + if (await closeWithdrawnSessionIfLedgerEmpty( + ds, + 'Root message withdrawn while forwarding final_output', + )) { + ds.lastBridgeEmittedUuid = finalOutputDedupeKey(ds, msg); + onComplete?.(true); + } else { + onComplete?.(false); + } return; } const next = attempt + 1; @@ -4183,10 +5767,11 @@ function deliverFinalOutput( logger.error(`[${t}] Bridge final_output gave up after ${next} attempts (turn ${msg.turnId.substring(0, 8)}): ${err.message}`); // Don't commit the dedup marker — leave room for any future // retransmit (e.g. daemon restart that re-fires the IPC). + onComplete?.(false); return; } logger.warn(`[${t}] Bridge final_output attempt ${next} failed (${err.message}); retrying in ${FINAL_OUTPUT_RETRY_BACKOFF_MS[next]}ms`); - deliverFinalOutput(ds, msg, t, next); + deliverFinalOutput(ds, msg, t, next, onComplete, isStillOwned, frozenReplyTarget); } }, FINAL_OUTPUT_RETRY_BACKOFF_MS[attempt] ?? 0); } @@ -4636,11 +6221,20 @@ function cleanupPersistentBackendSessions(backendType: 'tmux' | 'herdr', activeS let lastCliId: string | undefined; try { lastCliId = readFileSync(cliIdFile, 'utf-8').trim(); } catch { /* first run */ } const currentCliId = config.daemon.cliId; + const unsettledNames = new Set( + activeSessions_ + .filter(hasProtectedSessionMutationOwnership) + .map(session => backend.sessionName(session.sessionId)), + ); if (!multiBot && lastCliId && lastCliId !== currentCliId) { logger.info(`CLI_ID changed (${lastCliId} → ${currentCliId}), killing all ${backendType} sessions`); // Legacy per-topic hosts are still enumerable by bmx-* name. for (const name of backend.listBotmuxSessions()) { + if (unsettledNames.has(name)) { + logger.warn(`Preserving ${backendType} ${name}: unsettled Codex App dispatch must reconcile first`); + continue; + } backend.killSession(name); } // Machine-wide Herdr agents are not separate bmx-* sessions. Tear down diff --git a/src/daemon.ts b/src/daemon.ts index 9f990594d..966bfe758 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -66,10 +66,16 @@ import { bindResourcesToMessage, composeForwardFollowupContent, mergeMessageMent import { buildQuoteHint } from './im/lark/quote-hint.js'; import { logger } from './utils/logger.js'; import { withFileLock } from './utils/file-lock.js'; +import { + hasUnsettledCodexAppDispatch, + retireCodexAppDispatchAfterBackingMissing, + validateCodexAppManagedSendOrigin, +} from './utils/codex-app-dispatch-ledger.js'; +import { hasProtectedSessionMutationOwnership } from './core/session-mutation-guard.js'; import { delay } from './utils/timing.js'; import { BoundedMap } from './utils/bounded-map.js'; import { checkAllowedChatGroupsConfig } from './services/allowed-chat-groups.js'; -import type { Session, VcMeetingImTurnOrigin } from './types.js'; +import type { CliTurnPayload, Session, VcMeetingImTurnOrigin } from './types.js'; import { ensureCjkFontsInstalled } from './utils/font-installer.js'; import { scrubTmuxServerGlobalEnv } from './setup/ensure-tmux.js'; import { invalidWorkingDirs } from './utils/working-dir.js'; @@ -77,7 +83,13 @@ import { validateWorkingDir } from './core/working-dir.js'; import type { DaemonToWorker, LarkMessage } from './types.js'; export type { DaemonSession } from './core/types.js'; import type { DaemonSession } from './core/types.js'; -import { activeSessionKey, sessionKey, sessionAnchorId, storedSessionAnchorId } from './core/types.js'; +import { + activeSessionKey, + sessionKey, + sessionAnchorId, + storedSessionAnchorId, +} from './core/types.js'; +import { stagePendingRepoSetup, persistPendingRepoCardMessageId } from './core/pending-repo-journal.js'; import { buildTerminalUrl, setTerminalProxyPort, setTerminalExternalPort } from './core/terminal-url.js'; import { startTerminalProxy, type TerminalProxyHandle } from './core/terminal-proxy.js'; import type { CliId } from './adapters/cli/types.js'; @@ -93,6 +105,12 @@ import { setActiveSessionsRegistry, forkWorker, sendWorkerInput, + promoteQueuedActivationTail, + admitQueuedActivationTail, + reserveQueuedActivationTailAdmission, + type QueuedActivationTailReservation, + codexAppCleanInputAcceptedForSession, + hasQueuedActivationAdmissionGate, killWorker, reapOrphanWorkers, scheduleCardPatch, @@ -105,6 +123,7 @@ import { sweepGlobalBotmuxSkills, writableTerminalLinkFor, findActiveBySessionId, + withActiveSessionKeyLock, getDaemonBootId, type WorkerSessionReplyOptions, } from './core/worker-pool.js'; @@ -147,8 +166,15 @@ import { rememberLastCliInput, ensureTerminalWorkerPort, ensureSessionWhiteboard, + closeCliMismatchedSessionsForBot, } from './core/session-manager.js'; import { triggerSessionTurn } from './core/trigger-session.js'; +import { + runDetachedBotTurnMutation, + tryWithBotTurnMutation, + withBotTurnAdmission, + withBotTurnMutation, +} from './core/bot-turn-mutation-gate.js'; import { applyQueuedCodexAppLegacyFallback, mergeQueuedCodexAppTurn } from './core/session-create.js'; import { findOnlineDaemon, listOnlineDaemons } from './utils/daemon-discovery.js'; import { beginReplyTargetTurn, fallbackTurnId, isSubstituteTurn, resolveSessionReplyTarget, syncReplyTargetState } from './core/reply-target.js'; @@ -162,7 +188,7 @@ import { sweepOrphanSandboxes } from './adapters/backend/sandbox.js'; import { TmuxBackend } from './adapters/backend/tmux-backend.js'; import { HerdrBackend } from './adapters/backend/herdr-backend.js'; import { ZellijBackend } from './adapters/backend/zellij-backend.js'; -import { sweepIdleWorkers, DEFAULT_MAX_LIVE_WORKERS } from './core/idle-worker-sweeper.js'; +import { sweepIdleWorkersAfterTurnDrain, DEFAULT_MAX_LIVE_WORKERS } from './core/idle-worker-sweeper.js'; import { getSessionPersistentBackendType, killPersistentBackendTarget, @@ -276,7 +302,7 @@ let selfV3BootInstanceId: string | undefined; let selfDaemonLarkAppId: string | undefined; let vcMeetingTerminalReconciler: VcMeetingTerminalReconciler | undefined; import { isBotMentioned, probeBotOpenId, startLarkEventDispatcher, markForwardFollowupsSessionsReady, writeBotInfoFile, canOperate, canRunDaemonCommand, evaluateTalk, grantCommandRestriction, isKnownPeerBot, checkRequiredScopes, type RoutingContext, type TalkEvaluation, type DocCommentContext, type EventHandlers } from './im/lark/event-dispatcher.js'; -import { getDocSubscription, listAllDocSubscriptions, listDocSubscriptionsForSession, removeDocSubscription, setDocCommentPollCursor, type DocSubscription } from './services/doc-subs-store.js'; +import { getDocSubscription, listAllDocSubscriptions, listDocSubscriptionsForSession, putDocSubscription, removeDocSubscription, setDocCommentPollCursor, type DocSubscription } from './services/doc-subs-store.js'; import { BOT_REPLY_SENTINEL, subscribeDocFile, unsubscribeDocFile, addCommentReaction, hasBotSentinel, isBotAuthoredReply, listDocComments } from './im/lark/doc-comment.js'; import { learnFromMentions, resolveSender, flushIdentityCacheSync } from './im/lark/identity-cache.js'; import { normalizeBrand } from './im/lark/lark-hosts.js'; @@ -502,7 +528,11 @@ let vcMeetingReceiverRecoveryReady = false; let vcMeetingReceiverRecoverySchedulingComplete = false; const vcMeetingReceiverRecoveryPending = new Set(); const vcMeetingReceiverRecoveryTimers = new Map>(); -const vcMeetingReceiverRecoveryScopes = new Map(); +type VcMeetingBootRecoveryScope = VcMeetingDeliveryScope & { + turnId: string; + dispatchAttempt: number; +}; +const vcMeetingReceiverRecoveryScopes = new Map(); // Once the 10s ACK timeout fires, recovery has committed to kill + orphan // teardown. A delayed receiver_reset_ready from the old worker generation may // no longer cancel phase 2 or make the receiver ready early. @@ -529,13 +559,80 @@ function clearVcMeetingReceiverRecoveryPending(key: string): void { function addVcMeetingReceiverRecoveryPending( key: string, - scope: VcMeetingDeliveryScope, + scope: VcMeetingBootRecoveryScope, ): void { vcMeetingReceiverRecoveryPending.add(key); vcMeetingReceiverRecoveryScopes.set(key, scope); refreshVcMeetingReceiverRecoveryReady(); } +let testOnlyVcMeetingBootDispatchRetirement: + | ((sessionId: string, turnId: string, dispatchAttempt: number) => boolean) + | undefined; + +/** + * Retire the exact daemon-side Codex App ownership record after the owned CLI + * backing has been proved absent. This is intentionally stronger than the + * ordinary worker `cancel` transition: a dead generation may have accepted a + * successor behind this VC attempt, so recovery must remove only the fenced + * identity without requiring the old worker to prove FIFO state. + * + * A missing entry is idempotent success (the live worker may already have + * durably cancelled it before exiting). Multiple matches are corruption and + * remain fail-closed. The in-memory mutation is rolled back if the session + * store write fails, so callers can keep the delivery fence armed and retry. + */ +function retireVcMeetingCodexAppDispatchAfterBackingMissing( + sessionId: string, + turnId: string, + dispatchAttempt: number, +): boolean { + if (testOnlyVcMeetingBootDispatchRetirement) { + return testOnlyVcMeetingBootDispatchRetirement(sessionId, turnId, dispatchAttempt); + } + const session = findActiveBySessionId(sessionId)?.session ?? sessionStore.getSession(sessionId); + if (!session) return true; + const ledger = session.codexAppDispatchLedger ?? []; + const retired = retireCodexAppDispatchAfterBackingMissing(ledger, turnId, dispatchAttempt); + if (!retired.ok) { + logger.error( + `[vc-delivery] exact Codex App dispatch retirement failed (${retired.error}) ` + + `session=${sessionId} turn=${turnId.slice(0, 12)} attempt=${dispatchAttempt}`, + ); + return false; + } + if (retired.ledger.length === ledger.length) return true; + const priorLedger = session.codexAppDispatchLedger; + session.codexAppDispatchLedger = retired.ledger; + try { + sessionStore.updateSession(session); + const cleanupAppId = session.larkAppId; + if (cleanupAppId && !hasUnsettledCodexAppDispatch(session.codexAppDispatchLedger)) { + // Backing-missing retirement bypasses the normal worker ACK path, but it + // still creates the same durable empty-ledger boundary. Run mismatch + // cleanup only after persistence; failure is logged and never rolls back + // an already-authoritative retirement. + void runDetachedBotTurnMutation(cleanupAppId, async () => { + await closeCliMismatchedSessionsForBot(cleanupAppId); + }).catch(err => { + logger.error( + `[vc-delivery] post-retirement ledger-drain cleanup failed session=${sessionId}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + }); + } + return true; + } catch (err) { + session.codexAppDispatchLedger = priorLedger; + logger.error( + `[vc-delivery] failed to persist exact Codex App dispatch retirement ` + + `session=${sessionId} turn=${turnId.slice(0, 12)} attempt=${dispatchAttempt}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } +} + function isVcMeetingBootRecoveryBlocked(request: VcMeetingDeliveryRequest): boolean { // The brief pre-enumeration interval is the only global barrier. Once boot // has enumerated stale receipts, one broken receiver must not stall unrelated @@ -617,9 +714,20 @@ function scheduleVcMeetingBootRecoveryProbe( if (!vcMeetingReceiverRecoveryPending.has(key)) return; vcMeetingReceiverRecoveryTimers.delete(key); if (vcMeetingBootBackingMissing(sessionId, true)) { - logger.error(`[vc-delivery] boot receiver force-fenced session=${sessionId}`); - clearVcMeetingReceiverRecoveryPending(key); - return; + const scope = vcMeetingReceiverRecoveryScopes.get(key); + if (scope && retireVcMeetingCodexAppDispatchAfterBackingMissing( + sessionId, + scope.turnId, + scope.dispatchAttempt, + )) { + logger.error(`[vc-delivery] boot receiver force-fenced session=${sessionId}`); + clearVcMeetingReceiverRecoveryPending(key); + return; + } + logger.error( + `[vc-delivery] boot receiver backing missing but dispatch retirement unproven; ` + + `reprobe scheduled session=${sessionId}`, + ); } logger.error(`[vc-delivery] boot receiver teardown unproven; reprobe scheduled session=${sessionId}`); scheduleVcMeetingBootRecoveryProbe(key, sessionId, VC_MEETING_RUNTIME_EXPIRY_REPROBE_MS); @@ -654,8 +762,19 @@ function acknowledgeVcMeetingReceiverRecovery(key: string): boolean { const sessionId = vcMeetingReceiverRecoveryScopes.get(key)?.receiverSessionId; if (!sessionId) return false; if (vcMeetingBootBackingMissing(sessionId, false)) { - clearVcMeetingReceiverRecoveryPending(key); - return true; + const scope = vcMeetingReceiverRecoveryScopes.get(key); + if (scope && retireVcMeetingCodexAppDispatchAfterBackingMissing( + sessionId, + scope.turnId, + scope.dispatchAttempt, + )) { + clearVcMeetingReceiverRecoveryPending(key); + return true; + } + logger.error( + `[vc-delivery] boot receiver ACK has missing backing but dispatch retirement is unproven ` + + `session=${sessionId}`, + ); } logger.error(`[vc-delivery] boot receiver ACK lacked backing teardown proof session=${sessionId}`); escalateVcMeetingBootRecovery(key, sessionId); @@ -687,6 +806,8 @@ export const __testOnly_vcMeetingReceiverRecovery = { meetingId: scope.meetingId ?? 'meeting_test', memberId: scope.memberId ?? 'member_test', memberEpoch: scope.memberEpoch ?? 1, + turnId, + dispatchAttempt, }); armVcMeetingReceiverRecoveryTimeout(key, sessionId); return key; @@ -712,6 +833,14 @@ export const __testOnly_vcMeetingReceiverRecovery = { }, setBackingMissingProbe(probe: (sessionId: string, destroy: boolean) => boolean): void { testOnlyVcMeetingBootBackingMissing = probe; + // Test recovery lifecycles must not load or mutate the developer's real + // session store merely because they replaced the backing probe. + testOnlyVcMeetingBootDispatchRetirement ??= () => true; + }, + setDispatchRetirement( + retire: (sessionId: string, turnId: string, dispatchAttempt: number) => boolean, + ): void { + testOnlyVcMeetingBootDispatchRetirement = retire; }, reset(): void { for (const timer of vcMeetingReceiverRecoveryTimers.values()) clearTimeout(timer); @@ -722,6 +851,7 @@ export const __testOnly_vcMeetingReceiverRecovery = { vcMeetingReceiverRecoveryReady = false; vcMeetingReceiverRecoverySchedulingComplete = false; testOnlyVcMeetingBootBackingMissing = undefined; + testOnlyVcMeetingBootDispatchRetirement = undefined; }, }; @@ -749,6 +879,7 @@ type VcMeetingRuntimeLeaseRecoveryDeps = { backendAvailable: (backendType: PersistentBackendType) => boolean; killPersistent: (backendType: PersistentBackendType, sessionName: string, target?: PersistentBackendTarget) => void; probePersistent: (backendType: PersistentBackendType, sessionName: string, target?: PersistentBackendTarget) => 'exists' | 'missing' | 'unknown'; + retireDispatch: (sessionId: string, turnId: string, dispatchAttempt: number) => boolean; warn: (message: string) => void; error: (message: string) => void; }; @@ -853,8 +984,24 @@ function createVcMeetingRuntimeLeaseRecovery(deps: VcMeetingRuntimeLeaseRecovery // PTY has no surviving backing process once its worker has crossed the // SIGKILL backstop. Persistent backends unlock only after every applicable - // deterministic name probes authoritatively missing. + // deterministic name probes authoritatively missing. The backing proof is + // not itself enough: atomically retire this exact old daemon-ledger attempt + // before admitting hub replay, or accepted/prepared N can be restored next + // to replayed N+1 by a replacement worker. if (allMissing) { + if (!deps.retireDispatch( + fence.receiverSessionId, + fence.deliveryKey, + fence.dispatchAttempt, + )) { + deps.error( + `[vc-delivery] runtime lease backing missing but exact dispatch retirement failed; ` + + `delivery remains gated session=${fence.receiverSessionId} ` + + `attempt=${fence.dispatchAttempt}`, + ); + scheduleReprobe(fence, `${reason} dispatch retirement`); + return false; + } clearFence(fence); deps.warn( `[vc-delivery] runtime lease fence cleared after ${reason} ` @@ -1115,6 +1262,7 @@ const vcMeetingRuntimeLeaseRecovery = createVcMeetingRuntimeLeaseRecovery({ probePersistent: (backendType, sessionName, target) => target ? probePersistentBackendTarget(target) : probePersistentSession(backendType, sessionName), + retireDispatch: retireVcMeetingCodexAppDispatchAfterBackingMissing, warn: message => logger.warn(message), error: message => logger.error(message), }); @@ -2681,6 +2829,29 @@ async function sessionReply( return sendWithHookPolicy(chatId, content, msgType, opts.uuid); } } + if (opts?.replyTarget?.mode === 'thread') { + return replyWithHookPolicy( + opts.replyTarget.rootMessageId, + content, + msgType, + true, + opts.uuid, + ); + } + if (opts?.replyTarget?.mode === 'plain') { + if (opts.replyTarget.chatId !== chatId) { + throw new Error('frozen reply target does not match the session chat'); + } + // Preserve the existing regular-group → topic conversion safety without + // consulting the mutable per-turn target that may now belong to turn B. + if (ds?.scope === 'chat' && ds.session.rootMessageId) { + const mode = await getChatMode(appId, chatId, { forceRefresh: true }); + if (mode === 'topic') { + return replyWithHookPolicy(ds.session.rootMessageId, content, msgType, true, opts.uuid); + } + } + return sendWithHookPolicy(chatId, content, msgType, opts.uuid); + } if (ds?.scope === 'chat') { const fresh = readSessionFreshFromDisk(ds.session.sessionId, ds.larkAppId); if (fresh) syncReplyTargetState(ds, fresh); @@ -2710,6 +2881,12 @@ async function sessionReply( } // Thread-scope (or unknown / legacy): reply in thread. + if (opts?.replyTarget?.mode === 'plain') { + throw new Error('plain frozen reply target is invalid for a thread-scoped session'); + } + if (opts?.replyTarget?.mode === 'thread') { + return replyWithHookPolicy(opts.replyTarget.rootMessageId, content, msgType, true, opts.uuid); + } return replyWithHookPolicy(anchor, content, msgType, true, opts?.uuid); } @@ -3491,6 +3668,15 @@ async function prewarmDocCommentSession(ds: DaemonSession, sub: DocSubscription) const turnId = `doc-watch-${Date.now()}-${sub.fileToken.slice(0, 8)}`; const sender = sub.ownerOpenId ? await resolveSender(ds.larkAppId, sub.ownerOpenId, 'user') : undefined; + if ((!ds.worker || ds.worker.killed) + && (ds.pendingRepo + || ds.worktreeCreating + || hasProtectedSessionMutationOwnership(ds))) { + throw new Error( + `doc-watch warmup deferred: session ${ds.session.sessionId} has durable opening ownership`, + ); + } + beginNewTurn(ds, title); ds.lastMessageAt = Date.now(); ds.session.lastMessageAt = new Date(ds.lastMessageAt).toISOString(); @@ -3514,7 +3700,9 @@ async function prewarmDocCommentSession(ds: DaemonSession, sub: DocSubscription) }); rememberLastCliInput(ds, promptContent, cliInput); sessionStore.updateSession(ds.session); - sendWorkerInput(ds, cliInput, turnId); + if (!sendWorkerInput(ds, cliInput, turnId)) { + throw new Error('doc-watch warmup worker input was not accepted'); + } markSessionActivity(ds); } else { ensureSessionWhiteboard(ds); @@ -3534,6 +3722,8 @@ async function prewarmDocCommentSession(ds: DaemonSession, sub: DocSubscription) logger.info(`[${tag(ds)}] doc-comment watch prewarm injected file=${sub.fileToken.slice(0, 12)}`); } +export const __testOnly_prewarmDocCommentSession = prewarmDocCommentSession; + // Dependencies passed to command-handler const commandDeps: CommandHandlerDeps = { activeSessions, @@ -4549,6 +4739,20 @@ ipcRoute('POST', '/api/attention', async (req, res) => { if (!verified.ok) { return jsonRes(res, 403, { ok: false, error: verified.error }); } + if (!ds?.managedTurnOrigin + || typeof raw.originTurnId !== 'string' + || raw.originTurnId !== ds.managedTurnOrigin.turnId + || claimedAttempt !== ds.managedTurnOrigin.dispatchAttempt) { + return jsonRes(res, 403, { ok: false, error: 'origin_identity_mismatch' }); + } + const codexDecision = validateCodexAppManagedSendOrigin( + ds.session.codexAppDispatchLedger, + ds.managedTurnOrigin, + ds.initConfig?.cliId === 'codex-app' || ds.session.cliId === 'codex-app', + ); + if (!codexDecision.ok) { + return jsonRes(res, 409, { ok: false, error: 'origin_not_sendable' }); + } } if (!ds) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); @@ -4798,6 +5002,14 @@ ipcRoute('POST', '/api/hooks/emit', async (req, res) => { if (!verified.ok) { return jsonRes(res, 403, { ok: false, error: verified.error }); } + const codexDecision = validateCodexAppManagedSendOrigin( + ds!.session.codexAppDispatchLedger, + ds!.managedTurnOrigin!, + ds!.initConfig?.cliId === 'codex-app' || ds!.session.cliId === 'codex-app', + ); + if (!codexDecision.ok) { + return jsonRes(res, 409, { ok: false, error: 'origin_not_sendable' }); + } // Do not let a valid capability for session A forge session B's identity // inside the hook payload. Hook-specific fields remain caller supplied; // routing/identity fields come only from the authenticated daemon session. @@ -14493,6 +14705,119 @@ export const __testOnly_resolvePinnedWorkingDir = resolvePinnedWorkingDir; export const __testOnly_handleNewTopic = (data: any, ctx: RoutingContext): Promise => handleNewTopic(data, ctx); export const __testOnly_handleThreadReply = (data: any, ctx: RoutingContext): Promise => handleThreadReply(data, ctx); +type NewDaemonSessionClaim = + | { accepted: true; key: string; owner: DaemonSession } + | { + accepted: false; + key: string; + owner: DaemonSession; + reason: 'existing_owner'; + closedIncomingSessionId: string; + } + | { + accepted: false; + key: string; + owner: DaemonSession; + reason: 'both_pending' | 'incoming_pending'; + preservedIncomingSessionId: string; + }; + +/** + * Claim the canonical routing slot for a newly-created daemon session. + * + * Restore/transfer intentionally use setActiveSessionSafe's replacement + * policy to clean historical collisions. Live creation is different: once a + * caller has won a slot, a concurrent creator must never close that winner + * while the first caller is still preparing/forking it. First owner wins; + * ledger-empty losers are closed before the caller can observe rejection. + * Any incoming owner with an unsettled durable turn is preserved and fails + * closed because choosing either generation would risk dropping accepted + * work. + */ +async function claimNewDaemonSession( + map: Map, + incoming: DaemonSession, +): Promise { + const key = activeSessionKey(incoming); + return withActiveSessionKeyLock(map, key, async () => { + const incumbent = map.get(key); + if (!incumbent || incumbent === incoming) { + map.set(key, incoming); + return { accepted: true as const, key, owner: incoming }; + } + + const incumbentPending = hasProtectedSessionMutationOwnership(incumbent); + // A newly-created loser's transient initialStartPending is still owned by + // this caller and can be rerouted to the incumbent after the empty row is + // closed. Only durable session-level ownership makes the loser itself + // irreplaceable here. + const incomingPending = hasProtectedSessionMutationOwnership(incoming.session); + if (incomingPending) { + const reason = incumbentPending ? 'both_pending' as const : 'incoming_pending' as const; + logger.error( + `[session-registration] refusing ${reason} collision key=${key} ` + + `incumbent=${incumbent.session.sessionId} incoming=${incoming.session.sessionId}; ` + + 'preserving both rows and failing closed', + ); + return { + accepted: false as const, + key, + owner: incumbent, + reason, + preservedIncomingSessionId: incoming.session.sessionId, + }; + } + + // All live creation paths reach this helper before a worker/worktree is + // started, so persistence close is the complete loser cleanup. Do it while + // holding the key lock: rejection never exposes an active ghost row. + sessionStore.closeSession(incoming.session.sessionId); + logger.warn( + `[session-registration] canonical owner ${incumbent.session.sessionId.substring(0, 8)} ` + + `kept for key=${key}; closed incoming ${incoming.session.sessionId.substring(0, 8)}`, + ); + return { + accepted: false as const, + key, + owner: incumbent, + reason: 'existing_owner' as const, + closedIncomingSessionId: incoming.session.sessionId, + }; + }); +} + +export const __testOnly_claimNewDaemonSession = ( + map: Map, + incoming: DaemonSession, +): Promise => claimNewDaemonSession(map, incoming); + +/** Persist pending-repo N immediately after the caller wins the canonical + * runtime slot and before any card/worktree/fork publication. Staging before + * the claim creates a durable same-key loser that can poison restart with two + * protected owners. A failed write unpublishes the not-yet-durably-admitted + * runtime owner and fails loudly. This remains inside the documented residual + * WS-claim → durable-admission gap; no redelivery guarantee is asserted here. */ +function stageClaimedPendingRepoSetup( + map: Map, + ds: DaemonSession, + args: Parameters[1], +): void { + try { + stagePendingRepoSetup(ds, args); + } catch (err) { + const key = activeSessionKey(ds); + if (map.get(key) === ds) map.delete(key); + try { sessionStore.closeSession(ds.session.sessionId); } + catch (closeErr) { + logger.error( + `[session-registration] setup staging and loser cleanup both failed for ` + + `${ds.session.sessionId}: ${closeErr instanceof Error ? closeErr.message : String(closeErr)}`, + ); + } + throw err; + } +} + /** * 该新会话是否要走「仅默认目录 + 自动建 worktree」：pinned dir 来自本 bot 自己的 * defaultWorkingDir (layer 3) 且开关打开。为真时，spawn 路径不再同步 fork，而是把会话登记 @@ -14523,6 +14848,349 @@ function startAutoWorktreePending(ds: DaemonSession, args: { logger.info(`[${tag(ds)}] auto-worktree → pending, building worktree off ${args.baseDir}`); } +type AvailableBot = Awaited>[number]; + +/** Materialize the opening input at the final synchronous boundary before + * fork. All potentially-slow preparation must happen before this call; the + * pendingFollowUps snapshot and fork then cannot be interleaved by a later + * same-anchor handler. */ +function buildReservedInitialInput( + ds: DaemonSession, + availableBots: AvailableBot[], +) { + const selfBot = getBot(ds.larkAppId); + const botCfg = selfBot.config; + return buildNewTopicCliInput( + ds.pendingPrompt ?? '', + ds.session.sessionId, + ds.session.cliId ?? botCfg.cliId, + ds.session.cliPathOverride ?? botCfg.cliPathOverride, + ds.pendingAttachments, + ds.pendingMentions, + availableBots, + ds.pendingFollowUps, + { name: selfBot.botName, openId: selfBot.botOpenId }, + localeForBot(ds.larkAppId), + ds.pendingSender, + { + larkAppId: ds.larkAppId, + chatId: ds.chatId, + whiteboardId: ds.session.whiteboardId, + substituteTrigger: ds.pendingSubstituteTrigger, + codexAppText: ds.pendingCodexAppText, + codexAppApplicationContext: ds.pendingCodexAppApplicationContext, + codexAppMessageContext: ds.pendingCodexAppMessageContext, + codexAppFollowUps: ds.pendingCodexAppFollowUps, + codexAppFollowUpContexts: ds.pendingCodexAppFollowUpContexts, + }, + ); +} + +function clearInitialStartBuffers(ds: DaemonSession): void { + ds.pendingPrompt = undefined; + ds.pendingCodexAppText = undefined; + ds.pendingCodexAppApplicationContext = undefined; + ds.pendingCodexAppMessageContext = undefined; + ds.pendingAttachments = undefined; + ds.pendingMentions = undefined; + ds.pendingSubstituteTrigger = undefined; + ds.pendingSender = undefined; + ds.pendingFollowUps = undefined; + ds.pendingFollowUpTurnIds = undefined; + ds.pendingCodexAppFollowUps = undefined; + ds.pendingCodexAppFollowUpContexts = undefined; + ds.pendingCodexAppFollowUpGateAccepted = undefined; +} + +function clearInitialStartClaim(ds: DaemonSession, token?: string): boolean { + if (token !== undefined && ds.initialStartClaimToken !== token) return false; + ds.initialStartClaimToken = undefined; + ds.initialStartPending = false; + if ((ds.queuedActivationTailAdmissionsOutstanding ?? 0) === 0) { + ds.queuedActivationTailReleasePending = undefined; + if (ds.queuedActivationTailReleaseRetryTimer) { + clearTimeout(ds.queuedActivationTailReleaseRetryTimer); + ds.queuedActivationTailReleaseRetryTimer = undefined; + } + } + return true; +} + +/** Fence an arrival before any sender/prompt await. Besides reserving durable + * FIFO order, keep the opening route owned until this reservation either lands + * in the durable tail or fails. */ +function reserveAsyncQueuedActivationTailAdmission( + ds: DaemonSession, +): QueuedActivationTailReservation { + const reservation = reserveQueuedActivationTailAdmission(ds); + ds.queuedActivationTailAdmissionsOutstanding = + (ds.queuedActivationTailAdmissionsOutstanding ?? 0) + 1; + return reservation; +} + +function scheduleQueuedActivationTailReleaseRetry( + ds: DaemonSession, + acknowledgedToken?: string, +): void { + if (!ds.initialStartPending || ds.queuedActivationTailReleaseRetryTimer) return; + const timer = setTimeout(() => { + if (ds.queuedActivationTailReleaseRetryTimer !== timer) return; + ds.queuedActivationTailReleaseRetryTimer = undefined; + if (!ds.initialStartPending) return; + if (!releaseQueuedActivationReservation(ds, acknowledgedToken)) { + scheduleQueuedActivationTailReleaseRetry(ds, acknowledgedToken); + } + }, 100); + timer.unref?.(); + ds.queuedActivationTailReleaseRetryTimer = timer; +} + +/** Complete one async reservation. If the predecessor ACK arrived during its + * await, the final settler performs the deferred handoff immediately. */ +function settleAsyncQueuedActivationTailAdmission(ds: DaemonSession): void { + const outstanding = Math.max( + 0, + (ds.queuedActivationTailAdmissionsOutstanding ?? 0) - 1, + ); + ds.queuedActivationTailAdmissionsOutstanding = outstanding || undefined; + if (outstanding > 0 || !ds.queuedActivationTailReleasePending) return; + const pending = ds.queuedActivationTailReleasePending; + ds.queuedActivationTailReleasePending = undefined; + if (!releaseQueuedActivationReservation(ds, pending.acknowledgedToken)) { + scheduleQueuedActivationTailReleaseRetry(ds, pending.acknowledgedToken); + } +} + +export const __testOnly_reserveAsyncQueuedActivationTailAdmission = + reserveAsyncQueuedActivationTailAdmission; +export const __testOnly_settleAsyncQueuedActivationTailAdmission = + settleAsyncQueuedActivationTailAdmission; + +/** Release a queued activation's runtime route reservation only after the + * worker ACKs actual adapter submission. Turns that arrived meanwhile are sent + * as one ordered follow-up, never allowed to overtake the opening item. */ +function releaseQueuedActivationReservation(ds: DaemonSession, acknowledgedToken?: string): boolean { + if ((ds.queuedActivationTailAdmissionsOutstanding ?? 0) > 0) { + // The worker may ACK while N+1 is still awaiting sender/resource prompt + // construction. Keep the route gate and replay this release when the last + // reserved arrival either persists or fails. + ds.queuedActivationTailReleasePending = acknowledgedToken === undefined + ? {} + : { acknowledgedToken }; + return false; + } + ds.queuedActivationTailReleasePending = undefined; + // A prior callback retry may observe the successor already promoted. That is + // success for the acknowledged predecessor; never append/send the same tail + // head a second time while its fresh token owns the journal. + if (ds.session.queuedActivationPending) { + return acknowledgedToken !== undefined + && ds.session.queuedActivationToken !== acknowledgedToken; + } + const buffered = [...(ds.pendingFollowUps ?? [])]; + if (buffered.length > 0) { + const bot = getBot(ds.larkAppId); + const cliId = ds.session.cliId ?? bot.config.cliId; + const rawCodexText = (ds.pendingCodexAppFollowUps ?? []).join('\n\n'); + const followUp = buildFollowUpCliInput( + buffered.join('\n\n'), + ds.session.sessionId, + { + cliId, + cliPathOverride: ds.session.cliPathOverride ?? bot.config.cliPathOverride, + locale: localeForBot(ds.larkAppId), + larkAppId: ds.larkAppId, + chatId: ds.chatId, + whiteboardId: ds.session.whiteboardId, + codexAppText: rawCodexText || buffered.join('\n\n'), + codexAppApplicationContext: ds.pendingCodexAppApplicationContext, + codexAppMessageContext: (ds.pendingCodexAppFollowUpContexts ?? []) + .filter(Boolean) + .join('\n\n'), + }, + ); + const bufferedTurnIds = ds.pendingFollowUpTurnIds ?? []; + const turnId = bufferedTurnIds[bufferedTurnIds.length - 1] + ?? ds.currentReplyTarget?.turnId + ?? `queued-activation-followup-${randomUUID()}`; + const reservation = reserveQueuedActivationTailAdmission(ds); + const bufferedGateDecisions = ds.pendingCodexAppFollowUpGateAccepted ?? []; + // Runtime buffers are rolling-upgrade compatibility only. If they carry an + // arrival-time decision, preserve it; otherwise fail closed rather than + // sampling a later ON setting and inventing a sidecar at ACK time. + reservation.codexAppInputAccepted = bufferedGateDecisions.length > 0 + && bufferedGateDecisions.every(Boolean); + try { + admitQueuedActivationTail(ds, { + userPrompt: rawCodexText || buffered.join('\n\n'), + cliInput: followUp, + turnId, + }, reservation); + } catch (err) { + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Failed to persist buffered activation successor: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + // Ownership has moved into the durable exact FIFO. Clear the runtime source + // only after its store write succeeds. + ds.pendingFollowUps = undefined; + ds.pendingFollowUpTurnIds = undefined; + ds.pendingCodexAppFollowUps = undefined; + ds.pendingCodexAppFollowUpContexts = undefined; + ds.pendingCodexAppFollowUpGateAccepted = undefined; + ds.pendingCodexAppApplicationContext = undefined; + } + + // Migrate pre-durable runtime tails (and tests/rolling-upgrade state) before + // advancing. Persistence happens before the volatile cursor is cleared. + for (const staged of ds.pendingQueuedActivationFollowUps ?? []) { + const reservation = reserveQueuedActivationTailAdmission(ds); + try { + admitQueuedActivationTail(ds, { + userPrompt: staged.userPrompt, + cliInput: staged.cliInput, + turnId: staged.turnId, + dispatchAttempt: staged.dispatchAttempt, + }, reservation, { codexAppInputGateFrozen: true }); + } catch (err) { + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Failed to migrate activation successor: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + ds.pendingQueuedActivationFollowUps = undefined; + + if ((ds.session.queuedActivationTail?.length ?? 0) === 0) { + clearInitialStartBuffers(ds); + clearInitialStartClaim(ds); + return true; + } + const head = [...ds.session.queuedActivationTail!] + .sort((a, b) => a.order - b.order || a.id.localeCompare(b.id))[0]!; + const workerUnavailable = !ds.worker + || ds.worker.killed; + if (!promoteQueuedActivationTail(ds, workerUnavailable ? { send: false } : {})) return false; + try { + rememberLastCliInput(ds, head.userPrompt, head.cliInput, { + codexAppInputAccepted: !!head.cliInput.codexAppInput, + }); + } catch (err) { + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Failed to persist promoted activation metadata: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + clearInitialStartBuffers(ds); + if (workerUnavailable) { + // N's child can exit after its ACK while N+1 is still being prepared. Park + // the late exact head in the durable journal, but release the runtime claim + // so the next inbound turn can refork that head before accepting N+2. + clearInitialStartClaim(ds); + } + return true; +} + +export const __testOnly_releaseQueuedActivationReservation = releaseQueuedActivationReservation; + +/** Exact production callback passed to initWorkerPool. Keep the boolean return: + * false means the worker did not accept the staged FIFO head and the pool must + * retry instead of silently dropping the activation reservation. */ +function onQueuedActivationSubmitted(ds: DaemonSession, activationToken?: string): boolean { + return releaseQueuedActivationReservation(ds, activationToken); +} + +export const __testOnly_onQueuedActivationSubmitted = onQueuedActivationSubmitted; + +/** Fork an ordinary opening turn and release its route reservation. There is + * deliberately no await between the final buffered-input snapshot, fork, and + * release: a later handler either buffers before this block or observes the + * live worker after it. */ +function forkReservedInitialSession(ds: DaemonSession, availableBots: AvailableBot[]): void { + const userPrompt = ds.pendingPrompt ?? ''; + const input = buildReservedInitialInput(ds, availableBots); + rememberLastCliInput(ds, userPrompt, input); + const turnId = ds.pendingTurnId ?? ds.session.pendingRepoSetup?.turnId; + forkWorker(ds, input, turnId ? { turnId } : false); + ds.pendingTurnId = undefined; + // A no-project pendingRepo fallback enters forkWorker with `session.queued` + // still set. forkWorker materializes the exact opening as a tokened + // activation journal before returning; keep the route gate until the + // adapter-level ACK so a live-worker follower cannot bypass its FIFO tail. + const waitsForQueuedAck = ds.session.queuedActivationPending === true; + ds.initialStartPending = waitsForQueuedAck + || (ds.session.queuedActivationTail?.length ?? 0) > 0 + || (ds.queuedActivationTailAdmissionsOutstanding ?? 0) > 0; + clearInitialStartBuffers(ds); + if (!waitsForQueuedAck && ds.initialStartPending) { + // A follower may have reserved FIFO order while the opening prompt was + // still being built. Hand off now, or defer until its reservation settles. + void releaseQueuedActivationReservation(ds); + } +} + +/** Raw slash-command cold starts type the raw command only after prompt_ready. + * Buffer later same-anchor messages into the ordered follow-up payload carried + * on that same raw_input IPC; a separate worker message could overtake the + * command's text→Enter beat. */ +function forkReservedInitialRawSession(ds: DaemonSession, availableBots: AvailableBot[]): void { + const hasBufferedInput = + (ds.pendingPrompt?.trim().length ?? 0) > 0 + || (ds.pendingAttachments?.length ?? 0) > 0 + || (ds.pendingFollowUps?.length ?? 0) > 0; + if (hasBufferedInput) { + ensureSessionWhiteboard(ds); + const input = buildReservedInitialInput(ds, availableBots); + const botCfg = getBot(ds.larkAppId).config; + const bufferedGateDecisions = ds.pendingCodexAppFollowUpGateAccepted ?? []; + const bufferedCodexAppInputAccepted = bufferedGateDecisions.length > 0 + ? bufferedGateDecisions.every(Boolean) + : codexAppCleanInputAcceptedForSession(ds); + ds.pendingFollowUpInput = { + userPrompt: ds.pendingCodexAppText !== undefined || ds.pendingCodexAppFollowUps + ? [ds.pendingCodexAppText ?? '', ...(ds.pendingCodexAppFollowUps ?? [])].filter(Boolean).join('\n\n') + : ds.pendingPrompt || ds.pendingFollowUps?.join('\n\n') || '', + cliInput: input.content, + ...((ds.session.cliId === 'codex-app' || botCfg.cliId === 'codex-app') + && bufferedCodexAppInputAccepted + && input.codexAppInput + ? { codexAppInput: input.codexAppInput } + : {}), + codexAppInputGateFrozen: true, + }; + } + const rawInput = ds.pendingRawInput ?? ''; + rememberLastCliInput(ds, rawInput, rawInput); + // A non-owner turn can reserve/admit a durable successor while this literal + // raw opening is still preparing. Arm the same tokened activation journal + // used by pending-repo raw starts so that successor is released only after + // both the raw text and Enter have crossed the adapter boundary. + const armRawActivationAck = !ds.session.queuedActivationPending + && ((ds.session.queuedActivationTail?.length ?? 0) > 0 + || (ds.queuedActivationTailAdmissionsOutstanding ?? 0) > 0); + if (armRawActivationAck) ds.session.queued = true; + try { + forkWorker(ds, '', false); + } catch (err) { + // forkWorker's pre-init compensation restores its queued snapshot. This + // synthetic marker belongs only to the raw ACK fence, not the Dashboard + // backlog, so remove it while retaining pendingRawInput for a real retry. + if (armRawActivationAck && ds.session.queued === true) { + ds.session.queued = false; + sessionStore.updateSession(ds.session); + } + throw err; + } + // Raw text and Enter are a single adapter-submission boundary. When this + // cold start came through the durable pendingRepo fallback, retain the gate + // through that ACK so a follower cannot be sent between those two beats. + ds.initialStartPending = ds.session.queuedActivationPending === true; + clearInitialStartBuffers(ds); +} + async function replyInvalidWorkingDirs( anchor: string, larkAppId: string, @@ -14536,7 +15204,8 @@ async function replyInvalidWorkingDirs( if (invalid.length === 0) return false; ds.pendingRepo = false; - activeSessions.delete(sessionKey(anchor, larkAppId)); + const key = activeSessionKey(ds); + if (activeSessions.get(key) === ds) activeSessions.delete(key); sessionStore.closeSession(ds.session.sessionId); const msg = tr('cmd.repo.working_dir_not_exist', { dirs: invalid.map(d => `\`${d}\``).join(', '), @@ -14572,10 +15241,13 @@ async function startInitialPassthroughSession(args: { ownerOpenId: string | undefined; ownerUnionId: string | undefined; creatorOpenId: string | undefined; + /** Re-enter the existing-session route if another creator won the slot. */ + routeToCanonicalOwner: () => Promise; }): Promise { const { larkAppId, chatId, chatType, scope, anchor, messageId, replyRootId, parsed, commandContent, senderOpenId, senderUnionId, memberUnionId, ownerOpenId, ownerUnionId, creatorOpenId, + routeToCanonicalOwner, } = args; if (!await enforceMessageQuotaForCliInput(larkAppId, chatId, senderOpenId, messageId, anchor, senderUnionId, memberUnionId, chatType)) { return; @@ -14610,8 +15282,6 @@ async function startInitialPassthroughSession(args: { session.lastMessageAt = new Date(now).toISOString(); session.scope = scope; sessionStore.updateSession(session); - messageQueue.ensureQueue(anchor); - messageQueue.appendMessage(anchor, { ...parsed, content: commandContent }); const { pinnedWorkingDir, oncallEntry, inheritedFrom, pinnedFromBotDefault } = await resolvePinnedWorkingDir({ scope, anchor, chatId, chatType, larkAppId }); // Auto-worktree: register PENDING (router buffers, no force-fork) and build the @@ -14630,6 +15300,7 @@ async function startInitialPassthroughSession(args: { cliVersion: cliVersionCache.get(botCfg.cliId)?.version ?? 'unknown', lastMessageAt: now, hasHistory: false, + initialStartPending: true, pendingRepo: !pinnedWorkingDir || autoWt, pendingPrompt: '', pendingRawInput: commandContent, @@ -14645,10 +15316,27 @@ async function startInitialPassthroughSession(args: { } beginReplyTargetTurn(ds, replyRootId, messageId); sessionStore.updateSession(ds.session); - activeSessions.set(sessionKey(anchor, larkAppId), ds); + const registration = await claimNewDaemonSession(activeSessions, ds); + if (!registration.accepted) { + if (registration.reason === 'existing_owner' + && activeSessions.get(registration.key) === registration.owner) { + await routeToCanonicalOwner(); + } + return; + } + if (ds.pendingRepo) { + stageClaimedPendingRepoSetup(activeSessions, ds, { + mode: autoWt ? 'auto_worktree' : 'picker', + ...(autoWt && pinnedWorkingDir ? { baseDir: pinnedWorkingDir } : {}), + turnId: messageId, + }); + } + messageQueue.ensureQueue(anchor); + messageQueue.appendMessage(anchor, { ...parsed, content: commandContent }); if (pinnedWorkingDir && autoWt) { if (await replyInvalidWorkingDirs(anchor, larkAppId, ds)) return; + ds.initialStartPending = false; // pendingRepo/worktree now owns buffering // 挂起态提交：worktree 建好后经 runAutoWorktreeCommit → commitRepoSelection 拉起 // pendingRawInput 冷启动会话。detach → 立即返回，不阻塞本条消息处理。 startAutoWorktreePending(ds, { anchor, baseDir: pinnedWorkingDir, title: session.title, prompt: commandContent, operatorOpenId: ownerOpenId }); @@ -14657,10 +15345,8 @@ async function startInitialPassthroughSession(args: { if (pinnedWorkingDir) { if (await replyInvalidWorkingDirs(anchor, larkAppId, ds)) return; - rememberLastCliInput(ds, commandContent, commandContent); - // Raw passthrough gets its human turn only at the literal PTY write - // boundary (prompt_ready -> raw_input), never during CLI startup. - forkWorker(ds, '', false); + const availableBots = await getAvailableBots(larkAppId, chatId); + forkReservedInitialRawSession(ds, availableBots); const reason = oncallEntry ? `oncall-bound chat ${chatId}` : inheritedFrom @@ -14674,17 +15360,19 @@ async function startInitialPassthroughSession(args: { const scanDirs = getProjectScanDirs(ds).filter(d => existsSync(d)); const projects = scanDirs.length > 0 ? scanMultipleProjects(scanDirs, 3, repoPickerScanOptions()) : []; if (projects.length > 0) { + ds.initialStartPending = false; // pendingRepo/card now owns buffering lastRepoScan.set(chatId, projects); const cardJson = buildRepoSelectCard(projects, getSessionWorkingDir(ds), anchor, localeForBot(larkAppId), getBot(larkAppId).config.worktreeMultiPicker); ds.repoCardMessageId = await sessionReply(anchor, cardJson, 'interactive', larkAppId); + persistPendingRepoCardMessageId(ds, ds.repoCardMessageId); announcePendingRepoSession(ds); logger.info(`[${tag(ds)}] Waiting for repo selection before initial raw passthrough (${projects.length} projects)`); return; } ds.pendingRepo = false; - rememberLastCliInput(ds, commandContent, commandContent); - forkWorker(ds, '', false); + const availableBots = await getAvailableBots(larkAppId, chatId); + forkReservedInitialRawSession(ds, availableBots); logger.info(`[${tag(ds)}] No projects to select, queued initial raw passthrough ${commandContent.substring(0, 40)}`); } @@ -14710,6 +15398,13 @@ function mergeVcMeetingApplicationContext( } async function handleNewTopic(data: any, ctx: RoutingContext): Promise { + return withBotTurnAdmission( + ctx.larkAppId, + () => handleNewTopicAdmitted(data, ctx), + ); +} + +async function handleNewTopicAdmitted(data: any, ctx: RoutingContext): Promise { const { chatId, messageId, chatType, larkAppId, replyRootId, substituteTrigger } = ctx; // scope/anchor are mutable here: `/t` / `/topic` may flip a 普通群 chat-scope // routing into thread-scope so the bot's first reply seeds a Lark thread. @@ -14918,6 +15613,11 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { ownerOpenId: senderOpenId, ownerUnionId: senderUnionId, creatorOpenId: senderOpenId, + routeToCanonicalOwner: () => handleThreadReplyAdmitted(data, { + ...ctx, + scope, + anchor, + }), }); return; } @@ -14995,7 +15695,7 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { cmdPending = { pendingRepo: true, pendingPrompt: '', workingDir: pinnedWorkingDir }; } sessionStore.updateSession(session); - activeSessions.set(sessionKey(anchor, larkAppId), { + const cmdDs: DaemonSession = { session, worker: null, workerPort: null, @@ -15010,7 +15710,16 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { hasHistory: false, ownerOpenId: senderOpenId, ...cmdPending, - }); + }; + const registration = await claimNewDaemonSession(activeSessions, cmdDs); + if (!registration.accepted) { + if (registration.reason !== 'existing_owner') return; + } else if (cmdDs.pendingRepo) { + stageClaimedPendingRepoSetup(activeSessions, cmdDs, { + mode: 'picker', + turnId: messageId, + }); + } // Pass mention-stripped content so /command argument parsing works. await handleCommand(cmd, anchor, { ...parsed, content: commandContent }, commandDeps, larkAppId); return; @@ -15102,8 +15811,6 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { groupChatName, ); sessionStore.updateSession(session); - messageQueue.ensureQueue(anchor); - messageQueue.appendMessage(anchor, parsed); // Control card compensates for the card-less (avatar-style) chat-scope // substitute session. Topic-group substitute sessions (#475) are thread-scope @@ -15126,6 +15833,7 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { cliVersion: cliVersionCache.get(botCfg.cliId)?.version ?? 'unknown', lastMessageAt: now, hasHistory: false, + initialStartPending: true, pendingRepo: !pinnedWorkingDir || autoWt, pendingPrompt: promptContent, pendingTurnId: messageId, @@ -15150,13 +15858,30 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { : 'thread'; beginReplyTargetTurn(ds, replyRootId, messageId, new Date().toISOString(), { quoteOnly: substituteReplyMode === 'quote', substitute: !!substituteTrigger }); sessionStore.updateSession(ds.session); - activeSessions.set(sessionKey(anchor, larkAppId), ds); + const registration = await claimNewDaemonSession(activeSessions, ds); + if (!registration.accepted) { + if (registration.reason === 'existing_owner' + && activeSessions.get(registration.key) === registration.owner) { + await handleThreadReplyAdmitted(data, { ...ctx, scope, anchor }); + } + return; + } + if (ds.pendingRepo) { + stageClaimedPendingRepoSetup(activeSessions, ds, { + mode: autoWt ? 'auto_worktree' : 'picker', + ...(autoWt && pinnedWorkingDir ? { baseDir: pinnedWorkingDir } : {}), + turnId: messageId, + }); + } + messageQueue.ensureQueue(anchor); + messageQueue.appendMessage(anchor, parsed); // Auto-worktree: session is registered PENDING; build the worktree off the // critical path, then commitRepoSelection pins it + forks (folding in any // messages buffered during creation). detach → return immediately. if (pinnedWorkingDir && autoWt) { if (await replyInvalidWorkingDirs(anchor, larkAppId, ds)) return; + ds.initialStartPending = false; // pendingRepo/worktree now owns buffering startAutoWorktreePending(ds, { anchor, baseDir: pinnedWorkingDir, title: session.title, prompt: promptContent, operatorOpenId: senderOpenId }); return; } @@ -15164,13 +15889,10 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { // Pinned (oncall binding or inherited from sibling bot): spawn CLI immediately. if (pinnedWorkingDir) { if (await replyInvalidWorkingDirs(anchor, larkAppId, ds)) return; - const selfBot = getBot(larkAppId); ensureSessionWhiteboard(ds); - const prompt = buildNewTopicCliInput(promptContent, session.sessionId, botCfg.cliId, botCfg.cliPathOverride, attachments, parsed.mentions, await getAvailableBots(larkAppId, chatId), undefined, { name: selfBot.botName, openId: selfBot.botOpenId }, localeForBot(larkAppId), newTopicSender, { larkAppId, chatId, whiteboardId: ds.session.whiteboardId, substituteTrigger, codexAppText: codexAppVisibleText, codexAppApplicationContext, codexAppMessageContext }); + const availableBots = await getAvailableBots(larkAppId, chatId); await noteTurnReceived(ds, messageId, content, newTopicSender, messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); - rememberLastCliInput(ds, promptContent, prompt); - forkWorker(ds, prompt, { turnId: messageId }); - ds.pendingTurnId = undefined; + forkReservedInitialSession(ds, availableBots); const reason = oncallEntry ? `oncall-bound chat ${chatId}` : inheritedFrom @@ -15188,22 +15910,21 @@ async function handleNewTopic(data: any, ctx: RoutingContext): Promise { projects = scanMultipleProjects(scanDirs, 3, repoPickerScanOptions()); } if (projects.length > 0) { + ds.initialStartPending = false; // pendingRepo/card now owns buffering lastRepoScan.set(chatId, projects); const currentCwd = getSessionWorkingDir(ds); const cardJson = buildRepoSelectCard(projects, currentCwd, anchor, localeForBot(larkAppId), getBot(larkAppId).config.worktreeMultiPicker); ds.repoCardMessageId = await sessionReply(anchor, cardJson, 'interactive', larkAppId); + persistPendingRepoCardMessageId(ds, ds.repoCardMessageId); announcePendingRepoSession(ds); logger.info(`[${tag(ds)}] Waiting for repo selection (${projects.length} projects)`); } else { // No projects found — skip repo selection, spawn directly ds.pendingRepo = false; - const selfBot = getBot(larkAppId); ensureSessionWhiteboard(ds); - const prompt = buildNewTopicCliInput(promptContent, session.sessionId, botCfg.cliId, botCfg.cliPathOverride, attachments, parsed.mentions, await getAvailableBots(larkAppId, chatId), undefined, { name: selfBot.botName, openId: selfBot.botOpenId }, localeForBot(larkAppId), newTopicSender, { larkAppId, chatId, whiteboardId: ds.session.whiteboardId, substituteTrigger, codexAppText: codexAppVisibleText, codexAppApplicationContext, codexAppMessageContext }); + const availableBots = await getAvailableBots(larkAppId, chatId); await noteTurnReceived(ds, messageId, content, newTopicSender, messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); - rememberLastCliInput(ds, promptContent, prompt); - forkWorker(ds, prompt, { turnId: messageId }); - ds.pendingTurnId = undefined; + forkReservedInitialSession(ds, availableBots); logger.info(`Session ${session.sessionId} ready (no projects to select), total active: ${getActiveCount()}`); } } @@ -15362,6 +16083,7 @@ async function handleBotAdded(chatId: string, operatorOpenId: string | undefined cliVersion: cliVersionCache.get(botCfg.cliId)?.version ?? 'unknown', lastMessageAt: now, hasHistory: false, + initialStartPending: true, pendingRepo: !pinnedWorkingDir || autoWt, pendingPrompt: promptBody, pendingCodexAppText: botCfg.cliId === 'codex-app' ? codexAppText : undefined, @@ -15369,22 +16091,28 @@ async function handleBotAdded(chatId: string, operatorOpenId: string | undefined currentTurnTitle: title, workingDir: pinnedWorkingDir, }; - activeSessions.set(dsKey, ds); + const registration = await claimNewDaemonSession(activeSessions, ds); + if (!registration.accepted) { + if (activeSessions.get(registration.key) === registration.owner) { + groupJoinAnchorByChat.set(chatLiveKey, registration.key); + } + return; + } + if (ds.pendingRepo) { + stageClaimedPendingRepoSetup(activeSessions, ds, { + mode: autoWt ? 'auto_worktree' : 'picker', + ...(autoWt && pinnedWorkingDir ? { baseDir: pinnedWorkingDir } : {}), + turnId: anchor, + }); + } // Register the anchor so a later duplicate bot.added for this chat is deduped // even in 话题群 (where dsKey is the seed id, not chatId). groupJoinAnchorByChat.set(chatLiveKey, dsKey); - const selfBot = getBot(larkAppId); - const buildPrompt = async () => buildNewTopicCliInput( - promptBody, session.sessionId, botCfg.cliId, botCfg.cliPathOverride, - undefined, undefined, await getAvailableBots(larkAppId, chatId), undefined, - { name: selfBot.botName, openId: selfBot.botOpenId }, localeForBot(larkAppId), undefined, - { larkAppId, chatId, whiteboardId: ds.session.whiteboardId, codexAppText }, - ); - // Auto-worktree: register PENDING, build worktree off-path, commit+fork later. if (pinnedWorkingDir && autoWt) { if (await replyInvalidWorkingDirs(anchor, larkAppId, ds)) return; + ds.initialStartPending = false; // pendingRepo/worktree now owns buffering startAutoWorktreePending(ds, { anchor, baseDir: pinnedWorkingDir, title, prompt: promptBody, operatorOpenId }); return; } @@ -15393,10 +16121,9 @@ async function handleBotAdded(chatId: string, operatorOpenId: string | undefined if (pinnedWorkingDir) { if (await replyInvalidWorkingDirs(anchor, larkAppId, ds)) return; ensureSessionWhiteboard(ds); - const prompt = await buildPrompt(); + const availableBots = await getAvailableBots(larkAppId, chatId); await noteTurnReceived(ds, anchor, promptBody); - rememberLastCliInput(ds, promptBody, prompt); - forkWorker(ds, prompt); + forkReservedInitialSession(ds, availableBots); logger.info(`[auto-start:入群] ${chatId.substring(0, 12)} 自动开工（${mode}/${scope}），workingDir=${pinnedWorkingDir}`); return; } @@ -15406,18 +16133,19 @@ async function handleBotAdded(chatId: string, operatorOpenId: string | undefined const scanDirs = getProjectScanDirs(ds).filter(d => existsSync(d)); const projects = scanDirs.length > 0 ? scanMultipleProjects(scanDirs, 3, repoPickerScanOptions()) : []; if (projects.length > 0) { + ds.initialStartPending = false; // pendingRepo/card now owns buffering lastRepoScan.set(chatId, projects); const cardJson = buildRepoSelectCard(projects, getSessionWorkingDir(ds), anchor, localeForBot(larkAppId), getBot(larkAppId).config.worktreeMultiPicker); ds.repoCardMessageId = await sessionReply(anchor, cardJson, 'interactive', larkAppId); + persistPendingRepoCardMessageId(ds, ds.repoCardMessageId); announcePendingRepoSession(ds); logger.info(`[auto-start:入群] ${chatId.substring(0, 12)} 无默认目录，弹 repo 选择卡（${projects.length} 个项目）`); } else { ds.pendingRepo = false; ensureSessionWhiteboard(ds); - const prompt = await buildPrompt(); + const availableBots = await getAvailableBots(larkAppId, chatId); await noteTurnReceived(ds, anchor, promptBody); - rememberLastCliInput(ds, promptBody, prompt); - forkWorker(ds, prompt); + forkReservedInitialSession(ds, availableBots); logger.info(`[auto-start:入群] ${chatId.substring(0, 12)} 无默认目录且无可选项目，直接开工`); } } finally { @@ -15458,6 +16186,22 @@ function lookupForeignBotName(senderOpenId: string, larkAppId: string): string { } async function handleThreadReply(data: any, ctx: RoutingContext): Promise { + // Admission is bot-wide but intentionally concurrent. Add a narrower FIFO + // before entering it so two same-anchor deliveries can never invert while + // quota/resource/sender preparation awaits. Synthetic keys share the same + // lock registry without occupying or mutating an active-session slot. + const deliveryKey = `\u0000thread-delivery:${ctx.larkAppId}:${ctx.scope}:${ctx.anchor}`; + return withActiveSessionKeyLock( + activeSessions, + deliveryKey, + () => withBotTurnAdmission( + ctx.larkAppId, + () => handleThreadReplyAdmitted(data, ctx), + ), + ); +} + +async function handleThreadReplyAdmitted(data: any, ctx: RoutingContext): Promise { const { chatId: ctxChatId, chatType: ctxChatType, scope, anchor, larkAppId, replyRootId, substituteTrigger } = ctx; await resolveNonsupportMessage(data, larkAppId); const { parsed, resources } = parseEventMessage(data); @@ -15695,6 +16439,11 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise ownerOpenId: isForeignBot ? undefined : threadSenderOpenId, ownerUnionId: isForeignBot ? undefined : data?.sender?.sender_id?.union_id, creatorOpenId: threadSenderOpenId, + routeToCanonicalOwner: () => handleThreadReplyAdmitted(data, { + ...ctx, + scope, + anchor, + }), }); return; } @@ -15707,6 +16456,19 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise // 收紧到与 daemon 命令同档；这会同时改变真人 oncall 成员的现有行为，应单独评估。 const ds = existingDs; if (ds?.worker && !ds.worker.killed) { + if (hasQueuedActivationAdmissionGate(ds)) { + sessionReply( + anchor, + tr('daemon.cmd_activation_pending', { cmd }, localeForBot(larkAppId)), + 'text', + larkAppId, + ); + logger.warn( + `[${anchor.substring(0, 12)}] Refused passthrough ${cmd} before acceptance ` + + 'while queued activation owns raw submission order', + ); + return; + } // Passthrough commands bypass the normal message-forwarding block // below, so bind this accepted Lark turn here before the worker rotates // its marker at the actual PTY write boundary. @@ -15793,7 +16555,7 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise cmdPending = { pendingRepo: true, pendingPrompt: '', workingDir: pinnedWorkingDir }; } sessionStore.updateSession(session); - activeSessions.set(sessionKey(anchor, larkAppId), { + const cmdDs: DaemonSession = { session, worker: null, workerPort: null, @@ -15808,7 +16570,16 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise hasHistory: false, ownerOpenId: threadSenderOpenId, ...cmdPending, - }); + }; + const registration = await claimNewDaemonSession(activeSessions, cmdDs); + if (!registration.accepted) { + if (registration.reason !== 'existing_owner') return; + } else if (cmdDs.pendingRepo) { + stageClaimedPendingRepoSetup(activeSessions, cmdDs, { + mode: 'picker', + turnId: parsed.messageId, + }); + } } // Pass mention-stripped content so /command argument parsing works. // chatId lets session-less handlers (e.g. /group) reach the chat roster. @@ -15872,6 +16643,38 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise } } + // Reserve an existing worker:null owner before ANY post-routing await. + // withBotTurnAdmission permits concurrent turns, so two + // same-anchor handlers can both resolve the same cold owner and otherwise + // race through resource/sender preparation into two forks. The token makes + // one handler the opening owner; every other handler observes the gate and + // buffers. Queued activations retain the token through adapter ACK, while an + // ordinary refork releases it immediately after its buffered tail is handed + // to the worker. + let initialStartClaimToken: string | undefined; + let initialStartClaimWaitsForQueuedAck = false; + let retainInitialStartClaim = false; + const tryAcquireInitialStartClaim = (): void => { + if (initialStartClaimToken + || !ds + || (ds.worker && !ds.worker.killed) + || ds.pendingRepo + || ds.initialStartPending === true) return; + initialStartClaimToken = randomUUID(); + ds.initialStartClaimToken = initialStartClaimToken; + ds.initialStartPending = true; + initialStartClaimWaitsForQueuedAck = ds.session.queued === true + || ds.session.queuedActivationPending === true + || ds.session.queuedActivationInput !== undefined; + }; + tryAcquireInitialStartClaim(); + const ownsInitialStartClaim = (): boolean => !!( + ds + && initialStartClaimToken + && ds.initialStartClaimToken === initialStartClaimToken + ); + + try { const quotaSenderOpenId = threadSenderOpenId; if (!await enforceMessageQuotaForCliInput(larkAppId, ctxChatId ?? data?.message?.chat_id, quotaSenderOpenId, parsed.messageId, anchor, threadTeamTrustUnionId, threadSenderUnionId, ctxChatType)) { return; @@ -15924,8 +16727,57 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise sessionStore.updateSession(ds.session); } - // If waiting for repo selection, buffer the message and remind user - if (ds?.pendingRepo || ds?.pendingRepoCommitInFlight) { + // The first owner may have failed while this handler was awaiting resource + // preparation. Re-check once at the final routing boundary so the oldest + // surviving follower atomically takes over instead of proceeding unclaimed. + tryAcquireInitialStartClaim(); + + // A published worker:null owner is not necessarily ready to refork: its + // opening turn may still be preparing. Buffer both repo-pending and + // initial-start-pending messages into that opening turn so a later prompt + // cannot overtake it. + const initialStartPending = ds?.initialStartPending === true && !ownsInitialStartClaim(); + if (ds && initialStartPending && !ds.pendingRepo) { + const tailReservation = reserveAsyncQueuedActivationTailAdmission(ds); + try { + const botCfg = getBot(ds.larkAppId).config; + const followUp = buildFollowUpCliInput(promptContent, ds.session.sessionId, { + attachments, + mentions: parsed.mentions, + isAdoptMode: false, + cliId: ds.session.cliId ?? botCfg.cliId, + cliPathOverride: ds.session.cliPathOverride ?? botCfg.cliPathOverride, + sender: await getThreadSender(), + larkAppId, + chatId: ds.session.chatId, + whiteboardId: ds.session.whiteboardId, + substituteTrigger, + codexAppText: parsed.content, + codexAppApplicationContext, + codexAppMessageContext, + }); + admitQueuedActivationTail(ds, { + userPrompt: promptContent, + cliInput: followUp, + turnId: parsed.messageId, + }, tailReservation); + } finally { + settleAsyncQueuedActivationTailAdmission(ds); + } + logger.info( + `[${tag(ds)}] buffered same-anchor turn ${parsed.messageId.substring(0, 12)} ` + + 'behind queued activation submission ACK', + ); + return; + } + if (ds?.pendingRepo || initialStartPending) { + const durableTailReservation = ds.pendingRepo + ? reserveAsyncQueuedActivationTailAdmission(ds) + : undefined; + const bufferedCodexAppInputAccepted = initialStartPending && !ds.pendingRepo + ? codexAppCleanInputAcceptedForSession(ds) + : undefined; + try { // Enrich content with attachment hints and mention metadata (same as normal send) const codexAppFollowUpContextParts: string[] = []; if (codexAppMessageContext) codexAppFollowUpContextParts.push(codexAppMessageContext); @@ -15972,6 +16824,65 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise codexAppFollowUpContextParts.unshift(followUpSenderBlock); } } + if (ds.pendingRepo) { + const hasOpening = (ds.pendingPrompt?.trim().length ?? 0) > 0 + || (ds.pendingAttachments?.length ?? 0) > 0 + || !!ds.pendingRawInput; + ds.session.queued = true; + if (!hasOpening) { + // A bare /repo placeholder has no N yet. This inbound becomes the + // durable opening instead of an impossible successor to an empty ACK. + ds.pendingPrompt = promptContent; + ds.pendingTurnId = parsed.messageId; + ds.pendingCodexAppText = parsed.content; + ds.pendingCodexAppApplicationContext = codexAppApplicationContext; + ds.pendingCodexAppMessageContext = codexAppMessageContext; + ds.pendingAttachments = attachments.length > 0 ? attachments : undefined; + ds.pendingMentions = parsed.mentions; + ds.pendingSender = followUpSender; + const existingSetup = ds.session.pendingRepoSetup; + stagePendingRepoSetup(ds, { + mode: existingSetup?.mode ?? 'picker', + ...(existingSetup?.baseDir ? { baseDir: existingSetup.baseDir } : {}), + turnId: parsed.messageId, + }); + } else { + const botCfg = getBot(ds.larkAppId).config; + const exactFollowUp = buildFollowUpCliInput(promptContent, ds.session.sessionId, { + attachments, + mentions: parsed.mentions, + isAdoptMode: false, + cliId: ds.session.cliId ?? botCfg.cliId, + cliPathOverride: ds.session.cliPathOverride ?? botCfg.cliPathOverride, + sender: followUpSender, + larkAppId, + chatId: ds.session.chatId, + whiteboardId: ds.session.whiteboardId, + substituteTrigger, + codexAppText: parsed.content, + codexAppApplicationContext, + codexAppMessageContext, + }); + ds.session.queuedPrompt ??= ds.pendingPrompt; + ds.session.queuedCodexAppText ??= ds.pendingCodexAppText; + ds.session.queuedCodexAppMessageContext ??= ds.pendingCodexAppMessageContext; + admitQueuedActivationTail(ds, { + userPrompt: promptContent, + cliInput: exactFollowUp, + turnId: parsed.messageId, + }, durableTailReservation!); + } + const pendingReplyKey = ds.worktreeCreating + ? 'daemon.worktree_building_wait' + : 'daemon.choose_repo_first'; + await sessionReply(anchor, tr(pendingReplyKey, undefined, localeForBot(larkAppId)), 'text', larkAppId); + return; + } + + // Initial-start compatibility path after repo selection: only the durable + // queued-activation gate belongs here. pendingRepoCommitInFlight protects a + // second repo selection, but must not divert ordinary turns away from the + // already-forked worker into source buffers that will never be consumed. const sameInitialCaller = !!followUpSender?.openId && followUpSender.openId === ds.pendingSender?.openId; if (ds.pendingRawInput) { @@ -16002,16 +16913,29 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise } if (!ds.pendingFollowUps) ds.pendingFollowUps = []; ds.pendingFollowUps.push(enriched); + if (!ds.pendingFollowUpTurnIds) ds.pendingFollowUpTurnIds = []; + ds.pendingFollowUpTurnIds.push(parsed.messageId); if (!ds.pendingCodexAppFollowUps) ds.pendingCodexAppFollowUps = []; ds.pendingCodexAppFollowUps.push(parsed.content); if (!ds.pendingCodexAppFollowUpContexts) ds.pendingCodexAppFollowUpContexts = []; ds.pendingCodexAppFollowUpContexts.push(codexAppFollowUpContextParts.join('\n\n')); + if (!ds.pendingCodexAppFollowUpGateAccepted) { + ds.pendingCodexAppFollowUpGateAccepted = []; + } + ds.pendingCodexAppFollowUpGateAccepted.push(bufferedCodexAppInputAccepted === true); if (codexAppApplicationContext) { ds.pendingCodexAppApplicationContext = mergeVcMeetingApplicationContext( ds.pendingCodexAppApplicationContext, codexAppApplicationContext, ); } + if (initialStartPending && !ds.pendingRepo) { + logger.info( + `[${tag(ds)}] buffered same-anchor turn ${parsed.messageId.substring(0, 12)} ` + + 'behind reserved initial start', + ); + return; + } // Auto-worktree pending (worktreeCreating) has no repo card to point at — the // message IS buffered (folded on commit), so just say "hold on, building worktree" // instead of the misleading "pick a repo from the card above". @@ -16020,19 +16944,19 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise : 'daemon.choose_repo_first'; await sessionReply(anchor, tr(pendingReplyKey, undefined, localeForBot(larkAppId)), 'text', larkAppId); return; + } finally { + if (durableTailReservation) settleAsyncQueuedActivationTailAdmission(ds); + } } - // Route to file queue (keyed by anchor: rootMessageId for thread, chatId for chat) - messageQueue.ensureQueue(anchor); - messageQueue.appendMessage(anchor, parsed); - if (!ds) { // No active session at this anchor — auto-create. This branch is mostly a // safety net; the dispatcher routes here only when isSessionOwner() returns // true, but races (between check and execution, or session-closed events) // can land us here. if (activeSessions.has(sessionKey(anchor, larkAppId))) { - logger.info(`[${larkAppId}] Session already exists for ${scope}-scope ${anchor}, skipping auto-create`); + logger.info(`[${larkAppId}] Session already exists for ${scope}-scope ${anchor}, routing to canonical owner`); + await handleThreadReplyAdmitted(data, ctx); return; } @@ -16119,6 +17043,7 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise cliVersion: cliVersionCache.get(botCfg.cliId)?.version ?? 'unknown', lastMessageAt: now, hasHistory: false, + initialStartPending: true, pendingRepo: !pinnedWorkingDir || autoWt, pendingPrompt: promptContent, pendingTurnId: parsed.messageId, @@ -16143,11 +17068,30 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise : 'thread'; beginReplyTargetTurn(newDs, replyRootId, parsed.messageId, new Date().toISOString(), { quoteOnly: substituteReplyMode === 'quote', substitute: !!substituteTrigger }); sessionStore.updateSession(newDs.session); - activeSessions.set(sessionKey(anchor, larkAppId), newDs); + const registration = await claimNewDaemonSession(activeSessions, newDs); + if (!registration.accepted) { + if (registration.reason === 'existing_owner' + && activeSessions.get(registration.key) === registration.owner) { + await handleThreadReplyAdmitted(data, ctx); + } + return; + } + if (newDs.pendingRepo) { + stageClaimedPendingRepoSetup(activeSessions, newDs, { + mode: autoWt ? 'auto_worktree' : 'picker', + ...(autoWt && pinnedWorkingDir ? { baseDir: pinnedWorkingDir } : {}), + turnId: parsed.messageId, + }); + } + // Route to file queue only after ownership is committed. A rejected + // creator re-enters the canonical route above, which appends exactly once. + messageQueue.ensureQueue(anchor); + messageQueue.appendMessage(anchor, parsed); // Auto-worktree: register PENDING, build worktree off-path, commit+fork later. if (pinnedWorkingDir && autoWt) { if (await replyInvalidWorkingDirs(anchor, larkAppId, newDs)) return; + newDs.initialStartPending = false; // pendingRepo/worktree now owns buffering startAutoWorktreePending(newDs, { anchor, baseDir: pinnedWorkingDir, title: parsed.content.substring(0, 50), prompt: promptContent, operatorOpenId: ownerOpenId }); return; } @@ -16156,13 +17100,10 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise // spawn CLI immediately, skip repo selection. if (pinnedWorkingDir) { if (await replyInvalidWorkingDirs(anchor, larkAppId, newDs)) return; - const selfBot = getBot(larkAppId); ensureSessionWhiteboard(newDs); - const prompt = buildNewTopicCliInput(promptContent, session.sessionId, botCfg.cliId, botCfg.cliPathOverride, attachments, parsed.mentions, await getAvailableBots(larkAppId, autoCreateChatId), undefined, { name: selfBot.botName, openId: selfBot.botOpenId }, localeForBot(larkAppId), autoCreateSender, { larkAppId, chatId: autoCreateChatId, whiteboardId: newDs.session.whiteboardId, substituteTrigger, codexAppText: parsed.content, codexAppApplicationContext, codexAppMessageContext }); + const availableBots = await getAvailableBots(larkAppId, autoCreateChatId); await noteTurnReceived(newDs, parsed.messageId, parsed.content, autoCreateSender, parsed.messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); - rememberLastCliInput(newDs, promptContent, prompt); - forkWorker(newDs, prompt, { turnId: parsed.messageId }); - newDs.pendingTurnId = undefined; + forkReservedInitialSession(newDs, availableBots); const reason = oncallEntry ? `oncall-bound chat ${autoCreateChatId}` : inheritedFrom @@ -16180,27 +17121,30 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise projects = scanMultipleProjects(scanDirs2, 3, repoPickerScanOptions()); } if (projects.length > 0) { + newDs.initialStartPending = false; // pendingRepo/card now owns buffering lastRepoScan.set(autoCreateChatId, projects); const currentCwd = getSessionWorkingDir(newDs); const cardJson = buildRepoSelectCard(projects, currentCwd, anchor, localeForBot(larkAppId), getBot(larkAppId).config.worktreeMultiPicker); newDs.repoCardMessageId = await sessionReply(anchor, cardJson, 'interactive', larkAppId); + persistPendingRepoCardMessageId(newDs, newDs.repoCardMessageId); announcePendingRepoSession(newDs); logger.info(`[${tag(newDs)}] Waiting for repo selection (${projects.length} projects)`); } else { // No projects found — skip repo selection, spawn directly newDs.pendingRepo = false; - const selfBot = getBot(larkAppId); ensureSessionWhiteboard(newDs); - const prompt = buildNewTopicCliInput(promptContent, session.sessionId, botCfg.cliId, botCfg.cliPathOverride, attachments, parsed.mentions, await getAvailableBots(larkAppId, autoCreateChatId), undefined, { name: selfBot.botName, openId: selfBot.botOpenId }, localeForBot(larkAppId), autoCreateSender, { larkAppId, chatId: autoCreateChatId, whiteboardId: newDs.session.whiteboardId, substituteTrigger, codexAppText: parsed.content, codexAppApplicationContext, codexAppMessageContext }); + const availableBots = await getAvailableBots(larkAppId, autoCreateChatId); await noteTurnReceived(newDs, parsed.messageId, parsed.content, autoCreateSender, parsed.messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); - rememberLastCliInput(newDs, promptContent, prompt); - forkWorker(newDs, prompt, { turnId: parsed.messageId }); - newDs.pendingTurnId = undefined; + forkReservedInitialSession(newDs, availableBots); } return; } + // Existing-owner route: append once after all pending-repo early returns. + messageQueue.ensureQueue(anchor); + messageQueue.appendMessage(anchor, parsed); + // Send message to worker via IPC if (ds.worker && !ds.worker.killed) { const dsBotCfgForMsg = getBot(ds.larkAppId).config; @@ -16240,7 +17184,10 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise beginNewTurn(ds, parsed.content); await noteTurnReceived(ds, parsed.messageId, parsed.content, await getThreadSender(), parsed.messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); rememberLastCliInput(ds, promptContent, cliInput); - sendWorkerInput(ds, cliInput, parsed.messageId); + if (!sendWorkerInput(ds, cliInput, parsed.messageId)) { + logger.warn(`[${tag(ds)}] Inbound ${parsed.messageId} was not accepted by the live worker`); + return; + } } else { // Worker not running — re-fork with resume. This is a NEW turn, so drop // any restored streaming-card reference; worker_ready will POST a fresh @@ -16286,31 +17233,124 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise // bridge session whose worker died would gain a whiteboard binding (and a // block in its refork prompt) that its live turns never had. if (!ds.adoptedFrom) ensureSessionWhiteboard(ds); + const stageCurrentBehindQueuedActivation = async (): Promise => { + const tailReservation = reserveAsyncQueuedActivationTailAdmission(ds); + try { + const currentFollowUp = buildFollowUpCliInput(promptContent, ds.session.sessionId, { + attachments, + mentions: parsed.mentions, + isAdoptMode: false, + cliId: ds.session.cliId ?? dsBotCfgForFork.cliId, + cliPathOverride: ds.session.cliPathOverride ?? dsBotCfgForFork.cliPathOverride, + sender: await getThreadSender(), + larkAppId, + chatId: ds.session.chatId, + whiteboardId: ds.session.whiteboardId, + substituteTrigger, + codexAppText: parsed.content, + codexAppApplicationContext, + codexAppMessageContext, + }); + admitQueuedActivationTail(ds, { + userPrompt: promptContent, + cliInput: currentFollowUp, + turnId: parsed.messageId, + }, tailReservation); + } finally { + settleAsyncQueuedActivationTailAdmission(ds); + } + ds.initialStartPending = true; + await noteTurnReceived( + ds, + parsed.messageId, + parsed.content, + await getThreadSender(), + parsed.messageId, + substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined, + ); + }; + // Codex App may retain an ACK journal/FIFO head after its worker dies. A + // new inbound turn wakes a pure recovery fork first; it stays in the exact + // runtime tail until that recovered head produces its submission ACK. + if (ds.session.queuedActivationPending) { + await stageCurrentBehindQueuedActivation(); + try { + const recoverThroughCodexLedger = (ds.session.cliId ?? dsBotCfgForFork.cliId) === 'codex-app'; + forkWorker( + ds, + recoverThroughCodexLedger ? '' : (ds.session.queuedActivationInput ?? ''), + { + resume: ds.session.queuedActivationResume ?? ds.hasHistory, + turnId: ds.session.queuedActivationTurnId, + dispatchAttempt: ds.session.queuedActivationDispatchAttempt, + }, + ); + if (ownsInitialStartClaim()) retainInitialStartClaim = true; + } catch (err) { + ds.initialStartPending = false; + throw err; + } + return; + } + // A worker that died before submitting a queued activation leaves the + // exact opening payload parked in the durable journal. Wake that opening + // unchanged and stage this new inbound turn behind it as a separate FIFO + // item; rebuilding `queuedPrompt + current` here would lose any reply that + // had already been folded into the retained activation payload. + const retainedQueuedActivation = ds.session.queued === true + ? ds.session.queuedActivationInput + : undefined; + if (retainedQueuedActivation) { + await stageCurrentBehindQueuedActivation(); + try { + forkWorker(ds, retainedQueuedActivation, { + resume: ds.hasHistory, + turnId: ds.session.queuedActivationTurnId, + dispatchAttempt: ds.session.queuedActivationDispatchAttempt, + }); + if (ownsInitialStartClaim()) retainInitialStartClaim = true; + } catch (err) { + // Keep the staged tail for a later activation attempt, but release the + // route gate so another inbound turn can trigger that attempt. + ds.initialStartPending = false; + throw err; + } + return; + } // 待办池(queued)会话：CLI 从没起过，暂存的任务内容(queuedPrompt，已按角色包装好) // 必须当首轮发出去——否则群里来的这第一条消息会顶替掉它、把用户分配的任务丢掉。 // 把暂存任务前置、用户这条消息拼在后面，一并作为首轮。forkWorker 随后清 queued。 const queuedDashboardTurn = !!(ds.session.queued && ds.session.queuedPrompt); - const reforkContent = queuedDashboardTurn + // A restored pending-repo/backlog owner may already have durable N+1... + // behind its unopened N. Appending this inbound to N would overtake those + // persisted entries. Keep N exact and append the current turn after the + // existing tail before starting the activation journal. + const queuedHasDurableTail = queuedDashboardTurn + && (ds.session.queuedActivationTail?.length ?? 0) > 0; + if (queuedHasDurableTail) await stageCurrentBehindQueuedActivation(); + const reforkContent = queuedDashboardTurn && !queuedHasDurableTail ? `${ds.session.queuedPrompt}\n\n${promptContent}` - : promptContent; + : queuedHasDurableTail + ? ds.session.queuedPrompt! + : promptContent; const queuedCodexAppText = ds.session.queuedCodexAppText ?? ds.pendingCodexAppText; const reforkCodexApp = mergeQueuedCodexAppTurn({ queued: queuedDashboardTurn, queuedText: queuedCodexAppText, queuedMessageContext: ds.session.queuedCodexAppMessageContext ?? ds.pendingCodexAppMessageContext, - currentText: parsed.content, - currentMessageContext: codexAppMessageContext, + currentText: queuedHasDurableTail ? '' : parsed.content, + currentMessageContext: queuedHasDurableTail ? undefined : codexAppMessageContext, }); const builtReforkInput = buildReforkCliInput(ds, reforkContent, { - attachments, - mentions: parsed.mentions, + attachments: queuedHasDurableTail ? undefined : attachments, + mentions: queuedHasDurableTail ? undefined : parsed.mentions, cliId: ds.session.cliId ?? dsBotCfgForFork.cliId, cliPathOverride: ds.session.cliPathOverride ?? dsBotCfgForFork.cliPathOverride, selfMention: { name: selfBot.botName, openId: selfBot.botOpenId }, - sender: await getThreadSender(), - substituteTrigger, + sender: queuedHasDurableTail ? undefined : await getThreadSender(), + substituteTrigger: queuedHasDurableTail ? undefined : substituteTrigger, codexAppText: reforkCodexApp.text, - codexAppApplicationContext, + codexAppApplicationContext: queuedHasDurableTail ? undefined : codexAppApplicationContext, codexAppMessageContext: reforkCodexApp.messageContext, }); const wrappedInput = applyQueuedCodexAppLegacyFallback(builtReforkInput, { @@ -16324,13 +17364,35 @@ async function handleThreadReply(data: any, ctx: RoutingContext): Promise // contain the reply and would silently discard the original task. logger.warn(`[${tag(ds)}] Legacy queued dashboard task has no clean-input text; using the full legacy activation prompt`); } - await noteTurnReceived(ds, parsed.messageId, parsed.content, await getThreadSender(), parsed.messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); - rememberLastCliInput(ds, promptContent, wrappedInput); + if (!queuedHasDurableTail) { + await noteTurnReceived(ds, parsed.messageId, parsed.content, await getThreadSender(), parsed.messageId, substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined); + } + rememberLastCliInput(ds, queuedHasDurableTail ? ds.session.queuedPrompt! : promptContent, wrappedInput); sessionStore.updateSession(ds.session); forkWorker(ds, wrappedInput, { resume: ds.hasHistory, - turnId: parsed.messageId, + turnId: queuedHasDurableTail + ? (ds.session.queuedActivationTurnId ?? `queued-opening:${ds.session.sessionId}`) + : parsed.messageId, }); + if (ownsInitialStartClaim()) { + if (initialStartClaimWaitsForQueuedAck) { + retainInitialStartClaim = true; + } else { + // Ordinary cold reforks have no adapter-submission ACK. The child owns + // the opening init after forkWorker returns, so hand any concurrently + // buffered followers to its IPC queue now in exact arrival order. + retainInitialStartClaim = !releaseQueuedActivationReservation(ds); + } + } + } + } finally { + // Pre-fork quota/resource/persistence failures and any non-queued route + // that returned without accepting a worker must not strand the route gate. + // Fence by token so an older handler can never clear a newer generation. + if (ownsInitialStartClaim() && !retainInitialStartClaim) { + clearInitialStartClaim(ds!, initialStartClaimToken); + } } } @@ -16358,7 +17420,11 @@ async function autoCreateDocSession(sub: DocSubscription, larkAppId: string, ctx const title = `[Doc] ${sub.fileToken.slice(0, 8)}: ${ctx.text.slice(0, 40)}`; const virtualChatId = `doc:${sub.fileToken}`; - const virtualAnchor = sub.sessionAnchor; + // A native doc session is chat-scoped, therefore its canonical routing + // anchor is its virtual chat id. Persist that same identity back into the + // subscription; using the historical IM anchor here made close/restore and + // comment lookup disagree about which Map key owned the session. + const virtualAnchor = virtualChatId; const now = Date.now(); const session = sessionStore.createSession(virtualChatId, virtualAnchor, title, 'group'); @@ -16412,7 +17478,21 @@ async function autoCreateDocSession(sub: DocSubscription, larkAppId: string, ctx (ds.session.docCommentTargets ??= {})[turnId] = docTarget; try { sessionStore.updateSession(ds.session); } catch { /* best-effort */ } - activeSessions.set(sessionKey(virtualAnchor, larkAppId), ds); + const key = activeSessionKey(ds); + if (key !== sessionKey(virtualAnchor, larkAppId)) { + throw new Error(`doc session canonical key mismatch: ${key}`); + } + if (activeSessions.has(key)) { + throw new Error(`doc session canonical key already owned: ${key}`); + } + activeSessions.set(key, ds); + Object.assign(sub, { + sessionAnchor: virtualAnchor, + sessionId: session.sessionId, + scope: 'chat' as const, + chatId: virtualChatId, + }); + putDocSubscription(config.session.dataDir, larkAppId, sub); // 不在这里 forkWorker —— handleDocComment 会统一处理（它会检查 worker 状态、 // 加 reaction、设 docCommentTurns、然后 fork 或 send）。这里只建好 session 骨架。 @@ -16432,30 +17512,125 @@ async function autoCreateDocSession(sub: DocSubscription, larkAppId: string, ctx * 走 resume 重 fork）；已 /close 的会话其订阅在关闭时已退订，这里查不到 ds 即跳过。 */ async function handleDocComment(ctx: DocCommentContext): Promise { - const { larkAppId, sub, commentId, text } = ctx; - const turnId = ctx.replyId || commentId; - const claimKey = `${larkAppId}:${sub.fileToken}:${turnId}`; + return withBotTurnAdmission( + ctx.larkAppId, + () => handleDocCommentAdmitted(ctx), + ); +} + +async function handleDocCommentAdmitted(ctx: DocCommentContext): Promise { + const turnId = ctx.replyId || ctx.commentId; + const claimKey = `${ctx.larkAppId}:${ctx.sub.fileToken}:${turnId}`; if (handledDocCommentTurns.has(claimKey)) { - logger.info(`[doc-comment] duplicate turn skipped file=${sub.fileToken.slice(0, 12)} turn=${turnId.slice(0, 12)}`); + logger.info(`[doc-comment] duplicate turn skipped file=${ctx.sub.fileToken.slice(0, 12)} turn=${turnId.slice(0, 12)}`); return true; // 已处理过，算成功（让 poller 推进游标） } - const loc = localeForBot(larkAppId); + const existing = docCommentTurnsInFlight.get(claimKey); + if (existing) { + logger.info(`[doc-comment] duplicate turn awaiting in-flight result file=${ctx.sub.fileToken.slice(0, 12)} turn=${turnId.slice(0, 12)}`); + return existing; + } - let ds: DaemonSession | undefined | null = activeSessions.get(sessionKey(sub.sessionAnchor, larkAppId)); - if (!ds) { - // 无活跃 session → 自动为该文档创建一个（用虚拟 anchor = doc:{fileToken}） - logger.info(`[doc-comment] no active session for anchor=${sub.sessionAnchor.slice(0, 12)}; auto-creating for file=${sub.fileToken.slice(0, 12)}`); - ds = await autoCreateDocSession(sub, larkAppId, ctx); - if (!ds) { - // auto-create 失败：不设 claim（允许后续重试），返回 false 让 poller 不推进游标 - logger.warn(`[doc-comment] auto-create session failed for file=${sub.fileToken.slice(0, 12)}; will retry comment ${commentId.slice(0, 12)}`); - return false; + // Defer the body by one microtask so the Promise is published before any + // async lookup can yield. WS and poll deliveries of the same comment then + // observe one authoritative success/failure result. + const delivery = Promise.resolve().then(async () => { + // Different comments for one document are also ordered through initial + // worker creation. Without this full-delivery lane, both can resolve the + // same worker:null owner, await reactions, and each fork a generation. + const deliveryKey = `\u0000doc-delivery:${ctx.larkAppId}:${ctx.sub.fileToken}`; + const delivered = await withActiveSessionKeyLock( + activeSessions, + deliveryKey, + () => deliverDocCommentTurn(ctx, claimKey), + ); + if (delivered) handledDocCommentTurns.set(claimKey, Date.now()); + return delivered; + }); + docCommentTurnsInFlight.set(claimKey, delivery); + try { + return await delivery; + } finally { + if (docCommentTurnsInFlight.get(claimKey) === delivery) { + docCommentTurnsInFlight.delete(claimKey); } } - // ds 确认有效后才设 claim——auto-create 失败时不占坑，允许重试。 - handledDocCommentTurns.set(claimKey, Date.now()); +} + +async function deliverDocCommentTurn(ctx: DocCommentContext, claimKey: string): Promise { + const { larkAppId, commentId, text } = ctx; + let sub = ctx.sub; + const turnId = ctx.replyId || commentId; + const loc = localeForBot(larkAppId); + // The document identity, not a possibly-stale subscription anchor, is the + // creation lock. Otherwise one caller can lock an old IM anchor while a + // second locks doc:, and both can create a native session. + const docKey = sessionKey(`doc:${sub.fileToken}`, larkAppId); try { + const resolved = await withActiveSessionKeyLock(activeSessions, docKey, async () => { + sub = getDocSubscription(config.session.dataDir, larkAppId, sub.fileToken) ?? sub; + let current = sub.sessionId + ? [...activeSessions.values()].find(candidate => + candidate.larkAppId === larkAppId + && candidate.session.sessionId === sub.sessionId) + : undefined; + current ??= activeSessions.get(sessionKey(sub.sessionAnchor, larkAppId)); + // A legacy native doc session may still be stored under the subscription's + // old IM anchor. Prefer an already-canonical owner if one exists, otherwise + // migrate the exact object synchronously while holding the doc key lock. + if (current?.scope === 'chat' && current.chatId === `doc:${sub.fileToken}`) { + current = activeSessions.get(docKey) ?? current; + if (activeSessions.get(docKey) !== current) { + for (const [registeredKey, candidate] of activeSessions) { + if (candidate === current) activeSessions.delete(registeredKey); + } + activeSessions.set(docKey, current); + } + } + current ??= activeSessions.get(docKey); + if (!current) { + logger.info(`[doc-comment] no active session for anchor=${sub.sessionAnchor.slice(0, 12)}; auto-creating for file=${sub.fileToken.slice(0, 12)}`); + current = await autoCreateDocSession(sub, larkAppId, ctx) ?? undefined; + if (!current) return { failed: true as const }; + } + + const canonicalAnchor = sessionAnchorId(current); + if (sub.sessionAnchor !== canonicalAnchor + || sub.sessionId !== current.session.sessionId + || sub.scope !== current.scope + || sub.chatId !== current.chatId) { + sub = { + ...sub, + sessionAnchor: canonicalAnchor, + sessionId: current.session.sessionId, + scope: current.scope, + chatId: current.chatId, + }; + putDocSubscription(config.session.dataDir, larkAppId, sub); + } + return { ds: current }; + }); + if ('failed' in resolved) { + logger.warn(`[doc-comment] auto-create session failed for file=${sub.fileToken.slice(0, 12)}; will retry comment ${commentId.slice(0, 12)}`); + return false; + } + const ds = resolved.ds; + + if ((!ds.worker || ds.worker.killed) + && (ds.pendingRepo + || ds.worktreeCreating + || hasProtectedSessionMutationOwnership(ds))) { + // The provider cursor is the retry owner. A dormant setup/activation must + // not be replaced by a comment refork; returning false leaves this exact + // comment available after the opening owner reconciles. + logger.warn( + `[doc-comment] turn ${turnId.slice(0, 12)} deferred before acceptance ` + + `while session ${ds.session.sessionId.slice(0, 8)} has durable opening ownership`, + ); + return false; + } + // 给用户的回复加 "Typing" reaction，让评论者知道 bot 正在处理。 const userReplyId = ctx.replyId; let reactionId: string | undefined; @@ -16517,7 +17692,10 @@ async function handleDocComment(ctx: DocCommentContext): Promise { sessionStore.updateSession(ds.session); // 先落盘，botmux send 子进程才读得到落点 await noteTurnReceived(ds, commentId, text, sender, turnId); rememberLastCliInput(ds, promptContent, cliInput); - sendWorkerInput(ds, cliInput, turnId); + if (!sendWorkerInput(ds, cliInput, turnId)) { + logger.warn(`[${tag(ds)}] doc-comment turn was not accepted; provider retry retained`); + return false; + } logger.info(`[${tag(ds)}] doc-comment turn injected (turn ${turnId.slice(0, 8)})`); } else { // Worker 挂起 / 已退出 —— resume 重 fork（与 handleThreadReply 同路）。 @@ -16551,15 +17729,53 @@ async function handleDocComment(ctx: DocCommentContext): Promise { } return true; } catch (err) { - // 投递失败：清理 claim 允许重试，返回 false 让 poller 不推进游标。 - handledDocCommentTurns.delete(claimKey); - logger.warn(`[doc-comment] delivery failed, claim released for retry file=${sub.fileToken.slice(0, 12)} turn=${turnId.slice(0, 12)} err=${err instanceof Error ? err.message : String(err)}`); + // No completed claim exists yet. Return false to keep the poll cursor in + // place; every concurrent duplicate shares this same failure via the + // single-flight Promise and a later delivery can retry. + logger.warn(`[doc-comment] delivery failed, retry remains eligible file=${sub.fileToken.slice(0, 12)} turn=${turnId.slice(0, 12)} claim=${claimKey} err=${err instanceof Error ? err.message : String(err)}`); return false; } } /** 同一条评论可能同时被长连接通知和 --all 轮询看到；daemon 内统一去重。 */ const handledDocCommentTurns = new BoundedMap(5_000); +const docCommentTurnsInFlight = new Map>(); + +export const __testOnly_handleDocComment = (ctx: DocCommentContext): Promise => handleDocComment(ctx); +export function __testOnly_resetDocCommentClaims(): void { + handledDocCommentTurns.clear(); + docCommentTurnsInFlight.clear(); +} + +/** Preserve accepted/pending work when a normal group is converted to topic + * mode. The converted event is routed to a new thread key, so keeping the old + * chat owner does not block the new session; it only keeps the durable owner + * addressable for completion or explicit close. */ +function handleChatModeConverted(chatId: string, larkAppId: string): boolean { + const key = sessionKey(chatId, larkAppId); + const owner = activeSessions.get(key); + if (!owner) { + logger.info(`[chat-mode-converted] ${chatId.substring(0, 12)} evicted=false; no chat-scope owner`); + return false; + } + // Conversion preprocessing can run outside bot-turn admission. Converted + // turns route by their new thread key, so retaining a pending owner at the + // old chat key does not block them. + const pending = hasProtectedSessionMutationOwnership(owner) + || owner.pendingRepo === true; + if (pending) { + logger.warn( + `[chat-mode-converted] ${chatId.substring(0, 12)} preserved pending owner ` + + `${owner.session.sessionId.substring(0, 8)}; converted turns route by thread key`, + ); + return false; + } + const evicted = activeSessions.get(key) === owner && activeSessions.delete(key); + logger.info(`[chat-mode-converted] ${chatId.substring(0, 12)} evicted=${evicted}; worker (if any) keeps running until /close`); + return evicted; +} + +export const __testOnly_handleChatModeConverted = handleChatModeConverted; let docCommentPollRunning = false; @@ -16996,29 +18212,64 @@ export async function startDaemon(botIndex?: number): Promise { // One cap implementation shared by event-driven checks (process start / idle // edge) and the 60s safety-net timer below. Each daemon owns exactly one // bot's activeSessions map, so the configured limit is per bot. + let liveSessionCapSweepPending = false; const enforceLiveSessionCap = (source: 'session_change' | 'periodic'): void => { - const maxLiveWorkers = getBot(cfg.larkAppId).config.maxLiveWorkers; - const suspended = sweepIdleWorkers(activeSessions, { maxLiveWorkers }); - if (suspended.length > 0) { - logger.info( - `[idle-worker-sweeper] suspended ${suspended.length} session(s) over per-bot cap ` - + `${maxLiveWorkers ?? DEFAULT_MAX_LIVE_WORKERS} source=${source}`, - ); - } + // Coalesce spawn/idle/timer bursts. The detached mutation evaluates the + // current map and current config only after every admitted turn has drained, + // so one pending sweep subsumes all triggers that arrive before it runs. + if (liveSessionCapSweepPending) return; + liveSessionCapSweepPending = true; + void (async () => { + try { + const maxLiveWorkers = getBot(cfg.larkAppId).config.maxLiveWorkers; + const suspended = await sweepIdleWorkersAfterTurnDrain( + cfg.larkAppId, + activeSessions, + { maxLiveWorkers }, + ); + if (suspended.length > 0) { + logger.info( + `[idle-worker-sweeper] suspended ${suspended.length} session(s) over per-bot cap ` + + `${maxLiveWorkers ?? DEFAULT_MAX_LIVE_WORKERS} source=${source}`, + ); + } + } catch (err) { + logger.warn(`[idle-worker-sweeper] cap enforcement failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + liveSessionCapSweepPending = false; + } + })(); }; // Initialise worker pool with daemon callbacks initWorkerPool({ sessionReply, getSessionWorkingDir, getActiveCount, - closeSession(ds: DaemonSession) { + closeSession(ds: DaemonSession): Promise { // Route through the dashboard-aware helper so session.exited / session.update // events fire for withdrawn-message / crash / adopt-exit teardown paths too, // matching the dashboard-driven close. - void closeSessionHelper(ds.session.sessionId).catch(() => { /* idempotent */ }); - logger.info(`[${ds.session.sessionId.substring(0, 8)}] Session auto-closed (message withdrawn)`); + const sessionId = ds.session.sessionId; + return runDetachedBotTurnMutation(ds.larkAppId, async () => { + const current = findActiveBySessionId(sessionId); + if (current !== ds) return false; + if (hasProtectedSessionMutationOwnership(current)) { + logger.info(`[${sessionId.substring(0, 8)}] Withdraw auto-close deferred: Codex App FIFO became unsettled`); + return false; + } + await closeSessionHelper(sessionId); + logger.info(`[${sessionId.substring(0, 8)}] Session auto-closed (message withdrawn)`); + return true; + }).catch((err) => { + logger.warn( + `[${sessionId.substring(0, 8)}] Withdraw auto-close mutation failed; session retained: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return false; + }); }, enforceLiveSessionCap: () => enforceLiveSessionCap('session_change'), + onQueuedActivationSubmitted, onTurnTerminal(ds, terminal, context) { const enqueued = vcMeetingTerminalReconciler?.enqueue(terminal, context); if (terminal.dispatchAttempt !== undefined && enqueued?.accepted) { @@ -17074,6 +18325,11 @@ export async function startDaemon(botIndex?: number): Promise { onDurableExpiryReady(_ds, context) { vcMeetingRuntimeLeaseRecovery.acknowledge(context); }, + async onCodexAppLedgerDrained(ds) { + await withBotTurnMutation(ds.larkAppId, async () => { + await closeCliMismatchedSessionsForBot(ds.larkAppId); + }); + }, }); // Expose the activeSessions Map (owned by daemon) to worker-pool readers, // so dashboard IPC and other consumers can list/lookup live sessions. @@ -17103,10 +18359,13 @@ export async function startDaemon(botIndex?: number): Promise { loadOrCreateDashboardSecret( join(homedir(), '.botmux', '.dashboard-secret'), ); + let markIpcReady!: () => void; + const ipcReady = new Promise((resolve) => { markIpcReady = resolve; }); const ipcHandle = await startIpcServer({ port: ipcPort, host: '127.0.0.1', authRequired: true, + ready: ipcReady, }); // startIpcServer probes upward on EADDRINUSE (e.g. a second botmux instance on // this host already holds ipcBasePort+idx), so the bound port may differ from @@ -17116,6 +18375,20 @@ export async function startDaemon(botIndex?: number): Promise { process.env.BOTMUX_DAEMON_IPC_PORT = String(ipcHandle.port); logger.info(`[dashboard-ipc] listening on 127.0.0.1:${ipcHandle.port} (bot ${idx})`); + // Publish daemon ownership immediately after IPC binds, then perform the + // first session-file load under SessionStore's cross-process lock. An + // offline CLI either observes this descriptor and delegates, or it already + // holds the file lock; in the latter case this load waits and sees its atomic + // mutation. Never load a stale cache in an unadvertised startup window. + desc.lastHeartbeat = Date.now(); + writeDaemonDescriptor(desc); + sessionStore.listSessions(); + const descriptorHeartbeat = setInterval(() => { + desc.lastHeartbeat = Date.now(); + try { writeDaemonDescriptor(desc); } catch { /* best effort */ } + }, 30_000); + if (typeof descriptorHeartbeat.unref === 'function') descriptorHeartbeat.unref(); + // Single reverse-proxy port that fronts every session's web terminal under // /s/{sessionId}, so dev-machine users forward one port (proxyBasePort+idx) // instead of one per topic. Bound on the public host so `ssh -L` can reach it. @@ -17134,10 +18407,25 @@ export async function startDaemon(botIndex?: number): Promise { // Quiet-restart leaves sessions registered but worker-less until messaged. // Wake the worker on terminal access so links open without a manual ping. ensureWorkerPort: async (sessionId) => { + let owner: DaemonSession | undefined; for (const ds of activeSessions.values()) { - if (ds.session.sessionId === sessionId) return ensureTerminalWorkerPort(ds); + if (ds.session.sessionId === sessionId) { + owner = ds; + break; + } } - return undefined; + if (!owner) return undefined; + return withBotTurnAdmission(owner.larkAppId, () => { + // The gate may have held this request behind a mutation/close. Resolve + // the canonical record again before a lazy fork; never wake a stale ds + // captured before the wait. + for (const current of activeSessions.values()) { + if (current.session.sessionId === sessionId) { + return ensureTerminalWorkerPort(current); + } + } + return undefined; + }); }, }); // Only mark the proxy live after a successful bind — buildTerminalUrl then @@ -17158,17 +18446,6 @@ export async function startDaemon(botIndex?: number): Promise { logger.info(`[terminal-proxy] terminal links advertise external port ${config.web.externalPort + idx} (WEB_EXTERNAL_PORT ${config.web.externalPort} + bot ${idx})`); } - // Now that the IPC port is actually listening, publish the descriptor so - // the dashboard can discover us and successfully fetch /api/sessions etc. - desc.lastHeartbeat = Date.now(); - writeDaemonDescriptor(desc); - const descriptorHeartbeat = setInterval(() => { - desc.lastHeartbeat = Date.now(); - try { writeDaemonDescriptor(desc); } catch { /* best effort */ } - }, 30_000); - // Don't keep the event loop alive on this interval alone. - if (typeof descriptorHeartbeat.unref === 'function') descriptorHeartbeat.unref(); - // Reap a dead prior daemon's detached classifier before any prepared // proposal can be recovered. Per-proposal generation locks still serialize // live rolling-restart overlap; this sweep handles only dead-owner markers. @@ -17189,6 +18466,11 @@ export async function startDaemon(botIndex?: number): Promise { ); } + // Do not start any Lark dispatcher until durable sessions have been restored + // into the routing registry. Otherwise an inbound same-anchor turn can create + // a second owner while the old unsettled row exists only on disk. + const startEventDispatchers: Array<() => void> = []; + // Per-bot initialization for (const bot of getAllBots()) { const cfg = bot.config; @@ -17272,34 +18554,44 @@ export async function startDaemon(botIndex?: number): Promise { ); } - // Start event dispatcher for this bot + // Build the dispatcher now for authorization replay, but start it only + // after restore has published every durable route owner. const botEventHandlers: EventHandlers = { - handleCardAction: (data, appId) => handleCardAction(data, cardDeps, appId), + handleCardAction: (data, appId) => withBotTurnAdmission( + appId, + () => handleCardAction(data, cardDeps, appId), + ), handleNewTopic: (data, ctx) => handleNewTopic(data, ctx), handleThreadReply: (data, ctx) => handleThreadReply(data, ctx), - handleBotAdded: (chatId, operatorOpenId, appId) => handleBotAdded(chatId, operatorOpenId, appId), + handleBotAdded: (chatId, operatorOpenId, appId) => withBotTurnAdmission( + appId, + () => handleBotAdded(chatId, operatorOpenId, appId), + ), handleDocComment: (ctx) => handleDocComment(ctx), - handleVcMeetingPush: (ctx) => handleVcMeetingPush(ctx), + handleVcMeetingPush: (ctx) => withBotTurnAdmission( + ctx.larkAppId, + () => handleVcMeetingPush(ctx), + ), beforeSessionTurn: (data, ctx) => maybeCatchUpVcMeetingConsumerBeforeTurn(data, ctx), isSessionOwner: (anchor, appId) => activeSessions.has(sessionKey(anchor, appId)), resolveReplyThreadAlias: (rootId, chatId, appId) => findChatReplyAlias(rootId, chatId, appId), // Chat was converted 普通群 → 话题群 while we held a chat-scope session. - // Evict it from the routing map so subsequent inbound messages can land - // on a fresh thread-scope session (dispatcher already rerouted this turn - // to handleNewTopic). The worker is left running on purpose: the user may - // still have its web terminal open, and `/close` is the canonical cleanup - // path. Scheduler tasks tied to this session keep their `scope='chat'` - // semantics — that's an edge case worth following up on, not blocking - // the main fix. + // Idle legacy owners are evicted so subsequent inbound messages land on + // fresh thread-scope sessions. Owners with accepted/pending work remain + // addressable; the dispatcher already reroutes converted turns by their + // new thread root, so preserving that old chat key cannot steal them. onChatModeConverted: (chatId, appId) => { - const key = sessionKey(chatId, appId); - const evicted = activeSessions.delete(key); - logger.info(`[chat-mode-converted] ${chatId.substring(0, 12)} evicted=${evicted}; worker (if any) keeps running until /close`); + handleChatModeConverted(chatId, appId); }, }; // 存起来供授权成功后重放消息用（replayGrantedMessage → replayMessageEvent）。 botHandlers.set(cfg.larkAppId, botEventHandlers); - startLarkEventDispatcher(cfg.larkAppId, cfg.larkAppSecret, botEventHandlers, normalizeBrand(cfg.brand)); + startEventDispatchers.push(() => startLarkEventDispatcher( + cfg.larkAppId, + cfg.larkAppSecret, + botEventHandlers, + normalizeBrand(cfg.brand), + )); // A distillation command is durably prepared before its model run/card // delivery. Resume active prepared/proposed allocations after a daemon @@ -17332,6 +18624,12 @@ export async function startDaemon(botIndex?: number): Promise { for (const bot of getAllBots()) { markForwardFollowupsSessionsReady(bot.config.larkAppId); } + // The descriptor was intentionally published before restore so offline CLI + // mutations delegate to this daemon. Release those queued IPC calls only + // after every durable owner is visible in the canonical registry. + markIpcReady(); + + for (const startDispatcher of startEventDispatchers) startDispatcher(); try { await reconcileVcMeetingManagedActionsOnBoot(cfg.larkAppId); @@ -17372,7 +18670,11 @@ export async function startDaemon(botIndex?: number): Promise { || receiver.memberEpoch !== ref.memberEpoch) { // Corrupt identity must not restart an unrelated session or silently // un-gate delivery. Keep the daemon fail-closed for operator repair. - addVcMeetingReceiverRecoveryPending(recoveryKey, vcMeetingDeliveryScopeFromRef(ref)); + addVcMeetingReceiverRecoveryPending(recoveryKey, { + ...vcMeetingDeliveryScopeFromRef(ref), + turnId: ref.deliveryKey, + dispatchAttempt: ref.dispatchAttempt, + }); logger.error( `[vc-delivery] boot receiver identity conflict; delivery remains gated ` + `session=${ref.receiverSessionId}`, @@ -17385,7 +18687,11 @@ export async function startDaemon(botIndex?: number): Promise { vcMeetingRuntimeLeaseRecovery.arm(ref, cfg.larkAppId); continue; } - addVcMeetingReceiverRecoveryPending(recoveryKey, vcMeetingDeliveryScopeFromRef(ref)); + addVcMeetingReceiverRecoveryPending(recoveryKey, { + ...vcMeetingDeliveryScopeFromRef(ref), + turnId: ref.deliveryKey, + dispatchAttempt: ref.dispatchAttempt, + }); armVcMeetingReceiverRecoveryTimeout(recoveryKey, ref.receiverSessionId); try { ds.worker.send({ @@ -17473,7 +18779,10 @@ export async function startDaemon(botIndex?: number): Promise { // each filters to only execute tasks whose `larkAppId` matches its bot // (unmatched tasks are handled by the owning bot's daemon instead; a // missing larkAppId falls through to bot-0 as a legacy fallback). - scheduler.setExecuteCallback((task) => executeScheduledTask(task, activeSessions, refreshCliVersion)); + scheduler.setExecuteCallback((task) => withBotTurnAdmission( + task.larkAppId ?? cfg.larkAppId, + () => executeScheduledTask(task, activeSessions, refreshCliVersion), + )); scheduler.setOwnerFilter(cfg.larkAppId, idx === 0); scheduler.startScheduler(); diff --git a/src/dashboard/session-card-model.ts b/src/dashboard/session-card-model.ts index 54115593d..9530baa2e 100644 --- a/src/dashboard/session-card-model.ts +++ b/src/dashboard/session-card-model.ts @@ -21,6 +21,7 @@ export type SessionStatus = | 'working' | 'idle' | 'analyzing' + | 'stalled' | 'limited' | 'starting' | 'dormant' @@ -83,6 +84,8 @@ export function statusToDot(status: string): StatusDot { return { tone: 'success', pulse: true, label: 'sessions.status.working' }; case 'analyzing': return { tone: 'info', pulse: true, label: 'sessions.status.analyzing' }; + case 'stalled': + return { tone: 'danger', pulse: false, label: 'sessions.status.stalled' }; case 'starting': return { tone: 'info', pulse: true, label: 'sessions.status.starting' }; case 'idle': @@ -183,11 +186,12 @@ export function filterByCli(entries: ReadonlyArray, cliId: string const STATUS_ORDER: Record = { working: 0, analyzing: 1, - starting: 2, - idle: 3, - dormant: 4, - limited: 5, - closed: 6, + stalled: 2, + starting: 3, + idle: 4, + dormant: 5, + limited: 6, + closed: 7, }; function statusRank(status: string): number { diff --git a/src/dashboard/web/i18n.ts b/src/dashboard/web/i18n.ts index 0c6e94e94..dae4a0f0e 100644 --- a/src/dashboard/web/i18n.ts +++ b/src/dashboard/web/i18n.ts @@ -295,6 +295,7 @@ const zh: DashboardMessages = { 'sessions.status.idle': '空闲', 'sessions.status.dormant': '休眠', 'sessions.status.analyzing': '分析中', + 'sessions.status.stalled': '长时间无进展', 'sessions.status.active': '活动中', 'sessions.status.limited': '限额已达', 'sessions.status.closed': '已关闭', @@ -967,7 +968,7 @@ const zh: DashboardMessages = { 'sessions.topic.empty': '暂无匹配的话题会话', 'sessions.selectSession': '选择会话', 'sessions.board.needsYou': '需要你', - 'sessions.board.needsYouHint': '等待仓库、TUI 选择或额度处理', + 'sessions.board.needsYouHint': '等待仓库、TUI 选择、额度处理或无进展检查', 'sessions.board.starting': '启动中', 'sessions.board.startingHint': 'CLI 正在创建或恢复', 'sessions.board.working': '干活中', @@ -978,6 +979,7 @@ const zh: DashboardMessages = { 'sessions.board.signalRepo': '待选仓库', 'sessions.board.signalPrompt': '等待 TUI 选择', 'sessions.board.signalLimited': '额度受限', + 'sessions.board.signalStalled': '长时间无进展', 'sessions.board.signalAgent': '需要人工介入', 'sessions.board.waiting': '等待', 'sessions.board.dragHint': '拖动列头调整列顺序', @@ -2247,6 +2249,7 @@ const en: DashboardMessages = { 'sessions.status.idle': 'Idle', 'sessions.status.dormant': 'Dormant', 'sessions.status.analyzing': 'Analyzing', + 'sessions.status.stalled': 'No recent progress', 'sessions.status.active': 'Active', 'sessions.status.limited': 'Limit reached', 'sessions.status.closed': 'Closed', @@ -2919,7 +2922,7 @@ const en: DashboardMessages = { 'sessions.topic.empty': 'No matching topic sessions', 'sessions.selectSession': 'Select session', 'sessions.board.needsYou': 'Needs You', - 'sessions.board.needsYouHint': 'Repo, TUI, or usage limit waiting', + 'sessions.board.needsYouHint': 'Repo, TUI, usage limit, or no-progress attention', 'sessions.board.starting': 'Starting', 'sessions.board.startingHint': 'CLI is spawning or resuming', 'sessions.board.working': 'Working', @@ -2930,6 +2933,7 @@ const en: DashboardMessages = { 'sessions.board.signalRepo': 'Repo needed', 'sessions.board.signalPrompt': 'TUI choice needed', 'sessions.board.signalLimited': 'Usage limited', + 'sessions.board.signalStalled': 'No recent progress', 'sessions.board.signalAgent': 'Needs human input', 'sessions.board.waiting': 'Waiting', 'sessions.board.dragHint': 'Drag header to reorder columns', diff --git a/src/dashboard/web/kanban-model.ts b/src/dashboard/web/kanban-model.ts index 3cf616f10..846f94366 100644 --- a/src/dashboard/web/kanban-model.ts +++ b/src/dashboard/web/kanban-model.ts @@ -25,7 +25,7 @@ export function deriveKanbanColumn(s: KanbanRowLike): SessionKanbanColumn { if (s.status === 'closed') return 'done'; const manual = normalizeKanbanColumn(s.kanbanColumn); if (manual) return manual; - if (s.pendingRepo || s.tuiPromptActive || s.agentAttention || s.status === 'limited') return 'in_review'; + if (s.pendingRepo || s.tuiPromptActive || s.agentAttention || s.status === 'limited' || s.status === 'stalled') return 'in_review'; if (s.status === 'starting' || s.status === 'working' || s.status === 'analyzing' || s.status === 'active') { return 'in_progress'; } diff --git a/src/dashboard/web/sessions-kanban.tsx b/src/dashboard/web/sessions-kanban.tsx index 4e9b825ac..8fee92719 100644 --- a/src/dashboard/web/sessions-kanban.tsx +++ b/src/dashboard/web/sessions-kanban.tsx @@ -143,6 +143,7 @@ function boardSignalLabel(s: any): string { if (s.pendingRepo) return t('sessions.board.signalRepo'); if (s.tuiPromptActive) return t('sessions.board.signalPrompt'); if (s.status === 'limited') return t('sessions.board.signalLimited'); + if (s.status === 'stalled') return t('sessions.board.signalStalled'); return ''; } diff --git a/src/dashboard/web/sessions-page.tsx b/src/dashboard/web/sessions-page.tsx index c3e68dbde..50ed7905f 100644 --- a/src/dashboard/web/sessions-page.tsx +++ b/src/dashboard/web/sessions-page.tsx @@ -335,6 +335,7 @@ function boardSignalLabel(s: any): string { if (s.pendingRepo) return t('sessions.board.signalRepo'); if (s.tuiPromptActive) return t('sessions.board.signalPrompt'); if (s.status === 'limited') return t('sessions.board.signalLimited'); + if (s.status === 'stalled') return t('sessions.board.signalStalled'); return ''; } diff --git a/src/dashboard/web/sessions.ts b/src/dashboard/web/sessions.ts index 2a2a37f47..e1024aa4f 100644 --- a/src/dashboard/web/sessions.ts +++ b/src/dashboard/web/sessions.ts @@ -40,6 +40,7 @@ export const SESSION_STATUS_OPTIONS = [ 'idle', 'dormant', 'analyzing', + 'stalled', 'active', 'limited', 'closed', @@ -317,7 +318,7 @@ export function historySenderKey(message: any): string { export function deriveSessionBoardColumn(s: any): BoardColumnId | null { if (s.status === 'closed') return null; - if (s.pendingRepo || s.tuiPromptActive || s.agentAttention || s.status === 'limited') return 'needs-you'; + if (s.pendingRepo || s.tuiPromptActive || s.agentAttention || s.status === 'limited' || s.status === 'stalled') return 'needs-you'; if (s.status === 'starting') return 'starting'; if (s.status === 'working' || s.status === 'analyzing' || s.status === 'active') return 'working'; if (s.status === 'dormant') return 'idle'; diff --git a/src/dashboard/web/style.css b/src/dashboard/web/style.css index e889ad0fa..f7b6f6a01 100644 --- a/src/dashboard/web/style.css +++ b/src/dashboard/web/style.css @@ -2189,6 +2189,7 @@ button.primary { .status-analyzing { background: var(--warning-soft); color: var(--warning); } .status-starting { background: var(--success-soft); color: var(--success); } .status-limited { background: var(--danger-soft); color: var(--danger); } +.status-stalled { background: var(--danger-soft); color: var(--danger); } /* ── Sessions board — "control-room signal wall" ───────────────────────────── Each column is a signal channel: a LED dot + uppercase channel label in the @@ -8360,6 +8361,7 @@ button.contrast:hover { background: var(--danger-soft); border-color: var(--dang color: var(--warning); } .status-limited { background: var(--danger-soft); color: var(--danger); } +.status-stalled { background: var(--danger-soft); color: var(--danger); } /* ── 玻璃面板（dark 下生效；light 保持实底）─────────────────────────────── */ :root[data-theme="dark"] .panel, diff --git a/src/dashboard/web/ui.ts b/src/dashboard/web/ui.ts index 4d2bc0e2f..3a15ad5e3 100644 --- a/src/dashboard/web/ui.ts +++ b/src/dashboard/web/ui.ts @@ -368,6 +368,7 @@ export function attentionReason(s: Record): string | null { if (s.pendingRepo) return t('sessions.board.signalRepo'); if (s.tuiPromptActive) return t('sessions.board.signalPrompt'); if (s.status === 'limited') return t('sessions.board.signalLimited'); + if (s.status === 'stalled') return t('sessions.board.signalStalled'); return null; } diff --git a/src/i18n/en.ts b/src/i18n/en.ts index a5db8e1b3..677ab1076 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -33,9 +33,11 @@ export const messages: Record = { 'card.status.idle': 'Awaiting input', 'card.status.dormant': 'Dormant', 'card.status.analyzing': 'Analyzing…', + 'card.status.stalled': 'No recent progress', 'card.status.limited': 'Limit reached', 'card.status.retry_ready': 'Ready to retry', 'card.status.executing': 'Executing…', + 'worker.codex_app.no_progress': '⚠️ This Codex App turn has shown no observable progress for {seconds} seconds. It may still be running; check the web terminal and, if needed, restart only this session. Botmux will not restart or replay the task automatically.', 'card.status.session_closed': '🛑 Session Closed', 'card.status.relay_frozen': '🔄 Relayed away', 'card.status.selected': 'Selected', @@ -680,6 +682,7 @@ export const messages: Record = { 'card.voice.toast_session_gone': '⚠️ Session is offline; cannot generate a voice summary.', 'card.voice.toast_need_auth': '🔒 You are not authorized to use this bot, so you cannot generate a voice summary. Ask an admin for access.', 'card.voice.toast_worker_busy': '⚠️ The session is still running. Wait for it to go idle, then generate the voice summary.', + 'card.repo.toast_stale_picker': '⚠️ This repo picker is no longer current. Use the latest card.', 'card.voice.user_message': 'Generate a voice summary', 'card.action.takeover_retired': '⚠️ The old "Take Over" button is retired. In bridge mode, botmux bridges the original CLI so replies still come back to Lark — no takeover needed. Full takeover (`/adopt --takeover`) is on the roadmap.', 'card.action.terminal_not_ready': '⚠️ Terminal is not ready yet, please try again shortly.', @@ -772,6 +775,7 @@ export const messages: Record = { 'daemon.download_failed_need_login': '⚠️ Some images/files failed to download (missing User Token). Send `/login` in this topic to authorize, then resend.', 'daemon.foreign_bot_mention_prefix': '[@mention from {botName}]', 'daemon.cmd_needs_active_cli': '{cmd} needs an active CLI process; no running session in this topic.', + 'daemon.cmd_activation_pending': '{cmd} cannot be sent yet because the previous turn is still being submitted. Retry shortly.', 'daemon.enriched_mentions_label': '@mentions in this message:', 'daemon.choose_repo_first': 'Pick a repo from the card above first — your message is queued and will be sent once a repo is chosen.', 'daemon.worktree_building_wait': 'Creating a worktree (includes a git fetch, may take a few seconds) — your message is queued and will be sent together once it is ready.', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 190e891c4..5b0ef6670 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -36,9 +36,11 @@ export const messages: Record = { 'card.status.idle': '等待输入', 'card.status.dormant': '休眠', 'card.status.analyzing': '正在分析…', + 'card.status.stalled': '长时间无进展', 'card.status.limited': '限额已达', 'card.status.retry_ready': '可重试', 'card.status.executing': '正在执行…', + 'worker.codex_app.no_progress': '⚠️ Codex App 此轮已连续 {seconds} 秒没有可观察进展。任务可能仍在运行；请检查 Web 终端，必要时仅重启当前会话。Botmux 不会自动重启或重放任务。', 'card.status.session_closed': '🛑 会话已关闭', 'card.status.relay_frozen': '🔄 会话已搬迁', 'card.status.selected': '已选择', @@ -683,6 +685,7 @@ export const messages: Record = { 'card.voice.toast_session_gone': '⚠️ 会话已不在线，无法生成语音总结', 'card.voice.toast_need_auth': '🔒 你没有该机器人的使用权限，无法生成语音总结，请联系管理员授权', 'card.voice.toast_worker_busy': '⚠️ 会话当前还在执行中，请等它空闲后再生成语音总结。', + 'card.repo.toast_stale_picker': '⚠️ 此仓库选择卡已失效，请使用最新卡片。', 'card.voice.user_message': '生成语音总结', 'card.action.takeover_retired': '⚠️ 旧版"接管"按钮已停用。bridge 模式下原 CLI 由 botmux 桥接，无需接管即可在飞书中收到回答。如需 /resume 完整接管能力，请等待 /adopt --takeover 命令上线。', 'card.action.terminal_not_ready': '⚠️ 终端尚未就绪，请稍后再试。', @@ -775,6 +778,7 @@ export const messages: Record = { 'daemon.download_failed_need_login': '⚠️ 部分图片/文件下载失败（缺少 User Token）。请在话题中发送 /login 授权后重新发送。', 'daemon.foreign_bot_mention_prefix': '[来自 {botName} 的 @mention]', 'daemon.cmd_needs_active_cli': '{cmd} 需要活跃的 CLI 进程，当前话题无运行中的会话。', + 'daemon.cmd_activation_pending': '{cmd} 暂不能发送：上一条消息仍在提交中，请稍后重试。', 'daemon.enriched_mentions_label': '消息中的 @mention：', 'daemon.choose_repo_first': '请先在上方卡片中选择仓库，您的消息已暂存，选择后会自动发送。', 'daemon.worktree_building_wait': '正在创建 worktree（含 git fetch，可能需要几秒），您的消息已暂存，创建完成后会自动一并发送。', diff --git a/src/im/lark/card-builder.ts b/src/im/lark/card-builder.ts index 40c37d487..80ed2adae 100644 --- a/src/im/lark/card-builder.ts +++ b/src/im/lark/card-builder.ts @@ -637,7 +637,7 @@ export function truncateContent(content: string, locale?: Locale, maxBytes: numb const PRIVATE_SNAPSHOT_TEXT_MAX = 50_000; const STREAM_TEMPLATE_MAP = { - starting: 'yellow', working: 'blue', idle: 'green', analyzing: 'purple', limited: 'red', retry_ready: 'green', + starting: 'yellow', working: 'blue', idle: 'green', analyzing: 'purple', stalled: 'red', limited: 'red', retry_ready: 'green', } as const; /** Header status label for a streaming/snapshot card. Shared by the live card @@ -648,6 +648,7 @@ function streamStatusLabel(status: StreamStatus, usageLimit: CliUsageLimitState case 'working': return t('card.status.working', undefined, locale); case 'idle': return t('card.status.idle', undefined, locale); case 'analyzing': return t('card.status.analyzing', undefined, locale); + case 'stalled': return t('card.status.stalled', undefined, locale); case 'limited': return usageLimit?.retryReady ? t('card.status.retry_ready', undefined, locale) : t('card.status.limited', undefined, locale); diff --git a/src/im/lark/card-handler.ts b/src/im/lark/card-handler.ts index 373b7d9e8..7ec839abe 100644 --- a/src/im/lark/card-handler.ts +++ b/src/im/lark/card-handler.ts @@ -58,13 +58,13 @@ import { ttadkConfigModelChoices } from '../../setup/cli-selection.js'; import { logger } from '../../utils/logger.js'; import * as sessionStore from '../../services/session-store.js'; import { loadFrozenCards, saveFrozenCards } from '../../services/frozen-card-store.js'; -import { forkWorker, sendWorkerInput, killWorker, scheduleCardPatch, parkStreamCard, clearUsageLimitState, cardUsageLimit, writableTerminalLinkFor, resolvePrivateCardAudience, deliverWriteLinkCard, deliverEphemeralOrReply, CARD_POSTING_SENTINEL } from '../../core/worker-pool.js'; +import { forkWorker, sendWorkerInput, killWorker, scheduleCardPatch, parkStreamCard, clearUsageLimitState, cardUsageLimit, writableTerminalLinkFor, resolvePrivateCardAudience, deliverWriteLinkCard, deliverEphemeralOrReply, CARD_POSTING_SENTINEL, closeSession as closeWorkerPoolSession, withActiveSessionKeyLock } from '../../core/worker-pool.js'; import { getSessionWorkingDir, buildNewTopicCliInput, getAvailableBots, persistStreamCardState, resumeSession, rememberLastCliInput, ensureSessionWhiteboard } from '../../core/session-manager.js'; import { publishAttentionPatch, announcePendingRepoSession } from '../../core/session-activity.js'; import { fallbackTurnId } from '../../core/reply-target.js'; import { validateWorkingDir } from '../../core/working-dir.js'; import type { DaemonToWorker, DisplayMode, TermActionKey } from '../../types.js'; -import { sessionKey, sessionAnchorId, frozenDisplayMode, markRepoCardConsumed, isRepoCardConsumed, isActiveRepoCard, claimCurrentRepoCard } from '../../core/types.js'; +import { activeSessionKey, sessionKey, sessionAnchorId, frozenDisplayMode, markRepoCardConsumed, isActiveRepoCard } from '../../core/types.js'; import type { DaemonSession } from '../../core/types.js'; import { buildTerminalUrl } from '../../core/terminal-url.js'; import type { ProjectInfo } from '../../services/project-scanner.js'; @@ -81,6 +81,9 @@ import { openLocalCliInIterm, preflightLocalCliOpen, } from '../../services/local-cli-opener.js'; +import { hasProtectedSessionMutationOwnership } from '../../core/session-mutation-guard.js'; +import { persistPendingRepoCardMessageId } from '../../core/pending-repo-journal.js'; +import { runDetachedBotTurnAdmission, withBotTurnAdmission, withBotTurnMutation } from '../../core/bot-turn-mutation-gate.js'; // ─── Types ──────────────────────────────────────────────────────────────── @@ -374,77 +377,80 @@ export async function commitRepoSelection( // The worktree flow already posted a precise "worktree 已创建：path 分支 …" // line before funnelling in here — suppress the redundant "已选择/已切换" // confirmation so the user sees a single message, not two. - // pinWorkingDir=false: "直接开始"/skip keeps the default cwd for this launch - // without persisting it (esp. $HOME) onto session.workingDir — sibling bots - // must still get their own repo card instead of inheriting HOME. - // confirmReplyText: optional confirmation to send under the same claim - // (used by skip_repo so the post-fork reply window cannot race a second click). opts?: { suppressConfirmReply?: boolean; confirmReplyText?: string; pinWorkingDir?: boolean; riffRepoDirs?: string[]; }, -): Promise { +): Promise { const { ds, rootId, cardMessageId, larkAppId, operatorOpenId, activeSessions, sessionReply } = ctx; const locTarget = localeForBot(ds.larkAppId); // `/close` deletes the active-map entry without touching sessionId or // pendingRepo — identity against the map is the only tell that the session // this flow captured is gone. Checked alongside the generation snapshots. - const repoSessionKey = larkAppId ? sessionKey(rootId, larkAppId) : rootId; + const repoSessionKey = activeSessionKey(ds); const sessionStillActive = () => activeSessions.get(repoSessionKey) === ds; const commitGenSessionId = ds.session.sessionId; const pinWorkingDir = opts?.pinWorkingDir !== false; + // Card callbacks are replayable. Only the currently published picker may + // mutate this session; a withdrawn/replaced card must stay stale even after + // the activation FIFO eventually settles. Internal auto-worktree commits do + // not originate from a card and deliberately omit cardMessageId. + if (cardMessageId !== undefined + && (!ds.repoCardMessageId || cardMessageId !== ds.repoCardMessageId)) { + logger.warn( + `[${tag(ds)}] Ignoring stale repo-card callback ${cardMessageId} ` + + `(current=${ds.repoCardMessageId ?? 'none'})`, + ); + return false; + } + + if (!ds.pendingRepo + && hasProtectedSessionMutationOwnership(ds)) { + await sessionReply( + rootId, + '当前会话仍有待提交消息，暂不能切换仓库；请等待提交完成或关闭会话。', + ); + return false; + } + if (ds.pendingRepo) { - // Card select/skip and auto-worktree converge here; the text /repo path - // mirrors this transaction through the same in-memory claim. Only one may - // own the pending -> worker transition, including prompt preparation. - if (ds.pendingRepoCommitInFlight) { - logger.info(`[${tag(ds)}] Pending repo commit already in flight — ignoring duplicate selection`); - return; - } + const targetSessionId = ds.session.sessionId; ds.pendingRepoCommitInFlight = true; - let committed = false; try { - // Claim before mutating the selected directory so two different card - // choices cannot overwrite each other while one prompt is being built. - // Skip/bare-start may intentionally leave workingDir unpinned so the - // launch uses the bot default without persisting $HOME for siblings. + const started = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current || current !== ds || !current.pendingRepo) return false; + // "Start directly" launches in the resolved default cwd without pinning + // HOME onto the session for sibling-bot inheritance. if (pinWorkingDir) { ds.workingDir = dirPath; ds.session.workingDir = dirPath; } + // riff 多仓 stamp：只有多仓 worktree 流显式传入（保留用户选择顺序，首仓=primary）； + // 其它选仓路径一律清除旧 stamp——workingDir 变了，旧的多仓组合不再成立。 ds.session.riffRepoDirs = opts?.riffRepoDirs; sessionStore.updateSession(ds.session); const selfBot = getBot(ds.larkAppId); const botCfg = selfBot.config; const effectiveCliId = sessionCliId(ds); - - // Keep pendingRepo=true across this await. New topic messages therefore - // remain buffered on this same session instead of racing a worker-null - // safety-net fork. Snapshot every pending field only after it settles. - const needsPromptContext = !ds.pendingRawInput || - (ds.pendingPrompt?.trim().length ?? 0) > 0 || - (ds.pendingAttachments?.length ?? 0) > 0 || - (ds.pendingFollowUps?.length ?? 0) > 0; - const availableBots = needsPromptContext - ? await getAvailableBots(ds.larkAppId, ds.chatId) - : []; - if (!sessionStillActive() || ds.session.sessionId !== commitGenSessionId || - !ds.pendingRepo || (ds.worker && !ds.worker.killed)) { - logger.warn(`[${tag(ds)}] Session changed while preparing the pending-CLI prompt (${commitGenSessionId} → ${ds.session.sessionId}, active=${sessionStillActive()}, pending=${!!ds.pendingRepo}, worker=${!!ds.worker}) — aborting this fork`); - return; - } - + // First-time repo selection — now spawn CLI with the original prompt. const pendingPrompt = ds.pendingPrompt ?? ''; const pendingRawInput = ds.pendingRawInput; + // Raw-input cold start still wraps any input buffered while the repo card + // was pending — see the skip_repo branch for the rationale. const hasBufferedInput = pendingPrompt.trim().length > 0 || + ds.pendingCodexAppText !== undefined || (ds.pendingAttachments?.length ?? 0) > 0 || (ds.pendingFollowUps?.length ?? 0) > 0; - if (!pendingRawInput || hasBufferedInput) ensureSessionWhiteboard(ds); - const wrappedInput = (!pendingRawInput || hasBufferedInput) + if (hasBufferedInput) ensureSessionWhiteboard(ds); + const wrappedInput = hasBufferedInput ? buildNewTopicCliInput( pendingPrompt, ds.session.sessionId, @@ -452,7 +458,7 @@ export async function commitRepoSelection( botCfg.cliPathOverride, ds.pendingAttachments, ds.pendingMentions, - availableBots, + await getAvailableBots(ds.larkAppId, ds.chatId), ds.pendingFollowUps, { name: selfBot.botName, openId: selfBot.botOpenId }, locTarget, @@ -469,44 +475,47 @@ export async function commitRepoSelection( codexAppFollowUpContexts: ds.pendingCodexAppFollowUpContexts, }, ) - : { content: '' }; - const prompt = pendingRawInput ? '' : wrappedInput; - const pendingTurnId = ds.pendingTurnId; - const previousPendingFollowUpInput = ds.pendingFollowUpInput; - if (pendingRawInput && hasBufferedInput) { + : undefined; + const prompt = pendingRawInput ? '' : (wrappedInput ?? ''); + // Last-line defence: prompt prep awaited above — if anything replaced + // OR closed the session in that window, forking now would clobber it + // (or resurrect a /close'd session). + if (!sessionStillActive() || ds.session.sessionId !== commitGenSessionId) { + logger.warn(`[${tag(ds)}] Session replaced or closed while preparing the pending-CLI prompt (${commitGenSessionId} → ${ds.session.sessionId}, active=${sessionStillActive()}) — aborting this fork`); + return false; + } + if (pendingRawInput && hasBufferedInput && wrappedInput) { ds.pendingFollowUpInput = { userPrompt: ds.pendingCodexAppText !== undefined || ds.pendingCodexAppFollowUps ? [ds.pendingCodexAppText ?? '', ...(ds.pendingCodexAppFollowUps ?? [])].filter(Boolean).join('\n\n') : pendingPrompt || ds.pendingFollowUps?.join('\n\n') || '', cliInput: wrappedInput.content, - ...(ds.pendingFollowUpTurnId ? { turnId: ds.pendingFollowUpTurnId } : {}), + ...((ds.pendingFollowUpTurnIds?.at(-1) ?? ds.pendingFollowUpTurnId) + ? { turnId: ds.pendingFollowUpTurnIds?.at(-1) ?? ds.pendingFollowUpTurnId } + : {}), ...(effectiveCliId === 'codex-app' && botCfg.codexAppCleanInput === true && wrappedInput.codexAppInput ? { codexAppInput: wrappedInput.codexAppInput } : {}), codexAppInputGateFrozen: true, }; } - - // From here through forkWorker there is no await: publish the state - // transition atomically from the router's point of view. + if (pendingRawInput) rememberLastCliInput(ds, pendingRawInput, pendingRawInput); + else if (hasBufferedInput && wrappedInput) rememberLastCliInput(ds, pendingPrompt, wrappedInput); + // Keep the reservation and every buffered opening field intact through + // forkWorker's synchronous pre-accept/write-ahead phase. If it throws, + // the user can retry this exact selection without losing the first turn. + const pendingTurnId = ds.pendingTurnId ?? ds.session.pendingRepoSetup?.turnId; + forkWorker( + ds, + prompt, + !pendingRawInput && pendingTurnId ? { turnId: pendingTurnId } : false, + ); ds.pendingRepo = false; + // A queued activation owns the route through its adapter-level ACK. Every + // buffer below was synchronously folded into prompt N and is safe to clear; + // later inbounds observe this gate and enter the separate exact staged FIFO. + ds.initialStartPending = ds.session.queuedActivationPending === true; publishAttentionPatch(ds); - try { - if (pendingRawInput) { - // pendingRawTurnId is applied at the literal PTY write boundary. - forkWorker(ds, '', false); - } else if (pendingTurnId && hasBufferedInput) { - forkWorker(ds, prompt, { turnId: pendingTurnId }); - } else { - forkWorker(ds, prompt); - } - } catch (e) { - ds.pendingRepo = true; - ds.pendingFollowUpInput = previousPendingFollowUpInput; - publishAttentionPatch(ds); - throw e; - } - rememberLastCliInput(ds, pendingRawInput ?? pendingPrompt, pendingRawInput ?? wrappedInput); ds.pendingPrompt = undefined; ds.pendingCodexAppText = undefined; ds.pendingCodexAppApplicationContext = undefined; @@ -516,56 +525,56 @@ export async function commitRepoSelection( ds.pendingSubstituteTrigger = undefined; ds.pendingSender = undefined; ds.pendingFollowUps = undefined; + ds.pendingFollowUpTurnId = undefined; + ds.pendingFollowUpTurnIds = undefined; ds.pendingCodexAppFollowUps = undefined; ds.pendingCodexAppFollowUpContexts = undefined; - ds.pendingFollowUpTurnId = undefined; + ds.pendingCodexAppFollowUpGateAccepted = undefined; ds.pendingTurnId = undefined; - committed = true; - - // Local one-shot consume BEFORE any network await. Feishu withdraw is - // best-effort and may lag/fail; correctness for stale card clicks depends - // on this mark so a second callback cannot mid-session-switch after claim - // release (and so a sessionReply throw that jumps to finally still leaves - // the card locally invalid). + return true; + }); + if (!started) return false; + // Invalidate synchronously at the successful commit boundary. Card + // withdrawal and confirmation are best effort and may await/fail; + // neither may leave a replayable mutation capability behind. const cardToWithdraw = cardMessageId ?? ds.repoCardMessageId; markRepoCardConsumed(ds, cardToWithdraw); ds.repoCardMessageId = undefined; - - // Hold the claim through confirmation. Card withdrawal is deferred until - // after the callback can ACK; stale clicks are blocked by the local - // consume mark above rather than by the network request's lifetime. + // Keep the pending-selection claim until the confirmation attempt + // settles. This prevents a second picker action from reinterpreting the + // freshly-started session as a mid-session repository switch. try { if (!opts?.suppressConfirmReply) { - // A card click has no turn of its own — anchor the confirmation to the - // session's current reply-target turn so a shared fold-back topic keeps - // it in-thread (same leak as the /repo command path). - await sessionReply(rootId, t('cmd.repo.selected_in_pending', { name: dirLabel }, locTarget), undefined, fallbackTurnId(ds, undefined)); + await sessionReply( + rootId, + t('cmd.repo.selected_in_pending', { name: dirLabel }, locTarget), + undefined, + fallbackTurnId(ds, undefined), + ); } else if (opts.confirmReplyText) { - await sessionReply(rootId, opts.confirmReplyText, undefined, fallbackTurnId(ds, undefined)); + await sessionReply( + rootId, + opts.confirmReplyText, + undefined, + fallbackTurnId(ds, undefined), + ); } - } catch (e) { - logger.warn(`[${tag(ds)}] Confirm reply after pending repo commit failed: ${e instanceof Error ? e.message : e}`); + } catch (err) { + logger.warn( + `[${tag(ds)}] Confirm reply after pending repo commit failed: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); } + // Withdrawal is deliberately detached so the card callback can ACK + // before Lark observes that the source message disappeared. deferRepoCardWithdraw(larkAppId, cardToWithdraw); + logger.info(`[${tag(ds)}] Repo selected: ${dirPath}, spawning CLI`); + return true; } finally { ds.pendingRepoCommitInFlight = false; } - if (!committed) return; - logger.info(`[${tag(ds)}] Repo selected: ${dirPath}, spawning CLI`); - return; } else { // Mid-session repo switch — close old session, start fresh. - // Claim the current card BEFORE killWorker / any await. A concurrent click - // on the same Feishu card must not pass a second kill+fork while the first - // switch is still awaiting confirm replies. Correctness does not depend on - // deleteMessage succeeding. - const claimedCard = claimCurrentRepoCard(ds, cardMessageId); - if (cardMessageId && !claimedCard) { - // Stale / already-claimed card (entry check races can still reach here). - logger.info(`[${tag(ds)}] Ignoring stale mid-session repo card ${cardMessageId}`); - return; - } - // Safety net (mirrors the `/repo` text-command path): build the same // "session closed" card `/close` emits BEFORE displacing the old session // (it reads the live session's identity off `ds`). The new session reuses @@ -577,73 +586,90 @@ export async function commitRepoSelection( // the displaced session's stored workingDir (and the closed card), so // `claude --resume` later would reopen the old context in the new repo's // cwd. The new repo is pinned onto the fresh session below instead. - const closedCard = buildClosedSessionCard(ds, locTarget); - - killWorker(ds); - // Park the current card in `frozenCards` so the next POST under the new - // session sweeps it via recall. closeSession() wipes the on-disk - // frozen-cards file under the OLD sessionId, but the in-memory Map - // travels with `ds` into the new session and still carries the - // old messageId for deletion. If fork or POST fails, the parked card - // stays in the thread instead of vanishing prematurely. - parkStreamCard(ds); - sessionStore.closeSession(ds.session.sessionId); - + const targetSessionId = ds.session.sessionId; + const switched = await withBotTurnMutation(ds.larkAppId, async () => { + const candidate = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId, + ); + if (!candidate || candidate !== ds || candidate.session.status !== 'active') { + return { ok: false as const, error: 'session_replaced' as const }; + } + const key = activeSessionKey(candidate); + return withActiveSessionKeyLock(activeSessions, key, async () => { + const current = [...activeSessions.values()].find( + owner => owner.session.sessionId === targetSessionId, + ); + if (!current || current !== candidate + || activeSessions.get(key) !== current + || current.session.status !== 'active') { + return { ok: false as const, error: 'session_replaced' as const }; + } + if (hasProtectedSessionMutationOwnership(current)) { + return { ok: false as const, error: 'dispatch_pending' as const }; + } + const closedCard = buildClosedSessionCard(current, locTarget); + // Preserve the old card in memory before durable close clears its old + // frozen-card file; it is re-keyed under the replacement session below. + parkStreamCard(current); + const oldSession = current.session; + await closeWorkerPoolSession(targetSessionId); + if (activeSessions.get(key) === current) activeSessions.delete(key); + if (activeSessions.has(key)) { + return { ok: false as const, error: 'session_replaced' as const }; + } + const cardToWithdraw = cardMessageId ?? current.repoCardMessageId; + markRepoCardConsumed(current, cardToWithdraw); + current.repoCardMessageId = undefined; + + const session = sessionStore.createSession( + current.chatId, + current.scope === 'chat' ? oldSession.rootMessageId : rootId, + dirLabel, + current.chatType, + current.scope, + ); + current.session = session; + current.lastUserPrompt = undefined; + current.lastCliInput = undefined; + current.workingDir = dirPath; + session.workingDir = dirPath; + session.larkAppId = current.larkAppId; + session.chatDisplayName = oldSession.chatDisplayName; + session.ownerOpenId = oldSession.ownerOpenId; + session.creatorOpenId = oldSession.creatorOpenId; + session.lastCallerOpenId = oldSession.lastCallerOpenId; + session.riffRepoDirs = opts?.riffRepoDirs; + sessionStore.updateSession(session); + current.hasHistory = false; + if (current.frozenCards && current.frozenCards.size > 0) { + saveFrozenCards(session.sessionId, current.frozenCards); + } + current.streamCardId = undefined; + current.streamCardNonce = undefined; + current.streamCardPending = undefined; + current.lastScreenContent = undefined; + current.lastScreenStatus = undefined; + activeSessions.set(key, current); + forkWorker(current, '', false); + return { ok: true as const, current, closedCard, cardToWithdraw }; + }); + }); + if (!switched.ok) { + if (switched.error === 'dispatch_pending') { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能切换仓库；请等待本轮完成或关闭会话。', + ); + } + return false; + } await deliverEphemeralOrReply( - ds, + switched.current, operatorOpenId, - closedCard, + switched.closedCard, 'interactive', - () => sessionReply(rootId, closedCard, 'interactive'), - ); - - const oldSession = ds.session; - // `rootId` is the routing anchor. For chat-scope sessions it is the - // `oc_...` chat id, not the traceable `om_...` message root stored on - // Session. Preserve the old identity and explicitly persist scope so card - // switches cannot recreate the session as a legacy scope-less thread. - const session = sessionStore.createSession( - ds.chatId, - ds.scope === 'chat' ? oldSession.rootMessageId : rootId, - dirLabel, - ds.chatType, - ds.scope, + () => sessionReply(rootId, switched.closedCard, 'interactive'), ); - ds.session = session; - ds.lastUserPrompt = undefined; - ds.lastCliInput = undefined; - // Pin workingDir + larkAppId onto the new session before forkWorker. - // Without this, a daemon restart restores the session with an empty - // workingDir and the worker spawns in the bot's default cwd, so - // `claude --resume` looks in the wrong .claude/projects// dir and - // exits code 0 immediately, crash-looping until the rate-limiter trips. - ds.workingDir = dirPath; - ds.session.workingDir = dirPath; - ds.session.larkAppId = ds.larkAppId; - ds.session.chatDisplayName = oldSession.chatDisplayName; - ds.session.ownerOpenId = oldSession.ownerOpenId; - ds.session.creatorOpenId = oldSession.creatorOpenId; - ds.session.lastCallerOpenId = oldSession.lastCallerOpenId; - // Stamp the newly-created session, not the displaced session that was just - // closed. Plain/single-repo switches pass undefined and clear stale state. - ds.session.riffRepoDirs = opts?.riffRepoDirs; - sessionStore.updateSession(ds.session); - ds.hasHistory = false; - // Re-persist the parked card under the NEW sessionId so a daemon crash - // before the next POST doesn't strand it. closeSession() above wiped - // the on-disk file under the OLD sessionId; without this re-save, the - // in-memory Map only survives in process memory. - if (ds.frozenCards && ds.frozenCards.size > 0) { - saveFrozenCards(ds.session.sessionId, ds.frozenCards); - } - // Drop the old turn's streaming-card reference so worker_ready POSTs a - // fresh card for the new session instead of PATCHing the previous one. - ds.streamCardId = undefined; - ds.streamCardNonce = undefined; - ds.streamCardPending = undefined; - ds.lastScreenContent = undefined; - ds.lastScreenStatus = undefined; - forkWorker(ds, '', false); if (!opts?.suppressConfirmReply) { try { await sessionReply(rootId, t('cmd.repo.switched_to', { name: dirLabel }, locTarget)); @@ -651,11 +677,14 @@ export async function commitRepoSelection( logger.warn(`[${tag(ds)}] Confirm reply after mid-session repo switch failed: ${e instanceof Error ? e.message : e}`); } } - logger.info(`[${tag(ds)}] Repo switched to ${dirPath}, new session created`); - // The card was claimed locally above; withdraw it only after this callback - // can return its ACK so the delete request cannot race the callback frame. - deferRepoCardWithdraw(larkAppId, claimedCard); + // Return the callback ACK before recalling the card. Lark can keep the + // delete request pending; awaiting it here stalls the card action and lets + // the client retry an otherwise successful repository switch. + deferRepoCardWithdraw(larkAppId, switched.cardToWithdraw); + logger.info(`[${tag(switched.current)}] Repo switched to ${dirPath}, new session created`); } + + return true; } /** @@ -700,7 +729,11 @@ export async function runAutoWorktreeCommit(deps: { // pendingRepo session, folds any messages buffered during creation (pendingPrompt // + pendingFollowUps) into the first turn. suppressConfirmReply: the worktree // helper already posted the '已创建/回退' line, so skip the '已选择' confirmation. - await commitRepoSelection( + // The worktree build is intentionally detached from its caller's inbound + // admission. Re-enter with a fresh lease at the delayed commit/fork edge; + // the outer lease may have ended minutes ago and must not authorize this + // descendant across a bot-wide config mutation. + await runDetachedBotTurnAdmission(larkAppId, () => commitRepoSelection( { ds, rootId: anchor, larkAppId, operatorOpenId, activeSessions, // Never reached under suppressConfirmReply for a pendingRepo session. @@ -709,7 +742,7 @@ export async function runAutoWorktreeCommit(deps: { wt.dir, pathBasename(wt.dir), { suppressConfirmReply: true }, - ); + )); } catch (e) { // No recovery fork here: forking with an empty prompt would DROP the buffered // first turn (pendingPrompt lives only in-memory, not the message queue). Leave @@ -1586,8 +1619,10 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe logger.info(`[${tag(ds)}] voice_summary blocked because worker is busy: ${ds.lastScreenStatus ?? 'unknown'}`); return { toast: { type: 'warning', content: t('card.voice.toast_worker_busy', undefined, locDs) } }; } - voicedCardIds.add(dedupeKey); - if (voicedCardIds.size > 5000) { voicedCardIds.clear(); voicedCardIds.add(dedupeKey); } + if ((!ds.worker || ds.worker.killed) && hasProtectedSessionMutationOwnership(ds)) { + logger.info(`[${tag(ds)}] voice_summary deferred behind durable opening ownership`); + return { toast: { type: 'warning', content: t('card.voice.toast_worker_busy', undefined, locDs) } }; + } const instruction = voiceSummaryInstruction(locDs); const voiceInput = { content: instruction, @@ -1598,8 +1633,26 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe 'application', ), }; - if (ds.worker && !ds.worker.killed) sendWorkerInput(ds, voiceInput); - else forkWorker(ds, voiceInput, ds.hasHistory); + let accepted = false; + try { + if (ds.worker && !ds.worker.killed) accepted = sendWorkerInput(ds, voiceInput); + else { + forkWorker(ds, voiceInput, ds.hasHistory); + accepted = true; + } + } catch (err) { + logger.warn( + `[${tag(ds)}] voice_summary failed before acceptance: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + if (!accepted) { + return { toast: { type: 'warning', content: t('card.voice.toast_worker_busy', undefined, locDs) } }; + } + // Burn the replay key only after live IPC or the durable activation tail + // accepted the exact instruction. + voicedCardIds.add(dedupeKey); + if (voicedCardIds.size > 5000) { voicedCardIds.clear(); voicedCardIds.add(dedupeKey); } logger.info(`[${tag(ds)}] voice_summary triggered by ${operatorOpenId ?? '?'}`); return { toast: { type: 'success', content: t('card.voice.toast_wait', undefined, locDs) } }; } @@ -1610,26 +1663,54 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe // which violates the bridge invariant. Defense in depth — buildSessionCard // already omits the restart button when adoptMode=true, but a stale // pre-fix card or a malformed action payload could still arrive. - const locDs = localeForBot(ds.larkAppId); - if (ds.adoptedFrom) { - logger.warn(`[${tag(ds)}] Rejected restart on adopt session — would kill user's pane`); + const targetSessionId = ds.session.sessionId; + const targetAppId = ds.larkAppId; + const locDs = localeForBot(targetAppId); + const restart = await withBotTurnMutation(targetAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId + && candidate.session.status === 'active', + ); + if (!current) return { status: 'gone' as const }; + if (current.adoptedFrom || current.initConfig?.adoptMode) { + logger.warn(`[${tag(current)}] Rejected restart on adopt session — would kill user's pane`); + return { status: 'adopted' as const }; + } + if (hasProtectedSessionMutationOwnership(current)) { + return { status: 'pending' as const }; + } + const effectiveCliId = sessionCliId(current); + if (current.worker && !current.worker.killed) { + logger.info(`[${tag(current)}] Restart via card button`); + current.worker.send({ type: 'restart', reason: 'operator' } as DaemonToWorker); + return { status: 'restarted' as const, current, effectiveCliId }; + } + logger.info(`[${tag(current)}] Re-forking worker via card button`); + forkWorker(current, '', current.hasHistory); + return { status: 'reforked' as const, current, effectiveCliId }; + }); + if (restart.status === 'gone') { + return { toast: { type: 'warning', content: t('card.action.session_gone', undefined, locDs) } }; + } + if (restart.status === 'adopted') { await sessionReply(rootId, t('card.action.adopt_no_restart', undefined, locDs)); return; } - const botCfg = getBot(ds.larkAppId).config; - const effectiveCliId = sessionCliId(ds); - if (ds.worker) { - logger.info(`[${tag(ds)}] Restart via card button`); - ds.worker.send({ type: 'restart' } as DaemonToWorker); - const cliName = getCliDisplayName(effectiveCliId); + if (restart.status === 'pending') { + await sessionReply( + rootId, + '当前 Codex App 仍有未结算消息，暂不能重启；请等待本轮完成或关闭会话。', + ); + return; + } + if (restart.status === 'restarted') { + const cliName = getCliDisplayName(restart.effectiveCliId); const restartedMsg = t('card.action.restarted', { cliName }, locDs); - await deliverEphemeralOrReply(ds, operatorOpenId, restartedMsg, 'text', () => sessionReply(rootId, restartedMsg)); + await deliverEphemeralOrReply(restart.current, operatorOpenId, restartedMsg, 'text', () => sessionReply(rootId, restartedMsg)); } else { - logger.info(`[${tag(ds)}] Re-forking worker via card button`); - forkWorker(ds, '', ds.hasHistory); - const cliName = getCliDisplayName(effectiveCliId); + const cliName = getCliDisplayName(restart.effectiveCliId); const restartedFreshMsg = t('card.action.restarted_fresh', { cliName }, locDs); - await deliverEphemeralOrReply(ds, operatorOpenId, restartedFreshMsg, 'text', () => sessionReply(rootId, restartedFreshMsg)); + await deliverEphemeralOrReply(restart.current, operatorOpenId, restartedFreshMsg, 'text', () => sessionReply(rootId, restartedFreshMsg)); // DM card will be sent by the ready handler when worker starts } } @@ -1640,13 +1721,25 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe // 会话」却静默无反应会让人以为按钮坏了，给一条失败 toast（成功路径不弹，已关卡即反馈）。 return { toast: { type: 'warning', content: t('card.action.session_gone', undefined, localeForBot(larkAppId)) } }; } - const botCfg = getBot(ds.larkAppId).config; - // Build the closed card BEFORE killWorker/closeSession — it reads the - // live session's identity off `ds`. - const card = buildClosedSessionCard(ds, localeForBot(ds.larkAppId)); - killWorker(ds); - sessionStore.closeSession(ds.session.sessionId); - activeSessions.delete(sKey); + const targetSessionId = ds.session.sessionId; + const closed = await withBotTurnMutation(ds.larkAppId, async () => { + // Card payload roots survive transfers. Re-resolve by immutable session + // id after draining admissions, then let the pool remove the target's + // CURRENT activeSessionKey. Never delete the stale payload root, which + // may now belong to an unrelated pending FIFO owner. + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId, + ); + if (!current) return undefined; + const botCfg = getBot(current.larkAppId).config; + const card = buildClosedSessionCard(current, localeForBot(current.larkAppId)); + await closeWorkerPoolSession(targetSessionId); + return { status: 'closed' as const, current, botCfg, card }; + }); + if (!closed) { + return { toast: { type: 'warning', content: t('card.action.session_gone', undefined, localeForBot(larkAppId)) } }; + } + const { current, botCfg, card } = closed; // The closed card carries session title / CLI name / workingDir / resume // command. In private-card mode those must not leak to the group — send the // closed card ephemeral to the same owner audience instead. No group @@ -1655,15 +1748,15 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe // clicked, so a card built in private mode stays ephemeral even if the // bot's `privateCard` config was turned off in the meantime. if (value?.visibility === 'private' || botCfg.privateCard) { - const audience = resolvePrivateCardAudience(ds); + const audience = resolvePrivateCardAudience(current); for (const openId of audience) { - await sendEphemeralCard(ds.larkAppId, ds.chatId, openId, card).catch(err => - logger.warn(`[${tag(ds)}] private close card ephemeral send to ${openId.substring(0, 8)}… failed: ${err}`)); + await sendEphemeralCard(current.larkAppId, current.chatId, openId, card).catch(err => + logger.warn(`[${tag(current)}] private close card ephemeral send to ${openId.substring(0, 8)}… failed: ${err}`)); } - logger.info(`[${tag(ds)}] Closed via card button (private close card → ${audience.length} owner(s))`); + logger.info(`[${tag(current)}] Closed via card button (private close card → ${audience.length} owner(s))`); } else { - await deliverEphemeralOrReply(ds, operatorOpenId, card, 'interactive', () => sessionReply(rootId, card, 'interactive')); - logger.info(`[${tag(ds)}] Closed via card button`); + await deliverEphemeralOrReply(current, operatorOpenId, card, 'interactive', () => sessionReply(rootId, card, 'interactive')); + logger.info(`[${tag(current)}] Closed via card button`); } } @@ -1697,9 +1790,18 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe } if (actionType === 'disconnect' && ds) { - killWorker(ds); - sessionStore.closeSession(ds.session.sessionId); - activeSessions.delete(sKey); + const targetSessionId = ds.session.sessionId; + const disconnected = await withBotTurnMutation(ds.larkAppId, async () => { + const current = [...activeSessions.values()].find( + candidate => candidate.session.sessionId === targetSessionId, + ); + if (!current) return undefined; + await closeWorkerPoolSession(targetSessionId); + return current; + }); + if (!disconnected) { + return { toast: { type: 'warning', content: t('card.action.session_gone', undefined, localeForBot(larkAppId)) } }; + } await sessionReply(rootId, t('card.action.disconnected', undefined, localeForBot(ds.larkAppId))); logger.info(`[${tag(ds)}] Disconnected (adopt) via card button`); } @@ -1725,6 +1827,36 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe return; } + const retryCodexAppInput = ds.lastCodexAppInput + ? (({ clientUserMessageId: _priorMessageId, ...input }) => input)(ds.lastCodexAppInput) + : undefined; + const retryInput = { + content: cliInput, + ...(retryCodexAppInput ? { codexAppInput: retryCodexAppInput } : {}), + }; + if ((!ds.worker || ds.worker.killed) && hasProtectedSessionMutationOwnership(ds)) { + await sessionReply(rootId, t('card.action.retry_last_task_unavailable', undefined, locDs)); + return; + } + let accepted = false; + try { + if (ds.worker && !ds.worker.killed) accepted = sendWorkerInput(ds, retryInput); + else { + forkWorker(ds, retryInput, ds.hasHistory); + accepted = true; + } + } catch (err) { + logger.warn( + `[${tag(ds)}] retry_last_task failed before acceptance: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + if (!accepted) { + await sessionReply(rootId, t('card.action.retry_last_task_unavailable', undefined, locDs)); + return; + } + // Only now consume the one-shot retry state and advertise working: live + // IPC or the durable activation tail already owns the exact retry input. clearUsageLimitState(ds); ds.lastScreenStatus = 'working'; ds.streamCardPending = true; @@ -1755,16 +1887,6 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe ); scheduleCardPatch(ds, cardJson); } - - const retryCodexAppInput = ds.lastCodexAppInput - ? (({ clientUserMessageId: _priorMessageId, ...input }) => input)(ds.lastCodexAppInput) - : undefined; - const retryInput = { - content: cliInput, - ...(retryCodexAppInput ? { codexAppInput: retryCodexAppInput } : {}), - }; - if (ds.worker && !ds.worker.killed) sendWorkerInput(ds, retryInput); - else forkWorker(ds, retryInput, ds.hasHistory); logger.info(`[${tag(ds)}] Retrying last task after usage limit`); if (cardJson) { try { return JSON.parse(cardJson); } catch { /* fall through */ } @@ -2288,26 +2410,29 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe } if (ds.pendingRepo) { if (ds.worktreeCreating || ds.pendingRepoCommitInFlight) { - return { toast: { type: 'info', content: t('cmd.repo.worktree_in_progress', undefined, locDs) } }; + return { + toast: { + type: 'info', + content: t('cmd.repo.worktree_in_progress', undefined, locDs), + }, + }; } const cwd = getSessionWorkingDir(ds); - // Reuse the same claimed pending->worker transition as a normal repo - // selection. This keeps buffering active across prompt preparation and - // makes skip/select races single-winner. Do NOT pin the resolved default - // cwd (often $HOME) onto session.workingDir — that would let sibling - // bots inherit HOME instead of getting their own repo card. Confirmation - // + card withdraw run under the claim inside commitRepoSelection. - await commitRepoSelection( - { ds, rootId, cardMessageId, larkAppId, operatorOpenId, activeSessions, sessionReply }, + // "Start directly" is the same pending-start commit as choosing a + // directory, just pinned to the current/default cwd. Reusing the shared + // helper gives this card path the bot mutation, exact-owner recheck, + // buffered-sidecar handling, and fork-before-release ordering. + const started = await commitRepoSelection( + { ds, rootId, cardMessageId, larkAppId: larkAppId ?? ds.larkAppId, operatorOpenId, activeSessions, sessionReply }, + cwd, cwd, - pathBasename(cwd) || cwd, { suppressConfirmReply: true, confirmReplyText: t('cmd.skip.opened', { cwd }, locDs), pinWorkingDir: false, }, ); - if (!ds.pendingRepo) { + if (started) { logger.info(`[${tag(ds)}] Skip repo, spawning CLI in ${cwd}`); } } else { @@ -2354,10 +2479,12 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe // re-send a fresh repo card in the new mode — a form can't ride an // in-place patch, so the old card is withdrawn and a new one posted. const locDs = localeForBot(ds.larkAppId); - // Same active-card gate as skip/manual/worktree: a stale card must not - // flip bot config or replace the live repo card after claim/restart. - if (cardMessageId && !isActiveRepoCard(ds, cardMessageId)) { - return { toast: { type: 'info', content: t('cmd.repo.card_already_consumed', undefined, locDs) } }; + if (!cardMessageId || !ds.repoCardMessageId || cardMessageId !== ds.repoCardMessageId) { + logger.warn( + `[${tag(ds)}] Ignoring stale worktree-toggle picker ${cardMessageId ?? 'none'} ` + + `(current=${ds.repoCardMessageId ?? 'none'})`, + ); + return { toast: { type: 'warning', content: t('card.repo.toast_stale_picker', undefined, locDs) } }; } const spec = findConfigField('worktreeMultiPicker'); if (!spec) return; @@ -2365,18 +2492,47 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe const r = await applyConfigField(ds.larkAppId, spec, next); if (!r.ok) return { toast: { type: 'error', content: t('cmd.config.write_failed', { reason: r.reason }, locDs) } }; const projects = lastRepoScan.get(ds.chatId) ?? []; - // await so a rejected delete is caught here (not an unhandled rejection); - // a missing/already-gone card is fine — we post the fresh one regardless. - if (ds.repoCardMessageId && ds.larkAppId) { try { await deleteMessage(ds.larkAppId, ds.repoCardMessageId); } catch { /* card already gone */ } } const newCard = buildRepoSelectCard(projects, getSessionWorkingDir(ds), rootId, locDs, next); - ds.repoCardMessageId = await sessionReply(rootId, newCard, 'interactive'); + const oldCardMessageId = ds.repoCardMessageId; + let newCardMessageId: string; + try { + newCardMessageId = await sessionReply(rootId, newCard, 'interactive'); + } catch (err) { + logger.warn( + `[${tag(ds)}] Failed to publish replacement repo picker; old card remains authoritative: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return { toast: { type: 'error', content: t('cmd.config.write_failed', { reason: err instanceof Error ? err.message : String(err) }, locDs) } }; + } + try { + // Switch durable authority before changing runtime identity or + // withdrawing the old card. A failed write leaves the old picker fully + // usable and makes the newly-published card stale by exact-ID guard. + persistPendingRepoCardMessageId(ds, newCardMessageId); + } catch (err) { + logger.error( + `[${tag(ds)}] Failed to persist replacement repo picker ${newCardMessageId}; ` + + `old picker ${oldCardMessageId} retained: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return { toast: { type: 'error', content: t('cmd.config.write_failed', { reason: err instanceof Error ? err.message : String(err) }, locDs) } }; + } + ds.repoCardMessageId = newCardMessageId; + // The fresh ID is now durable. Withdrawal is best-effort and cannot + // invalidate the new authority if Lark reports the old card missing. + try { await deleteMessage(ds.larkAppId, oldCardMessageId); } + catch { /* card already gone */ } return { toast: { type: 'info', content: t(next ? 'card.repo.toast_worktree_mode_switched' : 'card.repo.toast_worktree_mode_switched_back', undefined, locDs) } }; } if (actionType === 'repo_worktree_submit' && ds) { const locDs = localeForBot(ds.larkAppId); - if (cardMessageId && !isActiveRepoCard(ds, cardMessageId)) { - return { toast: { type: 'info', content: t('cmd.repo.card_already_consumed', undefined, locDs) } }; + if (!cardMessageId || !ds.repoCardMessageId || cardMessageId !== ds.repoCardMessageId) { + logger.warn( + `[${tag(ds)}] Ignoring stale worktree-submit picker ${cardMessageId ?? 'none'} ` + + `(current=${ds.repoCardMessageId ?? 'none'})`, + ); + return { toast: { type: 'warning', content: t('card.repo.toast_stale_picker', undefined, locDs) } }; } const selectedPaths = stringListFromLarkMultiSelect(action?.form_value?.repo_worktree_paths); if (selectedPaths.length === 0) { @@ -2604,11 +2760,22 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe return; } - // Only the live posted card may drive selection. Covers: already-consumed, - // wrong/old card, and post-restart (repoCardMessageId is in-memory). Must - // reject before mid-session switch can killWorker. - if (cardMessageId && !isActiveRepoCard(targetDs, cardMessageId)) { - return { toast: { type: 'info', content: t('cmd.repo.card_already_consumed', undefined, localeForBot(targetDs.larkAppId)) } }; + // The picker message id is the capability for every repo dropdown. Check it + // before slug generation, worktreeCreating, git creation/push, or commit. + // In particular the direct single-select `repo_worktree` callback does not + // pass through the form-submit branch above, so relying on the later commit + // check would allow a stale card to create an orphan worktree first. + if (!cardMessageId || !targetDs.repoCardMessageId || cardMessageId !== targetDs.repoCardMessageId) { + logger.warn( + `[${tag(targetDs)}] Ignoring stale ${repoKey} picker ${cardMessageId ?? 'none'} ` + + `(current=${targetDs.repoCardMessageId ?? 'none'})`, + ); + return { + toast: { + type: 'warning', + content: t('card.repo.toast_stale_picker', undefined, localeForBot(targetDs.larkAppId)), + }, + }; } // 权限边界：pendingRepo（首次选 repo 才能 spawn）放行「会话发起人 或 canOperate」， diff --git a/src/im/lark/event-dispatcher.ts b/src/im/lark/event-dispatcher.ts index 93a08a4b2..ef1043c75 100644 --- a/src/im/lark/event-dispatcher.ts +++ b/src/im/lark/event-dispatcher.ts @@ -600,6 +600,28 @@ function eventIdForKey(data: any): string | undefined { return data?.event_id ?? data?.uuid ?? data?.header?.event_id ?? data?.event?.event_id; } +/** + * Synchronous first-stage routing lane for inbound Lark messages. + * + * Routing can await bot identity, chat mode, oncall binding, summary expansion, + * and VC catch-up before it discovers the canonical daemon anchor. Reserving + * only at that later anchor lets a faster N+1 reach the session before N. + * + * This barrier must be chat-wide, not thread-wide: a topic seed has no thread_id + * and initially looks chat-shaped, while its first reply carries thread_id; both + * later canonicalize to the seed message_id. Shared-topic aliases likewise fold + * a thread-shaped event back into a chat owner. Per-thread raw lanes therefore + * cannot prove arrival order. The barrier is released as soon as canonical work + * is synchronously enqueued below, so handlers for distinct canonical topics can + * still run concurrently. + */ +export function rawMessageIngressAnchor(larkAppId: string, message: any): string { + const chatId = typeof message?.chat_id === 'string' && message.chat_id.trim() + ? message.chat_id.trim() + : '__chatless__'; + return `lark-message-routing:${larkAppId}:${chatId}`; +} + // card.action.trigger is a synchronous callback with a 3s deadline and NO // re-push (unlike events). If a handler (e.g. restart, which spawns a worker) // might exceed the budget, we ACK before 3s with a toast and patch the card @@ -2084,14 +2106,13 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin const sessionsReady = new Promise(resolve => { resolveSessionsReady = resolve; }); if (sessionsReadyApps.has(larkAppId)) resolveSessionsReady(); sessionsReadyBarriers.set(larkAppId, { promise: sessionsReady, resolve: resolveSessionsReady }); - const dispatchHumanMessage = async (payload: PendingForwardTopicPayload): Promise => { - await serializeByAnchor(payload.ctx.anchor, () => { + const dispatchHumanMessage = (payload: PendingForwardTopicPayload): Promise => + serializeByAnchor(payload.ctx.anchor, () => { const ownsSession = handlers.isSessionOwner?.(payload.ctx.anchor, larkAppId) ?? payload.ownsSession; return ownsSession ? handlers.handleThreadReply(payload.data, payload.ctx) : handlers.handleNewTopic(payload.data, payload.ctx); - }); - }; + }, 0); const dispatchPersistedForwardFollowup = async ( seedMessageId: string, payload: PendingForwardTopicPayload, @@ -2131,6 +2152,26 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin } } const seedRoutingGates = new Map; complete: () => void }>(); + const waitForSeedRoutingGate = async (messageId: string): Promise => { + const gate = seedRoutingGates.get(messageId); + if (!gate) return; + const timeoutMs = Math.max(1, config.daemon.forwardFollowupWaitMs); + let timer: NodeJS.Timeout | undefined; + const outcome = await Promise.race([ + gate.ready.then(() => 'ready' as const), + new Promise<'timeout'>(resolve => { + timer = setTimeout(() => resolve('timeout'), timeoutMs); + timer.unref?.(); + }), + ]); + if (timer) clearTimeout(timer); + if (outcome === 'timeout') { + logger.warn( + `[forward-followup] seed routing gate timed out after ${timeoutMs}ms ` + + `for root=${messageId.substring(0, 12)}; continuing without pairing`, + ); + } + }; const registerSeedRoutingGate = (messageId: string) => { let resolveReady!: () => void; const ready = new Promise(resolve => { resolveReady = resolve; }); @@ -2217,8 +2258,8 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin // Serialize per anchor so back-to-back messages to the same thread // (e.g. dispatch's /repo prime + brief kickoff) don't interleave with // the first's async session-spawn. See anchor-serializer.ts. - await serializeByAnchor(ctx.anchor, () => - handlers.handleThreadReply(data, { ...ctx, chatId, messageId, chatType, larkAppId })) + void serializeByAnchor(ctx.anchor, () => + handlers.handleThreadReply(data, { ...ctx, chatId, messageId, chatType, larkAppId }), 0) .catch(err => logger.error(`Error handling message event: ${err}`)); return; } @@ -2361,8 +2402,8 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin logger.info(`Bot-to-bot @mention detected (scope=${ctx.scope}): routing to handleThreadReply`); // Serialize per anchor — a sub-bot dispatched a /repo prime + kickoff // back-to-back into this thread must be handled in order, not raced. - await serializeByAnchor(ctx.anchor, () => - handlers.handleThreadReply(data, { ...ctx, chatId, messageId, chatType, larkAppId, replyRootId })) + void serializeByAnchor(ctx.anchor, () => + handlers.handleThreadReply(data, { ...ctx, chatId, messageId, chatType, larkAppId, replyRootId }), 0) .catch(err => logger.error(`Error handling bot @mention: ${err}`)); return; } @@ -2634,6 +2675,7 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin : ''; const isControlCommand = strippedRoutingText.startsWith('/'); let pairedForwardSeed; + let stalePendingSeed; // Require isAllowed before pairing: a root-linked clarification from a // sender who was /revoked within the grace window must not consume the // seed from the buffer or overwrite the durable paired record. The seed @@ -2641,7 +2683,7 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin // only arise under never/ambient modes, where a legitimate merge always // requires isAllowed anyway. if (senderOpenId && isAllowed && message.root_id && !isControlCommand) { - await seedRoutingGates.get(message.root_id)?.ready; + await waitForSeedRoutingGate(message.root_id); const pairingInput = { larkAppId, chatId, @@ -2656,17 +2698,7 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin if (pairingDelayEnabled && !ambientRedirect) { pairedForwardSeed = forwardFollowups.take(pairingInput); } else if (!pairingDelayEnabled) { - const stalePendingSeed = forwardFollowups.take(pairingInput); - if (stalePendingSeed) { - try { - await dispatchPersistedForwardFollowup(stalePendingSeed.messageId, stalePendingSeed.payload); - } catch (err) { - logger.warn( - `[forward-followup] failed to flush stale seed=${stalePendingSeed.messageId.substring(0, 12)}; ` + - `continuing current msg=${messageId.substring(0, 12)}: ${err}`, - ); - } - } + stalePendingSeed = forwardFollowups.take(pairingInput); } } if (pairedForwardSeed) { @@ -2870,16 +2902,40 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin } catch (err) { logger.warn(`[forward-followup] failed to persist paired payload before dispatch: ${err}`); } - await dispatchPersistedForwardFollowup(pairedForwardSeed.messageId, payload) + void dispatchPersistedForwardFollowup(pairedForwardSeed.messageId, payload) .catch(err => logger.error(`Error handling paired message event: ${err}`)); return; } + if (stalePendingSeed) { + // The mention mode changed while a seed was held. Release the raw chat + // ingress lane immediately, but enqueue the stale seed before the + // current reply once session restoration is complete. Both calls append + // synchronously to the same canonical FIFO; their handler completion is + // deliberately not awaited by the raw lane. + void sessionsReady.then(() => { + void dispatchHumanMessage(stalePendingSeed.payload) + .then(() => removeForwardFollowup(larkAppId, stalePendingSeed.messageId)) + .catch(err => logger.warn( + `[forward-followup] failed to flush stale seed=${stalePendingSeed.messageId.substring(0, 12)}; ` + + `continuing current msg=${messageId.substring(0, 12)}: ${err}`, + )); + void dispatchHumanMessage(payload) + .catch(err => logger.error(`Error handling message event: ${err}`)); + }).catch(err => logger.error(`Error awaiting restored sessions for stale seed: ${err}`)); + return; + } + // Serialize per anchor so two messages to the same thread/chat are // processed in arrival order — never concurrently. Without this a fast // second message interleaves with the first's async session-spawn and is // dropped (worker-not-ready → re-fork branch). See anchor-serializer.ts. - await dispatchHumanMessage(payload) + // The chat-wide ingress lane protects only asynchronous routing. Once + // canonical work has been synchronously appended to its own anchor FIFO, + // release the raw lane so independent topics in the same chat can run + // concurrently. Same-anchor handlers remain strictly serialized by + // dispatchHumanMessage's canonical queue. + void dispatchHumanMessage(payload) .catch(err => logger.error(`Error handling message event: ${err}`)); } catch (err) { logger.error(`Error handling message event: ${err}`); @@ -2947,9 +3003,21 @@ export function startLarkEventDispatcher(larkAppId: string, larkAppSecret: strin const claim = messageIdForKey ? () => claimMessageOnce(larkAppId, messageIdForKey) : () => claimEventOnce(eventKey); - let seedRoutingGate: { ready: Promise; complete: () => void } | undefined; - const scheduled = scheduleAckSafeEvent(eventKey, () => processMessageEvent(data, seedRoutingGate), 'message event', claim); const rawMessage = data?.message; + const ingressAnchor = rawMessageIngressAnchor(larkAppId, rawMessage); + let seedRoutingGate: { ready: Promise; complete: () => void } | undefined; + // Reserve the app/chat ingress lane before any routing await. Canonical + // per-anchor serialization inside processMessageEvent remains the final + // execution fence after aliases and chat mode are resolved. + const scheduled = scheduleAckSafeEvent( + eventKey, + () => serializeByAnchor( + ingressAnchor, + () => processMessageEvent(data, seedRoutingGate), + ), + 'message event', + claim, + ); const rawSenderType = data?.sender?.sender_type; if ( scheduled diff --git a/src/im/lark/overview-card.ts b/src/im/lark/overview-card.ts index bc9c8d5cf..22c6f3132 100644 --- a/src/im/lark/overview-card.ts +++ b/src/im/lark/overview-card.ts @@ -31,10 +31,11 @@ export const OVERVIEW_ACTION_GOTO_SCHEDULES = 'dash_overview_goto_schedules' as export const OVERVIEW_ACTION_GOTO_SETTINGS = 'dash_overview_goto_settings' as const; export const OVERVIEW_ACTION_GOTO_GROUPS = 'dash_overview_goto_groups' as const; -/** Status set treated as "active" (working / analyzing / starting / limited). */ +/** Status set treated as active rather than falsely counted as idle. */ const ACTIVE_STATUSES: ReadonlySet = new Set([ 'working', 'analyzing', + 'stalled', 'starting', 'limited', ]); diff --git a/src/services/session-store.ts b/src/services/session-store.ts index e70c649b8..e5914daca 100644 --- a/src/services/session-store.ts +++ b/src/services/session-store.ts @@ -3,6 +3,7 @@ import { join, dirname } from 'node:path'; import { randomUUID } from 'node:crypto'; import { config } from '../config.js'; import { logger } from '../utils/logger.js'; +import { withFileLockSync } from '../utils/file-lock.js'; import { cleanupMaterializedDashboardImages } from '../core/dashboard-images.js'; import { deleteFrozenCards } from './frozen-card-store.js'; import type { Session } from '../types.js'; @@ -78,54 +79,58 @@ function load(): void { if (loaded) return; ensureDir(); const fp = getFilePath(); - if (existsSync(fp)) { - try { - const data = JSON.parse(readFileSync(fp, 'utf-8')); - sessions = new Map(Object.entries(data)); - } catch (err) { - logger.error(`Failed to load sessions: ${err}`); - sessions = new Map(); - loaded = true; - return; - } - const repaired = repairMissingChatScopes(); - if (repaired > 0) { + withFileLockSync(fp, () => { + if (existsSync(fp)) { try { - save(); - logger.info(`Repaired ${repaired} scope-less chat session(s) in ${fp}`); + const data = JSON.parse(readFileSync(fp, 'utf-8')); + sessions = new Map(Object.entries(data)); + const repaired = repairMissingChatScopes(); + if (repaired > 0) { + try { + const tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, JSON.stringify(Object.fromEntries(sessions), null, 2), 'utf-8'); + renameSync(tmpFp, fp); + logger.info(`Repaired ${repaired} scope-less chat session(s) in ${fp}`); + } catch (err) { + // Loading succeeded, so keep the in-memory sessions available even + // if the best-effort repair cannot be persisted yet. + logger.error(`Failed to persist repaired chat session scopes: ${err}`); + } + } + logger.info(`Loaded ${sessions.size} sessions from ${fp}`); } catch (err) { - // Loading succeeded, so keep the in-memory sessions available even if - // this best-effort migration cannot be persisted yet (ENOSPC/EACCES). - logger.error(`Failed to persist repaired chat session scopes: ${err}`); - } - } - logger.info(`Loaded ${sessions.size} sessions from ${fp}`); - } else if (currentAppId) { - // Per-bot file doesn't exist — migrate matching sessions from legacy sessions.json - const legacyFp = join(config.session.dataDir, 'sessions.json'); - if (existsSync(legacyFp)) { - try { - const data: Record = JSON.parse(readFileSync(legacyFp, 'utf-8')); + logger.error(`Failed to load sessions: ${err}`); sessions = new Map(); - for (const [k, v] of Object.entries(data)) { - if (v.larkAppId === currentAppId) { - sessions.set(k, v); + } + } else if (currentAppId) { + // Per-bot file doesn't exist — migrate matching legacy rows while still + // holding the same lock used by daemon saves and offline CLI mutations. + const legacyFp = join(config.session.dataDir, 'sessions.json'); + if (existsSync(legacyFp)) { + try { + const data: Record = JSON.parse(readFileSync(legacyFp, 'utf-8')); + sessions = new Map(); + for (const [k, v] of Object.entries(data)) { + if (v.larkAppId === currentAppId) sessions.set(k, v); } - } - if (sessions.size > 0) { - const repaired = repairMissingChatScopes(); - save(); - logger.info(`Migrated ${sessions.size} sessions from sessions.json to ${fp}`); - if (repaired > 0) { - logger.info(`Repaired ${repaired} scope-less chat session(s) during migration`); + if (sessions.size > 0) { + const repaired = repairMissingChatScopes(); + const obj = Object.fromEntries(sessions); + const tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, JSON.stringify(obj, null, 2), 'utf-8'); + renameSync(tmpFp, fp); + logger.info(`Migrated ${sessions.size} sessions from sessions.json to ${fp}`); + if (repaired > 0) { + logger.info(`Repaired ${repaired} scope-less chat session(s) during migration`); + } } + } catch (err) { + logger.error(`Failed to migrate sessions from legacy file: ${err}`); + sessions = new Map(); } - } catch (err) { - logger.error(`Failed to migrate sessions from legacy file: ${err}`); - sessions = new Map(); } } - } + }); loaded = true; } @@ -142,23 +147,23 @@ function readExistingSessionsFromDisk(fp: string): { raw: string; parsed: Record function save(): void { ensureDir(); const fp = getFilePath(); - const { raw: existingRaw } = readExistingSessionsFromDisk(fp); - const obj: Record = {}; - for (const [k, v] of sessions) { - stripLegacyPendingCardFields(v as unknown as Record); - obj[k] = v; - } - const json = JSON.stringify(obj, null, 2); - // The daemon fires several updateSession()/save() calls per inbound message - // (activity bump, pid, stream-card state, …) and many leave the serialized - // file byte-identical. Skipping the temp-file write + rename in that case - // elides the bulk of the redundant disk I/O — and writing identical bytes is - // a guaranteed no-op, so this can't drop state or race a concurrent writer - // (we compare against what's actually on disk right now). - if (json === existingRaw) return; - const tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; - writeFileSync(tmpFp, json, 'utf-8'); - renameSync(tmpFp, fp); + withFileLockSync(fp, () => { + const { raw: existingRaw } = readExistingSessionsFromDisk(fp); + const obj: Record = {}; + for (const [k, v] of sessions) { + stripLegacyPendingCardFields(v as unknown as Record); + obj[k] = v; + } + const json = JSON.stringify(obj, null, 2); + // The daemon fires several updateSession()/save() calls per inbound message + // (activity bump, pid, stream-card state, …) and many leave the serialized + // file byte-identical. Skipping the temp-file write + rename in that case + // elides the bulk of the redundant disk I/O. + if (json === existingRaw) return; + const tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, json, 'utf-8'); + renameSync(tmpFp, fp); + }); } export function createSession( @@ -190,6 +195,21 @@ export function getSession(sessionId: string): Session | undefined { return sessions.get(sessionId) ?? findInOtherFiles(sessionId); } +/** Cross-process fresh read ordered after daemon/CLI writes by the shared lock. */ +export function getSessionFresh(sessionId: string): Session | undefined { + ensureDir(); + const fp = getFilePath(); + return withFileLockSync(fp, () => { + if (!existsSync(fp)) return undefined; + try { + const data = JSON.parse(readFileSync(fp, 'utf-8')) as Record; + return data[sessionId]; + } catch { + return undefined; + } + }); +} + /** * Search all session files for a session not found in the current file. * @@ -236,6 +256,71 @@ export function closeSession(sessionId: string): void { } } +/** + * Reactivate one explicitly closed row and discard every queued/setup owner in + * the same durable file replacement. The close path has cleared these fields + * since 2026-07, but older closed rows can still contain prepared input. A + * generic resume is an explicit new lifecycle and must never revive that + * abandoned FIFO. + */ +export function reactivateClosedSession( + sessionId: string, +): { ok: true; session: Session } +| { ok: false; error: 'not_found' | 'not_closed' } { + load(); + const session = sessions.get(sessionId); + if (!session) return { ok: false, error: 'not_found' }; + if (session.status !== 'closed') return { ok: false, error: 'not_closed' }; + + const prior = { + status: session.status, + closedAt: session.closedAt, + lastMessageAt: session.lastMessageAt, + codexAppDispatchLedger: session.codexAppDispatchLedger, + codexAppGenerationCommits: session.codexAppGenerationCommits, + queued: session.queued, + queuedPrompt: session.queuedPrompt, + queuedCodexAppText: session.queuedCodexAppText, + queuedCodexAppMessageContext: session.queuedCodexAppMessageContext, + queuedActivationPending: session.queuedActivationPending, + queuedActivationToken: session.queuedActivationToken, + queuedActivationInput: session.queuedActivationInput, + queuedActivationTurnId: session.queuedActivationTurnId, + queuedActivationDispatchAttempt: session.queuedActivationDispatchAttempt, + queuedActivationResume: session.queuedActivationResume, + queuedActivationTail: session.queuedActivationTail, + queuedActivationTailNextOrder: session.queuedActivationTailNextOrder, + pendingRepoSetup: session.pendingRepoSetup, + }; + + session.status = 'active'; + session.closedAt = undefined; + session.lastMessageAt = new Date().toISOString(); + session.codexAppDispatchLedger = undefined; + session.codexAppGenerationCommits = undefined; + session.queued = undefined; + session.queuedPrompt = undefined; + session.queuedCodexAppText = undefined; + session.queuedCodexAppMessageContext = undefined; + session.queuedActivationPending = undefined; + session.queuedActivationToken = undefined; + session.queuedActivationInput = undefined; + session.queuedActivationTurnId = undefined; + session.queuedActivationDispatchAttempt = undefined; + session.queuedActivationResume = undefined; + session.queuedActivationTail = undefined; + session.queuedActivationTailNextOrder = undefined; + session.pendingRepoSetup = undefined; + + try { + save(); + } catch (err) { + Object.assign(session, prior); + throw err; + } + return { ok: true, session }; +} + export function updateSessionPid(sessionId: string, pid: number | null): void { load(); const session = sessions.get(sessionId); diff --git a/src/types.ts b/src/types.ts index 074cd9551..3ff269e9b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -98,7 +98,7 @@ export interface VcMeetingConsumerConfig { } /** Runtime status the worker derives from screen content. */ -export type ScreenStatus = 'working' | 'idle' | 'analyzing' | 'limited'; +export type ScreenStatus = 'working' | 'idle' | 'analyzing' | 'limited' | 'stalled'; /** Status shown on a streaming card — adds the pre-spawn 'starting' phase. */ export type StreamStatus = ScreenStatus | 'starting'; @@ -176,6 +176,28 @@ export interface Session { /** Dashboard-created image files retained while the session is active so * the CLI can read them, then removed by session-store on close. */ dashboardAttachments?: LarkAttachment[]; + /** Durable journal for a queued activation until the worker confirms that + * the exact initial input crossed its adapter submission boundary. */ + queuedActivationPending?: boolean; + /** Generation-scoped identity for the exact queued activation journal. */ + queuedActivationToken?: string; + /** Exact final worker input retained for retry. This may include a triggering + * group reply in addition to the original dashboard backlog prompt. */ + queuedActivationInput?: CliTurnPayload; + queuedActivationTurnId?: string; + queuedActivationDispatchAttempt?: number; + /** Frozen resume mode for the exact journal head. Backlog activations are + * fresh (`false`); post-history promoted successors resume (`true`). */ + queuedActivationResume?: boolean; + /** Durable FIFO of exact turns accepted while an activation/setup gate owns + * the route. Entries are removed only when promoted into the tokened journal. */ + queuedActivationTail?: QueuedActivationTailEntry[]; + /** Monotonic per-session reservation cursor. Reserved synchronously before + * async prompt materialization so completion order cannot reorder arrivals. */ + queuedActivationTailNextOrder?: number; + /** Crash-safe pending-repo owner. Presence means picker/worktree setup must + * be restored instead of classifying the active worker:null row as scratch. */ + pendingRepoSetup?: PendingRepoSetup; createdAt: string; /** Last user/bot/scheduler input that was routed into this session. */ lastMessageAt?: string; @@ -269,6 +291,9 @@ export interface Session { workerGeneration?: number; /** True once a substitute-mode control card has been DM'd to the owner(s). Persisted to avoid re-sends on worker restart or daemon recovery. */ substituteControlCardSent?: boolean; + /** Bounded exact destination captured at inbound turn start. Codex App copies + * this into its durable dispatch ledger after any intervening awaits. */ + turnReplyContexts?: Record; /** * 文档评论入口（/watch-comment / /subscribe-lark-doc）：当本会话「当前这一轮」由飞书文档评论 * 触发时，`botmux send` 的用户可见回复要回到该文档评论（而非飞书）。因 botmux @@ -300,6 +325,10 @@ export interface Session { /** Structured companion for lastCliInput so retry_last_task can preserve a * clean Codex App turn. The legacy string remains authoritative fallback. */ lastCodexAppInput?: CodexAppTurnInput; + /** Crash-safe Codex App accepted/prepared FIFO; daemon is the sole writer. */ + codexAppDispatchLedger?: CodexAppDispatchLedgerEntry[]; + /** Cumulative ACK boundary retained until a fresh runner retires the generation. */ + codexAppGenerationCommits?: CodexAppGenerationCommit[]; /** Default local project whiteboard bound to this session when the optional whiteboard feature is enabled. */ whiteboardId?: string; /** CLI-native resume id when it differs from botmux's sessionId (for example Codex thread id). */ @@ -561,22 +590,116 @@ export interface CodexAppTurnInput { clientUserMessageId?: string; } +/** Daemon-frozen Lark destination for one accepted turn. This is separate + * from `currentReplyTarget`, which is mutable session UI state. */ +export type FrozenSessionReplyTarget = + | { mode: 'plain'; chatId: string } + | { mode: 'thread'; rootMessageId: string } + | { mode: 'quote'; rootMessageId: string }; + +export interface FrozenSessionReplyContext { + target: FrozenSessionReplyTarget; + quoteTargetId?: string; + replyTargetSenderOpenId?: string; + replyTargetSenderIsBot?: boolean; +} + +/** Host-side destination frozen when a Codex App turn crosses daemon + * acceptance. Transient HTTP/silent sinks deliberately carry only their kind: + * after daemon restart their in-memory consumer no longer exists, so recovery + * must fail closed instead of leaking the result into ordinary Lark IM. */ +export type CodexAppDeliverySink = + | 'lark' + | 'doc_comment' + | 'http_wait' + | 'http_async' + | 'suppressed'; + +/** + * Daemon-owned durable attribution for one Codex App runner submission. + * + * `accepted` means the daemon accepted the IM turn but the worker has not yet + * crossed the runner write boundary. `prepared` is published by the worker + * immediately before that write. It is restored into a replacement FIFO, but + * remains uncertain until a signed final arrives. A replacement runner's idle + * marker cannot prove the prior generation never buffered the frame, so warm + * prepared recovery fails closed and requires exact generation fencing. + */ +export interface CodexAppDispatchLedgerEntry { + dispatchId: string; + turnId: string; + /** Links a queued activation journal to the exact accepted FIFO item. */ + queuedActivationToken?: string; + /** Frozen chat/thread routing identity. May differ from daemon-minted + * turnId for scheduler/card inputs that arrived without a native message id. */ + replyTurnId?: string; + /** Exact daemon-owned destination captured when the inbound turn began. */ + replyTarget?: FrozenSessionReplyTarget; + quoteTargetId?: string; + replyTargetSenderOpenId?: string; + replyTargetSenderIsBot?: boolean; + /** Exact output channel selected at acceptance. Undefined is legacy Lark. */ + deliverySink?: CodexAppDeliverySink; + dispatchAttempt?: number; + state: 'accepted' | 'prepared'; + content: string; + codexAppInput?: CodexAppTurnInput; + vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; +} + +/** Monotonic cumulative ACK boundary for one authenticated runner generation. */ +export interface CodexAppGenerationCommit { + generation: string; + committedThrough: number; +} + /** A legacy CLI prompt plus an optional backend-specific structured sidecar. */ export interface CliTurnPayload { content: string; codexAppInput?: CodexAppTurnInput; } +/** Exact ordered successor retained behind an adapter-ACKed activation. */ +export interface QueuedActivationTailEntry { + id: string; + order: number; + userPrompt: string; + cliInput: CliTurnPayload; + turnId: string; + dispatchAttempt?: number; +} + +/** Durable opening/setup state while a repository picker or detached default + * worktree owns the first turn. Runtime DaemonSession buffers are rebuilt from + * this record after restart; the record is cleared only with the activation + * journal's adapter-level ACK. */ +export interface PendingRepoSetup { + mode: 'picker' | 'auto_worktree'; + prompt: string; + rawInput?: string; + turnId?: string; + baseDir?: string; + repoCardMessageId?: string; + codexAppText?: string; + codexAppApplicationContext?: string; + codexAppMessageContext?: string; + attachments?: LarkAttachment[]; + mentions?: LarkMention[]; + substituteTrigger?: SubstituteTrigger; + sender?: { openId: string; type: 'user' | 'bot'; name?: string }; +} + /** Messages sent from Daemon to Worker */ export type DaemonToWorker = - | { type: 'init'; sessionId: string; chatId: string; chatType?: 'group' | 'p2p'; rootMessageId: string; workingDir: string; cliId: string; cliPathOverride?: string; wrapperCli?: string; launchShell?: string; model?: string; disableCliBypass?: boolean; codexRpcInput?: boolean; startupCommands?: string[]; env?: Record; sandbox?: boolean; sandboxPaths?: { readWrite?: string[]; readOnly?: string[]; deny?: string[] }; sandboxHidePaths?: string[]; sandboxReadonlyPaths?: string[]; sandboxNetwork?: boolean; readIsolation?: boolean; readDenyExtraPaths?: string[]; daemonBootId?: string; backendType: BackendType; persistentBackendTarget?: PersistentBackendTarget; backendConfig?: RiffBackendConfig; riffParentTaskId?: string; riffRepoDirs?: string[]; deferredScheduleRun?: Session['deferredScheduleRun']; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; prompt: string; promptCodexAppInput?: CodexAppTurnInput; resume?: boolean; cliSessionId?: string; originalSessionId?: string; ownerOpenId?: string; webPort?: number; larkAppId: string; larkAppSecret: string; brand?: 'feishu' | 'lark'; botName?: string; botOpenId?: string; locale?: 'zh' | 'en'; turnId?: string; dispatchAttempt?: number; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; pluginBindings?: string[]; skillPolicy?: BotSkillPolicy; skillPluginDir?: string; skillReadonlyRoots?: string[]; adoptMode?: boolean; adoptSource?: 'tmux' | 'herdr' | 'zellij'; adoptTmuxTarget?: string; adoptZellijSession?: string; adoptZellijPaneId?: string; adoptHerdrSessionName?: string; adoptHerdrTarget?: string; adoptHerdrPaneId?: string; adoptPaneCols?: number; adoptPaneRows?: number; bridgeJsonlPath?: string; adoptCliPid?: number; adoptCwd?: string; adoptRestoredFromMetadata?: boolean } - | { type: 'message'; content: string; codexAppInput?: CodexAppTurnInput; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; turnId?: string; dispatchAttempt?: number; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin } + | { type: 'init'; sessionId: string; chatId: string; chatType?: 'group' | 'p2p'; rootMessageId: string; workingDir: string; cliId: string; cliPathOverride?: string; wrapperCli?: string; launchShell?: string; model?: string; disableCliBypass?: boolean; codexRpcInput?: boolean; startupCommands?: string[]; env?: Record; sandbox?: boolean; sandboxPaths?: { readWrite?: string[]; readOnly?: string[]; deny?: string[] }; sandboxHidePaths?: string[]; sandboxReadonlyPaths?: string[]; sandboxNetwork?: boolean; readIsolation?: boolean; readDenyExtraPaths?: string[]; daemonBootId?: string; backendType: BackendType; persistentBackendTarget?: PersistentBackendTarget; backendConfig?: RiffBackendConfig; riffParentTaskId?: string; riffRepoDirs?: string[]; deferredScheduleRun?: Session['deferredScheduleRun']; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; prompt: string; promptCodexAppInput?: CodexAppTurnInput; queuedActivationToken?: string; resume?: boolean; cliSessionId?: string; originalSessionId?: string; ownerOpenId?: string; webPort?: number; larkAppId: string; larkAppSecret: string; brand?: 'feishu' | 'lark'; botName?: string; botOpenId?: string; locale?: 'zh' | 'en'; turnId?: string; replyTurnId?: string; dispatchAttempt?: number; codexAppDispatchId?: string; codexAppRecoveredDispatches?: CodexAppDispatchLedgerEntry[]; codexAppGenerationCommits?: CodexAppGenerationCommit[]; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; pluginBindings?: string[]; skillPolicy?: BotSkillPolicy; skillPluginDir?: string; skillReadonlyRoots?: string[]; adoptMode?: boolean; adoptSource?: 'tmux' | 'herdr' | 'zellij'; adoptTmuxTarget?: string; adoptZellijSession?: string; adoptZellijPaneId?: string; adoptHerdrSessionName?: string; adoptHerdrTarget?: string; adoptHerdrPaneId?: string; adoptPaneCols?: number; adoptPaneRows?: number; bridgeJsonlPath?: string; adoptCliPid?: number; adoptCwd?: string; adoptRestoredFromMetadata?: boolean } + | { type: 'message'; content: string; codexAppInput?: CodexAppTurnInput; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; turnId?: string; replyTurnId?: string; dispatchAttempt?: number; codexAppDispatchId?: string; queuedActivationToken?: string; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin } + | { type: 'codex_app_dispatch_persisted'; requestId: string; ok: boolean; error?: string } /** Literal slash-command passthrough. `followUpContent` rides along so the * worker enqueues it strictly AFTER the slash command's Enter — two separate * IPCs would race: process.on('message') handlers don't serialize, and the * raw_input branch awaits 200ms between sendText and Enter, a window where * a separate `message` IPC could write into the PTY first. */ - | { type: 'raw_input'; content: string; turnId?: string; followUpContent?: string; followUpTurnId?: string; followUpCodexAppInput?: CodexAppTurnInput } + | { type: 'raw_input'; content: string; turnId?: string; followUpContent?: string; followUpTurnId?: string; followUpCodexAppInput?: CodexAppTurnInput; queuedActivationToken?: string } /** Rename the current CLI-native interactive session. The worker queues this * administrative slash command until the TUI is idle and does not treat it * as a model turn. Only adapters declaring buildSessionRenameCommand handle @@ -584,7 +707,7 @@ export type DaemonToWorker = | { type: 'rename_session'; title: string } | { type: 'close' } | { type: 'suspend' } - | { type: 'restart' } + | { type: 'restart'; reason?: 'operator' | 'cli_crash' } /** Lease watchdog fencing: only the exact still-running durable attempt may * tear down/restart the CLI. A late command after terminal/current-turn * advance is ignored worker-side. */ @@ -636,6 +759,11 @@ export type WorkerToDaemon = cliPid?: number; cliProcStart?: string; } + | { + type: 'queued_activation_submitted'; + sessionId: string; + activationToken: string; + } | { type: 'cli_session_id'; cliSessionId: string; turnId?: string; dispatchAttempt?: number } | { type: 'native_session_title_generated'; title: string } | { type: 'claude_exit'; code: number | null; signal: string | null; logTail?: string; canParkDiagnostic?: boolean; turnId?: string; dispatchAttempt?: number } @@ -669,6 +797,14 @@ export type WorkerToDaemon = * exactly that generation and ignore a delayed revoke after the next turn * has already published a fresh token. */ | { type: 'managed_turn_origin_revoked'; sessionId: string; capability?: string; turnId?: string; dispatchAttempt?: number } + | { + type: 'codex_app_dispatch_transition'; + sessionId: string; + requestId: string; + operation: 'submit' | 'cancel' | 'retry'; + entries: Array>; + } + | { type: 'codex_app_generation_active'; sessionId: string; generation: string; fresh: boolean } | { type: 'final_output'; /** Worker-side botmux session identity. Daemon validates this before @@ -683,6 +819,7 @@ export type WorkerToDaemon = content: string; lastUuid: string; turnId: string; + replyTurnId?: string; /** Durable receiver attempt attribution. Final output suppression is * attempt-scoped so a late attempt-N event cannot affect attempt N+1. */ dispatchAttempt?: number; @@ -694,6 +831,15 @@ export type WorkerToDaemon = // which mixes presentation with payload). kind?: 'bridge' | 'local-turn' | 'local-turn-headless'; userText?: string; + /** Two-phase Codex App final settlement; daemon persists before ACKing worker. */ + codexAppSettlement?: { + requestId: string; + generation: string; + seq: number; + dispatchId: string; + }; + /** The model already delivered through botmux send; settle without fallback output. */ + suppressDelivery?: boolean; } | { type: 'turn_terminal'; diff --git a/src/utils/bot-routing.ts b/src/utils/bot-routing.ts index 587e8ec4e..02dfe48fb 100644 --- a/src/utils/bot-routing.ts +++ b/src/utils/bot-routing.ts @@ -139,7 +139,7 @@ export function stripCodeSpans(text: string): string { * default addressing (owner / oncall last-caller) is unchanged. */ export function buildFooterAddressing( - s: { ownerOpenId?: string; lastCallerOpenId?: string }, + s: { ownerOpenId?: string; lastCallerOpenId?: string; lastCallerIsBot?: boolean }, opts: { isOncall: boolean; isSubstitute?: boolean; @@ -159,7 +159,7 @@ export function buildFooterAddressing( if (!opts.isOncall && !opts.isSubstitute) return { sendTo: ownerHuman, cc: [] }; const caller = s.lastCallerOpenId ?? owner; - const callerIsBot = !!caller && botIds.has(caller); + const callerIsBot = !!caller && (s.lastCallerIsBot === true || botIds.has(caller)); // Oncall + last caller is a bot (but this reply itself has no explicit bot // target) → fall back to the human owner rather than pinging the bot caller. diff --git a/src/utils/child-env.ts b/src/utils/child-env.ts index ab9dd7f0e..6b896b319 100644 --- a/src/utils/child-env.ts +++ b/src/utils/child-env.ts @@ -132,6 +132,9 @@ export const BOTMUX_INJECTED_ENV_KEYS = [ 'BOTMUX_LARK_LIST_BOTS_API_ENABLED', 'BOTMUX_LARK_LIST_BOTS_API_TIMEOUT_MS', 'BOTMUX_READY_COMMAND', + // Path to a one-shot 0600 Codex App control bootstrap. Only the path reaches + // the pane; the runner consumes+unlinks the file before app-server starts. + 'BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP', // Hermes profile roots must match the worker-side transcript reader. 'HERMES_HOME', 'HERMES_BOTMUX_SOURCE_HOME', diff --git a/src/utils/codex-app-control.ts b/src/utils/codex-app-control.ts new file mode 100644 index 000000000..a2a2a3262 --- /dev/null +++ b/src/utils/codex-app-control.ts @@ -0,0 +1,3100 @@ +import { + createHash, + createPrivateKey, + createPublicKey, + generateKeyPairSync, + randomBytes, + sign, + verify, + type KeyObject, +} from 'node:crypto'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { + chmodSync, + closeSync, + constants as fsConstants, + fchmodSync, + fstatSync, + fsyncSync, + existsSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + renameSync, + rmdirSync, + unlinkSync, + writeSync, + type BigIntStats, + type Stats, +} from 'node:fs'; +import { homedir } from 'node:os'; +import { basename, dirname, isAbsolute, join, win32 } from 'node:path'; +import type { BackendType } from '../adapters/backend/types.js'; +import type { ScreenStatus } from '../types.js'; + +/** + * The launch boundary carries only a path to an owner-only, read-once file. + * That file contains a fresh Ed25519 private key. The runner consumes and + * unlinks it before app-server (and therefore model tools) can start. Only the + * public key is persisted by the worker. + */ +export const CODEX_APP_CONTROL_BOOTSTRAP_ENV = 'BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP'; + +export const CODEX_APP_CONTROL_LINE_MAX_BYTES = 4_096; +export const CODEX_APP_CONTROL_FINAL_MAX_BYTES = 1_048_576; +export const CODEX_APP_CONTROL_FINAL_CHUNK_BYTES = 1_536; +/** Keep control bootstrap/proof alive for the worker's existing cold-start cap. */ +export const CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS = 90_000; +/** Absolute per-connection challenge/accept deadline; transport activity does not extend it. */ +export const CODEX_APP_CONTROL_HANDSHAKE_TIMEOUT_MS = 5_000; + +const CONTROL_STATE_VERSION = 3 as const; +const CONTROL_WIRE_VERSION = 2 as const; +const CONTROL_BOOTSTRAP_VERSION = 3 as const; +const CONTROL_LOCATOR_VERSION = 1 as const; +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const BOOTSTRAP_MAX_BYTES = 8_192; +const LOCATOR_MAX_BYTES = 4_096; +const WINDOWS_SYSTEM_SID = 'S-1-5-18'; +const WINDOWS_PIPE_PREFIX = '\\\\?\\pipe\\botmux-codex-app-'; +const WINDOWS_OWNER_PIPE_PREFIX = '\\\\?\\pipe\\botmux-codex-app-owner-'; +const POSIX_SOCKET_LEAF_RE = /^endpoint-[a-f0-9]{32}\.sock$/; +const POSIX_OWNER_RECORD_RE = /^owner-[a-f0-9]{64}\.json$/; +const POSIX_OWNER_PENDING_RE = /^owner-[a-f0-9]{64}\.pending$/; +const POSIX_REAPER_RECORD_RE = /^reap-[a-f0-9]{64}\.json$/; +const POSIX_REAPER_PENDING_RE = /^reap-[a-f0-9]{64}\.pending$/; +const POSIX_DIRECTORY_GENERATION_RE = /^generation-([a-f0-9]{64})\.json$/; +const POSIX_LEASE_RECORD_MAX_BYTES = 4_096; +const windowsHardenedDirectories = new Set(); + +/** + * Arm the shared Codex App startup deadline. Keeping the scheduling in one + * helper prevents the runner auth gate, bootstrap cleanup, worker proof gate, + * and first-prompt hard cap from silently drifting to different lifetimes. + */ +export function armCodexAppControlStartupTimeout( + onTimeout: () => void, + timeoutMs = CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS, +): ReturnType { + return setTimeout(onTimeout, timeoutMs); +} + +export function armCodexAppControlHandshakeTimeout( + onTimeout: () => void, + timeoutMs = CODEX_APP_CONTROL_HANDSHAKE_TIMEOUT_MS, +): ReturnType { + return setTimeout(onTimeout, timeoutMs); +} + +export type CodexAppSignedStateReadiness = 'invalid' | 'waiting' | 'ready'; + +/** Authentication and a signed busy/idle bit are not sufficient readiness. + * The runner must explicitly assert that its initialized stdin/app-server path + * accepts input. Missing/non-boolean values are protocol violations; `false` + * is a valid not-ready state that leaves the absolute proof deadline armed. */ +export function codexAppSignedStateReadiness(payload: unknown): CodexAppSignedStateReadiness { + if (!payload || typeof payload !== 'object') return 'invalid'; + const acceptingInput = (payload as Record).acceptingInput; + if (typeof acceptingInput !== 'boolean') return 'invalid'; + return acceptingInput ? 'ready' : 'waiting'; +} + +/** Re-armable proof deadline shared by startup and authenticated disconnects. */ +export class CodexAppControlProofDeadline { + private timer: ReturnType | undefined; + + arm(onTimeout: () => void, timeoutMs = CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS): void { + this.clear(); + this.timer = armCodexAppControlStartupTimeout(() => { + this.timer = undefined; + onTimeout(); + }, timeoutMs); + this.timer.unref?.(); + } + + clear(): void { + if (this.timer) clearTimeout(this.timer); + this.timer = undefined; + } + + get armed(): boolean { + return this.timer !== undefined; + } +} + +/** + * Coalesce one signed control record while its worker-side effects are still + * awaiting a durable daemon ACK. Endpoint replacement can replay the same + * generation/sequence before the replay window is committed; every replay + * must observe the first application result instead of running queue or + * liveness effects again. + * + * The caller releases the entry only after committing the replay window (or + * after a rejected application). Keeping release outside the promise closes + * the resolution -> replay-commit gap. + */ +export class CodexAppControlRecordApplicationGate { + private readonly inFlight = new Map>(); + + run( + generation: string, + seq: number, + apply: () => boolean | Promise, + ): Promise { + const key = this.key(generation, seq); + const existing = this.inFlight.get(key); + if (existing) return existing; + const application = Promise.resolve().then(apply); + this.inFlight.set(key, application); + return application; + } + + release(generation: string, seq: number, application: Promise): void { + const key = this.key(generation, seq); + if (this.inFlight.get(key) === application) this.inFlight.delete(key); + } + + private key(generation: string, seq: number): string { + return `${generation}\0${seq}`; + } +} + +export type CodexAppRuntimeScreenStatus = Exclude; + +/** Authentication alone is not readiness. Until the active generation has + * published a signed accepting-input state, an idle screen is fail-closed to + * working so reconnect replay cannot expose a false-idle interval. */ +export function projectCodexAppControlReadinessStatus( + base: CodexAppRuntimeScreenStatus, + readiness: { + controlProven: boolean; + signedStateObserved: boolean; + inputReady: boolean; + }, +): CodexAppRuntimeScreenStatus { + if (base !== 'idle') return base; + return readiness.controlProven + && readiness.signedStateObserved + && readiness.inputReady + ? base + : 'working'; +} +const STATE_MAX_BYTES = 16_384; +const GENERATION_RE = /^[A-Za-z0-9_-]{43}$/; +const CHALLENGE_RE = /^[A-Za-z0-9_-]{43}$/; +const KEY_RE = /^[A-Za-z0-9_-]{32,512}$/; +const SIGNATURE_RE = /^[A-Za-z0-9_-]{86}$/; +const MAX_STATE_IDENTITIES = 4; + +export interface CodexAppControlPathOptions { + platform?: NodeJS.Platform; + localAppData?: string; + homeDirectory?: string; +} + +export interface CodexAppControlFilesystemPolicy { + useNoFollow: boolean; + verifyUid: boolean; + verifyExactMode: boolean; + chmodAfterCreate: boolean; + verifyPostUnlinkLinkCount: boolean; + fsyncDirectory: boolean; +} + +export function codexAppControlFilesystemPolicy( + platform: NodeJS.Platform = process.platform, +): CodexAppControlFilesystemPolicy { + if (platform === 'win32') { + return { + useNoFollow: false, + verifyUid: false, + verifyExactMode: false, + chmodAfterCreate: false, + verifyPostUnlinkLinkCount: false, + fsyncDirectory: false, + }; + } + return { + useNoFollow: true, + verifyUid: true, + verifyExactMode: true, + chmodAfterCreate: true, + verifyPostUnlinkLinkCount: true, + fsyncDirectory: true, + }; +} + +function ownerUid(): number | undefined { + return process.geteuid?.() ?? process.getuid?.(); +} + +function noFollowFlag(platform: NodeJS.Platform): number { + if (!codexAppControlFilesystemPolicy(platform).useNoFollow) return 0; + const flag = (fsConstants as typeof fsConstants & { O_NOFOLLOW?: number }).O_NOFOLLOW; + if (typeof flag !== 'number') throw new Error('O_NOFOLLOW is required for Codex App control files'); + return flag; +} + +function exactMode(mode: number): number { + return mode & 0o777; +} + +function assertOwned(stat: { uid: number }, label: string, platform: NodeJS.Platform): void { + if (!codexAppControlFilesystemPolicy(platform).verifyUid) return; + const uid = ownerUid(); + if (uid !== undefined && stat.uid !== uid) throw new Error(`${label} must be owned by the current uid`); +} + +function isAbsoluteForPlatform(path: string, platform: NodeJS.Platform): boolean { + return platform === 'win32' ? win32.isAbsolute(path) : isAbsolute(path); +} + +function joinForPlatform(platform: NodeJS.Platform, ...parts: string[]): string { + return platform === 'win32' ? win32.join(...parts) : join(...parts); +} + +function dirnameForPlatform(platform: NodeJS.Platform, path: string): string { + return platform === 'win32' ? win32.dirname(path) : dirname(path); +} + +/** + * Windows never places control material under SESSION_DATA_DIR: that setting + * is bot/user supplied and may point at a shared tree. The fixed root is + * instead anchored in the current Windows profile and hardened before any + * secret or locator is created beneath it. + */ +export function codexAppWindowsControlRoot( + options: CodexAppControlPathOptions = {}, +): string { + const base = options.localAppData?.trim() + || (options.platform === 'win32' || options.platform === undefined + ? process.env.LOCALAPPDATA?.trim() + : undefined) + || options.homeDirectory?.trim() + || homedir(); + // The locator protocol relies on local NTFS-style ACL and atomic-replace + // semantics. A UNC/redirected profile can have server-defined ACL and + // rename behavior, so do not silently place capabilities on it. The drive + // root itself remains configurable through LOCALAPPDATA/home for normal + // per-user profiles. + if (!base || !/^[A-Za-z]:[\\/]/.test(base) || !win32.isAbsolute(base)) { + throw new Error( + 'Windows Codex App control root requires a local drive-qualified LOCALAPPDATA or home directory', + ); + } + return win32.join(base, 'Botmux', 'codex-app-control'); +} + +function parseStrictCsvRow(row: string): string[] | undefined { + const fields: string[] = []; + let cursor = 0; + while (cursor < row.length) { + if (row[cursor] !== '"') return undefined; + cursor++; + let field = ''; + let closed = false; + while (cursor < row.length) { + const ch = row[cursor++]!; + if (ch !== '"') { + field += ch; + continue; + } + if (row[cursor] === '"') { + field += '"'; + cursor++; + continue; + } + closed = true; + break; + } + if (!closed) return undefined; + fields.push(field); + if (cursor === row.length) break; + if (row[cursor] !== ',') return undefined; + cursor++; + if (cursor === row.length) return undefined; + } + return fields; +} + +export function parseWindowsCurrentSid(output: string): string | undefined { + const rows = output.replace(/^\uFEFF/, '').split(/\r?\n/).filter(row => row.length > 0); + if (rows.length !== 1) return undefined; + const fields = parseStrictCsvRow(rows[0]!); + if (fields?.length !== 2) return undefined; + const sid = fields[1]!; + return /^S-\d-(?:\d+-)+\d+$/i.test(sid) ? sid.toUpperCase() : undefined; +} + +interface WindowsAclAce { + type: string; + flags: string; + rights: string; + objectGuid: string; + inheritObjectGuid: string; + trustee: string; +} + +export function decodeWindowsAclSnapshot(raw: Buffer): string { + if (raw.length >= 2 && raw[0] === 0xff && raw[1] === 0xfe) { + return raw.subarray(2).toString('utf16le'); + } + if (raw.length >= 4 && raw[1] === 0 && raw[3] === 0) return raw.toString('utf16le'); + return raw.toString('utf8'); +} + +function parseWindowsDaclSnapshot(snapshot: string): { flags: string; aces: WindowsAclAce[] } | undefined { + // `/save` layout is not a documented API. Locate the SDDL token by grammar, + // whether it follows the path on the same or next line. `D:\path` cannot + // match because a DACL has only uppercase control flags before its first ACE. + const daclMatch = snapshot.match(/D:[A-Z]*\([^\r\n]*/); + if (!daclMatch) return undefined; + let dacl = daclMatch[0].slice(2).trim(); + const saclStart = dacl.indexOf('S:'); + if (saclStart >= 0) dacl = dacl.slice(0, saclStart); + const firstAce = dacl.indexOf('('); + if (firstAce < 0) return undefined; + const flags = dacl.slice(0, firstAce); + const aceText = dacl.slice(firstAce); + const aces: WindowsAclAce[] = []; + let cursor = 0; + while (cursor < aceText.length) { + if (aceText[cursor] !== '(') return undefined; + const end = aceText.indexOf(')', cursor + 1); + if (end < 0) return undefined; + const fields = aceText.slice(cursor + 1, end).split(';'); + if (fields.length !== 6) return undefined; + aces.push({ + type: fields[0]!, + flags: fields[1]!, + rights: fields[2]!, + objectGuid: fields[3]!, + inheritObjectGuid: fields[4]!, + trustee: fields[5]!, + }); + cursor = end + 1; + } + return { flags, aces }; +} + +/** Locale-independent verification of the SDDL emitted by `icacls /save`. */ +export function verifyWindowsCodexAppControlDacl( + snapshot: string, + currentSid: string, + kind: 'directory' | 'file' = 'directory', +): boolean { + const parsed = parseWindowsDaclSnapshot(snapshot); + if (!parsed || !parsed.flags.includes('P')) return false; + // P = protected DACL. AI/AR are canonical auto-inheritance metadata that + // icacls may retain even after inherited ACEs are removed; reject anything + // else and separately reject every inherited ACE below. + if (parsed.flags.replace(/AI|AR|P/g, '')) return false; + const expected = new Set( + currentSid.toUpperCase() === WINDOWS_SYSTEM_SID + ? [WINDOWS_SYSTEM_SID] + : [currentSid.toUpperCase(), WINDOWS_SYSTEM_SID], + ); + if (parsed.aces.length !== expected.size) return false; + for (const ace of parsed.aces) { + const trustee = ace.trustee.toUpperCase() === 'SY' + ? WINDOWS_SYSTEM_SID + : ace.trustee.toUpperCase(); + if (ace.type !== 'A' || ace.rights !== 'FA' || ace.objectGuid || ace.inheritObjectGuid + || ace.flags.includes('ID') || !expected.delete(trustee)) return false; + if (kind === 'directory') { + if (!ace.flags.includes('OI') || !ace.flags.includes('CI') + || ace.flags.replace(/OI|CI/g, '')) return false; + } else if (ace.flags) return false; + } + return expected.size === 0; +} + +export type WindowsControlCommandRunner = ( + command: string, + args: string[], +) => Pick, 'status' | 'stdout' | 'stderr' | 'error'>; + +function defaultWindowsControlCommandRunner( + command: string, + args: string[], +): Pick, 'status' | 'stdout' | 'stderr' | 'error'> { + return spawnSync(command, args, { + encoding: 'utf8', + windowsHide: true, + shell: false, + timeout: 10_000, + maxBuffer: 1_048_576, + }); +} + +function runWindowsControlCommand( + runner: WindowsControlCommandRunner, + command: string, + args: string[], + label: string, +): string { + const result = runner(command, args); + if (result.error || result.status !== 0) { + throw new Error(`${label} failed closed (status=${String(result.status)})`); + } + return result.stdout ?? ''; +} + +/** + * Remove inherited ACLs, grant only the current SID and SYSTEM full control, + * then validate the resulting protected DACL via locale-independent SDDL. + * Unknown explicit ACEs are never silently retained: verification fails. + */ +function hardenWindowsCodexAppControlAcl( + path: string, + kind: 'directory' | 'file', + runner: WindowsControlCommandRunner = defaultWindowsControlCommandRunner, +): void { + if (!path || !win32.isAbsolute(path)) { + throw new Error(`Windows Codex App control ${kind} must be absolute`); + } + if (kind === 'directory') mkdirSync(path, { recursive: true }); + const stat = lstatSync(path); + if ((kind === 'directory' ? !stat.isDirectory() : !stat.isFile()) + || stat.isSymbolicLink() || (kind === 'file' && stat.nlink !== 1)) { + throw new Error(`Windows Codex App control ${kind} must be a real single-link ${kind}`); + } + const systemRoot = process.env.SystemRoot?.trim() || process.env.WINDIR?.trim(); + if (!systemRoot || !win32.isAbsolute(systemRoot)) { + throw new Error('Windows SystemRoot is required for trusted ACL tools'); + } + const whoamiCommand = win32.join(systemRoot, 'System32', 'whoami.exe'); + const icaclsCommand = win32.join(systemRoot, 'System32', 'icacls.exe'); + const whoami = runWindowsControlCommand( + runner, + whoamiCommand, + ['/user', '/fo', 'csv', '/nh'], + 'Windows current SID lookup', + ); + const sid = parseWindowsCurrentSid(whoami); + if (!sid) throw new Error('Windows current SID lookup returned no SID'); + let lastVerificationError: Error | undefined; + // Several workers can harden the shared per-user root concurrently. The + // transaction is idempotent; bounded full retries avoid snapshotting another + // worker's inheritance-removal→grant gap without ever accepting a loose ACL. + for (let attempt = 0; attempt < 3; attempt++) { + const snapshotPath = win32.join( + kind === 'directory' ? path : win32.dirname(path), + `.botmux-acl-${process.pid}-${randomBytes(16).toString('hex')}.txt`, + ); + try { + runWindowsControlCommand( + runner, + icaclsCommand, + [path, '/setowner', `*${sid}`, '/q'], + 'Windows control owner assignment', + ); + runWindowsControlCommand( + runner, + icaclsCommand, + [path, '/inheritance:r', '/q'], + 'Windows control ACL inheritance removal', + ); + runWindowsControlCommand( + runner, + icaclsCommand, + [path, '/grant:r', `*${sid}:${kind === 'directory' ? '(OI)(CI)F' : 'F'}`, + ...(sid === WINDOWS_SYSTEM_SID + ? [] + : [`*${WINDOWS_SYSTEM_SID}:${kind === 'directory' ? '(OI)(CI)F' : 'F'}`]), '/q'], + 'Windows control ACL grant', + ); + runWindowsControlCommand( + runner, + icaclsCommand, + [path, '/verify', '/q'], + 'Windows control ACL canonical verification', + ); + runWindowsControlCommand( + runner, + icaclsCommand, + [path, '/save', snapshotPath, '/q'], + 'Windows control ACL snapshot', + ); + const snapshot = decodeWindowsAclSnapshot(readFileSync(snapshotPath)); + if (verifyWindowsCodexAppControlDacl(snapshot, sid, kind)) return; + lastVerificationError = new Error( + `Windows Codex App control ${kind} ACL is not exactly current SID + SYSTEM`, + ); + } catch (err) { + lastVerificationError = err instanceof Error ? err : new Error(String(err)); + } finally { + try { unlinkSync(snapshotPath); } catch { /* no snapshot on command failure */ } + } + } + throw lastVerificationError ?? new Error(`Windows Codex App control ${kind} ACL verification failed`); +} + +/** + * Remove inherited ACLs, set the current SID as owner, grant only that SID and + * SYSTEM full control, and verify the protected DACL. Unknown explicit ACEs + * are retained by icacls but cause exact verification to fail closed. + */ +export function hardenWindowsCodexAppControlDirectory( + directory: string, + runner: WindowsControlCommandRunner = defaultWindowsControlCommandRunner, +): void { + hardenWindowsCodexAppControlAcl(directory, 'directory', runner); +} + +/** Validate/harden a pre-existing fixed-name state file before trusting it. */ +export function hardenWindowsCodexAppControlFile( + path: string, + runner: WindowsControlCommandRunner = defaultWindowsControlCommandRunner, +): void { + hardenWindowsCodexAppControlAcl(path, 'file', runner); +} + +/** Create/validate a non-symlink owner-only directory. */ +export function ensureCodexAppControlDirectory( + directory: string, + platform: NodeJS.Platform = process.platform, +): void { + if (!directory || !isAbsoluteForPlatform(directory, platform)) { + throw new Error('Codex App control directory must be absolute'); + } + if (platform === 'win32') { + const cacheKey = win32.normalize(directory).toLowerCase(); + if (windowsHardenedDirectories.has(cacheKey)) { + const stat = lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + windowsHardenedDirectories.delete(cacheKey); + throw new Error('Windows Codex App control directory changed after ACL hardening'); + } + return; + } + hardenWindowsCodexAppControlDirectory(directory); + // Re-running whoami/icacls for every atomic locator replacement would + // block the worker event loop and amplify the named-pipe DoS boundary. + // Once exact ACL verification succeeds, cross-user mutation is excluded; + // same-SID compromise is outside this control plane's trust boundary. + windowsHardenedDirectories.add(cacheKey); + return; + } + mkdirSync(directory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE }); + const stat = lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new Error('Codex App control directory must be a real directory'); + } + assertOwned(stat, 'Codex App control directory', platform); + if (exactMode(stat.mode) !== PRIVATE_DIRECTORY_MODE) { + chmodSync(directory, PRIVATE_DIRECTORY_MODE); + const tightened = lstatSync(directory); + assertOwned(tightened, 'Codex App control directory', platform); + if (!tightened.isDirectory() || tightened.isSymbolicLink() + || exactMode(tightened.mode) !== PRIVATE_DIRECTORY_MODE) { + throw new Error('Codex App control directory could not be secured to 0700'); + } + } +} + +function sessionKey(sessionId: string): string { + return createHash('sha256').update(sessionId, 'utf8').digest('hex'); +} + +export function codexAppControlStatePath(sessionDataDir: string, sessionId: string): string { + if (process.platform === 'win32') { + return win32.join(codexAppWindowsControlRoot(), 'state', `${sessionKey(sessionId)}.json`); + } + return join(sessionDataDir, 'codex-app-control', `${sessionKey(sessionId)}.json`); +} + +export function codexAppControlStatePathForPlatform( + sessionDataDir: string, + sessionId: string, + options: CodexAppControlPathOptions = {}, +): string { + const platform = options.platform ?? process.platform; + if (platform === 'win32') { + return win32.join(codexAppWindowsControlRoot(options), 'state', `${sessionKey(sessionId)}.json`); + } + return join(sessionDataDir, 'codex-app-control', `${sessionKey(sessionId)}.json`); +} + +export function codexAppControlSocketPath(directory: string, sessionId: string): string { + // Unix-domain paths are short on macOS. A fixed 32-hex leaf keeps the path + // comfortably below the platform limit for normal BOT_HOME/tmp roots. + return join(directory, `${sessionKey(sessionId).slice(0, 32)}.sock`); +} + +export function codexAppPosixControlRoot(uid = process.getuid?.()): string { + return join('/tmp', `botmux-codex-app-${uid ?? 'unknown'}`); +} + +export function codexAppPosixOwnerLeaseDirectory(controlRoot: string, sessionId: string): string { + return join(controlRoot, 'leases', `${sessionKey(sessionId).slice(0, 32)}.lease`); +} + +export function generateCodexAppPosixSocketEndpoint(socketDirectory: string): string { + return join(socketDirectory, `endpoint-${randomBytes(16).toString('hex')}.sock`); +} + +export function codexAppControlLocatorPath( + controlRoot: string, + sessionId: string, + platform: NodeJS.Platform = process.platform, +): string { + return joinForPlatform(platform, controlRoot, 'locators', `${sessionKey(sessionId)}.json`); +} + +export function generateCodexAppWindowsPipeEndpoint(): string { + return `${WINDOWS_PIPE_PREFIX}${randomBytes(32).toString('hex')}`; +} + +/** + * A fixed coordination pipe is held for the lifetime of one Windows worker + * process. It is not a control endpoint and is never put in a runner locator. + * Its only purpose is to serialize locator publishers across the daemon's + * kill-then-fork overlap window. The OS releases it if the old worker is + * terminated, avoiding a crash-stale filesystem lock. + */ +export function codexAppWindowsOwnerPipeEndpoint(sessionId: string): string { + return `${WINDOWS_OWNER_PIPE_PREFIX}${sessionKey(sessionId)}`; +} + +export interface CodexAppControlOwnerLeaseOptions { + bind: () => Promise; + timeoutMs?: number; + retryDelayMs?: number; + now?: () => number; + wait?: (delayMs: number) => Promise; +} + +/** Retry only a competing bind; every other owner-lease error fails closed. */ +export async function acquireCodexAppControlOwnerLease( + options: CodexAppControlOwnerLeaseOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? 10_000; + const retryDelayMs = options.retryDelayMs ?? 50; + const now = options.now ?? Date.now; + const wait = options.wait ?? (delayMs => new Promise(resolve => setTimeout(resolve, delayMs))); + const deadline = now() + timeoutMs; + let lastBusyError: unknown; + while (true) { + try { + return await options.bind(); + } catch (err) { + if ((err as NodeJS.ErrnoException)?.code !== 'EADDRINUSE') throw err; + lastBusyError = err; + } + if (now() >= deadline) { + throw lastBusyError instanceof Error + ? lastBusyError + : new Error('Windows Codex App control owner lease timed out'); + } + await wait(Math.min(retryDelayMs, Math.max(0, deadline - now()))); + } +} + +interface CodexAppPosixFileIdentity { + dev: string; + ino: string; +} + +interface CodexAppPosixDirectoryIdentity extends CodexAppPosixFileIdentity { + generation?: string; +} + +interface CodexAppPosixOwnerTarget { + nonce: string; + recordIdentity: CodexAppPosixFileIdentity; + pid: number | null; + processStartToken: string | null; +} + +interface CodexAppPosixOwnerRecord { + version: 2 | 3; + role: 'owner' | 'reaper'; + sessionId: string; + nonce: string; + pid: number; + processStartToken: string; + createdAtMs: number; + directoryIdentity: CodexAppPosixDirectoryIdentity; + targetOwner?: CodexAppPosixOwnerTarget; +} + +export type CodexAppOwnerProcessStatus = 'alive' | 'dead' | 'unknown'; + +export interface CodexAppPosixOwnerLease { + directory: string; + ownerRecordPath: string; + isOwned(): boolean; + release(): void; +} + +export interface CodexAppPosixOwnerLeaseOptions { + controlRoot: string; + sessionId: string; + platform?: NodeJS.Platform; + timeoutMs?: number; + retryDelayMs?: number; + initializationGraceMs?: number; + pid?: number; + processStartToken?: string; + now?: () => number; + wait?: (delayMs: number) => Promise; + inspectOwner?: (pid: number, processStartToken: string) => CodexAppOwnerProcessStatus; + /** Deterministic handoff-race seam: invoked after mkdir(EEXIST), before observation. */ + onContended?: () => void; + /** Deterministic crash-resume seam: invoked after this creator pins the new directory inode. */ + onOwnerDirectoryCreated?: (directory: string) => void | Promise; + /** Deterministic crash-window seam: pauses after owner publication, before the directory CAS. */ + onOwnerRecordPublished?: (directory: string, ownerRecordPath: string) => void | Promise; + /** Deterministic replacement seam: pauses after a stale owner is pinned, before reaper publication. */ + onBeforeReaperRecordPublished?: (directory: string) => void | Promise; + /** Deterministic crash-window seam: pauses after reaper publication, before the directory CAS. */ + onReaperRecordPublished?: (directory: string, reaperRecordPath: string) => void | Promise; +} + +function linuxProcessStartToken(raw: string): string | undefined { + const closeParen = raw.lastIndexOf(')'); + if (closeParen < 0) return undefined; + const fields = raw.slice(closeParen + 2).trim().split(/\s+/); + return fields[19] || undefined; +} + +export function codexAppPosixProcessProbeEnv( + base: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + // `ps -o lstart` is formatted through locale and timezone. Pin both so the + // same live process cannot acquire a different token after the worker's + // ambient LC_*/TZ changes. Preserve PATH and the remaining launch context. + return { ...base, LC_ALL: 'C', LANG: 'C', TZ: 'UTC' }; +} + +export function readCodexAppProcessStartToken( + pid: number, + platform: NodeJS.Platform = process.platform, +): string | undefined { + if (!Number.isSafeInteger(pid) || pid <= 0 || platform === 'win32') return undefined; + if (platform === 'linux') { + try { return linuxProcessStartToken(readFileSync(`/proc/${pid}/stat`, 'utf8')); } + catch { return undefined; } + } + const result = spawnSync('/bin/ps', ['-p', String(pid), '-o', 'lstart='], { + encoding: 'utf8', + env: codexAppPosixProcessProbeEnv(), + shell: false, + timeout: 5_000, + maxBuffer: 64 * 1024, + }); + if (result.error || result.status !== 0) return undefined; + const token = result.stdout.trim(); + return token || undefined; +} + +function inspectCodexAppOwnerProcess( + pid: number, + expectedStartToken: string, + platform: NodeJS.Platform, +): CodexAppOwnerProcessStatus { + try { + process.kill(pid, 0); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ESRCH') return 'dead'; + // EPERM proves that a process occupies the pid. Any other probe failure is + // also fail-closed: it must not authorize stale-owner deletion. + if (code === 'EPERM') return 'alive'; + return 'unknown'; + } + const actualStartToken = readCodexAppProcessStartToken(pid, platform); + if (!actualStartToken) return 'unknown'; + // lstart has one-second resolution on non-Linux POSIX. A same-second PID + // reuse can therefore look alive and block stale recovery, but cannot grant + // ownership or reap a live owner: the ambiguity remains fail-closed. + return actualStartToken === expectedStartToken ? 'alive' : 'dead'; +} + +function validPosixFileIdentity(value: unknown): value is CodexAppPosixFileIdentity { + if (!value || typeof value !== 'object') return false; + const identity = value as Record; + return typeof identity.dev === 'string' && /^(?:0|[1-9]\d*)$/.test(identity.dev) + && typeof identity.ino === 'string' && /^(?:0|[1-9]\d*)$/.test(identity.ino); +} + +function validPosixDirectoryIdentity( + value: unknown, + requireGeneration: boolean, +): value is CodexAppPosixDirectoryIdentity { + if (!validPosixFileIdentity(value)) return false; + const generation = (value as unknown as Record).generation; + return requireGeneration + ? typeof generation === 'string' && /^[a-f0-9]{64}$/.test(generation) + : generation === undefined; +} + +function validPosixOwnerTarget(value: unknown): value is CodexAppPosixOwnerTarget { + if (!value || typeof value !== 'object') return false; + const target = value as Record; + const processIdentityValid = (target.pid === null && target.processStartToken === null) + || (Number.isSafeInteger(target.pid) && (target.pid as number) > 0 + && typeof target.processStartToken === 'string' + && target.processStartToken.length > 0 && target.processStartToken.length <= 256); + return typeof target.nonce === 'string' && /^[a-f0-9]{64}$/.test(target.nonce) + && validPosixFileIdentity(target.recordIdentity) + && processIdentityValid; +} + +function validPosixOwnerRecord( + value: unknown, + sessionId: string, + expectedRole?: 'owner' | 'reaper', +): value is CodexAppPosixOwnerRecord { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + const roleValid = record.role === 'owner' || record.role === 'reaper'; + const targetValid = record.role === 'owner' + ? record.targetOwner === undefined + : validPosixOwnerTarget(record.targetOwner); + const versionValid = record.version === 2 + ? validPosixDirectoryIdentity(record.directoryIdentity, false) + : record.version === 3 && validPosixDirectoryIdentity(record.directoryIdentity, true); + return versionValid + && roleValid + && (expectedRole === undefined || record.role === expectedRole) + && record.sessionId === sessionId + && typeof record.nonce === 'string' && /^[a-f0-9]{64}$/.test(record.nonce) + && Number.isSafeInteger(record.pid) && (record.pid as number) > 0 + && typeof record.processStartToken === 'string' + && record.processStartToken.length > 0 && record.processStartToken.length <= 256 + && typeof record.createdAtMs === 'number' && Number.isFinite(record.createdAtMs) + && targetValid; +} + +function readPosixOwnerRecord( + path: string, + sessionId: string, + platform: NodeJS.Platform, + expectedRole: 'owner' | 'reaper' = 'owner', +): CodexAppPosixOwnerRecord | undefined { + try { + const parsed: unknown = JSON.parse(readOwnedRegularFile( + path, + 4_096, + 'Codex App POSIX owner record', + platform, + )); + return validPosixOwnerRecord(parsed, sessionId, expectedRole) ? parsed : undefined; + } catch { + return undefined; + } +} + +interface ObservedPosixLeaseRecord { + path: string; + name: string; + nonce: string; + role: 'owner' | 'reaper'; + stat: BigIntStats; + ageMs: number; + record?: CodexAppPosixOwnerRecord; +} + +interface ObservedPosixLeaseDirectory { + stat: BigIntStats; + identity: CodexAppPosixDirectoryIdentity; + names: string[]; + generationName?: string; + generationStat?: BigIntStats; + generationNames?: string[]; + generationStats?: Map; +} + +function errnoCode(err: unknown): string | undefined { + return (err as NodeJS.ErrnoException)?.code; +} + +function posixLeaseRaceError(message: string): NodeJS.ErrnoException { + return Object.assign(new Error(message), { code: 'EAGAIN' }); +} + +function posixFileIdentity(stat: BigIntStats): CodexAppPosixFileIdentity { + return { dev: stat.dev.toString(10), ino: stat.ino.toString(10) }; +} + +function samePosixFileIdentity( + left: CodexAppPosixFileIdentity, + right: CodexAppPosixFileIdentity, +): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function samePosixStatIdentity(before: BigIntStats, after: BigIntStats): boolean { + return before.dev === after.dev && before.ino === after.ino; +} + +function exactBigIntMode(mode: bigint): number { + return Number(mode & 0o777n); +} + +function assertPosixOwned( + stat: { uid: bigint }, + label: string, + platform: NodeJS.Platform, +): void { + if (!codexAppControlFilesystemPolicy(platform).verifyUid) return; + const uid = ownerUid(); + if (uid !== undefined && stat.uid !== BigInt(uid)) { + throw new Error(`${label} must be owned by the current uid`); + } +} + +function assertPrivatePosixLeaseRecord( + stat: BigIntStats, + label: string, + platform: NodeJS.Platform, +): void { + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1n + || exactBigIntMode(stat.mode) !== PRIVATE_FILE_MODE) { + throw new Error(`${label} must be a single-link regular 0600 file`); + } + assertPosixOwned(stat, label, platform); +} + +function posixLeaseRecordNonce(name: string): string | undefined { + const match = name.match(/^(?:owner|reap)-([a-f0-9]{64})\.(?:json|pending)$/); + return match?.[1]; +} + +function posixLeaseRecordRole(name: string): 'owner' | 'reaper' | undefined { + if (POSIX_OWNER_RECORD_RE.test(name) || POSIX_OWNER_PENDING_RE.test(name)) return 'owner'; + if (POSIX_REAPER_RECORD_RE.test(name) || POSIX_REAPER_PENDING_RE.test(name)) return 'reaper'; + return undefined; +} + +/** + * Observe one lease actor record without treating a crash-partial payload as + * insecure metadata. Wrong type/owner/mode/link count fails hard; a secure + * empty/truncated/oversized file is returned as a grace-reclaimable record. + */ +function observePosixLeaseRecord( + path: string, + name: string, + sessionId: string, + platform: NodeJS.Platform, + nowMs: number, + label: string, +): ObservedPosixLeaseRecord | undefined { + let before: BigIntStats; + try { + before = lstatSync(path, { bigint: true }); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + assertPrivatePosixLeaseRecord(before, label, platform); + const nonce = posixLeaseRecordNonce(name); + if (!nonce) throw new Error(`${label} has an invalid filename`); + const role = posixLeaseRecordRole(name); + if (!role) throw new Error(`${label} has an invalid role`); + + let record: CodexAppPosixOwnerRecord | undefined; + if (before.size > 0n && before.size <= BigInt(POSIX_LEASE_RECORD_MAX_BYTES)) { + try { + const parsed: unknown = JSON.parse(readOwnedRegularFile( + path, + POSIX_LEASE_RECORD_MAX_BYTES, + label, + platform, + )); + if (validPosixOwnerRecord(parsed, sessionId, role) && parsed.nonce === nonce) { + record = parsed; + } + } catch (err) { + // JSON syntax failures are crash-partial data. A path race is handled by + // the post-read identity check below and never reclassified as stale. + if (errnoCode(err) === 'ENOENT') return undefined; + } + } + + let after: BigIntStats; + try { + after = lstatSync(path, { bigint: true }); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + assertPrivatePosixLeaseRecord(after, label, platform); + if (!samePosixStatIdentity(before, after)) return undefined; + return { + path, + name, + nonce, + role, + stat: after, + ageMs: Math.max(0, nowMs - Number(after.mtimeMs)), + ...(record ? { record } : {}), + }; +} + +function observePosixLeaseDirectory( + directory: string, + platform: NodeJS.Platform, +): ObservedPosixLeaseDirectory | undefined { + let before: BigIntStats; + try { + before = lstatSync(directory, { bigint: true }); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + assertPosixLeaseDirectoryStat(before, platform); + let names: string[]; + try { + names = readdirSync(directory).sort(); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + let after: BigIntStats; + try { + after = lstatSync(directory, { bigint: true }); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + if (!samePosixStatIdentity(before, after)) return undefined; + assertPosixLeaseDirectoryStat(after, platform); + const generationNames = names.filter(name => POSIX_DIRECTORY_GENERATION_RE.test(name)); + const generationStats = new Map(); + for (const candidateName of generationNames) { + const candidateGeneration = candidateName.match(POSIX_DIRECTORY_GENERATION_RE)?.[1]; + const candidatePath = join(directory, candidateName); + try { + const candidateStat = lstatSync(candidatePath, { bigint: true }); + assertPrivatePosixLeaseRecord( + candidateStat, + 'Codex App POSIX directory generation', + platform, + ); + const persistedGeneration = readOwnedRegularFile( + candidatePath, + 128, + 'Codex App POSIX directory generation', + platform, + ); + const verifiedStat = lstatSync(candidatePath, { bigint: true }); + assertPrivatePosixLeaseRecord( + verifiedStat, + 'Codex App POSIX directory generation', + platform, + ); + if (!candidateGeneration || persistedGeneration !== candidateGeneration + || !samePosixStatIdentity(candidateStat, verifiedStat)) { + throw new Error('Codex App POSIX directory generation verification failed'); + } + generationStats.set(candidateName, verifiedStat); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + } + if (generationNames.length > 1) { + return { + stat: after, + identity: posixFileIdentity(after), + names, + generationNames, + generationStats, + }; + } + const generationName = generationNames[0]; + if (!generationName) return { stat: after, identity: posixFileIdentity(after), names }; + const generation = generationName.match(POSIX_DIRECTORY_GENERATION_RE)?.[1]; + return { + stat: after, + identity: { ...posixFileIdentity(after), generation }, + names, + generationName, + generationStat: generationStats.get(generationName), + }; +} + +function createStagedPosixLeaseRecord(input: { + directory: string; + finalPath: string; + pendingPath: string; + record: CodexAppPosixOwnerRecord; + sessionId: string; + label: string; + platform: NodeJS.Platform; +}): void { + try { + createExclusiveFile( + input.pendingPath, + JSON.stringify(input.record), + `${input.label} pending file`, + input.platform, + ); + // A complete synced inode becomes authoritative in one namespace step. + // SIGKILL during the write can therefore leave only a `.pending` file. + renameSync(input.pendingPath, input.finalPath); + fsyncDirectory(input.directory, input.platform); + const persisted = readPosixOwnerRecord( + input.finalPath, + input.sessionId, + input.platform, + input.record.role, + ); + if (!persisted || persisted.nonce !== input.record.nonce) { + throw new Error(`${input.label} verification failed`); + } + } finally { + try { unlinkSync(input.pendingPath); } catch { /* renamed or crash-partial cleanup */ } + } +} + +/** + * Rename the exact observed random-name inode out of the lease directory. + * Random actor filenames make this a compare-and-swap boundary: a concurrent + * cleaner can only receive ENOENT and can never unlink a successor's record. + */ +function retireObservedPosixLeaseRecord( + observed: ObservedPosixLeaseRecord, + leasesRoot: string, + sessionId: string, + platform: NodeJS.Platform, +): boolean { + let current: BigIntStats; + try { + current = lstatSync(observed.path, { bigint: true }); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return false; + throw err; + } + assertPrivatePosixLeaseRecord(current, 'Codex App POSIX lease record', platform); + if (!samePosixStatIdentity(observed.stat, current)) return false; + const retired = join( + leasesRoot, + `.retired-${sessionKey(sessionId).slice(0, 16)}-${observed.nonce}-${randomBytes(16).toString('hex')}`, + ); + try { + renameSync(observed.path, retired); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return false; + throw err; + } + try { + const moved = lstatSync(retired, { bigint: true }); + assertPrivatePosixLeaseRecord(moved, 'Retired Codex App POSIX lease record', platform); + if (!samePosixStatIdentity(observed.stat, moved)) { + throw new Error('Codex App POSIX lease record changed during retirement'); + } + } finally { + try { unlinkSync(retired); } catch { /* crash residue is outside the authority directory */ } + } + return true; +} + +/** Remove the exact generation marker only while it is the directory's sole + * remaining entry. The marker gives each mkdir generation an unforgeable + * identity even when the filesystem immediately reuses the same dev+ino. */ +function retirePosixDirectoryGeneration( + directory: string, + observed: ObservedPosixLeaseDirectory, + leasesRoot: string, + sessionId: string, + platform: NodeJS.Platform, +): boolean { + if (!observed.generationName || !observed.generationStat) return false; + const current = observePosixLeaseDirectory(directory, platform); + if (!current || current.names.length !== 1 + || current.generationName !== observed.generationName + || current.identity.generation !== observed.identity.generation + || !samePosixFileIdentity(current.identity, observed.identity) + || !current.generationStat + || !samePosixStatIdentity(current.generationStat, observed.generationStat)) return false; + const markerPath = join(directory, observed.generationName); + const retired = join( + leasesRoot, + `.retired-generation-${sessionKey(sessionId).slice(0, 16)}-${observed.identity.generation}`, + ); + try { + renameSync(markerPath, retired); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return false; + throw err; + } + try { + const moved = lstatSync(retired, { bigint: true }); + assertPrivatePosixLeaseRecord(moved, 'Retired Codex App POSIX directory generation', platform); + if (!samePosixStatIdentity(observed.generationStat, moved)) { + throw new Error('Codex App POSIX directory generation changed during retirement'); + } + } finally { + try { unlinkSync(retired); } catch { /* crash residue is outside authority */ } + } + try { rmdirSync(directory); } catch (err) { + if (errnoCode(err) !== 'ENOENT' && errnoCode(err) !== 'ENOTEMPTY') throw err; + } + return true; +} + +/** Recover only the crash residue in which the directory contains two or more + * individually valid generation markers and no actor can still be live. Each + * exact marker inode is moved out before unlink, so a concurrent replacement + * turns into a retry instead of deleting successor authority. */ +function retireAmbiguousPosixDirectoryGenerations( + directory: string, + observed: ObservedPosixLeaseDirectory, + leasesRoot: string, + sessionId: string, + platform: NodeJS.Platform, +): boolean { + const observedNames = observed.generationNames; + const observedStats = observed.generationStats; + if (!observedNames || observedNames.length < 2 || !observedStats) return false; + const current = observePosixLeaseDirectory(directory, platform); + if (!current + || !samePosixFileIdentity(current.identity, observed.identity) + || current.names.length !== observedNames.length + || current.generationNames?.length !== observedNames.length + || current.generationNames.some((name, index) => name !== observedNames[index])) return false; + for (const name of observedNames) { + const expectedStat = observedStats.get(name); + const currentStat = current.generationStats?.get(name); + if (!expectedStat || !currentStat || !samePosixStatIdentity(expectedStat, currentStat)) { + return false; + } + } + for (const name of observedNames) { + const expectedStat = observedStats.get(name)!; + const markerPath = join(directory, name); + const retired = join( + leasesRoot, + `.retired-generation-${sessionKey(sessionId).slice(0, 16)}-${randomBytes(16).toString('hex')}`, + ); + try { + renameSync(markerPath, retired); + } catch (err) { + if (errnoCode(err) === 'ENOENT') return false; + throw err; + } + try { + const moved = lstatSync(retired, { bigint: true }); + assertPrivatePosixLeaseRecord(moved, 'Retired Codex App POSIX directory generation', platform); + if (!samePosixStatIdentity(expectedStat, moved)) { + throw new Error('Codex App POSIX directory generation changed during retirement'); + } + } finally { + try { unlinkSync(retired); } catch { /* residue is outside authority */ } + } + } + try { rmdirSync(directory); } catch (err) { + if (errnoCode(err) !== 'ENOENT' && errnoCode(err) !== 'ENOTEMPTY') throw err; + } + return true; +} + +function assertPosixLeaseDirectoryStat(stat: BigIntStats, platform: NodeJS.Platform): void { + if (!stat.isDirectory() || stat.isSymbolicLink() || exactBigIntMode(stat.mode) !== PRIVATE_DIRECTORY_MODE) { + throw new Error('Codex App POSIX owner lease must be a real 0700 directory'); + } + assertPosixOwned(stat, 'Codex App POSIX owner lease', platform); +} + +function posixLeaseRecordMatchesDirectory( + observed: ObservedPosixLeaseRecord, + directory: ObservedPosixLeaseDirectory | CodexAppPosixDirectoryIdentity, +): boolean { + if (!observed.record) return false; + const identity = 'identity' in directory ? directory.identity : directory; + if (!samePosixFileIdentity(observed.record.directoryIdentity, identity)) return false; + return observed.record.version === 2 + ? identity.generation === undefined + : observed.record.directoryIdentity.generation === identity.generation; +} + +function posixOwnerTarget(observed: ObservedPosixLeaseRecord): CodexAppPosixOwnerTarget { + return { + nonce: observed.nonce, + recordIdentity: posixFileIdentity(observed.stat), + pid: observed.record?.pid ?? null, + processStartToken: observed.record?.processStartToken ?? null, + }; +} + +function posixReaperTargetsOwner( + reaper: ObservedPosixLeaseRecord, + owner: ObservedPosixLeaseRecord, +): boolean { + const target = reaper.record?.targetOwner; + if (!target || target.nonce !== owner.nonce + || !samePosixFileIdentity(target.recordIdentity, posixFileIdentity(owner.stat))) return false; + if (!owner.record) return target.pid === null && target.processStartToken === null; + return target.pid === owner.record.pid + && target.processStartToken === owner.record.processStartToken; +} + +/** + * Process-lifetime publisher lease for POSIX. + * + * Both owners and stale cleaners publish complete, random-name actor records. + * An actor record is written to `.pending`, synced, and atomically renamed to + * `.json`, so SIGKILL cannot create a partial authoritative record. Existing + * partial records remain recoverable after a grace interval. Every complete + * actor is bound to the exact directory dev+ino it intended to mutate; reapers + * additionally bind the exact owner-record inode/process tuple. A cleaner moves + * the exact observed random-name inode out of the authority directory before + * deleting it; that rename is the CAS boundary which prevents a delayed actor + * from poisoning or deleting a successor's record after path replacement. + */ +export async function acquireCodexAppPosixOwnerLease( + options: CodexAppPosixOwnerLeaseOptions, +): Promise { + const platform = options.platform ?? process.platform; + if (platform === 'win32') throw new Error('POSIX owner lease is unavailable on Windows'); + const now = options.now ?? Date.now; + const wait = options.wait ?? (delayMs => new Promise(resolve => setTimeout(resolve, delayMs))); + const timeoutMs = options.timeoutMs ?? 10_000; + const retryDelayMs = options.retryDelayMs ?? 50; + const initializationGraceMs = options.initializationGraceMs ?? 1_000; + const pid = options.pid ?? process.pid; + const processStartToken = options.processStartToken + ?? readCodexAppProcessStartToken(pid, platform); + if (!processStartToken) { + throw new Error('Cannot verify current process start token for Codex App POSIX owner lease'); + } + const inspectOwner = options.inspectOwner + ?? ((ownerPid: number, token: string) => inspectCodexAppOwnerProcess(ownerPid, token, platform)); + ensureCodexAppControlDirectory(options.controlRoot, platform); + const leasesRoot = join(options.controlRoot, 'leases'); + ensureCodexAppControlDirectory(leasesRoot, platform); + const directory = codexAppPosixOwnerLeaseDirectory(options.controlRoot, options.sessionId); + const deadline = now() + timeoutMs; + + const retry = async (): Promise => { + if (now() >= deadline) throw new Error('Codex App POSIX owner lease timed out'); + await wait(Math.min(retryDelayMs, Math.max(0, deadline - now()))); + }; + + acquireLoop: while (true) { + const nonce = randomBytes(32).toString('hex'); + const directoryGeneration = randomBytes(32).toString('hex'); + const generationName = `generation-${directoryGeneration}.json`; + const generationPath = join(directory, generationName); + const ownerRecordPath = join(directory, `owner-${nonce}.json`); + const ownerPendingPath = join(directory, `owner-${nonce}.pending`); + let createdDirectory = false; + try { + mkdirSync(directory, { mode: PRIVATE_DIRECTORY_MODE }); + createdDirectory = true; + const ownedDirectoryStat = lstatSync(directory, { bigint: true }); + assertPosixLeaseDirectoryStat(ownedDirectoryStat, platform); + createExclusiveFile( + generationPath, + directoryGeneration, + 'Codex App POSIX directory generation', + platform, + ); + fsyncDirectory(directory, platform); + const ownedDirectoryIdentity: CodexAppPosixDirectoryIdentity = { + ...posixFileIdentity(ownedDirectoryStat), + generation: directoryGeneration, + }; + if (options.onOwnerDirectoryCreated) { + await options.onOwnerDirectoryCreated(directory); + } + const record: CodexAppPosixOwnerRecord = { + version: 3, + role: 'owner', + sessionId: options.sessionId, + nonce, + pid, + processStartToken, + createdAtMs: now(), + directoryIdentity: ownedDirectoryIdentity, + }; + try { + createStagedPosixLeaseRecord({ + directory, + finalPath: ownerRecordPath, + pendingPath: ownerPendingPath, + record, + sessionId: options.sessionId, + label: 'Codex App POSIX owner record', + platform, + }); + const persisted = readPosixOwnerRecord(ownerRecordPath, options.sessionId, platform, 'owner'); + if (!persisted || persisted.nonce !== nonce) { + throw new Error('Codex App POSIX owner record verification failed'); + } + if (options.onOwnerRecordPublished) { + await options.onOwnerRecordPublished(directory, ownerRecordPath); + } + const installed = observePosixLeaseDirectory(directory, platform); + if (!installed || !samePosixStatIdentity(installed.stat, ownedDirectoryStat) + || installed.identity.generation !== directoryGeneration + || installed.names.length !== 2 + || !installed.names.includes(generationName) + || !installed.names.includes(basename(ownerRecordPath))) { + throw posixLeaseRaceError( + 'Codex App POSIX owner directory changed during record publication', + ); + } + } catch (err) { + try { unlinkSync(ownerPendingPath); } catch { /* not installed */ } + try { unlinkSync(ownerRecordPath); } catch { /* not installed */ } + try { unlinkSync(generationPath); } catch { /* replacement directory owns a different generation */ } + try { rmdirSync(directory); } catch { /* competing reaper or residue */ } + throw err; + } + let released = false; + const isOwned = (): boolean => { + if (released) return false; + try { + const currentDirectory = observePosixLeaseDirectory(directory, platform); + if (!currentDirectory + || !samePosixFileIdentity(currentDirectory.identity, ownedDirectoryIdentity)) return false; + const currentOwner = observePosixLeaseRecord( + ownerRecordPath, + basename(ownerRecordPath), + options.sessionId, + platform, + now(), + 'Codex App POSIX owner record', + ); + if (!currentOwner || currentOwner.record?.nonce !== nonce + || !posixLeaseRecordMatchesDirectory(currentOwner, ownedDirectoryIdentity)) return false; + for (const name of currentDirectory.names.filter(candidate => ( + POSIX_REAPER_RECORD_RE.test(candidate) || POSIX_REAPER_PENDING_RE.test(candidate) + ))) { + const reaper = observePosixLeaseRecord( + join(directory, name), + name, + options.sessionId, + platform, + now(), + 'Codex App POSIX reaper record', + ); + if (!reaper) return false; + // A delayed cleaner may publish into a replacement directory. Only + // a complete actor bound to this inode and this exact owner tuple + // can suspend authority. Partial/foreign actors never authorize a + // takeover and are reclaimed by the acquisition loop. + if (posixLeaseRecordMatchesDirectory(reaper, currentDirectory) + && posixReaperTargetsOwner(reaper, currentOwner)) return false; + } + return true; + } catch { + return false; + } + }; + return { + directory, + ownerRecordPath, + isOwned, + release: () => { + if (released) return; + released = true; + const persisted = readPosixOwnerRecord(ownerRecordPath, options.sessionId, platform, 'owner'); + if (persisted?.nonce === nonce + && samePosixFileIdentity(persisted.directoryIdentity, ownedDirectoryIdentity)) { + try { unlinkSync(ownerRecordPath); } catch { /* already reaped */ } + } + try { unlinkSync(ownerPendingPath); } catch { /* never published or already retired */ } + try { + const currentDirectory = observePosixLeaseDirectory(directory, platform); + if (currentDirectory + && currentDirectory.identity.generation === directoryGeneration + && currentDirectory.names.length === 1 + && currentDirectory.names[0] === generationName) { + retirePosixDirectoryGeneration( + directory, + currentDirectory, + leasesRoot, + options.sessionId, + platform, + ); + } + } catch { /* another contender owns cleanup */ } + }, + }; + } catch (err) { + if (createdDirectory && (errnoCode(err) === 'EAGAIN' || errnoCode(err) === 'ENOENT')) { + await retry(); + continue; + } + if (createdDirectory || errnoCode(err) !== 'EEXIST') throw err; + } + + options.onContended?.(); + const snapshot = observePosixLeaseDirectory(directory, platform); + if (!snapshot) { + await retry(); + continue; + } + + const knownNames = snapshot.names.filter(name => ( + POSIX_OWNER_RECORD_RE.test(name) + || POSIX_OWNER_PENDING_RE.test(name) + || POSIX_REAPER_RECORD_RE.test(name) + || POSIX_REAPER_PENDING_RE.test(name) + || POSIX_DIRECTORY_GENERATION_RE.test(name) + )); + if (knownNames.length !== snapshot.names.length) { + throw new Error('Codex App POSIX owner lease contains an unknown record'); + } + + if ((snapshot.generationNames?.length ?? 0) > 1) { + let retiredActor = false; + for (const name of snapshot.names.filter(candidate => ( + POSIX_OWNER_RECORD_RE.test(candidate) + || POSIX_OWNER_PENDING_RE.test(candidate) + || POSIX_REAPER_RECORD_RE.test(candidate) + || POSIX_REAPER_PENDING_RE.test(candidate) + ))) { + const observed = observePosixLeaseRecord( + join(directory, name), + name, + options.sessionId, + platform, + now(), + 'Codex App POSIX ambiguous-generation actor', + ); + if (!observed) { + await retry(); + continue acquireLoop; + } + if (observed.record) { + const status = inspectOwner( + observed.record.pid, + observed.record.processStartToken, + ); + if (status !== 'dead') { + await retry(); + continue acquireLoop; + } + } else if (observed.ageMs < initializationGraceMs) { + await retry(); + continue acquireLoop; + } + if (!retireObservedPosixLeaseRecord( + observed, + leasesRoot, + options.sessionId, + platform, + )) { + await retry(); + continue acquireLoop; + } + retiredActor = true; + } + if (retiredActor) continue; + const ageMs = Math.max(0, now() - Number(snapshot.stat.mtimeMs)); + if (ageMs < initializationGraceMs + || !retireAmbiguousPosixDirectoryGenerations( + directory, + snapshot, + leasesRoot, + options.sessionId, + platform, + )) { + await retry(); + } + continue; + } + + // A live/unknown cleaner is authority and blocks fail-closed. A dead or + // crash-partial cleaner has a unique filename, so retiring that exact inode + // cannot clobber a successor cleaner. + const reaperNames = snapshot.names.filter(name => ( + POSIX_REAPER_RECORD_RE.test(name) || POSIX_REAPER_PENDING_RE.test(name) + )); + if (reaperNames.length > 0) { + let retiredAny = false; + for (const name of reaperNames) { + const observed = observePosixLeaseRecord( + join(directory, name), + name, + options.sessionId, + platform, + now(), + 'Codex App POSIX reaper record', + ); + if (!observed) { + await retry(); + continue acquireLoop; + } + if (observed.record && !posixLeaseRecordMatchesDirectory(observed, snapshot)) { + // The actor was published by a delayed cleaner that pinned an older + // directory inode. It has no authority in this replacement directory, + // regardless of whether that process is still alive. + } else if (observed.record) { + const status = inspectOwner( + observed.record.pid, + observed.record.processStartToken, + ); + if (status !== 'dead') { + await retry(); + continue acquireLoop; + } + } else if (observed.ageMs < initializationGraceMs) { + await retry(); + continue acquireLoop; + } + if (!retireObservedPosixLeaseRecord( + observed, + leasesRoot, + options.sessionId, + platform, + )) { + await retry(); + continue acquireLoop; + } + retiredAny = true; + } + if (retiredAny) continue; + } + + const ownerNames = snapshot.names.filter(name => ( + POSIX_OWNER_RECORD_RE.test(name) || POSIX_OWNER_PENDING_RE.test(name) + )); + const observedOwners: ObservedPosixLeaseRecord[] = []; + let retiredForeignOwner = false; + for (const name of ownerNames) { + const observed = observePosixLeaseRecord( + join(directory, name), + name, + options.sessionId, + platform, + now(), + 'Codex App POSIX owner record', + ); + if (!observed) { + await retry(); + continue acquireLoop; + } + if (observed.record && !posixLeaseRecordMatchesDirectory(observed, snapshot)) { + // This can only be a creator that pinned an older directory and resumed + // after the fixed path was replaced. Its process liveness is irrelevant: + // the record never held authority over this inode. + if (!retireObservedPosixLeaseRecord( + observed, + leasesRoot, + options.sessionId, + platform, + )) { + await retry(); + continue acquireLoop; + } + retiredForeignOwner = true; + continue; + } + observedOwners.push(observed); + } + if (retiredForeignOwner) continue; + if (observedOwners.length > 1) { + let retiredNonAuthoritativeOwner = false; + for (const observed of observedOwners) { + const reclaimable = observed.record + ? inspectOwner(observed.record.pid, observed.record.processStartToken) === 'dead' + : observed.ageMs >= initializationGraceMs; + if (!reclaimable) continue; + // A creator can be paused after mkdir while another contender replaces + // the empty inode, then resume and publish into that replacement before + // noticing another owner. Exact actor-file CAS makes dead/partial losing + // candidates independently reclaimable without touching a live/unknown + // same-directory candidate. + if (!retireObservedPosixLeaseRecord( + observed, + leasesRoot, + options.sessionId, + platform, + )) { + await retry(); + continue acquireLoop; + } + retiredNonAuthoritativeOwner = true; + } + if (retiredNonAuthoritativeOwner) continue; + // Multiple live/unknown candidates are ambiguous. None can be newly + // granted by this contender; wait for a creator to finish its directory + // CAS or for liveness to become provable. + await retry(); + continue; + } + + if (observedOwners.length === 0) { + const ageMs = Math.max(0, now() - Number(snapshot.stat.mtimeMs)); + if (ageMs < initializationGraceMs) { + await retry(); + continue; + } + if (snapshot.generationName) { + if (!retirePosixDirectoryGeneration( + directory, + snapshot, + leasesRoot, + options.sessionId, + platform, + )) await retry(); + } else { + try { + rmdirSync(directory); + } catch (err) { + if (errnoCode(err) !== 'ENOENT' && errnoCode(err) !== 'ENOTEMPTY') throw err; + await retry(); + } + } + continue; + } + + const observedOwner = observedOwners[0]!; + if (observedOwner.record) { + const status = inspectOwner( + observedOwner.record.pid, + observedOwner.record.processStartToken, + ); + if (status !== 'dead') { + await retry(); + continue; + } + } else if (observedOwner.ageMs < initializationGraceMs) { + await retry(); + continue; + } + + const reaperNonce = randomBytes(32).toString('hex'); + const reaperPath = join(directory, `reap-${reaperNonce}.json`); + const reaperPendingPath = join(directory, `reap-${reaperNonce}.pending`); + const reaperRecord: CodexAppPosixOwnerRecord = { + version: snapshot.identity.generation ? 3 : 2, + role: 'reaper', + sessionId: options.sessionId, + nonce: reaperNonce, + pid, + processStartToken, + createdAtMs: now(), + directoryIdentity: snapshot.identity, + targetOwner: posixOwnerTarget(observedOwner), + }; + try { + if (options.onBeforeReaperRecordPublished) { + await options.onBeforeReaperRecordPublished(directory); + } + createStagedPosixLeaseRecord({ + directory, + finalPath: reaperPath, + pendingPath: reaperPendingPath, + record: reaperRecord, + sessionId: options.sessionId, + label: 'Codex App POSIX reaper record', + platform, + }); + if (options.onReaperRecordPublished) { + await options.onReaperRecordPublished(directory, reaperPath); + } + const reaperDirectory = observePosixLeaseDirectory(directory, platform); + if (!reaperDirectory || !samePosixStatIdentity(reaperDirectory.stat, snapshot.stat)) { + throw posixLeaseRaceError( + 'Codex App POSIX owner directory changed during reaper election', + ); + } + } catch (err) { + try { unlinkSync(reaperPendingPath); } catch { /* never installed */ } + try { unlinkSync(reaperPath); } catch { /* never installed */ } + if (errnoCode(err) === 'EEXIST' || errnoCode(err) === 'ENOENT' + || errnoCode(err) === 'EAGAIN') { + await retry(); + continue; + } + throw err; + } + + try { + const electedSnapshot = observePosixLeaseDirectory(directory, platform); + if (!electedSnapshot) continue; + const electedReapers: ObservedPosixLeaseRecord[] = []; + let electionMustRetry = false; + for (const name of electedSnapshot.names.filter(candidate => ( + POSIX_REAPER_RECORD_RE.test(candidate) || POSIX_REAPER_PENDING_RE.test(candidate) + ))) { + const observed = observePosixLeaseRecord( + join(directory, name), + name, + options.sessionId, + platform, + now(), + 'Codex App POSIX reaper record', + ); + if (!observed || !observed.record) { + electionMustRetry = true; + break; + } + if (!posixLeaseRecordMatchesDirectory(observed, electedSnapshot) + || !posixReaperTargetsOwner(observed, observedOwner)) { + electionMustRetry = true; + break; + } + if (observed.nonce !== reaperNonce) { + const status = inspectOwner( + observed.record.pid, + observed.record.processStartToken, + ); + if (status !== 'alive') { + // Unknown actors block fail-closed; dead actors are reclaimed by + // the outer observation loop before a new election. + electionMustRetry = true; + break; + } + } + electedReapers.push(observed); + } + if (electionMustRetry) continue; + electedReapers.sort((a, b) => ( + (a.record!.createdAtMs - b.record!.createdAtMs) || a.nonce.localeCompare(b.nonce) + )); + // Simultaneous contenders may both have published before observing each + // other. The oldest deterministic actor proceeds; all others withdraw. + if (electedReapers[0]?.nonce !== reaperNonce) continue; + + const currentOwner = observePosixLeaseRecord( + observedOwner.path, + observedOwner.name, + options.sessionId, + platform, + now(), + 'Codex App POSIX owner record', + ); + if (!currentOwner || !samePosixStatIdentity(currentOwner.stat, observedOwner.stat) + || (currentOwner.record && !posixLeaseRecordMatchesDirectory(currentOwner, electedSnapshot)) + || !posixReaperTargetsOwner( + electedReapers.find(candidate => candidate.nonce === reaperNonce)!, + currentOwner, + )) continue; + if (currentOwner.record) { + if (inspectOwner( + currentOwner.record.pid, + currentOwner.record.processStartToken, + ) !== 'dead') continue; + } else if (currentOwner.ageMs < initializationGraceMs) { + continue; + } + if (!retireObservedPosixLeaseRecord( + currentOwner, + leasesRoot, + options.sessionId, + platform, + )) { + continue; + } + } finally { + try { unlinkSync(reaperPendingPath); } catch { /* renamed or already retired */ } + try { unlinkSync(reaperPath); } catch { /* another cleaner retired it */ } + } + const emptied = observePosixLeaseDirectory(directory, platform); + if (emptied?.generationName && emptied.names.length === 1) { + retirePosixDirectoryGeneration( + directory, + emptied, + leasesRoot, + options.sessionId, + platform, + ); + } else { + try { rmdirSync(directory); } catch (err) { + if (errnoCode(err) !== 'ENOENT' && errnoCode(err) !== 'ENOTEMPTY') throw err; + } + } + } +} + +export function generateCodexAppControlEpoch(): string { + return randomBytes(32).toString('base64url'); +} + +export interface CodexAppControlLocator { + version: typeof CONTROL_LOCATOR_VERSION; + sessionId: string; + epoch: string; + endpoint: string; +} + +export function isValidCodexAppWindowsPipeEndpoint(endpoint: unknown): endpoint is string { + return typeof endpoint === 'string' + && /^\\\\\?\\pipe\\botmux-codex-app-[a-f0-9]{64}$/.test(endpoint); +} + +function isValidCodexAppPosixSocketEndpoint( + endpoint: unknown, + locatorPath: string | undefined, + expectedControlRoot: string | undefined, + expectedSessionId: string, + platform: NodeJS.Platform, +): endpoint is string { + if (typeof endpoint !== 'string' || !locatorPath || !expectedControlRoot + || !isAbsolute(endpoint) || !isAbsolute(locatorPath) || !isAbsolute(expectedControlRoot)) { + return false; + } + // Do not infer trust from an attacker-selected locator path. The caller must + // supply the already trusted control root and both paths must be the exact + // canonical children for this session. + if (locatorPath !== codexAppControlLocatorPath( + expectedControlRoot, + expectedSessionId, + platform, + )) return false; + const endpointDirectory = join(expectedControlRoot, 'sockets'); + const leaf = basename(endpoint); + return POSIX_SOCKET_LEAF_RE.test(leaf) + && endpoint === join(endpointDirectory, leaf) + // Darwin's sockaddr_un.sun_path is 104 bytes including the terminator. + && Buffer.byteLength(endpoint, 'utf8') < 104; +} + +export interface CodexAppControlLocatorValidationOptions { + platform?: NodeJS.Platform; + locatorPath?: string; + /** Trusted, worker-owned root; never derive this authority from locatorPath. */ + expectedControlRoot?: string; +} + +export function validateCodexAppControlLocator( + value: unknown, + expectedSessionId: string, + options: CodexAppControlLocatorValidationOptions = {}, +): CodexAppControlLocator | undefined { + if (!value || typeof value !== 'object') return undefined; + const record = value as Record; + const platform = options.platform ?? process.platform; + const endpointValid = platform === 'win32' + ? isValidCodexAppWindowsPipeEndpoint(record.endpoint) + : isValidCodexAppPosixSocketEndpoint( + record.endpoint, + options.locatorPath, + options.expectedControlRoot, + expectedSessionId, + platform, + ); + if (record.version !== CONTROL_LOCATOR_VERSION + || record.sessionId !== expectedSessionId + || !isValidGeneration(record.epoch) + || !endpointValid) return undefined; + return record as unknown as CodexAppControlLocator; +} + +export interface CodexAppControlIdentity { + generation: string; + publicKey: string; + createdAtMs: number; +} + +/** + * pending may contain an old live identity plus a fresh spawn candidate. The + * first identity that answers this worker's fresh socket challenge is atomically + * collapsed to the sole active identity. No private material is persisted. + */ +export interface CodexAppControlState { + version: typeof CONTROL_STATE_VERSION; + status: 'pending' | 'active'; + identities: CodexAppControlIdentity[]; + updatedAtMs: number; + activatedAtMs?: number; +} + +function isValidGeneration(value: unknown): value is string { + return typeof value === 'string' && GENERATION_RE.test(value); +} + +function isValidChallenge(value: unknown): value is string { + return typeof value === 'string' && CHALLENGE_RE.test(value); +} + +function importPublicKey(encoded: string): KeyObject { + return createPublicKey({ key: Buffer.from(encoded, 'base64url'), format: 'der', type: 'spki' }); +} + +function importPrivateKey(encoded: string): KeyObject { + return createPrivateKey({ key: Buffer.from(encoded, 'base64url'), format: 'der', type: 'pkcs8' }); +} + +function validIdentity(value: unknown): value is CodexAppControlIdentity { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + if (!isValidGeneration(record.generation) + || typeof record.publicKey !== 'string' || !KEY_RE.test(record.publicKey) + || typeof record.createdAtMs !== 'number' || !Number.isFinite(record.createdAtMs)) return false; + try { + return importPublicKey(record.publicKey).asymmetricKeyType === 'ed25519'; + } catch { + return false; + } +} + +function validState(value: unknown): value is CodexAppControlState { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + if (record.version !== CONTROL_STATE_VERSION + || (record.status !== 'pending' && record.status !== 'active') + || !Array.isArray(record.identities) + || record.identities.length < 1 + || record.identities.length > MAX_STATE_IDENTITIES + || !record.identities.every(validIdentity) + || new Set(record.identities.map(identity => identity.generation)).size !== record.identities.length + || typeof record.updatedAtMs !== 'number' || !Number.isFinite(record.updatedAtMs) + || (record.status === 'active' && record.identities.length !== 1) + || (record.activatedAtMs !== undefined + && (typeof record.activatedAtMs !== 'number' || !Number.isFinite(record.activatedAtMs)))) return false; + return true; +} + +function assertRegularSingleLinkWithinLimit( + stat: Stats, + maxBytes: number, + label: string, + platform: NodeJS.Platform, +): void { + const policy = codexAppControlFilesystemPolicy(platform); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 + || (policy.verifyExactMode && exactMode(stat.mode) !== PRIVATE_FILE_MODE) + || stat.size <= 0 || stat.size > maxBytes) { + const mode = policy.verifyExactMode ? ' 0600' : ''; + throw new Error(`${label} must be a single-link regular${mode} file within the size limit`); + } + assertOwned(stat, label, platform); +} + +function sameFileIdentity( + before: Stats, + after: Stats, +): boolean { + return Number.isFinite(before.dev) && Number.isFinite(before.ino) + && before.dev === after.dev && before.ino === after.ino; +} + +function readOwnedRegularFile( + path: string, + maxBytes: number, + label: string, + platform: NodeJS.Platform = process.platform, +): string { + const before = lstatSync(path); + assertRegularSingleLinkWithinLimit(before, maxBytes, label, platform); + const fd = openSync(path, fsConstants.O_RDONLY | noFollowFlag(platform)); + try { + const stat = fstatSync(fd); + assertRegularSingleLinkWithinLimit(stat, maxBytes, label, platform); + if (!sameFileIdentity(before, stat)) throw new Error(`${label} changed between lstat and open`); + return readFileSync(fd, 'utf8'); + } finally { + closeSync(fd); + } +} + +export function readCodexAppControlState( + path: string, + platform: NodeJS.Platform = process.platform, +): CodexAppControlState | undefined { + try { + const parsed: unknown = JSON.parse(readOwnedRegularFile( + path, + STATE_MAX_BYTES, + 'Codex App control state', + platform, + )); + return validState(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function createExclusiveFile( + path: string, + contents: string, + label: string, + platform: NodeJS.Platform = process.platform, +): void { + const policy = codexAppControlFilesystemPolicy(platform); + const fd = openSync( + path, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | noFollowFlag(platform), + PRIVATE_FILE_MODE, + ); + try { + if (policy.chmodAfterCreate) fchmodSync(fd, PRIVATE_FILE_MODE); + const stat = fstatSync(fd); + if (!stat.isFile() || stat.nlink !== 1 + || (policy.verifyExactMode && exactMode(stat.mode) !== PRIVATE_FILE_MODE)) { + throw new Error(`${label} was not created as a single-link regular private file`); + } + assertOwned(stat, label, platform); + const data = Buffer.from(contents, 'utf8'); + let offset = 0; + while (offset < data.length) offset += writeSync(fd, data, offset, data.length - offset); + fsyncSync(fd); + } finally { + closeSync(fd); + } +} + +function fsyncDirectory(directory: string, platform: NodeJS.Platform): void { + if (!codexAppControlFilesystemPolicy(platform).fsyncDirectory) return; + const fd = openSync(directory, fsConstants.O_RDONLY | noFollowFlag(platform)); + try { + const stat = fstatSync(fd); + if (!stat.isDirectory()) throw new Error('Codex App control state parent must be a directory'); + assertOwned(stat, 'Codex App control state parent', platform); + fsyncSync(fd); + } finally { + closeSync(fd); + } +} + +/** Symlink-safe atomic state replacement inside a private directory. */ +export function writeCodexAppControlState(path: string, state: CodexAppControlState): void { + writeCodexAppControlStateForPlatform(path, state, process.platform); +} + +export function writeCodexAppControlStateForPlatform( + path: string, + state: CodexAppControlState, + platform: NodeJS.Platform, +): void { + if (!validState(state)) throw new Error('Codex App control state is invalid'); + const directory = dirnameForPlatform(platform, path); + ensureCodexAppControlDirectory(directory, platform); + const tmp = joinForPlatform( + platform, + directory, + `.${sessionKey(path)}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`, + ); + try { + createExclusiveFile(tmp, JSON.stringify(state), 'Codex App control state temp file', platform); + renameSync(tmp, path); + // The file itself was synced before rename; sync the parent as well so a + // proved generation does not silently roll back after a host crash. + fsyncDirectory(directory, platform); + const persisted = readCodexAppControlState(path, platform); + if (!persisted || persisted.status !== state.status + || persisted.identities.map(identity => identity.generation).join(',') + !== state.identities.map(identity => identity.generation).join(',')) { + throw new Error('Codex App control state verification failed after rename'); + } + } finally { + try { unlinkSync(tmp); } catch { /* renamed or never created */ } + } +} + +export function readCodexAppControlLocator( + path: string, + expectedSessionId: string, + platform: NodeJS.Platform = process.platform, + expectedControlRoot: string | undefined = platform === 'win32' + ? undefined + : codexAppPosixControlRoot(), +): CodexAppControlLocator | undefined { + try { + const value: unknown = JSON.parse(readOwnedRegularFile( + path, + LOCATOR_MAX_BYTES, + 'Codex App control locator', + platform, + )); + return validateCodexAppControlLocator(value, expectedSessionId, { + platform, + locatorPath: path, + expectedControlRoot, + }); + } catch { + return undefined; + } +} + +export function writeCodexAppControlLocator( + path: string, + locator: CodexAppControlLocator, + platform: NodeJS.Platform = process.platform, + expectedControlRoot: string | undefined = platform === 'win32' + ? undefined + : codexAppPosixControlRoot(), +): void { + if (!validateCodexAppControlLocator(locator, locator.sessionId, { + platform, + locatorPath: path, + expectedControlRoot, + })) { + throw new Error('Codex App control locator is invalid'); + } + const directory = dirnameForPlatform(platform, path); + ensureCodexAppControlDirectory(directory, platform); + const tmp = joinForPlatform( + platform, + directory, + `.${sessionKey(path)}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`, + ); + try { + createExclusiveFile(tmp, JSON.stringify(locator), 'Codex App control locator temp file', platform); + renameSync(tmp, path); + fsyncDirectory(directory, platform); + const persisted = readCodexAppControlLocator( + path, + locator.sessionId, + platform, + expectedControlRoot, + ); + if (!persisted || persisted.epoch !== locator.epoch || persisted.endpoint !== locator.endpoint) { + throw new Error('Codex App control locator verification failed after rename'); + } + } finally { + try { unlinkSync(tmp); } catch { /* renamed or never created */ } + } +} + +/** + * The ordering contract is security-sensitive on every platform: the random + * pipe/socket must already be bound before its locator becomes visible. + */ +export async function bindThenPublishCodexAppControlLocator(input: { + sessionId: string; + epoch: string; + endpoint: string; + listen: (endpoint: string) => Promise; + publish: (locator: CodexAppControlLocator) => void; + platform?: NodeJS.Platform; + locatorPath?: string; + expectedControlRoot?: string; + isCurrent?: () => boolean; + retire?: () => void; +}): Promise { + const locator = validateCodexAppControlLocator({ + version: CONTROL_LOCATOR_VERSION, + sessionId: input.sessionId, + epoch: input.epoch, + endpoint: input.endpoint, + }, input.sessionId, { + platform: input.platform, + locatorPath: input.locatorPath, + expectedControlRoot: input.expectedControlRoot, + }); + if (!locator) throw new Error('Codex App control endpoint publication metadata is invalid'); + await input.listen(locator.endpoint); + if (input.isCurrent && !input.isCurrent()) { + input.retire?.(); + return undefined; + } + try { + input.publish(locator); + } catch (err) { + input.retire?.(); + throw err; + } + return locator; +} + +export function shouldFailCodexAppControlChannel(input: { + channelId: number; + currentChannelId: number; + stopping: boolean; +}): boolean { + return !input.stopping && input.channelId === input.currentChannelId; +} + +/** + * Runner-side endpoint policy for locator rotations. A never-accepted + * locator may be retried (the independently random, protected epoch is still + * required for acceptance) until the shared startup deadline. Once accepted, + * the pipe name is permanently burned and only a newly published endpoint can + * be used. + */ +export class CodexAppControlEndpointTracker { + private readonly attemptsByEndpoint = new Map(); + private readonly acceptedEndpoints = new Set(); + + take(locator: CodexAppControlLocator): string | undefined { + if (this.acceptedEndpoints.has(locator.endpoint)) return undefined; + const attempts = this.attemptsByEndpoint.get(locator.endpoint) ?? 0; + this.attemptsByEndpoint.set(locator.endpoint, attempts + 1); + return locator.endpoint; + } + + noteAccepted(endpoint: string): void { + if (!this.attemptsByEndpoint.has(endpoint)) { + throw new Error('Cannot accept an unattempted Codex App control endpoint'); + } + this.acceptedEndpoints.add(endpoint); + } + + wasAttempted(endpoint: string): boolean { + return this.attemptsByEndpoint.has(endpoint); + } + + attemptCount(endpoint: string): number { + return this.attemptsByEndpoint.get(endpoint) ?? 0; + } + + wasAccepted(endpoint: string): boolean { + return this.acceptedEndpoints.has(endpoint); + } +} + +/** Read and select one locator endpoint; missing/corrupt files are a poll miss. */ +export function takeCodexAppControlLocatorEndpoint(input: { + locatorPath: string; + sessionId: string; + tracker: CodexAppControlEndpointTracker; + platform?: NodeJS.Platform; + expectedControlRoot?: string; +}): { endpoint: string; epoch: string } | undefined { + const platform = input.platform ?? process.platform; + const locator = readCodexAppControlLocator( + input.locatorPath, + input.sessionId, + platform, + input.expectedControlRoot ?? (platform === 'win32' ? undefined : codexAppPosixControlRoot()), + ); + if (!locator) return undefined; + const endpoint = input.tracker.take(locator); + return endpoint ? { endpoint, epoch: locator.epoch } : undefined; +} + +const PERSISTENT_BACKEND_TYPES: ReadonlySet = new Set(['tmux', 'herdr', 'zellij']); + +/** Missing/corrupt/legacy generations cannot prove a warm reattach. */ +export function shouldColdStartCodexAppReattach(input: { + cliId?: string; + backendType: BackendType; + isReattach: boolean; + persistedState?: CodexAppControlState; +}): boolean { + return input.cliId === 'codex-app' + && input.isReattach + && PERSISTENT_BACKEND_TYPES.has(input.backendType) + && !input.persistedState; +} + +export interface CodexAppControlBootstrap { + path: string; + identity: CodexAppControlIdentity; +} + +export interface ConsumedCodexAppControlBootstrap { + generation: string; + privateKey: KeyObject; + socketPath?: string; + locatorPath?: string; +} + +export type CodexAppControlBootstrapTarget = + | { kind: 'endpoint'; socketPath: string } + | { kind: 'locator'; locatorPath: string }; + +/** Create a fresh asymmetric candidate and one O_EXCL private bootstrap. */ +export function createCodexAppControlBootstrap( + directory: string, + sessionId: string, + target: string | CodexAppControlBootstrapTarget = codexAppControlSocketPath(directory, sessionId), + platform: NodeJS.Platform = process.platform, +): CodexAppControlBootstrap { + ensureCodexAppControlDirectory(directory, platform); + const resolvedTarget: CodexAppControlBootstrapTarget = typeof target === 'string' + ? { kind: 'endpoint', socketPath: target } + : target; + if (resolvedTarget.kind === 'endpoint' + && !isAbsoluteForPlatform(resolvedTarget.socketPath, platform)) { + throw new Error('Codex App control socket path must be absolute'); + } + if (resolvedTarget.kind === 'locator' + && !isAbsoluteForPlatform(resolvedTarget.locatorPath, platform)) { + throw new Error('Codex App control locator path must be absolute'); + } + const generation = randomBytes(32).toString('base64url'); + const pair = generateKeyPairSync('ed25519'); + const publicKey = pair.publicKey.export({ format: 'der', type: 'spki' }).toString('base64url'); + const privateKey = pair.privateKey.export({ format: 'der', type: 'pkcs8' }).toString('base64url'); + const identity: CodexAppControlIdentity = { generation, publicKey, createdAtMs: Date.now() }; + const key = sessionKey(sessionId).slice(0, 16); + const path = joinForPlatform(platform, directory, `${key}.${randomBytes(16).toString('hex')}.bootstrap`); + createExclusiveFile( + path, + JSON.stringify({ + version: CONTROL_BOOTSTRAP_VERSION, + sessionId, + generation, + privateKey, + ...(resolvedTarget.kind === 'endpoint' + ? { socketPath: resolvedTarget.socketPath } + : { locatorPath: resolvedTarget.locatorPath }), + }), + 'Codex App control bootstrap', + platform, + ); + return { path, identity }; +} + +/** Remove crash-orphaned one-shot files for exactly one session generation. */ +export function cleanupStaleCodexAppControlBootstraps( + directory: string, + sessionId: string, + platform: NodeJS.Platform = process.platform, +): void { + ensureCodexAppControlDirectory(directory, platform); + const key = sessionKey(sessionId).slice(0, 16); + const pattern = new RegExp(`^${key}\\.[a-f0-9]{32}\\.bootstrap$`); + for (const name of readdirSync(directory)) { + if (!pattern.test(name)) continue; + try { unlinkSync(joinForPlatform(platform, directory, name)); } catch { /* raced with runner consume */ } + } +} + +/** + * One-shot consume on one O_NOFOLLOW fd. The file is unlinked before its bytes + * are read and the post-unlink link count must be zero. The private key is + * imported here so callers do not retain or forward its encoded form. + */ +export function consumeCodexAppControlBootstrap( + path: string, + expectedSessionId?: string, + platform: NodeJS.Platform = process.platform, +): ConsumedCodexAppControlBootstrap { + if (!path || !isAbsoluteForPlatform(path, platform)) { + throw new Error('Codex App control bootstrap path must be absolute'); + } + let fd: number | undefined; + try { + const policy = codexAppControlFilesystemPolicy(platform); + const beforePath = lstatSync(path); + assertRegularSingleLinkWithinLimit( + beforePath, + BOOTSTRAP_MAX_BYTES, + 'Codex App control bootstrap', + platform, + ); + fd = openSync(path, fsConstants.O_RDONLY | noFollowFlag(platform)); + const before = fstatSync(fd); + assertRegularSingleLinkWithinLimit( + before, + BOOTSTRAP_MAX_BYTES, + 'Codex App control bootstrap', + platform, + ); + if (!sameFileIdentity(beforePath, before)) { + throw new Error('Codex App control bootstrap changed between lstat and open'); + } + unlinkSync(path); + const after = fstatSync(fd); + if (policy.verifyPostUnlinkLinkCount && after.nlink !== 0) { + throw new Error('Codex App control bootstrap was not consumed by unlink'); + } + const decoded = JSON.parse(readFileSync(fd, 'utf8')) as Record; + const hasSocketPath = typeof decoded.socketPath === 'string' + && isAbsoluteForPlatform(decoded.socketPath, platform); + const hasLocatorPath = typeof decoded.locatorPath === 'string' + && isAbsoluteForPlatform(decoded.locatorPath, platform); + const posixLocatorPathValid = platform === 'win32' || !hasLocatorPath + || decoded.locatorPath === codexAppControlLocatorPath( + codexAppPosixControlRoot(), + typeof decoded.sessionId === 'string' ? decoded.sessionId : '', + platform, + ); + if (decoded.version !== CONTROL_BOOTSTRAP_VERSION + || typeof decoded.sessionId !== 'string' + || (expectedSessionId !== undefined && decoded.sessionId !== expectedSessionId) + || !isValidGeneration(decoded.generation) + || typeof decoded.privateKey !== 'string' || !KEY_RE.test(decoded.privateKey) + || hasSocketPath === hasLocatorPath + // Windows always follows its protected locator. Current POSIX workers + // also emit locators; a legacy direct socket bootstrap remains accepted + // because the read-once bootstrap itself is the launch capability. Any + // POSIX locator is nevertheless pinned to the fixed per-UID root here. + || (platform === 'win32' && !hasLocatorPath) + || !posixLocatorPathValid) { + throw new Error('Codex App control bootstrap is invalid'); + } + const privateKey = importPrivateKey(decoded.privateKey); + if (privateKey.asymmetricKeyType !== 'ed25519') throw new Error('Codex App control private key is invalid'); + return { + generation: decoded.generation, + privateKey, + ...(hasSocketPath ? { socketPath: decoded.socketPath as string } : {}), + ...(hasLocatorPath ? { locatorPath: decoded.locatorPath as string } : {}), + }; + } catch (err) { + try { unlinkSync(path); } catch { /* remove a rejected bootstrap/symlink */ } + throw err; + } finally { + if (fd !== undefined) closeSync(fd); + } +} + +export function mergeCodexAppControlCandidate( + existing: CodexAppControlState | undefined, + candidate: CodexAppControlIdentity, + nowMs = Date.now(), +): CodexAppControlState { + if (!validIdentity(candidate)) { + throw new Error('Codex App control candidate is invalid'); + } + const identities = [candidate, ...(existing?.identities ?? [])] + .filter((identity, index, all) => all.findIndex(item => item.generation === identity.generation) === index) + .slice(0, MAX_STATE_IDENTITIES); + return { + version: CONTROL_STATE_VERSION, + status: 'pending', + identities, + updatedAtMs: nowMs, + }; +} + +export function activateCodexAppControlIdentity( + state: CodexAppControlState, + generation: string, + nowMs = Date.now(), +): CodexAppControlState { + const identity = state.identities.find(candidate => candidate.generation === generation); + if (!identity) throw new Error('Codex App control generation is not a persisted candidate'); + return { + version: CONTROL_STATE_VERSION, + status: 'active', + identities: [identity], + updatedAtMs: nowMs, + activatedAtMs: nowMs, + }; +} + +export function generateCodexAppControlChallenge(): string { + return randomBytes(32).toString('base64url'); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return JSON.stringify(value); + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error('non-finite control payload number'); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map(key => ( + `${JSON.stringify(key)}:${canonicalJson(record[key])}` + )).join(',')}}`; + } + throw new Error('unsupported control payload value'); +} + +function signingBytes(domain: 'auth' | 'marker', value: Record): Buffer { + return Buffer.from(`botmux/codex-app/control/${domain}/v2\0${canonicalJson(value)}`, 'utf8'); +} + +function signature(privateKey: KeyObject, domain: 'auth' | 'marker', value: Record): string { + if (privateKey.type !== 'private' || privateKey.asymmetricKeyType !== 'ed25519') { + throw new Error('Codex App control signer must be an Ed25519 private key'); + } + return sign(null, signingBytes(domain, value), privateKey).toString('base64url'); +} + +function signatureValid( + publicKey: string, + signatureValue: string, + domain: 'auth' | 'marker', + value: Record, +): boolean { + if (!KEY_RE.test(publicKey) || !SIGNATURE_RE.test(signatureValue)) return false; + try { + return verify( + null, + signingBytes(domain, value), + importPublicKey(publicKey), + Buffer.from(signatureValue, 'base64url'), + ); + } catch { + return false; + } +} + +export interface CodexAppControlChallenge { + version: typeof CONTROL_WIRE_VERSION; + type: 'challenge'; + sessionId: string; + challenge: string; +} + +export interface CodexAppControlAuth { + version: typeof CONTROL_WIRE_VERSION; + type: 'auth'; + sessionId: string; + generation: string; + challenge: string; + signature: string; +} + +export interface CodexAppControlAccepted { + version: typeof CONTROL_WIRE_VERSION; + type: 'accepted'; + sessionId: string; + generation: string; + challenge: string; + endpointEpoch?: string; +} + +export interface CodexAppControlAck { + version: typeof CONTROL_WIRE_VERSION; + type: 'ack'; + sessionId: string; + generation: string; + challenge: string; + seq: number; +} + +export interface CodexAppSignedControlMarker { + version: typeof CONTROL_WIRE_VERSION; + type: 'marker'; + sessionId: string; + generation: string; + challenge: string; + seq: number; + kind: string; + payload: Record; + signature: string; +} + +export type CodexAppControlWireRecord = + | CodexAppControlChallenge + | CodexAppControlAuth + | CodexAppControlAccepted + | CodexAppControlAck + | CodexAppSignedControlMarker; + +export function encodeCodexAppControlChallenge(sessionId: string, challenge: string): string { + if (!sessionId || !isValidChallenge(challenge)) throw new Error('Codex App control challenge is invalid'); + return JSON.stringify({ version: CONTROL_WIRE_VERSION, type: 'challenge', sessionId, challenge }); +} + +export function encodeCodexAppControlAccepted( + sessionId: string, + generation: string, + challenge: string, + endpointEpoch?: string, +): string { + if (!sessionId || !isValidGeneration(generation) || !isValidChallenge(challenge)) { + throw new Error('Codex App control acceptance metadata is invalid'); + } + if (endpointEpoch !== undefined && !isValidGeneration(endpointEpoch)) { + throw new Error('Codex App control endpoint epoch is invalid'); + } + return JSON.stringify({ + version: CONTROL_WIRE_VERSION, + type: 'accepted', + sessionId, + generation, + challenge, + ...(endpointEpoch ? { endpointEpoch } : {}), + }); +} + +export function encodeCodexAppControlAck( + sessionId: string, + generation: string, + challenge: string, + seq: number, +): string { + if (!sessionId || !isValidGeneration(generation) || !isValidChallenge(challenge) + || !Number.isSafeInteger(seq) || seq <= 0) { + throw new Error('Codex App control acknowledgement metadata is invalid'); + } + return JSON.stringify({ + version: CONTROL_WIRE_VERSION, + type: 'ack', + sessionId, + generation, + challenge, + seq, + }); +} + +function authUnsigned(sessionId: string, generation: string, challenge: string): Record { + return { sessionId, generation, challenge }; +} + +export function encodeCodexAppControlAuth( + privateKey: KeyObject, + sessionId: string, + generation: string, + challenge: string, +): string { + if (!sessionId || !isValidGeneration(generation) || !isValidChallenge(challenge)) { + throw new Error('Codex App control authentication metadata is invalid'); + } + const unsigned = authUnsigned(sessionId, generation, challenge); + return JSON.stringify({ + version: CONTROL_WIRE_VERSION, + type: 'auth', + ...unsigned, + signature: signature(privateKey, 'auth', unsigned), + }); +} + +function markerUnsigned( + sessionId: string, + generation: string, + challenge: string, + seq: number, + kind: string, + payload: Record, +): Record { + return { sessionId, generation, challenge, seq, kind, payload }; +} + +export function encodeCodexAppSignedControlMarker( + privateKey: KeyObject, + sessionId: string, + generation: string, + challenge: string, + seq: number, + kind: string, + payload: Record, +): string { + if (!sessionId || !isValidGeneration(generation) || !isValidChallenge(challenge) + || !Number.isSafeInteger(seq) || seq <= 0 || !kind) { + throw new Error('Codex App control marker metadata is invalid'); + } + const unsigned = markerUnsigned(sessionId, generation, challenge, seq, kind, payload); + return JSON.stringify({ + version: CONTROL_WIRE_VERSION, + type: 'marker', + ...unsigned, + signature: signature(privateKey, 'marker', unsigned), + }); +} + +export function parseCodexAppControlWireRecord(line: string): CodexAppControlWireRecord | undefined { + try { + const value = JSON.parse(line) as Record; + if (value.version !== CONTROL_WIRE_VERSION || typeof value.sessionId !== 'string' || !value.sessionId) return undefined; + if (value.type === 'challenge' && isValidChallenge(value.challenge)) { + return value as unknown as CodexAppControlChallenge; + } + if (value.type === 'auth' && isValidGeneration(value.generation) + && isValidChallenge(value.challenge) + && typeof value.signature === 'string' && SIGNATURE_RE.test(value.signature)) { + return value as unknown as CodexAppControlAuth; + } + if (value.type === 'accepted' && isValidGeneration(value.generation) + && isValidChallenge(value.challenge) + && (value.endpointEpoch === undefined || isValidGeneration(value.endpointEpoch))) { + return value as unknown as CodexAppControlAccepted; + } + if (value.type === 'ack' && isValidGeneration(value.generation) + && isValidChallenge(value.challenge) + && Number.isSafeInteger(value.seq) && Number(value.seq) > 0) { + return value as unknown as CodexAppControlAck; + } + if (value.type === 'marker' && isValidGeneration(value.generation) + && isValidChallenge(value.challenge) + && Number.isSafeInteger(value.seq) && Number(value.seq) > 0 + && typeof value.kind === 'string' && value.kind.length > 0 + && value.payload && typeof value.payload === 'object' && !Array.isArray(value.payload) + && typeof value.signature === 'string' && SIGNATURE_RE.test(value.signature)) { + return value as unknown as CodexAppSignedControlMarker; + } + return undefined; + } catch { + return undefined; + } +} + +export type CodexAppControlRunnerHandshakeAction = + | { type: 'authenticate'; challenge: string } + | { type: 'accepted'; challenge: string } + | { type: 'ack'; seq: number } + | { type: 'reject' }; + +/** + * Pure runner-side handshake state machine. Keeping the phase checks here + * makes repeated challenges, wrong locator epochs, and out-of-order ACKs + * executable in unit tests instead of relying on source-string assertions. + */ +export class CodexAppControlRunnerHandshake { + private phase: 'challenge' | 'accepted' | 'active' = 'challenge'; + private challengeValue: string | undefined; + + constructor( + private readonly expectedSessionId: string, + private readonly expectedGeneration: string, + private readonly expectedEndpointEpoch?: string, + ) {} + + handle( + record: CodexAppControlWireRecord | undefined, + sentThrough: number, + ): CodexAppControlRunnerHandshakeAction { + if (!record || record.sessionId !== this.expectedSessionId) return { type: 'reject' }; + if (this.phase === 'challenge' && record.type === 'challenge') { + this.challengeValue = record.challenge; + this.phase = 'accepted'; + return { type: 'authenticate', challenge: record.challenge }; + } + if (this.phase === 'accepted' + && record.type === 'accepted' + && record.generation === this.expectedGeneration + && record.challenge === this.challengeValue + && (this.expectedEndpointEpoch === undefined + || record.endpointEpoch === this.expectedEndpointEpoch)) { + this.phase = 'active'; + return { type: 'accepted', challenge: record.challenge }; + } + if (this.phase === 'active' + && record.type === 'ack' + && record.generation === this.expectedGeneration + && record.challenge === this.challengeValue + && record.seq > 0 + && record.seq <= sentThrough) { + return { type: 'ack', seq: record.seq }; + } + return { type: 'reject' }; + } + + get active(): boolean { + return this.phase === 'active'; + } +} + +export function verifyCodexAppControlAuth(auth: CodexAppControlAuth, publicKey: string): boolean { + return signatureValid( + publicKey, + auth.signature, + 'auth', + authUnsigned(auth.sessionId, auth.generation, auth.challenge), + ); +} + +export function authenticateCodexAppControlCandidate(input: { + state: CodexAppControlState | undefined; + auth: CodexAppControlAuth; + sessionId: string; + challenge: string; +}): CodexAppControlIdentity | undefined { + if (!input.state + || input.auth.sessionId !== input.sessionId + || input.auth.challenge !== input.challenge) return undefined; + const identity = input.state.identities.find( + candidate => candidate.generation === input.auth.generation, + ); + return identity && verifyCodexAppControlAuth(input.auth, identity.publicKey) + ? identity + : undefined; +} + +export function verifyCodexAppSignedControlMarker( + marker: CodexAppSignedControlMarker, + publicKey: string, +): boolean { + return signatureValid( + publicKey, + marker.signature, + 'marker', + markerUnsigned( + marker.sessionId, + marker.generation, + marker.challenge, + marker.seq, + marker.kind, + marker.payload, + ), + ); +} + +/** + * Marker sequences may begin above one after a worker replacement, but every + * record on one authenticated connection must then be contiguous. This keeps + * a skipped final fragment from being hidden by a later cumulative ACK. + */ +export class CodexAppControlSequenceFence { + private previous: number | undefined; + + accept(seq: number): boolean { + if (!Number.isSafeInteger(seq) || seq <= 0) return false; + if (this.previous !== undefined && seq !== this.previous + 1) return false; + this.previous = seq; + return true; + } +} + +export type CodexAppFinalAssemblyResult = + | { status: 'not-final' } + | { status: 'accepted' } + | { status: 'complete'; payload: Record } + | { status: 'reject'; reason: string }; + +interface CodexAppFinalAssemblyState { + id: string; + total: number; + metadata: Record; + chunks: Buffer[]; + bytes: number; +} + +const FINAL_ID_MAX_BYTES = 512; +const STRICT_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; + +/** + * Per-connection final transaction assembler. Start and chunks deliberately + * remain unacknowledged; only a valid, complete end record is eligible for the + * cumulative ACK. Any gap or interleaving rejects the connection, so the + * runner must re-sign and replay the complete transaction under a new + * challenge. + */ +export class CodexAppControlFinalAssembler { + private active: CodexAppFinalAssemblyState | undefined; + + accept(kind: string, payload: Record): CodexAppFinalAssemblyResult { + const isTransactionKind = kind === 'final-start' || kind === 'final-chunk' || kind === 'final-end'; + if (!isTransactionKind) { + if (kind === 'final' || kind.startsWith('final-')) { + return this.reject(`unsupported final record ${kind}`); + } + return this.active + ? this.reject(`interleaved ${kind} record inside final transaction`) + : { status: 'not-final' }; + } + + const id = typeof payload.id === 'string' ? payload.id : ''; + if (!id || Buffer.byteLength(id, 'utf8') > FINAL_ID_MAX_BYTES) { + return this.reject('invalid final transaction id'); + } + + if (kind === 'final-start') { + const total = typeof payload.total === 'number' && Number.isSafeInteger(payload.total) + ? payload.total + : -1; + if (this.active || total < 0 || total > 2_048) { + return this.reject('invalid or overlapping final-start'); + } + const { id: _id, total: _total, ...metadata } = payload; + this.active = { id, total, metadata, chunks: [], bytes: 0 }; + return { status: 'accepted' }; + } + + const assembly = this.active; + if (!assembly || id !== assembly.id) { + return this.reject('final fragment has no matching start'); + } + + if (kind === 'final-chunk') { + const index = typeof payload.index === 'number' && Number.isSafeInteger(payload.index) + ? payload.index + : -1; + if (index !== assembly.chunks.length || index >= assembly.total + || typeof payload.data !== 'string' || !STRICT_BASE64_RE.test(payload.data)) { + return this.reject('invalid, duplicate, or out-of-order final chunk'); + } + const chunk = Buffer.from(payload.data, 'base64'); + if (chunk.length === 0 || chunk.length > CODEX_APP_CONTROL_FINAL_CHUNK_BYTES) { + return this.reject('final chunk size is invalid'); + } + assembly.bytes += chunk.length; + if (assembly.bytes > CODEX_APP_CONTROL_FINAL_MAX_BYTES) { + return this.reject('final transaction exceeds the byte limit'); + } + assembly.chunks.push(chunk); + return { status: 'accepted' }; + } + + const endTotal = typeof payload.total === 'number' && Number.isSafeInteger(payload.total) + ? payload.total + : -1; + if (endTotal !== assembly.total || assembly.chunks.length !== assembly.total) { + return this.reject('incomplete or inconsistent final-end'); + } + this.active = undefined; + return { + status: 'complete', + payload: { + ...assembly.metadata, + content: Buffer.concat(assembly.chunks, assembly.bytes).toString('utf8'), + }, + }; + } + + clear(): void { + this.active = undefined; + } + + private reject(reason: string): CodexAppFinalAssemblyResult { + this.active = undefined; + return { status: 'reject', reason }; + } +} + +/** + * Per-worker replay window for authenticated runner generations. A runner may + * reconnect after losing an ACK and legitimately re-sign the same sequence + * under the new connection challenge; the worker ACKs that retry without + * applying its lifecycle/final side effects twice. + */ +export class CodexAppControlReplayWindow { + private readonly highWaterByGeneration = new Map(); + + highWater(generation: string): number { + return this.highWaterByGeneration.get(generation) ?? 0; + } + + hasSeen(generation: string, seq: number): boolean { + return seq <= this.highWater(generation); + } + + commit(generation: string, seq: number): void { + if (!isValidGeneration(generation) || !Number.isSafeInteger(seq) || seq <= 0) { + throw new Error('Codex App control replay sequence is invalid'); + } + if (seq > this.highWater(generation)) this.highWaterByGeneration.set(generation, seq); + } + + retainOnly(generation: string): void { + const retained = this.highWaterByGeneration.get(generation); + this.highWaterByGeneration.clear(); + if (retained !== undefined) this.highWaterByGeneration.set(generation, retained); + } +} + +/** Bounded newline decoder; oversized attacker input is discarded until resync. */ +export class CodexAppControlLineDecoder { + private pending = Buffer.alloc(0); + private droppingOversized = false; + + push(chunk: Buffer): { lines: string[]; droppedMalformed: boolean } { + const lines: string[] = []; + let droppedMalformed = false; + let cursor = 0; + while (cursor < chunk.length) { + const nl = chunk.indexOf(0x0a, cursor); + const end = nl >= 0 ? nl : chunk.length; + const piece = chunk.subarray(cursor, end); + cursor = nl >= 0 ? nl + 1 : chunk.length; + + if (this.droppingOversized) { + droppedMalformed = true; + if (nl >= 0) this.droppingOversized = false; + continue; + } + if (this.pending.length + piece.length > CODEX_APP_CONTROL_LINE_MAX_BYTES) { + this.pending = Buffer.alloc(0); + droppedMalformed = true; + if (nl < 0) this.droppingOversized = true; + continue; + } + this.pending = Buffer.concat([this.pending, piece]); + if (nl >= 0) { + const line = this.pending.toString('utf8').trim(); + this.pending = Buffer.alloc(0); + if (line) lines.push(line); + } + } + return { lines, droppedMalformed }; + } + + clear(): void { + this.pending = Buffer.alloc(0); + this.droppingOversized = false; + } +} diff --git a/src/utils/codex-app-dispatch-ledger.ts b/src/utils/codex-app-dispatch-ledger.ts new file mode 100644 index 000000000..2a248f528 --- /dev/null +++ b/src/utils/codex-app-dispatch-ledger.ts @@ -0,0 +1,207 @@ +import type { + CodexAppDispatchLedgerEntry, + CodexAppGenerationCommit, +} from '../types.js'; + +export type CodexAppDispatchIdentity = Pick< + CodexAppDispatchLedgerEntry, + 'dispatchId' | 'turnId' | 'dispatchAttempt' +>; + +function sameIdentity( + left: CodexAppDispatchIdentity, + right: CodexAppDispatchIdentity, +): boolean { + return left.dispatchId === right.dispatchId + && left.turnId === right.turnId + && left.dispatchAttempt === right.dispatchAttempt; +} + +/** Any ledger entry is daemon-owned unfinished work. Lifecycle mutations that + * can replace a worker, pane, cwd, or chat route must reject while this is true; + * explicit session close is the sole abandon operation. */ +export function hasUnsettledCodexAppDispatch( + ledger: readonly CodexAppDispatchLedgerEntry[] | undefined, +): boolean { + return (ledger?.length ?? 0) > 0; +} + +/** + * Authorize a worker-hosted `botmux send` relay against the immutable Codex + * App origin. `requireExact` is true for a live Codex App turn even when the + * ledger has just become empty: settlement winning before watcher admission + * must reject, never downgrade the old capability into an ordinary send. + */ +export function validateCodexAppManagedSendOrigin( + ledger: readonly CodexAppDispatchLedgerEntry[] | undefined, + origin: { turnId?: string; dispatchAttempt?: number }, + requireExact: boolean, +): { ok: true; requiresLedger: boolean } | { ok: false; error: string } { + const entries = ledger ?? []; + if (!requireExact && entries.length === 0) { + return { ok: true, requiresLedger: false }; + } + if (!origin.turnId) { + return { ok: false, error: 'unsettled Codex App output has no live turn identity' }; + } + const sameTurn = entries.filter(entry => entry.turnId === origin.turnId); + const exact = origin.dispatchAttempt === undefined + ? sameTurn + : sameTurn.filter(entry => entry.dispatchAttempt === origin.dispatchAttempt); + if (exact.length !== 1) { + return { + ok: false, + error: `${exact.length} Codex App ledger entries match the live relay origin`, + }; + } + const sink = exact[0]!.deliverySink ?? 'lark'; + if (sink === 'http_wait' || sink === 'http_async' || sink === 'suppressed') { + return { ok: false, error: `Codex App output is bound to ${sink}` }; + } + return { ok: true, requiresLedger: true }; +} + +export function appendAcceptedCodexAppDispatch( + ledger: readonly CodexAppDispatchLedgerEntry[], + entry: Omit, +): CodexAppDispatchLedgerEntry[] { + if (!entry.dispatchId || !entry.turnId) throw new Error('Codex App dispatch identity is incomplete'); + if (ledger.some(candidate => candidate.dispatchId === entry.dispatchId)) { + throw new Error('Codex App dispatch id is already present'); + } + return [...ledger, { ...entry, state: 'accepted' }]; +} + +export function prepareCodexAppDispatch( + ledger: readonly CodexAppDispatchLedgerEntry[], + identity: CodexAppDispatchIdentity, +): { ok: true; ledger: CodexAppDispatchLedgerEntry[] } | { ok: false; error: string } { + const index = ledger.findIndex(entry => entry.dispatchId === identity.dispatchId); + if (index < 0 || !sameIdentity(ledger[index], identity)) { + return { ok: false, error: 'dispatch_not_found' }; + } + if (ledger.slice(0, index).some(entry => entry.state !== 'prepared')) { + return { ok: false, error: 'dispatch_out_of_order' }; + } + if (ledger[index].state === 'prepared') return { ok: true, ledger: [...ledger] }; + const next = ledger.map((entry, candidateIndex) => candidateIndex === index + ? { ...entry, state: 'prepared' as const } + : entry); + return { ok: true, ledger: next }; +} + +/** A worker proved that a prepared write left the runner input untouched (or + * fully flushed invalid), so the exact item may return to accepted and retry in + * the same FIFO without minting a second dispatch. */ +export function retryPreparedCodexAppDispatch( + ledger: readonly CodexAppDispatchLedgerEntry[], + identity: CodexAppDispatchIdentity, +): { ok: true; ledger: CodexAppDispatchLedgerEntry[] } | { ok: false; error: string } { + const index = ledger.findIndex(entry => entry.dispatchId === identity.dispatchId); + if (index < 0 || !sameIdentity(ledger[index], identity)) { + return { ok: false, error: 'dispatch_not_found' }; + } + if (ledger[index].state !== 'prepared') { + return { ok: false, error: 'dispatch_not_prepared' }; + } + if (ledger.slice(index + 1).some(entry => entry.state === 'prepared')) { + return { ok: false, error: 'prepared_successor_exists' }; + } + return { + ok: true, + ledger: ledger.map((entry, candidateIndex) => candidateIndex === index + ? { ...entry, state: 'accepted' as const } + : entry), + }; +} + +export function cancelCodexAppDispatch( + ledger: readonly CodexAppDispatchLedgerEntry[], + identity: CodexAppDispatchIdentity, +): { ok: true; ledger: CodexAppDispatchLedgerEntry[] } | { ok: false; error: string } { + const index = ledger.findIndex(entry => entry.dispatchId === identity.dispatchId); + if (index < 0 || !sameIdentity(ledger[index], identity)) { + return { ok: false, error: 'dispatch_not_found' }; + } + if (ledger.slice(index + 1).some(entry => entry.state === 'prepared')) { + return { ok: false, error: 'prepared_successor_exists' }; + } + return { ok: true, ledger: ledger.filter((_, candidateIndex) => candidateIndex !== index) }; +} + +/** + * Remove one exact VC delivery attempt after its owned CLI backing has been + * authoritatively proved absent. Unlike ordinary cancellation, this does not + * reject a prepared successor: the dead generation can no longer execute the + * fenced attempt, and retaining it would make a replacement restore N beside + * the delivery hub's replayed N+1. + * + * No match is idempotent success because the worker may have durably retired + * the entry before it exited. More than one match is corruption and remains + * fail-closed because the receipt identity does not contain a dispatchId. + */ +export function retireCodexAppDispatchAfterBackingMissing( + ledger: readonly CodexAppDispatchLedgerEntry[], + turnId: string, + dispatchAttempt: number, +): { ok: true; ledger: CodexAppDispatchLedgerEntry[] } | { ok: false; error: string } { + const sameTurnIndexes = ledger.flatMap((entry, index) => entry.turnId === turnId ? [index] : []); + if (sameTurnIndexes.length === 0) return { ok: true, ledger: [...ledger] }; + if (sameTurnIndexes.length !== 1) return { ok: false, error: 'dispatch_identity_ambiguous' }; + const matchIndex = sameTurnIndexes[0]!; + if (ledger[matchIndex]!.dispatchAttempt !== dispatchAttempt) { + return { ok: false, error: 'dispatch_attempt_conflict' }; + } + return { + ok: true, + ledger: ledger.filter((_, index) => index !== matchIndex), + }; +} + +export function settleCodexAppDispatch( + ledger: readonly CodexAppDispatchLedgerEntry[], + commits: readonly CodexAppGenerationCommit[], + identity: CodexAppDispatchIdentity, + generation: string, + seq: number, +): { + ok: true; + ledger: CodexAppDispatchLedgerEntry[]; + commits: CodexAppGenerationCommit[]; + settledEntry: CodexAppDispatchLedgerEntry; + } | { ok: false; error: string } { + const head = ledger[0]; + if (!head || head.state !== 'prepared' || !sameIdentity(head, identity)) { + return { ok: false, error: 'prepared_head_mismatch' }; + } + if (!generation || !Number.isSafeInteger(seq) || seq <= 0) { + return { ok: false, error: 'invalid_control_identity' }; + } + const existing = commits.find(commit => commit.generation === generation)?.committedThrough ?? 0; + const nextCommit = { generation, committedThrough: Math.max(existing, seq) }; + return { + ok: true, + ledger: ledger.slice(1), + settledEntry: { ...head }, + commits: [ + ...commits.filter(commit => commit.generation !== generation), + nextCommit, + ], + }; +} + +export function committedCodexAppSequence( + commits: readonly CodexAppGenerationCommit[], + generation: string, + seq: number, +): boolean { + return commits.some(commit => commit.generation === generation && seq <= commit.committedThrough); +} + +/** A proved fresh runner retires every prior generation and its ACK history. */ +export function retainFreshCodexAppGeneration( + commits: readonly CodexAppGenerationCommit[], + generation: string, +): CodexAppGenerationCommit[] { + return commits.filter(commit => commit.generation === generation); +} diff --git a/src/utils/codex-app-turn-dispatch.ts b/src/utils/codex-app-turn-dispatch.ts new file mode 100644 index 000000000..d8def7803 --- /dev/null +++ b/src/utils/codex-app-turn-dispatch.ts @@ -0,0 +1,196 @@ +/** + * Worker-owned attribution for Codex App's serial runner queue. + * + * The runner can echo a stable clientUserMessageId, but it must never choose + * the daemon delivery attempt. A worker may also finish writing turn N+1 + * before turn N's final transaction arrives, so the mutable "current turn" + * globals are not an attribution boundary. Reserve one immutable entry per + * control-line write and settle final transactions strictly from the head. + */ + +export interface CodexAppTurnDispatchReservation { + handle: number; + dispatchId?: string; + turnId: string; + replyTurnId?: string; + dispatchAttempt?: number; + recovered?: boolean; +} + +export type CodexAppTurnDispatchSettlement = + | { + ok: true; + handle: number; + dispatchId?: string; + turnId: string; + replyTurnId?: string; + dispatchAttempt?: number; + nativeTurnId?: string; + remaining: number; + } + | { + ok: false; + reason: 'no_pending_turn' | 'turn_mismatch' | 'dispatch_attempt_mismatch'; + markerTurnId?: string; + expectedTurnId?: string; + markerDispatchAttempt?: unknown; + expectedDispatchAttempt?: number; + }; + +export class CodexAppTurnDispatchQueue { + private readonly queue: CodexAppTurnDispatchReservation[] = []; + private nextHandle = 1; + + reserve( + turnId: string, + dispatchAttempt?: number, + dispatchId?: string, + recovered = false, + replyTurnId?: string, + ): CodexAppTurnDispatchReservation { + if (!turnId) throw new Error('Codex App dispatch turn id must be non-empty'); + const reservation = { + handle: this.nextHandle++, + ...(dispatchId ? { dispatchId } : {}), + turnId, + ...(replyTurnId ? { replyTurnId } : {}), + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + ...(recovered ? { recovered: true } : {}), + }; + this.queue.push(reservation); + return { ...reservation }; + } + + /** + * A replacement worker cannot recover the old in-memory queue. It may + * recover exactly the daemon-frozen active identity supplied in `init`, and + * only while no locally submitted entry exists. A runner marker still has + * to assert equality before this entry can settle. + */ + recoverWarmReattach( + turnId: string | undefined, + dispatchAttempt?: number, + dispatchId?: string, + replyTurnId?: string, + ): CodexAppTurnDispatchReservation | undefined { + if (!turnId || this.queue.length > 0) return undefined; + return this.reserve(turnId, dispatchAttempt, dispatchId, true, replyTurnId); + } + + restore(entries: ReadonlyArray<{ + dispatchId: string; + turnId: string; + dispatchAttempt?: number; + replyTurnId?: string; + }>): void { + if (this.queue.length > 0) throw new Error('Codex App dispatch queue is already populated'); + for (const entry of entries) { + this.reserve(entry.turnId, entry.dispatchAttempt, entry.dispatchId, true, entry.replyTurnId); + } + } + + recoveredPrefix(): CodexAppTurnDispatchReservation[] { + const prefix: CodexAppTurnDispatchReservation[] = []; + for (const entry of this.queue) { + if (!entry.recovered) break; + prefix.push({ ...entry }); + } + return prefix; + } + + /** Remove the exact write that threw or reported submitted=false. */ + cancelExact(handle: number): boolean { + const index = this.queue.findIndex(entry => entry.handle === handle); + if (index < 0) return false; + this.queue.splice(index, 1); + return true; + } + + markRecovered(handle: number): boolean { + const entry = this.queue.find(candidate => candidate.handle === handle); + if (!entry) return false; + entry.recovered = true; + return true; + } + + findExact( + turnId: string, + dispatchAttempt?: number, + ): CodexAppTurnDispatchReservation | undefined { + const entry = this.queue.find(candidate => candidate.turnId === turnId + && candidate.dispatchAttempt === dispatchAttempt); + return entry ? { ...entry } : undefined; + } + + /** + * Validate and consume one complete final transaction. The head remains in + * place on every rejection, so a stale/mismatched marker cannot steal the + * next turn or advance the FIFO. + */ + settleFinal(payload: { + turnId?: unknown; + nativeTurnId?: unknown; + dispatchAttempt?: unknown; + }, consume = true): CodexAppTurnDispatchSettlement { + const head = this.queue[0]; + if (!head) return { ok: false, reason: 'no_pending_turn' }; + + const markerTurnId = typeof payload.turnId === 'string' && payload.turnId.length > 0 + ? payload.turnId + : undefined; + if (markerTurnId && markerTurnId !== head.turnId) { + return { + ok: false, + reason: 'turn_mismatch', + markerTurnId, + expectedTurnId: head.turnId, + }; + } + + // Attempt identity is worker-owned. A runner may redundantly assert it, + // but an assertion of any other value (including a malformed one) is a + // rejection and can never select another queue entry. + if (payload.dispatchAttempt !== undefined + && payload.dispatchAttempt !== head.dispatchAttempt) { + return { + ok: false, + reason: 'dispatch_attempt_mismatch', + markerDispatchAttempt: payload.dispatchAttempt, + ...(head.dispatchAttempt !== undefined + ? { expectedDispatchAttempt: head.dispatchAttempt } + : {}), + }; + } + + const nativeTurnId = typeof payload.nativeTurnId === 'string' && payload.nativeTurnId.length > 0 + ? payload.nativeTurnId + : undefined; + if (consume) this.queue.shift(); + return { + ok: true, + handle: head.handle, + ...(head.dispatchId ? { dispatchId: head.dispatchId } : {}), + turnId: head.turnId, + ...(head.replyTurnId ? { replyTurnId: head.replyTurnId } : {}), + ...(head.dispatchAttempt !== undefined + ? { dispatchAttempt: head.dispatchAttempt } + : {}), + ...(nativeTurnId ? { nativeTurnId } : {}), + remaining: this.queue.length - (consume ? 0 : 1), + }; + } + + commitExactHead(handle: number): boolean { + if (this.queue[0]?.handle !== handle) return false; + this.queue.shift(); + return true; + } + + size(): number { + return this.queue.length; + } + + clear(): void { + this.queue.length = 0; + } +} diff --git a/src/utils/codex-app-turn-liveness.ts b/src/utils/codex-app-turn-liveness.ts new file mode 100644 index 000000000..0ca44ce82 --- /dev/null +++ b/src/utils/codex-app-turn-liveness.ts @@ -0,0 +1,364 @@ +import type { BackendType } from '../adapters/backend/types.js'; + +/** + * Codex App turn liveness, driven by the app-server runner's explicit turn + * activity markers. This deliberately reports "no observable progress" + * rather than failure: a long-running tool may recover and emit activity + * later, at which point the stalled projection clears without replaying work. + */ + +export const CODEX_APP_NO_PROGRESS_TIMEOUT_MS = 90_000; + +const PERSISTENT_BACKEND_TYPES: ReadonlySet = new Set(['tmux', 'herdr', 'zellij']); + +/** + * Decide whether an existing persistent pane needs a synthetic observation. + * This deliberately does not depend on pipe mode: Zellij reattaches through + * its own PTY (`isPipeMode=false`) but preserves the same running CLI. + */ +export function shouldBeginCodexAppReattachObservation(input: { + cliId?: string; + backendType: BackendType; + isReattach: boolean; +}): boolean { + return input.cliId === 'codex-app' + && input.isReattach + && PERSISTENT_BACKEND_TYPES.has(input.backendType); +} + +export interface CodexAppLivenessPoll { + active: boolean; + stalled: boolean; + /** True only on the working -> stalled edge. */ + newlyStalled: boolean; + /** True at most once for one submitted turn, even if it later recovers. */ + shouldNotify: boolean; + turnId?: string; +} + +export interface CodexAppActivityApplyResult { + accepted: boolean; + phase?: 'submitted' | 'progress' | 'completed'; + /** A previously rejected inter-turn prompt became authoritative. */ + shouldReplayPrompt?: boolean; +} + +export interface CodexAppStateApplyResult { + accepted: boolean; + busy?: boolean; + tracksTurn?: boolean; + /** Signed idle arrived after the tracker's explicit queue drained. */ + shouldPublishReady?: boolean; + atMs?: number; +} + +/** + * Signed runner state is the Codex App ready authority. Terminal prompt bytes + * remain useful as a recovery hint, but PTY/tmux/Herdr/Zellij delivery can be + * delayed or lost and must never publish idle ahead of the signed final queue. + */ +export class CodexAppReadyAuthority { + private signedIdle = false; + private latePromptRecoveryArmed = false; + + reset(): void { + this.signedIdle = false; + this.latePromptRecoveryArmed = false; + } + + beginWork(): void { + this.signedIdle = false; + this.latePromptRecoveryArmed = false; + } + + /** Returns true only for an authenticated runner's idle boundary. */ + noteSignedState(busy: boolean): boolean { + this.latePromptRecoveryArmed = false; + this.signedIdle = !busy; + return this.signedIdle; + } + + canPublishPromptReady(): boolean { + return this.signedIdle; + } + + /** Arm only after the worker cancels the exact local submit slot. */ + armLatePromptRecovery(): void { + this.signedIdle = false; + this.latePromptRecoveryArmed = true; + } + + /** + * Consume one late terminal prompt after the cancelled slot left no tracked + * work. New work, authenticated activity/state, or reset clears the arm. + */ + consumeLatePromptRecovery(trackerEmpty: boolean): boolean { + if (!trackerEmpty || !this.latePromptRecoveryArmed) return false; + this.latePromptRecoveryArmed = false; + return true; + } +} + +/** Apply a state payload from an already authenticated runner connection. */ +export function applyTrustedCodexAppStateMarker( + tracker: CodexAppTurnLiveness, + authority: CodexAppReadyAuthority, + payload: unknown, + receivedAtMs = Date.now(), +): CodexAppStateApplyResult { + if (!payload || typeof payload !== 'object') return { accepted: false }; + const marker = payload as Record; + if (typeof marker.busy !== 'boolean') return { accepted: false }; + const runnerAtMs = typeof marker.atMs === 'number' && Number.isFinite(marker.atMs) + ? Math.min(marker.atMs, receivedAtMs) + : receivedAtMs; + if (marker.busy) { + authority.noteSignedState(true); + // Goal auto-continuations are native work but do not own a Botmux dispatch + // slot. Tracking them as a synthetic user turn would leave an extra slot + // after a later Lark message steers the same native turn and completes. + const tracksTurn = marker.tracksTurn !== false; + if (tracksTurn) tracker.noteSubmitted(runnerAtMs); + else tracker.discardReattachObservation(); + return { accepted: true, busy: true, tracksTurn, shouldPublishReady: false, atMs: runnerAtMs }; + } + authority.noteSignedState(false); + return { + accepted: true, + busy: false, + shouldPublishReady: tracker.notePrompt(runnerAtMs), + atMs: runnerAtMs, + }; +} + +interface ActiveTurn { + handle: number; + turnId?: string; + lastActivityAtMs: number; + stalled: boolean; + notified: boolean; + reattachObservation: boolean; +} + +export class CodexAppTurnLiveness { + private readonly turns: ActiveTurn[] = []; + private nextHandle = 1; + private promptDeferred = false; + + constructor(private readonly timeoutMs = CODEX_APP_NO_PROGRESS_TIMEOUT_MS) { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error('Codex App liveness timeout must be a positive finite number'); + } + } + + /** + * Queue one Botmux input immediately before its control line is submitted. + * Codex App's runner is serial, so only the head turn owns activity/stall + * state; queued turns get a fresh clock when the head completes. + */ + begin(turnId?: string, nowMs = Date.now()): number { + // A prompt deferred for an earlier queue has no authority over a new queue. + if (this.turns.length === 0) this.promptDeferred = false; + const handle = this.nextHandle++; + this.turns.push({ + handle, + turnId, + lastActivityAtMs: nowMs, + stalled: false, + notified: false, + reattachObservation: false, + }); + return handle; + } + + /** Observe an authenticated runner whose in-memory turn state survived reattach. */ + beginReattachObservation(nowMs = Date.now()): number | undefined { + if (this.turns.length > 0) return undefined; + const handle = this.nextHandle++; + this.turns.push({ + handle, + lastActivityAtMs: nowMs, + stalled: false, + notified: false, + reattachObservation: true, + }); + return handle; + } + + /** Drop only the synthetic reconnect slot when signed state identifies work + * as a native Goal continuation rather than a Botmux-owned turn. */ + discardReattachObservation(): void { + if (!this.turns[0]?.reattachObservation) return; + this.turns.shift(); + this.promptDeferred = false; + } + + /** + * Record runner/app-server activity. Activity after a stall makes the turn + * working again, but the same turn will not notify the user a second time. + */ + noteActivity(nowMs = Date.now()): void { + const active = this.turns[0]; + if (!active) return; + // Real work for the head turn supersedes a prompt rendered in the narrow + // inter-turn gap before its chunked control line finished arriving. + this.promptDeferred = false; + // Signed socket records can be buffered behind other process work. Never + // let an older runner timestamp move the worker's clock backwards. + active.lastActivityAtMs = Math.max(active.lastActivityAtMs, nowMs); + active.stalled = false; + } + + /** + * A signed submitted record can be the first event after worker reattach. + * Recover an explicit slot when no local flush state survived the restart. + */ + noteSubmitted(nowMs = Date.now()): void { + if (this.turns.length === 0) this.begin(undefined, nowMs); + else if (this.turns[0].reattachObservation) this.turns[0].reattachObservation = false; + this.noteActivity(nowMs); + } + + /** Complete only the runner's current turn and activate the next queued one. */ + completeCurrent(nowMs = Date.now()): boolean { + if (this.turns.length === 0) return false; + this.turns.shift(); + const next = this.turns[0]; + if (!next) return this.consumeDeferredPrompt(); + // Time spent waiting behind the previous turn is not lack of progress for + // this turn. Its own timeout begins when it becomes runner-current. + next.lastActivityAtMs = Math.max(next.lastActivityAtMs, nowMs); + next.stalled = false; + return false; + } + + /** Remove the exact control-line submission that failed, preserving peers. */ + cancelExact(handle: number, nowMs = Date.now()): { + cancelled: boolean; + shouldReplayPrompt: boolean; + } { + const index = this.turns.findIndex(turn => turn.handle === handle); + if (index < 0) return { cancelled: false, shouldReplayPrompt: false }; + const wasCurrent = index === 0; + this.turns.splice(index, 1); + if (wasCurrent && this.turns[0]) { + this.turns[0].lastActivityAtMs = Math.max(this.turns[0].lastActivityAtMs, nowMs); + this.turns[0].stalled = false; + } + return { cancelled: true, shouldReplayPrompt: this.consumeDeferredPrompt() }; + } + + /** Backwards-compatible shorthand for callers that only need replay state. */ + cancel(handle: number, nowMs = Date.now()): boolean { + return this.cancelExact(handle, nowMs).shouldReplayPrompt; + } + + /** + * A prompt is authoritative only for the synthetic reattach observation. + * Returns whether no explicit/queued turn remains, so the worker can reject + * a transient inter-turn prompt instead of publishing prompt_ready / idle. + */ + notePrompt(nowMs = Date.now()): boolean { + if (this.turns[0]?.reattachObservation) this.completeCurrent(nowMs); + if (this.turns.length === 0) { + this.promptDeferred = false; + return true; + } + this.promptDeferred = true; + return false; + } + + hasActiveTurn(): boolean { + return this.turns.length > 0; + } + + /** Drop every queued turn on CLI exit, kill, or worker reinitialization. */ + clear(): void { + this.turns.length = 0; + this.promptDeferred = false; + } + + poll(nowMs = Date.now()): CodexAppLivenessPoll { + const active = this.turns[0]; + if (!active) return { active: false, stalled: false, newlyStalled: false, shouldNotify: false }; + + const timedOut = nowMs - active.lastActivityAtMs >= this.timeoutMs; + const newlyStalled = timedOut && !active.stalled; + if (newlyStalled) active.stalled = true; + + const shouldNotify = newlyStalled && !active.notified; + if (shouldNotify) active.notified = true; + + return { + active: true, + stalled: active.stalled, + newlyStalled, + shouldNotify, + turnId: active.turnId, + }; + } + + private consumeDeferredPrompt(): boolean { + if (this.turns.length > 0 || !this.promptDeferred) return false; + this.promptDeferred = false; + return true; + } +} + +/** + * One flush may cancel a queued liveness slot because writeInput throws or + * it returns submitted=false. Preserve the deferred real prompt across that + * async boundary, but replay it only after every peer slot has drained. + */ +export class CodexAppFlushPromptReplay { + private replayRequested = false; + + cancelSubmission( + tracker: CodexAppTurnLiveness, + authority: CodexAppReadyAuthority, + handle: number | undefined, + nowMs = Date.now(), + ): void { + if (handle === undefined) return; + const cancelled = tracker.cancelExact(handle, nowMs); + if (!cancelled.cancelled) return; + authority.armLatePromptRecovery(); + if (cancelled.shouldReplayPrompt) this.replayRequested = true; + } + + consumeAfterFlush(tracker: CodexAppTurnLiveness): boolean { + const shouldReplay = this.replayRequested && !tracker.hasActiveTurn(); + this.replayRequested = false; + return shouldReplay; + } +} + +/** Apply an activity payload whose signed socket connection was authenticated. */ +export function applyTrustedCodexAppActivityMarker( + tracker: CodexAppTurnLiveness, + payload: unknown, + receivedAtMs = Date.now(), +): CodexAppActivityApplyResult { + if (!payload || typeof payload !== 'object') return { accepted: false }; + const marker = payload as Record; + const phase = marker.phase; + if (phase !== 'submitted' && phase !== 'progress' && phase !== 'completed') { + return { accepted: false }; + } + const runnerAtMs = typeof marker.atMs === 'number' && Number.isFinite(marker.atMs) + ? marker.atMs + : receivedAtMs; + // Runner and worker share one host clock. A marker may arrive late, but it + // may never push the activity clock into the future and suppress stalls. + const atMs = Math.min(runnerAtMs, receivedAtMs); + if (phase === 'completed') { + return { + accepted: true, + phase, + shouldReplayPrompt: tracker.completeCurrent(atMs), + }; + } + if (phase === 'submitted') tracker.noteSubmitted(atMs); + else tracker.noteActivity(atMs); + return { accepted: true, phase }; +} diff --git a/src/utils/pending-input-queue.ts b/src/utils/pending-input-queue.ts index 574767381..17a5d9fc3 100644 --- a/src/utils/pending-input-queue.ts +++ b/src/utils/pending-input-queue.ts @@ -7,7 +7,10 @@ export interface PendingCliInput { * receives `content`. */ logicalContent?: string; turnId?: string; + replyTurnId?: string; dispatchAttempt?: number; + codexAppDispatchId?: string; + queuedActivationToken?: string; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; codexAppInput?: CodexAppTurnInput; } @@ -24,6 +27,8 @@ export function mergeQueuedCliInput( // per-message attribution/context, so concatenating only their visible text // would drop or mis-attach the sidecar. if (tail.dispatchAttempt !== undefined || next.dispatchAttempt !== undefined + || tail.codexAppDispatchId || next.codexAppDispatchId + || tail.queuedActivationToken || next.queuedActivationToken || tail.vcMeetingImTurnOrigin || next.vcMeetingImTurnOrigin || tail.codexAppInput || next.codexAppInput || tail.logicalContent || next.logicalContent) return false; @@ -55,10 +60,11 @@ export function shouldDeferArgsBakedDurablePrompt(opts: { passesInitialPromptViaArgs: boolean; adoptMode: boolean; dispatchAttempt?: number; + queuedActivationToken?: string; }): boolean { return opts.passesInitialPromptViaArgs && !opts.adoptMode - && opts.dispatchAttempt !== undefined; + && (opts.dispatchAttempt !== undefined || !!opts.queuedActivationToken); } /** Some backends (tmux in particular) reject long launch command strings before @@ -114,6 +120,8 @@ export function shouldStopPendingBatch( ): boolean { return written.dispatchAttempt !== undefined || next?.dispatchAttempt !== undefined + || !!written.queuedActivationToken + || !!next?.queuedActivationToken || !!written.vcMeetingImTurnOrigin || !!next?.vcMeetingImTurnOrigin; } diff --git a/src/worker.ts b/src/worker.ts index 21c31bcb7..9779f8a9a 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -12,8 +12,8 @@ * 6. On 'close', kills CLI and exits * 7. On 'restart', kills CLI and re-spawns with --resume */ -import { randomBytes } from 'node:crypto'; -import { mkdirSync, writeFileSync, unlinkSync, rmdirSync, existsSync, statSync, lstatSync, readdirSync, readlinkSync, readFileSync, realpathSync, copyFileSync, watch as fsWatch, createWriteStream, openSync, closeSync, fstatSync, constants as fsConstants, type FSWatcher, type WriteStream } from 'node:fs'; +import { createHash, randomBytes } from 'node:crypto'; +import { chmodSync, mkdirSync, writeFileSync, unlinkSync, rmdirSync, existsSync, statSync, lstatSync, readdirSync, readlinkSync, readFileSync, realpathSync, copyFileSync, watch as fsWatch, createWriteStream, openSync, closeSync, fstatSync, constants as fsConstants, type FSWatcher, type WriteStream } from 'node:fs'; import { atomicWriteFileSync } from './utils/atomic-write.js'; import { join, basename, dirname, delimiter } from 'node:path'; import { homedir, tmpdir } from 'node:os'; @@ -34,6 +34,7 @@ import { } from './adapters/cli/read-isolation.js'; import { buildFsPolicy, compileToSeatbelt, migrateLegacySandboxFields } from './adapters/cli/fs-policy.js'; import { killPersistentBackendTarget, killPersistentSession, probePersistentBackendTarget, type PersistentBackendType } from './core/persistent-backend.js'; +import { finalizeRawCommandDelivery, writeRawCommandLine } from './core/raw-command-writer.js'; import { readProcessStartIdentity } from './core/session-marker.js'; import { drainTranscript, joinAssistantText, trailingAssistantText, findJsonlContainingFingerprint, findJsonlsContainingExactContent, findLatestJsonl, extractLastAssistantTurn, stringifyUserContent, extractTurnStartText, splitTranscriptEventsByCutoff, type TranscriptEvent } from './services/claude-transcript.js'; import { BridgeTurnQueue, makeFingerprint, normaliseForFingerprint } from './services/bridge-turn-queue.js'; @@ -136,11 +137,14 @@ import { extractKiroSessionIdFromOutput } from './services/kiro-session.js'; import { baselineJsonlCursor } from './services/jsonl-cursor.js'; import { fileURLToPath } from 'node:url'; import { createServer as createHttpServer, type IncomingMessage } from 'node:http'; +import { createServer as createNetServer, type Server as NetServer, type Socket } from 'node:net'; import { WebSocketServer, WebSocket } from 'ws'; import { listenWebTerminalWithFallback } from './utils/web-terminal-listen.js'; import { HerdrWebTerminalBinding } from './utils/herdr-web-terminal-binding.js'; import { TERMINAL_FAVICON_DATA_URI } from './utils/terminal-favicon.js'; import type { + CodexAppDispatchLedgerEntry, + CodexAppGenerationCommit, CodexAppTurnInput, DaemonToWorker, WorkerToDaemon, @@ -225,7 +229,6 @@ import { decideSubmitConfirmationAction, type SubmitActivityEvidence } from './s import { config, resolveChatBotDiscoveryConfig } from './config.js'; import * as sessionStore from './services/session-store.js'; import * as pty from 'node-pty'; -import { createHash } from 'node:crypto'; import { hasInstalledSessionReadyHook, installHook, @@ -260,6 +263,68 @@ import { CodexRpcEngine } from './codex-rpc-engine.js'; // dirs. See SESSION_CLI_HOME_ENV_KEYS for why deleting beats pinning a // default (~/.claude relocates Claude's state file → onboarding rerun) and // why GROK_HOME is exempt. +import { + applyTrustedCodexAppActivityMarker, + applyTrustedCodexAppStateMarker, + CODEX_APP_NO_PROGRESS_TIMEOUT_MS, + CodexAppFlushPromptReplay, + CodexAppReadyAuthority, + CodexAppTurnLiveness, +} from './utils/codex-app-turn-liveness.js'; +import { CodexAppTurnDispatchQueue } from './utils/codex-app-turn-dispatch.js'; +import { + committedCodexAppSequence, + validateCodexAppManagedSendOrigin, +} from './utils/codex-app-dispatch-ledger.js'; +import { + CODEX_APP_CONTROL_BOOTSTRAP_ENV, + CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS, + CodexAppControlFinalAssembler, + CodexAppControlLineDecoder, + CodexAppControlProofDeadline, + CodexAppControlRecordApplicationGate, + CodexAppControlReplayWindow, + CodexAppControlSequenceFence, + activateCodexAppControlIdentity, + acquireCodexAppControlOwnerLease, + acquireCodexAppPosixOwnerLease, + armCodexAppControlStartupTimeout, + authenticateCodexAppControlCandidate, + bindThenPublishCodexAppControlLocator, + cleanupStaleCodexAppControlBootstraps, + codexAppSignedStateReadiness, + codexAppControlLocatorPath, + codexAppPosixControlRoot, + codexAppControlStatePath, + codexAppWindowsOwnerPipeEndpoint, + codexAppWindowsControlRoot, + createCodexAppControlBootstrap, + encodeCodexAppControlAck, + encodeCodexAppControlAccepted, + encodeCodexAppControlChallenge, + ensureCodexAppControlDirectory, + generateCodexAppControlEpoch, + generateCodexAppControlChallenge, + generateCodexAppPosixSocketEndpoint, + generateCodexAppWindowsPipeEndpoint, + hardenWindowsCodexAppControlFile, + mergeCodexAppControlCandidate, + parseCodexAppControlWireRecord, + projectCodexAppControlReadinessStatus, + readCodexAppControlState, + shouldColdStartCodexAppReattach, + shouldFailCodexAppControlChannel, + verifyCodexAppSignedControlMarker, + writeCodexAppControlLocator, + writeCodexAppControlState, + type CodexAppControlState, + type CodexAppControlIdentity, + type CodexAppPosixOwnerLease, + type CodexAppSignedControlMarker, +} from './utils/codex-app-control.js'; + +// Never inherit a session-level CLI home from the daemon's launch environment. +// Each worker re-pins the current bot's home after this scrub. scrubSessionCliHomeEnv(process.env); // ─── State ─────────────────────────────────────────────────────────────────── @@ -734,6 +799,16 @@ async function engageCodexRpc(cfg: Extract): P // would permanently block every later turn's drain — the notify + Web // terminal are the authoritative recovery surface (Codex P1). if (shouldPreMarkFirstTurn(first)) codexBridgeMarkPendingTurn(cfg.prompt, cfg.turnId); + if (first === 'accepted' && cfg.queuedActivationToken) { + // Fresh RPC input bypasses pendingMessages/flushPending entirely. The + // app-server's accepted turn/start (or positive rollout proof) is the + // durable submission boundary for the daemon's queued-opening journal. + send({ + type: 'queued_activation_submitted', + sessionId: cfg.sessionId, + activationToken: cfg.queuedActivationToken, + }); + } outcome = first; // 'accepted' | 'ambiguous' — both stay engaged, prompt never re-queued } codexRpcEngine = engine; @@ -798,6 +873,15 @@ function armRpcStartupDialogDismiss(): void { }; rpcDialogDismissTimer = setTimeout(tick, 2500); } + +let cliSpawnGeneration = 0; + +class CliSpawnSupersededError extends Error { + constructor() { + super('CLI spawn was superseded by a newer lifecycle operation'); + this.name = 'CliSpawnSupersededError'; + } +} let cliPidMarker: string | null = null; // path to .botmux-cli-pids/ let seatbeltProfilePath: string | null = null; // per-session Seatbelt .sb profile to rm at exit (external-wrapper read isolation) let sandboxStopWatcher: (() => void) | null = null; // stop fn for the sandbox outbox watcher @@ -1235,7 +1319,7 @@ const READY_SIGNAL_TIMEOUT_MS = 45_000; const FIRST_PROMPT_TIMEOUT_MS = 15_000; /** Hard cap for startup screens that outlive the soft fallback. Prevents a * changed/missing readyPattern from trapping the first queued input forever. */ -const FIRST_PROMPT_HARD_TIMEOUT_MS = 90_000; +const FIRST_PROMPT_HARD_TIMEOUT_MS = CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS; /** Epoch ms of the most recent PTY output — used to settle for quiescence * before the first flush (see settleThenFlush). */ let lastPtyOutputAtMs = 0; @@ -1336,46 +1420,12 @@ const COCO_SLASH_TYPE_THROTTLE_MS = 40; * the match before submit) → a separate Enter. Non-TUI backends fall back to a * single write + CR. Shared by the `raw_input` IPC handler and runStartupCommands * so both stay in lockstep. */ -async function sendRawCommandLine(be: NonNullable, content: string): Promise { - if ('sendText' in be && 'sendSpecialKeys' in be) { - if (lastInitConfig?.cliId === 'coco') { - // CoCo (Trae CLI, Ink TUI) detects "several bytes in one PTY read = paste", - // so a one-shot sendText('/model') lands as PASTED text: command mode + the - // slash picker never activate and the trailing Enter submits `/model` to the - // model (the "/model 多一个换行" bug). Fix: type char-by-char (throttled) so - // each char is a distinct keystroke that opens command mode, and append ONE - // trailing space to a bare command so the suggestion popup is dismissed. - // Without that dismissal the popup captures the first Enter (CoCo then needs - // two), and for an interactive command like /model — which opens a model - // selector — a stray second Enter would confirm whatever model is highlighted. - // Popup gone ⇒ exactly one Enter executes (/model just opens the selector and - // waits). trim() first so a trailing newline carried from the Lark message - // isn't typed as a literal newline that re-breaks command detection. - const cmd = content.trim(); - const typed = cmd.includes(' ') ? cmd : `${cmd} `; - for (const ch of typed) { - (be as any).sendText(ch); - await new Promise(r => setTimeout(r, COCO_SLASH_TYPE_THROTTLE_MS)); - } - await new Promise(r => setTimeout(r, 200)); - (be as any).sendSpecialKeys('Enter'); - return; - } - (be as any).sendText(content); - await new Promise(r => setTimeout(r, 200)); - (be as any).sendSpecialKeys('Enter'); - } else { - // PtyBackend has no sendText/sendSpecialKeys, so write the keystrokes - // directly — but still beat between the text and the Enter. Writing - // `content + '\r'` in one chunk submits before the CLI's slash-command - // parser has registered the `/cmd` match, so the command is left - // unsent in the input box (observed with `/goal ` on a pty - // workflow worker: typed but never executed). Mirror the tmux path's - // 200ms beat. - be.write(content); - await new Promise(r => setTimeout(r, 200)); - be.write('\r'); - } +async function sendRawCommandLine(be: NonNullable, content: string): Promise { + return writeRawCommandLine(be, content, { + coco: lastInitConfig?.cliId === 'coco', + cocoThrottleMs: COCO_SLASH_TYPE_THROTTLE_MS, + submitBeatMs: 200, + }); } /** Serialize only the literal command-line write window (text -> beat -> Enter). @@ -1384,14 +1434,14 @@ async function sendRawCommandLine(be: NonNullable, content: stri * from splicing keystrokes into one another. */ let commandLineWriteTail: Promise = Promise.resolve(); let commandLineWritesPending = 0; -async function sendRawCommandLineSerially(be: NonNullable, content: string): Promise { +async function sendRawCommandLineSerially(be: NonNullable, content: string): Promise { const previous = commandLineWriteTail; let release!: () => void; commandLineWriteTail = new Promise(resolve => { release = resolve; }); commandLineWritesPending += 1; await previous; try { - await sendRawCommandLine(be, content); + return await sendRawCommandLine(be, content); } finally { commandLineWritesPending -= 1; release(); @@ -1435,7 +1485,11 @@ async function runStartupCommands(): Promise { for (const cmd of cmds) { if (!backend) break; try { - await sendRawCommandLineSerially(backend, cmd); + const accepted = await sendRawCommandLineSerially(backend, cmd); + if (!accepted) { + log(`Startup command skipped (backend rejected write): ${cmd}`); + continue; + } await awaitPtyQuiescence(STARTUP_CMD_QUIET_MS, STARTUP_CMD_CAP_MS); log(`Startup command sent: ${cmd}`); } catch (e: any) { @@ -1448,6 +1502,8 @@ async function runStartupCommands(): Promise { } const pendingMessages: PendingCliInput[] = []; +/** Async init must materialize its opening input before follow-ups may flush. */ +let initPromptMaterialized = false; /** Literal commands that arrived while native /rename owned the TUI or while * an owned CLI restart was fenced. Normal raw_input commands are still * delivered immediately (including while busy). */ @@ -1529,25 +1585,46 @@ async function deliverRawInput(msg: Extract Enter window cannot be interrupted by the follow-up. - if (sent && msg.followUpContent) { - sendToPty(msg.followUpContent, msg.followUpTurnId, { - codexAppInput: msg.followUpCodexAppInput, - }); - log(`Enqueued follow-up after raw input (${msg.followUpContent.length} chars)`); + finalizeRawCommandDelivery({ + accepted: sent, + durableActivation: !!msg.queuedActivationToken, + acknowledgeActivation: !!msg.queuedActivationToken, + hasFollowUp: !!msg.followUpContent, + onAccepted: () => { + isPromptReady = false; + idleDetector?.reset(); + log(`Passthrough slash command: ${msg.content}`); + }, + // Follow-up rides on the same IPC and is enqueued only after this command's + // Enter lands. sendToPty also observes commandLineWritesPending, so another + // raw command's text -> Enter window cannot be interrupted by the follow-up. + onFollowUp: () => { + sendToPty(msg.followUpContent!, msg.followUpTurnId, { + codexAppInput: msg.followUpCodexAppInput, + }); + log(`Enqueued follow-up after raw input (${msg.followUpContent!.length} chars)`); + }, + // Pending-repo raw openings are durable too. ACK only after text + Enter. + onActivationAck: () => send({ + type: 'queued_activation_submitted', + sessionId, + activationToken: msg.queuedActivationToken!, + }), + onDurableFailure: () => { + isPromptReady = false; + log(`Durable raw activation write rejected; retiring worker generation`); + void sendFatalWorkerErrorAndExit( + new Error('durable raw activation was not accepted by the backend'), + msg.turnId, + ); + }, + }); + if (!sent && !msg.queuedActivationToken) { + log(`Passthrough slash command was not accepted by the backend: ${msg.content}`); } // A pending /rename may have been held by the command-write mutex. It still // waits for a genuine prompt because isPromptReady was cleared above. @@ -1717,7 +1794,14 @@ function revokeManagedTurnOriginForTerminal( } function authorizeManagedSend( claim: { capability?: string }, -): { ok: true; origin: { turnId?: string; dispatchAttempt?: number } } | { ok: false; error: string } { +): { + ok: true; + origin: { + turnId?: string; + dispatchAttempt?: number; + requiresCodexAppLedger?: boolean; + }; +} | { ok: false; error: string } { if (!sessionId) return { ok: false, error: 'VC policy cannot resolve session id' }; const dataDir = process.env.SESSION_DATA_DIR; if (!dataDir) return { ok: false, error: 'VC policy cannot resolve session data' }; @@ -1729,7 +1813,22 @@ function authorizeManagedSend( if (!capability || !claim.capability || claim.capability !== capability.token) { return { ok: false, error: 'origin_mismatch: relay capability is stale or missing' }; } - const session = sessionStore.getSession(sessionId); + // The daemon owns and rotates this ledger in another process. Never consult + // SessionStore's worker-local cache here: it was loaded at init and would + // stale-authorize turn N or reject turn N+1 after daemon settlement. + const session = sessionStore.getSessionFresh(sessionId); + const ledger = session?.codexAppDispatchLedger ?? []; + const codexAppManagedOrigin = lastInitConfig?.cliId === 'codex-app' + || session?.cliId === 'codex-app'; + const codexLedgerDecision = validateCodexAppManagedSendOrigin( + ledger, + capability, + codexAppManagedOrigin, + ); + if (!codexLedgerDecision.ok) { + return { ok: false, error: `origin_mismatch: ${codexLedgerDecision.error}` }; + } + const requiresCodexAppLedger = codexLedgerDecision.requiresLedger; const currentImOrigin = currentVcMeetingImTurnOrigin; const imOrigin = currentImOrigin?.larkMessageId === capability.turnId && currentImOrigin?.receiverSessionId === sessionId @@ -1750,6 +1849,7 @@ function authorizeManagedSend( ...(capability.dispatchAttempt !== undefined ? { dispatchAttempt: capability.dispatchAttempt } : {}), + ...(requiresCodexAppLedger ? { requiresCodexAppLedger: true } : {}), }, } : { ok: false, error: `${decision.errorCode}: ${decision.error}` }; @@ -1797,9 +1897,971 @@ function writeCliPidMarker(): void { } } } -let lastStructuredBridgeActivityAtMs = 0; - -type RuntimeScreenStatus = Exclude; +let lastStructuredBridgeActivityAtMs = 0; +const codexAppTurnLiveness = new CodexAppTurnLiveness(); +const codexAppReadyAuthority = new CodexAppReadyAuthority(); +const codexAppTurnDispatchQueue = new CodexAppTurnDispatchQueue(); +let codexAppFallbackTurnSequence = 0; +let codexAppRecoveredDispatches: CodexAppDispatchLedgerEntry[] = []; +let codexAppGenerationCommits: CodexAppGenerationCommit[] = []; +const codexAppPendingDaemonAcks = new Map void; + timer: NodeJS.Timeout; +}>(); +let codexAppCompletionAwaitingFinal = false; +let codexAppControlBootstrapPathForSpawn: string | undefined; +let codexAppControlStateValue: CodexAppControlState | undefined; +let codexAppControlProven = false; +/** Authentication proves identity, not app-server readiness. These become true + * only after the active generation publishes a valid signed state marker. */ +let codexAppSignedStateObserved = false; +let codexAppInputReady = false; +let codexAppControlFatal = false; +let codexAppControlPersistentGeneration = false; +let codexAppControlDirectoryForSpawn: string | undefined; +let codexAppControlSocketPathValue: string | undefined; +let codexAppControlSocketDirectory: string | undefined; +let codexAppControlLocatorPathValue: string | undefined; +let codexAppControlEndpointEpoch: string | undefined; +let codexAppControlServer: NetServer | undefined; +let codexAppWindowsOwnerLeaseServer: NetServer | undefined; +let codexAppWindowsOwnerLeaseSessionId: string | undefined; +let codexAppWindowsOwnerLeasePromise: Promise | undefined; +let codexAppPosixOwnerLease: CodexAppPosixOwnerLease | undefined; +let codexAppPosixOwnerLeaseSessionId: string | undefined; +let codexAppPosixOwnerLeasePromise: Promise | undefined; +let codexAppControlActiveSocket: Socket | undefined; +let codexAppControlActiveIdentity: CodexAppControlIdentity | undefined; +let codexAppFreshCandidateGeneration: string | undefined; +let codexAppUnprovenPromptDeferred = false; +let codexAppRejectedControlLogged = false; +let codexAppMalformedControlLogged = false; +let codexAppBootstrapCleanupTimer: NodeJS.Timeout | undefined; +const codexAppProofDeadline = new CodexAppControlProofDeadline(); +let codexAppControlStopping = false; +let codexAppControlChannelId = 0; +let codexAppControlRotation: Promise | undefined; +const CODEX_APP_CONTROL_ENDPOINT_RETRY_MS = 5_000; + +interface CodexAppControlConnection { + socket: Socket; + endpoint: string; + epoch: string; + channelId: number; + challenge: string; + decoder: CodexAppControlLineDecoder; + sequenceFence: CodexAppControlSequenceFence; + finalAssembler: CodexAppControlFinalAssembler; + authenticated: boolean; + pendingLines: string[]; + processingLines: boolean; + identity?: CodexAppControlIdentity; + authTimer: NodeJS.Timeout; +} +const codexAppControlConnections = new Map(); +const codexAppControlReplayWindow = new CodexAppControlReplayWindow(); +const codexAppControlRecordApplicationGate = new CodexAppControlRecordApplicationGate(); + +function cleanupCodexAppControlBootstrap(): void { + if (codexAppBootstrapCleanupTimer) { + clearTimeout(codexAppBootstrapCleanupTimer); + codexAppBootstrapCleanupTimer = undefined; + } + const path = codexAppControlBootstrapPathForSpawn; + codexAppControlBootstrapPathForSpawn = undefined; + if (!path) return; + try { unlinkSync(path); } catch { /* runner normally unlinks it first */ } +} + +function readPersistedCodexAppControlState( + cfg: Extract, +): CodexAppControlState | undefined { + const dataDir = process.env.SESSION_DATA_DIR; + if (!dataDir && process.platform !== 'win32') return undefined; + const statePath = codexAppControlStatePath(dataDir ?? '', cfg.sessionId); + if (process.platform === 'win32') { + ensureCodexAppControlDirectory(codexAppWindowsControlRoot()); + ensureCodexAppControlDirectory(dirname(statePath)); + if (existsSync(statePath)) hardenWindowsCodexAppControlFile(statePath); + } + return readCodexAppControlState(statePath); +} + +function persistCodexAppControlState( + cfg: Extract, + state: CodexAppControlState, +): void { + const dataDir = process.env.SESSION_DATA_DIR; + if (!dataDir && process.platform !== 'win32') { + throw new Error('SESSION_DATA_DIR is required for persistent Codex App control state'); + } + const statePath = codexAppControlStatePath(dataDir ?? '', cfg.sessionId); + if (process.platform === 'win32') { + ensureCodexAppControlDirectory(codexAppWindowsControlRoot()); + ensureCodexAppControlDirectory(dirname(statePath)); + } + writeCodexAppControlState(statePath, state); +} + +function stopCodexAppControlChannel( + opts: { preserveDispatchRecovery?: boolean } = {}, +): void { + codexAppControlStopping = true; + // Attribution belongs to exactly one worker/control generation. A fresh or + // replacement worker may recover only the daemon-frozen active identity + // after the old runner proves a warm reattach; stale queued entries must not + // cross a stop/restart boundary. + if (!opts.preserveDispatchRecovery) { + codexAppTurnDispatchQueue.clear(); + codexAppRecoveredDispatches = []; + } + for (const pending of codexAppPendingDaemonAcks.values()) { + clearTimeout(pending.timer); + pending.resolve(false); + } + codexAppPendingDaemonAcks.clear(); + codexAppControlChannelId++; + codexAppControlRotation = undefined; + codexAppProofDeadline.clear(); + codexAppSignedStateObserved = false; + codexAppInputReady = false; + for (const connection of codexAppControlConnections.values()) { + clearTimeout(connection.authTimer); + connection.socket.destroy(); + } + codexAppControlConnections.clear(); + codexAppControlActiveSocket = undefined; + codexAppControlActiveIdentity = undefined; + const server = codexAppControlServer; + codexAppControlServer = undefined; + try { server?.close(); } catch { /* worker exit/restart */ } + const socketPath = codexAppControlSocketPathValue; + // Named pipes are kernel objects, not filesystem entries. Never lstat, + // chmod, or unlink them on Windows; closing the server retires the endpoint. + if (socketPath && process.platform !== 'win32') { + try { + const stat = lstatSync(socketPath); + const uid = process.geteuid?.() ?? process.getuid?.(); + if (stat.isSocket() && !stat.isSymbolicLink() && (uid === undefined || stat.uid === uid)) { + unlinkSync(socketPath); + } + } catch { /* absent or already removed */ } + } + // Do not read-then-unlink the fixed locator here. That is not an + // atomic compare-and-delete: a replacement process could publish a new epoch + // between those operations. A stale locator is harmless (the random pipe is + // closed and the runner validates its independent epoch) and the next owner + // overwrites it atomically after binding a fresh endpoint. + codexAppControlEndpointEpoch = undefined; +} + +function failCodexAppControlGeneration(reason: string): void { + if (codexAppControlFatal) return; + codexAppControlFatal = true; + log(`Codex App control generation failed closed: ${reason}`); + codexAppControlProven = false; + codexAppSignedStateObserved = false; + codexAppInputReady = false; + codexAppTurnLiveness.clear(); + codexAppReadyAuthority.reset(); + cleanupCodexAppControlBootstrap(); + stopCodexAppControlChannel(); + const cfg = lastInitConfig; + if (cfg && effectiveBackendType !== 'pty') { + const name = effectiveBackendType === 'tmux' + ? TmuxBackend.sessionName(cfg.sessionId) + : effectiveBackendType === 'zellij' + ? ZellijBackend.sessionName(cfg.sessionId) + : effectiveBackendType === 'herdr' + ? HerdrBackend.sessionName(cfg.sessionId) + : undefined; + if (name) { + try { killPersistentSession(effectiveBackendType as PersistentBackendType, name); } + catch (err: any) { log(`Failed to kill rejected Codex App generation: ${err?.message ?? err}`); } + } + } + try { backend?.kill(); } catch { /* process exit is the final fail-close */ } + backend = null; + queueMicrotask(() => { + void sendFatalWorkerErrorAndExit( + new Error(reason), + undefined, + undefined, + { hardExit: true }, + ); + }); +} + +function activateCodexAppControlConnection( + connection: CodexAppControlConnection, + identity: CodexAppControlIdentity, +): void { + const cfg = lastInitConfig; + const state = codexAppControlStateValue; + if (!cfg || !state) { + connection.socket.destroy(); + return; + } + const proofKind = identity.generation === codexAppFreshCandidateGeneration + ? 'fresh runner' + : 'warm reattach'; + if (proofKind === 'fresh runner' + && codexAppRecoveredDispatches.some(entry => entry.state === 'prepared')) { + // A fresh process cannot prove whether the prior generation buffered a + // prepared frame. Fail before persisting/announcing this generation or + // ACKing authentication; otherwise the daemon may trim recovery state and + // observers can briefly treat an unusable runner as active. + connection.socket.destroy(); + failCodexAppControlGeneration( + 'Fresh Codex App runner cannot adopt a recovered prepared dispatch', + ); + return; + } + let active: CodexAppControlState; + try { + active = activateCodexAppControlIdentity(state, identity.generation); + if (codexAppControlPersistentGeneration) persistCodexAppControlState(cfg, active); + } catch (err: any) { + failCodexAppControlGeneration(`Could not persist authenticated Codex App generation: ${err?.message ?? err}`); + return; + } + codexAppControlStateValue = active; + codexAppControlProven = true; + codexAppSignedStateObserved = false; + codexAppInputReady = false; + if (!codexAppProofDeadline.armed) { + codexAppProofDeadline.arm(() => { + failCodexAppControlGeneration( + `Authenticated Codex App runner did not publish signed state within ${CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS / 1000} seconds`, + ); + }); + } + codexAppControlActiveIdentity = identity; + codexAppControlReplayWindow.retainOnly(identity.generation); + connection.authenticated = true; + connection.identity = identity; + clearTimeout(connection.authTimer); + const previous = codexAppControlActiveSocket; + codexAppControlActiveSocket = connection.socket; + if (previous && previous !== connection.socket) previous.destroy(); + cleanupCodexAppControlBootstrap(); + connection.socket.write(`${encodeCodexAppControlAccepted( + cfg.sessionId, + identity.generation, + connection.challenge, + connection.epoch, + )}\n`); + log(`Authenticated Codex App ${proofKind} by Ed25519 challenge proof (generation=${identity.generation.slice(0, 8)})`); + send({ + type: 'codex_app_generation_active', + sessionId: cfg.sessionId, + generation: identity.generation, + fresh: proofKind === 'fresh runner', + }); + if (proofKind === 'fresh runner') { + codexAppGenerationCommits = codexAppGenerationCommits.filter( + commit => commit.generation === identity.generation, + ); + } + if (proofKind === 'warm reattach') { + codexAppTurnLiveness.beginReattachObservation(); + } + // Authentication is not an input-ready barrier. A warm runner may still + // replay old final/state records immediately after auth. Only the later + // signed idle record, after recovered-prefix reconciliation, may flush. +} + +async function handleCodexAppControlLine( + connection: CodexAppControlConnection, + line: string, +): Promise { + const record = parseCodexAppControlWireRecord(line); + const cfg = lastInitConfig; + if (!record || !cfg || record.sessionId !== cfg.sessionId + || connection.channelId !== codexAppControlChannelId + || connection.epoch !== codexAppControlEndpointEpoch) { + rejectCodexAppControlMarker('malformed socket'); + connection.socket.destroy(); + return; + } + if (!connection.authenticated) { + if (record.type !== 'auth' || record.challenge !== connection.challenge) { + rejectCodexAppControlMarker(record.type); + connection.socket.destroy(); + return; + } + const identity = authenticateCodexAppControlCandidate({ + state: codexAppControlStateValue, + auth: record, + sessionId: cfg.sessionId, + challenge: connection.challenge, + }); + if (!identity) { + rejectCodexAppControlMarker('challenge response'); + connection.socket.destroy(); + return; + } + activateCodexAppControlConnection(connection, identity); + return; + } + const identity = connection.identity; + if (record.type !== 'marker' || !identity + || connection.socket !== codexAppControlActiveSocket + || record.generation !== identity.generation + || record.challenge !== connection.challenge + || !verifyCodexAppSignedControlMarker(record, identity.publicKey)) { + rejectCodexAppControlMarker(record.type === 'marker' ? record.kind : record.type); + connection.socket.destroy(); + return; + } + if (!connection.sequenceFence.accept(record.seq)) { + rejectCodexAppControlMarker('non-contiguous signed sequence'); + connection.socket.destroy(); + return; + } + // A replacement worker may see the runner replay a complete final whose + // daemon-side delivery and FIFO pop were committed just before the old + // worker died. The per-generation high-water mark is a durable cumulative + // ACK boundary: acknowledge each replayed prefix record without assembling + // or re-emitting the final. + if (committedCodexAppSequence(codexAppGenerationCommits, identity.generation, record.seq)) { + connection.socket.write(`${encodeCodexAppControlAck( + cfg.sessionId, + identity.generation, + connection.challenge, + record.seq, + )}\n`); + return; + } + // ACK loss is expected around endpoint replacement. The runner re-signs its + // unacknowledged record against the fresh challenge; acknowledge that retry + // without applying completed/final side effects twice. + if (codexAppControlReplayWindow.hasSeen(identity.generation, record.seq)) { + connection.socket.write(`${encodeCodexAppControlAck( + cfg.sessionId, + identity.generation, + connection.challenge, + record.seq, + )}\n`); + return; + } + const finalResult = connection.finalAssembler.accept(record.kind, record.payload); + if (finalResult.status === 'reject') { + rejectCodexAppControlMarker(finalResult.reason); + connection.socket.destroy(); + return; + } + // start/chunk records intentionally remain uncommitted so a replacement + // connection can rebuild the complete final. Every record that does apply + // worker effects is single-flight by signed generation/sequence until the + // replay window below is committed. + if (finalResult.status === 'accepted') return; + const application = codexAppControlRecordApplicationGate.run( + identity.generation, + record.seq, + () => finalResult.status === 'not-final' + ? handleTrustedCodexAppMarker( + record.kind, + record.payload, + { generation: identity.generation, seq: record.seq }, + ) + : handleTrustedCodexAppMarker( + 'final', + finalResult.payload, + { generation: identity.generation, seq: record.seq }, + ), + ); + let applied: boolean; + try { + applied = await application; + } catch (err) { + codexAppControlRecordApplicationGate.release( + identity.generation, + record.seq, + application, + ); + throw err; + } + if (!applied) { + codexAppControlRecordApplicationGate.release( + identity.generation, + record.seq, + application, + ); + // Do not commit or cumulatively ACK a semantically rejected signed marker. + // In particular, a final for the wrong FIFO head must remain replayable + // rather than becoming a permanently lost answer. + connection.socket.destroy(); + return; + } + codexAppControlReplayWindow.commit(identity.generation, record.seq); + codexAppControlRecordApplicationGate.release( + identity.generation, + record.seq, + application, + ); + if (!connection.socket.destroyed) { + connection.socket.write(`${encodeCodexAppControlAck( + cfg.sessionId, + identity.generation, + connection.challenge, + record.seq, + )}\n`); + } +} + +const CODEX_APP_CONTROL_PENDING_LINE_LIMIT = 4_096; + +async function drainCodexAppControlLines(connection: CodexAppControlConnection): Promise { + if (connection.processingLines) return; + connection.processingLines = true; + connection.socket.pause(); + try { + while (!connection.socket.destroyed && connection.pendingLines.length > 0) { + const line = connection.pendingLines.shift()!; + await handleCodexAppControlLine(connection, line); + } + } catch (err: any) { + log(`Codex App control record processing failed: ${err?.message ?? err}`); + connection.socket.destroy(); + } finally { + connection.processingLines = false; + if (!connection.socket.destroyed) connection.socket.resume(); + } +} + +function acceptCodexAppControlSocket( + socket: Socket, + endpoint: string, + epoch: string, + channelId: number, +): void { + if (codexAppControlFatal || codexAppControlStopping || !lastInitConfig + || channelId !== codexAppControlChannelId) { + socket.destroy(); + return; + } + // Never let a stalled unauthenticated client monopolize the endpoint. Every + // connection gets its own challenge and timeout; new accepts continue while + // bad same-UID clients are rejected independently. + const unauthenticated = [...codexAppControlConnections.values()] + .filter(connection => !connection.authenticated); + if (unauthenticated.length >= 16) { + socket.destroy(); + return; + } + const challenge = generateCodexAppControlChallenge(); + const connection: CodexAppControlConnection = { + socket, + endpoint, + epoch, + channelId, + challenge, + decoder: new CodexAppControlLineDecoder(), + sequenceFence: new CodexAppControlSequenceFence(), + finalAssembler: new CodexAppControlFinalAssembler(), + authenticated: false, + pendingLines: [], + processingLines: false, + authTimer: setTimeout(() => socket.destroy(), CODEX_APP_CONTROL_ENDPOINT_RETRY_MS), + }; + connection.authTimer.unref?.(); + codexAppControlConnections.set(socket, connection); + socket.setNoDelay(true); + socket.on('data', chunk => { + const decoded = connection.decoder.push(chunk); + if (decoded.droppedMalformed) { + if (!codexAppMalformedControlLogged) { + codexAppMalformedControlLogged = true; + log('Dropped malformed/oversized Codex App socket control line'); + } + socket.destroy(); + return; + } + if (connection.pendingLines.length + decoded.lines.length > CODEX_APP_CONTROL_PENDING_LINE_LIMIT) { + log('Dropped Codex App control connection whose pending line queue exceeded the bound'); + socket.destroy(); + return; + } + connection.pendingLines.push(...decoded.lines); + void drainCodexAppControlLines(connection); + }); + socket.on('error', () => { /* close performs local cleanup */ }); + socket.on('close', () => { + clearTimeout(connection.authTimer); + codexAppControlConnections.delete(socket); + connection.pendingLines.length = 0; + const wasActive = codexAppControlActiveSocket === socket; + if (wasActive) { + codexAppControlActiveSocket = undefined; + codexAppControlActiveIdentity = undefined; + codexAppControlProven = false; + codexAppSignedStateObserved = false; + codexAppInputReady = false; + codexAppReadyAuthority.beginWork(); + codexAppTurnLiveness.beginReattachObservation(); + if (!codexAppControlStopping + && !codexAppControlFatal + && connection.channelId === codexAppControlChannelId) { + codexAppProofDeadline.arm(() => { + failCodexAppControlGeneration( + `Codex App runner did not re-authenticate within ${CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS / 1000} seconds after its control socket closed`, + ); + }); + } + log('Codex App authenticated control socket closed; waiting for a fresh challenge proof'); + } + // Rotate only after the authenticated runner closes. An unauthenticated + // close cannot prove that the legitimate runner has no pending connect to + // this name; retiring it could let another local process rebind that name + // under the pending attempt. Pre-auth failures therefore remain bound until + // the shared 90-second fail-close deadline (availability-only boundary). + if (wasActive + && !codexAppControlStopping + && !codexAppControlFatal + && connection.channelId === codexAppControlChannelId + && connection.epoch === codexAppControlEndpointEpoch) { + void rotateCodexAppControlEndpoint('active socket closed'); + } + }); + socket.write(`${encodeCodexAppControlChallenge(lastInitConfig.sessionId, challenge)}\n`); +} + +function removeStaleCodexAppSocket(path: string): void { + if (process.platform === 'win32') return; + try { + const stat = lstatSync(path); + const uid = process.geteuid?.() ?? process.getuid?.(); + if (!stat.isSocket() || stat.isSymbolicLink() || (uid !== undefined && stat.uid !== uid)) { + throw new Error('existing Codex App control socket path is not an owned socket'); + } + unlinkSync(path); + } catch (err: any) { + if (err?.code !== 'ENOENT') throw err; + } +} + +function listenCodexAppControlServer(server: NetServer, endpoint: string): Promise { + return new Promise((resolve, reject) => { + const onError = (err: Error) => { + server.off('listening', onListening); + reject(err); + }; + const onListening = () => { + server.off('error', onError); + resolve(); + }; + server.once('error', onError); + server.once('listening', onListening); + server.listen(endpoint); + }); +} + +async function ensureCodexAppWindowsOwnerLease(sessionId: string): Promise { + if (process.platform !== 'win32') return; + if (codexAppWindowsOwnerLeaseServer) { + if (codexAppWindowsOwnerLeaseSessionId !== sessionId) { + throw new Error('Windows Codex App owner lease belongs to another session'); + } + return; + } + if (codexAppWindowsOwnerLeasePromise) return codexAppWindowsOwnerLeasePromise; + const endpoint = codexAppWindowsOwnerPipeEndpoint(sessionId); + let acquisition!: Promise; + acquisition = (async () => { + const server = await acquireCodexAppControlOwnerLease({ + bind: async () => { + const candidate = createNetServer(socket => socket.destroy()); + try { + await listenCodexAppControlServer(candidate, endpoint); + return candidate; + } catch (err) { + try { candidate.close(); } catch { /* bind never completed */ } + throw err; + } + }, + }); + if (codexAppWindowsOwnerLeaseServer) { + try { server.close(); } catch { /* another local acquire won */ } + if (codexAppWindowsOwnerLeaseSessionId !== sessionId) { + throw new Error('Windows Codex App owner lease changed session during acquisition'); + } + return; + } + codexAppWindowsOwnerLeaseServer = server; + codexAppWindowsOwnerLeaseSessionId = sessionId; + server.on('error', err => { + if (codexAppWindowsOwnerLeaseServer === server && !codexAppControlStopping) { + failCodexAppControlGeneration(`Windows Codex App owner lease failed: ${err.message}`); + } + }); + })().finally(() => { + if (codexAppWindowsOwnerLeasePromise === acquisition) { + codexAppWindowsOwnerLeasePromise = undefined; + } + }); + codexAppWindowsOwnerLeasePromise = acquisition; + return acquisition; +} + +async function ensureCodexAppPosixOwnerLease( + controlRoot: string, + ownerSessionId: string, +): Promise { + if (process.platform === 'win32') return; + if (codexAppPosixOwnerLease) { + if (codexAppPosixOwnerLeaseSessionId !== ownerSessionId || !codexAppPosixOwnerLease.isOwned()) { + throw new Error('POSIX Codex App owner lease is not owned by this worker/session'); + } + return; + } + if (codexAppPosixOwnerLeasePromise) return codexAppPosixOwnerLeasePromise; + let acquisition!: Promise; + acquisition = (async () => { + const lease = await acquireCodexAppPosixOwnerLease({ + controlRoot, + sessionId: ownerSessionId, + }); + if (codexAppPosixOwnerLease) { + lease.release(); + if (codexAppPosixOwnerLeaseSessionId !== ownerSessionId) { + throw new Error('POSIX Codex App owner lease changed session during acquisition'); + } + return; + } + codexAppPosixOwnerLease = lease; + codexAppPosixOwnerLeaseSessionId = ownerSessionId; + })().finally(() => { + if (codexAppPosixOwnerLeasePromise === acquisition) { + codexAppPosixOwnerLeasePromise = undefined; + } + }); + codexAppPosixOwnerLeasePromise = acquisition; + return acquisition; +} + +function releaseCodexAppPosixOwnerLease(): void { + codexAppPosixOwnerLease?.release(); + codexAppPosixOwnerLease = undefined; + codexAppPosixOwnerLeaseSessionId = undefined; +} + +interface StartedCodexAppControlEndpoint { + server: NetServer; + endpoint: string; + epoch: string; +} + +async function startCodexAppControlEndpoint( + cfg: Extract, + channelId: number, +): Promise { + const epoch = generateCodexAppControlEpoch(); + const endpoint = process.platform === 'win32' + ? generateCodexAppWindowsPipeEndpoint() + : codexAppControlSocketDirectory + ? generateCodexAppPosixSocketEndpoint(codexAppControlSocketDirectory) + : undefined; + if (!endpoint) throw new Error('Codex App control endpoint path was not prepared'); + const locatorPath = codexAppControlLocatorPathValue; + if (!locatorPath) throw new Error('Codex App control locator path was not prepared'); + const server = createNetServer(socket => { + acceptCodexAppControlSocket(socket, endpoint, epoch, channelId); + }); + const priorServer = codexAppControlServer; + const priorEndpoint = codexAppControlSocketPathValue; + const priorEpoch = codexAppControlEndpointEpoch; + try { + const publisherLeaseOwned = (): boolean => process.platform === 'win32' + ? !!codexAppWindowsOwnerLeaseServer + : codexAppPosixOwnerLease?.isOwned() === true; + const published = await bindThenPublishCodexAppControlLocator({ + sessionId: cfg.sessionId, + epoch, + endpoint, + platform: process.platform, + locatorPath, + expectedControlRoot: process.platform === 'win32' + ? undefined + : codexAppPosixControlRoot(), + listen: async () => { + await listenCodexAppControlServer(server, endpoint); + if (process.platform !== 'win32') { + chmodSync(endpoint, 0o600); + const stat = lstatSync(endpoint); + const uid = process.geteuid?.() ?? process.getuid?.(); + if (!stat.isSocket() || stat.isSymbolicLink() || (uid !== undefined && stat.uid !== uid)) { + throw new Error('listening path is not the owned Codex App socket'); + } + } + }, + isCurrent: () => !codexAppControlStopping + && !codexAppControlFatal + && channelId === codexAppControlChannelId + && publisherLeaseOwned(), + retire: () => { try { server.close(); } catch { /* already retired */ } }, + publish: locator => { + if (!publisherLeaseOwned()) throw new Error('Codex App locator publisher lease was lost'); + // Install the bound epoch before making the locator visible. Any + // immediate connection is therefore checked against this exact epoch. + codexAppControlServer = server; + codexAppControlSocketPathValue = endpoint; + codexAppControlEndpointEpoch = epoch; + writeCodexAppControlLocator(locatorPath, locator); + }, + }); + if (!published) throw new Error('Codex App control endpoint was retired before locator publication'); + } catch (err) { + if (codexAppControlServer === server) { + codexAppControlServer = priorServer; + codexAppControlSocketPathValue = priorEndpoint; + codexAppControlEndpointEpoch = priorEpoch; + } + try { server.close(); } catch { /* bind may not have completed */ } + if (process.platform !== 'win32') { + try { removeStaleCodexAppSocket(endpoint); } catch { /* preserve original error */ } + } + throw err; + } + server.on('error', err => { + if (codexAppControlServer === server + && channelId === codexAppControlChannelId + && !codexAppControlStopping) { + failCodexAppControlGeneration(`Codex App control endpoint failed: ${err.message}`); + } + }); + return { server, endpoint, epoch }; +} + +function installCodexAppControlEndpoint(started: StartedCodexAppControlEndpoint): NetServer | undefined { + const previous = codexAppControlServer; + codexAppControlServer = started.server; + codexAppControlSocketPathValue = started.endpoint; + codexAppControlEndpointEpoch = started.epoch; + return previous; +} + +function closeRetiredCodexAppControlEndpoint( + server: NetServer | undefined, + endpoint: string | undefined, +): void { + try { server?.close(); } catch { /* already closed */ } + if (endpoint && process.platform !== 'win32') { + try { removeStaleCodexAppSocket(endpoint); } catch { /* next bind will verify */ } + } +} + +function rotateCodexAppControlEndpoint(reason: string): Promise { + if (codexAppControlStopping || codexAppControlFatal) { + return Promise.resolve(); + } + if (codexAppControlRotation) return codexAppControlRotation; + const cfg = lastInitConfig; + const channelId = codexAppControlChannelId; + if (!cfg) return Promise.resolve(); + const oldServer = codexAppControlServer; + const oldEndpoint = codexAppControlSocketPathValue; + let rotation!: Promise; + rotation = (async () => { + try { + const started = await startCodexAppControlEndpoint(cfg, channelId); + if (codexAppControlStopping || codexAppControlFatal + || channelId !== codexAppControlChannelId) { + closeRetiredCodexAppControlEndpoint(started.server, started.endpoint); + return; + } + installCodexAppControlEndpoint(started); + // Bind + publish the fresh random endpoint before retiring the old one. + closeRetiredCodexAppControlEndpoint(oldServer, oldEndpoint); + for (const connection of [...codexAppControlConnections.values()]) { + if (connection.epoch !== started.epoch) connection.socket.destroy(); + } + log(`Rotated Codex App control endpoint (${reason}, epoch=${started.epoch.slice(0, 8)})`); + } catch (err: any) { + if (shouldFailCodexAppControlChannel({ + channelId, + currentChannelId: codexAppControlChannelId, + stopping: codexAppControlStopping, + })) { + failCodexAppControlGeneration(`Could not rotate Codex App control endpoint: ${err?.message ?? err}`); + } + } finally { + if (codexAppControlRotation === rotation) codexAppControlRotation = undefined; + } + })(); + codexAppControlRotation = rotation; + return rotation; +} + +async function prepareCodexAppControlGeneration( + cfg: Extract, + _willReattachPersistent: boolean, + persistentGeneration: boolean, +): Promise { + cleanupCodexAppControlBootstrap(); + // init already restored the daemon-owned FIFO before spawn. Retiring a + // previous/empty socket endpoint must not erase that recovery snapshot just + // before a warm runner replays its unacked final. + stopCodexAppControlChannel({ preserveDispatchRecovery: true }); + codexAppControlStopping = false; + const channelId = codexAppControlChannelId; + codexAppControlStateValue = undefined; + codexAppControlProven = false; + codexAppSignedStateObserved = false; + codexAppInputReady = false; + codexAppReadyAuthority.reset(); + codexAppControlFatal = false; + codexAppControlPersistentGeneration = false; + codexAppControlDirectoryForSpawn = undefined; + codexAppControlSocketPathValue = undefined; + codexAppControlSocketDirectory = undefined; + codexAppControlLocatorPathValue = undefined; + codexAppControlEndpointEpoch = undefined; + codexAppFreshCandidateGeneration = undefined; + codexAppUnprovenPromptDeferred = false; + codexAppRejectedControlLogged = false; + codexAppMalformedControlLogged = false; + if (cfg.cliId !== 'codex-app') return; + + const dataDir = process.env.SESSION_DATA_DIR; + const windowsRoot = process.platform === 'win32' ? codexAppWindowsControlRoot() : undefined; + const controlRoot = windowsRoot ?? codexAppPosixControlRoot(); + // The daemon's kill-then-fork replacement path can briefly overlap worker + // processes. Hold a process-lifetime per-session publisher lease before + // touching the fixed locator so an old process cannot publish after its + // replacement. The lease is never exposed as a runner control endpoint. + if (windowsRoot) await ensureCodexAppWindowsOwnerLease(cfg.sessionId); + else await ensureCodexAppPosixOwnerLease(controlRoot, cfg.sessionId); + const controlDirectory = windowsRoot + ? join(windowsRoot, 'bootstraps') + : dataDir + ? join(botHomePath(dirname(dataDir), cfg.larkAppId), 'codex-app-control-bootstrap') + : join(tmpdir(), `botmux-codex-app-control-${process.getuid?.() ?? 'unknown'}`); + // AF_UNIX paths are capped at roughly 104 bytes on macOS. Keep random POSIX + // endpoints in the short private control root; both platforms publish them + // through a fixed protected locator while the process-lifetime owner lease + // serializes replacement workers. + const socketDirectory = join(controlRoot, 'sockets'); + ensureCodexAppControlDirectory(socketDirectory); + ensureCodexAppControlDirectory(controlDirectory); + cleanupStaleCodexAppControlBootstraps(controlDirectory, cfg.sessionId); + codexAppControlDirectoryForSpawn = controlDirectory; + codexAppControlSocketDirectory = socketDirectory; + codexAppControlLocatorPathValue = codexAppControlLocatorPath(controlRoot, cfg.sessionId); + ensureCodexAppControlDirectory(dirname(codexAppControlLocatorPathValue)); + // A crashed worker may leave a stale locator. Keep it until the new random + // endpoint has bound, then atomically overwrite it. Deleting first would + // introduce a cross-process read/unlink race and is unnecessary because + // accepted carries the protected, independently random locator epoch. + // Preserve old public identities for every persistent spawn attempt, even + // when the existence probe predicted fresh. The backend may race between + // probe and spawn; the socket proof, not that prediction, selects old reuse + // versus the fresh candidate. + if (persistentGeneration) codexAppControlStateValue = readPersistedCodexAppControlState(cfg); + const started = await startCodexAppControlEndpoint(cfg, channelId); + if (codexAppControlStopping || channelId !== codexAppControlChannelId) { + closeRetiredCodexAppControlEndpoint(started.server, started.endpoint); + throw new Error('Codex App control endpoint was retired during startup'); + } + installCodexAppControlEndpoint(started); +} + +/** Late-create the only secret-bearing file immediately before backend.spawn. */ +function prepareFreshCodexAppControlBootstrap( + cfg: Extract, + persistentGeneration: boolean, +): void { + if (cfg.cliId !== 'codex-app') return; + if (!codexAppControlDirectoryForSpawn || !codexAppControlLocatorPathValue) { + throw new Error('Codex App control channel was not prepared'); + } + const bootstrap = createCodexAppControlBootstrap( + codexAppControlDirectoryForSpawn, + cfg.sessionId, + { kind: 'locator', locatorPath: codexAppControlLocatorPathValue }, + ); + codexAppControlBootstrapPathForSpawn = bootstrap.path; + codexAppFreshCandidateGeneration = bootstrap.identity.generation; + codexAppControlStateValue = mergeCodexAppControlCandidate( + codexAppControlStateValue, + bootstrap.identity, + ); + codexAppControlPersistentGeneration = persistentGeneration; + if (persistentGeneration) persistCodexAppControlState(cfg, codexAppControlStateValue); + codexAppBootstrapCleanupTimer = armCodexAppControlStartupTimeout(cleanupCodexAppControlBootstrap); + codexAppBootstrapCleanupTimer.unref?.(); +} + +function finalizeCodexAppControlGeneration( + cfg: Extract, + _actuallyReattached: boolean, + persistentGeneration: boolean, +): void { + if (cfg.cliId !== 'codex-app') return; + codexAppControlPersistentGeneration = persistentGeneration; + // A live runner can answer while synchronous backend.spawn is still + // returning. Authentication already persisted the active public identity; + // do not mistake that success for a missing pending bootstrap. + if (codexAppControlProven && codexAppControlStateValue?.status === 'active') { + cleanupCodexAppControlBootstrap(); + if (!codexAppSignedStateObserved && !codexAppProofDeadline.armed) { + codexAppProofDeadline.arm(() => { + failCodexAppControlGeneration( + `Authenticated Codex App runner did not publish signed state within ${CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS / 1000} seconds`, + ); + }); + } + return; + } + if (codexAppControlStateValue?.status !== 'pending' + || !codexAppControlBootstrapPathForSpawn) { + cleanupCodexAppControlBootstrap(); + throw new Error('Codex App pending asymmetric control generation was not prepared'); + } + codexAppProofDeadline.arm(() => { + failCodexAppControlGeneration( + `Codex App runner did not authenticate and publish signed state within ${CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS / 1000} seconds`, + ); + }); +} + +function rejectCodexAppControlMarker(kind: string): void { + if (codexAppRejectedControlLogged) return; + codexAppRejectedControlLogged = true; + log(`Ignored unauthenticated Codex App ${kind} control record`); +} + +type RuntimeScreenStatus = Exclude; + +/** + * Project an explicit Codex App no-progress state above the screen heuristic. + * The warning is once-per-turn and intentionally does not restart or replay + * anything: both actions could duplicate model/tool side effects. + */ +function codexAppLivenessStatus(base: RuntimeScreenStatus, nowMs = Date.now()): RuntimeScreenStatus { + if (lastInitConfig?.cliId !== 'codex-app') return base; + base = projectCodexAppControlReadinessStatus(base, { + controlProven: codexAppControlProven, + signedStateObserved: codexAppSignedStateObserved, + inputReady: codexAppInputReady, + }); + const liveness = codexAppTurnLiveness.poll(nowMs); + if (liveness.shouldNotify) { + send({ + type: 'user_notify', + turnId: liveness.turnId ?? currentBotmuxTurnId, + ...(liveness.turnId === currentBotmuxTurnId + && currentBotmuxDispatchAttempt !== undefined + ? { dispatchAttempt: currentBotmuxDispatchAttempt } + : {}), + message: t('worker.codex_app.no_progress', { + seconds: Math.round(CODEX_APP_NO_PROGRESS_TIMEOUT_MS / 1000), + }), + }); + } + if (liveness.stalled) return 'stalled'; + return liveness.active && base === 'idle' ? 'working' : base; +} // Per-turn usage-limit state machine. Owns the turn counter plus the // "did this turn hit a limit" / "suppress a stale retry-ready banner" flags, so @@ -4352,6 +5414,7 @@ async function captureAndUpload(): Promise { let status: RuntimeScreenStatus = isPromptReady ? 'idle' : 'working'; if (screenAnalyzer?.isAnalyzing) status = 'analyzing'; + status = codexAppLivenessStatus(status); send({ type: 'screenshot_uploaded', imageKey, @@ -4728,10 +5791,10 @@ function dismissAidenCodexUpdateDialog(data: string): boolean { return true; } -// Codex App runner sends botmux control messages as OSC sequences so they do -// not pollute the visible terminal. Strip them before xterm rendering and -// translate them back into worker IPC. -const APP_RUNNER_OSC_CLI_IDS = new Set(['codex-app', 'mira', 'mir']); +// Mira/Mir still send terminal OSC control messages. Codex App deliberately +// does not: its signed Unix-socket channel is independent of terminal/backend +// rendering (including Herdr and Zellij). +const APP_RUNNER_OSC_CLI_IDS = new Set(['mira', 'mir']); const appRunnerControlDecoder = new RunnerControlDecoder(); let kiroSessionIdCaptureArmed = false; let kiroSessionIdCaptureBuffer = ''; @@ -4744,74 +5807,392 @@ function decodeCodexAppPayload(payload: string): any | undefined { } } -function handleCodexAppMarker(body: string): void { - const sep = body.indexOf(':'); - if (sep < 0) return; - const kind = body.slice(0, sep); - const payload = decodeCodexAppPayload(body.slice(sep + 1)); - if (!payload || typeof payload !== 'object') return; +const CODEX_APP_DAEMON_PERSIST_TIMEOUT_MS = 30_000; + +function waitForCodexAppDaemonPersistence( + requestId: string, + publish: () => void, +): Promise { + return new Promise(resolve => { + const timer = setTimeout(() => { + const pending = codexAppPendingDaemonAcks.get(requestId); + if (!pending) return; + codexAppPendingDaemonAcks.delete(requestId); + resolve(false); + }, CODEX_APP_DAEMON_PERSIST_TIMEOUT_MS); + timer.unref?.(); + codexAppPendingDaemonAcks.set(requestId, { resolve, timer }); + try { + publish(); + } catch { + clearTimeout(timer); + codexAppPendingDaemonAcks.delete(requestId); + resolve(false); + } + }); +} + +function requestCodexAppDispatchTransition( + operation: 'submit' | 'cancel' | 'retry', + entries: Array<{ dispatchId: string; turnId: string; dispatchAttempt?: number }>, +): Promise { + if (!sessionId || entries.length === 0) return Promise.resolve(entries.length === 0); + const requestId = randomBytes(16).toString('hex'); + return waitForCodexAppDaemonPersistence(requestId, () => send({ + type: 'codex_app_dispatch_transition', + sessionId, + requestId, + operation, + entries, + })); +} + +/** Retire one exact durable dispatch before its receipt/lease owner is allowed + * to replay it. The daemon ledger is the authority; local queue removal is + * deliberately after the durable ACK so a failed session-file write cannot + * create a replay window while the old worker still owns the turn. */ +async function retireCodexAppDispatchForDurableReplay( + turnId: string, + dispatchAttempt: number, +): Promise { + if (lastInitConfig?.cliId !== 'codex-app') { + for (let index = pendingMessages.length - 1; index >= 0; index--) { + const item = pendingMessages[index]; + if (item.turnId === turnId && item.dispatchAttempt === dispatchAttempt) { + pendingMessages.splice(index, 1); + } + } + return true; + } + const reservation = codexAppTurnDispatchQueue.findExact(turnId, dispatchAttempt); + const pending = pendingMessages.find(item => item.turnId === turnId + && item.dispatchAttempt === dispatchAttempt); + const recovered = codexAppRecoveredDispatches.find(entry => entry.turnId === turnId + && entry.dispatchAttempt === dispatchAttempt); + const dispatchIds = new Set([ + reservation?.dispatchId, + pending?.codexAppDispatchId, + recovered?.dispatchId, + ].filter((value): value is string => !!value)); + if (dispatchIds.size !== 1) { + log( + `Cannot retire Codex App durable dispatch turn=${turnId.slice(0, 12)} ` + + `attempt=${dispatchAttempt}: exact dispatch id is ${dispatchIds.size === 0 ? 'missing' : 'ambiguous'}`, + ); + return false; + } + const dispatchId = [...dispatchIds][0]!; + if (!await requestCodexAppDispatchTransition('cancel', [{ + dispatchId, + turnId, + dispatchAttempt, + }])) return false; + + if (reservation && !codexAppTurnDispatchQueue.cancelExact(reservation.handle)) return false; + for (let index = pendingMessages.length - 1; index >= 0; index--) { + const item = pendingMessages[index]; + if (item.turnId === turnId && item.dispatchAttempt === dispatchAttempt) { + pendingMessages.splice(index, 1); + } + } + codexAppRecoveredDispatches = codexAppRecoveredDispatches.filter( + entry => entry.dispatchId !== dispatchId, + ); + return true; +} +async function handleTrustedCodexAppMarker( + kind: string, + payload: Record, + control?: { generation: string; seq: number }, +): Promise { if (kind === 'thread' && typeof payload.threadId === 'string') { persistCliSessionId(payload.threadId); - return; + return true; } - if (kind === 'final' && typeof payload.content === 'string') { - const startedAtMs = typeof payload.startedAtMs === 'number' ? payload.startedAtMs : undefined; - const completedAtMs = typeof payload.completedAtMs === 'number' ? payload.completedAtMs : Date.now(); - // Codex App keeps the app-server-generated id separately for diagnostics. - // Routing must use its stable clientUserMessageId marker; legacy envelopes - // intentionally omit it and fall back to the worker's frozen botmux turn. - const identity = resolveCodexAppFinalTurnIdentity( + if (kind === 'state' && lastInitConfig?.cliId === 'codex-app') { + const readiness = codexAppSignedStateReadiness(payload); + if (readiness === 'invalid') { + rejectCodexAppControlMarker('signed state missing boolean acceptingInput'); + failCodexAppControlGeneration( + 'Codex App runner published signed state without boolean acceptingInput', + ); + return false; + } + if (readiness === 'waiting') { + if (typeof payload.busy !== 'boolean') { + rejectCodexAppControlMarker('invalid signed state'); + return false; + } + codexAppSignedStateObserved = false; + codexAppInputReady = false; + codexAppReadyAuthority.beginWork(); + isPromptReady = false; + idleDetector?.reset(); + if (!codexAppProofDeadline.armed) { + codexAppProofDeadline.arm(() => { + failCodexAppControlGeneration( + `Codex App runner did not become input-ready within ${CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS / 1000} seconds`, + ); + }); + } + log('Authenticated Codex App runner is not yet accepting input; readiness deadline remains armed'); + return true; + } + const state = applyTrustedCodexAppStateMarker( + codexAppTurnLiveness, + codexAppReadyAuthority, payload, - currentBotmuxTurnId, - `${lastInitConfig?.cliId ?? 'app'}-${Date.now()}`, ); - if (!identity.ok) { - log( - `${cliName()} rejected final marker with mismatched turn ` - + `(marker=${identity.markerTurnId.substring(0, 12)}, ` - + `current=${identity.currentBotmuxTurnId?.substring(0, 12) ?? '-'})`, + if (!state.accepted) { + rejectCodexAppControlMarker('invalid signed state'); + return false; + } + // This is the first actual readiness proof. Authentication alone cannot + // clear startup/reconnect failure detection or release queued input. + codexAppSignedStateObserved = true; + codexAppInputReady = true; + codexAppProofDeadline.clear(); + cleanupCodexAppControlBootstrap(); + if (codexAppInputReady && awaitingFirstPrompt) { + awaitingFirstPrompt = false; + renderer?.markNewTurn(); + } + if (state.busy) { + isPromptReady = false; + idleDetector?.reset(); + if (codexAppInputReady) queueMicrotask(() => { void flushPending(); }); + } else { + if (codexAppCompletionAwaitingFinal) { + // Every runner generation that can authenticate this new signed + // channel emits an explicit final transaction, including zero chunks + // for an empty answer. Guessing an empty final here would let a + // mismatched/rejected final advance the FIFO and be ACKed as success. + failCodexAppControlGeneration( + 'Codex App runner published idle before the required final transaction', + ); + return false; + } + // Signed idle proves only that the runner has no queued/active turn. It + // does NOT prove its raw stdin inputBuffer is empty: the old worker can + // die after writing the complete frame but before Enter, then this warm + // runner reconnects and reports idle. Resetting prepared→accepted would + // let the replacement pre-flush submit that old frame and then replay it + // a second time. A recovered prepared entry therefore advances only by + // replaying its final/high-water ACK; otherwise fail closed for explicit + // operator abandon. + if (codexAppTurnDispatchQueue.recoveredPrefix().length > 0) { + failCodexAppControlGeneration( + 'Codex App signed idle cannot prove the recovered prepared frame was never buffered', + ); + return false; + } + if (state.shouldPublishReady) { + codexAppUnprovenPromptDeferred = false; + queueMicrotask(() => markPromptReady()); + } + } + return true; + } + + if (kind === 'diagnostic' && lastInitConfig?.cliId === 'codex-app') { + if (payload.code !== 'native_turn_identity_conflict' + || typeof payload.message !== 'string') { + rejectCodexAppControlMarker('invalid signed diagnostic'); + return false; + } + log(`Codex App fail-closed diagnostic: ${payload.message}`); + send({ + type: 'user_notify', + message: payload.message, + ...(typeof payload.turnId === 'string' ? { turnId: payload.turnId } : {}), + }); + return true; + } + + if (kind === 'activity' && lastInitConfig?.cliId === 'codex-app') { + const activity = applyTrustedCodexAppActivityMarker( + codexAppTurnLiveness, + payload, + ); + if (!activity.accepted) { + rejectCodexAppControlMarker('invalid signed activity'); + return false; + } + cleanupCodexAppControlBootstrap(); + if (activity.phase === 'completed') { + codexAppCompletionAwaitingFinal = true; + } else { + if (activity.phase === 'submitted' && codexAppCompletionAwaitingFinal) { + failCodexAppControlGeneration( + 'Codex App runner submitted the next turn before the required final transaction', + ); + return false; + } + codexAppReadyAuthority.beginWork(); + isPromptReady = false; + idleDetector?.reset(); + } + return true; + } + + if (kind === 'final' && typeof payload.content === 'string') { + const finalContent = payload.content; + const startedAtMs = typeof payload.startedAtMs === 'number' && Number.isFinite(payload.startedAtMs) + ? payload.startedAtMs + : undefined; + const receivedAtMs = Date.now(); + const completedAtMs = typeof payload.completedAtMs === 'number' && Number.isFinite(payload.completedAtMs) + ? Math.min(payload.completedAtMs, receivedAtMs) + : receivedAtMs; + let turnId: string; + let nativeTurnId: string | undefined; + let replyTurnId: string | undefined; + let dispatchAttempt: number | undefined; + let codexAppDispatchId: string | undefined; + let codexAppDispatchHandle: number | undefined; + if (lastInitConfig?.cliId === 'codex-app') { + // flushPending may finish writing N+1 before final N arrives. Attribute + // only from the immutable worker-owned FIFO head; currentBotmuxTurnId is + // intentionally not consulted here. + const settlement = codexAppTurnDispatchQueue.settleFinal(payload, false); + if (!settlement.ok) { + log( + `${cliName()} rejected final marker (${settlement.reason}; ` + + `marker=${settlement.markerTurnId?.substring(0, 12) ?? '-'}, ` + + `expected=${settlement.expectedTurnId?.substring(0, 12) ?? '-'})`, + ); + return false; + } + ({ + turnId, + replyTurnId, + nativeTurnId, + dispatchAttempt, + dispatchId: codexAppDispatchId, + handle: codexAppDispatchHandle, + } = settlement); + if (!codexAppCompletionAwaitingFinal) { + // turn/start failures emit a final transaction without an app-server + // completed activity edge. Close exactly one liveness slot here. + codexAppTurnLiveness.completeCurrent(completedAtMs); + } + codexAppCompletionAwaitingFinal = false; + // A terminal prompt may already have been observed, but ready waits for + // the later signed state{busy:false}. The runner sequences that state + // after this complete final transaction. + } else { + // Mira/Mir retain their terminal OSC control path and do not use the + // Codex App serial dispatch FIFO. + const identity = resolveCodexAppFinalTurnIdentity( + payload, + currentBotmuxTurnId, + `${lastInitConfig?.cliId ?? 'app'}-${Date.now()}`, ); - return; + if (!identity.ok) { + log( + `${cliName()} rejected final marker with mismatched turn ` + + `(marker=${identity.markerTurnId.substring(0, 12)}, ` + + `current=${identity.currentBotmuxTurnId?.substring(0, 12) ?? '-'})`, + ); + return false; + } + ({ turnId, nativeTurnId } = identity); + if (payload.dispatchAttempt !== undefined + && payload.dispatchAttempt !== currentBotmuxDispatchAttempt) { + log( + `${cliName()} rejected final marker with mismatched dispatch attempt ` + + `(marker=${String(payload.dispatchAttempt)}, current=${currentBotmuxDispatchAttempt ?? '-'})`, + ); + return false; + } + dispatchAttempt = currentBotmuxDispatchAttempt; } - const { turnId, nativeTurnId } = identity; if (nativeTurnId && nativeTurnId !== turnId) { log(`${cliName()} native turn ${nativeTurnId.substring(0, 12)} mapped to botmux turn ${turnId.substring(0, 12)}`); } - if (payload.dispatchAttempt !== undefined - && payload.dispatchAttempt !== currentBotmuxDispatchAttempt) { - log( - `${cliName()} rejected final marker with mismatched dispatch attempt ` - + `(marker=${String(payload.dispatchAttempt)}, current=${currentBotmuxDispatchAttempt ?? '-'})`, - ); - return; - } - // Attempt authority is worker-owned. A runner marker may redundantly assert - // equality for compatibility, but can never select another attempt. - const dispatchAttempt = currentBotmuxDispatchAttempt; - if (startedAtMs !== undefined) { - const sentByModel = shouldSuppressBridgeEmit( - { markTimeMs: startedAtMs, isLocal: false, finalText: payload.content }, + let suppressDelivery = false; + if (finalContent && startedAtMs !== undefined) { + suppressDelivery = shouldSuppressBridgeEmit( + { markTimeMs: startedAtMs, isLocal: false, finalText: finalContent }, completedAtMs + 5_001, readSendMarkers(), false, ); - if (sentByModel) { + if (suppressDelivery) { log(`${cliName()} final_output suppressed (model already called botmux send)`); - emitTurnTerminal(turnId, 'completed', undefined, dispatchAttempt); - return; } } - send({ - type: 'final_output', - content: payload.content, - lastUuid: turnId, - turnId, - ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), - }); + + if (codexAppDispatchId) { + if (!control || codexAppDispatchHandle === undefined) return false; + const requestId = randomBytes(16).toString('hex'); + const persisted = await waitForCodexAppDaemonPersistence(requestId, () => send({ + type: 'final_output', + content: suppressDelivery ? '' : finalContent, + lastUuid: turnId, + turnId, + ...(replyTurnId ? { replyTurnId } : {}), + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + ...(suppressDelivery ? { suppressDelivery: true } : {}), + codexAppSettlement: { + requestId, + generation: control.generation, + seq: control.seq, + dispatchId: codexAppDispatchId!, + }, + })); + if (!persisted || !codexAppTurnDispatchQueue.commitExactHead(codexAppDispatchHandle)) { + return false; + } + codexAppGenerationCommits = [ + ...codexAppGenerationCommits.filter(commit => commit.generation !== control.generation), + { + generation: control.generation, + committedThrough: Math.max( + control.seq, + codexAppGenerationCommits.find( + commit => commit.generation === control.generation, + )?.committedThrough ?? 0, + ), + }, + ]; + codexAppRecoveredDispatches = codexAppRecoveredDispatches.filter( + entry => entry.dispatchId !== codexAppDispatchId, + ); + } else { + if (lastInitConfig?.cliId === 'codex-app' + && codexAppDispatchHandle !== undefined + && !codexAppTurnDispatchQueue.commitExactHead(codexAppDispatchHandle)) { + return false; + } + if (finalContent && !suppressDelivery) { + send({ + type: 'final_output', + content: finalContent, + lastUuid: turnId, + turnId, + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + }); + } else if (!finalContent) { + log(`${cliName()} empty final settled for botmux turn ${turnId.substring(0, 12)}`); + } + } emitTurnTerminal(turnId, 'completed', undefined, dispatchAttempt); + return true; } + rejectCodexAppControlMarker(`unsupported signed ${kind}`); + return false; +} + +function handleAppRunnerOscMarker(body: string): void { + const sep = body.indexOf(':'); + if (sep < 0) return; + const kind = body.slice(0, sep); + const payload = decodeCodexAppPayload(body.slice(sep + 1)); + if (!payload || typeof payload !== 'object') return; + void handleTrustedCodexAppMarker(kind, payload); } function maybeCaptureKiroSessionId(data: string): void { @@ -4828,7 +6209,7 @@ function splitCodexAppControl(data: string): string { return appRunnerControlDecoder.push( data, APP_RUNNER_OSC_CLI_IDS.has(lastInitConfig?.cliId ?? ''), - handleCodexAppMarker, + handleAppRunnerOscMarker, ); } @@ -4964,6 +6345,13 @@ function markPromptReadyFromPty(): void { function markPromptReady(): void { if (isPromptReady) return; // guard against duplicate calls + if (lastInitConfig?.cliId === 'codex-app' && !codexAppControlProven) { + if (!codexAppUnprovenPromptDeferred) { + log('Ignoring Codex App prompt until this worker verifies a fresh Ed25519 challenge response'); + } + codexAppUnprovenPromptDeferred = true; + return; + } stopBusyPatternIdleProbe(); // Ready-gate: a startup selector's ❯ (cjadk et al.) falsely matches // readyPattern → the IdleDetector fires idle while the CLI is NOT actually at @@ -4993,6 +6381,22 @@ function markPromptReady(): void { log('Idle detected during ready-gate settle; deferring prompt-ready until settle completes'); return; } + // Authenticated runners advance their explicit control queue on signed + // completed/final records. + // A prompt is authoritative only for the synthetic observation installed + // after a verified warm reattach; clearing normal queued turns here could + // lose follow-up inputs written by one flushPending() call. + if (lastInitConfig?.cliId === 'codex-app' && !codexAppTurnLiveness.notePrompt()) { + log('Ignoring transient Codex App prompt while an explicit runner turn remains queued'); + return; + } + if (lastInitConfig?.cliId === 'codex-app' && !codexAppReadyAuthority.canPublishPromptReady()) { + if (!codexAppReadyAuthority.consumeLatePromptRecovery(!codexAppTurnLiveness.hasActiveTurn())) { + log('Deferring Codex App terminal prompt until signed runner state confirms busy=false'); + return; + } + log('Accepted one late Codex App prompt after exact local submit cancellation'); + } isPromptReady = true; clearSessionRenameInFlight(); // An old backend can still report idle while its async teardown is running. @@ -5055,15 +6459,11 @@ function persistCliSessionId(cliSessionId: string): void { turnId: currentBotmuxTurnId, dispatchAttempt: currentBotmuxDispatchAttempt, }); - try { - const session = sessionStore.getSession(sessionId); - if (!session || session.cliSessionId === cliSessionId) return; - session.cliSessionId = cliSessionId; - sessionStore.updateSession(session); - log(`Persisted CLI session id: ${cliSessionId}`); - } catch (err: any) { - log(`Failed to persist CLI session id: ${err.message}`); - } + // The daemon is the sole sessions-file writer for Codex App durability. + // Writing the worker's stale in-process snapshot here could roll a freshly + // persisted accepted→prepared transition back to accepted. The ordered IPC + // above already has an authoritative daemon handler that persists this id. + log(`Published CLI session id for daemon persistence: ${cliSessionId}`); } function observeCursorCliSessionId(pid: number, label = 'spawn'): void { @@ -5317,6 +6717,26 @@ function detectBareShellLaunch(): boolean { * sends the next message (type-ahead) without waiting for idle detection. * Messages pushed during a flush are picked up by the while loop. */ +function requeueUnsubmittedQueuedActivation(item: PendingCliInput): void { + if (!item.queuedActivationToken) return; + // backend.onExit already moved the same object into carryOver; spawnCli will + // restore it after the fenced restart, so do not create a second owner here. + if (!backend) return; + inflightInputs.forget(item); + if (!pendingMessages.some(candidate => + candidate.queuedActivationToken === item.queuedActivationToken)) { + pendingMessages.unshift(item); + } + log(`Retained queued activation ${item.queuedActivationToken.substring(0, 8)} for retry`); +} + +function codexAppRuntimeTypeAheadReady(): boolean { + return lastInitConfig?.cliId === 'codex-app' + && codexAppControlProven + && codexAppSignedStateObserved + && codexAppInputReady; +} + async function flushPending(): Promise { // destroySession() may be asynchronous while `backend` still references the // old CLI. Never let a new flush (including one triggered by the old @@ -5387,7 +6807,7 @@ async function flushPending(): Promise { const claudeBridgeActive = !!bridgeJsonlPath && !lastInitConfig?.adoptMode; const codexBridgeActive = codexBridgeFallbackActive(); const typeAheadAllowed = pendingInputAllowsTypeAhead( - cliAdapter.supportsTypeAhead === true, + cliAdapter.supportsTypeAhead === true || codexAppRuntimeTypeAheadReady(), durableTurnInFlight, pendingMessages[0], ); @@ -5407,6 +6827,7 @@ async function flushPending(): Promise { if (!isPromptReady && !typeAheadAllowed) return; isFlushing = true; + const codexAppPromptReplay = new CodexAppFlushPromptReplay(); if (isPromptReady) { isPromptReady = false; idleDetector?.reset(); @@ -5530,6 +6951,50 @@ async function flushPending(): Promise { log('Refused durable Claude submit: transcript terminal bridge is unavailable'); break; } + // Unlike the transcript bridge, Codex App has an explicit app-server + // runner. Start its liveness clock before the control line is written so + // an accepted-but-never-dequeued input is diagnosed too; authenticated + // activity records can arrive while writeInput awaits chunk delivery. + const tracksCodexAppLiveness = lastInitConfig?.cliId === 'codex-app'; + const codexAppFrozenTurnId = tracksCodexAppLiveness + ? item.turnId + || item.codexAppInput?.clientUserMessageId + || `codex-app-${sessionId || 'unknown'}-${Date.now().toString(36)}-${++codexAppFallbackTurnSequence}` + : undefined; + // Reserve attribution before the first chunk is written. The runner can + // dequeue and finish a small turn while writeRunnerInput is still + // awaiting later chunk throttles, and this flush may then write N+1. + const codexAppDispatchReservation = codexAppFrozenTurnId + ? codexAppTurnDispatchQueue.reserve( + codexAppFrozenTurnId, + item.dispatchAttempt, + item.codexAppDispatchId, + false, + item.replyTurnId, + ) + : undefined; + if (tracksCodexAppLiveness && item.codexAppDispatchId && codexAppFrozenTurnId) { + const prepared = await requestCodexAppDispatchTransition('submit', [{ + dispatchId: item.codexAppDispatchId, + turnId: codexAppFrozenTurnId, + ...(item.dispatchAttempt !== undefined + ? { dispatchAttempt: item.dispatchAttempt } + : {}), + }]); + if (!prepared) { + if (codexAppDispatchReservation) { + codexAppTurnDispatchQueue.cancelExact(codexAppDispatchReservation.handle); + } + failCodexAppControlGeneration( + 'Daemon could not persist the Codex App prepared dispatch before runner write', + ); + break; + } + } + const codexAppLivenessHandle = tracksCodexAppLiveness + ? codexAppTurnLiveness.begin(codexAppFrozenTurnId) + : undefined; + if (tracksCodexAppLiveness) codexAppReadyAuthority.beginWork(); log(`Writing to PTY (flush): "${msg.substring(0, 80)}"`); // Defense in depth: TmuxPipeBackend's send methods no longer throw on a // dead pane (they fire onExit instead), but writeInput can still throw @@ -5554,8 +7019,53 @@ async function flushPending(): Promise { } scheduleBusyPatternIdleProbe(`${cliName()} post-submit`); } catch (err: any) { + codexAppPromptReplay.cancelSubmission( + codexAppTurnLiveness, + codexAppReadyAuthority, + codexAppLivenessHandle, + ); + if (tracksCodexAppLiveness) { + // A throwing framed write has unknown side effects: the runner may + // hold a complete valid line without its newline. Never cancel and + // continue in this generation. Ordinary IM ownership stays in the + // daemon's prepared ledger for explicit crash recovery. Durable + // ownership also stays intact: the worker-exit path atomically marks + // its receipt ambiguous and arms the runtime fence before replay. + log(`writeInput threw with unknown Codex App runner state: ${err?.message ?? err}`); + failCodexAppControlGeneration( + 'Codex App input write became ambiguous; fenced the runner before any successor could submit', + ); + break; + } + // Legacy/non-control adapters keep their existing submit-failure path. + // A durable receiver attempt transfers replay ownership to the + // receipt/lease reconciler on the ambiguous terminal below, so remove + // any exact local reservation first to avoid two independent replayers. + let dispatchStillPending = true; + if (codexAppDispatchReservation) { + if (item.codexAppDispatchId && durableWrite && codexAppFrozenTurnId) { + const cancelled = codexAppTurnDispatchQueue.cancelExact( + codexAppDispatchReservation.handle, + ) && await requestCodexAppDispatchTransition('cancel', [{ + dispatchId: item.codexAppDispatchId, + turnId: codexAppFrozenTurnId, + dispatchAttempt: item.dispatchAttempt!, + }]); + if (!cancelled) { + failCodexAppControlGeneration( + 'Could not transfer an ambiguous Codex App delivery to durable recovery', + ); + break; + } + } else if (!item.codexAppDispatchId) { + dispatchStillPending = codexAppTurnDispatchQueue.cancelExact( + codexAppDispatchReservation.handle, + ); + } + } log(`writeInput threw: ${err?.message ?? err}`); - if (durableWrite && item.turnId) { + requeueUnsubmittedQueuedActivation(item); + if (dispatchStillPending && durableWrite && item.turnId) { // A throwing backend cannot prove whether bytes reached the CLI. // Reconcile as ambiguous (not a definitive failure) so the receiver // can replay the same frozen delivery behind the action gate. @@ -5565,7 +7075,7 @@ async function flushPending(): Promise { // nulled `backend` and told the user the CLI exited) — nothing more to // do. Otherwise surface it as a submit failure so the message isn't // silently lost. - if (backend) scheduleSubmitFailureNotify( + if (backend && dispatchStillPending) scheduleSubmitFailureNotify( logicalMsg, undefined, '会话 JSONL', @@ -5596,16 +7106,77 @@ async function flushPending(): Promise { // `&& backend`: if the CLI exited during this write (pane gone → onExit // nulled backend) the user already got a "CLI exited" notice; don't also // nag that the submit wasn't confirmed. - if (result && result.submitted === false && backend) { - scheduleSubmitFailureNotify( - logicalMsg, - result.recheck, - '会话 JSONL', - bridgeTurnId, - result.failureReason, - turnSeq, - item, + if (result && result.submitted === false) { + codexAppPromptReplay.cancelSubmission( + codexAppTurnLiveness, + codexAppReadyAuthority, + codexAppLivenessHandle, ); + const codexAppSafeNonSubmission = result.submissionDisposition === 'untouched' + || result.submissionDisposition === 'flushed_invalid'; + if (tracksCodexAppLiveness && !codexAppSafeNonSubmission) { + // final-Enter exhaustion, cleanup failure, and thrown/unknown writes + // can leave a complete valid frame buffered. Retain an ordinary + // prepared ledger entry and fence the entire generation. For a + // durable attempt, worker exit owns the ambiguous receipt + runtime + // fence transition; publishing an earlier terminal would open a + // replay window before the old runner teardown is proven. + failCodexAppControlGeneration( + 'Codex App runner input buffer is not provably clean; fenced before any successor could submit', + ); + break; + } + const dispatchStillPending = codexAppDispatchReservation + ? codexAppTurnDispatchQueue.cancelExact(codexAppDispatchReservation.handle) + : true; + if (dispatchStillPending && item.codexAppDispatchId && codexAppFrozenTurnId) { + const retryQueuedActivation = tracksCodexAppLiveness + && !!item.queuedActivationToken + && codexAppSafeNonSubmission; + const transitioned = await requestCodexAppDispatchTransition( + retryQueuedActivation ? 'retry' : 'cancel', [{ + dispatchId: item.codexAppDispatchId, + turnId: codexAppFrozenTurnId, + ...(item.dispatchAttempt !== undefined + ? { dispatchAttempt: item.dispatchAttempt } + : {}), + }], + ); + if (!transitioned) { + failCodexAppControlGeneration( + retryQueuedActivation + ? 'Daemon could not return a safely untouched queued activation to accepted' + : 'Daemon could not cancel a Codex App dispatch rejected before submission', + ); + break; + } + if (!retryQueuedActivation) { + codexAppRecoveredDispatches = codexAppRecoveredDispatches.filter( + entry => entry.dispatchId !== item.codexAppDispatchId, + ); + } + } + if (backend && dispatchStillPending) { + scheduleSubmitFailureNotify( + logicalMsg, + result.recheck, + '会话 JSONL', + bridgeTurnId, + result.failureReason, + turnSeq, + item, + ); + } + requeueUnsubmittedQueuedActivation(item); + } else if (item.queuedActivationToken) { + // The daemon keeps the exact journal and route reservation until this + // adapter-level boundary. IPC loss may replay at-least-once; an early + // worker/daemon crash can never silently consume the opening turn. + send({ + type: 'queued_activation_submitted', + sessionId, + activationToken: item.queuedActivationToken, + }); } // All structured bridges now drain every pending message in one flush: // Claude's BridgeTurnQueue handles `attachment(queued_command)` events @@ -5620,6 +7191,14 @@ async function flushPending(): Promise { } } finally { isFlushing = false; + // A prompt can arrive after turn N completes but before turn N+1's chunked + // control line finishes. We reject it while N+1 has a liveness slot. If that + // write then fails, the prompt is still the runner's real ready boundary; + // replay it after releasing the flush mutex so queued peers can drain. + if (codexAppPromptReplay.consumeAfterFlush(codexAppTurnLiveness)) { + log('Replaying deferred Codex App prompt after a queued submission was cancelled'); + markPromptReady(); + } } } @@ -5629,6 +7208,9 @@ function sendToPty( opts: { codexAppInput?: CodexAppTurnInput; dispatchAttempt?: number; + codexAppDispatchId?: string; + queuedActivationToken?: string; + replyTurnId?: string; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; } = {}, ): boolean { @@ -5636,12 +7218,21 @@ function sendToPty( const next: PendingCliInput = { content, turnId, + ...(opts.replyTurnId ? { replyTurnId: opts.replyTurnId } : {}), + ...(opts.codexAppDispatchId ? { codexAppDispatchId: opts.codexAppDispatchId } : {}), + ...(opts.queuedActivationToken ? { queuedActivationToken: opts.queuedActivationToken } : {}), ...(opts.codexAppInput ? { codexAppInput: opts.codexAppInput } : {}), ...(opts.dispatchAttempt !== undefined ? { dispatchAttempt: opts.dispatchAttempt } : {}), ...(opts.vcMeetingImTurnOrigin ? { vcMeetingImTurnOrigin: opts.vcMeetingImTurnOrigin } : {}), }; + if (!initPromptMaterialized) { + pendingMessages.push(next); + log(`Queued message behind async init materialization (${pendingMessages.length} pending)`); + return true; + } + if (!cliAdapter) return false; // During an exact lease-fenced CLI restart the worker stays alive while the // backend is rebuilt. Preserve incoming attempt N+1 in the worker queue; the // old early-return silently dropped it after receiver had already persisted @@ -5652,7 +7243,7 @@ function sendToPty( return true; } const supportsTypeAhead = pendingInputAllowsTypeAhead( - cliAdapter.supportsTypeAhead === true, + cliAdapter.supportsTypeAhead === true || codexAppRuntimeTypeAheadReady(), durableTurnInFlight, next, ); @@ -5733,6 +7324,7 @@ function startScreenUpdates(): void { if (awaitingFirstPrompt) return; let status: RuntimeScreenStatus = isPromptReady ? 'idle' : 'working'; if (screenAnalyzer?.isAnalyzing) status = 'analyzing'; + status = codexAppLivenessStatus(status); void (async () => { let content = lastContent; @@ -6053,6 +7645,7 @@ async function spawnCli( cfg: Extract, opts: { pluginGenerationPrepared?: boolean } = {}, ): Promise { + const spawnGeneration = ++cliSpawnGeneration; clearSessionRenameInFlight(); currentCliCredentialIsolated = false; // Enrollment writes the fixed marker before any device credential appears. @@ -6602,6 +8195,38 @@ async function spawnCli( } } + // A pane created before asymmetric control framing has no persisted public + // identity capable of answering this worker's fresh challenge. Never fall + // back to terminal OSC trust: terminate it and cold-spawn a signed runner. + if (persistentSessionName && shouldColdStartCodexAppReattach({ + cliId: cfg.cliId, + backendType: effectiveBackendType, + isReattach: willReattachPersistent, + persistedState: readPersistedCodexAppControlState(cfg), + })) { + log(`Codex App persistent pane ${persistentSessionName} has no valid public control identity — killing + cold-spawning authenticated runner`); + try { + killPersistentSession(effectiveBackendType as PersistentBackendType, persistentSessionName); + } catch (err: any) { + throw new Error(`Refusing unauthenticated Codex App reattach: could not kill stale pane (${err?.message ?? err})`); + } + willReattachPersistent = false; + } + + // The worker establishes trust before any runner output can be parsed. A + // fresh runner gets a new capability; a persistent reattach reloads the + // capability created by that same runner generation. + // The control endpoint is an authenticated lifecycle prerequisite. Await + // bind + protected locator publication before backend.spawn so + // a runner can never observe a not-yet-owned or stale endpoint. + try { + await prepareCodexAppControlGeneration(cfg, willReattachPersistent, !!persistentSessionName); + } catch (err) { + if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError(); + throw err; + } + if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError(); + // The plugin set is stable only for the lifetime of one real CLI process. // A warm worker reattach keeps the existing Gateway and catalog untouched; // every fresh/resumed CLI spawn atomically refreshes both from current Bot config. @@ -6610,6 +8235,7 @@ async function spawnCli( ? readSessionMcpRuntimeManifest(cfg.sessionId, config.session.dataDir) : await prepareCliPluginGenerationAndGateway(cfg, cliAdapter); } + if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError(); // Re-arm the startup-commands one-shot ONLY for a genuinely fresh CLI process. // A reattach to a LIVE persistent pane (daemon-restart recovery) is the SAME @@ -6784,6 +8410,7 @@ async function spawnCli( passesInitialPromptViaArgs: cliAdapter.passesInitialPromptViaArgs === true, adoptMode: cfg.adoptMode === true, dispatchAttempt: cfg.dispatchAttempt, + queuedActivationToken: cfg.queuedActivationToken, }) || (effectiveResume && cliAdapter.initialPromptArgsIgnoredOnResume === true) || (!promptArgPreparationChanged && shouldDeferInitialPromptForArgLimit({ passesInitialPromptViaArgs: cliAdapter.passesInitialPromptViaArgs === true, @@ -6932,6 +8559,9 @@ async function spawnCli( delete childEnv[MCP_GATEWAY_SOCKET_ENV]; delete childEnv[MCP_GATEWAY_REQUIRED_ENV]; } + // Never inherit an ambient/stale bootstrap path from the daemon's own launch + // environment. The raw capability is never placed in any environment. + delete childEnv[CODEX_APP_CONTROL_BOOTSTRAP_ENV]; // Put the daemon-written wrapper dir (~/.botmux/bin/botmux = THIS build) ahead of any // stale npm-global botmux in PATH, so the agent's `botmux` is always this build. Matters // most under read isolation: only this build has the send-cred reader — a shadowing stale @@ -7012,7 +8642,6 @@ async function spawnCli( if (claudeDataDir) childEnv.CLAUDE_CONFIG_DIR = canonicalizeForSandbox(claudeDataDir); // = /claude else childEnv.CODEX_HOME = canonicalizeForSandbox(isolatedCodexHome!); } - // Per-bot env (bots.json `env`): extra vars for THIS bot's CLI only — e.g. // ANTHROPIC_BASE_URL/ANTHROPIC_AUTH_TOKEN to run a bot on GLM/a 3rd-party // provider, an HTTPS_PROXY, or a CLI feature flag. Passed as injectEnv (NOT @@ -7587,14 +9216,57 @@ async function spawnCli( log(`Failed to capture reproduce command: ${err?.message ?? err}`); } - backend.spawn(spawnBin, spawnArgs, { - cwd: spawnCwd, - cols: PTY_COLS, - rows: PTY_ROWS, - env: childEnv as Record, - injectEnv: perBotInjectKeys.length ? perBotInjectEnv : undefined, - launchShell: lastInitConfig?.launchShell, - }); + // Create the asymmetric candidate as late as possible. Persistent backends + // receive it even on a predicted reattach: an actual reuse ignores the + // launch env and proves the old key, while an actual fresh start consumes + // the candidate and proves the new key. The worker trusts cryptographic + // proof, not a backend's prediction flag. + try { + if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError(); + prepareFreshCodexAppControlBootstrap(cfg, !!persistentSessionName); + if (codexAppControlBootstrapPathForSpawn) { + childEnv[CODEX_APP_CONTROL_BOOTSTRAP_ENV] = codexAppControlBootstrapPathForSpawn; + } + backend.spawn(spawnBin, spawnArgs, { + cwd: spawnCwd, + cols: PTY_COLS, + rows: PTY_ROWS, + env: childEnv as Record, + injectEnv: perBotInjectKeys.length ? perBotInjectEnv : undefined, + launchShell: lastInitConfig?.launchShell, + }); + } catch (err) { + cleanupCodexAppControlBootstrap(); + throw err; + } finally { + delete childEnv[CODEX_APP_CONTROL_BOOTSTRAP_ENV]; + } + const actuallyReattachedPersistent = 'isReattach' in backend + && backend.isReattach === true; + try { + finalizeCodexAppControlGeneration( + cfg, + actuallyReattachedPersistent, + !!persistentSessionName, + ); + } catch (err) { + cleanupCodexAppControlBootstrap(); + // A runner whose generation cannot be authenticated durably must not keep + // executing behind an apparently failed worker. This covers both a + // fresh-persist failure or a generation that cannot complete challenge + // setup. + if (persistentSessionName) { + try { + killPersistentSession( + effectiveBackendType as PersistentBackendType, + persistentSessionName, + ); + } catch (killErr: any) { + log(`Failed to kill unauthenticated Codex App persistent generation: ${killErr?.message ?? killErr}`); + } + } + throw err; + } if (selectedBackend.createdHerdrSessionName) { send({ @@ -7935,6 +9607,7 @@ async function spawnCli( // prompt-ready — without this hook a follow-up arriving mid-task would sit // in pendingMessages forever once the task finishes. backend.onTaskDone?.(() => { + if (fatalWorkerErrorPending) return; log(`${cliName()} task finished — re-arming prompt-ready for queued follow-ups`); markPromptReady(); }); @@ -7969,6 +9642,32 @@ async function spawnCli( exitedDispatchAttempt, ); log(`${cliName()} exited (code: ${code}, signal: ${signal})`); + if (lastInitConfig?.cliId === 'codex-app' && codexAppControlFatal) { + // An earlier control-path failure already made this whole worker + // generation terminal and scheduled its hard OS exit. That failure may + // synchronously kill the backend, invoking this callback after + // stopCodexAppControlChannel() cleared the local FIFO. Never reinterpret + // that cleared snapshot as safe ownership: publishing ambiguous or + // claude_exit here would let the daemon admit N+1 before the Node-worker + // exit arms the durable recovery fence. + log('Suppressed Codex App backend-exit signals for a fatal worker generation'); + return; + } + const codexAppPreparedAtExit = !intentionalRestart + && lastInitConfig?.cliId === 'codex-app' + && (codexAppTurnDispatchQueue.size() > 0 + || codexAppRecoveredDispatches.some(entry => entry.state === 'prepared')); + if (codexAppPreparedAtExit) { + // Do not publish a standalone ambiguous terminal or claude_exit first. + // Either signal can let the durable receiver admit N+1 before the later + // Node-worker exit arms its persistent-pane fence. Exit this worker + // generation directly; daemon onWorkerExit performs the ambiguous + // transition and recovery-arm as one ordering boundary. + failCodexAppControlGeneration( + 'Codex App runner exited with a prepared dispatch; worker replacement requires exact recovery', + ); + return; + } if (cliAdapter?.reliableTurnTerminal === true && exitedTurnId && exitedDispatchAttempt !== undefined) { @@ -8003,6 +9702,22 @@ async function spawnCli( stopCodexRpcEngine(); if (rpcDialogDismissTimer) { clearTimeout(rpcDialogDismissTimer); rpcDialogDismissTimer = null; } } + cleanupCodexAppControlBootstrap(); + // A natural Codex App runner exit is only the CLI-generation terminal, + // not proof that any prepared frame was unconsumed. Keep the exact local + // FIFO/recovery snapshot until the daemon answers claude_exit with its + // cli_crash decision. The restart handler then sees prepared ownership and + // exits this Node worker too, which is what drives the daemon's + // onWorkerExit durable-receipt fence. Intentional in-worker teardown has + // already established its own replay/cancel boundary and may clear here. + stopCodexAppControlChannel({ + preserveDispatchRecovery: lastInitConfig?.cliId === 'codex-app' && !intentionalRestart, + }); + codexAppControlStateValue = undefined; + codexAppControlProven = false; + codexAppTurnLiveness.clear(); + codexAppReadyAuthority.reset(); + codexAppCompletionAwaitingFinal = false; const logTail = recentTerminalLogTail(); // Don't park a diagnostic shell here: most exits are immediately // auto-restarted by the daemon, so an inline park would just be torn down @@ -8036,8 +9751,16 @@ async function spawnCli( } }); - if (isPipeMode && backend && 'isReattach' in backend && backend.isReattach) { + const isPersistentBackendReattach = actuallyReattachedPersistent; + // Codex App warm observation is armed only after the old generation proves + // its private key over this worker's fresh socket challenge. Backend flags + // remain useful for screen seeding but are not authentication evidence. + + if (isPipeMode && backend && isPersistentBackendReattach) { log(`Re-attached to existing ${effectiveBackendType} session via pipe backend: ${persistentSessionName}`); + // Pipe backends expose an authoritative snapshot + busy-pattern probe. + // Keep those operations pipe-only; the liveness slot above is backend- + // independent and Zellij receives its screen through normal PTY output. seedBackendScreen(`${effectiveBackendType} reattach`, backend); scheduleReattachIdleProbe(`${effectiveBackendType} reattach`, backend); } @@ -8109,11 +9832,22 @@ async function spawnCli( } } +// Invalidate an in-flight async spawn before any other teardown hook runs. The +// remaining cleanup is synchronous today, but generation ownership must not +// depend on that implementation detail. function killCli(opts: { preservePending?: boolean } = {}): void { + cliSpawnGeneration++; currentCliCredentialIsolated = false; stopNativeSessionTitleSync(); stopSessionMcpGatewayHost(); stopCodexRpcEngine(); + cleanupCodexAppControlBootstrap(); + stopCodexAppControlChannel(); + codexAppControlStateValue = undefined; + codexAppControlProven = false; + codexAppTurnLiveness.clear(); + codexAppReadyAuthority.reset(); + codexAppCompletionAwaitingFinal = false; if (!opts.preservePending) cleanupPiInitialPromptFiles(); destroyCrashDiagnosticTerminal('killCli'); idleDetector?.dispose(); @@ -8220,12 +9954,6 @@ async function restartCliProcess( const rpcThreadId = remoteThreadId; const restartingBackend = backend; if (restartingBackend) intentionalRestartBackend = restartingBackend; - // Riff teardown cancels a remote task asynchronously. Wait for the cancel - // (bounded) before respawning, otherwise the old and replacement tasks can - // overlap and both emit into the same Lark thread. Local backends normally - // return void here and continue synchronously. A synchronous throw is - // handled by the same fatal restart path below, so it cannot leave the - // input gate permanently armed. const teardown = restartingBackend?.destroySession?.(); if (teardown && typeof (teardown as Promise).then === 'function') { try { @@ -8259,6 +9987,7 @@ async function restartCliProcess( if (codexRpcEngine) armRpcStartupDialogDismiss(); } catch (err) { cliRestartInProgress = false; + if (err instanceof CliSpawnSupersededError) return; await sendFatalWorkerErrorAndExit(err); return; } @@ -9468,10 +11197,21 @@ function sendAndFlush(msg: WorkerToDaemon): Promise { resolve(); return; } + let settled = false; + const finish = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(); + }; + // IPC callbacks are best-effort transport evidence, not an exit gate. A + // parent that has stopped draining (or a runtime callback edge) must not + // keep a fail-closed worker and its stale ownership alive indefinitely. + const timer = setTimeout(finish, 1_000); try { - process.send(payload, () => resolve()); + process.send(payload, finish); } catch { - resolve(); + finish(); } }); } @@ -9486,6 +11226,7 @@ async function sendFatalWorkerErrorAndExit( err: unknown, turnId = currentBotmuxTurnId, dispatchAttempt = currentBotmuxDispatchAttempt, + opts: { hardExit?: boolean } = {}, ): Promise { if (fatalWorkerErrorPending) return; fatalWorkerErrorPending = true; @@ -9497,6 +11238,19 @@ async function sendFatalWorkerErrorAndExit( // failure to the receipt/lease recovery path instead of replying out-of-band. ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), }); + log('Fatal worker error delivered; exiting process'); + if (opts.hardExit) { + // A fail-closed Codex App generation must produce a real OS-level worker + // exit so the daemon can arm its worker-generation receipt fence. On some + // runtimes process.exit() can hang joining a native/libuv fs worker (the + // control/terminal stack uses long-lived reads). Perform the synchronous + // cleanup we control, then use an uncatchable self-signal for this one + // replacement path. Persistent-session and SIGKILL residue both have + // daemon/next-worker reconcilers as additional backstops. + try { teardownSandboxBestEffort(); } catch { /* best effort */ } + try { cleanup(); } catch { /* best effort */ } + try { process.kill(process.pid, 'SIGKILL'); } catch { /* fall through */ } + } process.exit(1); } @@ -9533,6 +11287,23 @@ process.on('message', async (raw: unknown) => { if (msg.larkAppId && process.env.BOTMUX_WORKFLOW !== '1') { sessionStore.init(msg.larkAppId); } + if (msg.cliId === 'codex-app') { + codexAppRecoveredDispatches = (msg.codexAppRecoveredDispatches ?? []).map(entry => ({ + ...entry, + codexAppInput: entry.codexAppInput ? { ...entry.codexAppInput } : undefined, + })); + codexAppGenerationCommits = [...(msg.codexAppGenerationCommits ?? [])]; + codexAppTurnDispatchQueue.restore( + codexAppRecoveredDispatches + .filter(entry => entry.state === 'prepared') + .map(entry => ({ + dispatchId: entry.dispatchId, + turnId: entry.turnId, + replyTurnId: entry.replyTurnId, + dispatchAttempt: entry.dispatchAttempt, + })), + ); + } // Capture credentials for direct image upload from worker larkAppIdForUpload = msg.larkAppId; larkAppSecretForUpload = msg.larkAppSecret; @@ -9635,6 +11406,22 @@ process.on('message', async (raw: unknown) => { // execution — exactly-once, Codex P1-1). Ambiguous never reaches here. // - RPC RESUME: queuePrompt=true → queue for post-ready flush (bridge // marked at flush time, P0-1). + const recoveredAcceptedEntries = codexAppRecoveredDispatches + .filter(entry => entry.state === 'accepted'); + const recoveredAcceptedInputs = recoveredAcceptedEntries + .map(entry => ({ + content: entry.content, + turnId: entry.turnId, + replyTurnId: entry.replyTurnId, + dispatchAttempt: entry.dispatchAttempt, + codexAppDispatchId: entry.dispatchId, + queuedActivationToken: entry.queuedActivationToken + ?? (recoveredAcceptedEntries.length === 1 + ? msg.queuedActivationToken + : undefined), + codexAppInput: entry.codexAppInput, + vcMeetingImTurnOrigin: entry.vcMeetingImTurnOrigin, + })); let initialInputCommitted = false; if (shouldQueueInitialPrompt({ hasPrompt: !!msg.prompt, @@ -9643,23 +11430,33 @@ process.on('message', async (raw: unknown) => { passesInitialPromptViaArgs: cliAdapter?.passesInitialPromptViaArgs === true, deferInitialPrompt, })) { - pendingMessages.push({ + // Accepted entries that never crossed the old worker's prepared/write + // boundary are replayable payload, not runner attribution. Queue + // them before this fork's new prompt while prepared predecessors stay + // preloaded in the exact final FIFO above. + pendingMessages.unshift(...recoveredAcceptedInputs, { content: lastSpawnQueuedInitialPrompt ?? msg.prompt, ...(lastSpawnQueuedInitialPromptLogicalContent ? { logicalContent: lastSpawnQueuedInitialPromptLogicalContent } : {}), turnId: msg.turnId, + replyTurnId: msg.replyTurnId, dispatchAttempt: msg.dispatchAttempt, + codexAppDispatchId: msg.codexAppDispatchId, + queuedActivationToken: msg.queuedActivationToken, vcMeetingImTurnOrigin: msg.vcMeetingImTurnOrigin, codexAppInput: msg.promptCodexAppInput, }); initialInputCommitted = true; + } else if (msg.cliId === 'codex-app') { + pendingMessages.unshift(...recoveredAcceptedInputs); } else if (msg.prompt) { // A successful spawn with a non-queued prompt means the adapter baked // it into argv or the RPC engine already accepted it. initialInputCommitted = true; } if (initialInputCommitted) acknowledgeTurnInputCommitted(msg.turnId); + initPromptMaterialized = true; // A backend may become prompt-ready before spawnCli() returns. The // initial prompt is queued only afterwards, so the earlier @@ -9685,12 +11482,25 @@ process.on('message', async (raw: unknown) => { dispatchAttempt: currentBotmuxDispatchAttempt, }); } catch (err: any) { + if (err instanceof CliSpawnSupersededError) return; await sendFatalWorkerErrorAndExit(err); return; } break; } + case 'codex_app_dispatch_persisted': { + const pending = codexAppPendingDaemonAcks.get(msg.requestId); + if (!pending) break; + codexAppPendingDaemonAcks.delete(msg.requestId); + clearTimeout(pending.timer); + pending.resolve(msg.ok); + if (!msg.ok) { + log(`Daemon rejected Codex App dispatch persistence: ${msg.error ?? 'unknown error'}`); + } + break; + } + case 'message': { // Mark new turn baseline so the streaming card only shows this turn's content renderer?.markNewTurn(); @@ -9746,6 +11556,7 @@ process.on('message', async (raw: unknown) => { // Pass the message's own attempt (not the stale currentBotmux* from a // prior IM turn) so a durable delivery relaunch failure carries the // right attribution for the daemon's receipt/lease gate. + if (err instanceof CliSpawnSupersededError) return; await sendFatalWorkerErrorAndExit(err, msg.turnId, msg.dispatchAttempt); return; } @@ -9845,6 +11656,9 @@ process.on('message', async (raw: unknown) => { const inputCommitted = sendToPty(content, msg.turnId, { codexAppInput, dispatchAttempt: msg.dispatchAttempt, + codexAppDispatchId: msg.codexAppDispatchId, + queuedActivationToken: msg.queuedActivationToken, + replyTurnId: msg.replyTurnId, vcMeetingImTurnOrigin: msg.vcMeetingImTurnOrigin, }); if (inputCommitted) acknowledgeTurnInputCommitted(msg.turnId); @@ -9921,6 +11735,10 @@ process.on('message', async (raw: unknown) => { log('Refused durable expiry ACK for adopt-mode session'); break; } + if (!await retireCodexAppDispatchForDurableReplay(msg.turnId, msg.dispatchAttempt)) { + log('Withholding durable expiry ACK because the daemon dispatch ledger could not retire exactly'); + break; + } log( `Expiring active durable turn=${msg.turnId.slice(0, 12)} attempt=${msg.dispatchAttempt}; ` + 'restarting CLI with queued follow-ups preserved', @@ -9939,15 +11757,10 @@ process.on('message', async (raw: unknown) => { // ordinary IM turn. Remove that exact queued generation and ACK without // restarting the unrelated active turn; otherwise attempt N survives in // the queue and executes after the hub has already replayed N+1. - let removedPending = 0; - for (let i = pendingMessages.length - 1; i >= 0; i--) { - const item = pendingMessages[i]; - if (item.turnId === msg.turnId && item.dispatchAttempt === msg.dispatchAttempt) { - pendingMessages.splice(i, 1); - removedPending++; - } - } - if (removedPending > 0) { + const removedPending = pendingMessages.filter(item => item.turnId === msg.turnId + && item.dispatchAttempt === msg.dispatchAttempt).length; + if (removedPending > 0 + && await retireCodexAppDispatchForDurableReplay(msg.turnId, msg.dispatchAttempt)) { log( `Expired ${removedPending} queued durable input(s) turn=${msg.turnId.slice(0, 12)} ` + `attempt=${msg.dispatchAttempt}`, @@ -9980,6 +11793,10 @@ process.on('message', async (raw: unknown) => { `Boot recovery fencing ambiguous receiver turn=${msg.turnId.slice(0, 12)} ` + `attempt=${msg.dispatchAttempt}`, ); + if (!await retireCodexAppDispatchForDurableReplay(msg.turnId, msg.dispatchAttempt)) { + log('Withholding receiver reset ACK because the daemon dispatch ledger could not retire exactly'); + break; + } durableTurnInFlight = false; inflightInputs.onTurnComplete(); await restartCliProcess('ambiguous receiver boot recovery', { immediate: true, preservePending: true }); @@ -10222,6 +12039,10 @@ function cleanup(): void { try { workflowPtyLogStream.end(); } catch { /* already closed */ } workflowPtyLogStream = undefined; } + // Publisher ownership spans every CLI generation in this worker. Releasing + // from killCli/restart would reopen a stale-publish window; only process-level + // cleanup retires the lease. SIGKILL residue is reclaimed by the next worker. + releaseCodexAppPosixOwnerLease(); } process.on('SIGTERM', () => { stopScreenshotLoop(); killCli(); cleanup(); process.exit(0); }); diff --git a/test/anchor-serializer.test.ts b/test/anchor-serializer.test.ts index a9f23372c..63cff9bf4 100644 --- a/test/anchor-serializer.test.ts +++ b/test/anchor-serializer.test.ts @@ -10,7 +10,7 @@ * * Run: pnpm vitest run test/anchor-serializer.test.ts */ -import { describe, it, expect, beforeEach } from 'vitest'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; import { serializeByAnchor, __resetAnchorQueues } from '../src/utils/anchor-serializer.js'; const delayPush = (order: string[], label: string, ms: number) => () => @@ -84,4 +84,38 @@ describe('serializeByAnchor — wait cap (head-of-line-blocking guard)', () => { await Promise.all([p1, p2]); expect(order).toEqual(['s:1', 'e:1', 's:2', 'e:2']); }); + + it('keeps strict admission ordered past five seconds when cap=0', async () => { + vi.useFakeTimers(); + try { + const order: string[] = []; + let releaseFirst!: () => void; + const first = serializeByAnchor('strict', () => new Promise(resolve => { + order.push('start:1'); + releaseFirst = () => { + order.push('end:1'); + resolve(); + }; + }), 0); + const second = serializeByAnchor('strict', async () => { + order.push('start:2'); + }, 0); + + await Promise.resolve(); + await Promise.resolve(); + expect(order).toEqual(['start:1']); + + // The ordinary serializer deliberately escapes after 5s. A raw inbound + // admission lane must not: concurrent cold-session handlers can reorder + // or drop accepted turns even when the first await is merely slow. + await vi.advanceTimersByTimeAsync(5_100); + expect(order).toEqual(['start:1']); + + releaseFirst(); + await Promise.all([first, second]); + expect(order).toEqual(['start:1', 'end:1', 'start:2']); + } finally { + vi.useRealTimers(); + } + }); }); diff --git a/test/bot-registry.test.ts b/test/bot-registry.test.ts index 3cdc0169d..25c34ea79 100644 --- a/test/bot-registry.test.ts +++ b/test/bot-registry.test.ts @@ -72,6 +72,12 @@ describe('registerBot', () => { expect(client.opts.appSecret).toBe('secret_001'); }); + it('bounds the SDK HTTP transport timeout when the client exposes axios defaults', () => { + const client = { httpInstance: { defaults: { timeout: 0 } } }; + mod.configureLarkClientHttpTimeout(client); + expect(client.httpInstance.defaults.timeout).toBe(mod.LARK_REQUEST_TIMEOUT_MS); + }); + it('should default the SDK Client domain to feishu when brand is unset', () => { const state = mod.registerBot(makeCfg()); const client = state.client as unknown as { opts: Record }; diff --git a/test/bot-routing.test.ts b/test/bot-routing.test.ts index 30b4d5457..90fda0b29 100644 --- a/test/bot-routing.test.ts +++ b/test/bot-routing.test.ts @@ -188,6 +188,13 @@ describe('buildFooterAddressing', () => { )).toEqual({ sendTo: 'ou_owner', cc: [] }); }); + it('honors a frozen bot-sender bit even before cross-ref learns that bot id', () => { + expect(buildFooterAddressing( + { ownerOpenId: 'ou_owner', lastCallerOpenId: 'ou_new_bot', lastCallerIsBot: true }, + { isOncall: true, knownBotOpenIds }, + )).toEqual({ sendTo: 'ou_owner', cc: [] }); + }); + it('drops addressing when the resolved recipient would be a bot', () => { expect(buildFooterAddressing( { ownerOpenId: 'ou_codex_bot', lastCallerOpenId: 'ou_claude_bot' }, diff --git a/test/bridge-final-output-retry.test.ts b/test/bridge-final-output-retry.test.ts index 7126496a8..fffc3615a 100644 --- a/test/bridge-final-output-retry.test.ts +++ b/test/bridge-final-output-retry.test.ts @@ -238,6 +238,223 @@ describe('Bridge final_output delivery (P2 retry)', () => { expect(ds.lastBridgeEmittedUuid).toBe(SCOPED_DEDUPE_KEY); }); + it('routes synthetic Codex App identities through their frozen reply turn and uses dispatch-stable Lark UUIDs', async () => { + const sessionReply = vi.fn(async () => 'om_reply'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + const { __testOnly_deliverFinalOutput } = await import('../src/core/worker-pool.js') as any; + + __testOnly_deliverFinalOutput(ds, { + type: 'final_output', + sessionId: ds.session.sessionId, + content: 'scheduled answer one', + lastUuid: 'synthetic-1', + turnId: 'codex-app-dispatch-synthetic-1', + replyTurnId: 'om_shared_route', + codexAppSettlement: { + requestId: 'request-1', generation: 'generation-1', seq: 1, dispatchId: 'dispatch-1', + }, + }, 'tag', 0); + __testOnly_deliverFinalOutput(ds, { + type: 'final_output', + sessionId: ds.session.sessionId, + content: 'scheduled answer two', + lastUuid: 'synthetic-2', + turnId: 'codex-app-dispatch-synthetic-2', + replyTurnId: 'om_shared_route', + codexAppSettlement: { + requestId: 'request-2', generation: 'generation-1', seq: 2, dispatchId: 'dispatch-2', + }, + }, 'tag', 0); + await vi.advanceTimersByTimeAsync(10); + + expect(sessionReply).toHaveBeenCalledTimes(2); + expect(sessionReply.mock.calls.map(call => call[4])).toEqual([ + 'om_shared_route', + 'om_shared_route', + ]); + expect(sessionReply.mock.calls.map(call => call[5]?.uuid)).toEqual([ + 'ca_dispatch-1', + 'ca_dispatch-2', + ]); + expect(sessionReply.mock.calls[0][5]).not.toHaveProperty('suppressHook'); + }); + + it('routes sequential Codex App settlements through each ledger-frozen shared-chat root', async () => { + const sessionReply = vi.fn(async () => 'om_reply'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.adoptedFrom = undefined; + ds.scope = 'chat'; + ds.session.scope = 'chat'; + ds.session.cliId = 'codex-app'; + ds.currentReplyTarget = { + rootMessageId: 'om_topic_b', turnId: 'turn-b', updatedAt: new Date().toISOString(), + }; + ds.session.currentReplyTarget = ds.currentReplyTarget; + ds.session.codexAppDispatchLedger = [ + { + dispatchId: 'dispatch-a', turnId: 'turn-a', state: 'prepared', content: 'A', + replyTarget: { mode: 'thread', rootMessageId: 'om_topic_a' }, + }, + { + dispatchId: 'dispatch-b', turnId: 'turn-b', state: 'accepted', content: 'B', + replyTarget: { mode: 'thread', rootMessageId: 'om_topic_b' }, + }, + ]; + __testOnly_setupWorkerHandlers(ds, ds.worker as any); + + (ds.worker as any).emit('message', { + type: 'final_output', sessionId: ds.session.sessionId, + content: 'answer A', lastUuid: 'uuid-a', turnId: 'turn-a', + codexAppSettlement: { + requestId: 'settle-a', generation: 'generation-1', seq: 1, dispatchId: 'dispatch-a', + }, + } satisfies Extract); + await vi.advanceTimersByTimeAsync(10); + await vi.waitFor(() => expect(sessionReply).toHaveBeenCalledTimes(1)); + expect(sessionReply.mock.calls[0][5]?.replyTarget) + .toEqual({ mode: 'thread', rootMessageId: 'om_topic_a' }); + + (ds.worker as any).emit('message', { + type: 'codex_app_dispatch_transition', + sessionId: ds.session.sessionId, + requestId: 'prepare-b', + operation: 'submit', + entries: [{ dispatchId: 'dispatch-b', turnId: 'turn-b' }], + } satisfies Extract); + await vi.waitFor(() => expect(ds.session.codexAppDispatchLedger?.[0]?.state).toBe('prepared')); + (ds.worker as any).emit('message', { + type: 'final_output', sessionId: ds.session.sessionId, + content: 'answer B', lastUuid: 'uuid-b', turnId: 'turn-b', + codexAppSettlement: { + requestId: 'settle-b', generation: 'generation-1', seq: 2, dispatchId: 'dispatch-b', + }, + } satisfies Extract); + await vi.advanceTimersByTimeAsync(10); + await vi.waitFor(() => expect(sessionReply).toHaveBeenCalledTimes(2)); + expect(sessionReply.mock.calls[1][5]?.replyTarget) + .toEqual({ mode: 'thread', rootMessageId: 'om_topic_b' }); + }); + + it.each(['doc_comment', 'http_wait', 'http_async', 'suppressed'] as const)( + 'fails closed instead of leaking a recovered %s settlement into Lark', + async deliverySink => { + const sessionReply = vi.fn(async () => 'om_forbidden'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.adoptedFrom = undefined; + ds.scope = 'chat'; + ds.session.scope = 'chat'; + ds.session.cliId = 'codex-app'; + ds.session.codexAppDispatchLedger = [{ + dispatchId: `dispatch-${deliverySink}`, + turnId: `turn-${deliverySink}`, + state: 'prepared', + content: 'recovered payload', + deliverySink, + }]; + __testOnly_setupWorkerHandlers(ds, ds.worker as any); + + (ds.worker as any).emit('message', { + type: 'final_output', + sessionId: ds.session.sessionId, + content: 'must not cross channels', + lastUuid: `uuid-${deliverySink}`, + turnId: `turn-${deliverySink}`, + codexAppSettlement: { + requestId: `settle-${deliverySink}`, + generation: 'generation-recovered', + seq: 1, + dispatchId: `dispatch-${deliverySink}`, + }, + } satisfies Extract); + + await vi.waitFor(() => expect(ds.session.codexAppDispatchLedger).toEqual([])); + expect(sessionReply).not.toHaveBeenCalled(); + await vi.waitFor(() => expect((ds.worker as any).send).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'codex_app_dispatch_persisted', + requestId: `settle-${deliverySink}`, + ok: true, + }), + )); + }, + ); + + it('still resolves a live HTTP wait sink without posting to Lark', async () => { + const sessionReply = vi.fn(async () => 'om_forbidden'); + const resolveWait = vi.fn(); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.adoptedFrom = undefined; + ds.session.cliId = 'codex-app'; + ds.pendingWaitPromises = new Map([['turn-wait-live', { resolve: resolveWait }]]); + ds.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-wait-live', + turnId: 'turn-wait-live', + state: 'prepared', + content: 'live payload', + deliverySink: 'http_wait', + }]; + __testOnly_setupWorkerHandlers(ds, ds.worker as any); + + (ds.worker as any).emit('message', { + type: 'final_output', sessionId: ds.session.sessionId, + content: 'HTTP answer', lastUuid: 'uuid-wait-live', turnId: 'turn-wait-live', + codexAppSettlement: { + requestId: 'settle-wait-live', generation: 'generation-live', seq: 1, + dispatchId: 'dispatch-wait-live', + }, + } satisfies Extract); + + await vi.waitFor(() => expect(resolveWait).toHaveBeenCalledWith('HTTP answer')); + await vi.waitFor(() => expect(ds.session.codexAppDispatchLedger).toEqual([])); + expect(sessionReply).not.toHaveBeenCalled(); + }); + + it('abandons a delayed final before external delivery when worker/session ownership changes', async () => { + const sessionReply = vi.fn(async () => 'om_reply'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + const { __testOnly_deliverFinalOutput } = await import('../src/core/worker-pool.js') as any; + let owned = true; + const complete = vi.fn(); + + __testOnly_deliverFinalOutput(ds, finalOutputMsg(), 'tag', 0, complete, () => owned); + owned = false; + await vi.advanceTimersByTimeAsync(10); + + expect(sessionReply).not.toHaveBeenCalled(); + expect(complete).toHaveBeenCalledWith(false); + expect(ds.lastBridgeEmittedUuid).toBeUndefined(); + }); + it('drops final_output whose worker sessionId does not match the daemon session', async () => { const sessionReply = vi.fn(async () => 'om_reply'); initWorkerPool({ @@ -1197,6 +1414,7 @@ describe('Bridge final_output delivery (P2 retry)', () => { }); const ds = makeDs(); + const worker = ds.worker as any; const { __testOnly_deliverFinalOutput } = await import('../src/core/worker-pool.js') as any; __testOnly_deliverFinalOutput(ds, finalOutputMsg(), 'tag', 0); @@ -1206,6 +1424,37 @@ describe('Bridge final_output delivery (P2 retry)', () => { expect(sessionReply).toHaveBeenCalledTimes(1); expect(ds.lastBridgeEmittedUuid).toBe(SCOPED_DEDUPE_KEY); expect(closeSession).toHaveBeenCalledWith(ds); + expect(worker.send).not.toHaveBeenCalledWith({ type: 'close' }); + expect(worker.kill).not.toHaveBeenCalled(); + }); + + it('preserves an unsettled Codex FIFO owner when the root is withdrawn', async () => { + const sessionReply = vi.fn().mockRejectedValue(new MessageWithdrawnError('om_root')); + const closeSession = vi.fn(); + const complete = vi.fn(); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession, + }); + const ds = makeDs(); + ds.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-pending', + turnId: 'turn-1', + state: 'prepared', + content: 'pending answer', + }]; + const { __testOnly_deliverFinalOutput } = await import('../src/core/worker-pool.js') as any; + __testOnly_deliverFinalOutput(ds, finalOutputMsg(), 'tag', 0, complete); + await vi.advanceTimersByTimeAsync(0); + + expect(sessionReply).toHaveBeenCalledOnce(); + expect(closeSession).not.toHaveBeenCalled(); + expect((ds.worker as any).kill).not.toHaveBeenCalled(); + expect(ds.session.codexAppDispatchLedger).toHaveLength(1); + expect(ds.lastBridgeEmittedUuid).toBeUndefined(); + expect(complete).toHaveBeenCalledWith(false); }); it('aborts pending retry if the session was closed in the meantime', async () => { diff --git a/test/card-builder.test.ts b/test/card-builder.test.ts index 40bf6f544..6c23d40b2 100644 --- a/test/card-builder.test.ts +++ b/test/card-builder.test.ts @@ -520,6 +520,19 @@ describe('buildStreamingCard', () => { expect(card.header.title.content).toContain('工作中'); }); + it('shows a red, neutral no-progress label for stalled turns', () => { + const zh = parse(buildStreamingCard(SID, ROOT, URL, TITLE, '', 'stalled')); + expect(zh.header.template).toBe('red'); + expect(zh.header.title.content).toContain('长时间无进展'); + + const en = parse(buildStreamingCard( + SID, ROOT, URL, TITLE, '', 'stalled', undefined, 'hidden', + undefined, undefined, false, false, 'en', + )); + expect(en.header.template).toBe('red'); + expect(en.header.title.content).toContain('No recent progress'); + }); + it('should show green template and "等待输入" for idle status', () => { const card = parse(buildStreamingCard(SID, ROOT, URL, TITLE, '', 'idle')); expect(card.header.template).toBe('green'); diff --git a/test/card-handler-repo-select.test.ts b/test/card-handler-repo-select.test.ts index 52f14192c..250646b39 100644 --- a/test/card-handler-repo-select.test.ts +++ b/test/card-handler-repo-select.test.ts @@ -61,7 +61,27 @@ vi.mock('../src/services/session-store.js', () => ({ getSession: vi.fn(), })); -vi.mock('../src/core/worker-pool.js', () => ({ +vi.mock('../src/core/worker-pool.js', () => { + const lockTails = new WeakMap, Map>>(); + const withActiveSessionKeyLock = vi.fn(async (map: Map, key: string, action: () => T | Promise) => { + let tails = lockTails.get(map); + if (!tails) { + tails = new Map(); + lockTails.set(map, tails); + } + const previous = tails.get(key) ?? Promise.resolve(); + let release!: () => void; + const hold = new Promise(resolve => { release = resolve; }); + const tail = previous.catch(() => {}).then(() => hold); + tails.set(key, tail); + await previous.catch(() => {}); + try { return await action(); } + finally { + release(); + if (tails.get(key) === tail) tails.delete(key); + } + }); + return { forkWorker: vi.fn(), killWorker: vi.fn(), scheduleCardPatch: vi.fn(), @@ -72,8 +92,11 @@ vi.mock('../src/core/worker-pool.js', () => ({ resolvePrivateCardAudience: vi.fn(() => []), deliverWriteLinkCard: vi.fn(), deliverEphemeralOrReply: vi.fn(), + closeSession: vi.fn(async () => ({ ok: true, alreadyClosed: false })), + withActiveSessionKeyLock, CARD_POSTING_SENTINEL: '__posting__', -})); + }; +}); vi.mock('../src/core/session-manager.js', () => ({ getSessionWorkingDir: vi.fn(() => '/tmp'), @@ -93,6 +116,11 @@ vi.mock('../src/im/lark/event-dispatcher.js', () => ({ vi.mock('../src/core/session-activity.js', () => ({ publishAttentionPatch: vi.fn(), + announcePendingRepoSession: vi.fn(), +})); + +vi.mock('../src/services/default-worktree.js', () => ({ + maybeCreateDefaultWorktree: vi.fn(), })); vi.mock('../src/services/frozen-card-store.js', () => ({ @@ -123,11 +151,11 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ // ─── Imports ────────────────────────────────────────────────────────────── -import { handleCardAction, type CardHandlerDeps } from '../src/im/lark/card-handler.js'; -import { forkWorker, killWorker, deliverEphemeralOrReply, deliverWriteLinkCard } from '../src/core/worker-pool.js'; +import { handleCardAction, runAutoWorktreeCommit, type CardHandlerDeps } from '../src/im/lark/card-handler.js'; +import { forkWorker, killWorker, deliverEphemeralOrReply, deliverWriteLinkCard, closeSession as closeWorkerPoolSession, withActiveSessionKeyLock } from '../src/core/worker-pool.js'; import { buildNewTopicCliInput, getAvailableBots, getSessionWorkingDir } from '../src/core/session-manager.js'; import { getBot } from '../src/bot-registry.js'; -import { createSession, closeSession, updateSession } from '../src/services/session-store.js'; +import { createSession, updateSession } from '../src/services/session-store.js'; import { createRepoWorktree, pushWorktreeBranch, removeRepoWorktree } from '../src/services/git-worktree.js'; import { applyConfigField } from '../src/services/bot-config-store.js'; import { deleteMessage } from '../src/im/lark/client.js'; @@ -138,6 +166,11 @@ import type { ProjectInfo } from '../src/services/project-scanner.js'; import { mkdtempSync, rmSync } from 'node:fs'; import { homedir, tmpdir } from 'node:os'; import { basename, join } from 'node:path'; +import { maybeCreateDefaultWorktree } from '../src/services/default-worktree.js'; +import { + __testOnly_resetBotTurnMutationGates, + withBotTurnMutation, +} from '../src/core/bot-turn-mutation-gate.js'; // ─── Helpers ────────────────────────────────────────────────────────────── @@ -172,8 +205,9 @@ function makeDs(overrides?: Partial): DaemonSession { cliVersion: '1.0.0', lastMessageAt: Date.now(), hasHistory: true, - // Match makeSelectEvent / makeSkipEvent / makeManualEvent open_message_id — - // only the live posted card may drive selection. + // Every card callback below carries context.open_message_id=om_card. The + // production handler now requires that capability to match the currently + // published picker exactly, so the fixture must model a real live card. repoCardMessageId: 'om_card', ...overrides, } as unknown as DaemonSession; @@ -235,6 +269,7 @@ function deferred() { } beforeEach(() => { + __testOnly_resetBotTurnMutationGates(); vi.clearAllMocks(); vi.mocked(deleteMessage).mockReset().mockResolvedValue(true); vi.mocked(getBot).mockImplementation(() => ({ @@ -266,6 +301,42 @@ afterEach(async () => { // ─── Tests ──────────────────────────────────────────────────────────────── describe('repo select card — plain switch', () => { + it('rejects a callback from any card id other than the currently published picker', async () => { + const ds = makeDs({ + pendingRepo: true, + pendingPrompt: 'OPENING_N', + worker: null, + repoCardMessageId: 'om_current_picker', + }); + const { deps, sessionReply } = makeDeps(ds); + const stale = makeSelectEvent('repo_switch', '/repos/alpha'); + stale.context.open_message_id = 'om_stale_picker'; + + await handleCardAction(stale, deps, APP_ID); + + expect(ds.pendingRepo).toBe(true); + expect(ds.pendingPrompt).toBe('OPENING_N'); + expect(ds.repoCardMessageId).toBe('om_current_picker'); + expect(forkWorker).not.toHaveBeenCalled(); + expect(sessionReply).not.toHaveBeenCalled(); + }); + + it('invalidates the exact picker before awaiting confirmation and keeps replays inert when confirmation fails', async () => { + const ds = makeDs({ pendingRepo: true, pendingPrompt: 'OPENING_N', worker: null }); + const { deps, sessionReply } = makeDeps(ds); + sessionReply.mockRejectedValueOnce(new Error('confirmation unavailable')); + const event = makeSelectEvent('repo_switch', '/repos/alpha'); + + await expect(handleCardAction(event, deps, APP_ID)).resolves.toBeUndefined(); + + expect(forkWorker).toHaveBeenCalledTimes(1); + expect(ds.pendingRepo).toBe(false); + expect(ds.repoCardMessageId).toBeUndefined(); + + await handleCardAction(event, deps, APP_ID); + expect(forkWorker).toHaveBeenCalledTimes(1); + }); + it('pendingRepo selection forks the CLI with the buffered prompt', async () => { const ds = makeDs({ pendingRepo: true, @@ -326,7 +397,7 @@ describe('repo select card — plain switch', () => { content: 'mock-prompt', codexAppInput, }); - expect(vi.mocked(forkWorker).mock.calls[0]).toHaveLength(2); + expect(vi.mocked(forkWorker).mock.calls[0]![2]).toBe(false); expect(vi.mocked(buildNewTopicCliInput).mock.calls[0]![11]).toEqual(expect.objectContaining({ substituteTrigger, })); @@ -373,24 +444,55 @@ describe('repo select card — plain switch', () => { })); }); - it('keeps the pending repo card untouched when skip_repo is clicked while a worktree is being created', async () => { + it('keeps the pending reservation and opening buffers when forkWorker throws synchronously', async () => { const ds = makeDs({ pendingRepo: true, - pendingPrompt: 'hello world', - pendingTurnId: 'om_pending_turn', - repoCardMessageId: 'om_card', - worktreeCreating: true, + initialStartPending: true, + pendingPrompt: 'first prompt', + pendingFollowUps: ['buffered follow-up'], worker: null, }); const { deps } = makeDeps(ds); + vi.mocked(forkWorker).mockImplementationOnce(() => { + expect(ds.pendingRepo).toBe(true); + expect(ds.initialStartPending).toBe(true); + expect(ds.pendingPrompt).toBe('first prompt'); + expect(ds.pendingFollowUps).toEqual(['buffered follow-up']); + throw new Error('fork preaccept failed'); + }); + + await expect(handleCardAction( + makeSelectEvent('repo_switch', '/repos/alpha'), + deps, + APP_ID, + )).rejects.toThrow('fork preaccept failed'); - const result = await handleCardAction(makeSkipEvent(), deps, APP_ID); + expect(ds.pendingRepo).toBe(true); + expect(ds.initialStartPending).toBe(true); + expect(ds.pendingPrompt).toBe('first prompt'); + expect(ds.pendingFollowUps).toEqual(['buffered follow-up']); + }); + + it('skip_repo keeps its reservation through roster lookup and cannot resurrect a closed session', async () => { + const ds = makeDs({ pendingRepo: true, pendingPrompt: 'first prompt', worker: null }); + const { deps, sessionReply } = makeDeps(ds); + const roster = deferred(); + vi.mocked(getAvailableBots).mockReturnValueOnce(roster.promise); + + const action = handleCardAction(makeSkipEvent(), deps, APP_ID); + await vi.waitFor(() => expect(getAvailableBots).toHaveBeenCalledTimes(1)); + expect(ds.pendingRepo).toBe(true); + expect(ds.pendingPrompt).toBe('first prompt'); + + deps.activeSessions.delete(sessionKey(ROOT_ID, APP_ID)); + ds.session.status = 'closed'; + roster.resolve([]); + await action; - expect(result?.toast?.content).toContain('已有一个 worktree 正在创建'); expect(forkWorker).not.toHaveBeenCalled(); expect(ds.pendingRepo).toBe(true); - expect(ds.pendingPrompt).toBe('hello world'); - expect(ds.pendingTurnId).toBe('om_pending_turn'); + expect(ds.pendingPrompt).toBe('first prompt'); + expect(ds.pendingTurnId).toBeUndefined(); expect(ds.repoCardMessageId).toBe('om_card'); expect(deleteMessage).not.toHaveBeenCalled(); }); @@ -474,7 +576,7 @@ describe('repo select card — plain switch', () => { const late = await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); // After successful fork the card is marked consumed before confirm reply, // so the second click is rejected even while claim is still held. - expect(late?.toast?.content).toMatch(/仓库已选定|worktree 正在创建|ignore the old card/i); + expect(late?.toast?.content).toMatch(/失效|最新卡片|仓库已选定|worktree 正在创建|ignore the old card/i); expect(forkWorker).toHaveBeenCalledTimes(1); expect(killWorker).not.toHaveBeenCalled(); expect(createSession).not.toHaveBeenCalled(); @@ -552,7 +654,7 @@ describe('repo select card — plain switch', () => { expect(ds.session.sessionId).toBe('uuid-old'); const lateWhileDeletePending = await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); - expect(lateWhileDeletePending?.toast?.content).toMatch(/仓库已选定|worktree 正在创建|ignore the old card/i); + expect(lateWhileDeletePending?.toast?.content).toMatch(/失效|最新卡片|仓库已选定|worktree 正在创建|ignore the old card/i); expect(killWorker).not.toHaveBeenCalled(); expect(createSession).not.toHaveBeenCalled(); expect(forkWorker).toHaveBeenCalledTimes(1); @@ -567,7 +669,7 @@ describe('repo select card — plain switch', () => { // After claim release, Feishu may still show the card (delete returned false). // Consume mark alone must keep rejecting. const lateAfter = await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); - expect(lateAfter?.toast?.content).toMatch(/仓库已选定|ignore the old card/i); + expect(lateAfter?.toast?.content).toMatch(/失效|最新卡片|仓库已选定|ignore the old card/i); expect(killWorker).not.toHaveBeenCalled(); expect(createSession).not.toHaveBeenCalled(); expect(forkWorker).toHaveBeenCalledTimes(1); @@ -632,7 +734,7 @@ describe('repo select card — plain switch', () => { expect(ds.workingDir).toBe('/repos/alpha'); const late = await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); - expect(late?.toast?.content).toMatch(/仓库已选定|ignore the old card/i); + expect(late?.toast?.content).toMatch(/失效|最新卡片|仓库已选定|ignore the old card/i); expect(killWorker).not.toHaveBeenCalled(); expect(createSession).not.toHaveBeenCalled(); expect(forkWorker).toHaveBeenCalledTimes(1); @@ -679,8 +781,8 @@ describe('repo select card — plain switch', () => { await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); - expect(killWorker).toHaveBeenCalledTimes(1); - expect(closeSession).toHaveBeenCalledWith('uuid-old'); + expect(killWorker).not.toHaveBeenCalled(); + expect(closeWorkerPoolSession).toHaveBeenCalledWith('uuid-old'); expect(ds.session.sessionId).toMatch(/^uuid-new-/); expect(ds.workingDir).toBe('/repos/beta'); expect(ds.session.workingDir).toBe('/repos/beta'); @@ -725,8 +827,9 @@ describe('repo select card — plain switch', () => { const sessionAfterFirst = ds.session.sessionId; const late = await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); - expect(late?.toast?.content).toMatch(/仓库已选定|ignore the old card/i); - expect(killWorker).toHaveBeenCalledTimes(1); + expect(late?.toast?.content).toMatch(/失效|最新卡片|仓库已选定|ignore the old card/i); + expect(killWorker).not.toHaveBeenCalled(); + expect(closeWorkerPoolSession).toHaveBeenCalledTimes(1); expect(createSession).toHaveBeenCalledTimes(1); expect(forkWorker).toHaveBeenCalledTimes(1); expect(ds.session.sessionId).toBe(sessionAfterFirst); @@ -734,7 +837,8 @@ describe('repo select card — plain switch', () => { releaseReply!(); await first; - expect(killWorker).toHaveBeenCalledTimes(1); + expect(killWorker).not.toHaveBeenCalled(); + expect(closeWorkerPoolSession).toHaveBeenCalledTimes(1); expect(forkWorker).toHaveBeenCalledTimes(1); expect(ds.session.sessionId).toBe(sessionAfterFirst); }); @@ -747,7 +851,7 @@ describe('repo select card — plain switch', () => { const { deps } = makeDeps(ds); const late = await handleCardAction(makeSelectEvent('repo_switch', '/repos/beta'), deps, APP_ID); - expect(late?.toast?.content).toMatch(/仓库已选定|ignore the old card/i); + expect(late?.toast?.content).toMatch(/失效|最新卡片|仓库已选定|ignore the old card/i); expect(killWorker).not.toHaveBeenCalled(); expect(createSession).not.toHaveBeenCalled(); expect(forkWorker).not.toHaveBeenCalled(); @@ -795,6 +899,39 @@ describe('repo select card — plain switch', () => { })); }); + it('holds the canonical anchor lock across mid-session close and replacement publication', async () => { + const ds = makeDs(); + const { deps } = makeDeps(ds); + const close = deferred(); + vi.mocked(closeWorkerPoolSession).mockReturnValueOnce(close.promise); + + const switching = handleCardAction( + makeSelectEvent('repo_switch', '/repos/beta'), + deps, + APP_ID, + ); + await vi.waitFor(() => expect(closeWorkerPoolSession).toHaveBeenCalledWith('uuid-old')); + + let contenderEntered = false; + const contender = withActiveSessionKeyLock( + deps.activeSessions, + sessionKey(ROOT_ID, APP_ID), + () => { + contenderEntered = true; + return deps.activeSessions.get(sessionKey(ROOT_ID, APP_ID)); + }, + ); + await Promise.resolve(); + expect(contenderEntered).toBe(false); + + close.resolve({ ok: true, alreadyClosed: false }); + await switching; + const ownerAfterSwitch = await contender; + expect(contenderEntered).toBe(true); + expect(ownerAfterSwitch).toBe(ds); + expect(ds.session.sessionId).toMatch(/^uuid-new-/); + }); + it('ignores a keyless dropdown (option + root_id, no repo_switch/repo_worktree key)', async () => { // Security seal: a hand-crafted card (e.g. via `botmux send --card-json`) can // supply a bare `option + value.root_id` with no recognized key. It must NOT @@ -818,6 +955,30 @@ describe('repo select card — plain switch', () => { }); describe('repo select card — worktree open', () => { + it('rejects a stale direct single-select picker before any worktree side effect', async () => { + const ds = makeDs({ + pendingRepo: true, + pendingPrompt: 'hi', + worker: null, + repoCardMessageId: 'om_current_picker', + }); + const { deps, sessionReply } = makeDeps(ds); + + const result = await handleCardAction( + makeSelectEvent('repo_worktree', '/repos/alpha'), + deps, + APP_ID, + ); + + expect(result?.toast?.type).toBe('warning'); + expect(createRepoWorktree).not.toHaveBeenCalled(); + expect(pushWorktreeBranch).not.toHaveBeenCalled(); + expect(sessionReply).not.toHaveBeenCalled(); + expect(forkWorker).not.toHaveBeenCalled(); + expect(killWorker).not.toHaveBeenCalled(); + expect(ds.worktreeCreating).not.toBe(true); + }); + it('double click starts ONE background creation and commits once', async () => { const ds = makeDs({ pendingRepo: true, pendingPrompt: 'hi', worker: null }); const { deps, sessionReply } = makeDeps(ds); @@ -1131,7 +1292,7 @@ describe('repo select card — worktree open', () => { await handleCardAction(makeWorktreeSubmitEvent('feat/mid', ['/repos/alpha', '/repos/beta']), deps, APP_ID); await vi.waitFor(() => expect(ds.worktreeCreating).toBe(false)); - expect(closeSession).toHaveBeenCalledWith('uuid-old'); + expect(closeWorkerPoolSession).toHaveBeenCalledWith('uuid-old'); expect(ds.session).not.toBe(oldSession); expect(oldSession.riffRepoDirs).toBeUndefined(); expect(ds.session.riffRepoDirs).toEqual([ @@ -1261,14 +1422,37 @@ describe('repo select card — worktree open', () => { expect(ds.workingDir).toBe('/repos/alpha-feat-one'); }); + it('rejects a stale worktree form before slug generation or git work', async () => { + const ds = makeDs({ + pendingRepo: true, + pendingPrompt: 'hi', + worker: null, + repoCardMessageId: 'om_current_picker', + }); + const { deps, sessionReply } = makeDeps(ds); + + const result = await handleCardAction( + makeWorktreeSubmitEvent('feat/stale', ['/repos/alpha']), + deps, + APP_ID, + ); + + expect(result?.toast?.type).toBe('warning'); + expect(createRepoWorktree).not.toHaveBeenCalled(); + expect(pushWorktreeBranch).not.toHaveBeenCalled(); + expect(sessionReply).not.toHaveBeenCalled(); + expect(forkWorker).not.toHaveBeenCalled(); + expect(ds.worktreeCreating).not.toBe(true); + }); + it('worktree_toggle_mode flips the persisted picker mode and re-sends a fresh repo card', async () => { - // Callback open_message_id must match the live posted card. - const ds = makeDs({ pendingRepo: true, pendingPrompt: 'hi', worker: null, repoCardMessageId: 'om_card' }); + const ds = makeDs({ pendingRepo: true, pendingPrompt: 'hi', worker: null, repoCardMessageId: 'om_old_card' }); + ds.session.pendingRepoSetup = { mode: 'picker', prompt: 'hi', repoCardMessageId: 'om_old_card' }; const { deps, sessionReply } = makeDeps(ds); const event = { operator: { open_id: OWNER }, action: { value: { action: 'worktree_toggle_mode', root_id: ROOT_ID } }, - context: { open_message_id: 'om_card' }, + context: { open_message_id: 'om_old_card' }, }; const res = await handleCardAction(event, deps, APP_ID); @@ -1277,51 +1461,69 @@ describe('repo select card — worktree open', () => { // persisted the flipped mode (config undefined → true) expect(vi.mocked(applyConfigField)).toHaveBeenCalledWith('app_test', expect.objectContaining({ configKey: 'worktreeMultiPicker' }), true); // withdrew the old card and posted a fresh interactive repo card - expect(vi.mocked(deleteMessage)).toHaveBeenCalledWith('app_test', 'om_card'); + expect(vi.mocked(deleteMessage)).toHaveBeenCalledWith('app_test', 'om_old_card'); const interactiveCall = sessionReply.mock.calls.find(c => c[2] === 'interactive'); expect(interactiveCall).toBeDefined(); + expect(ds.repoCardMessageId).toBe('om_reply'); + expect(ds.session.pendingRepoSetup.repoCardMessageId).toBe('om_reply'); expect(createRepoWorktree).not.toHaveBeenCalled(); expect(forkWorker).not.toHaveBeenCalled(); expect(ds.worktreeCreating).not.toBe(true); }); - it('worktree_toggle_mode rejects a stale/wrong card id before flipping config', async () => { - const ds = makeDs({ pendingRepo: true, pendingPrompt: 'hi', worker: null, repoCardMessageId: 'om_live_card' }); - const { deps } = makeDeps(ds); - const event = { + it('rejects a stale worktree mode toggle before config, publish, or deletion', async () => { + const ds = makeDs({ pendingRepo: true, worker: null, repoCardMessageId: 'om_current_picker' }); + const { deps, sessionReply } = makeDeps(ds); + + const result = await handleCardAction({ operator: { open_id: OWNER }, action: { value: { action: 'worktree_toggle_mode', root_id: ROOT_ID } }, - context: { open_message_id: 'om_stale_card' }, - }; + context: { open_message_id: 'om_old_picker' }, + }, deps, APP_ID); - const res = await handleCardAction(event, deps, APP_ID); + expect(result?.toast?.type).toBe('warning'); + expect(applyConfigField).not.toHaveBeenCalled(); + expect(sessionReply).not.toHaveBeenCalled(); + expect(deleteMessage).not.toHaveBeenCalled(); + expect(ds.repoCardMessageId).toBe('om_current_picker'); + }); - expect(res?.toast?.content).toMatch(/仓库已选定|ignore the old card/i); - expect(vi.mocked(applyConfigField)).not.toHaveBeenCalled(); - expect(vi.mocked(deleteMessage)).not.toHaveBeenCalled(); - expect(ds.repoCardMessageId).toBe('om_live_card'); + it('keeps the old picker authoritative when replacement publication fails', async () => { + const ds = makeDs({ pendingRepo: true, pendingPrompt: 'hi', worker: null, repoCardMessageId: 'om_old_card' }); + ds.session.pendingRepoSetup = { mode: 'picker', prompt: 'hi', repoCardMessageId: 'om_old_card' }; + const { deps, sessionReply } = makeDeps(ds); + sessionReply.mockRejectedValueOnce(new Error('publish unavailable')); + + const result = await handleCardAction({ + operator: { open_id: OWNER }, + action: { value: { action: 'worktree_toggle_mode', root_id: ROOT_ID } }, + context: { open_message_id: 'om_old_card' }, + }, deps, APP_ID); + + expect(result?.toast?.type).toBe('error'); + expect(ds.repoCardMessageId).toBe('om_old_card'); + expect(ds.session.pendingRepoSetup.repoCardMessageId).toBe('om_old_card'); + expect(deleteMessage).not.toHaveBeenCalled(); }); - it('worktree_toggle_mode rejects restart-like state with no live repoCardMessageId', async () => { - const ds = makeDs({ - pendingRepo: true, - pendingPrompt: 'hi', - worker: null, - repoCardMessageId: undefined, - consumedRepoCardMessageIds: undefined, - }); + it('keeps the old picker authoritative when the replacement id cannot persist', async () => { + const ds = makeDs({ pendingRepo: true, pendingPrompt: 'hi', worker: null, repoCardMessageId: 'om_old_card' }); + ds.session.pendingRepoSetup = { mode: 'picker', prompt: 'hi', repoCardMessageId: 'om_old_card' }; const { deps } = makeDeps(ds); - const event = { + vi.mocked(updateSession).mockImplementationOnce(() => { + throw new Error('picker id save unavailable'); + }); + + const result = await handleCardAction({ operator: { open_id: OWNER }, action: { value: { action: 'worktree_toggle_mode', root_id: ROOT_ID } }, - context: { open_message_id: 'om_card' }, - }; - - const res = await handleCardAction(event, deps, APP_ID); + context: { open_message_id: 'om_old_card' }, + }, deps, APP_ID); - expect(res?.toast?.content).toMatch(/仓库已选定|ignore the old card/i); - expect(vi.mocked(applyConfigField)).not.toHaveBeenCalled(); - expect(vi.mocked(deleteMessage)).not.toHaveBeenCalled(); + expect(result?.toast?.type).toBe('error'); + expect(ds.repoCardMessageId).toBe('om_old_card'); + expect(ds.session.pendingRepoSetup.repoCardMessageId).toBe('om_old_card'); + expect(deleteMessage).not.toHaveBeenCalled(); }); it('worktree_toggle_mode requires canOperate — a non-operator (even the pending-session owner) cannot flip bot config', async () => { @@ -1384,6 +1586,43 @@ describe('repo select card — worktree open', () => { }); }); +describe('auto-worktree detached commit admission', () => { + it('holds the delayed commit/fork behind a same-bot mutation after the caller lease ended', async () => { + const ds = makeDs({ + pendingRepo: true, + pendingPrompt: 'delayed first turn', + worker: null, + }); + const { deps } = makeDeps(ds); + const { activeSessions } = deps; + const worktreeReady = deferred<{ dir: string }>(); + vi.mocked(maybeCreateDefaultWorktree).mockReturnValueOnce(worktreeReady.promise); + + const detached = runAutoWorktreeCommit({ + ds, + anchor: ROOT_ID, + larkAppId: APP_ID, + baseDir: '/repos/alpha', + prompt: 'delayed first turn', + activeSessions, + notify: vi.fn(), + }); + await vi.waitFor(() => expect(maybeCreateDefaultWorktree).toHaveBeenCalledOnce()); + + const finishMutation = deferred(); + const mutation = withBotTurnMutation(APP_ID, () => finishMutation.promise); + worktreeReady.resolve({ dir: '/repos/alpha-wt' }); + await Promise.resolve(); + await Promise.resolve(); + expect(forkWorker).not.toHaveBeenCalled(); + + finishMutation.resolve(); + await Promise.all([mutation, detached]); + expect(forkWorker).toHaveBeenCalledOnce(); + expect(ds.workingDir).toBe('/repos/alpha-wt'); + }); +}); + describe('repo select card — manual directory entry', () => { let tmpDir: string; beforeEach(() => { tmpDir = mkdtempSync(join(tmpdir(), 'botmux-manual-repo-')); }); @@ -1412,8 +1651,8 @@ describe('repo select card — manual directory entry', () => { await handleCardAction(makeManualEvent(tmpDir), deps, APP_ID); - expect(killWorker).toHaveBeenCalledTimes(1); - expect(closeSession).toHaveBeenCalledWith('uuid-old'); + expect(killWorker).not.toHaveBeenCalled(); + expect(closeWorkerPoolSession).toHaveBeenCalledWith('uuid-old'); expect(ds.session.sessionId).toMatch(/^uuid-new-/); expect(ds.session.workingDir).toBe(tmpDir); expect(forkWorker).toHaveBeenCalledTimes(1); diff --git a/test/card-handler-voice.test.ts b/test/card-handler-voice.test.ts index b25c0759b..8aa6765f3 100644 --- a/test/card-handler-voice.test.ts +++ b/test/card-handler-voice.test.ts @@ -107,6 +107,21 @@ describe('card-handler voice_summary', () => { expect(send).toHaveBeenCalledTimes(1); }); + it('does not consume the dedupe key when live IPC rejects the voice turn', async () => { + const { types, handler } = await fresh(); + const send = vi.fn() + .mockImplementationOnce(() => { throw new Error('worker exited'); }) + .mockImplementationOnce(() => {}); + deps.activeSessions.set(types.sessionKey('om_root', 'h1'), fakeSession(send)); + + const failed = await handler.handleCardAction(voiceAction('om_retryable_voice'), deps, 'h1'); + const retried = await handler.handleCardAction(voiceAction('om_retryable_voice'), deps, 'h1'); + + expect(failed?.toast?.type).toBe('warning'); + expect(retried?.toast?.type).toBe('success'); + expect(send).toHaveBeenCalledTimes(2); + }); + it('a re-click while the voice is still generating (worker back to working) says "already", not "busy"', async () => { const { types, handler } = await fresh(); const send = vi.fn(); @@ -176,6 +191,34 @@ describe('card-handler voice_summary', () => { }); describe('card-handler retry_last_task', () => { + it('keeps retry eligibility and visible state unchanged when live IPC rejects acceptance', async () => { + const { types, handler } = await fresh(); + const send = vi.fn() + .mockImplementationOnce(() => { throw new Error('worker exited'); }) + .mockImplementationOnce(() => {}); + const ds = fakeSession(send); + ds.lastCliInput = 'retry me'; + ds.lastUserPrompt = 'retry me'; + ds.usageLimit = { retryReady: true, retryAtMs: 0, retryLabel: 'now' }; + deps.activeSessions.set(types.sessionKey('om_root', 'h1'), ds); + + await handler.handleCardAction(retryAction(), deps, 'h1'); + + expect(ds.usageLimit).toMatchObject({ retryReady: true, retryLabel: 'now' }); + expect(ds.lastScreenStatus).toBe('idle'); + expect(deps.sessionReply).toHaveBeenCalledWith( + 'om_root', + expect.stringContaining('重试状态已失效'), + undefined, + 'h1', + ); + + await handler.handleCardAction(retryAction(), deps, 'h1'); + expect(send).toHaveBeenCalledTimes(2); + expect(ds.usageLimit).toBeUndefined(); + expect(ds.lastScreenStatus).toBe('working'); + }); + it('preserves the clean Codex App sidecar when retrying a completed turn', async () => { const dir = mkdtempSync(join(tmpdir(), 'botmux-cardretry-clean-')); const cfg = join(dir, 'bots.json'); @@ -214,6 +257,12 @@ describe('card-handler retry_last_task', () => { additionalContext: ds.lastCodexAppInput.additionalContext, }, }); - expect(send.mock.calls[0][0].codexAppInput).not.toHaveProperty('clientUserMessageId'); + // A replay must never reuse the completed native turn's routing id. The + // durable Codex App dispatcher assigns a fresh synthetic identity so this + // retry has its own ledger ownership and final attribution. + expect(send.mock.calls[0][0].codexAppInput.clientUserMessageId) + .toMatch(/^codex-app-dispatch-/); + expect(send.mock.calls[0][0].codexAppInput.clientUserMessageId).not.toBe('om_original'); + expect(send.mock.calls[0][0].codexAppDispatchId).toEqual(expect.any(String)); }); }); diff --git a/test/card-integration.test.ts b/test/card-integration.test.ts index f5bf3943d..b3f7ae49a 100644 --- a/test/card-integration.test.ts +++ b/test/card-integration.test.ts @@ -162,9 +162,8 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ // ─── Imports ────────────────────────────────────────────────────────────── import { handleCardAction, type CardHandlerDeps } from '../src/im/lark/card-handler.js'; -import { scheduleCardPatch } from '../src/core/worker-pool.js'; -import { killWorker, forkWorker } from '../src/core/worker-pool.js'; -import { sessionKey } from '../src/core/types.js'; +import { scheduleCardPatch, forkWorker, setActiveSessionsRegistry } from '../src/core/worker-pool.js'; +import { activeSessionKey, sessionKey } from '../src/core/types.js'; import type { DaemonSession } from '../src/core/types.js'; import { buildStreamingCard } from '../src/im/lark/card-builder.js'; @@ -189,7 +188,14 @@ function makeDaemonSession(overrides?: Partial): DaemonSession { chatType: 'group', scope: 'chat', }, - worker: { killed: false, send: vi.fn() } as any, + worker: { + killed: false, + send: vi.fn(), + kill: vi.fn(), + once: vi.fn(), + exitCode: null, + signalCode: null, + } as any, workerPort: 8080, workerToken: 'tok_secret', larkAppId: APP_ID, @@ -214,6 +220,10 @@ function makeDaemonSession(overrides?: Partial): DaemonSession { function makeDeps(activeSessions: Map): CardHandlerDeps { sessionReplyCallIndex = 0; + // closeSession is intentionally real in this partial worker-pool mock; point + // its authoritative registry at the integration fixture so close-card tests + // exercise the same identity-safe eviction contract as production. + setActiveSessionsRegistry(activeSessions); return { activeSessions, sessionReply: vi.fn(async () => { @@ -471,7 +481,7 @@ describe('Card integration: full event flow', () => { await handleCardAction(makeRestartEvent(ROOT_ID), deps, APP_ID); - expect(workerSend).toHaveBeenCalledWith({ type: 'restart' }); + expect(workerSend).toHaveBeenCalledWith({ type: 'restart', reason: 'operator' }); // The confirmation is delivered ephemeral to the clicker (group chat + an // operator open_id), not as a visible group reply. expect(vi.mocked(clientMod.sendEphemeralCard)).toHaveBeenCalledWith( @@ -491,17 +501,16 @@ describe('Card integration: full event flow', () => { expect(forkWorker).toHaveBeenCalledWith(ds, '', false); }); - it('close should kill worker and remove session', async () => { + it('close should remove session and deliver the closed card', async () => { const clientMod = await import('../src/im/lark/client.js'); const ds = makeDaemonSession(); const sessions = new Map(); - const sKey = sessionKey(ROOT_ID, APP_ID); + const sKey = activeSessionKey(ds); sessions.set(sKey, ds); const deps = makeDeps(sessions); - await handleCardAction(makeCloseEvent(ROOT_ID), deps, APP_ID); + await handleCardAction(makeCloseEvent(ROOT_ID, 'ou_user', undefined, ds.session.sessionId), deps, APP_ID); - expect(killWorker).toHaveBeenCalledWith(ds); expect(sessions.has(sKey)).toBe(false); // Closed reply is an interactive card with a Resume button, delivered // ephemeral to the clicker (group chat + operator open_id); the mocked @@ -525,13 +534,16 @@ describe('Card integration: full event flow', () => { try { const ds = makeDaemonSession(); const sessions = new Map(); - const sKey = sessionKey(ROOT_ID, APP_ID); + const sKey = activeSessionKey(ds); sessions.set(sKey, ds); const deps = makeDeps(sessions); - await handleCardAction(makeCloseEvent(ROOT_ID, 'ou_owner'), deps, APP_ID); + await handleCardAction( + makeCloseEvent(ROOT_ID, 'ou_owner', undefined, ds.session.sessionId), + deps, + APP_ID, + ); - expect(killWorker).toHaveBeenCalledWith(ds); expect(sessions.has(sKey)).toBe(false); // Closed card goes ephemeral to the owner … expect(vi.mocked(clientMod.sendEphemeralCard)).toHaveBeenCalledWith( @@ -566,13 +578,16 @@ describe('Card integration: full event flow', () => { try { const ds = makeDaemonSession(); const sessions = new Map(); - const sKey = sessionKey(ROOT_ID, APP_ID); + const sKey = activeSessionKey(ds); sessions.set(sKey, ds); const deps = makeDeps(sessions); - await handleCardAction(makeCloseEvent(ROOT_ID, 'ou_owner', 'private'), deps, APP_ID); + await handleCardAction( + makeCloseEvent(ROOT_ID, 'ou_owner', 'private', ds.session.sessionId), + deps, + APP_ID, + ); - expect(killWorker).toHaveBeenCalledWith(ds); // Closed card still goes ephemeral to the owner … expect(vi.mocked(clientMod.sendEphemeralCard)).toHaveBeenCalledWith( APP_ID, ds.chatId, 'ou_owner', expect.stringContaining('"type":"closed"'), @@ -788,7 +803,7 @@ describe('Card integration: full event flow', () => { await handleCardAction(makeRestartEvent(ROOT_ID), deps, APP_ID); - expect(workerSend).toHaveBeenCalledWith({ type: 'restart' }); + expect(workerSend).toHaveBeenCalledWith({ type: 'restart', reason: 'operator' }); expect(vi.mocked(clientMod.sendEphemeralCard)).not.toHaveBeenCalled(); expect(deps.sessionReply).toHaveBeenCalledWith( ROOT_ID, expect.stringContaining('重启'), undefined, APP_ID, @@ -799,13 +814,12 @@ describe('Card integration: full event flow', () => { const clientMod = await import('../src/im/lark/client.js'); const ds = makeDaemonSession({ scope: 'thread' }); const sessions = new Map(); - const sKey = sessionKey(ROOT_ID, APP_ID); + const sKey = activeSessionKey(ds); sessions.set(sKey, ds); const deps = makeDeps(sessions); await handleCardAction(makeCloseEvent(ROOT_ID), deps, APP_ID); - expect(killWorker).toHaveBeenCalledWith(ds); expect(sessions.has(sKey)).toBe(false); expect(vi.mocked(clientMod.sendEphemeralCard)).not.toHaveBeenCalled(); expect(deps.sessionReply).toHaveBeenCalledWith( @@ -1137,6 +1151,7 @@ describe('Card integration: full event flow', () => { // sessionReply was used to surface the rejection message. expect(deps.sessionReply).toHaveBeenCalled(); }); + }); describe('Scenario 8: usage-limit retry action', () => { diff --git a/test/cli-send-hook-context.test.ts b/test/cli-send-hook-context.test.ts index cd2dfd9bf..610373778 100644 --- a/test/cli-send-hook-context.test.ts +++ b/test/cli-send-hook-context.test.ts @@ -1,11 +1,42 @@ -import { readFileSync } from 'node:fs'; +import { spawn, spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; +import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { describe, expect, it } from 'vitest'; const __dirname = dirname(fileURLToPath(import.meta.url)); const cliSource = readFileSync(join(__dirname, '..', 'src', 'cli.ts'), 'utf8'); +function runCli(args: string[], env: NodeJS.ProcessEnv): Promise<{ + code: number | null; + stdout: string; + stderr: string; +}> { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [ + '--import', 'tsx', join(__dirname, '..', 'src', 'cli.ts'), ...args, + ], { + cwd: join(__dirname, '..'), + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { stdout += String(chunk); }); + child.stderr.on('data', chunk => { stderr += String(chunk); }); + const timer = setTimeout(() => { + child.kill('SIGKILL'); + reject(new Error(`CLI timed out: ${stderr}`)); + }, 10_000); + child.on('error', reject); + child.on('close', code => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + }); +} + describe('cmdSend hook context wiring', () => { it('repairs scope-less chat records in both CLI session file loaders', () => { const loadSessionsStart = cliSource.indexOf('function loadSessions()'); @@ -32,6 +63,196 @@ describe('cmdSend hook context wiring', () => { expect(cliSource).toMatch(/mentions\.push\(\{ open_id: replyTargetSenderOpenId, name: '' \}\)/); }); + it('prefers the current Codex App ledger entry over mutable shared-chat reply state', () => { + expect(cliSource).toContain( + 'originSession?.codexAppDispatchLedger', + ); + expect(cliSource).toContain("turnMatches.filter(entry => entry.dispatchAttempt === originDispatchAttempt)"); + expect(cliSource).toContain('if (exactMatches.length !== 1)'); + expect(cliSource).toContain('const frozenTurnDispatch = originSessionId === sid'); + expect(cliSource).toMatch( + /const sendTarget = !sendInto && !sendTopLevel && !overrideChatId && frozenTurnReplyTarget\s*\? frozenTurnReplyTarget/, + ); + expect(cliSource).toContain('?? frozenTurnDispatch?.replyTargetSenderOpenId'); + expect(cliSource).toContain('?? frozenTurnDispatch?.quoteTargetId'); + expect(cliSource).toContain('lastCallerOpenId: frozenTurnDispatch.replyTargetSenderOpenId'); + expect(cliSource).toContain('lastCallerIsBot: frozenTurnDispatch.replyTargetSenderIsBot'); + }); + + it('fails closed when a durable turn is bound to a non-Lark delivery sink', () => { + expect(cliSource).toContain("exactOriginDispatch?.deliverySink === 'http_wait'"); + expect(cliSource).toContain("exactOriginDispatch?.deliverySink === 'http_async'"); + expect(cliSource).toContain("exactOriginDispatch?.deliverySink === 'suppressed'"); + expect(cliSource).toContain("exactOriginDispatch?.deliverySink === 'doc_comment'"); + expect(cliSource).toContain('a document-comment turn supports only its exact plain-text comment reply'); + }); + + it('retains per-turn document-comment routing for non-Codex CLI adapters only', () => { + expect(cliSource).toContain("originSession?.cliId !== 'codex-app' && !!docTarget"); + expect(cliSource).toContain('const isOriginDocCommentTurn ='); + expect(cliSource.match(/if \(isOriginDocCommentTurn\)/g)).toHaveLength(2); + }); + + it('binds delivery-sink authority to the trusted origin, not --session-id destination', () => { + const cmdSendStart = cliSource.indexOf('async function cmdSend('); + const cmdDispatchStart = cliSource.indexOf('async function cmdDispatch(', cmdSendStart); + const cmdSend = cliSource.slice(cmdSendStart, cmdDispatchStart); + expect(cmdSend).toContain('const originSessionId = trustedRelayCtx?.sessionId'); + expect(cmdSend).toContain('const authoritativeOriginTurnCtx = trustedRelayCtx'); + expect(cmdSend).toContain(': (liveMarkerCtx?.turnId'); + expect(cmdSend).not.toMatch(/const originSessionId =[\s\S]{0,200}sessionIdArg/); + expect(cmdSend).toContain('const sid = sessionIdArg ?? ancestorCtx?.sessionId'); + expect(cmdSend).toContain('const exactOriginDispatch = (() => {'); + expect(cmdSend).toContain('if (sid !== originSessionId'); + expect(cmdSend).toContain('const exactDocSession = originSession!;'); + }); + + it('does not promote detached spawn-time turn env while durable output is unsettled', () => { + const dataDir = mkdtempSync(join(tmpdir(), 'botmux-send-stale-origin-')); + try { + writeFileSync(join(dataDir, 'sessions-app-a.json'), JSON.stringify({ + origin: { + sessionId: 'origin', + chatId: 'oc_origin', + rootMessageId: 'om_origin', + title: 'origin', + status: 'active', + createdAt: new Date(0).toISOString(), + larkAppId: 'app-a', + codexAppDispatchLedger: [{ + dispatchId: 'dispatch-live', + turnId: 'turn-live', + dispatchAttempt: 1, + state: 'accepted', + content: 'private result', + deliverySink: 'http_wait', + }], + }, + destination: { + sessionId: 'destination', + chatId: 'oc_destination', + rootMessageId: 'om_destination', + title: 'destination', + status: 'active', + createdAt: new Date(0).toISOString(), + larkAppId: 'app-a', + }, + })); + const result = spawnSync(process.execPath, [ + '--import', 'tsx', + join(__dirname, '..', 'src', 'cli.ts'), + 'send', 'must-not-leak', '--session-id', 'destination', '--no-mention', + ], { + cwd: join(__dirname, '..'), + encoding: 'utf8', + env: { + ...process.env, + SESSION_DATA_DIR: dataDir, + BOTMUX_SESSION_ID: 'origin', + // These are inherited spawn-time fallbacks, not a live marker or + // protected capability. They must not select (or bypass) a sink. + BOTMUX_TURN_ID: 'turn-stale', + BOTMUX_DISPATCH_ATTEMPT: '99', + BOTMUX_HOST_RELAY_AUTHORIZED: '', + BOTMUX_SEND_RELAY: '', + BOTMUX_WORKFLOW: '', + BOTMUX_LARK_APP_ID: '', + BOTMUX_LARK_APP_SECRET: '', + }, + }); + expect(result.status).toBe(2); + expect(result.stderr).toContain('unsettled durable output but no fresh authoritative dispatch identity'); + expect(result.stderr).not.toContain('must-not-leak'); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); + + + it('rejects a trusted host re-exec when its authorized Codex App ledger was already settled', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'botmux-send-host-ledger-gone-')); + writeFileSync(join(dataDir, 'sessions-app-a.json'), JSON.stringify({ + session: { + sessionId: 'session', chatId: 'oc_chat', rootMessageId: 'om_root', + title: 'settled', status: 'active', createdAt: new Date(0).toISOString(), + larkAppId: 'app-a', cliId: 'codex-app', pid: process.pid, + }, + })); + try { + const result = await runCli( + ['send', 'must not downgrade', '--session-id', 'session', '--no-mention'], + { + ...process.env, + SESSION_DATA_DIR: dataDir, + BOTMUX_SESSION_ID: 'session', + BOTMUX_TURN_ID: 'turn-settled', + BOTMUX_DISPATCH_ATTEMPT: '4', + BOTMUX_HOST_RELAY_AUTHORIZED: '1', + BOTMUX_HOST_RELAY_REQUIRES_CODEX_APP_LEDGER: '1', + BOTMUX_SEND_RELAY: '', + BOTMUX_WORKFLOW: '', + BOTMUX_LARK_APP_ID: '', BOTMUX_LARK_APP_SECRET: '', + }, + ); + expect(result.code).toBe(2); + expect(result.stderr).toContain('authorized Codex App origin session/turn-settled is no longer unsettled'); + expect(result.stderr).not.toContain('must not downgrade'); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); + + it('authorizes a host relay from its forced trusted origin before any provider side effect', () => { + const cmdSendStart = cliSource.indexOf('async function cmdSend('); + const cmdDispatchStart = cliSource.indexOf('async function cmdDispatch(', cmdSendStart); + const cmdSend = cliSource.slice(cmdSendStart, cmdDispatchStart); + expect(cmdSend).toContain("const trustedHostRelay = process.env.BOTMUX_HOST_RELAY_AUTHORIZED === '1';"); + expect(cmdSend).toContain('isTrustedVcMeetingHostRelayParent('); + expect(cmdSend).toContain("BOTMUX_HOST_RELAY_REQUIRES_CODEX_APP_LEDGER === '1'"); + expect(cmdSend.indexOf('const exactOriginDispatch = (() => {')) + .toBeLessThan(cmdSend.indexOf("const { synthesizeVoiceOpus }")); + expect(cmdSend.indexOf("exactOriginDispatch?.deliverySink === 'http_wait'")) + .toBeLessThan(cmdSend.indexOf("const { sendMessage, replyMessage, uploadImage, uploadFile")); + }); + + it('validates the exact document text path before reading content or invoking TTS/uploads', () => { + const cmdSendStart = cliSource.indexOf('async function cmdSend('); + const cmdDispatchStart = cliSource.indexOf('async function cmdDispatch(', cmdSendStart); + const cmdSend = cliSource.slice(cmdSendStart, cmdDispatchStart); + const docGuard = cmdSend.indexOf('if (isOriginDocCommentTurn)'); + expect(docGuard).toBeGreaterThan(-1); + expect(docGuard).toBeLessThan(cmdSend.indexOf('// Read content from:')); + expect(docGuard).toBeLessThan(cmdSend.indexOf('content = readFileSync(contentFile')); + expect(docGuard).toBeLessThan(cmdSend.indexOf('content = await readStdin()')); + expect(docGuard).toBeLessThan(cmdSend.indexOf("const { synthesizeVoiceOpus }")); + const guardSource = cmdSend.slice(docGuard, cmdSend.indexOf('// Read content from:')); + for (const forbidden of [ + 'asVoice', + 'images.length > 0', + 'files.length > 0', + 'videoAttachments.length > 0', + 'videoCovers.length > 0', + 'customCardRequested', + 'sendTopLevel', + 'overrideChatId', + 'sendInto', + ]) { + expect(guardSource).toContain(forbidden); + } + }); + + it('never writes the startup session snapshot after an async document reply', () => { + const cmdSendStart = cliSource.indexOf('async function cmdSend('); + const cmdDispatchStart = cliSource.indexOf('async function cmdDispatch(', cmdSendStart); + const cmdSend = cliSource.slice(cmdSendStart, cmdDispatchStart); + const docSendStart = cmdSend.indexOf('if (isOriginDocCommentTurn)', cmdSend.indexOf('// Read content from:')); + const mentionParsing = cmdSend.indexOf('// Parse mentions:', docSendStart); + const docSend = cmdSend.slice(docSendStart, mentionParsing); + expect(docSend).toContain('Daemon settlement owns exact target retirement.'); + expect(docSend).not.toContain('saveSession('); + expect(docSend).not.toMatch(/delete\s+exactDocSession\.docCommentTargets/); + }); + it('freezes VC listener replay content and indexes only the successful primary output', () => { const cmdSendStart = cliSource.indexOf('async function cmdSend('); const cmdDispatchStart = cliSource.indexOf('async function cmdDispatch(', cmdSendStart); @@ -67,7 +288,7 @@ describe('cmdSend hook context wiring', () => { expect(cmdSend.indexOf('const managedRenderedPayloadError = managedVcSendPayloadError({')) .toBeGreaterThan(cmdSend.indexOf('BOTMUX_CARD_PREPARED_CONTENT_FILE')); expect(cmdSend.indexOf('const managedRenderedPayloadError = managedVcSendPayloadError({')) - .toBeLessThan(cmdSend.indexOf('const results = await Promise.all(images.map')); + .toBeLessThan(cmdSend.indexOf('Promise.all(images.map')); expect(cmdSend).toContain('const managedQuoteError = managedVcQuoteError({'); expect(cmdSend).toContain('const managedCustomCardError = managedVcCustomCardError('); expect(cmdSend).toMatch(/sessionQuoteTargetId: vcMeetingDeliveryReplyOrigin\s*\? undefined/); diff --git a/test/codex-app-control.test.ts b/test/codex-app-control.test.ts new file mode 100644 index 000000000..c47282b63 --- /dev/null +++ b/test/codex-app-control.test.ts @@ -0,0 +1,1722 @@ +import { + chmodSync, + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { createConnection, createServer, type Server } from 'node:net'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + CODEX_APP_CONTROL_LINE_MAX_BYTES, + CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS, + CodexAppControlFinalAssembler, + CodexAppControlLineDecoder, + CodexAppControlEndpointTracker, + CodexAppControlReplayWindow, + CodexAppControlProofDeadline, + CodexAppControlRunnerHandshake, + CodexAppControlSequenceFence, + activateCodexAppControlIdentity, + acquireCodexAppControlOwnerLease, + acquireCodexAppPosixOwnerLease, + armCodexAppControlHandshakeTimeout, + armCodexAppControlStartupTimeout, + authenticateCodexAppControlCandidate, + bindThenPublishCodexAppControlLocator, + cleanupStaleCodexAppControlBootstraps, + codexAppControlFilesystemPolicy, + codexAppControlLocatorPath, + codexAppPosixControlRoot, + codexAppPosixOwnerLeaseDirectory, + codexAppPosixProcessProbeEnv, + codexAppControlStatePathForPlatform, + codexAppWindowsOwnerPipeEndpoint, + codexAppWindowsControlRoot, + consumeCodexAppControlBootstrap, + createCodexAppControlBootstrap, + decodeWindowsAclSnapshot, + encodeCodexAppControlAccepted, + encodeCodexAppControlAck, + encodeCodexAppControlAuth, + encodeCodexAppControlChallenge, + encodeCodexAppSignedControlMarker, + generateCodexAppControlChallenge, + generateCodexAppControlEpoch, + generateCodexAppPosixSocketEndpoint, + generateCodexAppWindowsPipeEndpoint, + mergeCodexAppControlCandidate, + parseCodexAppControlWireRecord, + parseWindowsCurrentSid, + readCodexAppControlState, + readCodexAppControlLocator, + shouldColdStartCodexAppReattach, + shouldFailCodexAppControlChannel, + takeCodexAppControlLocatorEndpoint, + validateCodexAppControlLocator, + verifyCodexAppControlAuth, + verifyCodexAppSignedControlMarker, + verifyWindowsCodexAppControlDacl, + writeCodexAppControlLocator, + writeCodexAppControlState, +} from '../src/utils/codex-app-control.js'; + +const tempDirs: string[] = []; + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-control-')); + tempDirs.push(dir); + return dir; +} + +function shortPosixControlRoot(): string { + const dir = mkdtempSync('/tmp/bca-'); + tempDirs.push(dir); + return dir; +} + +function writePosixLeaseActorRecord(input: { + path: string; + sessionId: string; + nonce: string; + pid: number; + processStartToken: string; + createdAtMs?: number; + intendedDirectory?: string; + targetOwnerPath?: string; + version?: 2 | 3; +}): void { + const role = basename(input.path).startsWith('reap-') ? 'reaper' : 'owner'; + const intendedDirectory = input.intendedDirectory ?? dirname(input.path); + const directoryStat = statSync(intendedDirectory, { bigint: true }); + const generationNames = readdirSync(intendedDirectory) + .filter(name => /^generation-[a-f0-9]{64}\.json$/.test(name)); + const directoryGeneration = generationNames.length === 1 + ? generationNames[0]!.slice('generation-'.length, -'.json'.length) + : undefined; + const version = input.version ?? (directoryGeneration ? 3 : 2); + let targetOwner: Record | undefined; + if (role === 'reaper') { + const targetPath = input.targetOwnerPath ?? readdirSync(dirname(input.path)) + .find(name => /^owner-[a-f0-9]{64}\.json$/.test(name)); + const resolvedTargetPath = targetPath + ? (targetPath.includes('/') ? targetPath : join(dirname(input.path), targetPath)) + : undefined; + if (resolvedTargetPath && existsSync(resolvedTargetPath)) { + const targetRecord = JSON.parse(readFileSync(resolvedTargetPath, 'utf8')) as { + nonce: string; + pid: number; + processStartToken: string; + }; + const targetStat = statSync(resolvedTargetPath, { bigint: true }); + targetOwner = { + nonce: targetRecord.nonce, + recordIdentity: { dev: targetStat.dev.toString(10), ino: targetStat.ino.toString(10) }, + pid: targetRecord.pid, + processStartToken: targetRecord.processStartToken, + }; + } else { + targetOwner = { + nonce: '0'.repeat(64), + recordIdentity: { dev: '0', ino: '0' }, + pid: null, + processStartToken: null, + }; + } + } + writeFileSync(input.path, JSON.stringify({ + version, + role, + sessionId: input.sessionId, + nonce: input.nonce, + pid: input.pid, + processStartToken: input.processStartToken, + createdAtMs: input.createdAtMs ?? Date.now(), + directoryIdentity: { + dev: directoryStat.dev.toString(10), + ino: directoryStat.ino.toString(10), + ...(version === 3 ? { generation: directoryGeneration } : {}), + }, + ...(targetOwner ? { targetOwner } : {}), + }), { mode: 0o600 }); + chmodSync(input.path, 0o600); +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe('Codex App asymmetric control bootstrap and state', () => { + it('keeps a delayed runner alive beyond 30 seconds and expires at the shared 90-second hard cap', async () => { + vi.useFakeTimers(); + const onTimeout = vi.fn(); + const timer = armCodexAppControlStartupTimeout(onTimeout); + try { + expect(CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS).toBe(90_000); + await vi.advanceTimersByTimeAsync(30_001); + expect(onTimeout).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(59_998); + expect(onTimeout).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(onTimeout).toHaveBeenCalledTimes(1); + } finally { + clearTimeout(timer); + vi.useRealTimers(); + } + }); + + it('returns only a public identity, consumes the 0600 private bootstrap once, and binds session', () => { + const dir = tempDir(); + const bootstrap = createCodexAppControlBootstrap(dir, 'session-one'); + const raw = readFileSync(bootstrap.path, 'utf8'); + const serialized = JSON.parse(raw); + + expect(Object.keys(bootstrap).sort()).toEqual(['identity', 'path']); + expect(bootstrap.identity.publicKey).toEqual(expect.any(String)); + expect(JSON.stringify(bootstrap)).not.toContain(serialized.privateKey); + expect(serialized).toMatchObject({ + version: 3, + sessionId: 'session-one', + generation: bootstrap.identity.generation, + privateKey: expect.any(String), + socketPath: expect.stringMatching(/\.sock$/), + }); + expect(statSync(bootstrap.path).mode & 0o777).toBe(0o600); + + const consumed = consumeCodexAppControlBootstrap(bootstrap.path, 'session-one'); + expect(consumed.generation).toBe(bootstrap.identity.generation); + expect(consumed.privateKey.type).toBe('private'); + expect(consumed.privateKey.asymmetricKeyType).toBe('ed25519'); + expect(existsSync(bootstrap.path)).toBe(false); + expect(() => consumeCodexAppControlBootstrap(bootstrap.path, 'session-one')).toThrow(); + + const wrongSession = createCodexAppControlBootstrap(dir, 'session-two'); + expect(() => consumeCodexAppControlBootstrap(wrongSession.path, 'session-other')).toThrow(/invalid/); + expect(existsSync(wrongSession.path)).toBe(false); + }); + + it('fails closed on broad mode, symlink, and hard-linked bootstraps', () => { + const dir = tempDir(); + const loose = createCodexAppControlBootstrap(dir, 'loose'); + chmodSync(loose.path, 0o644); + expect(() => consumeCodexAppControlBootstrap(loose.path, 'loose')).toThrow(/regular 0600 file/); + expect(existsSync(loose.path)).toBe(false); + + const target = join(dir, 'target'); + const symlink = join(dir, 'symlink.bootstrap'); + writeFileSync(target, '{}', { mode: 0o600 }); + symlinkSync(target, symlink); + expect(() => consumeCodexAppControlBootstrap(symlink, 'x')).toThrow(); + expect(existsSync(symlink)).toBe(false); + + const linked = createCodexAppControlBootstrap(dir, 'linked'); + const secondLink = join(dir, 'second-link.bootstrap'); + linkSync(linked.path, secondLink); + expect(() => consumeCodexAppControlBootstrap(linked.path, 'linked')).toThrow(/single-link/); + rmSync(secondLink, { force: true }); + }); + + it('pins POSIX locator bootstraps to the fixed per-UID control root', () => { + const dir = tempDir(); + const sessionId = 'session-locator-root'; + const canonical = createCodexAppControlBootstrap(dir, sessionId, { + kind: 'locator', + locatorPath: codexAppControlLocatorPath(codexAppPosixControlRoot(), sessionId), + }); + expect(consumeCodexAppControlBootstrap(canonical.path, sessionId).locatorPath) + .toBe(codexAppControlLocatorPath(codexAppPosixControlRoot(), sessionId)); + + const attackerRoot = tempDir(); + const forged = createCodexAppControlBootstrap(dir, sessionId, { + kind: 'locator', + locatorPath: codexAppControlLocatorPath(attackerRoot, sessionId), + }); + expect(() => consumeCodexAppControlBootstrap(forged.path, sessionId)).toThrow(/invalid/); + }); + + it('cleans crash-orphaned bootstrap files for only the requested session', () => { + const dir = tempDir(); + const first = createCodexAppControlBootstrap(dir, 'session-clean'); + const other = createCodexAppControlBootstrap(dir, 'session-other'); + cleanupStaleCodexAppControlBootstraps(dir, 'session-clean'); + expect(existsSync(first.path)).toBe(false); + expect(existsSync(other.path)).toBe(true); + }); + + it('persists public candidates pending, then atomically collapses the proven generation active', () => { + const dir = tempDir(); + const statePath = join(dir, 'state', 'session.json'); + const oldBootstrap = createCodexAppControlBootstrap(dir, 'session-state'); + const oldPending = mergeCodexAppControlCandidate(undefined, oldBootstrap.identity, 10); + const oldActive = activateCodexAppControlIdentity(oldPending, oldBootstrap.identity.generation, 20); + writeCodexAppControlState(statePath, oldActive); + + const fresh = createCodexAppControlBootstrap(dir, 'session-state'); + const pending = mergeCodexAppControlCandidate(oldActive, fresh.identity, 30); + writeCodexAppControlState(statePath, pending); + const persistedPending = readCodexAppControlState(statePath)!; + expect(persistedPending.status).toBe('pending'); + expect(persistedPending.identities.map(identity => identity.generation)).toEqual([ + fresh.identity.generation, + oldBootstrap.identity.generation, + ]); + expect(JSON.stringify(persistedPending)).not.toContain('privateKey'); + expect(statSync(statePath).mode & 0o777).toBe(0o600); + + const challenge = generateCodexAppControlChallenge(); + const oldKey = consumeCodexAppControlBootstrap(oldBootstrap.path, 'session-state'); + const oldAuth = parseCodexAppControlWireRecord(encodeCodexAppControlAuth( + oldKey.privateKey, + 'session-state', + oldKey.generation, + challenge, + )); + if (!oldAuth || oldAuth.type !== 'auth') throw new Error('old auth parse failed'); + expect(authenticateCodexAppControlCandidate({ + state: persistedPending, + auth: oldAuth, + sessionId: 'session-state', + challenge, + })?.generation).toBe(oldBootstrap.identity.generation); + expect(authenticateCodexAppControlCandidate({ + state: persistedPending, + auth: { ...oldAuth, challenge: generateCodexAppControlChallenge() }, + sessionId: 'session-state', + challenge, + })).toBeUndefined(); + + const reused = activateCodexAppControlIdentity( + persistedPending, + oldBootstrap.identity.generation, + 40, + ); + writeCodexAppControlState(statePath, reused); + expect(readCodexAppControlState(statePath)).toMatchObject({ + status: 'active', + identities: [{ generation: oldBootstrap.identity.generation }], + activatedAtMs: 40, + }); + }); + + it('cold-starts only persistent panes that have no valid public state', () => { + const dir = tempDir(); + const bootstrap = createCodexAppControlBootstrap(dir, 'session-cold'); + const pending = mergeCodexAppControlCandidate(undefined, bootstrap.identity); + for (const backendType of ['tmux', 'herdr', 'zellij'] as const) { + expect(shouldColdStartCodexAppReattach({ + cliId: 'codex-app', backendType, isReattach: true, + })).toBe(true); + expect(shouldColdStartCodexAppReattach({ + cliId: 'codex-app', backendType, isReattach: true, persistedState: pending, + })).toBe(false); + } + expect(shouldColdStartCodexAppReattach({ + cliId: 'codex-app', backendType: 'pty', isReattach: true, + })).toBe(false); + expect(shouldColdStartCodexAppReattach({ + cliId: 'codex', backendType: 'tmux', isReattach: true, + })).toBe(false); + }); +}); + +describe('Codex App final transaction fencing', () => { + it('allows an arbitrary first sequence after replacement but requires continuity thereafter', () => { + const fence = new CodexAppControlSequenceFence(); + expect(fence.accept(41)).toBe(true); + expect(fence.accept(42)).toBe(true); + expect(fence.accept(44)).toBe(false); + + const duplicate = new CodexAppControlSequenceFence(); + expect(duplicate.accept(9)).toBe(true); + expect(duplicate.accept(9)).toBe(false); + }); + + it('publishes a final only after a complete ordered transaction', () => { + const assembler = new CodexAppControlFinalAssembler(); + expect(assembler.accept('final-start', { + id: 'turn:1', total: 2, turnId: 'om_1', completedAtMs: 100, + })).toEqual({ status: 'accepted' }); + expect(assembler.accept('final-chunk', { + id: 'turn:1', index: 0, data: Buffer.from('hello ').toString('base64'), + })).toEqual({ status: 'accepted' }); + expect(assembler.accept('final-chunk', { + id: 'turn:1', index: 1, data: Buffer.from('world').toString('base64'), + })).toEqual({ status: 'accepted' }); + expect(assembler.accept('final-end', { id: 'turn:1', total: 2 })).toEqual({ + status: 'complete', + payload: { turnId: 'om_1', completedAtMs: 100, content: 'hello world' }, + }); + }); + + it('rejects an incomplete final-end and accepts a full replay from a fresh connection', () => { + const incomplete = new CodexAppControlFinalAssembler(); + expect(incomplete.accept('final-start', { id: 'turn:2', total: 2 })).toEqual({ status: 'accepted' }); + expect(incomplete.accept('final-chunk', { + id: 'turn:2', index: 0, data: Buffer.from('first').toString('base64'), + })).toEqual({ status: 'accepted' }); + expect(incomplete.accept('final-end', { id: 'turn:2', total: 2 })).toMatchObject({ status: 'reject' }); + + const replay = new CodexAppControlFinalAssembler(); + expect(replay.accept('final-start', { id: 'turn:2', total: 2 })).toEqual({ status: 'accepted' }); + expect(replay.accept('final-chunk', { + id: 'turn:2', index: 0, data: Buffer.from('first').toString('base64'), + })).toEqual({ status: 'accepted' }); + expect(replay.accept('final-chunk', { + id: 'turn:2', index: 1, data: Buffer.from('second').toString('base64'), + })).toEqual({ status: 'accepted' }); + expect(replay.accept('final-end', { id: 'turn:2', total: 2 })).toMatchObject({ + status: 'complete', + payload: { content: 'firstsecond' }, + }); + }); + + it.each([ + ['out-of-order chunk', [ + ['final-start', { id: 'turn:3', total: 2 }], + ['final-chunk', { id: 'turn:3', index: 1, data: Buffer.from('late').toString('base64') }], + ]], + ['duplicate chunk', [ + ['final-start', { id: 'turn:3', total: 2 }], + ['final-chunk', { id: 'turn:3', index: 0, data: Buffer.from('one').toString('base64') }], + ['final-chunk', { id: 'turn:3', index: 0, data: Buffer.from('again').toString('base64') }], + ]], + ['invalid base64', [ + ['final-start', { id: 'turn:3', total: 1 }], + ['final-chunk', { id: 'turn:3', index: 0, data: '***not-base64***' }], + ]], + ['interleaved marker', [ + ['final-start', { id: 'turn:3', total: 1 }], + ['state', { busy: false }], + ]], + ['mismatched final total', [ + ['final-start', { id: 'turn:3', total: 1 }], + ['final-chunk', { id: 'turn:3', index: 0, data: Buffer.from('one').toString('base64') }], + ['final-end', { id: 'turn:3', total: 2 }], + ]], + ] as const)('rejects %s instead of making it cumulatively ACK-eligible', (_name, records) => { + const assembler = new CodexAppControlFinalAssembler(); + let result: ReturnType = { status: 'not-final' }; + for (const [kind, payload] of records) result = assembler.accept(kind, payload); + expect(result.status).toBe('reject'); + }); +}); + +describe('Windows Codex App control root, locator, and filesystem policy', () => { + it('anchors state and locator paths under LOCALAPPDATA instead of SESSION_DATA_DIR', () => { + const options = { + platform: 'win32' as const, + localAppData: 'C:\\Users\\alice\\AppData\\Local', + homeDirectory: 'C:\\Users\\alice', + }; + const root = codexAppWindowsControlRoot(options); + expect(root).toBe('C:\\Users\\alice\\AppData\\Local\\Botmux\\codex-app-control'); + const state = codexAppControlStatePathForPlatform('Z:\\shared\\untrusted', 'session-win', options); + expect(state.startsWith(root)).toBe(true); + expect(state).not.toContain('shared'); + expect(codexAppControlLocatorPath(root, 'session-win', 'win32').startsWith(root)).toBe(true); + expect(codexAppWindowsOwnerPipeEndpoint('session-win')) + .toMatch(/^\\\\\?\\pipe\\botmux-codex-app-owner-[a-f0-9]{64}$/); + expect(() => codexAppWindowsControlRoot({ + platform: 'win32', + localAppData: '\\\\fileserver\\profiles\\alice', + homeDirectory: 'C:\\Users\\alice', + })).toThrow(/local drive-qualified/); + }); + + it('uses Windows file semantics without weakening POSIX owner-only policy', () => { + expect(codexAppControlFilesystemPolicy('win32')).toEqual({ + useNoFollow: false, + verifyUid: false, + verifyExactMode: false, + chmodAfterCreate: false, + verifyPostUnlinkLinkCount: false, + fsyncDirectory: false, + }); + expect(codexAppControlFilesystemPolicy('linux')).toEqual({ + useNoFollow: true, + verifyUid: true, + verifyExactMode: true, + chmodAfterCreate: true, + verifyPostUnlinkLinkCount: true, + fsyncDirectory: true, + }); + }); + + it('parses the current SID and accepts only a protected exact current-SID + SYSTEM DACL', () => { + const sid = 'S-1-5-21-111-222-333-1001'; + expect(parseWindowsCurrentSid(`"DOMAIN\\alice","${sid}"`)).toBe(sid); + // The username column is attacker-influenced and can itself be SID-shaped; + // whoami's second CSV column is the only SID authority. + expect(parseWindowsCurrentSid(`"S-1-5-18","${sid}"\r\n`)).toBe(sid); + expect(parseWindowsCurrentSid(`"DOMAIN\\alice, admin","${sid}"\r\n`)).toBe(sid); + expect(parseWindowsCurrentSid(`"${sid}","not-a-sid"`)).toBeUndefined(); + expect(parseWindowsCurrentSid(`"DOMAIN\\alice","${sid}","extra"`)).toBeUndefined(); + const exact = [ + 'D:\\Botmux\\codex-app-control', + `D:P(A;OICI;FA;;;${sid})(A;OICI;FA;;;SY)`, + '', + ].join('\r\n'); + expect(verifyWindowsCodexAppControlDacl(exact, sid)).toBe(true); + expect(verifyWindowsCodexAppControlDacl( + `D:\\Botmux\\control D:P(A;OICI;FA;;;${sid})(A;OICI;FA;;;SY)`, + sid, + )).toBe(true); + const utf16 = Buffer.concat([ + Buffer.from([0xff, 0xfe]), + Buffer.from(exact, 'utf16le'), + ]); + expect(verifyWindowsCodexAppControlDacl(decodeWindowsAclSnapshot(utf16), sid)).toBe(true); + expect(verifyWindowsCodexAppControlDacl( + `D:P(A;;FA;;;${sid})(A;;FA;;;SY)`, + sid, + 'file', + )).toBe(true); + expect(verifyWindowsCodexAppControlDacl(exact, sid, 'file')).toBe(false); + expect(verifyWindowsCodexAppControlDacl( + `${exact.trim()}(A;OICI;FA;;;S-1-1-0)`, + sid, + )).toBe(false); + expect(verifyWindowsCodexAppControlDacl( + `D:AI(A;OICI;FA;;;${sid})(A;OICI;FA;;;SY)`, + sid, + )).toBe(false); + expect(verifyWindowsCodexAppControlDacl( + `D:P(A;OICIID;FA;;;${sid})(A;OICI;FA;;;SY)`, + sid, + )).toBe(false); + }); + + it('uses trusted absolute whoami/icacls argv without a shell and verifies saved SDDL', () => { + const source = readFileSync(join(process.cwd(), 'src/utils/codex-app-control.ts'), 'utf8'); + const start = source.indexOf('function defaultWindowsControlCommandRunner('); + const end = source.indexOf('/**\n * Remove inherited ACLs', start); + const acl = source.slice(start, end); + expect(acl).toContain('spawnSync(command, args, {'); + expect(acl).toContain('shell: false'); + expect(source).toContain("win32.join(systemRoot, 'System32', 'whoami.exe')"); + expect(source).toContain("win32.join(systemRoot, 'System32', 'icacls.exe')"); + expect(source).toContain("['/user', '/fo', 'csv', '/nh']"); + expect(source).toContain("[path, '/inheritance:r', '/q']"); + expect(source).toContain("[path, '/save', snapshotPath, '/q']"); + expect(source).toContain('verifyWindowsCodexAppControlDacl(snapshot, sid, kind)'); + }); + + it('validates random 256-bit pipe endpoints and rejects wrong/corrupt locators', () => { + const endpoint = generateCodexAppWindowsPipeEndpoint(); + const epoch = generateCodexAppControlEpoch(); + expect(endpoint).toMatch(/^\\\\\?\\pipe\\botmux-codex-app-[a-f0-9]{64}$/); + const locator = validateCodexAppControlLocator({ + version: 1, + sessionId: 'session-win', + epoch, + endpoint, + }, 'session-win', { platform: 'win32' }); + expect(locator).toEqual({ version: 1, sessionId: 'session-win', epoch, endpoint }); + expect(validateCodexAppControlLocator(locator, 'wrong-session', { platform: 'win32' })).toBeUndefined(); + expect(validateCodexAppControlLocator( + { ...locator, endpoint: '\\\\?\\pipe\\fixed' }, + 'session-win', + { platform: 'win32' }, + )) + .toBeUndefined(); + expect(validateCodexAppControlLocator( + { ...locator, epoch: 'short' }, + 'session-win', + { platform: 'win32' }, + )).toBeUndefined(); + }); + + it('publishes only after listen succeeds and never publishes on bind failure', async () => { + const order: string[] = []; + const endpoint = generateCodexAppWindowsPipeEndpoint(); + const epoch = generateCodexAppControlEpoch(); + await bindThenPublishCodexAppControlLocator({ + sessionId: 'session-bind', + epoch, + endpoint, + platform: 'win32', + listen: async () => { order.push('listen'); }, + publish: () => { order.push('publish'); }, + }); + expect(order).toEqual(['listen', 'publish']); + + const publish = vi.fn(); + await expect(bindThenPublishCodexAppControlLocator({ + sessionId: 'session-bind', + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppWindowsPipeEndpoint(), + platform: 'win32', + listen: async () => { throw new Error('EADDRINUSE'); }, + publish, + })).rejects.toThrow('EADDRINUSE'); + expect(publish).not.toHaveBeenCalled(); + }); + + it('serializes owner publishers, bounds EADDRINUSE waiting, and fails other bind errors immediately', async () => { + let nowMs = 0; + let oldLeaseHeld = true; + const bind = vi.fn(async () => { + if (oldLeaseHeld) throw Object.assign(new Error('old worker owns lease'), { code: 'EADDRINUSE' }); + return 'new-lease'; + }); + await expect(acquireCodexAppControlOwnerLease({ + bind, + timeoutMs: 1_000, + retryDelayMs: 100, + now: () => nowMs, + wait: async delayMs => { + nowMs += delayMs; + if (nowMs >= 200) oldLeaseHeld = false; + }, + })).resolves.toBe('new-lease'); + expect(bind).toHaveBeenCalledTimes(3); + + nowMs = 0; + const alwaysBusy = vi.fn(async () => { + throw Object.assign(new Error('still owned'), { code: 'EADDRINUSE' }); + }); + await expect(acquireCodexAppControlOwnerLease({ + bind: alwaysBusy, + timeoutMs: 200, + retryDelayMs: 100, + now: () => nowMs, + wait: async delayMs => { nowMs += delayMs; }, + })).rejects.toThrow('still owned'); + expect(nowMs).toBe(200); + + const denied = Object.assign(new Error('access denied'), { code: 'EACCES' }); + const failFast = vi.fn(async () => { throw denied; }); + await expect(acquireCodexAppControlOwnerLease({ bind: failFast })).rejects.toBe(denied); + expect(failFast).toHaveBeenCalledTimes(1); + }); + + it('keeps B published when superseded A finishes binding late', async () => { + let releaseListen!: () => void; + const listen = new Promise(resolve => { releaseListen = resolve; }); + let currentChannelId = 1; + let stopping = false; + const publishA = vi.fn(); + const retireA = vi.fn(); + const pendingA = bindThenPublishCodexAppControlLocator({ + sessionId: 'session-retired', + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppWindowsPipeEndpoint(), + platform: 'win32', + listen: () => listen, + publish: publishA, + isCurrent: () => !stopping && currentChannelId === 1, + retire: retireA, + }); + // Model stop(A), then bind+publish B, before A's delayed bind resolves. + stopping = true; + currentChannelId = 2; + stopping = false; + let published = ''; + await bindThenPublishCodexAppControlLocator({ + sessionId: 'session-retired', + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppWindowsPipeEndpoint(), + platform: 'win32', + listen: async () => {}, + publish: () => { published = 'B'; }, + isCurrent: () => !stopping && currentChannelId === 2, + }); + releaseListen(); + await expect(pendingA).resolves.toBeUndefined(); + expect(published).toBe('B'); + expect(publishA).not.toHaveBeenCalled(); + expect(retireA).toHaveBeenCalledTimes(1); + expect(shouldFailCodexAppControlChannel({ + channelId: 1, + currentChannelId, + stopping, + })).toBe(false); + + const retireB = vi.fn(); + await expect(bindThenPublishCodexAppControlLocator({ + sessionId: 'session-retired', + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppWindowsPipeEndpoint(), + platform: 'win32', + listen: async () => {}, + publish: () => { throw new Error('B locator write failed'); }, + isCurrent: () => !stopping && currentChannelId === 2, + retire: retireB, + })).rejects.toThrow('B locator write failed'); + expect(retireB).toHaveBeenCalledTimes(1); + expect(shouldFailCodexAppControlChannel({ + channelId: 2, + currentChannelId, + stopping, + })).toBe(true); + }); + + it('retires a bound endpoint when locator publication fails', async () => { + const retire = vi.fn(); + await expect(bindThenPublishCodexAppControlLocator({ + sessionId: 'session-publish-failure', + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppWindowsPipeEndpoint(), + platform: 'win32', + listen: async () => {}, + publish: () => { throw new Error('locator rename failed'); }, + retire, + })).rejects.toThrow('locator rename failed'); + expect(retire).toHaveBeenCalledTimes(1); + }); + + it('polls missing locators, retries unaccepted A, burns accepted A, then selects B', () => { + const dir = shortPosixControlRoot(); + const sessionId = 'session-track'; + const locatorPath = codexAppControlLocatorPath(dir, sessionId); + const socketDirectory = join(dir, 'sockets'); + mkdirSync(join(dir, 'locators'), { recursive: true, mode: 0o700 }); + mkdirSync(socketDirectory, { recursive: true, mode: 0o700 }); + const tracker = new CodexAppControlEndpointTracker(); + const first = { + version: 1 as const, + sessionId, + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppPosixSocketEndpoint(socketDirectory), + }; + expect(takeCodexAppControlLocatorEndpoint({ + locatorPath, + sessionId: first.sessionId, + tracker, + expectedControlRoot: dir, + })).toBeUndefined(); + writeCodexAppControlLocator(locatorPath, first, process.platform, dir); + for (let attempt = 1; attempt <= 5; attempt++) { + expect(takeCodexAppControlLocatorEndpoint({ + locatorPath, + sessionId: first.sessionId, + tracker, + expectedControlRoot: dir, + })).toEqual({ endpoint: first.endpoint, epoch: first.epoch }); + expect(tracker.attemptCount(first.endpoint)).toBe(attempt); + } + tracker.noteAccepted(first.endpoint); + expect(takeCodexAppControlLocatorEndpoint({ + locatorPath, + sessionId: first.sessionId, + tracker, + expectedControlRoot: dir, + })).toBeUndefined(); + expect(tracker.wasAttempted(first.endpoint)).toBe(true); + expect(tracker.wasAccepted(first.endpoint)).toBe(true); + const rotated = { + ...first, + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppPosixSocketEndpoint(socketDirectory), + }; + writeCodexAppControlLocator(locatorPath, rotated, process.platform, dir); + expect(takeCodexAppControlLocatorEndpoint({ + locatorPath, + sessionId: first.sessionId, + tracker, + expectedControlRoot: dir, + })).toEqual({ endpoint: rotated.endpoint, epoch: rotated.epoch }); + }); +}); + +describe('POSIX Codex App locator replacement', () => { + it('pins non-Linux process-start probes to a stable locale and timezone', () => { + expect(codexAppPosixProcessProbeEnv({ + PATH: '/trusted/bin', + LC_ALL: 'zh_CN.UTF-8', + LANG: 'zh_CN.UTF-8', + TZ: 'Asia/Shanghai', + })).toMatchObject({ + PATH: '/trusted/bin', + LC_ALL: 'C', + LANG: 'C', + TZ: 'UTC', + }); + }); + + it('accepts only random endpoints inside the locator control root', () => { + const root = shortPosixControlRoot(); + const sessionId = 'session-posix-locator'; + const locatorPath = codexAppControlLocatorPath(root, sessionId); + const socketDirectory = join(root, 'sockets'); + mkdirSync(join(root, 'locators'), { recursive: true, mode: 0o700 }); + mkdirSync(socketDirectory, { recursive: true, mode: 0o700 }); + const locator = { + version: 1 as const, + sessionId, + epoch: generateCodexAppControlEpoch(), + endpoint: generateCodexAppPosixSocketEndpoint(socketDirectory), + }; + + expect(validateCodexAppControlLocator(locator, sessionId, { + platform: 'linux', + locatorPath, + expectedControlRoot: root, + })).toEqual(locator); + expect(validateCodexAppControlLocator( + { ...locator, endpoint: join(root, 'outside.sock') }, + sessionId, + { platform: 'linux', locatorPath, expectedControlRoot: root }, + )).toBeUndefined(); + expect(validateCodexAppControlLocator(locator, sessionId, { + platform: 'linux', + locatorPath: join(root, 'locator.json'), + expectedControlRoot: root, + })).toBeUndefined(); + expect(validateCodexAppControlLocator(locator, sessionId, { + platform: 'linux', + locatorPath, + })).toBeUndefined(); + expect(validateCodexAppControlLocator(locator, sessionId, { + platform: 'linux', + locatorPath, + expectedControlRoot: join(root, 'attacker-selected'), + })).toBeUndefined(); + }); + + it('keeps B in the POSIX locator when superseded A finishes binding late', async () => { + const root = shortPosixControlRoot(); + const sessionId = 'session-posix-delayed-publish'; + const locatorPath = codexAppControlLocatorPath(root, sessionId); + const socketDirectory = join(root, 'sockets'); + mkdirSync(join(root, 'locators'), { recursive: true, mode: 0o700 }); + mkdirSync(socketDirectory, { recursive: true, mode: 0o700 }); + let channelId = 1; + let releaseA!: () => void; + const delayedListen = new Promise(resolvePromise => { releaseA = resolvePromise; }); + const retireA = vi.fn(); + const endpointA = generateCodexAppPosixSocketEndpoint(socketDirectory); + const pendingA = bindThenPublishCodexAppControlLocator({ + sessionId, + epoch: generateCodexAppControlEpoch(), + endpoint: endpointA, + platform: process.platform, + locatorPath, + expectedControlRoot: root, + listen: () => delayedListen, + isCurrent: () => channelId === 1, + publish: locator => writeCodexAppControlLocator( + locatorPath, + locator, + process.platform, + root, + ), + retire: retireA, + }); + + channelId = 2; + const endpointB = generateCodexAppPosixSocketEndpoint(socketDirectory); + await bindThenPublishCodexAppControlLocator({ + sessionId, + epoch: generateCodexAppControlEpoch(), + endpoint: endpointB, + platform: process.platform, + locatorPath, + expectedControlRoot: root, + listen: async () => undefined, + isCurrent: () => channelId === 2, + publish: locator => writeCodexAppControlLocator( + locatorPath, + locator, + process.platform, + root, + ), + }); + releaseA(); + await expect(pendingA).resolves.toBeUndefined(); + expect(retireA).toHaveBeenCalledTimes(1); + expect(readCodexAppControlLocator(locatorPath, sessionId, process.platform, root)?.endpoint) + .toBe(endpointB); + }); + + it('holds publisher ownership across overlap and fails closed on unknown owner liveness', async () => { + const root = tempDir(); + const statuses = new Map([ + [101, 'alive'], + [202, 'alive'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + const first = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: 'session-posix-owner', + pid: 101, + processStartToken: 'start-A', + inspectOwner, + }); + let nowMs = 0; + let waits = 0; + const second = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: 'session-posix-owner', + pid: 202, + processStartToken: 'start-B', + inspectOwner, + timeoutMs: 1_000, + retryDelayMs: 10, + now: () => nowMs, + wait: async delayMs => { + nowMs += delayMs; + waits++; + if (waits === 2) first.release(); + }, + }); + expect(waits).toBeGreaterThanOrEqual(2); + expect(first.isOwned()).toBe(false); + expect(second.isOwned()).toBe(true); + second.release(); + + const unknown = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: 'session-posix-unknown', + pid: 101, + processStartToken: 'start-A', + inspectOwner, + }); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: 'session-posix-unknown', + pid: 202, + processStartToken: 'start-B', + inspectOwner: () => 'unknown', + timeoutMs: 20, + retryDelayMs: 10, + now: () => nowMs, + wait: async delayMs => { nowMs += delayMs; }, + })).rejects.toThrow(/timed out/); + expect(unknown.isOwned()).toBe(true); + unknown.release(); + }); + + it('does not reap a fresh empty initialization window before its grace expires', async () => { + const root = tempDir(); + const sessionId = 'session-posix-initializing'; + const directory = codexAppPosixOwnerLeaseDirectory(root, sessionId); + mkdirSync(join(root, 'leases'), { recursive: true, mode: 0o700 }); + mkdirSync(directory, { mode: 0o700 }); + let nowMs = statSync(directory).mtimeMs; + let waits = 0; + const lease = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 304, + processStartToken: 'start-after-empty', + inspectOwner: () => 'alive', + initializationGraceMs: 30, + retryDelayMs: 10, + timeoutMs: 100, + now: () => nowMs, + wait: async delayMs => { waits++; nowMs += delayMs; }, + }); + expect(waits).toBeGreaterThanOrEqual(3); + expect(lease.isOwned()).toBe(true); + lease.release(); + }); + + it('does not let a v2 actor claim authority over a v3 generation directory', async () => { + const root = tempDir(); + const sessionId = 'session-posix-v2-v3-boundary'; + const directory = codexAppPosixOwnerLeaseDirectory(root, sessionId); + mkdirSync(join(root, 'leases'), { recursive: true, mode: 0o700 }); + mkdirSync(directory, { mode: 0o700 }); + const generation = '1'.repeat(64); + writeFileSync(join(directory, `generation-${generation}.json`), generation, { mode: 0o600 }); + writePosixLeaseActorRecord({ + path: join(directory, `owner-${'2'.repeat(64)}.json`), + sessionId, + nonce: '2'.repeat(64), + pid: 401, + processStartToken: 'legacy-live', + version: 2, + }); + + const lease = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 402, + processStartToken: 'v3-successor', + inspectOwner: pid => pid === 401 ? 'alive' : 'unknown', + initializationGraceMs: 0, + }); + expect(lease.isOwned()).toBe(true); + lease.release(); + }); + + it('recovers multiple valid generation markers after the crash residue is actor-free', async () => { + const root = tempDir(); + const sessionId = 'session-posix-multiple-generations'; + const directory = codexAppPosixOwnerLeaseDirectory(root, sessionId); + mkdirSync(join(root, 'leases'), { recursive: true, mode: 0o700 }); + mkdirSync(directory, { mode: 0o700 }); + for (const generation of ['3'.repeat(64), '4'.repeat(64)]) { + writeFileSync(join(directory, `generation-${generation}.json`), generation, { mode: 0o600 }); + } + + const lease = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 403, + processStartToken: 'multiple-generation-successor', + inspectOwner: () => 'unknown', + initializationGraceMs: 0, + }); + expect(lease.isOwned()).toBe(true); + lease.release(); + }); + + it('keeps an ambiguous multi-generation directory fail-closed while an actor is live', async () => { + const root = tempDir(); + const sessionId = 'session-posix-multiple-generations-live'; + const directory = codexAppPosixOwnerLeaseDirectory(root, sessionId); + mkdirSync(join(root, 'leases'), { recursive: true, mode: 0o700 }); + mkdirSync(directory, { mode: 0o700 }); + for (const generation of ['5'.repeat(64), '6'.repeat(64)]) { + writeFileSync(join(directory, `generation-${generation}.json`), generation, { mode: 0o600 }); + } + writePosixLeaseActorRecord({ + path: join(directory, `owner-${'7'.repeat(64)}.json`), + sessionId, + nonce: '7'.repeat(64), + pid: 404, + processStartToken: 'ambiguous-live', + }); + let nowMs = Date.now(); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 405, + processStartToken: 'blocked-successor', + inspectOwner: pid => pid === 404 ? 'alive' : 'unknown', + initializationGraceMs: 0, + timeoutMs: 20, + retryDelayMs: 10, + now: () => nowMs, + wait: async delayMs => { nowMs += delayMs; }, + })).rejects.toThrow(/timed out/); + + const lease = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 405, + processStartToken: 'recovered-successor', + inspectOwner: pid => pid === 404 ? 'dead' : 'unknown', + initializationGraceMs: 0, + }); + expect(lease.isOwned()).toBe(true); + lease.release(); + }); + + it('recovers a dead reaper crash both before and after stale-owner retirement', async () => { + const root = tempDir(); + const statuses = new Map([ + [311, 'alive'], + [312, 'dead'], + [313, 'alive'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + + const beforeSession = 'session-posix-reaper-before-retire'; + const stale = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: beforeSession, + pid: 311, + processStartToken: 'owner-before', + inspectOwner, + }); + statuses.set(311, 'dead'); + const beforeReaperNonce = 'a'.repeat(64); + writePosixLeaseActorRecord({ + path: join(stale.directory, `reap-${beforeReaperNonce}.json`), + sessionId: beforeSession, + nonce: beforeReaperNonce, + pid: 312, + processStartToken: 'reaper-before', + }); + const recoveredBefore = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: beforeSession, + pid: 313, + processStartToken: 'successor-before', + inspectOwner, + }); + expect(stale.isOwned()).toBe(false); + expect(recoveredBefore.isOwned()).toBe(true); + recoveredBefore.release(); + + const afterSession = 'session-posix-reaper-after-retire'; + const afterDirectory = codexAppPosixOwnerLeaseDirectory(root, afterSession); + mkdirSync(afterDirectory, { recursive: true, mode: 0o700 }); + chmodSync(afterDirectory, 0o700); + const afterReaperNonce = 'b'.repeat(64); + writePosixLeaseActorRecord({ + path: join(afterDirectory, `reap-${afterReaperNonce}.json`), + sessionId: afterSession, + nonce: afterReaperNonce, + pid: 312, + processStartToken: 'reaper-after', + }); + const recoveredAfter = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: afterSession, + pid: 313, + processStartToken: 'successor-after', + inspectOwner, + initializationGraceMs: 0, + }); + expect(recoveredAfter.isOwned()).toBe(true); + recoveredAfter.release(); + }); + + it('keeps live and unknown reaper actors fail-closed', async () => { + for (const [suffix, reaperStatus] of [ + ['live', 'alive'], + ['unknown', 'unknown'], + ] as const) { + const root = tempDir(); + const sessionId = `session-posix-reaper-${suffix}`; + const statuses = new Map([ + [321, 'dead'], + [322, reaperStatus], + [323, 'alive'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + const stale = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 321, + processStartToken: `owner-${suffix}`, + inspectOwner, + }); + const reaperNonce = suffix === 'live' ? 'c'.repeat(64) : 'd'.repeat(64); + writePosixLeaseActorRecord({ + path: join(stale.directory, `reap-${reaperNonce}.json`), + sessionId, + nonce: reaperNonce, + pid: 322, + processStartToken: `reaper-${suffix}`, + }); + let nowMs = Date.now(); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 323, + processStartToken: `successor-${suffix}`, + inspectOwner, + timeoutMs: 30, + retryDelayMs: 10, + now: () => nowMs, + wait: async delayMs => { nowMs += delayMs; }, + })).rejects.toThrow(/timed out/); + expect(stale.isOwned()).toBe(false); // a reaper record suspends owner authority + rmSync(stale.directory, { recursive: true, force: true }); + } + }); + + it('grace-recovers secure crash-partial owner and reaper records', async () => { + for (const kind of ['owner', 'reap'] as const) { + const root = tempDir(); + const sessionId = `session-posix-partial-${kind}`; + const directory = codexAppPosixOwnerLeaseDirectory(root, sessionId); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + chmodSync(directory, 0o700); + const nonce = (kind === 'owner' ? 'e' : 'f').repeat(64); + const partialPath = join(directory, `${kind}-${nonce}.json`); + writeFileSync(partialPath, '{"version":1', { mode: 0o600 }); + chmodSync(partialPath, 0o600); + let nowMs = statSync(partialPath).mtimeMs; + let waits = 0; + const lease = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 333, + processStartToken: `successor-partial-${kind}`, + inspectOwner: () => 'alive', + initializationGraceMs: 30, + retryDelayMs: 10, + timeoutMs: 200, + now: () => nowMs, + wait: async delayMs => { waits++; nowMs += delayMs; }, + }); + expect(waits).toBeGreaterThanOrEqual(3); + expect(lease.isOwned()).toBe(true); + lease.release(); + } + }); + + it('retries when graceful release wins exactly after mkdir reports EEXIST', async () => { + const root = tempDir(); + const sessionId = 'session-posix-release-observation-gap'; + const first = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 341, + processStartToken: 'first-owner', + inspectOwner: () => 'alive', + }); + let contended = 0; + const second = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 342, + processStartToken: 'second-owner', + inspectOwner: () => 'alive', + onContended: () => { + contended++; + first.release(); + }, + }); + expect(contended).toBeGreaterThanOrEqual(1); + expect(first.isOwned()).toBe(false); + expect(second.isOwned()).toBe(true); + second.release(); + }); + + it('never lets a delayed original creator publish into a successor directory inode', async () => { + const root = tempDir(); + const sessionId = 'session-posix-delayed-original-writer'; + const successorNonce = '3'.repeat(64); + let replaced = false; + let nowMs = Date.now(); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 346, + processStartToken: 'delayed-original', + inspectOwner: pid => pid === 347 ? 'alive' : 'dead', + timeoutMs: 30, + retryDelayMs: 10, + now: () => nowMs, + wait: async delayMs => { nowMs += delayMs; }, + onOwnerDirectoryCreated: directory => { + if (replaced) return; + replaced = true; + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { mode: 0o700 }); + writePosixLeaseActorRecord({ + path: join(directory, `owner-${successorNonce}.json`), + sessionId, + nonce: successorNonce, + pid: 347, + processStartToken: 'successor-owner', + }); + }, + })).rejects.toThrow(/timed out/); + expect(replaced).toBe(true); + const successorPath = join( + codexAppPosixOwnerLeaseDirectory(root, sessionId), + `owner-${successorNonce}.json`, + ); + expect(existsSync(successorPath)).toBe(true); + }); + + it('retires a live foreign owner left in the publish-to-directory-CAS crash window', async () => { + const root = tempDir(); + const sessionId = 'session-posix-foreign-owner-crash-window'; + const statuses = new Map([ + [361, 'alive'], + [362, 'alive'], + [363, 'alive'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + let successor!: Awaited>; + let replaced = false; + let paused = false; + let foreignOwnerPath = ''; + let signalPublished!: () => void; + const published = new Promise(resolvePromise => { signalPublished = resolvePromise; }); + let resumePublisher!: () => void; + const publisherMayResume = new Promise(resolvePromise => { resumePublisher = resolvePromise; }); + + const delayed = acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 361, + processStartToken: 'delayed-owner', + inspectOwner, + timeoutMs: 1_000, + retryDelayMs: 5, + onOwnerDirectoryCreated: async directory => { + if (replaced) return; + replaced = true; + rmSync(directory, { recursive: true, force: true }); + successor = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 362, + processStartToken: 'successor-owner', + inspectOwner, + }); + }, + onOwnerRecordPublished: async (_directory, ownerRecordPath) => { + if (paused) return; + paused = true; + foreignOwnerPath = ownerRecordPath; + signalPublished(); + await publisherMayResume; + }, + }); + + await published; + expect(existsSync(foreignOwnerPath)).toBe(true); + expect(successor.isOwned()).toBe(true); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 363, + processStartToken: 'observing-contender', + inspectOwner, + timeoutMs: 30, + retryDelayMs: 5, + })).rejects.toThrow(/timed out/); + // The delayed actor is still live, but its D1-bound record has no authority + // in D2 and cannot permanently create a multiple-owner poison pill. + expect(existsSync(foreignOwnerPath)).toBe(false); + expect(successor.isOwned()).toBe(true); + + successor.release(); + resumePublisher(); + const recovered = await delayed; + expect(recovered.isOwned()).toBe(true); + recovered.release(); + }); + + it('reconciles a dead same-directory losing owner without disturbing the live winner', async () => { + const root = tempDir(); + const sessionId = 'session-posix-same-directory-owner-crash'; + const statuses = new Map([ + [365, 'alive'], + [366, 'dead'], + [367, 'alive'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + const winner = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 365, + processStartToken: 'same-dir-winner', + inspectOwner, + }); + const losingNonce = '4'.repeat(64); + const losingPath = join(winner.directory, `owner-${losingNonce}.json`); + writePosixLeaseActorRecord({ + path: losingPath, + sessionId, + nonce: losingNonce, + pid: 366, + processStartToken: 'same-dir-dead-loser', + }); + expect(winner.isOwned()).toBe(true); + + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 367, + processStartToken: 'same-dir-observer', + inspectOwner, + timeoutMs: 30, + retryDelayMs: 5, + })).rejects.toThrow(/timed out/); + expect(existsSync(losingPath)).toBe(false); + expect(winner.isOwned()).toBe(true); + + winner.release(); + const successor = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 367, + processStartToken: 'same-dir-observer', + inspectOwner, + }); + expect(successor.isOwned()).toBe(true); + successor.release(); + }); + + it('ignores and retires a live stale reaper published into a replacement directory', async () => { + const root = tempDir(); + const sessionId = 'session-posix-foreign-reaper-crash-window'; + const statuses = new Map([ + [371, 'alive'], + [372, 'alive'], + [373, 'alive'], + [374, 'alive'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + const stale = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 371, + processStartToken: 'stale-owner', + inspectOwner, + }); + statuses.set(371, 'dead'); + + let successor!: Awaited>; + let replaced = false; + let paused = false; + let foreignReaperPath = ''; + let signalPublished!: () => void; + const published = new Promise(resolvePromise => { signalPublished = resolvePromise; }); + let resumePublisher!: () => void; + const publisherMayResume = new Promise(resolvePromise => { resumePublisher = resolvePromise; }); + const delayedCleaner = acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 372, + processStartToken: 'delayed-reaper', + inspectOwner, + timeoutMs: 1_000, + retryDelayMs: 5, + onBeforeReaperRecordPublished: async directory => { + if (replaced) return; + replaced = true; + rmSync(directory, { recursive: true, force: true }); + successor = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 373, + processStartToken: 'replacement-owner', + inspectOwner, + }); + }, + onReaperRecordPublished: async (_directory, reaperRecordPath) => { + if (paused) return; + paused = true; + foreignReaperPath = reaperRecordPath; + signalPublished(); + await publisherMayResume; + }, + }); + + await published; + expect(existsSync(foreignReaperPath)).toBe(true); + // A foreign live reaper must not revoke the real D2 owner while its delayed + // publisher is paused at the exact post-publication crash boundary. + expect(successor.isOwned()).toBe(true); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 374, + processStartToken: 'replacement-observer', + inspectOwner, + timeoutMs: 30, + retryDelayMs: 5, + })).rejects.toThrow(/timed out/); + expect(existsSync(foreignReaperPath)).toBe(false); + expect(successor.isOwned()).toBe(true); + + successor.release(); + resumePublisher(); + const recovered = await delayedCleaner; + expect(recovered.isOwned()).toBe(true); + recovered.release(); + stale.release(); + }); + + it('hard-fails wrong record mode instead of grace-reclassifying it as partial', async () => { + const root = tempDir(); + const sessionId = 'session-posix-insecure-partial'; + const directory = codexAppPosixOwnerLeaseDirectory(root, sessionId); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + chmodSync(directory, 0o700); + const nonce = '1'.repeat(64); + writeFileSync(join(directory, `owner-${nonce}.json`), '{', { mode: 0o644 }); + chmodSync(join(directory, `owner-${nonce}.json`), 0o644); + await expect(acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId, + pid: 351, + processStartToken: 'insecure-successor', + inspectOwner: () => 'dead', + initializationGraceMs: 0, + })).rejects.toThrow(/0600/); + }); + + it('reclaims SIGKILL residue with multiple contenders but never grants two owners', async () => { + const root = tempDir(); + const statuses = new Map([ + [301, 'alive'], + [302, 'alive'], + [303, 'alive'], + [304, 'dead'], + ]); + const inspectOwner = (pid: number) => statuses.get(pid) ?? 'unknown'; + const stale = await acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: 'session-posix-race', + pid: 301, + processStartToken: 'start-stale', + inspectOwner, + }); + statuses.set(301, 'dead'); // model SIGKILL: no graceful release + const crashedReaperNonce = '2'.repeat(64); + writePosixLeaseActorRecord({ + path: join(stale.directory, `reap-${crashedReaperNonce}.json`), + sessionId: 'session-posix-race', + nonce: crashedReaperNonce, + pid: 304, + processStartToken: 'start-dead-reaper', + }); + + let secondResolved = false; + const contender = (pid: number, token: string) => acquireCodexAppPosixOwnerLease({ + controlRoot: root, + sessionId: 'session-posix-race', + pid, + processStartToken: token, + inspectOwner, + timeoutMs: 2_000, + retryDelayMs: 2, + }); + const bPromise = contender(302, 'start-B'); + const cPromise = contender(303, 'start-C').then(lease => { + secondResolved = true; + return lease; + }); + const firstWinner = await Promise.race([ + bPromise.then(lease => ({ name: 'B' as const, lease })), + cPromise.then(lease => ({ name: 'C' as const, lease })), + ]); + expect(stale.isOwned()).toBe(false); + expect(firstWinner.lease.isOwned()).toBe(true); + if (firstWinner.name === 'B') expect(secondResolved).toBe(false); + firstWinner.lease.release(); + const secondWinner = firstWinner.name === 'B' ? await cPromise : await bPromise; + expect(secondWinner.isOwned()).toBe(true); + expect(firstWinner.lease.isOwned()).toBe(false); + secondWinner.release(); + }); + + it('keeps random endpoint B reachable after old endpoint A closes late', async () => { + const root = shortPosixControlRoot(); + const sessionId = 'session-posix-close'; + const socketDirectory = join(root, 'sockets'); + const locatorPath = codexAppControlLocatorPath(root, sessionId); + mkdirSync(socketDirectory, { recursive: true, mode: 0o700 }); + mkdirSync(join(root, 'locators'), { recursive: true, mode: 0o700 }); + const endpointA = generateCodexAppPosixSocketEndpoint(socketDirectory); + const endpointB = generateCodexAppPosixSocketEndpoint(socketDirectory); + const serverA = createServer(socket => socket.end()); + const serverB = createServer(socket => socket.end('B')); + const listen = (server: Server, endpoint: string) => new Promise((resolvePromise, rejectPromise) => { + server.once('error', rejectPromise); + server.listen(endpoint, () => { + server.off('error', rejectPromise); + resolvePromise(); + }); + }); + await listen(serverA, endpointA); + await listen(serverB, endpointB); + const locator = { + version: 1 as const, + sessionId, + epoch: generateCodexAppControlEpoch(), + endpoint: endpointB, + }; + writeCodexAppControlLocator(locatorPath, locator, process.platform, root); + + await new Promise(resolvePromise => serverA.close(() => resolvePromise())); + expect(readCodexAppControlLocator( + locatorPath, + sessionId, + process.platform, + root, + )?.endpoint).toBe(endpointB); + expect(existsSync(endpointB)).toBe(true); + await new Promise((resolvePromise, rejectPromise) => { + const socket = createConnection(endpointB); + socket.once('data', data => { + expect(data.toString('utf8')).toBe('B'); + socket.destroy(); + resolvePromise(); + }); + socket.once('error', rejectPromise); + }); + await new Promise(resolvePromise => serverB.close(() => resolvePromise())); + }); +}); + +describe('Codex App signed challenge protocol', () => { + it('enforces challenge → matching locator epoch → ACK ordering and rejects repeats', () => { + const generation = generateCodexAppControlEpoch(); + const epoch = generateCodexAppControlEpoch(); + const challenge = generateCodexAppControlChallenge(); + const parse = (line: string) => parseCodexAppControlWireRecord(line); + const handshake = new CodexAppControlRunnerHandshake('session-handshake', generation, epoch); + + expect(handshake.handle(parse(encodeCodexAppControlChallenge( + 'session-handshake', challenge, + )), 0)).toEqual({ type: 'authenticate', challenge }); + expect(handshake.handle(parse(encodeCodexAppControlChallenge( + 'session-handshake', generateCodexAppControlChallenge(), + )), 0)).toEqual({ type: 'reject' }); + + const wrongEpoch = new CodexAppControlRunnerHandshake('session-handshake', generation, epoch); + expect(wrongEpoch.handle(parse(encodeCodexAppControlChallenge( + 'session-handshake', challenge, + )), 0).type).toBe('authenticate'); + expect(wrongEpoch.handle(parse(encodeCodexAppControlAccepted( + 'session-handshake', generation, challenge, generateCodexAppControlEpoch(), + )), 0)).toEqual({ type: 'reject' }); + + expect(handshake.handle(parse(encodeCodexAppControlAccepted( + 'session-handshake', generation, challenge, epoch, + )), 0)).toEqual({ type: 'accepted', challenge }); + expect(handshake.active).toBe(true); + expect(handshake.handle(parse(encodeCodexAppControlAck( + 'session-handshake', generation, challenge, 2, + )), 1)).toEqual({ type: 'reject' }); + expect(handshake.handle(parse(encodeCodexAppControlAck( + 'session-handshake', generation, challenge, 1, + )), 1)).toEqual({ type: 'ack', seq: 1 }); + }); + + it('uses an absolute handshake deadline that slow transport activity cannot extend', async () => { + vi.useFakeTimers(); + const timedOut = vi.fn(); + const timer = armCodexAppControlHandshakeTimeout(timedOut, 100); + try { + await vi.advanceTimersByTimeAsync(40); + // Model two partial/slow-drip transport reads. There is deliberately no + // activity/reset API: only a matching accepted record clears the timer. + await vi.advanceTimersByTimeAsync(40); + expect(timedOut).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(20); + expect(timedOut).toHaveBeenCalledTimes(1); + } finally { + clearTimeout(timer); + vi.useRealTimers(); + } + }); + + it('re-arms the shared 90-second proof deadline after an authenticated disconnect', async () => { + vi.useFakeTimers(); + const deadline = new CodexAppControlProofDeadline(); + const startupTimeout = vi.fn(); + const disconnectTimeout = vi.fn(); + try { + deadline.arm(startupTimeout); + expect(deadline.armed).toBe(true); + // Successful startup authentication clears the first deadline. + deadline.clear(); + await vi.advanceTimersByTimeAsync(CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS); + expect(startupTimeout).not.toHaveBeenCalled(); + + // Losing that authenticated socket starts a fresh proof deadline. + deadline.arm(disconnectTimeout); + await vi.advanceTimersByTimeAsync(CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS - 1); + expect(disconnectTimeout).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(disconnectTimeout).toHaveBeenCalledTimes(1); + expect(deadline.armed).toBe(false); + } finally { + deadline.clear(); + vi.useRealTimers(); + } + }); + + it('authenticates possession and signs every marker without putting a reusable secret on the wire', () => { + const dir = tempDir(); + const bootstrap = createCodexAppControlBootstrap(dir, 'session-signed'); + const consumed = consumeCodexAppControlBootstrap(bootstrap.path, 'session-signed'); + const challenge = generateCodexAppControlChallenge(); + + const authLine = encodeCodexAppControlAuth( + consumed.privateKey, + 'session-signed', + consumed.generation, + challenge, + ); + const auth = parseCodexAppControlWireRecord(authLine); + expect(auth?.type).toBe('auth'); + expect(authLine).not.toContain('privateKey'); + expect(auth && auth.type === 'auth' + ? verifyCodexAppControlAuth(auth, bootstrap.identity.publicKey) + : false).toBe(true); + + const markerLine = encodeCodexAppSignedControlMarker( + consumed.privateKey, + 'session-signed', + consumed.generation, + challenge, + 1, + 'activity', + { phase: 'progress', atMs: 123 }, + ); + const marker = parseCodexAppControlWireRecord(markerLine); + expect(marker?.type).toBe('marker'); + expect(marker && marker.type === 'marker' + ? verifyCodexAppSignedControlMarker(marker, bootstrap.identity.publicKey) + : false).toBe(true); + }); + + it('rejects replay across connection challenges and mutation of every signed domain field', () => { + const dir = tempDir(); + const bootstrap = createCodexAppControlBootstrap(dir, 'session-replay'); + const consumed = consumeCodexAppControlBootstrap(bootstrap.path, 'session-replay'); + const challenge = generateCodexAppControlChallenge(); + const otherChallenge = generateCodexAppControlChallenge(); + const auth = parseCodexAppControlWireRecord(encodeCodexAppControlAuth( + consumed.privateKey, 'session-replay', consumed.generation, challenge, + )); + expect(auth?.type).toBe('auth'); + if (!auth || auth.type !== 'auth') throw new Error('auth parse failed'); + expect(verifyCodexAppControlAuth({ ...auth, challenge: otherChallenge }, bootstrap.identity.publicKey)).toBe(false); + expect(verifyCodexAppControlAuth({ ...auth, sessionId: 'other-session' }, bootstrap.identity.publicKey)).toBe(false); + + const marker = parseCodexAppControlWireRecord(encodeCodexAppSignedControlMarker( + consumed.privateKey, + 'session-replay', + consumed.generation, + challenge, + 7, + 'final', + { content: 'real' }, + )); + expect(marker?.type).toBe('marker'); + if (!marker || marker.type !== 'marker') throw new Error('marker parse failed'); + for (const mutated of [ + { ...marker, sessionId: 'other' }, + { ...marker, generation: 'A'.repeat(43) }, + { ...marker, challenge: otherChallenge }, + { ...marker, seq: 8 }, + { ...marker, kind: 'activity' }, + { ...marker, payload: { content: 'forged' } }, + ]) { + expect(verifyCodexAppSignedControlMarker(mutated, bootstrap.identity.publicKey)).toBe(false); + } + }); + + it('deduplicates a re-signed sequence retry across socket reconnects', () => { + const replay = new CodexAppControlReplayWindow(); + const firstGeneration = 'A'.repeat(43); + const nextGeneration = 'B'.repeat(43); + expect(replay.hasSeen(firstGeneration, 1)).toBe(false); + replay.commit(firstGeneration, 1); + expect(replay.hasSeen(firstGeneration, 1)).toBe(true); + expect(replay.hasSeen(firstGeneration, 0)).toBe(true); + expect(replay.hasSeen(firstGeneration, 2)).toBe(false); + replay.commit(firstGeneration, 3); + expect(replay.hasSeen(firstGeneration, 2)).toBe(true); + + replay.commit(nextGeneration, 7); + replay.retainOnly(firstGeneration); + expect(replay.highWater(firstGeneration)).toBe(3); + expect(replay.highWater(nextGeneration)).toBe(0); + }); + + it('bounds socket line buffering and resynchronizes after oversized input', () => { + const decoder = new CodexAppControlLineDecoder(); + expect(decoder.push(Buffer.from('one\npar')).lines).toEqual(['one']); + expect(decoder.push(Buffer.from('tial\n')).lines).toEqual(['partial']); + expect(decoder.push(Buffer.alloc(CODEX_APP_CONTROL_LINE_MAX_BYTES + 1, 0x78)).droppedMalformed).toBe(true); + const recovered = decoder.push(Buffer.from('\nvalid\n')); + expect(recovered).toEqual({ lines: ['valid'], droppedMalformed: true }); + }); +}); diff --git a/test/codex-app-dispatch-ledger.test.ts b/test/codex-app-dispatch-ledger.test.ts new file mode 100644 index 000000000..f0574710c --- /dev/null +++ b/test/codex-app-dispatch-ledger.test.ts @@ -0,0 +1,201 @@ +import { describe, expect, it } from 'vitest'; +import { + appendAcceptedCodexAppDispatch, + cancelCodexAppDispatch, + committedCodexAppSequence, + hasUnsettledCodexAppDispatch, + prepareCodexAppDispatch, + retryPreparedCodexAppDispatch, + retireCodexAppDispatchAfterBackingMissing, + settleCodexAppDispatch, + validateCodexAppManagedSendOrigin, +} from '../src/utils/codex-app-dispatch-ledger.js'; + +describe('Codex App durable dispatch ledger', () => { + it('preserves frozen payload and settles only the exact prepared FIFO head', () => { + let ledger = appendAcceptedCodexAppDispatch([], { + dispatchId: 'dispatch-1', + turnId: 'turn-1', + replyTurnId: 'route-1', + deliverySink: 'http_wait', + content: 'frozen content', + codexAppInput: { text: 'clean content' }, + }); + ledger = appendAcceptedCodexAppDispatch(ledger, { + dispatchId: 'dispatch-2', + turnId: 'turn-2', + content: 'next content', + }); + + const first = prepareCodexAppDispatch(ledger, { + dispatchId: 'dispatch-1', turnId: 'turn-1', + }); + expect(first).toMatchObject({ ok: true }); + if (!first.ok) return; + expect(first.ledger[0]).toMatchObject({ + state: 'prepared', + replyTurnId: 'route-1', + deliverySink: 'http_wait', + content: 'frozen content', + codexAppInput: { text: 'clean content' }, + }); + + const settled = settleCodexAppDispatch( + first.ledger, + [], + { dispatchId: 'dispatch-1', turnId: 'turn-1' }, + 'generation-1', + 7, + ); + expect(settled).toMatchObject({ ok: true }); + if (!settled.ok) return; + expect(settled.ledger.map(entry => entry.dispatchId)).toEqual(['dispatch-2']); + expect(committedCodexAppSequence(settled.commits, 'generation-1', 7)).toBe(true); + }); + + it('treats every accepted or prepared entry as unsettled lifecycle ownership', () => { + expect(hasUnsettledCodexAppDispatch(undefined)).toBe(false); + expect(hasUnsettledCodexAppDispatch([])).toBe(false); + expect(hasUnsettledCodexAppDispatch([ + { dispatchId: 'd1', turnId: 't1', state: 'accepted', content: 'one' }, + ])).toBe(true); + expect(hasUnsettledCodexAppDispatch([ + { dispatchId: 'd1', turnId: 't1', state: 'prepared', content: 'one' }, + ])).toBe(true); + }); + + it('authorizes host relay only for the exact live Lark-bound Codex App entry', () => { + const ledger = [{ + dispatchId: 'd1', turnId: 't1', dispatchAttempt: 3, + state: 'prepared' as const, content: 'one', deliverySink: 'lark' as const, + }]; + expect(validateCodexAppManagedSendOrigin( + ledger, + { turnId: 't1', dispatchAttempt: 3 }, + true, + )).toEqual({ ok: true, requiresLedger: true }); + expect(validateCodexAppManagedSendOrigin( + ledger, + { turnId: 't1', dispatchAttempt: 4 }, + true, + )).toMatchObject({ ok: false }); + expect(validateCodexAppManagedSendOrigin([ + ...ledger, + { ...ledger[0]!, dispatchId: 'd2' }, + ], { turnId: 't1', dispatchAttempt: 3 }, true)).toEqual({ + ok: false, + error: '2 Codex App ledger entries match the live relay origin', + }); + }); + + it('rejects a Codex App capability when settlement removed the ledger before relay admission', () => { + expect(validateCodexAppManagedSendOrigin( + [], + { turnId: 't1', dispatchAttempt: 3 }, + true, + )).toEqual({ + ok: false, + error: '0 Codex App ledger entries match the live relay origin', + }); + // A non-Codex managed receiver may legitimately carry a dispatch attempt + // without owning the Codex ledger; do not blanket-block that path. + expect(validateCodexAppManagedSendOrigin( + [], + { turnId: 'vc-turn', dispatchAttempt: 3 }, + false, + )).toEqual({ ok: true, requiresLedger: false }); + }); + + it('rejects Codex App host sends bound to non-Lark sinks', () => { + for (const deliverySink of ['http_wait', 'http_async', 'suppressed'] as const) { + expect(validateCodexAppManagedSendOrigin([{ + dispatchId: 'd1', turnId: 't1', state: 'prepared', content: 'one', deliverySink, + }], { turnId: 't1' }, true)).toEqual({ + ok: false, + error: `Codex App output is bound to ${deliverySink}`, + }); + } + }); + + it('does not cancel a predecessor while a prepared successor exists', () => { + const ledger = [ + { dispatchId: 'd1', turnId: 't1', state: 'prepared' as const, content: 'one' }, + { dispatchId: 'd2', turnId: 't2', state: 'prepared' as const, content: 'two' }, + ]; + expect(cancelCodexAppDispatch(ledger, { dispatchId: 'd1', turnId: 't1' })) + .toEqual({ ok: false, error: 'prepared_successor_exists' }); + }); + + it('returns an exactly untouched queued activation from prepared to accepted without losing its token', () => { + const ledger = [{ + dispatchId: 'activation-dispatch', + turnId: 'activation-turn', + dispatchAttempt: 2, + state: 'prepared' as const, + content: 'exact queued opening', + queuedActivationToken: 'activation-token', + }]; + expect(retryPreparedCodexAppDispatch(ledger, { + dispatchId: 'activation-dispatch', + turnId: 'activation-turn', + dispatchAttempt: 2, + })).toEqual({ + ok: true, + ledger: [{ ...ledger[0], state: 'accepted' }], + }); + }); + + it('refuses a prepared retry when a prepared successor could overtake it', () => { + const ledger = [ + { dispatchId: 'activation', turnId: 't1', state: 'prepared' as const, content: 'one' }, + { dispatchId: 'successor', turnId: 't2', state: 'prepared' as const, content: 'two' }, + ]; + expect(retryPreparedCodexAppDispatch(ledger, { + dispatchId: 'activation', turnId: 't1', + })).toEqual({ ok: false, error: 'prepared_successor_exists' }); + }); + + it.each(['accepted', 'prepared'] as const)( + 'retires an exact %s crashed delivery after backing-missing proof without dropping its successor', + state => { + const ledger = [ + { + dispatchId: 'old-dispatch', turnId: 'delivery-old', dispatchAttempt: 3, + state, content: 'old', + }, + { + dispatchId: 'successor', turnId: 'ordinary-successor', + state: 'prepared' as const, content: 'new', + }, + ]; + expect(retireCodexAppDispatchAfterBackingMissing(ledger, 'delivery-old', 3)) + .toEqual({ ok: true, ledger: [ledger[1]] }); + }, + ); + + it('keeps exact crash retirement idempotent and fails closed on ambiguous receipt identity', () => { + expect(retireCodexAppDispatchAfterBackingMissing([], 'delivery-old', 3)) + .toEqual({ ok: true, ledger: [] }); + const ambiguous = [ + { dispatchId: 'd1', turnId: 'delivery-old', dispatchAttempt: 3, state: 'accepted' as const, content: 'one' }, + { dispatchId: 'd2', turnId: 'delivery-old', dispatchAttempt: 3, state: 'prepared' as const, content: 'two' }, + ]; + expect(retireCodexAppDispatchAfterBackingMissing(ambiguous, 'delivery-old', 3)) + .toEqual({ ok: false, error: 'dispatch_identity_ambiguous' }); + for (const conflictingAttempt of [2, undefined]) { + const conflict = [{ + dispatchId: 'old', turnId: 'delivery-old', dispatchAttempt: conflictingAttempt, + state: 'accepted' as const, content: 'old', + }]; + expect(retireCodexAppDispatchAfterBackingMissing(conflict, 'delivery-old', 3)) + .toEqual({ ok: false, error: 'dispatch_attempt_conflict' }); + } + + const duplicateDispatchId = [ + { dispatchId: 'same', turnId: 'delivery-old', dispatchAttempt: 3, state: 'accepted' as const, content: 'old' }, + { dispatchId: 'same', turnId: 'unrelated', state: 'prepared' as const, content: 'new' }, + ]; + expect(retireCodexAppDispatchAfterBackingMissing(duplicateDispatchId, 'delivery-old', 3)) + .toEqual({ ok: true, ledger: [duplicateDispatchId[1]] }); + }); +}); diff --git a/test/codex-app-runner.integration.test.ts b/test/codex-app-runner.integration.test.ts index d961f20cd..26563b978 100644 --- a/test/codex-app-runner.integration.test.ts +++ b/test/codex-app-runner.integration.test.ts @@ -3,21 +3,46 @@ import { chmodSync, copyFileSync, existsSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, + unlinkSync, writeFileSync, } from 'node:fs'; +import { createServer, type Server, type Socket } from 'node:net'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import { encodeRunnerInput } from '../src/adapters/cli/runner-input.js'; +import { + CodexAppControlFinalAssembler, + CodexAppControlLineDecoder, + CodexAppControlSequenceFence, + codexAppControlLocatorPath, + codexAppPosixControlRoot, + codexAppControlSocketPath, + createCodexAppControlBootstrap, + encodeCodexAppControlAck, + encodeCodexAppControlAccepted, + encodeCodexAppControlChallenge, + ensureCodexAppControlDirectory, + generateCodexAppControlChallenge, + generateCodexAppControlEpoch, + generateCodexAppPosixSocketEndpoint, + parseCodexAppControlWireRecord, + verifyCodexAppControlAuth, + verifyCodexAppSignedControlMarker, + writeCodexAppControlLocator, + type CodexAppControlLocator, + type CodexAppSignedControlMarker, +} from '../src/utils/codex-app-control.js'; import type { CodexAppTurnInput } from '../src/types.js'; const RUNNER_PATH = resolve('src/codex-app-runner.ts'); const FAKE_SERVER_FIXTURE = resolve('test/fixtures/fake-codex-app-server.mjs'); const CONTROL_PREFIX = '::botmux-codex-app:'; -const FINAL_MARKER = /\x1b\]777;botmux:final:([A-Za-z0-9+/=]+)\x07/; +const SESSION_ID = 'session-integration'; interface Harness { child: ChildProcessWithoutNullStreams; @@ -31,9 +56,388 @@ interface RunResult { imagePath: string; missingImagePath: string; final: Record; + finals: Array>; + activities: Array>; + states: Array>; + markers: Array<{ kind: string; payload: Record }>; + wireLines: string[]; + privateKeyEncoding: string; } const liveChildren = new Set(); +const liveCollectors = new Set(); +const liveLocatorCollectors = new Set(); + +class ControlCollector { + readonly bootstrap; + readonly socketPath: string; + readonly privateKeyEncoding: string; + readonly activities: Array> = []; + readonly states: Array> = []; + readonly finals: Array> = []; + readonly markers: Array<{ kind: string; payload: Record }> = []; + readonly wireLines: string[] = []; + authCount = 0; + readonly authObserved: Promise; + private resolveAuthObserved!: () => void; + private readonly server: Server; + private readonly sockets = new Set(); + private pendingAcceptance?: { socket: Socket; challenge: string }; + private lastSeq = 0; + private disconnectedFinalChunk = false; + private omittedFinalChunk = false; + private disconnectedFinalEndAck = false; + private readonly socketDirectory: string; + + constructor( + readonly directory: string, + private readonly manualAcceptance = false, + private readonly disconnectOnFirstFinalChunk = false, + private readonly omitFirstFinalChunk = false, + private readonly disconnectAfterFirstFinalEndBeforeAck = false, + ) { + this.socketDirectory = mkdtempSync('/tmp/bca-sock-'); + this.socketPath = codexAppControlSocketPath(this.socketDirectory, SESSION_ID); + this.bootstrap = createCodexAppControlBootstrap(directory, SESSION_ID, this.socketPath); + this.privateKeyEncoding = JSON.parse(readFileSync(this.bootstrap.path, 'utf8')).privateKey; + this.authObserved = new Promise(resolvePromise => { this.resolveAuthObserved = resolvePromise; }); + this.server = createServer(socket => this.accept(socket)); + liveCollectors.add(this); + } + + listen(): Promise { + return new Promise((resolvePromise, rejectPromise) => { + this.server.once('error', rejectPromise); + this.server.listen(this.socketPath, () => { + this.server.off('error', rejectPromise); + resolvePromise(); + }); + }); + } + + releaseAcceptance(): void { + const pending = this.pendingAcceptance; + if (!pending) throw new Error('no authenticated runner is awaiting acceptance'); + this.pendingAcceptance = undefined; + pending.socket.write(`${encodeCodexAppControlAccepted( + SESSION_ID, + this.bootstrap.identity.generation, + pending.challenge, + )}\n`); + } + + async restartEndpoint(): Promise { + for (const socket of this.sockets) socket.destroy(); + this.sockets.clear(); + if (this.server.listening) { + await new Promise(resolvePromise => this.server.close(() => resolvePromise())); + } + await this.listen(); + } + + async close(): Promise { + for (const socket of this.sockets) socket.destroy(); + this.sockets.clear(); + if (this.server.listening) { + await new Promise(resolvePromise => this.server.close(() => resolvePromise())); + } + liveCollectors.delete(this); + rmSync(this.socketDirectory, { recursive: true, force: true }); + } + + private accept(socket: Socket): void { + this.sockets.add(socket); + const decoder = new CodexAppControlLineDecoder(); + const sequenceFence = new CodexAppControlSequenceFence(); + const finalAssembler = new CodexAppControlFinalAssembler(); + const challenge = generateCodexAppControlChallenge(); + let authenticated = false; + socket.on('data', chunk => { + const decoded = decoder.push(chunk); + if (decoded.droppedMalformed) socket.destroy(); + for (const line of decoded.lines) { + this.wireLines.push(line); + const record = parseCodexAppControlWireRecord(line); + if (!record || record.sessionId !== SESSION_ID) { + socket.destroy(); + continue; + } + if (!authenticated) { + if (record.type !== 'auth' + || record.challenge !== challenge + || record.generation !== this.bootstrap.identity.generation + || !verifyCodexAppControlAuth(record, this.bootstrap.identity.publicKey)) { + socket.destroy(); + continue; + } + authenticated = true; + this.authCount++; + this.resolveAuthObserved(); + if (this.manualAcceptance) this.pendingAcceptance = { socket, challenge }; + else { + socket.write(`${encodeCodexAppControlAccepted( + SESSION_ID, + this.bootstrap.identity.generation, + challenge, + )}\n`); + } + continue; + } + if (record.type !== 'marker' + || record.challenge !== challenge + || record.generation !== this.bootstrap.identity.generation + || !sequenceFence.accept(record.seq) + || !verifyCodexAppSignedControlMarker(record, this.bootstrap.identity.publicKey)) { + socket.destroy(); + continue; + } + // The worker checks connection continuity before its cumulative replay + // window. A reconnect may therefore replay an already-committed final + // transaction contiguously: ACK every duplicate without reassembling + // or publishing its final side effect again. + if (record.seq <= this.lastSeq) { + socket.write(`${encodeCodexAppControlAck( + SESSION_ID, + this.bootstrap.identity.generation, + challenge, + record.seq, + )}\n`); + continue; + } + this.markers.push({ kind: record.kind, payload: record.payload }); + if (record.kind === 'final-chunk' + && this.disconnectOnFirstFinalChunk + && !this.disconnectedFinalChunk) { + this.disconnectedFinalChunk = true; + // Model a replacement worker: the per-connection assembly and + // in-memory cumulative sequence window both disappear. + this.lastSeq = 0; + socket.destroy(); + return; + } + if (record.kind === 'final-chunk' + && this.omitFirstFinalChunk + && !this.omittedFinalChunk) { + // Simulate a missing fragment inside this connection. final-end must + // be rejected without an ACK, forcing a complete replay. + this.omittedFinalChunk = true; + continue; + } + const finalResult = finalAssembler.accept(record.kind, record.payload); + if (finalResult.status === 'reject') { + socket.destroy(); + return; + } + if (finalResult.status === 'not-final') this.collectNonFinalMarker(record); + else if (finalResult.status === 'complete') this.finals.push(finalResult.payload); + if (finalResult.status === 'accepted') continue; + this.lastSeq = record.seq; + if (record.kind === 'final-end' + && this.disconnectAfterFirstFinalEndBeforeAck + && !this.disconnectedFinalEndAck) { + this.disconnectedFinalEndAck = true; + socket.destroy(); + return; + } + socket.write(`${encodeCodexAppControlAck( + SESSION_ID, + this.bootstrap.identity.generation, + challenge, + record.seq, + )}\n`); + } + }); + socket.on('error', () => undefined); + socket.on('close', () => this.sockets.delete(socket)); + socket.write(`${encodeCodexAppControlChallenge(SESSION_ID, challenge)}\n`); + } + + private collectNonFinalMarker(marker: CodexAppSignedControlMarker): void { + if (marker.kind === 'state') { + this.states.push(marker.payload); + return; + } + if (marker.kind === 'activity') { + this.activities.push(marker.payload); + } + } +} + +type LocatorEndpointMode = 'accept' | 'wrong-epoch' | 'repeat-challenge' | 'slow-drip'; + +interface LocatorEndpointHandle { + locator: CodexAppControlLocator; + readonly connections: number; + readonly closedConnections: number; + readonly authCount: number; + authObserved: Promise; + closeObserved: Promise; + close(): Promise; +} + +/** + * Runs the real runner locator loop on POSIX with the same strict random + * AF_UNIX endpoint + protected locator shape used by the worker. + */ +class LocatorControlCollector { + readonly locatorPath: string; + readonly bootstrap; + private readonly endpoints = new Set(); + private readonly socketDirectory: string; + + constructor(readonly directory: string) { + const controlRoot = codexAppPosixControlRoot(); + this.locatorPath = codexAppControlLocatorPath(controlRoot, SESSION_ID); + this.socketDirectory = join(controlRoot, 'sockets'); + ensureCodexAppControlDirectory(controlRoot); + ensureCodexAppControlDirectory(join(controlRoot, 'locators')); + ensureCodexAppControlDirectory(this.socketDirectory); + this.bootstrap = createCodexAppControlBootstrap(directory, SESSION_ID, { + kind: 'locator', + locatorPath: this.locatorPath, + }); + liveLocatorCollectors.add(this); + } + + async publish(mode: LocatorEndpointMode): Promise { + const endpoint = generateCodexAppPosixSocketEndpoint(this.socketDirectory); + const epoch = generateCodexAppControlEpoch(); + const locator: CodexAppControlLocator = { + version: 1, + sessionId: SESSION_ID, + endpoint, + epoch, + }; + const server = createServer(); + const sockets = new Set(); + let connectionCount = 0; + let closedConnectionCount = 0; + let authCount = 0; + let resolveAuth!: () => void; + let resolveClose!: () => void; + const authObserved = new Promise(resolvePromise => { resolveAuth = resolvePromise; }); + const closeObserved = new Promise(resolvePromise => { resolveClose = resolvePromise; }); + server.on('connection', socket => { + sockets.add(socket); + connectionCount++; + const decoder = new CodexAppControlLineDecoder(); + const challenge = generateCodexAppControlChallenge(); + let accepted = false; + let lastSeq = 0; + let dripTimer: ReturnType | undefined; + socket.on('data', chunk => { + const decoded = decoder.push(chunk); + if (decoded.droppedMalformed) socket.destroy(); + for (const line of decoded.lines) { + const record = parseCodexAppControlWireRecord(line); + if (!record || record.sessionId !== SESSION_ID) { + socket.destroy(); + continue; + } + if (!accepted) { + if (record.type !== 'auth' + || record.generation !== this.bootstrap.identity.generation + || record.challenge !== challenge + || !verifyCodexAppControlAuth(record, this.bootstrap.identity.publicKey)) { + socket.destroy(); + continue; + } + authCount++; + resolveAuth(); + if (mode === 'repeat-challenge') continue; + accepted = true; + socket.write(`${encodeCodexAppControlAccepted( + SESSION_ID, + this.bootstrap.identity.generation, + challenge, + mode === 'wrong-epoch' ? generateCodexAppControlEpoch() : epoch, + )}\n`); + continue; + } + if (record.type !== 'marker' + || record.generation !== this.bootstrap.identity.generation + || record.challenge !== challenge + || record.seq <= lastSeq + || !verifyCodexAppSignedControlMarker(record, this.bootstrap.identity.publicKey)) { + socket.destroy(); + continue; + } + lastSeq = record.seq; + socket.write(`${encodeCodexAppControlAck( + SESSION_ID, + this.bootstrap.identity.generation, + challenge, + record.seq, + )}\n`); + } + }); + socket.on('error', () => undefined); + socket.on('close', () => { + if (dripTimer) clearInterval(dripTimer); + sockets.delete(socket); + closedConnectionCount++; + resolveClose(); + }); + if (mode === 'slow-drip') { + const line = `${encodeCodexAppControlChallenge(SESSION_ID, challenge)}\n`; + let offset = 0; + socket.write(line[offset++]!); + dripTimer = setInterval(() => { + if (socket.destroyed || offset >= line.length) { + if (dripTimer) clearInterval(dripTimer); + dripTimer = undefined; + return; + } + socket.write(line[offset++]!); + }, 200); + } else { + socket.write(`${encodeCodexAppControlChallenge(SESSION_ID, challenge)}\n`); + } + if (mode === 'repeat-challenge') { + socket.write(`${encodeCodexAppControlChallenge( + SESSION_ID, + generateCodexAppControlChallenge(), + )}\n`); + } + }); + await new Promise((resolvePromise, rejectPromise) => { + server.once('error', rejectPromise); + server.listen(endpoint, () => { + server.off('error', rejectPromise); + resolvePromise(); + }); + }); + writeCodexAppControlLocator(this.locatorPath, locator); + let closed = false; + const handle: LocatorEndpointHandle = { + locator, + get connections() { return connectionCount; }, + get closedConnections() { return closedConnectionCount; }, + get authCount() { return authCount; }, + authObserved, + closeObserved, + close: async () => { + if (closed) return; + closed = true; + for (const socket of sockets) socket.destroy(); + sockets.clear(); + if (server.listening) { + await new Promise(resolvePromise => server.close(() => resolvePromise())); + } + try { unlinkSync(endpoint); } catch { /* libuv may already remove it */ } + this.endpoints.delete(handle); + }, + }; + this.endpoints.add(handle); + return handle; + } + + async close(): Promise { + await Promise.all([...this.endpoints].map(endpoint => endpoint.close())); + try { unlinkSync(this.locatorPath); } catch { /* absent or already replaced */ } + liveLocatorCollectors.delete(this); + } +} function startRunner( fakeCodex: string, @@ -41,27 +445,32 @@ function startRunner( logPath: string, version: string, behavior: string, + controlBootstrapPath: string | null, + options: { threadId?: string; env?: Record } = {}, ): Harness { let stdout = ''; let stderr = ''; - const child = spawn(process.execPath, [ - '--import', - 'tsx', - RUNNER_PATH, - '--session-id', - 'session-integration', - '--codex-bin', - fakeCodex, - '--cwd', - cwd, - ], { + const env = { + ...process.env, + FAKE_CODEX_LOG: logPath, + FAKE_CODEX_VERSION: version, + FAKE_CODEX_BEHAVIOR: behavior, + NODE_ENV: 'test', + ...options.env, + }; + delete env.BOTMUX_CODEX_APP_CONTROL_NONCE; + delete env.BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP; + if (controlBootstrapPath !== null) env.BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP = controlBootstrapPath; + const runnerArgs = [ + '--import', 'tsx', RUNNER_PATH, + '--session-id', SESSION_ID, + '--codex-bin', fakeCodex, + '--cwd', cwd, + ...(options.threadId ? ['--thread-id', options.threadId] : []), + ]; + const child = spawn(process.execPath, runnerArgs, { cwd: resolve('.'), - env: { - ...process.env, - FAKE_CODEX_LOG: logPath, - FAKE_CODEX_VERSION: version, - FAKE_CODEX_BEHAVIOR: behavior, - }, + env, stdio: ['pipe', 'pipe', 'pipe'], }); liveChildren.add(child); @@ -75,28 +484,31 @@ function startRunner( }; } -function waitForOutput(harness: Harness, predicate: (output: string) => boolean, timeoutMs = 10_000): Promise { - if (predicate(harness.stdout)) return Promise.resolve(); +function waitFor( + harness: Harness, + predicate: () => boolean, + timeoutMs = 10_000, +): Promise { + if (predicate()) return Promise.resolve(); return new Promise((resolvePromise, rejectPromise) => { + const poll = setInterval(() => { + if (!predicate()) return; + cleanup(); + resolvePromise(); + }, 10); const timer = setTimeout(() => { cleanup(); - rejectPromise(new Error(`runner output timed out\nstdout:\n${harness.stdout}\nstderr:\n${harness.stderr}`)); + rejectPromise(new Error(`runner timed out\nstdout:\n${harness.stdout}\nstderr:\n${harness.stderr}`)); }, timeoutMs); - const onData = () => { - if (!predicate(harness.stdout)) return; - cleanup(); - resolvePromise(); - }; const onExit = (code: number | null, signal: NodeJS.Signals | null) => { cleanup(); - rejectPromise(new Error(`runner exited before expected output (code=${code}, signal=${signal})\nstdout:\n${harness.stdout}\nstderr:\n${harness.stderr}`)); + rejectPromise(new Error(`runner exited early (code=${code}, signal=${signal})\nstdout:\n${harness.stdout}\nstderr:\n${harness.stderr}`)); }; const cleanup = () => { + clearInterval(poll); clearTimeout(timer); - harness.child.stdout.off('data', onData); harness.child.off('exit', onExit); }; - harness.child.stdout.on('data', onData); harness.child.once('exit', onExit); }); } @@ -104,9 +516,7 @@ function waitForOutput(harness: Harness, predicate: (output: string) => boolean, async function stopChild(child: ChildProcessWithoutNullStreams): Promise { if (child.exitCode !== null || child.signalCode !== null) return; await new Promise(resolvePromise => { - const forceTimer = setTimeout(() => { - child.kill('SIGKILL'); - }, 1_000); + const forceTimer = setTimeout(() => child.kill('SIGKILL'), 1_000); child.once('exit', () => { clearTimeout(forceTimer); resolvePromise(); @@ -115,25 +525,17 @@ async function stopChild(child: ChildProcessWithoutNullStreams): Promise { }); } -function decodeFinalMarker(output: string): Record { - const match = output.match(FINAL_MARKER); - if (!match) throw new Error(`final marker missing from output:\n${output}`); - return JSON.parse(Buffer.from(match[1], 'base64').toString('utf8')); -} - function readRequests(logPath: string): Array> { if (!existsSync(logPath)) return []; - return readFileSync(logPath, 'utf8') - .split('\n') - .filter(Boolean) - .map(line => JSON.parse(line)); + return readFileSync(logPath, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); } async function exerciseRunner(opts: { version: string; - behavior?: 'success' | 'capability-error' | 'generic-error' | 'osc-injection'; + behavior?: 'success' | 'capability-error' | 'generic-error' | 'osc-injection' | 'empty-final' | 'start-response-last'; includeMissingImage?: boolean; includeSidecar?: boolean; + turnCount?: number; }): Promise { const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-')); const fakeCodex = join(dir, 'fake-codex'); @@ -146,7 +548,8 @@ async function exerciseRunner(opts: { 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zg0sAAAAASUVORK5CYII=', 'base64', )); - + const control = new ControlCollector(dir); + await control.listen(); const sidecar: CodexAppTurnInput = { text: 'clean user text', additionalContext: { @@ -161,37 +564,338 @@ async function exerciseRunner(opts: { ], clientUserMessageId: 'om_integration_123', }; - const harness = startRunner(fakeCodex, dir, logPath, opts.version, opts.behavior ?? 'success'); + const harness = startRunner( + fakeCodex, dir, logPath, opts.version, opts.behavior ?? 'success', control.bootstrap.path, + ); try { - await waitForOutput(harness, output => output.includes('Codex App connected.')); - const encoded = encodeRunnerInput( - 'legacy prompt', - opts.includeSidecar === false ? undefined : sidecar, - ); - harness.child.stdin.write(`${CONTROL_PREFIX}${encoded}\r`); - await waitForOutput(harness, output => FINAL_MARKER.test(output)); - + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + expect(existsSync(control.bootstrap.path)).toBe(false); + const turnCount = opts.turnCount ?? 1; + for (let i = 0; i < turnCount; i++) { + const legacyContent = turnCount === 1 + ? 'legacy prompt' + : `legacy prompt ${i + 1}`; + const turnSidecar = opts.includeSidecar === false + ? undefined + : turnCount === 1 + ? sidecar + : { ...sidecar, clientUserMessageId: `om_integration_${i + 1}` }; + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput( + legacyContent, + turnSidecar, + )}\r`); + } + await waitFor(harness, () => ( + control.finals.length >= turnCount + && control.states.filter(state => state.busy === false).length >= 2 + && (harness.stdout.match(/› /g)?.length ?? 0) >= 2 + )); const output = harness.stdout; - const final = decodeFinalMarker(output); + const requests = readRequests(logPath); await stopChild(harness.child); - return { output, requests: readRequests(logPath), imagePath, missingImagePath, final }; + return { + output, + requests, + imagePath, + missingImagePath, + final: control.finals[0]!, + finals: [...control.finals], + activities: [...control.activities], + states: [...control.states], + markers: [...control.markers], + wireLines: [...control.wireLines], + privateKeyEncoding: control.privateKeyEncoding, + }; } finally { await stopChild(harness.child); + await control.close(); rmSync(dir, { recursive: true, force: true }); } } afterEach(async () => { await Promise.all([...liveChildren].map(stopChild)); + await Promise.all([...liveCollectors].map(collector => collector.close())); + await Promise.all([...liveLocatorCollectors].map(collector => collector.close())); }); describe('codex-app-runner app-server protocol integration', () => { + it('refuses to start without a worker-established control bootstrap', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-no-key-')); + const harness = startRunner('/does/not/matter', dir, join(dir, 'requests.jsonl'), '0.136.0', 'success', null); + try { + const exitCode = harness.child.exitCode ?? await new Promise(resolvePromise => { + harness.child.once('exit', code => resolvePromise(code)); + }); + expect(exitCode).toBe(2); + expect(harness.stderr).toContain('BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP is required'); + } finally { + await stopChild(harness.child); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('does not start app-server until the worker verifies proof and accepts the generation', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-auth-gate-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir, true); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + await control.authObserved; + expect(readRequests(logPath)).toEqual([]); + expect(harness.stdout).not.toContain('Codex App connected.'); + control.releaseAcceptance(); + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + expect(readRequests(logPath).some(request => request.method === 'initialize')).toBe(true); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps retrying when the worker socket begins listening after the runner starts', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-late-socket-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + await new Promise(resolvePromise => setTimeout(resolvePromise, 300)); + expect(harness.child.exitCode).toBeNull(); + expect(readRequests(logPath)).toEqual([]); + await control.listen(); + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + expect(control.authCount).toBe(1); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('polls locators, rejects repeated/wrong-epoch/slow-drip handshakes, burns A, and connects B', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-locator-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new LocatorControlCollector(dir); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + // Missing locator is a poll miss, not a fatal bootstrap/app-server start. + await new Promise(resolvePromise => setTimeout(resolvePromise, 350)); + expect(harness.child.exitCode).toBeNull(); + expect(readRequests(logPath)).toEqual([]); + + const repeated = await control.publish('repeat-challenge'); + await waitFor(harness, () => repeated.closedConnections >= 1); + expect(readRequests(logPath)).toEqual([]); + + const wrongEpoch = await control.publish('wrong-epoch'); + await waitFor(harness, () => wrongEpoch.authCount >= 1 && wrongEpoch.closedConnections >= 1); + expect(readRequests(logPath)).toEqual([]); + + const slowDripStartedAt = Date.now(); + const slowDrip = await control.publish('slow-drip'); + await waitFor(harness, () => slowDrip.closedConnections >= 1); + expect(Date.now() - slowDripStartedAt).toBeGreaterThanOrEqual(4_500); + expect(readRequests(logPath)).toEqual([]); + + const acceptedA = await control.publish('accept'); + await waitFor(harness, () => ( + acceptedA.authCount >= 1 && harness.stdout.includes('Codex App connected.') + )); + expect(readRequests(logPath).filter(request => request.method === 'initialize')).toHaveLength(1); + + await acceptedA.close(); + const acceptedAConnections = acceptedA.connections; + await new Promise(resolvePromise => setTimeout(resolvePromise, 600)); + expect(acceptedA.connections).toBe(acceptedAConnections); + + const acceptedB = await control.publish('accept'); + await waitFor(harness, () => acceptedB.authCount >= 1); + expect(readRequests(logPath).filter(request => request.method === 'initialize')).toHaveLength(1); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('emits signed submitted/progress/completed boundaries without a reusable secret on the wire', async () => { + const result = await exerciseRunner({ version: '0.136.0', turnCount: 2 }); + expect(result.activities.map(activity => activity.phase)).toEqual( + expect.arrayContaining(['submitted', 'progress', 'completed']), + ); + expect(result.activities + .filter(activity => activity.phase === 'submitted' || activity.phase === 'completed') + .map(activity => activity.phase)) + .toEqual(['submitted', 'completed', 'submitted', 'completed']); + expect(result.activities.filter(activity => activity.phase === 'completed')).toMatchObject([ + { turnId: 'turn-fake-1', atMs: expect.any(Number) }, + { turnId: 'turn-fake-2', atMs: expect.any(Number) }, + ]); + expect(result.wireLines.join('\n')).not.toContain(result.privateKeyEncoding); + expect(result.requests.find(request => request.fixtureEnv)?.fixtureEnv).toEqual({ + controlNoncePresent: false, + controlBootstrapPresent: false, + argvContainsControlNonce: false, + }); + expect(result.finals).toHaveLength(2); + expect(result.finals.map(final => final.turnId)).toEqual([ + 'om_integration_1', + 'om_integration_2', + ]); + expect(result.output.match(/› /g)).toHaveLength(2); + + // One idle state belongs to initialized startup and one to the fully + // drained two-turn queue. There must be no transient idle between turns. + const idleMarkerIndexes = result.markers + .map((marker, index) => marker.kind === 'state' && marker.payload.busy === false ? index : -1) + .filter(index => index >= 0); + expect(idleMarkerIndexes).toHaveLength(2); + const lastFinalEndIndex = result.markers.findLastIndex(marker => marker.kind === 'final-end'); + const lastCompletedIndex = result.markers.findLastIndex( + marker => marker.kind === 'activity' && marker.payload.phase === 'completed', + ); + expect(idleMarkerIndexes[1]).toBeGreaterThan(lastCompletedIndex); + expect(idleMarkerIndexes[1]).toBeGreaterThan(lastFinalEndIndex); + }); + + it('emits a zero-chunk final transaction for an empty answer before the signed idle boundary', async () => { + const result = await exerciseRunner({ version: '0.136.0', behavior: 'empty-final' }); + expect(result.finals).toEqual([ + expect.objectContaining({ + turnId: 'om_integration_123', + nativeTurnId: 'turn-fake-1', + content: '', + }), + ]); + const finalStart = result.markers.find(marker => marker.kind === 'final-start'); + expect(finalStart?.payload).toMatchObject({ total: 0, turnId: 'om_integration_123' }); + expect(result.markers.some(marker => marker.kind === 'final-chunk')).toBe(false); + const finalEndIndex = result.markers.findIndex(marker => marker.kind === 'final-end'); + const drainedIdleIndex = result.markers.findLastIndex( + marker => marker.kind === 'state' && marker.payload.busy === false, + ); + expect(finalEndIndex).toBeGreaterThan(-1); + expect(drainedIdleIndex).toBeGreaterThan(finalEndIndex); + }); + + it('re-authenticates the same live runner with a fresh challenge after worker endpoint restart', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-warm-proof-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + await waitFor(harness, () => ( + harness.stdout.includes('Codex App connected.') + && control.authCount === 1 + && control.states.length === 1 + )); + const initializeCount = readRequests(logPath).filter(request => request.method === 'initialize').length; + await control.restartEndpoint(); + await waitFor(harness, () => control.authCount === 2 && control.states.length === 2); + expect(readRequests(logPath).filter(request => request.method === 'initialize')).toHaveLength(initializeCount); + expect(control.states[1]).toMatchObject({ busy: false, atMs: expect.any(Number) }); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('warm follow-up')}\r`); + await waitFor(harness, () => control.finals.length === 1); + expect(control.finals[0]).toMatchObject({ content: 'fake answer 1' }); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('replays a complete final transaction when the worker is replaced after its first chunk', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-final-replay-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir, false, true); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('final replay')}\r`); + await waitFor(harness, () => control.authCount >= 2 && control.finals.length === 1); + expect(control.finals).toEqual([ + expect.objectContaining({ content: 'fake answer 1' }), + ]); + expect(readRequests(logPath).filter(request => request.method === 'turn/start')).toHaveLength(1); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('does not ACK an incomplete final-end and replays the complete transaction after re-authentication', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-incomplete-final-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir, false, false, true); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('incomplete final replay')}\r`); + await waitFor(harness, () => control.authCount >= 2 && control.finals.length === 1); + expect(control.finals).toEqual([ + expect.objectContaining({ content: 'fake answer 1' }), + ]); + expect(readRequests(logPath).filter(request => request.method === 'turn/start')).toHaveLength(1); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('ACKs a committed final replay after ACK loss without publishing the final twice', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-final-ack-loss-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir, false, false, false, true); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.136.0', 'success', control.bootstrap.path); + try { + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('final ACK loss')}\r`); + await waitFor(harness, () => control.authCount >= 2 && control.states.length >= 2); + expect(control.finals).toEqual([ + expect.objectContaining({ content: 'fake answer 1' }), + ]); + expect(readRequests(logPath).filter(request => request.method === 'turn/start')).toHaveLength(1); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('sends clean text, hidden context, localImage, and clientUserMessageId on codex >= 0.136', async () => { const result = await exerciseRunner({ version: '0.136.0', includeMissingImage: true }); const initialize = result.requests.find(request => request.method === 'initialize'); expect(initialize?.params.capabilities).toEqual({ experimentalApi: true }); - const turns = result.requests.filter(request => request.method === 'turn/start'); expect(turns).toHaveLength(1); expect(turns[0].params.input).toEqual([ @@ -212,7 +916,339 @@ describe('codex-app-runner app-server protocol integration', () => { expect(result.final.nativeTurnId).toBe('turn-fake-1'); }); - it('preserves the full legacy prompt on codex < 0.135 even if the server would ignore new fields', async () => { + it('buffers start notifications until a response-last RPC proves the authoritative native id', async () => { + const result = await exerciseRunner({ version: '0.144.1', behavior: 'start-response-last' }); + expect(result.final).toMatchObject({ + turnId: 'om_integration_123', + nativeTurnId: 'turn-fake-1', + content: 'fake answer 1', + }); + expect(result.final.content).not.toContain('unrelated autonomous output'); + expect(result.requests.filter(request => request.method === 'thread/turns/list')).toHaveLength(0); + expect(result.markers.some(marker => marker.kind === 'diagnostic')).toBe(false); + expect(result.activities.map(activity => activity.phase)).toEqual([ + 'submitted', + 'progress', + 'completed', + ]); + }); + + it('does not let response-last A overwrite a newer Goal B native lifecycle', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-response-last-goal-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner( + fakeCodex, + dir, + logPath, + '0.144.1', + 'start-response-last-goal', + control.bootstrap.path, + ); + try { + await waitFor(harness, () => control.states.some(state => state.busy === false)); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('first legacy', { + text: 'first exact', clientUserMessageId: 'om_response_last_a', + })}\r`); + await waitFor(harness, () => control.finals.length === 1 + && control.states.some(state => state.busy === true && state.tracksTurn === false)); + + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('confirm legacy', { + text: 'confirm exact', clientUserMessageId: 'om_response_last_confirm', + })}\r`); + await waitFor(harness, () => control.finals.length === 2 + && control.states.filter(state => state.busy === false).length >= 2); + + const requests = readRequests(logPath); + expect(requests.filter(request => request.method === 'turn/start')).toHaveLength(1); + expect(requests.filter(request => request.method === 'turn/steer')).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ + expectedTurnId: 'turn-goal-auto', + clientUserMessageId: 'om_response_last_confirm', + }), + }), + ]); + expect(control.finals).toEqual([ + expect.objectContaining({ + turnId: 'om_response_last_a', + nativeTurnId: 'turn-fake-1', + content: 'fake answer 1', + }), + expect.objectContaining({ + turnId: 'om_response_last_confirm', + nativeTurnId: 'turn-goal-auto', + content: 'goal steer answer', + }), + ]); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps a Goal auto-continuation native-busy and steers the next exact Lark turn into it', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-goal-steer-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.144.1', 'goal-continuation', control.bootstrap.path); + const sidecar = (text: string, id: string): CodexAppTurnInput => ({ + text, + clientUserMessageId: id, + }); + try { + await waitFor(harness, () => control.states.some(state => state.busy === false)); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('first legacy', sidecar('first', 'om_goal_a'))}\r`); + await waitFor(harness, () => ( + control.finals.length === 1 + && control.states.some(state => state.busy === true && state.tracksTurn === false) + )); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('confirm legacy', sidecar('confirm', 'om_goal_confirm'))}\r`); + await waitFor(harness, () => control.finals.length === 2 + && control.states.filter(state => state.busy === false).length >= 2); + + const requests = readRequests(logPath); + expect( + requests.filter(request => request.method === 'turn/start'), + JSON.stringify(requests.filter(request => request.method?.startsWith('turn/')), null, 2), + ).toHaveLength(1); + const steer = requests.filter(request => request.method === 'turn/steer'); + expect(steer).toHaveLength(1); + expect(steer[0].params).toMatchObject({ + threadId: 'thread-fake', + expectedTurnId: 'turn-goal-auto', + clientUserMessageId: 'om_goal_confirm', + input: [{ type: 'text', text: 'confirm', text_elements: [] }], + }); + expect(steer[0].params).not.toHaveProperty('cwd'); + expect(control.finals).toEqual([ + expect.objectContaining({ turnId: 'om_goal_a', content: 'fake answer 1' }), + expect.objectContaining({ + turnId: 'om_goal_confirm', + nativeTurnId: 'turn-goal-auto', + content: 'goal steer answer', + }), + ]); + const firstEnd = control.markers.findIndex(marker => + marker.kind === 'final-end' && marker.payload.id?.startsWith('om_goal_a:')); + const secondStart = control.markers.findIndex(marker => + marker.kind === 'final-start' && marker.payload.turnId === 'om_goal_confirm'); + expect(control.markers.slice(firstEnd + 1, secondStart)).not.toContainEqual( + expect.objectContaining({ kind: 'state', payload: expect.objectContaining({ busy: false }) }), + ); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('falls back to turn/start only after an explicit stale expected-turn rejection', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-steer-race-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.144.1', 'goal-steer-race', control.bootstrap.path); + try { + await waitFor(harness, () => control.states.some(state => state.busy === false)); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('first', { + text: 'first', clientUserMessageId: 'om_race_a', + })}\r`); + await waitFor(harness, () => control.finals.length === 1 + && control.states.some(state => state.busy === true && state.tracksTurn === false)); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('confirm legacy', { + text: 'confirm exact', clientUserMessageId: 'om_race_confirm', + })}\r`); + await waitFor(harness, () => control.finals.length === 2 + && control.states.filter(state => state.busy === false).length >= 2); + + const requests = readRequests(logPath); + const starts = requests.filter(request => request.method === 'turn/start'); + const steers = requests.filter(request => request.method === 'turn/steer'); + expect(steers).toHaveLength(1); + expect(steers[0].params).toMatchObject({ + expectedTurnId: 'turn-goal-auto', + clientUserMessageId: 'om_race_confirm', + }); + expect(starts).toHaveLength(2); + expect(starts[1].params).toMatchObject({ + clientUserMessageId: 'om_race_confirm', + input: [{ type: 'text', text: 'confirm exact', text_elements: [] }], + }); + expect(control.finals[1]).toMatchObject({ + turnId: 'om_race_confirm', + nativeTurnId: 'turn-fake-2', + content: 'fake answer 2', + }); + expect(control.finals[1].content).not.toContain('autonomous goal text before Lark input'); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reconciles a mismatched completion only through one exact full-history client id match', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-history-reconcile-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.144.1', 'history-reconcile', control.bootstrap.path); + try { + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('legacy', { + text: 'exact input', + clientUserMessageId: 'om_exact_reconcile', + })}\r`); + await waitFor(harness, () => control.finals.length === 1 + && control.states.filter(state => state.busy === false).length >= 2); + expect(readRequests(logPath).filter(request => request.method === 'thread/turns/list')) + .toEqual([expect.objectContaining({ + params: expect.objectContaining({ + threadId: 'thread-fake', + limit: 50, + sortDirection: 'desc', + itemsView: 'full', + }), + })]); + expect(control.finals[0]).toMatchObject({ + turnId: 'om_exact_reconcile', + nativeTurnId: 'turn-fake-1', + content: 'reconciled answer 1', + }); + expect(control.finals[0].content).not.toContain('autonomous text before exact input'); + expect(control.markers.some(marker => marker.kind === 'diagnostic')).toBe(false); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + for (const [behavior, expectedReason] of [ + ['history-no-match', 'found no match'], + ['history-multi-match', 'found multiple matches'], + ] as const) { + it(`fails closed with an explicit settled error when bounded history ${expectedReason}`, async () => { + const dir = mkdtempSync(join(tmpdir(), `botmux-codex-runner-${behavior}-`)); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner(fakeCodex, dir, logPath, '0.144.1', behavior, control.bootstrap.path); + try { + await waitFor(harness, () => harness.stdout.includes('Codex App connected.')); + harness.child.stdin.write(`${CONTROL_PREFIX}${encodeRunnerInput('legacy', { + text: 'must match exactly', + clientUserMessageId: 'om_conflict', + })}\r`); + await waitFor(harness, () => control.markers.some(marker => marker.kind === 'diagnostic') + && control.finals.length === 1 + && control.states.filter(state => state.busy === false).length >= 2); + const diagnostic = control.markers.find(marker => marker.kind === 'diagnostic'); + expect(diagnostic?.payload).toMatchObject({ + code: 'native_turn_identity_conflict', + turnId: 'om_conflict', + message: expect.stringContaining(expectedReason), + }); + expect(control.finals[0]).toMatchObject({ + turnId: 'om_conflict', + content: expect.stringContaining('Codex App native turn identity conflict'), + }); + expect(control.states.filter(state => state.busy === false).length).toBeGreaterThanOrEqual(2); + expect(harness.child.exitCode).toBeNull(); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + } + + it('keeps the startup deadline armed through initialize and never exposes a pre-ready prompt', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-startup-deadline-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner( + fakeCodex, + dir, + logPath, + '0.144.1', + 'hang-initialize', + control.bootstrap.path, + { env: { BOTMUX_TEST_CODEX_APP_STARTUP_TIMEOUT_MS: '1000' } }, + ); + try { + const exitCode = await new Promise(resolvePromise => harness.child.once('exit', resolvePromise)); + expect(exitCode).toBe(2); + expect(readRequests(logPath).filter(request => request.method === 'initialize')).toHaveLength(1); + expect(harness.stdout).not.toContain('Codex App connected.'); + expect(harness.stdout).not.toContain('› '); + expect(control.states).toEqual([]); + expect(harness.stderr).toContain('startup timed out before the first signed runner state'); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('does not turn an ambiguous resume timeout into a fresh thread', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-resume-timeout-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner( + fakeCodex, + dir, + logPath, + '0.144.1', + 'hang-resume', + control.bootstrap.path, + { + threadId: 'thread-existing', + env: { BOTMUX_TEST_CODEX_APP_STARTUP_TIMEOUT_MS: '1000' }, + }, + ); + try { + await new Promise(resolvePromise => harness.child.once('exit', () => resolvePromise())); + const requests = readRequests(logPath); + expect(requests.filter(request => request.method === 'thread/resume')).toHaveLength(1); + expect(requests.filter(request => request.method === 'thread/start')).toHaveLength(0); + expect(harness.stdout).not.toContain('Codex App connected.'); + expect(control.states).toEqual([]); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('preserves the full legacy prompt on codex < 0.135 even if the server ignores new fields', async () => { const result = await exerciseRunner({ version: '0.134.9' }); const turns = result.requests.filter(request => request.method === 'turn/start'); expect(turns).toHaveLength(1); @@ -269,11 +1305,11 @@ describe('codex-app-runner app-server protocol integration', () => { expect(result.final.nativeTurnId).toBe('turn-fake-1'); }); - it('escapes split agent/command OSC injections and emits only the trusted final marker', async () => { + it('escapes split agent/command OSC injections and emits the trusted final only out of band', async () => { const result = await exerciseRunner({ version: '0.136.0', behavior: 'osc-injection' }); expect(result.output).toContain('␛]777;botmux:final:'); - expect(result.output.match(/\x1b\]777;botmux:final:/g)).toHaveLength(1); + expect(result.output.match(/\x1b\]777;botmux:final:/g)).toBeNull(); expect(result.final).toMatchObject({ turnId: 'om_integration_123', nativeTurnId: 'turn-fake-1', diff --git a/test/codex-app-turn-dispatch.test.ts b/test/codex-app-turn-dispatch.test.ts new file mode 100644 index 000000000..846d37e1f --- /dev/null +++ b/test/codex-app-turn-dispatch.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from 'vitest'; +import { CodexAppTurnDispatchQueue } from '../src/utils/codex-app-turn-dispatch.js'; + +describe('CodexAppTurnDispatchQueue', () => { + it('attributes queued finals to immutable FIFO heads after later writes change worker globals', () => { + const queue = new CodexAppTurnDispatchQueue(); + queue.reserve('turn-1', 4); + queue.reserve('turn-2', 9); + + expect(queue.settleFinal({ + turnId: 'turn-1', + nativeTurnId: 'native-1', + })).toMatchObject({ + ok: true, + turnId: 'turn-1', + dispatchAttempt: 4, + nativeTurnId: 'native-1', + remaining: 1, + }); + expect(queue.settleFinal({ turnId: 'turn-2' })).toMatchObject({ + ok: true, + turnId: 'turn-2', + dispatchAttempt: 9, + remaining: 0, + }); + }); + + it('uses a complete empty final as the exact FIFO boundary', () => { + const queue = new CodexAppTurnDispatchQueue(); + queue.reserve('empty-turn'); + queue.reserve('next-turn'); + + expect(queue.settleFinal({ turnId: 'empty-turn' })).toMatchObject({ + ok: true, + turnId: 'empty-turn', + remaining: 1, + }); + expect(queue.settleFinal({ turnId: 'next-turn' })).toMatchObject({ + ok: true, + turnId: 'next-turn', + remaining: 0, + }); + }); + + it('rejects mismatched turn and attempt assertions without advancing the head', () => { + const queue = new CodexAppTurnDispatchQueue(); + queue.reserve('turn-1', 7); + queue.reserve('turn-2', 8); + + expect(queue.settleFinal({ turnId: 'turn-2' })).toEqual({ + ok: false, + reason: 'turn_mismatch', + markerTurnId: 'turn-2', + expectedTurnId: 'turn-1', + }); + expect(queue.settleFinal({ turnId: 'turn-1', dispatchAttempt: 8 })).toEqual({ + ok: false, + reason: 'dispatch_attempt_mismatch', + markerDispatchAttempt: 8, + expectedDispatchAttempt: 7, + }); + expect(queue.size()).toBe(2); + expect(queue.settleFinal({ turnId: 'turn-1', dispatchAttempt: 7 })).toMatchObject({ + ok: true, + turnId: 'turn-1', + dispatchAttempt: 7, + remaining: 1, + }); + }); + + it('cancels only the exact failed write and preserves peers in FIFO order', () => { + const queue = new CodexAppTurnDispatchQueue(); + const first = queue.reserve('turn-1', 1); + const second = queue.reserve('turn-2', 2); + const third = queue.reserve('turn-3', 3); + + expect(queue.cancelExact(second.handle)).toBe(true); + expect(queue.cancelExact(second.handle)).toBe(false); + expect(queue.settleFinal({ turnId: 'turn-1' })).toMatchObject({ + ok: true, + turnId: first.turnId, + remaining: 1, + }); + expect(queue.settleFinal({ turnId: 'turn-3' })).toMatchObject({ + ok: true, + turnId: third.turnId, + remaining: 0, + }); + // A late adapter rejection after an already-applied final must not emit a + // second ambiguous/failed terminal for the settled turn. + expect(queue.cancelExact(first.handle)).toBe(false); + }); + + it('recovers at most one daemon-frozen warm-reattach identity and clears on reset', () => { + const queue = new CodexAppTurnDispatchQueue(); + expect(queue.recoverWarmReattach(undefined, 3)).toBeUndefined(); + expect(queue.recoverWarmReattach('reattached', 3)).toMatchObject({ + turnId: 'reattached', + dispatchAttempt: 3, + }); + expect(queue.recoverWarmReattach('must-not-overwrite', 4)).toBeUndefined(); + expect(queue.settleFinal({ turnId: 'reattached' })).toMatchObject({ + ok: true, + turnId: 'reattached', + dispatchAttempt: 3, + }); + + queue.reserve('stale'); + queue.clear(); + expect(queue.size()).toBe(0); + expect(queue.settleFinal({ turnId: 'stale' })).toEqual({ + ok: false, + reason: 'no_pending_turn', + }); + }); +}); diff --git a/test/codex-app-turn-liveness.test.ts b/test/codex-app-turn-liveness.test.ts new file mode 100644 index 000000000..29e4cc3a6 --- /dev/null +++ b/test/codex-app-turn-liveness.test.ts @@ -0,0 +1,513 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + applyTrustedCodexAppActivityMarker, + applyTrustedCodexAppStateMarker, + CodexAppFlushPromptReplay, + CodexAppReadyAuthority, + CodexAppTurnLiveness, + shouldBeginCodexAppReattachObservation, +} from '../src/utils/codex-app-turn-liveness.js'; +import { + CodexAppControlRecordApplicationGate, + projectCodexAppControlReadinessStatus, +} from '../src/utils/codex-app-control.js'; + +describe('CodexAppReadyAuthority', () => { + it('does not allocate a Botmux liveness slot for a native Goal continuation', () => { + const authority = new CodexAppReadyAuthority(); + const tracker = new CodexAppTurnLiveness(1_000); + const state = applyTrustedCodexAppStateMarker(tracker, authority, { + busy: true, + tracksTurn: false, + acceptingInput: true, + atMs: 10_000, + }, 10_000); + expect(state).toMatchObject({ accepted: true, busy: true, tracksTurn: false }); + expect(tracker.hasActiveTurn()).toBe(false); + expect(authority.canPublishPromptReady()).toBe(false); + + tracker.beginReattachObservation(10_100); + expect(tracker.hasActiveTurn()).toBe(true); + applyTrustedCodexAppStateMarker(tracker, authority, { + busy: true, + tracksTurn: false, + acceptingInput: true, + atMs: 10_110, + }, 10_110); + expect(tracker.hasActiveTurn()).toBe(false); + }); + + it('suppresses a terminal prompt and publishes ready only after signed idle, preserving final-before-ready', () => { + const authority = new CodexAppReadyAuthority(); + const tracker = new CodexAppTurnLiveness(1_000); + const events: string[] = []; + + tracker.begin('turn-ready', 10_000); + authority.beginWork(); + applyTrustedCodexAppActivityMarker(tracker, { + phase: 'completed', + atMs: 10_100, + }, 10_100); + // Model a prompt byte that wins the terminal/backend race and arrives + // before the signed final transaction. It is not an idle authority. + if (tracker.notePrompt(10_101) && authority.canPublishPromptReady()) events.push('ready-from-terminal'); + else events.push('terminal-prompt-suppressed'); + + events.push('final-output'); + const state = applyTrustedCodexAppStateMarker(tracker, authority, { + busy: false, + atMs: 10_102, + }, 10_102); + expect(state).toMatchObject({ accepted: true, busy: false, shouldPublishReady: true }); + if (state.shouldPublishReady && authority.canPublishPromptReady()) events.push('prompt-ready'); + + expect(events).toEqual([ + 'terminal-prompt-suppressed', + 'final-output', + 'prompt-ready', + ]); + }); + + it('does not retain an old idle grant across a new queued turn', () => { + const authority = new CodexAppReadyAuthority(); + authority.noteSignedState(false); + expect(authority.canPublishPromptReady()).toBe(true); + + authority.beginWork(); + expect(authority.canPublishPromptReady()).toBe(false); + authority.noteSignedState(true); + expect(authority.canPublishPromptReady()).toBe(false); + }); + + it('grants exactly one late terminal prompt after an exact submit cancellation', () => { + const authority = new CodexAppReadyAuthority(); + const tracker = new CodexAppTurnLiveness(1_000); + const replay = new CodexAppFlushPromptReplay(); + const handle = tracker.begin('cancelled-turn', 10_000); + authority.beginWork(); + + // No prompt was observed during writeInput; it arrives only after the + // exact failed slot has been removed and the flush has returned. + replay.cancelSubmission(tracker, authority, handle, 10_100); + expect(replay.consumeAfterFlush(tracker)).toBe(false); + expect(tracker.notePrompt(10_110)).toBe(true); + expect(authority.canPublishPromptReady()).toBe(false); + expect(authority.consumeLatePromptRecovery(!tracker.hasActiveTurn())).toBe(true); + expect(authority.canPublishPromptReady()).toBe(false); + expect(authority.consumeLatePromptRecovery(true)).toBe(false); + }); + + it('does not arm recovery for an absent or already-cancelled submit handle', () => { + const authority = new CodexAppReadyAuthority(); + const tracker = new CodexAppTurnLiveness(1_000); + const replay = new CodexAppFlushPromptReplay(); + + replay.cancelSubmission(tracker, authority, undefined); + replay.cancelSubmission(tracker, authority, 999); + expect(authority.consumeLatePromptRecovery(true)).toBe(false); + + const exact = tracker.begin('exact-turn'); + replay.cancelSubmission(tracker, authority, exact); + expect(authority.consumeLatePromptRecovery(true)).toBe(true); + replay.cancelSubmission(tracker, authority, exact); + expect(authority.consumeLatePromptRecovery(true)).toBe(false); + }); + + it('does not let a cancelled turn prompt release newer work', () => { + const authority = new CodexAppReadyAuthority(); + const tracker = new CodexAppTurnLiveness(1_000); + const replay = new CodexAppFlushPromptReplay(); + const cancelled = tracker.begin('cancelled-turn', 10_000); + authority.beginWork(); + replay.cancelSubmission(tracker, authority, cancelled, 10_100); + + tracker.begin('new-turn', 10_110); + authority.beginWork(); + expect(tracker.notePrompt(10_120)).toBe(false); + expect(authority.consumeLatePromptRecovery(!tracker.hasActiveTurn())).toBe(false); + expect(authority.canPublishPromptReady()).toBe(false); + }); + + it('clears late-prompt recovery on trusted work, signed state, and reset', () => { + const authority = new CodexAppReadyAuthority(); + const tracker = new CodexAppTurnLiveness(1_000); + const replay = new CodexAppFlushPromptReplay(); + const arm = () => { + const handle = tracker.begin('cancelled-turn'); + authority.beginWork(); + replay.cancelSubmission(tracker, authority, handle); + }; + + arm(); + authority.beginWork(); // trusted submitted/progress handler + expect(authority.consumeLatePromptRecovery(true)).toBe(false); + + arm(); + authority.noteSignedState(false); + expect(authority.consumeLatePromptRecovery(true)).toBe(false); + expect(authority.canPublishPromptReady()).toBe(true); + + arm(); + authority.reset(); + expect(authority.consumeLatePromptRecovery(true)).toBe(false); + expect(authority.canPublishPromptReady()).toBe(false); + }); +}); + +describe('shouldBeginCodexAppReattachObservation', () => { + it('covers a Zellij reattach even though Zellij is not pipe mode', () => { + for (const backendType of ['tmux', 'herdr', 'zellij'] as const) { + expect(shouldBeginCodexAppReattachObservation({ + cliId: 'codex-app', + backendType, + isReattach: true, + })).toBe(true); + } + }); + + it('does not observe a fresh spawn, non-persistent backend, or another CLI', () => { + expect(shouldBeginCodexAppReattachObservation({ + cliId: 'codex-app', + backendType: 'zellij', + isReattach: false, + })).toBe(false); + expect(shouldBeginCodexAppReattachObservation({ + cliId: 'codex-app', + backendType: 'pty', + isReattach: true, + })).toBe(false); + expect(shouldBeginCodexAppReattachObservation({ + cliId: 'codex', + backendType: 'tmux', + isReattach: true, + })).toBe(false); + }); +}); + +describe('CodexAppTurnLiveness', () => { + it('coalesces a replayed final while persistence is pending and keeps type-ahead working', async () => { + const tracker = new CodexAppTurnLiveness(1_000); + const gate = new CodexAppControlRecordApplicationGate(); + tracker.begin('turn-n', 10_000); + tracker.begin('turn-n-plus-1', 10_010); + + let completionAwaitingFinal = false; + const completed = applyTrustedCodexAppActivityMarker(tracker, { + phase: 'completed', + atMs: 10_100, + }, 10_100); + expect(completed).toMatchObject({ accepted: true, phase: 'completed' }); + completionAwaitingFinal = true; + + let releasePersistence!: () => void; + const persistence = new Promise(resolve => { + releasePersistence = resolve; + }); + let applicationCount = 0; + const applyFinal = async (): Promise => { + applicationCount++; + if (!completionAwaitingFinal) tracker.completeCurrent(10_110); + completionAwaitingFinal = false; + await persistence; + return true; + }; + + const first = gate.run('generation-a', 42, applyFinal); + await Promise.resolve(); + const replayEffect = vi.fn(async () => true); + const replay = gate.run('generation-a', 42, replayEffect); + + expect(applicationCount).toBe(1); + expect(replayEffect).not.toHaveBeenCalled(); + expect(tracker.poll(10_110)).toMatchObject({ + active: true, + turnId: 'turn-n-plus-1', + }); + expect(projectCodexAppControlReadinessStatus('idle', { + controlProven: true, + signedStateObserved: false, + inputReady: false, + })).toBe('working'); + + releasePersistence(); + await expect(first).resolves.toBe(true); + await expect(replay).resolves.toBe(true); + gate.release('generation-a', 42, first); + + expect(applicationCount).toBe(1); + expect(tracker.poll(10_120)).toMatchObject({ + active: true, + turnId: 'turn-n-plus-1', + }); + }); + + it('projects idle only after authenticated signed input readiness', () => { + expect(projectCodexAppControlReadinessStatus('idle', { + controlProven: false, + signedStateObserved: false, + inputReady: false, + })).toBe('working'); + expect(projectCodexAppControlReadinessStatus('idle', { + controlProven: true, + signedStateObserved: false, + inputReady: false, + })).toBe('working'); + expect(projectCodexAppControlReadinessStatus('idle', { + controlProven: true, + signedStateObserved: true, + inputReady: false, + })).toBe('working'); + expect(projectCodexAppControlReadinessStatus('idle', { + controlProven: true, + signedStateObserved: true, + inputReady: true, + })).toBe('idle'); + expect(projectCodexAppControlReadinessStatus('stalled', { + controlProven: false, + signedStateObserved: false, + inputReady: false, + })).toBe('stalled'); + }); + + it('stays working before the no-progress threshold, then stalls and notifies once', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + + expect(tracker.poll(10_999)).toEqual({ + active: true, + stalled: false, + newlyStalled: false, + shouldNotify: false, + turnId: 'turn-1', + }); + expect(tracker.poll(11_000)).toEqual({ + active: true, + stalled: true, + newlyStalled: true, + shouldNotify: true, + turnId: 'turn-1', + }); + expect(tracker.poll(12_000)).toEqual({ + active: true, + stalled: true, + newlyStalled: false, + shouldNotify: false, + turnId: 'turn-1', + }); + }); + + it('uses runner activity as the clock and clears a stalled projection on recovery', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + tracker.noteActivity(10_900); + expect(tracker.poll(11_500).stalled).toBe(false); + + expect(tracker.poll(11_900)).toMatchObject({ stalled: true, shouldNotify: true }); + tracker.noteActivity(12_000); + expect(tracker.poll(12_000)).toMatchObject({ stalled: false, newlyStalled: false }); + + // A recovered turn may become quiet again, but it must not spam the same + // warning a second time. + expect(tracker.poll(13_000)).toMatchObject({ + stalled: true, + newlyStalled: true, + shouldNotify: false, + }); + }); + + it('does not move the activity clock backwards for a delayed OSC marker', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + tracker.noteActivity(11_000); + tracker.noteActivity(10_500); + expect(tracker.poll(11_999).stalled).toBe(false); + expect(tracker.poll(12_000).stalled).toBe(true); + }); + + it('keeps flushPending inputs FIFO and starts a queued turn clock only after its predecessor completes', () => { + const tracker = new CodexAppTurnLiveness(1_000); + // flushPending() may write both Botmux inputs before the serial runner + // finishes the first one. Completion must advance one queue slot, not + // clear or overwrite the later turn. + tracker.begin('turn-1', 10_000); + tracker.begin('turn-2', 10_010); + tracker.noteActivity(10_900); + tracker.completeCurrent(11_000); + + expect(tracker.poll(11_999)).toEqual({ + active: true, + stalled: false, + newlyStalled: false, + shouldNotify: false, + turnId: 'turn-2', + }); + expect(tracker.poll(12_000)).toEqual({ + active: true, + stalled: true, + newlyStalled: true, + shouldNotify: true, + turnId: 'turn-2', + }); + }); + + it('cancels only the exact failed submission and preserves queued peers', () => { + const tracker = new CodexAppTurnLiveness(1_000); + const first = tracker.begin('turn-1', 10_000); + const second = tracker.begin('turn-2', 10_010); + + tracker.cancel(second, 10_100); + expect(tracker.poll(11_000)).toMatchObject({ + stalled: true, + shouldNotify: true, + turnId: 'turn-1', + }); + + tracker.cancel(first, 11_100); + expect(tracker.poll(20_000)).toEqual({ + active: false, + stalled: false, + newlyStalled: false, + shouldNotify: false, + }); + }); + + it('replays a deferred inter-turn prompt when the queued control-line write is cancelled', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + const second = tracker.begin('turn-2', 10_010); + + expect(tracker.completeCurrent(10_100)).toBe(false); + // The runner briefly became empty while turn-2 was still arriving in + // chunks, so its real prompt is held behind turn-2's liveness slot. + expect(tracker.notePrompt(10_110)).toBe(false); + expect(tracker.cancel(second, 10_120)).toBe(true); + expect(tracker.hasActiveTurn()).toBe(false); + }); + + it('replays a cancelled flush prompt exactly once after every peer liveness slot drains', () => { + const tracker = new CodexAppTurnLiveness(1_000); + const authority = new CodexAppReadyAuthority(); + const replay = new CodexAppFlushPromptReplay(); + tracker.begin('turn-1', 10_000); + const second = tracker.begin('turn-2', 10_010); + const third = tracker.begin('turn-3', 10_020); + + tracker.completeCurrent(10_100); + expect(tracker.notePrompt(10_110)).toBe(false); + replay.cancelSubmission(tracker, authority, second, 10_120); + expect(replay.consumeAfterFlush(tracker)).toBe(false); + + // A later flush can observe the same real prompt only after the remaining + // peer is cancelled; the one-shot gate cannot replay it twice. + expect(tracker.notePrompt(10_130)).toBe(false); + replay.cancelSubmission(tracker, authority, third, 10_140); + expect(replay.consumeAfterFlush(tracker)).toBe(true); + expect(replay.consumeAfterFlush(tracker)).toBe(false); + }); + + it('does not replay cancellation state after an authenticated next submit supersedes the prompt', () => { + const tracker = new CodexAppTurnLiveness(1_000); + const authority = new CodexAppReadyAuthority(); + const replay = new CodexAppFlushPromptReplay(); + tracker.begin('turn-1', 10_000); + const second = tracker.begin('turn-2', 10_010); + tracker.completeCurrent(10_100); + expect(tracker.notePrompt(10_110)).toBe(false); + tracker.noteSubmitted(10_120); + authority.beginWork(); + replay.cancelSubmission(tracker, authority, second, 10_130); + expect(replay.consumeAfterFlush(tracker)).toBe(false); + }); + + it('discards a deferred inter-turn prompt once the next turn really starts', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + tracker.begin('turn-2', 10_010); + tracker.completeCurrent(10_100); + expect(tracker.notePrompt(10_110)).toBe(false); + + tracker.noteSubmitted(10_120); + expect(tracker.completeCurrent(10_200)).toBe(false); + }); + + it('applies only valid activity from the authenticated socket and bounds future timestamps', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + tracker.begin('turn-2', 10_010); + + expect(applyTrustedCodexAppActivityMarker(tracker, { + phase: 'forged', + atMs: 999_999, + }, 10_100)).toEqual({ accepted: false }); + expect(tracker.poll(10_100)).toMatchObject({ turnId: 'turn-1', stalled: false }); + expect(tracker.poll(11_000)).toMatchObject({ turnId: 'turn-1', stalled: true }); + + expect(applyTrustedCodexAppActivityMarker(tracker, { + phase: 'completed', + atMs: 11_100, + }, 11_100)).toMatchObject({ accepted: true, phase: 'completed' }); + expect(tracker.poll(11_100)).toMatchObject({ turnId: 'turn-2', stalled: false }); + + // Even an authenticated runner timestamp is bounded by receipt time. + expect(applyTrustedCodexAppActivityMarker(tracker, { + phase: 'progress', + atMs: 999_999, + }, 11_200)).toMatchObject({ accepted: true, phase: 'progress' }); + expect(tracker.poll(12_200)).toMatchObject({ turnId: 'turn-2', stalled: true }); + }); + + it('uses a prompt only to close a synthetic authenticated reattach observation', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.beginReattachObservation(10_000); + expect(tracker.notePrompt(10_100)).toBe(true); + expect(tracker.poll(20_000).stalled).toBe(false); + + tracker.begin('turn-1', 20_000); + tracker.begin('turn-2', 20_010); + expect(tracker.notePrompt(20_100)).toBe(false); + expect(tracker.poll(21_000)).toMatchObject({ + stalled: true, + turnId: 'turn-1', + }); + }); + + it('does not install duplicate reattach observations over a tracked turn', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + expect(tracker.beginReattachObservation(10_100)).toBeUndefined(); + expect(tracker.poll(11_000)).toMatchObject({ + stalled: true, + turnId: 'turn-1', + }); + }); + + it('recovers an explicit active slot from a submitted marker after worker reattach', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.noteSubmitted(10_000); + + expect(tracker.notePrompt(10_100)).toBe(false); + expect(tracker.poll(11_000)).toMatchObject({ + active: true, + stalled: true, + shouldNotify: true, + }); + }); + + it('clears all state on CLI exit, kill, or worker reinitialization', () => { + const tracker = new CodexAppTurnLiveness(1_000); + tracker.begin('turn-1', 10_000); + tracker.begin('turn-2', 10_010); + tracker.clear(); + expect(tracker.poll(20_000)).toEqual({ + active: false, + stalled: false, + newlyStalled: false, + shouldNotify: false, + }); + }); + + it('rejects invalid timeout configuration', () => { + expect(() => new CodexAppTurnLiveness(0)).toThrow(/positive finite/); + expect(() => new CodexAppTurnLiveness(Number.NaN)).toThrow(/positive finite/); + }); +}); diff --git a/test/command-handler.test.ts b/test/command-handler.test.ts index 050e1c6e8..13ca95c63 100644 --- a/test/command-handler.test.ts +++ b/test/command-handler.test.ts @@ -285,6 +285,7 @@ vi.mock('../src/core/worker-pool.js', () => ({ // resolves as idempotent close so unrelated tests don't need to think // about it. closeSession: vi.fn(async () => ({ ok: true, alreadyClosed: false })), + withActiveSessionKeyLock: vi.fn(async (_map: Map, _key: string, action: () => any) => action()), // `isRelayableRealSession(ds)` — true when ds.worker is set OR persisted // CLI markers exist (session.cliId / session.lastCliInput). The default // makeSession fixture sets cliId='claude-code' so most tests pass the @@ -464,7 +465,7 @@ import { sessionKey } from '../src/core/types.js'; import { setTerminalProxyPort } from '../src/core/terminal-url.js'; import type { DaemonSession } from '../src/core/types.js'; import type { LarkMessage, Session } from '../src/types.js'; -import { killWorker, suspendWorker, forkWorker, getCurrentCliVersion, deliverEphemeralOrReply, deliverWritableTerminalCardTo } from '../src/core/worker-pool.js'; +import { killWorker, forkWorker, suspendWorker, getCurrentCliVersion, deliverEphemeralOrReply, deliverWritableTerminalCardTo, closeSession as closeWorkerPoolSession, withActiveSessionKeyLock } from '../src/core/worker-pool.js'; import { getOwnerOpenId } from '../src/bot-registry.js'; import { canOperate } from '../src/im/lark/event-dispatcher.js'; import { getSessionWorkingDir, buildNewTopicPrompt, buildNewTopicCliInput, ensureSessionWhiteboard, getAvailableBots } from '../src/core/session-manager.js'; @@ -1392,9 +1393,7 @@ describe('handleCommand', () => { await handleCommand('/close', ROOT_ID, makeLarkMessage('/close'), deps, LARK_APP_ID); - expect(killWorker).toHaveBeenCalledWith(ds); - expect(sessionStore.closeSession).toHaveBeenCalledWith('sess-001'); - expect(deps.activeSessions.has(sessionKey(ROOT_ID, LARK_APP_ID))).toBe(false); + expect(closeWorkerPoolSession).toHaveBeenCalledWith('sess-001'); // The「会话已关闭」card is delivered「仅自己可见」-first: it routes through // deliverEphemeralOrReply targeting the user who ran /close (message.senderId), // so plain groups get an ephemeral (visible-to-you) card and topic groups @@ -1477,7 +1476,7 @@ describe('handleCommand', () => { await handleCommand('/restart', ROOT_ID, makeLarkMessage('/restart'), deps, LARK_APP_ID); - expect(workerSend).toHaveBeenCalledWith({ type: 'restart' }); + expect(workerSend).toHaveBeenCalledWith({ type: 'restart', reason: 'operator' }); expect(deps.sessionReply).toHaveBeenCalledWith( ROOT_ID, expect.stringContaining('正在重启'), @@ -1706,6 +1705,24 @@ describe('handleCommand', () => { expect(replyContent).toContain('已自动创建并切换'); }); + it('should reject before path creation while Codex App dispatch ownership is non-empty', async () => { + vi.mocked(existsSync).mockReturnValue(false); + const ds = makeDaemonSession(); + ds.session.codexAppDispatchLedger = [ + { dispatchId: 'd-1', turnId: 't-1', state: 'accepted', content: 'owned' }, + ]; + const originalWorkingDir = ds.workingDir; + const deps = makeDeps(ds); + + await handleCommand('/cd', ROOT_ID, makeLarkMessage('/cd /brand-new/owned'), deps, LARK_APP_ID); + + expect(mkdirSync).not.toHaveBeenCalled(); + expect(killWorker).not.toHaveBeenCalled(); + expect(ds.workingDir).toBe(originalWorkingDir); + const replyContent = (deps.sessionReply as ReturnType).mock.calls[0][1] as string; + expect(replyContent).toContain('未结算'); + }); + it('should reject /cd when auto-create fails', async () => { vi.mocked(existsSync).mockReturnValue(false); vi.mocked(mkdirSync).mockImplementationOnce(() => { throw new Error('EACCES: permission denied'); }); @@ -1917,14 +1934,18 @@ describe('handleCommand', () => { expect(ds.workingDir).toBe('/home/testuser/project-b'); // ds.session is replaced by createSession result (pendingRepo is false → else branch) - expect(killWorker).toHaveBeenCalledWith(ds); - expect(sessionStore.closeSession).toHaveBeenCalled(); + expect(closeWorkerPoolSession).toHaveBeenCalledWith('sess-001'); expect(sessionStore.createSession).toHaveBeenCalledWith( CHAT_ID, ROOT_ID, 'project-b (dev)', 'group', undefined, ); expect(ds.session.sessionId).toBe('new-session-123'); expect(ds.hasHistory).toBe(false); expect(forkWorker).toHaveBeenCalledWith(ds, '', false); + expect(withActiveSessionKeyLock).toHaveBeenCalledWith( + deps.activeSessions, + sessionKey(ROOT_ID, LARK_APP_ID), + expect.any(Function), + ); }); it('mid-session chat switch preserves scope and the original message root', async () => { @@ -2266,25 +2287,29 @@ describe('handleCommand', () => { // ─── bare /repo while pending (replaces the old /skip command) ──────────── describe('/repo (bare) while pending', () => { - it('does not consume the pending launch while a card commit owns the shared claim', async () => { + it('retains the opening reservation and buffers when forkWorker fails before accept', async () => { const ds = makeDaemonSession({ pendingRepo: true, - pendingRepoCommitInFlight: true, - pendingPrompt: 'hello world', - pendingTurnId: 'om_pending_turn', + initialStartPending: true, + pendingPrompt: 'first prompt', + pendingFollowUps: ['buffered follow-up'], }); const deps = makeDeps(ds); + vi.mocked(forkWorker).mockImplementationOnce(() => { + expect(ds.pendingRepo).toBe(true); + expect(ds.initialStartPending).toBe(true); + expect(ds.pendingPrompt).toBe('first prompt'); + expect(ds.pendingFollowUps).toEqual(['buffered follow-up']); + throw new Error('fork preaccept failed'); + }); await handleCommand('/repo', ROOT_ID, makeLarkMessage('/repo'), deps, LARK_APP_ID); - expect(forkWorker).not.toHaveBeenCalled(); - expect(getAvailableBots).not.toHaveBeenCalled(); expect(ds.pendingRepo).toBe(true); - expect(ds.pendingRepoCommitInFlight).toBe(true); - expect(ds.pendingPrompt).toBe('hello world'); - expect(ds.pendingTurnId).toBe('om_pending_turn'); - const replies = vi.mocked(deps.sessionReply).mock.calls.map(c => c[1]).join(); - expect(replies).toContain('已有一个 worktree 正在创建'); + expect(ds.initialStartPending).toBe(true); + expect(ds.pendingPrompt).toBe('first prompt'); + expect(ds.pendingFollowUps).toEqual(['buffered follow-up']); + expect(deleteMessage).not.toHaveBeenCalled(); }); it('should boot the CLI idle (no prompt submitted) when launched via /repo itself', async () => { @@ -2300,7 +2325,7 @@ describe('handleCommand', () => { // No buffered message → spawn idle with an empty prompt so the user's NEXT // message becomes the first prompt (not an empty/boilerplate user_message). - expect(forkWorker).toHaveBeenCalledWith(ds, '', false); + expect(forkWorker).toHaveBeenCalledWith(ds, '', { turnId: 'om_repo_command_only' }); expect(buildNewTopicPrompt).not.toHaveBeenCalled(); expect(killWorker).not.toHaveBeenCalled(); expect(sessionStore.createSession).not.toHaveBeenCalled(); @@ -2439,7 +2464,7 @@ describe('handleCommand', () => { expect(forkWorker).toHaveBeenCalledWith(ds, { content: 'WRAPPED:clean', codexAppInput, - }); + }, false); expect(ds.pendingSubstituteTrigger).toBeUndefined(); }); diff --git a/test/crash-loop-diagnostic.test.ts b/test/crash-loop-diagnostic.test.ts index d79c97787..fff0d6ad1 100644 --- a/test/crash-loop-diagnostic.test.ts +++ b/test/crash-loop-diagnostic.test.ts @@ -180,7 +180,7 @@ describe("crash-loop diagnostic terminal (daemon 'claude_exit' handler)", () => // First 3 auto-restart in place; the 4th asks the worker to park a // diagnostic shell (deferred park) and keeps it alive (no close). - expect(worker.send).toHaveBeenCalledWith({ type: 'restart' }); + expect(worker.send).toHaveBeenCalledWith({ type: 'restart', reason: 'cli_crash' }); expect(worker.send).toHaveBeenCalledWith({ type: 'park_diagnostic' }); expect(worker.send).not.toHaveBeenCalledWith({ type: 'close' }); // Survives daemon restart: lazy cold-resume + idle, restart counter reset. @@ -216,4 +216,5 @@ describe("crash-loop diagnostic terminal (daemon 'claude_exit' handler)", () => expect(worker.send).toHaveBeenCalledWith({ type: 'close' }); expect(ds.session.suspendedColdResume).toBeFalsy(); }); + }); diff --git a/test/daemon-codex-app-workflow-wiring.test.ts b/test/daemon-codex-app-workflow-wiring.test.ts index 7d374fc3a..0824d0cb7 100644 --- a/test/daemon-codex-app-workflow-wiring.test.ts +++ b/test/daemon-codex-app-workflow-wiring.test.ts @@ -24,7 +24,8 @@ describe('daemon Codex App workflow prompt lanes', () => { expect(block).toContain("const codexAppMessageContext = codexAppQuoteContext + (workflowGrillPrompt ?? '');"); expect(block).toContain('const promptContent = codexAppQuoteContext + codexAppApplicationContext + content;'); expect(block).toContain('pendingCodexAppText: codexAppVisibleText'); - expect(block.match(/codexAppText: codexAppVisibleText/g)).toHaveLength(2); + expect(source).toContain('codexAppText: ds.pendingCodexAppText'); + expect(block.match(/forkReservedInitialSession\(ds, availableBots\)/g)).toHaveLength(2); }); it('retains VC lifecycle context in rewritten legacy prompts without demoting it to untrusted', () => { @@ -46,4 +47,13 @@ describe('daemon Codex App workflow prompt lanes', () => { 'const codexAppApplicationContext = initialCodexAppApplicationContext;', ); }); + + it('does not buffer ordinary turns solely because repo commit UI cleanup is still in flight', () => { + const block = region( + 'async function handleThreadReply', + 'async function autoCreateDocSession', + ); + expect(block).toContain('if (ds?.pendingRepo || initialStartPending) {'); + expect(block).not.toContain('if (ds?.pendingRepo || ds?.pendingRepoCommitInFlight || initialStartPending)'); + }); }); diff --git a/test/daemon-refork-substitute-wiring.test.ts b/test/daemon-refork-substitute-wiring.test.ts index f02424866..d5ef63e66 100644 --- a/test/daemon-refork-substitute-wiring.test.ts +++ b/test/daemon-refork-substitute-wiring.test.ts @@ -13,13 +13,15 @@ describe('daemon stopped-worker substitute refork wiring', () => { expect(forkStart).toBeGreaterThan(start); expect(end).toBeGreaterThan(start); const reforkBlock = source.slice(start, end); - expect(reforkBlock).toMatch(/\n\s+substituteTrigger,\n/); + expect(reforkBlock).toContain( + 'substituteTrigger: queuedHasDurableTail ? undefined : substituteTrigger', + ); expect(reforkBlock).toContain('substituteTrigger ? SUBSTITUTE_RECEIVED_REACTION_EMOJI_TYPE : undefined'); expect(reforkBlock).toContain('applyQueuedCodexAppLegacyFallback(builtReforkInput'); expect(reforkBlock).toContain('Legacy queued dashboard task has no clean-input text'); expect(reforkBlock).toContain('wrappedInput !== builtReforkInput && dsBotCfgForFork.codexAppCleanInput === true'); expect(reforkBlock.indexOf('applyQueuedCodexAppLegacyFallback(builtReforkInput')) - .toBeLessThan(reforkBlock.indexOf('rememberLastCliInput(ds, promptContent, wrappedInput)')); + .toBeLessThan(reforkBlock.indexOf('rememberLastCliInput(ds, queuedHasDurableTail')); }); it('clears stale multi-Riff repo state when doc-watch replaces a stopped session cwd', () => { diff --git a/test/daemon-rename-route.test.ts b/test/daemon-rename-route.test.ts index 7b1897d7c..c30a6ebf3 100644 --- a/test/daemon-rename-route.test.ts +++ b/test/daemon-rename-route.test.ts @@ -33,6 +33,7 @@ const mocks = vi.hoisted(() => { delete process.env.BOTMUX_SESSION_ID; delete process.env.BOTMUX_LARK_APP_ID; let seq = 0; + const sessions = new Map(); return { replyMessage: vi.fn(async () => 'om_reply'), sendMessage: vi.fn(async () => 'om_top'), @@ -43,17 +44,32 @@ const mocks = vi.hoisted(() => { ? { openId, type: senderType === 'app' || senderType === 'bot' ? 'bot' as const : 'user' as const } : undefined )), - forkWorker: vi.fn(), - createSession: vi.fn((chatId: string, rootMessageId: string, title: string, chatType?: 'group' | 'p2p') => ({ - sessionId: `sess-fake-${++seq}`, - chatId, - rootMessageId, - title, - status: 'active' as const, - createdAt: new Date().toISOString(), - chatType, - })), - updateSession: vi.fn(), + sessions, + createSession: vi.fn((chatId: string, rootMessageId: string, title: string, chatType?: 'group' | 'p2p') => { + const session = { + sessionId: `sess-fake-${++seq}`, + chatId, + rootMessageId, + title, + status: 'active' as const, + createdAt: new Date().toISOString(), + chatType, + }; + sessions.set(session.sessionId, session); + return session; + }), + updateSession: vi.fn((session: any) => { sessions.set(session.sessionId, session); }), + getSession: vi.fn((sessionId: string) => sessions.get(sessionId)), + closeSession: vi.fn((sessionId: string) => { + const session = sessions.get(sessionId); + if (session) session.status = 'closed'; + }), + forkWorker: vi.fn((ds: any) => { + ds.worker = { killed: false, send: vi.fn() }; + }), + scanMultipleProjects: vi.fn(() => [] as any[]), + getAvailableBots: vi.fn(async () => [] as any[]), + downloadResources: vi.fn(async () => ({ attachments: [], needLogin: false })), }; }); @@ -75,7 +91,32 @@ vi.mock('../src/im/lark/client.js', async () => { vi.mock('../src/services/session-store.js', async () => { const actual = await vi.importActual('../src/services/session-store.js'); - return { ...actual, createSession: mocks.createSession, updateSession: mocks.updateSession }; + return { + ...actual, + createSession: mocks.createSession, + updateSession: mocks.updateSession, + getSession: mocks.getSession, + closeSession: mocks.closeSession, + }; +}); + +vi.mock('../src/core/worker-pool.js', async () => { + const actual = await vi.importActual('../src/core/worker-pool.js'); + return { ...actual, forkWorker: mocks.forkWorker }; +}); + +vi.mock('../src/core/session-manager.js', async () => { + const actual = await vi.importActual('../src/core/session-manager.js'); + return { + ...actual, + getAvailableBots: mocks.getAvailableBots, + downloadResources: mocks.downloadResources, + }; +}); + +vi.mock('../src/services/project-scanner.js', async () => { + const actual = await vi.importActual('../src/services/project-scanner.js'); + return { ...actual, scanMultipleProjects: mocks.scanMultipleProjects }; }); vi.mock('../src/im/lark/identity-cache.js', async () => { @@ -83,19 +124,26 @@ vi.mock('../src/im/lark/identity-cache.js', async () => { return { ...actual, resolveSender: (...args: any[]) => mocks.resolveSender(...args) }; }); -vi.mock('../src/core/worker-pool.js', async () => { - const actual = await vi.importActual('../src/core/worker-pool.js'); - return { ...actual, forkWorker: (...args: any[]) => mocks.forkWorker(...args) }; -}); - import { registerBot } from '../src/bot-registry.js'; -import { sessionKey } from '../src/core/types.js'; +import { sessionAnchorId, sessionKey } from '../src/core/types.js'; import { __testOnly_activeSessions as activeSessions, + __testOnly_claimNewDaemonSession as claimNewDaemonSession, + __testOnly_handleChatModeConverted as handleChatModeConverted, + __testOnly_handleDocComment as handleDocComment, __testOnly_handleNewTopic as handleNewTopic, __testOnly_handleThreadReply as handleThreadReply, + __testOnly_onQueuedActivationSubmitted as onQueuedActivationSubmitted, + __testOnly_prewarmDocCommentSession as prewarmDocCommentSession, + __testOnly_releaseQueuedActivationReservation as releaseQueuedActivationReservation, + __testOnly_reserveAsyncQueuedActivationTailAdmission as reserveAsyncQueuedActivationTailAdmission, + __testOnly_resetDocCommentClaims as resetDocCommentClaims, + __testOnly_settleAsyncQueuedActivationTailAdmission as settleAsyncQueuedActivationTailAdmission, } from '../src/daemon.js'; +import { admitQueuedActivationTail } from '../src/core/worker-pool.js'; import type { DaemonSession } from '../src/core/types.js'; +import { getDocSubscription, putDocSubscription, removeDocSubscription } from '../src/services/doc-subs-store.js'; +import { config } from '../src/config.js'; const APP = 'rename_route_app'; const CHAT = 'oc_rename_route_chat'; @@ -165,56 +213,47 @@ function seedThreadSession(anchor: string, title: string): DaemonSession { return ds; } -function seedLiveChatSession(send = vi.fn()): DaemonSession { - const ds = { - scope: 'chat', - chatId: CHAT, - chatType: 'group', +function makeChatSession(sessionId: string, chatId: string, options?: { + pendingLedger?: boolean; + pendingRepo?: boolean; + queued?: boolean; +}): DaemonSession { + const session = { + sessionId, + chatId, + rootMessageId: `om_${sessionId}`, + title: sessionId, + status: 'active' as const, + createdAt: NOW, larkAppId: APP, - worker: { killed: false, send }, + scope: 'chat' as const, + chatType: 'group' as const, + queued: options?.queued, + codexAppDispatchLedger: options?.pendingLedger ? [{ + dispatchId: `dispatch-${sessionId}`, + turnId: `turn-${sessionId}`, + dispatchAttempt: 1, + state: 'prepared' as const, + content: 'durable work', + deliverySink: 'lark' as const, + }] : undefined, + }; + mocks.sessions.set(sessionId, session); + return { + session, + worker: null, workerPort: null, workerToken: null, + larkAppId: APP, + chatId, + chatType: 'group', + scope: 'chat', spawnedAt: Date.now(), cliVersion: '1.0.0', lastMessageAt: Date.now(), hasHistory: false, - ownerOpenId: OWNER, - currentReplyTarget: { - rootMessageId: 'om_stale_root', - turnId: 'om_stale_turn', - updatedAt: NOW, - }, - session: { - sessionId: 'sess-live-chat-' + Math.random().toString(36).slice(2), - chatId: CHAT, - rootMessageId: 'om_original_root', - title: 'live chat', - status: 'active', - createdAt: NOW, - larkAppId: APP, - scope: 'chat', - quoteTargetId: 'om_stale_quote', - quoteTargetSenderOpenId: 'ou_stale_caller', - lastCallerOpenId: 'ou_stale_caller', - currentReplyTarget: { - rootMessageId: 'om_stale_root', - turnId: 'om_stale_turn', - updatedAt: NOW, - }, - }, - } as unknown as DaemonSession; - activeSessions.set(sessionKey(CHAT, APP), ds); - return ds; -} - -function seedPendingRawSession(anchor: string): DaemonSession { - const ds = seedThreadSession(anchor, 'pending raw'); - ds.pendingRepo = true; - ds.pendingPrompt = ''; - ds.pendingRawInput = '/goal start'; - ds.pendingRawTurnId = 'om_initial_raw'; - ds.pendingSender = { openId: OWNER, type: 'user' }; - return ds; + pendingRepo: options?.pendingRepo, + } as DaemonSession; } /** All text replied through the mocked Lark client in this test, joined. */ @@ -231,13 +270,20 @@ describe('/rename production routing — must not pre-create a session (review P mocks.sendMessage.mockResolvedValue('om_top'); mocks.getChatMode.mockResolvedValue('group'); mocks.getChatNameAndMode.mockResolvedValue({ name: null, mode: 'group' }); + mocks.sessions.clear(); + mocks.forkWorker.mockImplementation((ds: any) => { + ds.worker = { killed: false, send: vi.fn() }; + }); + mocks.scanMultipleProjects.mockReturnValue([]); + mocks.getAvailableBots.mockResolvedValue([]); + mocks.downloadResources.mockResolvedValue({ attachments: [], needLogin: false }); activeSessions.clear(); + resetDocCommentClaims(); const bot = registerBot({ larkAppId: APP, larkAppSecret: 's', cliId: 'claude-code', allowedUsers: [OWNER], - oncallChats: [{ chatId: CHAT, workingDir: '/tmp' }], }); bot.resolvedAllowedUsers = [OWNER]; }); @@ -315,6 +361,28 @@ describe('/rename production routing — must not pre-create a session (review P expect(activeSessions.has(sessionKey('om_new_2', APP))).toBe(true); }); + it('routes a colliding daemon command to the canonical pending owner and closes only the loser', async () => { + const anchor = 'om_pending_owner'; + const incumbent = seedThreadSession(anchor, 'durable owner'); + incumbent.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-incumbent', + turnId: 'turn-incumbent', + dispatchAttempt: 1, + state: 'prepared', + content: 'accepted input', + deliverySink: 'lark', + }]; + + await handleNewTopic(makeEventData(anchor, '/status'), makeCtx(anchor, anchor)); + + expect(activeSessions.get(sessionKey(anchor, APP))).toBe(incumbent); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + const incomingId = mocks.createSession.mock.results[0]!.value.sessionId; + expect(mocks.closeSession).toHaveBeenCalledWith(incomingId); + expect(mocks.closeSession).not.toHaveBeenCalledWith(incumbent.session.sessionId); + expect(repliedText()).toContain(`Session: ${incumbent.session.sessionId}`); + }); + it('new topic: passes the accepted Lark message id into the first worker', async () => { await handleNewTopic( makeEventData('om_workflow_new', '/workflow new 修复首轮授权'), @@ -360,85 +428,954 @@ describe('/rename production routing — must not pre-create a session (review P expect(mocks.getChatNameAndMode).toHaveBeenCalledTimes(2); }); - it('thread safety-net: passes the accepted reply id into the first worker', async () => { + it('delivers an ordinary loser turn to the canonical owner exactly once', async () => { + const anchor = 'om_collision_delivery'; + const incumbent = seedThreadSession(anchor, 'canonical owner'); + const send = vi.fn(); + incumbent.worker = { killed: false, send } as any; + + await handleNewTopic( + makeEventData(anchor, 'deliver this once'), + makeCtx(anchor, anchor), + ); + + expect(activeSessions.get(sessionKey(anchor, APP))).toBe(incumbent); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + expect(mocks.closeSession).toHaveBeenCalledWith( + mocks.createSession.mock.results[0]!.value.sessionId, + ); + const inputCalls = send.mock.calls.filter(call => call[0]?.type === 'message'); + expect(inputCalls).toHaveLength(1); + expect(JSON.stringify(inputCalls[0]![0])).toContain('deliver this once'); + }); + + it('buffers a later turn while the winning initial start is paused, then forks once in input order', async () => { + registerBot({ + larkAppId: APP, + larkAppSecret: 's', + cliId: 'claude-code', + allowedUsers: [OWNER], + defaultWorkingDir: '/tmp', + }).resolvedAllowedUsers = [OWNER]; + + let announcePreparation!: () => void; + const preparationStarted = new Promise(resolve => { announcePreparation = resolve; }); + let releasePreparation!: (bots: any[]) => void; + const preparationGate = new Promise(resolve => { releasePreparation = resolve; }); + mocks.getAvailableBots.mockImplementationOnce(async () => { + announcePreparation(); + return preparationGate; + }); + + const anchor = 'om_initial_order_root'; + const first = handleNewTopic( + makeEventData(anchor, 'first task'), + makeCtx(anchor, anchor), + ); + await preparationStarted; + + const owner = activeSessions.get(sessionKey(anchor, APP))!; + expect(owner.initialStartPending).toBe(true); + expect(owner.worker).toBeNull(); + await handleThreadReply( - makeEventData('om_workflow_reply', '/workflow new 修复首轮授权', 'om_fresh_root'), - makeCtx('om_fresh_root', 'om_workflow_reply'), + makeEventData('om_initial_order_second', 'second task', anchor), + makeCtx(anchor, 'om_initial_order_second'), ); + expect(mocks.forkWorker).not.toHaveBeenCalled(); + expect(owner.pendingFollowUps).toBeUndefined(); + expect(owner.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'om_initial_order_second', + cliInput: expect.objectContaining({ + content: expect.stringContaining('second task'), + }), + }), + ]); + + releasePreparation([]); + await first; + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); - expect(mocks.forkWorker.mock.calls[0]?.[2]).toEqual({ turnId: 'om_workflow_reply' }); + const openingInput = mocks.forkWorker.mock.calls[0]![1]; + expect(openingInput.content.indexOf('first task')).toBeGreaterThanOrEqual(0); + expect(openingInput.content).not.toContain('second task'); + expect(owner.worker!.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'om_initial_order_second', + content: expect.stringContaining('second task'), + queuedActivationToken: expect.any(String), + })); + expect(owner.initialStartPending).toBe(true); + expect(owner.pendingFollowUps).toBeUndefined(); + }); + + it('keeps the no-project text fallback gated until its queued activation ACK', async () => { + const anchor = 'om_no_project_text_cutpoint'; + const openingToken = 'token-no-project-text'; + const send = vi.fn(); + mocks.forkWorker.mockImplementationOnce((owner: any, input: any) => { + expect(owner.session.queued).toBe(true); + expect(input.content).toContain('OPENING_TEXT_N'); + Object.assign(owner.session, { + queued: false, + queuedActivationPending: true, + queuedActivationToken: openingToken, + queuedActivationInput: input, + queuedActivationTurnId: anchor, + queuedActivationResume: false, + }); + owner.worker = { killed: false, send }; + }); + + await handleNewTopic( + makeEventData(anchor, 'OPENING_TEXT_N'), + makeCtx(anchor, anchor), + ); + + const owner = activeSessions.get(sessionKey(anchor, APP))!; + expect(mocks.scanMultipleProjects).toHaveBeenCalled(); + expect(owner.pendingRepo).toBe(false); + expect(owner.initialStartPending).toBe(true); + expect(owner.session.queuedActivationToken).toBe(openingToken); + + await handleThreadReply( + makeEventData('om_no_project_text_n1', 'FOLLOWER_TEXT_N_PLUS_1', anchor), + makeCtx(anchor, 'om_no_project_text_n1'), + ); + + expect(send).not.toHaveBeenCalled(); + expect(owner.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'om_no_project_text_n1', + cliInput: expect.objectContaining({ + content: expect.stringContaining('FOLLOWER_TEXT_N_PLUS_1'), + }), + }), + ]); + + Object.assign(owner.session, { + queuedActivationPending: undefined, + queuedActivationToken: undefined, + queuedActivationInput: undefined, + queuedActivationTurnId: undefined, + queuedActivationResume: undefined, + pendingRepoSetup: undefined, + }); + expect(onQueuedActivationSubmitted(owner, openingToken)).toBe(true); + expect(send).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'om_no_project_text_n1', + content: expect.stringContaining('FOLLOWER_TEXT_N_PLUS_1'), + queuedActivationToken: expect.any(String), + })); }); - it('live passthrough binds raw input and reply metadata to the accepted message', async () => { + it('keeps the no-project raw fallback gated through raw text-to-Enter ACK', async () => { + const anchor = 'om_no_project_raw_cutpoint'; + const openingToken = 'token-no-project-raw'; + const rawOpening = '/goal OPENING_RAW_N'; const send = vi.fn(); - const ds = seedLiveChatSession(send); - const messageId = 'om_model_turn'; - const replyRootId = 'om_model_reply_root'; + mocks.forkWorker.mockImplementationOnce((owner: any, input: any) => { + expect(owner.session.queued).toBe(true); + expect(input).toBe(''); + expect(owner.pendingRawInput).toBe(rawOpening); + Object.assign(owner.session, { + queued: false, + queuedActivationPending: true, + queuedActivationToken: openingToken, + queuedActivationInput: { content: '' }, + queuedActivationTurnId: anchor, + queuedActivationResume: false, + }); + owner.worker = { killed: false, send }; + }); + + await handleNewTopic( + makeEventData(anchor, rawOpening), + makeCtx(anchor, anchor), + ); + + const owner = activeSessions.get(sessionKey(anchor, APP))!; + expect(mocks.scanMultipleProjects).toHaveBeenCalled(); + expect(owner.pendingRepo).toBe(false); + expect(owner.initialStartPending).toBe(true); + expect(owner.session.queuedActivationToken).toBe(openingToken); await handleThreadReply( - makeEventData(messageId, '/model opus', replyRootId), - { - chatId: CHAT, - messageId, - chatType: 'group' as const, - scope: 'chat' as const, - anchor: CHAT, - replyRootId, + makeEventData('om_no_project_raw_n1', 'FOLLOWER_AFTER_RAW_N_PLUS_1', anchor), + makeCtx(anchor, 'om_no_project_raw_n1'), + ); + + // The follower must stay durable until the adapter confirms that both the + // raw command text and its Enter beat were submitted. + expect(send).not.toHaveBeenCalled(); + expect(owner.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'om_no_project_raw_n1', + cliInput: expect.objectContaining({ + content: expect.stringContaining('FOLLOWER_AFTER_RAW_N_PLUS_1'), + }), + }), + ]); + + Object.assign(owner.session, { + queuedActivationPending: undefined, + queuedActivationToken: undefined, + queuedActivationInput: undefined, + queuedActivationTurnId: undefined, + queuedActivationResume: undefined, + pendingRepoSetup: undefined, + }); + expect(onQueuedActivationSubmitted(owner, openingToken)).toBe(true); + expect(send).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'om_no_project_raw_n1', + content: expect.stringContaining('FOLLOWER_AFTER_RAW_N_PLUS_1'), + queuedActivationToken: expect.any(String), + })); + }); + + it('refuses an existing raw CLI passthrough while activation admission owns the route', async () => { + const anchor = 'om_passthrough_activation'; + const owner = seedThreadSession(anchor, 'activation owner'); + const send = vi.fn(); + owner.worker = { killed: false, send } as any; + Object.assign(owner.session, { + queuedActivationPending: true, + queuedActivationToken: 'passthrough-token', + queuedActivationInput: { content: 'OPENING_N' }, + queuedActivationTurnId: 'turn-opening', + }); + + await handleThreadReply( + makeEventData('om_passthrough_n1', '/model opus', anchor), + makeCtx(anchor, 'om_passthrough_n1'), + ); + + expect(send).not.toHaveBeenCalled(); + expect(owner.currentTurnTitle).toBeUndefined(); + expect(owner.session.queuedActivationToken).toBe('passthrough-token'); + expect(repliedText()).toContain('仍在提交中'); + }); + + it('replays a retained queued activation exactly, then releases the later inbound with its own turn id', async () => { + const anchor = 'om_reparked_activation_root'; + const ds = seedThreadSession(anchor, 're-parked activation'); + const exactOpening = { + content: 'BACKLOG_N\n\nPRIOR_TRIGGER_REPLY', + }; + Object.assign(ds.session, { + cliId: 'claude-code', + workingDir: '/tmp', + queued: true, + queuedPrompt: 'STALE_REBUILD_SOURCE_MUST_NOT_BE_USED', + queuedActivationInput: exactOpening, + queuedActivationTurnId: 'om_prior_trigger', + queuedActivationDispatchAttempt: 4, + }); + ds.workingDir = '/tmp'; + + await handleThreadReply( + makeEventData('om_later_n_plus_1', 'LATER_REPLY_N_PLUS_1', anchor), + makeCtx(anchor, 'om_later_n_plus_1'), + ); + + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); + expect(mocks.forkWorker).toHaveBeenCalledWith(ds, exactOpening, { + resume: false, + turnId: 'om_prior_trigger', + dispatchAttempt: 4, + }); + expect(mocks.forkWorker.mock.calls[0]![1]).toBe(exactOpening); + expect(JSON.stringify(mocks.forkWorker.mock.calls[0]![1])) + .not.toContain('STALE_REBUILD_SOURCE_MUST_NOT_BE_USED'); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'om_later_n_plus_1', + userPrompt: expect.stringContaining('LATER_REPLY_N_PLUS_1'), + cliInput: expect.objectContaining({ + content: expect.stringContaining('LATER_REPLY_N_PLUS_1'), + }), + }), + ]); + + const send = vi.mocked(ds.worker!.send); + expect(releaseQueuedActivationReservation(ds)).toBe(true); + expect(send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'om_later_n_plus_1', + content: expect.stringContaining('LATER_REPLY_N_PLUS_1'), + queuedActivationToken: expect.any(String), + })); + expect(send).toHaveBeenCalledTimes(1); + expect(mocks.forkWorker.mock.invocationCallOrder[0]) + .toBeLessThan(send.mock.invocationCallOrder[0]!); + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(ds.session.queuedActivationPending).toBe(true); + expect(ds.session.queuedActivationInput?.content).toContain('LATER_REPLY_N_PLUS_1'); + const successorToken = ds.session.queuedActivationToken!; + // The worker-pool clears the tokened journal durably before invoking the + // daemon callback. Model that adapter ACK boundary explicitly here. + Object.assign(ds.session, { + queuedActivationPending: undefined, + queuedActivationToken: undefined, + queuedActivationInput: undefined, + queuedActivationTurnId: undefined, + queuedActivationDispatchAttempt: undefined, + queuedActivationResume: undefined, + }); + expect(onQueuedActivationSubmitted(ds, successorToken)).toBe(true); + expect(ds.initialStartPending).toBe(false); + }); + + it('atomically claims a fresh queued refork so a concurrent reply buffers behind its owner', async () => { + const anchor = 'om_fresh_queued_claim_root'; + const ds = seedThreadSession(anchor, 'fresh queued claim'); + Object.assign(ds.session, { + cliId: 'claude-code', + workingDir: '/tmp', + queued: true, + queuedPrompt: 'BACKLOG_N', + }); + ds.workingDir = '/tmp'; + + let announceFirstDownload!: () => void; + const firstDownloadStarted = new Promise(resolve => { announceFirstDownload = resolve; }); + let releaseFirstDownload!: () => void; + const firstDownloadGate = new Promise(resolve => { releaseFirstDownload = resolve; }); + mocks.downloadResources.mockImplementationOnce(async () => { + announceFirstDownload(); + await firstDownloadGate; + return { attachments: [], needLogin: false }; + }); + + const first = handleThreadReply( + makeEventData('om_fresh_owner_n', 'OWNER_REPLY_N', anchor), + makeCtx(anchor, 'om_fresh_owner_n'), + ); + await firstDownloadStarted; + + expect(ds.initialStartPending).toBe(true); + expect(ds.initialStartClaimToken).toEqual(expect.any(String)); + const ownerToken = ds.initialStartClaimToken; + + const follower = handleThreadReply( + makeEventData('om_fresh_follower_n1', 'FOLLOWER_REPLY_N_PLUS_1', anchor), + makeCtx(anchor, 'om_fresh_follower_n1'), + ); + + expect(mocks.forkWorker).not.toHaveBeenCalled(); + expect(ds.initialStartClaimToken).toBe(ownerToken); + + releaseFirstDownload(); + await first; + await follower; + + expect(ds.pendingFollowUps).toBeUndefined(); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'om_fresh_follower_n1', + cliInput: expect.objectContaining({ + content: expect.stringContaining('FOLLOWER_REPLY_N_PLUS_1'), + }), + }), + ]); + + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); + const opening = mocks.forkWorker.mock.calls[0]![1]; + expect(opening.content.indexOf('BACKLOG_N')).toBeGreaterThanOrEqual(0); + expect(opening.content.indexOf('OWNER_REPLY_N')).toBeGreaterThan(opening.content.indexOf('BACKLOG_N')); + expect(opening.content).not.toContain('FOLLOWER_REPLY_N_PLUS_1'); + expect(ds.initialStartPending).toBe(true); + expect(ds.initialStartClaimToken).toBe(ownerToken); + + const send = vi.mocked(ds.worker!.send); + expect(onQueuedActivationSubmitted(ds)).toBe(true); + expect(send).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'om_fresh_follower_n1', + content: expect.stringContaining('FOLLOWER_REPLY_N_PLUS_1'), + queuedActivationToken: expect.any(String), + })); + expect(ds.session.queuedActivationPending).toBe(true); + expect(ds.session.queuedActivationInput?.content).toContain('FOLLOWER_REPLY_N_PLUS_1'); + const successorToken = ds.session.queuedActivationToken!; + Object.assign(ds.session, { + queuedActivationPending: undefined, + queuedActivationToken: undefined, + queuedActivationInput: undefined, + queuedActivationTurnId: undefined, + queuedActivationDispatchAttempt: undefined, + queuedActivationResume: undefined, + }); + expect(onQueuedActivationSubmitted(ds, successorToken)).toBe(true); + expect(ds.initialStartPending).toBe(false); + expect(ds.initialStartClaimToken).toBeUndefined(); + }); + + it.each([ + { arrivalGate: true, laterGate: false, expectsSidecar: true, label: 'ON→OFF' }, + { arrivalGate: false, laterGate: true, expectsSidecar: false, label: 'OFF→ON' }, + ])( + 'freezes a queued follower clean-input decision at reservation time ($label)', + ({ arrivalGate, laterGate, expectsSidecar }) => { + const bot = registerBot({ larkAppId: APP, - }, + larkAppSecret: 's', + cliId: 'codex-app', + codexAppCleanInput: arrivalGate, + allowedUsers: [OWNER], + }); + bot.resolvedAllowedUsers = [OWNER]; + const ds = seedThreadSession(`om_gate_${arrivalGate}`, 'clean-input reservation'); + const send = vi.fn(); + ds.worker = { killed: false, send } as any; + ds.initialStartPending = true; + ds.hasHistory = true; + ds.session.cliId = 'codex-app'; + + const reservation = reserveAsyncQueuedActivationTailAdmission(ds); + expect(ds.queuedActivationTailAdmissionsOutstanding).toBe(1); + bot.config.codexAppCleanInput = laterGate; + + // Model N's ACK landing while N+1 is still awaiting prompt materialization. + expect(releaseQueuedActivationReservation(ds, 'opening-token')).toBe(false); + const sidecar = { + text: 'FOLLOWER_CLEAN_N1', + additionalContext: { + hidden: { kind: 'application' as const, value: 'arrival' }, + }, + }; + admitQueuedActivationTail(ds, { + userPrompt: 'FOLLOWER_CLEAN_N1', + cliInput: { + content: 'FOLLOWER_LEGACY_N1', + codexAppInput: sidecar, + }, + turnId: 'turn-clean-follower', + dispatchAttempt: 2, + }, reservation); + settleAsyncQueuedActivationTailAdmission(ds); + + const expectedSidecar = expectsSidecar + ? { ...sidecar, clientUserMessageId: 'turn-clean-follower' } + : undefined; + expect(ds.queuedActivationTailAdmissionsOutstanding).toBeUndefined(); + expect(ds.queuedActivationTailReleasePending).toBeUndefined(); + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(ds.session.queuedActivationInput?.codexAppInput).toEqual(expectedSidecar); + expect(ds.session.codexAppDispatchLedger?.at(-1)?.codexAppInput) + .toEqual(expectedSidecar); + expect(ds.session.lastCodexAppInput).toEqual(expectedSidecar); + expect(send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'turn-clean-follower', + ...(expectedSidecar ? { codexAppInput: expectedSidecar } : {}), + })); + if (!expectedSidecar) { + expect(send.mock.calls[0]![0]).not.toHaveProperty('codexAppInput'); + } + }, + ); + + it('parks a late follower after ACK→worker-exit and reforks it before the next inbound', async () => { + const anchor = 'om_ack_exit_late_admission'; + const ds = seedThreadSession(anchor, 'ACK exit late admission'); + ds.session.cliId = 'claude-code'; + ds.workingDir = '/tmp'; + ds.session.workingDir = '/tmp'; + ds.hasHistory = true; + ds.initialStartPending = true; + ds.worker = { killed: false, send: vi.fn() } as any; + + const reservation = reserveAsyncQueuedActivationTailAdmission(ds); + expect(releaseQueuedActivationReservation(ds, 'opening-token')).toBe(false); + // N's worker exits after its ACK but before the reserved N+1 finishes. + ds.worker = null; + admitQueuedActivationTail(ds, { + userPrompt: 'LATE_N_PLUS_1', + cliInput: { content: 'LATE_N_PLUS_1' }, + turnId: 'turn-late-n1', + }, reservation); + settleAsyncQueuedActivationTailAdmission(ds); + + expect(ds.session).toMatchObject({ + queuedActivationPending: true, + queuedActivationInput: { content: 'LATE_N_PLUS_1' }, + queuedActivationTurnId: 'turn-late-n1', + }); + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(ds.initialStartPending).toBe(false); + expect(ds.initialStartClaimToken).toBeUndefined(); + expect(ds.queuedActivationTailReleaseRetryTimer).toBeUndefined(); + + mocks.forkWorker.mockClear(); + await handleThreadReply( + makeEventData('turn-after-late-n2', 'AFTER_LATE_N_PLUS_2', anchor), + makeCtx(anchor, 'turn-after-late-n2'), ); - expect(send).toHaveBeenCalledWith({ - type: 'raw_input', - content: '/model opus', - turnId: messageId, + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); + expect(mocks.forkWorker.mock.calls[0]![1]).toBe(ds.session.queuedActivationInput); + expect(mocks.forkWorker.mock.calls[0]![1].content).toBe('LATE_N_PLUS_1'); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'turn-after-late-n2', + cliInput: expect.objectContaining({ + content: expect.stringContaining('AFTER_LATE_N_PLUS_2'), + }), + }), + ]); + }); + + it('releases the route when a post-ACK async follower admission fails', () => { + const ds = seedThreadSession('om_failed_late_admission', 'failed late admission'); + ds.session.cliId = 'claude-code'; + ds.initialStartPending = true; + ds.worker = { killed: false, send: vi.fn() } as any; + const reservation = reserveAsyncQueuedActivationTailAdmission(ds); + expect(releaseQueuedActivationReservation(ds, 'opening-token')).toBe(false); + mocks.updateSession.mockImplementationOnce(() => { + throw new Error('tail persistence unavailable'); }); - expect(ds.session.quoteTargetId).toBe(messageId); - expect(ds.session.quoteTargetSenderOpenId).toBe(OWNER); - expect(ds.session.lastCallerOpenId).toBe(OWNER); - expect(ds.currentReplyTarget).toMatchObject({ rootMessageId: replyRootId, turnId: messageId }); - expect(ds.session.currentReplyTarget).toMatchObject({ rootMessageId: replyRootId, turnId: messageId }); - expect(mocks.updateSession).toHaveBeenCalledWith(ds.session); + + expect(() => { + try { + admitQueuedActivationTail(ds, { + userPrompt: 'FAILED_N_PLUS_1', + cliInput: { content: 'FAILED_N_PLUS_1' }, + turnId: 'turn-failed-n1', + }, reservation); + } finally { + settleAsyncQueuedActivationTailAdmission(ds); + } + }).toThrow('tail persistence unavailable'); + + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(ds.queuedActivationTailAdmissionsOutstanding).toBeUndefined(); + expect(ds.queuedActivationTailReleasePending).toBeUndefined(); + expect(ds.initialStartPending).toBe(false); + expect(ds.initialStartClaimToken).toBeUndefined(); + expect(ds.queuedActivationTailReleaseRetryTimer).toBeUndefined(); }); - it('pending raw same-caller follow-up rotates both staged turns to the latest message', async () => { - const anchor = 'om_pending_raw_root'; - const ds = seedPendingRawSession(anchor); - const messageId = 'om_pending_raw_followup'; + it('releases a failed queued-refork claim so a later inbound can become the owner', async () => { + const anchor = 'om_failed_queued_claim_root'; + const ds = seedThreadSession(anchor, 'failed queued claim'); + Object.assign(ds.session, { + cliId: 'claude-code', + workingDir: '/tmp', + queued: true, + queuedPrompt: 'BACKLOG_RETRY', + }); + ds.workingDir = '/tmp'; + mocks.forkWorker.mockImplementationOnce(() => { + throw new Error('pre-fork acceptance failed'); + }); + + await expect(handleThreadReply( + makeEventData('om_failed_owner', 'FAILED_OWNER_REPLY', anchor), + makeCtx(anchor, 'om_failed_owner'), + )).rejects.toThrow('pre-fork acceptance failed'); + expect(ds.worker).toBeNull(); + expect(ds.initialStartPending).toBe(false); + expect(ds.initialStartClaimToken).toBeUndefined(); + + mocks.forkWorker.mockImplementation((owner: any) => { + owner.worker = { killed: false, send: vi.fn() }; + }); await handleThreadReply( - makeEventData(messageId, '补充同一个人的要求', anchor), - makeCtx(anchor, messageId), + makeEventData('om_retry_owner', 'RETRY_OWNER_REPLY', anchor), + makeCtx(anchor, 'om_retry_owner'), ); - expect(ds.pendingRawTurnId).toBe(messageId); - expect(ds.pendingFollowUpTurnId).toBe(messageId); - expect(ds.pendingFollowUps).toHaveLength(1); - expect(ds.session.quoteTargetId).toBe(messageId); + expect(mocks.forkWorker).toHaveBeenCalledTimes(2); + expect(mocks.forkWorker.mock.calls[1]![1].content).toContain('RETRY_OWNER_REPLY'); + expect(ds.initialStartPending).toBe(true); + expect(ds.initialStartClaimToken).toEqual(expect.any(String)); }); - it('pending raw mixed-caller follow-up clears both staged turns', async () => { - const anchor = 'om_pending_raw_mixed_root'; - const ds = seedPendingRawSession(anchor); - const ownerMessageId = 'om_pending_owner_followup'; + it('retains a tokened generic ACK successor after IPC failure and recovers it before the next inbound', async () => { + const anchor = 'om_ack_tail_handoff_root'; + const ds = seedThreadSession(anchor, 'ACK tail handoff'); + Object.assign(ds.session, { + cliId: 'claude-code', + workingDir: '/tmp', + queued: false, + }); + ds.workingDir = '/tmp'; + ds.hasHistory = true; + ds.initialStartPending = true; + ds.pendingFollowUps = ['GENERIC_TAIL_N_PLUS_1']; + ds.pendingFollowUpTurnIds = ['om_generic_tail_n_plus_1']; + const failedSend = vi.fn(() => { throw new Error('worker exited before accepting tail'); }); + const kill = vi.fn(); + ds.worker = { killed: false, send: failedSend, kill } as any; + // Promotion is already a durable acceptance boundary: an IPC throw fences + // this child but keeps one tokened journal owner for exact recovery. + expect(onQueuedActivationSubmitted(ds)).toBe(true); + expect(failedSend).toHaveBeenCalledTimes(1); + expect(kill).toHaveBeenCalledTimes(1); + expect(ds.pendingFollowUps).toBeUndefined(); + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(ds.session).toMatchObject({ + queued: false, + queuedActivationPending: true, + queuedActivationToken: expect.any(String), + queuedActivationTurnId: 'om_generic_tail_n_plus_1', + queuedActivationInput: expect.objectContaining({ + content: expect.stringContaining('GENERIC_TAIL_N_PLUS_1'), + }), + }); + const retainedToken = ds.session.queuedActivationToken; + // Model the worker error/exit fence (the route test uses a lightweight + // child stub without worker-pool event handlers). + ds.worker = null; + ds.initialStartPending = false; + ds.initialStartClaimToken = undefined; + expect(ds.initialStartPending).toBe(false); + + mocks.forkWorker.mockClear(); await handleThreadReply( - makeEventData(ownerMessageId, '先由原调用者补充', anchor), - makeCtx(anchor, ownerMessageId), + makeEventData('om_after_tail_n_plus_2', 'AFTER_TAIL_N_PLUS_2', anchor), + makeCtx(anchor, 'om_after_tail_n_plus_2'), ); - expect(ds.pendingRawTurnId).toBe(ownerMessageId); - expect(ds.pendingFollowUpTurnId).toBe(ownerMessageId); - - const strangerMessageId = 'om_pending_stranger_followup'; - const strangerData = makeEventData(strangerMessageId, '再由另一个人补充', anchor); - strangerData.sender = { sender_id: { open_id: 'ou_stranger' }, sender_type: 'user' }; - await handleThreadReply(strangerData, makeCtx(anchor, strangerMessageId)); - - expect(ds.pendingRawTurnId).toBeUndefined(); - expect(ds.pendingFollowUpTurnId).toBeUndefined(); - expect(ds.pendingFollowUps).toHaveLength(2); - expect(ds.session.quoteTargetId).toBe(strangerMessageId); - expect(ds.session.lastCallerOpenId).toBe('ou_stranger'); + + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); + const reforkedHead = mocks.forkWorker.mock.calls[0]![1]; + expect(reforkedHead).toBe(ds.session.queuedActivationInput); + expect(reforkedHead.content).toContain('GENERIC_TAIL_N_PLUS_1'); + expect(reforkedHead.content).not.toContain('AFTER_TAIL_N_PLUS_2'); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ turnId: 'om_after_tail_n_plus_2' }), + ]); + + const resumedSend = vi.mocked(ds.worker!.send); + Object.assign(ds.session, { + queuedActivationPending: undefined, + queuedActivationToken: undefined, + queuedActivationInput: undefined, + queuedActivationTurnId: undefined, + queuedActivationDispatchAttempt: undefined, + queuedActivationResume: undefined, + }); + expect(onQueuedActivationSubmitted(ds, retainedToken)).toBe(true); + expect(resumedSend).toHaveBeenCalledTimes(1); + expect(resumedSend).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'om_after_tail_n_plus_2', + content: expect.stringContaining('AFTER_TAIL_N_PLUS_2'), + queuedActivationToken: expect.any(String), + })); + }); +}); + +describe('daemon live-session registration claims', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.sessions.clear(); + mocks.getAvailableBots.mockResolvedValue([]); + }); + + it('is first-owner-wins under concurrent creation and closes the empty loser', async () => { + const registry = new Map(); + const first = makeChatSession('claim-first', 'oc_claim'); + const second = makeChatSession('claim-second', 'oc_claim'); + + const [firstResult, secondResult] = await Promise.all([ + claimNewDaemonSession(registry, first), + claimNewDaemonSession(registry, second), + ]); + + expect(firstResult.accepted).toBe(true); + expect(secondResult).toMatchObject({ + accepted: false, + reason: 'existing_owner', + owner: first, + closedIncomingSessionId: second.session.sessionId, + }); + expect(registry.get(sessionKey('oc_claim', APP))).toBe(first); + expect(mocks.closeSession).toHaveBeenCalledTimes(1); + expect(mocks.closeSession).toHaveBeenCalledWith(second.session.sessionId); + }); + + it('fails closed and preserves both persistence rows when both owners are pending', async () => { + const registry = new Map(); + const incumbent = makeChatSession('claim-pending-a', 'oc_pending', { pendingLedger: true }); + const incoming = makeChatSession('claim-pending-b', 'oc_pending', { pendingLedger: true }); + registry.set(sessionKey('oc_pending', APP), incumbent); + + const result = await claimNewDaemonSession(registry, incoming); + + expect(result).toMatchObject({ + accepted: false, + reason: 'both_pending', + owner: incumbent, + preservedIncomingSessionId: incoming.session.sessionId, + }); + expect(registry.get(sessionKey('oc_pending', APP))).toBe(incumbent); + expect(mocks.closeSession).not.toHaveBeenCalled(); + expect(mocks.sessions.get(incoming.session.sessionId)?.status).toBe('active'); + }); + + it('retires a losing group-join-style pending runtime before any durable setup is staged', async () => { + const registry = new Map(); + const incumbent = makeChatSession('join-canonical', 'oc_join_claim'); + const duplicate = makeChatSession('join-duplicate', 'oc_join_claim'); + duplicate.pendingRepo = true; + duplicate.pendingPrompt = 'SYNTHETIC_JOIN_OPENING'; + duplicate.initialStartPending = true; + registry.set(sessionKey('oc_join_claim', APP), incumbent); + + const result = await claimNewDaemonSession(registry, duplicate); + + expect(result).toMatchObject({ + accepted: false, + reason: 'existing_owner', + owner: incumbent, + closedIncomingSessionId: duplicate.session.sessionId, + }); + expect(duplicate.session.pendingRepoSetup).toBeUndefined(); + expect(mocks.closeSession).toHaveBeenCalledWith(duplicate.session.sessionId); + expect(registry.get(sessionKey('oc_join_claim', APP))).toBe(incumbent); + }); +}); + +describe('chat-mode conversion ownership', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.sessions.clear(); + mocks.getAvailableBots.mockResolvedValue([]); + activeSessions.clear(); + }); + + it.each([ + ['durable ledger', { pendingLedger: true }], + ['repo selection', { pendingRepo: true }], + ['queued dashboard task', { queued: true }], + ])('preserves a pending chat owner (%s)', (_label, options) => { + const owner = makeChatSession(`converted-${_label}`, CHAT, options); + activeSessions.set(sessionKey(CHAT, APP), owner); + + expect(handleChatModeConverted(CHAT, APP)).toBe(false); + expect(activeSessions.get(sessionKey(CHAT, APP))).toBe(owner); + }); + + it('still evicts a ledger-empty idle owner', () => { + const owner = makeChatSession('converted-idle', CHAT); + activeSessions.set(sessionKey(CHAT, APP), owner); + + expect(handleChatModeConverted(CHAT, APP)).toBe(true); + expect(activeSessions.has(sessionKey(CHAT, APP))).toBe(false); + }); + +}); + +describe('document comment canonical ownership and single-flight delivery', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.sessions.clear(); + mocks.forkWorker.mockImplementation((ds: any) => { + ds.worker = { killed: false, send: vi.fn() }; + }); + mocks.getAvailableBots.mockResolvedValue([]); + activeSessions.clear(); + resetDocCommentClaims(); + const bot = registerBot({ larkAppId: APP, larkAppSecret: 's', cliId: 'claude-code', allowedUsers: [OWNER] }); + bot.resolvedAllowedUsers = [OWNER]; + }); + + function docSub(fileToken: string): any { + const sub = { + fileToken, + fileType: 'docx', + sessionAnchor: `om_legacy_${fileToken}`, + scope: 'thread' as const, + chatId: CHAT, + commentTriggerMode: 'all' as const, + managedBy: 'watch-comment' as const, + createdAt: Date.now(), + }; + putDocSubscription(config.session.dataDir, APP, sub); + return sub; + } + + function docCtx(sub: any, suffix: string): any { + return { + larkAppId: APP, + sub, + commentId: `comment-${suffix}`, + replyId: `reply-${suffix}`, + text: `question ${suffix}`, + }; + } + + function bindSubToSession(sub: any, ds: DaemonSession): void { + Object.assign(sub, { + sessionAnchor: sessionAnchorId(ds), + sessionId: ds.session.sessionId, + scope: ds.scope, + chatId: ds.chatId, + }); + putDocSubscription(config.session.dataDir, APP, sub); + } + + it('leaves the provider cursor retryable when a worker-null setup owner exists', async () => { + const sub = docSub(`doc-protected-${Date.now()}`); + const ds = seedThreadSession(sub.sessionAnchor, 'protected doc owner'); + ds.pendingRepo = true; + ds.session.queued = true; + ds.session.queuedPrompt = 'OPENING_N'; + ds.session.pendingRepoSetup = { mode: 'picker', prompt: 'OPENING_N' }; + bindSubToSession(sub, ds); + mocks.forkWorker.mockClear(); + + await expect(handleDocComment(docCtx(sub, 'protected'))).resolves.toBe(false); + + expect(mocks.forkWorker).not.toHaveBeenCalled(); + expect(ds.worker).toBeNull(); + expect(ds.session.docCommentTargets).toBeUndefined(); + expect(ds.session.pendingRepoSetup).toMatchObject({ mode: 'picker', prompt: 'OPENING_N' }); + removeDocSubscription(config.session.dataDir, APP, sub.fileToken); + }); + + it('durably queues a live document comment behind the activation head without worker IPC', async () => { + const sub = docSub(`doc-live-gate-${Date.now()}`); + const ds = seedThreadSession(sub.sessionAnchor, 'live gated doc owner'); + const send = vi.fn(); + ds.worker = { killed: false, send } as any; + Object.assign(ds.session, { + queuedActivationPending: true, + queuedActivationToken: 'doc-opening-token', + queuedActivationInput: { content: 'OPENING_N' }, + queuedActivationTurnId: 'opening-turn', + }); + bindSubToSession(sub, ds); + + await expect(handleDocComment(docCtx(sub, 'live'))).resolves.toBe(true); + + expect(send).not.toHaveBeenCalled(); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'reply-live', + cliInput: expect.objectContaining({ content: expect.stringContaining('question live') }), + }), + ]); + removeDocSubscription(config.session.dataDir, APP, sub.fileToken); + }); + + it('refuses doc-watch prewarm before turn mutation when a worker-null setup owner exists', async () => { + const sub = docSub(`doc-prewarm-protected-${Date.now()}`); + const ds = seedThreadSession(sub.sessionAnchor, 'prewarm protected'); + ds.session.pendingRepoSetup = { mode: 'picker', prompt: 'OPENING_N' }; + ds.pendingRepo = true; + const priorLastMessageAt = ds.lastMessageAt; + mocks.forkWorker.mockClear(); + + await expect(prewarmDocCommentSession(ds, sub)).rejects.toThrow('durable opening ownership'); + + expect(ds.lastMessageAt).toBe(priorLastMessageAt); + expect(ds.currentTurnTitle).toBeUndefined(); + expect(mocks.forkWorker).not.toHaveBeenCalled(); + removeDocSubscription(config.session.dataDir, APP, sub.fileToken); + }); + + it('durably queues live doc-watch prewarm behind activation without worker IPC', async () => { + const sub = docSub(`doc-prewarm-live-${Date.now()}`); + const ds = seedThreadSession(sub.sessionAnchor, 'prewarm live'); + const send = vi.fn(); + ds.worker = { killed: false, send } as any; + Object.assign(ds.session, { + queuedActivationPending: true, + queuedActivationToken: 'prewarm-opening-token', + queuedActivationInput: { content: 'OPENING_N' }, + queuedActivationTurnId: 'opening-turn', + }); + + await expect(prewarmDocCommentSession(ds, sub)).resolves.toBeUndefined(); + + expect(send).not.toHaveBeenCalled(); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: expect.stringMatching(/^doc-watch-/), + cliInput: expect.objectContaining({ content: expect.stringContaining(sub.fileToken) }), + }), + ]); + removeDocSubscription(config.session.dataDir, APP, sub.fileToken); + }); + + it('serializes concurrent get-or-create, merges targets, and reuses canonical state after restart', async () => { + const fileToken = `doc-concurrent-${Date.now()}`; + const sub = docSub(fileToken); + + await expect(Promise.all([ + handleDocComment(docCtx(sub, 'one')), + handleDocComment(docCtx(sub, 'two')), + ])).resolves.toEqual([true, true]); + + const key = sessionKey(`doc:${fileToken}`, APP); + const owner = activeSessions.get(key)!; + expect(owner).toBeDefined(); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); + expect(Object.keys(owner.session.docCommentTargets ?? {}).sort()).toEqual(['reply-one', 'reply-two']); + + const persisted = getDocSubscription(config.session.dataDir, APP, fileToken)!; + expect(persisted).toMatchObject({ + sessionAnchor: `doc:${fileToken}`, + sessionId: owner.session.sessionId, + scope: 'chat', + chatId: `doc:${fileToken}`, + }); + // This is the exact anchor closeSession uses to find subscriptions. + expect(sessionAnchorId(owner)).toBe(persisted.sessionAnchor); + + // Simulate a daemon memory restart restoring the same persisted session at + // activeSessionKey(ds), then deliver another comment from a stale snapshot. + activeSessions.clear(); + owner.worker = null; + activeSessions.set(key, owner); + const staleSnapshot = { ...sub }; + await expect(handleDocComment(docCtx(staleSnapshot, 'three'))).resolves.toBe(true); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + expect(activeSessions.get(key)).toBe(owner); + expect(Object.keys(owner.session.docCommentTargets ?? {}).sort()).toEqual([ + 'reply-one', + 'reply-three', + 'reply-two', + ]); + + removeDocSubscription(config.session.dataDir, APP, fileToken); + }); + + it('makes duplicate WS/poll deliveries share failure so neither advances its cursor', async () => { + const fileToken = `doc-failure-${Date.now()}`; + const sub = docSub(fileToken); + const ctx = docCtx(sub, 'same'); + mocks.forkWorker.mockImplementationOnce(() => { throw new Error('simulated fork failure'); }); + + const results = await Promise.all([ + handleDocComment(ctx), + handleDocComment({ ...ctx, sub: { ...sub } }), + ]); + + expect(results).toEqual([false, false]); + expect(mocks.forkWorker).toHaveBeenCalledTimes(1); + + // Failure was not recorded as completed: a later poll retry can deliver. + mocks.forkWorker.mockImplementation((ds: any) => { + ds.worker = { killed: false, send: vi.fn() }; + }); + await expect(handleDocComment(ctx)).resolves.toBe(true); + expect(mocks.forkWorker).toHaveBeenCalledTimes(2); + + removeDocSubscription(config.session.dataDir, APP, fileToken); }); }); diff --git a/test/dashboard-attention-signals.test.ts b/test/dashboard-attention-signals.test.ts index 956179f31..71306a12e 100644 --- a/test/dashboard-attention-signals.test.ts +++ b/test/dashboard-attention-signals.test.ts @@ -218,10 +218,11 @@ describe('attention signals', () => { const src = readFileSync(new URL('../src/daemon.ts', import.meta.url), 'utf-8'); const start = src.indexOf('async function handleThreadReply('); expect(start).toBeGreaterThanOrEqual(0); - // 20000：窗口需罩住函数头到最后一个拦截点 (findPendingAskByAnchor) 的全部 - // 源码——passthrough 冷启动等合法插入会把后续 marker 往后推，窗口太紧会误报。 - // 语义断言不变：clear 在所有拦截点之前。 - const region = src.slice(start, start + 20000); + const end = src.indexOf('async function autoCreateDocSession', start); + expect(end).toBeGreaterThan(start); + // Bound the assertion by the function's next top-level sibling rather than + // a byte count: admission and recovery guards legitimately grow over time. + const region = src.slice(start, end); const clearIdx = region.indexOf('clearAgentAttentionForHumanInbound();'); expect(clearIdx).toBeGreaterThanOrEqual(0); for (const marker of [ diff --git a/test/dashboard-create-session.test.ts b/test/dashboard-create-session.test.ts index f9e3e4784..e1bcd6c01 100644 --- a/test/dashboard-create-session.test.ts +++ b/test/dashboard-create-session.test.ts @@ -35,28 +35,50 @@ vi.mock('../src/services/message-queue.js', () => ({ ensureQueue: vi.fn() })); const sendMessageMock = vi.fn(async () => 'om_banner_123'); const uploadImageMock = vi.fn(async () => 'img_dashboard_123'); +const replyMessageMock = vi.fn(async () => 'om_reply_123'); +const deleteMessageMock = vi.fn(async () => {}); vi.mock('../src/im/lark/client.js', () => ({ sendMessage: (...a: any[]) => sendMessageMock(...a), uploadImage: (...a: any[]) => uploadImageMock(...a), downloadMessageResource: vi.fn(), listChatBotMembers: vi.fn(async () => []), - getChatMode: vi.fn(), - replyMessage: vi.fn(), + getChatMode: vi.fn(async () => 'topic'), + replyMessage: (...a: any[]) => replyMessageMock(...a), + deleteMessage: (...a: any[]) => deleteMessageMock(...a), UserTokenMissingError: class extends Error {}, })); +const scanMultipleProjectsMock = vi.fn(() => [] as Array>); +vi.mock('../src/services/project-scanner.js', () => ({ + scanMultipleProjects: (...a: any[]) => scanMultipleProjectsMock(...a), +})); + const forkWorkerMock = vi.fn(); +const sendWorkerInputMock = vi.fn(); +const closeWorkerSessionMock = vi.fn(async () => ({ ok: true, alreadyClosed: false })); +const runAutoWorktreeCommitMock = vi.fn(async () => {}); +let activeRegistryMock: Map | null = null; vi.mock('../src/core/worker-pool.js', () => ({ forkWorker: (...a: any[]) => forkWorkerMock(...a), + sendWorkerInput: (...a: any[]) => sendWorkerInputMock(...a), forkAdoptWorker: vi.fn(), adoptSandboxBlocked: vi.fn((botCfg, session) => botCfg?.sandbox === true || botCfg?.readIsolation === true || session?.sandbox === true || process.env.BOTMUX_SANDBOX === '1'), killStalePids: vi.fn(), getCurrentCliVersion: vi.fn(() => 'test-cli-v1'), restoreUsageLimitRuntimeState: vi.fn(), - setActiveSessionSafe: vi.fn(async (map: Map, k: string, ds: any) => { map.set(k, ds); }), - getActiveSessionsRegistry: vi.fn(() => null), - isRelayableRealSession: vi.fn(() => false), - closeSession: vi.fn(), + setActiveSessionSafe: vi.fn(async (map: Map, k: string, ds: any) => { + map.set(k, ds); + return { accepted: true }; + }), + getActiveSessionsRegistry: vi.fn(() => activeRegistryMock), + isRelayableRealSession: vi.fn((ds: any) => !!ds?.worker || !!ds?.session?.cliId || !!ds?.session?.lastCliInput), + closeSession: (...a: any[]) => closeWorkerSessionMock(...a), + promoteQueuedActivationTail: vi.fn(() => false), + withActiveSessionKeyLock: vi.fn(async (_map: Map, _key: string, action: () => any) => action()), +})); + +vi.mock('../src/im/lark/card-handler.js', () => ({ + runAutoWorktreeCommit: (...args: any[]) => runAutoWorktreeCommitMock(...args), })); vi.mock('../src/bot-registry.js', () => ({ @@ -94,6 +116,7 @@ vi.mock('../src/services/whiteboard-store.js', () => ({ import { activateQueuedSession, buildReforkCliInput, + executeScheduledTask, rememberLastCliInput, restoreActiveSessions, spawnDashboardSession, @@ -101,10 +124,14 @@ import { import { sessionKey } from '../src/core/types.js'; import { dashboardEventBus } from '../src/core/dashboard-events.js'; import { getBot } from '../src/bot-registry.js'; +import { getAllBots } from '../src/bot-registry.js'; +import type { ScheduledTask } from '../src/types.js'; +import { setActiveSessionSafe } from '../src/core/worker-pool.js'; import { applyQueuedCodexAppLegacyFallback, mergeQueuedCodexAppTurn, } from '../src/core/session-create.js'; +import * as sessionStore from '../src/services/session-store.js'; const APP = 'cli_app_test'; const CHAT = 'oc_newgroup'; @@ -115,6 +142,21 @@ beforeEach(() => { forkWorkerMock.mockClear(); sendMessageMock.mockClear(); uploadImageMock.mockClear(); + replyMessageMock.mockClear(); + deleteMessageMock.mockClear(); + scanMultipleProjectsMock.mockReset(); + scanMultipleProjectsMock.mockReturnValue([]); + sendWorkerInputMock.mockClear(); + closeWorkerSessionMock.mockClear(); + runAutoWorktreeCommitMock.mockClear(); + vi.mocked(sessionStore.closeSession).mockClear(); + activeRegistryMock = null; + vi.mocked(sessionStore.updateSession).mockImplementation((s: Session) => { store.set(s.sessionId, s); }); + vi.mocked(setActiveSessionSafe).mockImplementation(async (map: Map, key: string, ds: any) => { + map.set(key, ds); + return { accepted: true } as any; + }); + vi.mocked(getAllBots).mockReturnValue([]); (dashboardEventBus.publish as any).mockClear(); vi.mocked(getBot).mockReturnValue({ config: { cliId: 'claude-code', cliPathOverride: undefined, defaultWorkingDir: '/tmp' }, @@ -249,8 +291,91 @@ describe('spawnDashboardSession — backlog (待办池) parks without starting t .toEqual(expect.arrayContaining([ expect.objectContaining({ kind: 'untrusted', value: expect.stringContaining('') }), ])); - expect(restored.session.queuedCodexAppText).toBeUndefined(); - expect(restored.session.queuedCodexAppMessageContext).toBeUndefined(); + // Activation ownership is accepted, but the exact queued journal remains + // until the real worker reports adapter-level submission. + expect(restored.session.queuedCodexAppText).toBe('重启后仍保持纯净'); + expect(restored.session.queuedCodexAppMessageContext).toContain(''); + }); + + it('retains and eagerly replays a non-Codex pre-init journal after a daemon crash', async () => { + const beforeRestart = new Map(); + await spawnDashboardSession(beforeRestart, undefined, { + larkAppId: APP, + chatId: CHAT, + content: 'recover at least once', + column: 'backlog', + role: 'solo', + }); + const interrupted = beforeRestart.get(sessionKey(CHAT, APP))!; + interrupted.session.queued = false; + interrupted.session.queuedActivationPending = true; + interrupted.session.queuedActivationToken = 'activation-before-crash'; + interrupted.session.queuedActivationInput = { content: interrupted.session.queuedPrompt! }; + interrupted.session.queuedActivationTurnId = 'turn-before-crash'; + interrupted.session.queuedActivationResume = false; + // The journal deliberately retains all queued payload fields until init + // IPC has returned and the success cleanup is durably committed. + store.set(interrupted.session.sessionId, interrupted.session); + + const afterRestart = new Map(); + await restoreActiveSessions(afterRestart); + const restored = afterRestart.get(sessionKey(CHAT, APP))!; + + expect(restored.session.queued).toBe(false); + expect(restored.session.queuedActivationPending).toBe(true); + expect(restored.session.queuedActivationToken).toBe('activation-before-crash'); + expect(restored.session.queuedActivationInput).toEqual({ + content: expect.stringContaining('recover at least once'), + }); + expect(restored.session.queuedPrompt).toContain('recover at least once'); + expect(restored.pendingPrompt).toBeUndefined(); + expect(restored.hasHistory).toBe(false); + expect(forkWorkerMock).toHaveBeenCalledWith( + restored, + restored.session.queuedActivationInput, + { + resume: false, + turnId: 'turn-before-crash', + dispatchAttempt: undefined, + }, + ); + }); + + it('restores a Codex pre-init journal through its accepted ledger without re-parking', async () => { + vi.mocked(getBot).mockReturnValue({ + config: { cliId: 'codex-app', cliPathOverride: undefined, defaultWorkingDir: '/tmp' }, + botName: 'TestBot', + botOpenId: 'ou_bot', + } as any); + const beforeRestart = new Map(); + await spawnDashboardSession(beforeRestart, undefined, { + larkAppId: APP, + chatId: CHAT, + content: 'recover from exact FIFO', + column: 'backlog', + role: 'solo', + }); + const interrupted = beforeRestart.get(sessionKey(CHAT, APP))!; + interrupted.session.cliId = 'codex-app'; + interrupted.session.queued = false; + interrupted.session.queuedActivationPending = true; + interrupted.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-before-crash', + turnId: 'turn-before-crash', + state: 'accepted', + content: 'recover from exact FIFO', + }]; + store.set(interrupted.session.sessionId, interrupted.session); + forkWorkerMock.mockClear(); + + const afterRestart = new Map(); + await restoreActiveSessions(afterRestart); + const restored = afterRestart.get(sessionKey(CHAT, APP))!; + + expect(restored.session.queued).toBe(false); + expect(restored.hasHistory).toBe(false); + expect(restored.session.codexAppDispatchLedger).toHaveLength(1); + expect(forkWorkerMock).toHaveBeenCalledWith(restored, '', true); }); it('starts a restored pre-clean-input backlog task safely from the Dashboard button', async () => { @@ -340,6 +465,265 @@ describe('spawnDashboardSession — backlog (待办池) parks without starting t expect(restored.lastCodexAppInput).toBeUndefined(); expect(restored.session.lastCodexAppInput).toBeUndefined(); }); + + it('restores a durable picker by publishing and persisting a fresh authoritative card id', async () => { + const pending: Session = { + sessionId: 'pending-picker', + chatId: CHAT, + rootMessageId: CHAT, + scope: 'chat', + larkAppId: APP, + title: 'pick a repo', + status: 'active', + createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + queued: true, + queuedPrompt: 'OPENING_N', + pendingRepoSetup: { + mode: 'picker', + prompt: 'OPENING_N', + repoCardMessageId: 'om_stale_picker', + }, + }; + store.set(pending.sessionId, pending); + scanMultipleProjectsMock.mockReturnValue([{ + name: 'repo', path: '/tmp', type: 'repo', branch: 'main', + }]); + sendMessageMock.mockResolvedValueOnce('om_fresh_picker'); + + const active = new Map(); + await restoreActiveSessions(active); + + const restored = active.get(sessionKey(CHAT, APP))!; + expect(restored.pendingRepo).toBe(true); + expect(restored.pendingPrompt).toBe('OPENING_N'); + expect(restored.repoCardMessageId).toBe('om_fresh_picker'); + expect(restored.session.pendingRepoSetup?.repoCardMessageId).toBe('om_fresh_picker'); + expect(sendMessageMock).toHaveBeenCalledWith(APP, CHAT, expect.any(String), 'interactive'); + expect(deleteMessageMock).toHaveBeenCalledWith(APP, 'om_stale_picker'); + expect(forkWorkerMock).not.toHaveBeenCalled(); + }); + + it('isolates a picker publish failure and keeps restoring later sessions with the exact setup journal', async () => { + const pending: Session = { + sessionId: 'pending-picker-failure', + chatId: CHAT, + rootMessageId: CHAT, + scope: 'chat', + larkAppId: APP, + title: 'pick a repo', + status: 'active', + createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + queued: true, + queuedPrompt: 'OPENING_N', + pendingRepoSetup: { + mode: 'picker', + prompt: 'OPENING_N', + repoCardMessageId: 'om_old_picker', + }, + }; + const sibling: Session = { + sessionId: 'later-backlog', + chatId: 'oc_later', + rootMessageId: 'oc_later', + scope: 'chat', + larkAppId: APP, + title: 'later', + status: 'active', + createdAt: new Date('2026-01-01T00:00:01Z').toISOString(), + queued: true, + queuedPrompt: 'LATER_TASK', + }; + store.set(pending.sessionId, pending); + store.set(sibling.sessionId, sibling); + scanMultipleProjectsMock.mockReturnValue([{ + name: 'repo', path: '/tmp', type: 'repo', branch: 'main', + }]); + sendMessageMock.mockRejectedValueOnce(new Error('picker publish unavailable')); + + const active = new Map(); + await expect(restoreActiveSessions(active)).resolves.toBeUndefined(); + + const restored = active.get(sessionKey(CHAT, APP))!; + expect(restored.pendingRepo).toBe(true); + expect(restored.pendingPrompt).toBe('OPENING_N'); + expect(restored.repoCardMessageId).toBe('om_old_picker'); + expect(restored.session.pendingRepoSetup).toMatchObject({ + mode: 'picker', prompt: 'OPENING_N', repoCardMessageId: 'om_old_picker', + }); + expect(active.get(sessionKey('oc_later', APP))?.session.queuedPrompt).toBe('LATER_TASK'); + }); + + it('contains detached auto-worktree recovery rejection and leaves the setup retryable', async () => { + const pending: Session = { + sessionId: 'pending-auto-worktree', + chatId: CHAT, + rootMessageId: CHAT, + scope: 'chat', + larkAppId: APP, + title: 'make worktree', + status: 'active', + createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + queued: true, + queuedPrompt: 'OPENING_N', + pendingRepoSetup: { + mode: 'auto_worktree', prompt: 'OPENING_N', baseDir: '/tmp', + }, + }; + store.set(pending.sessionId, pending); + runAutoWorktreeCommitMock.mockRejectedValueOnce(new Error('worktree publish unavailable')); + + const active = new Map(); + await expect(restoreActiveSessions(active)).resolves.toBeUndefined(); + await Promise.resolve(); + + const restored = active.get(sessionKey(CHAT, APP))!; + expect(restored.pendingRepo).toBe(true); + expect(restored.pendingPrompt).toBe('OPENING_N'); + expect(restored.session.pendingRepoSetup).toMatchObject({ + mode: 'auto_worktree', prompt: 'OPENING_N', baseDir: '/tmp', + }); + }); + + it('isolates a historical same-anchor protected loser so one collision cannot abort daemon restart', async () => { + const incumbent: Session = { + sessionId: 'canonical-pending-owner', + chatId: CHAT, + rootMessageId: CHAT, + scope: 'chat', + larkAppId: APP, + title: 'canonical', + status: 'active', + createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + cliId: 'claude-code', + queuedActivationPending: true, + queuedActivationToken: 'canonical-token', + queuedActivationInput: { content: 'CANONICAL_N' }, + queuedActivationTurnId: 'turn-canonical', + queuedActivationResume: false, + }; + const stagedLoser: Session = { + sessionId: 'historical-staged-loser', + chatId: CHAT, + rootMessageId: CHAT, + scope: 'chat', + larkAppId: APP, + title: 'loser', + status: 'active', + createdAt: new Date('2026-01-01T00:00:01Z').toISOString(), + queued: true, + queuedPrompt: 'LOSER_OPENING_N', + pendingRepoSetup: { mode: 'picker', prompt: 'LOSER_OPENING_N' }, + }; + store.set(incumbent.sessionId, incumbent); + store.set(stagedLoser.sessionId, stagedLoser); + vi.mocked(setActiveSessionSafe).mockImplementation(async (map, key, ds) => { + const current = map.get(key); + if (!current) { + map.set(key, ds); + return { accepted: true } as any; + } + return { + accepted: false, + reason: 'both_pending', + keptSessionId: current.session.sessionId, + preservedIncomingSessionId: ds.session.sessionId, + } as any; + }); + + const active = new Map(); + await expect(restoreActiveSessions(active)).resolves.toBeUndefined(); + + expect(active.get(sessionKey(CHAT, APP))?.session.sessionId).toBe(incumbent.sessionId); + expect(store.get(stagedLoser.sessionId)).toMatchObject({ + status: 'active', + queued: true, + queuedPrompt: 'LOSER_OPENING_N', + pendingRepoSetup: { mode: 'picker', prompt: 'LOSER_OPENING_N' }, + }); + }); + + it('isolates a malformed queued+unsettled row and restores the later healthy row', async () => { + const malformed: Session = { + sessionId: 'malformed-queued', chatId: CHAT, rootMessageId: CHAT, scope: 'chat', larkAppId: APP, + title: 'bad', status: 'active', createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + queued: true, queuedPrompt: 'BAD_N', + codexAppDispatchLedger: [{ dispatchId: 'bad-dispatch', turnId: 'bad-turn', state: 'prepared', content: 'BAD_N' }], + }; + const healthy: Session = { + sessionId: 'healthy-after-malformed', chatId: 'oc_healthy_1', rootMessageId: 'oc_healthy_1', scope: 'chat', larkAppId: APP, + title: 'good', status: 'active', createdAt: new Date('2026-01-01T00:00:01Z').toISOString(), + queued: true, queuedPrompt: 'GOOD_N', + }; + store.set(malformed.sessionId, malformed); + store.set(healthy.sessionId, healthy); + + const active = new Map(); + await expect(restoreActiveSessions(active)).resolves.toBeUndefined(); + + expect(active.has(sessionKey(CHAT, APP))).toBe(false); + expect(store.get(malformed.sessionId)).toMatchObject({ status: 'active', queued: true, queuedPrompt: 'BAD_N' }); + expect(active.get(sessionKey('oc_healthy_1', APP))?.session.sessionId).toBe(healthy.sessionId); + }); + + it('isolates a tail-promotion write failure, rolls the row back, and restores the later row', async () => { + const failed: Session = { + sessionId: 'failed-tail-promotion', chatId: CHAT, rootMessageId: CHAT, scope: 'chat', larkAppId: APP, + title: 'bad tail', status: 'active', createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + cliId: 'claude-code', + queuedActivationTail: [{ id: 'tail-1', order: 1, userPrompt: 'TAIL_N', cliInput: { content: 'TAIL_N' }, turnId: 'tail-turn' }], + queuedActivationTailNextOrder: 1, + }; + const healthy: Session = { + sessionId: 'healthy-after-tail', chatId: 'oc_healthy_2', rootMessageId: 'oc_healthy_2', scope: 'chat', larkAppId: APP, + title: 'good', status: 'active', createdAt: new Date('2026-01-01T00:00:01Z').toISOString(), + queued: true, queuedPrompt: 'GOOD_AFTER_TAIL', + }; + store.set(failed.sessionId, failed); + store.set(healthy.sessionId, healthy); + vi.mocked(sessionStore.updateSession).mockImplementation((s: Session) => { + if (s.sessionId === failed.sessionId) throw new Error('tail promotion save unavailable'); + store.set(s.sessionId, s); + }); + + const active = new Map(); + await expect(restoreActiveSessions(active)).resolves.toBeUndefined(); + + expect(active.has(sessionKey(CHAT, APP))).toBe(false); + expect(failed.queuedActivationPending).toBeUndefined(); + expect(failed.queuedActivationTail?.[0]?.cliInput.content).toBe('TAIL_N'); + expect(active.get(sessionKey('oc_healthy_2', APP))?.session.sessionId).toBe(healthy.sessionId); + }); + + it('rolls back a failed terminal-empty cleanup and continues restoring later rows', async () => { + const failed: Session = { + sessionId: 'failed-terminal-cleanup', chatId: CHAT, rootMessageId: CHAT, scope: 'chat', larkAppId: APP, + title: 'terminal', status: 'active', createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + cliId: 'codex-app', queuedActivationPending: true, queuedActivationToken: 'terminal-token', + queuedActivationInput: { content: 'SETTLED_N' }, queuedActivationTurnId: 'terminal-turn', + }; + const healthy: Session = { + sessionId: 'healthy-after-terminal', chatId: 'oc_healthy_3', rootMessageId: 'oc_healthy_3', scope: 'chat', larkAppId: APP, + title: 'good', status: 'active', createdAt: new Date('2026-01-01T00:00:01Z').toISOString(), + queued: true, queuedPrompt: 'GOOD_AFTER_TERMINAL', + }; + store.set(failed.sessionId, failed); + store.set(healthy.sessionId, healthy); + vi.mocked(sessionStore.updateSession).mockImplementation((s: Session) => { + if (s.sessionId === failed.sessionId) throw new Error('terminal cleanup save unavailable'); + store.set(s.sessionId, s); + }); + + const active = new Map(); + await expect(restoreActiveSessions(active)).resolves.toBeUndefined(); + + expect(active.has(sessionKey(CHAT, APP))).toBe(false); + expect(failed).toMatchObject({ + queuedActivationPending: true, + queuedActivationToken: 'terminal-token', + queuedActivationInput: { content: 'SETTLED_N' }, + }); + expect(active.get(sessionKey('oc_healthy_3', APP))?.session.sessionId).toBe(healthy.sessionId); + }); }); describe('spawnDashboardSession — in_progress starts immediately', () => { @@ -367,6 +751,108 @@ describe('spawnDashboardSession — in_progress starts immediately', () => { }); expect(prompt.content).toContain('Sub1'); }); + + it('unpublishes and closes only the new row when fork pre-accept throws', async () => { + const active = new Map(); + forkWorkerMock.mockImplementationOnce(() => { throw new Error('spawn preaccept failed'); }); + + await expect(spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'opening task', column: 'in_progress', role: 'solo', + })).resolves.toEqual({ ok: false, error: 'spawn preaccept failed' }); + + expect(active.has(sessionKey(CHAT, APP))).toBe(false); + expect(sessionStore.closeSession).toHaveBeenCalledWith('sess-1'); + }); + + it('unpublishes and closes only the new row when auto-worktree staging fails', async () => { + vi.mocked(getBot).mockReturnValue({ + config: { + cliId: 'claude-code', defaultWorkingDir: '/tmp', defaultWorkingDirAutoWorktree: true, + }, + botName: 'TestBot', botOpenId: 'ou_bot', + } as any); + vi.mocked(sessionStore.updateSession).mockImplementation((s: Session) => { + if (s.pendingRepoSetup?.mode === 'auto_worktree') throw new Error('stage setup unavailable'); + store.set(s.sessionId, s); + }); + const active = new Map(); + activeRegistryMock = active; + + await expect(spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'opening worktree task', column: 'in_progress', role: 'solo', + })).resolves.toEqual({ ok: false, error: 'stage setup unavailable' }); + + expect(active.has(sessionKey(CHAT, APP))).toBe(false); + expect(sessionStore.closeSession).toHaveBeenCalledWith('sess-1'); + expect(runAutoWorktreeCommitMock).not.toHaveBeenCalled(); + expect(forkWorkerMock).not.toHaveBeenCalled(); + }); + + it('keeps a visible picker fail-closed when persisting its authoritative id fails', async () => { + vi.mocked(getBot).mockReturnValue({ + config: { cliId: 'claude-code', workingDir: '/tmp', defaultWorkingDir: undefined }, + botName: 'TestBot', botOpenId: 'ou_bot', + } as any); + scanMultipleProjectsMock.mockReturnValue([{ name: 'repo', path: '/tmp', type: 'repo', branch: 'main' }]); + sendMessageMock.mockResolvedValueOnce('om_visible_picker'); + vi.mocked(sessionStore.updateSession).mockImplementation((s: Session) => { + if (s.pendingRepoSetup?.repoCardMessageId === 'om_visible_picker') { + throw new Error('picker id save unavailable'); + } + store.set(s.sessionId, s); + }); + const active = new Map(); + + const result = await spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'pick before start', column: 'in_progress', role: 'solo', + }); + + expect(result).toEqual({ ok: true, sessionId: 'sess-1' }); + const ds = active.get(sessionKey(CHAT, APP))!; + expect(ds.pendingRepo).toBe(true); + expect(ds.repoCardMessageId).toBe('om_visible_picker'); + expect(ds.session.pendingRepoSetup).toMatchObject({ mode: 'picker', prompt: expect.stringContaining('pick before start') }); + expect(ds.session.pendingRepoSetup?.repoCardMessageId).toBeUndefined(); + expect(forkWorkerMock).not.toHaveBeenCalled(); + expect(sessionStore.closeSession).not.toHaveBeenCalled(); + expect(dashboardEventBus.publish).toHaveBeenCalledWith(expect.objectContaining({ + type: 'session.spawned', + })); + }); + + it('retains a durable picker owner when publish fallback and fork both fail', async () => { + vi.mocked(getBot).mockReturnValue({ + config: { cliId: 'claude-code', workingDir: '/tmp', defaultWorkingDir: undefined }, + botName: 'TestBot', botOpenId: 'ou_bot', + } as any); + scanMultipleProjectsMock.mockReturnValue([{ name: 'repo', path: '/tmp', type: 'repo', branch: 'main' }]); + sendMessageMock.mockRejectedValueOnce(new Error('picker publish unavailable')); + forkWorkerMock.mockImplementationOnce(() => { throw new Error('fallback fork unavailable'); }); + const active = new Map(); + + const result = await spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'retain exact opening', column: 'in_progress', role: 'solo', + }); + + expect(result).toEqual({ ok: false, error: 'fallback fork unavailable' }); + const ds = active.get(sessionKey(CHAT, APP))!; + expect(ds).toBeDefined(); + expect(ds.pendingRepo).toBe(true); + expect(ds.pendingPrompt).toContain('retain exact opening'); + expect(ds.session).toMatchObject({ + status: 'active', + queued: true, + queuedPrompt: expect.stringContaining('retain exact opening'), + pendingRepoSetup: { + mode: 'picker', + prompt: expect.stringContaining('retain exact opening'), + }, + }); + expect(sessionStore.closeSession).not.toHaveBeenCalled(); + expect(dashboardEventBus.publish).toHaveBeenCalledWith(expect.objectContaining({ + type: 'session.spawned', + })); + }); }); describe('spawnDashboardSession — guards', () => { @@ -376,6 +862,64 @@ describe('spawnDashboardSession — guards', () => { const r2 = await spawnDashboardSession(active, undefined, { larkAppId: APP, chatId: CHAT, content: 'b', column: 'in_progress', role: 'solo' }); expect(r2).toMatchObject({ ok: false, error: 'session_exists' }); }); + + it.each([ + ['pendingRepo', { pendingRepo: true }], + ['initialStartPending', { initialStartPending: true }], + ['worktreeCreating', { worktreeCreating: true }], + ])('does not replace an existing %s opening reservation', async (_name, flags) => { + const reserved = { + session: { sessionId: 'reserved', status: 'active', queued: false }, + worker: null, + larkAppId: APP, + chatId: CHAT, + scope: 'chat', + ...flags, + } as unknown as DaemonSession; + const active = new Map([[sessionKey(CHAT, APP), reserved]]); + + const result = await spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'must not replace', column: 'in_progress', role: 'solo', + }); + + expect(result).toEqual({ ok: false, error: 'session_exists' }); + expect(active.get(sessionKey(CHAT, APP))).toBe(reserved); + expect(closeWorkerSessionMock).not.toHaveBeenCalled(); + expect(forkWorkerMock).not.toHaveBeenCalled(); + }); + + it('rechecks first-owner after banner preparation and preserves a concurrent reservation', async () => { + const active = new Map(); + let bannerStarted!: () => void; + let releaseBanner!: () => void; + const started = new Promise(resolve => { bannerStarted = resolve; }); + const paused = new Promise(resolve => { releaseBanner = resolve; }); + sendMessageMock.mockImplementationOnce(async () => { + bannerStarted(); + await paused; + return 'om_delayed_banner'; + }); + + const spawning = spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'late contender', column: 'in_progress', role: 'solo', postBanner: true, + }); + await started; + const incumbent = { + session: { sessionId: 'concurrent-reservation', status: 'active', queued: false }, + worker: null, + initialStartPending: true, + larkAppId: APP, + chatId: CHAT, + scope: 'chat', + } as unknown as DaemonSession; + active.set(sessionKey(CHAT, APP), incumbent); + releaseBanner(); + + await expect(spawning).resolves.toEqual({ ok: false, error: 'session_exists' }); + expect(active.get(sessionKey(CHAT, APP))).toBe(incumbent); + expect(closeWorkerSessionMock).not.toHaveBeenCalled(); + expect(forkWorkerMock).not.toHaveBeenCalled(); + }); }); describe('activateQueuedSession', () => { @@ -395,7 +939,9 @@ describe('activateQueuedSession', () => { expect(prompt).toMatchObject({ content: expect.stringContaining('排队的任务') }); expect(prompt.content).toContain(''); // preamble survived park→activate expect(ds.session.queued).toBe(false); - expect(ds.session.queuedPrompt).toBeUndefined(); + // The worker-pool ACK handler, not activation acceptance, clears this + // exact replay source after adapter submission. + expect(ds.session.queuedPrompt).toContain('排队的任务'); expect(ds.session.kanbanColumn).toBe('in_progress'); }); @@ -403,4 +949,198 @@ describe('activateQueuedSession', () => { const ds = { worker: null, session: { queued: false } } as unknown as DaemonSession; expect(await activateQueuedSession(ds)).toMatchObject({ ok: false, error: 'not_queued' }); }); + + it('keeps the backlog payload retryable when fork pre-accept throws', async () => { + const active = new Map(); + await spawnDashboardSession(active, undefined, { + larkAppId: APP, chatId: CHAT, content: 'do not lose this task', column: 'backlog', role: 'solo', + }); + activeRegistryMock = active; + const ds = active.get(sessionKey(CHAT, APP))!; + forkWorkerMock.mockImplementationOnce(() => { throw new Error('fork preaccept failed'); }); + + const result = await activateQueuedSession(ds); + + expect(result).toEqual({ ok: false, error: 'fork preaccept failed' }); + expect(ds.session.queued).toBe(true); + expect(ds.session.queuedPrompt).toContain('do not lose this task'); + expect(ds.pendingPrompt).toContain('do not lose this task'); + expect(ds.initialStartPending).toBe(false); + expect(ds.pendingRepo).toBe(false); + }); + + it('returns success after worker ownership even when kanban metadata persistence fails', async () => { + const active = new Map(); + await spawnDashboardSession(active, undefined, { + larkAppId: APP, + chatId: CHAT, + content: 'owned before metadata write', + column: 'backlog', + role: 'solo', + }); + const ds = active.get(sessionKey(CHAT, APP))!; + forkWorkerMock.mockImplementationOnce((owned: DaemonSession) => { + owned.worker = { killed: false, send: vi.fn() } as any; + owned.session.queuedActivationPending = true; + owned.session.queuedActivationToken = 'activation-token'; + }); + vi.mocked(sessionStore.updateSession) + .mockImplementationOnce((s: Session) => { store.set(s.sessionId, s); }) + .mockImplementationOnce(() => { + throw new Error('kanban projection unavailable'); + }); + + await expect(activateQueuedSession(ds)).resolves.toEqual({ ok: true }); + expect(forkWorkerMock).toHaveBeenCalledOnce(); + expect(ds.worker).toBeTruthy(); + expect(ds.session.queued).toBe(false); + expect(ds.session.kanbanColumn).toBe('in_progress'); + expect(ds.session.queuedPrompt).toContain('owned before metadata write'); + }); + + it('keeps the durable queued payload while an auto-worktree owns the pending fork', async () => { + vi.mocked(getBot).mockReturnValue({ + config: { + cliId: 'codex-app', + cliPathOverride: undefined, + defaultWorkingDir: '/tmp', + defaultWorkingDirAutoWorktree: true, + codexAppCleanInput: true, + }, + botName: 'TestBot', + botOpenId: 'ou_bot', + } as any); + const active = new Map(); + await spawnDashboardSession(active, undefined, { + larkAppId: APP, + chatId: CHAT, + content: 'survive pending worktree restart', + column: 'backlog', + role: 'solo', + }); + activeRegistryMock = active; + const ds = active.get(sessionKey(CHAT, APP))!; + + expect(await activateQueuedSession(ds)).toEqual({ ok: true }); + + expect(forkWorkerMock).not.toHaveBeenCalled(); + expect(runAutoWorktreeCommitMock).toHaveBeenCalledTimes(1); + expect(ds.pendingRepo).toBe(true); + expect(ds.session).toMatchObject({ + queued: true, + queuedPrompt: expect.stringContaining('survive pending worktree restart'), + queuedCodexAppText: 'survive pending worktree restart', + kanbanColumn: 'in_progress', + }); + + // A repeated dashboard start is idempotent while the delayed commit owns + // the attempt; it must not schedule a second worktree or picker. + expect(await activateQueuedSession(ds)).toEqual({ ok: true }); + expect(runAutoWorktreeCommitMock).toHaveBeenCalledTimes(1); + }); +}); + +describe('executeScheduledTask — workerless owner semantics', () => { + const ROOT = 'om_scheduler_owner'; + + function task(): ScheduledTask { + return { + id: 'schedule-owner-test', + name: 'owner test', + schedule: 'once', + parsed: { kind: 'once', at: '2026-01-01T00:00:00.000Z' }, + prompt: 'scheduled prompt', + workingDir: '/tmp', + chatId: CHAT, + rootMessageId: ROOT, + scope: 'thread', + larkAppId: APP, + enabled: true, + createdAt: '2026-01-01T00:00:00.000Z', + } as ScheduledTask; + } + + function owner(overrides: Partial & { session?: Partial } = {}): DaemonSession { + const { session: sessionPatch, ...daemonPatch } = overrides; + return { + session: { + sessionId: 'existing-owner', chatId: CHAT, rootMessageId: ROOT, + status: 'active', scope: 'thread', createdAt: '2026-01-01T00:00:00.000Z', + ...sessionPatch, + } as Session, + worker: null, + workerPort: null, + workerToken: null, + larkAppId: APP, + chatId: CHAT, + chatType: 'group', + scope: 'thread', + spawnedAt: 1, + cliVersion: 'test', + lastMessageAt: 1, + hasHistory: false, + ...daemonPatch, + } as DaemonSession; + } + + beforeEach(() => { + vi.mocked(getAllBots).mockReturnValue([{ + config: { larkAppId: APP, cliId: 'claude-code' }, + botName: 'TestBot', botOpenId: 'ou_bot', resolvedAllowedUsers: [], + }] as any); + }); + + it.each([ + ['pending_repo', { pendingRepo: true }], + ['initial_start_pending', { initialStartPending: true }], + ['worktree_creating', { worktreeCreating: true }], + ['queued_backlog', { session: { queued: true } }], + ])('preserves a %s reservation instead of forking the scheduled prompt', async (state, patch) => { + const ds = owner(patch as any); + const active = new Map([[sessionKey(ROOT, APP), ds]]); + + await expect(executeScheduledTask(task(), active, vi.fn())).rejects.toThrow(state); + + expect(active.get(sessionKey(ROOT, APP))).toBe(ds); + expect(forkWorkerMock).not.toHaveBeenCalled(); + expect(sendWorkerInputMock).not.toHaveBeenCalled(); + expect(closeWorkerSessionMock).not.toHaveBeenCalled(); + }); + + it('reforks a cold real owner and keeps its history', async () => { + const ds = owner({ session: { cliId: 'claude-code', lastCliInput: 'prior turn' }, hasHistory: true }); + const active = new Map([[sessionKey(ROOT, APP), ds]]); + + await executeScheduledTask(task(), active, vi.fn()); + + expect(active.get(sessionKey(ROOT, APP))).toBe(ds); + expect(forkWorkerMock).toHaveBeenCalledWith(ds, expect.anything(), expect.objectContaining({ + resume: true, + turnId: expect.stringMatching(/^schedule:schedule-owner-test:/), + })); + expect(closeWorkerSessionMock).not.toHaveBeenCalled(); + }); + + it('retires a scratch owner and creates a fresh scheduled session', async () => { + const scratch = owner(); + const active = new Map([[sessionKey(ROOT, APP), scratch]]); + + await executeScheduledTask(task(), active, vi.fn()); + + expect(closeWorkerSessionMock).toHaveBeenCalledWith('existing-owner'); + expect(active.get(sessionKey(ROOT, APP))).not.toBe(scratch); + expect(forkWorkerMock).toHaveBeenCalledTimes(1); + }); + + it('retains a fresh-session reservation and opening prompt when fork pre-accept throws', async () => { + const active = new Map(); + forkWorkerMock.mockImplementationOnce(() => { throw new Error('scheduler fork failed'); }); + + await expect(executeScheduledTask(task(), active, vi.fn())).rejects.toThrow('scheduler fork failed'); + + const ds = active.get(sessionKey(ROOT, APP)); + expect(ds?.initialStartPending).toBe(true); + expect(ds?.pendingPrompt).toBe('scheduled prompt'); + expect(ds?.worker).toBeNull(); + }); }); diff --git a/test/dashboard-ipc.test.ts b/test/dashboard-ipc.test.ts index afece1f41..eae11118a 100644 --- a/test/dashboard-ipc.test.ts +++ b/test/dashboard-ipc.test.ts @@ -13,7 +13,7 @@ import * as oncallStore from '../src/services/oncall-store.js'; import * as sessionStore from '../src/services/session-store.js'; import * as workerPool from '../src/core/worker-pool.js'; import * as scheduler from '../src/core/scheduler.js'; -import { __testOnly_resetBotRegistry, loadBotConfigs, registerBot } from '../src/bot-registry.js'; +import { __testOnly_resetBotRegistry, getBot, loadBotConfigs, registerBot } from '../src/bot-registry.js'; import { config } from '../src/config.js'; import { sessionKey } from '../src/core/types.js'; import { writeRoleFile, writeTeamRoleFile } from '../src/core/role-resolver.js'; @@ -118,6 +118,40 @@ describe('dashboard IPC server', () => { expect(res.status).toBe(404); }); + it('binds and serves health early but holds authenticated state routes behind readiness', async () => { + setIpcAuthSecret(TEST_IPC_SECRET); + let releaseReady!: () => void; + const ready = new Promise(resolve => { releaseReady = resolve; }); + let mutations = 0; + const path = '/api/test-startup-readiness-mutation'; + ipcRoute('POST', path, (_req, res) => { + mutations += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + }); + handle = await startIpcServer({ + port: 0, + host: '127.0.0.1', + authRequired: true, + ready, + }); + const base = `http://127.0.0.1:${handle.port}`; + + const health = await fetch(`${base}/__health`); + expect(health.status).toBe(200); + const pending = fetch(`${base}${path}`, { + method: 'POST', + headers: trustedHostHeaders('POST', path, handle.port), + }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(mutations).toBe(0); + + releaseReady(); + const response = await pending; + expect(response.status).toBe(200); + expect(mutations).toBe(1); + }); + it('denies sandbox-like loopback reads and mutations but accepts route-bound trusted-host calls', async () => { setIpcAuthSecret(TEST_IPC_SECRET); let mutations = 0; @@ -655,6 +689,97 @@ describe('POST /api/sessions/:sessionId/lock', () => { }); }); +describe('POST /api/sessions/:sessionId/board queued activation', () => { + it('returns the activation failure without publishing a false in-progress success', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'dashboard-ipc-board-activation-')); + const previousDataDir = config.session.dataDir; + const previousRegistry = workerPool.getActiveSessionsRegistry(); + const appId = 'test-board-activation-app'; + const events: any[] = []; + const off = dashboardEventBus.subscribe(event => events.push(event)); + try { + config.session.dataDir = dataDir; + registerBot({ + larkAppId: appId, + larkAppSecret: 'secret', + cliId: 'codex', + defaultWorkingDir: '/tmp', + workingDir: '/tmp', + workingDirs: ['/tmp'], + } as any); + setLarkAppId(appId); + sessionStore.init(appId); + const session = sessionStore.createSession('oc_board', 'om_board', 'queued board task', 'group'); + Object.assign(session, { + larkAppId: appId, + scope: 'thread', + workingDir: '/tmp', + queued: true, + queuedPrompt: 'queued board payload', + kanbanColumn: 'backlog', + }); + sessionStore.updateSession(session); + const ds = { + session, + worker: null, + workerPort: null, + workerToken: null, + larkAppId: appId, + chatId: session.chatId, + chatType: 'group', + scope: 'thread', + spawnedAt: Date.now(), + cliVersion: 'test', + lastMessageAt: Date.now(), + hasHistory: false, + workingDir: '/tmp', + pendingPrompt: session.queuedPrompt, + } as any; + workerPool.setActiveSessionsRegistry(new Map([[sessionKey(session.rootMessageId, appId), ds]])); + workerPool.initWorkerPool({ + sessionReply: vi.fn(async () => 'om_reply'), + getSessionWorkingDir: () => { throw new Error('forced pre-init failure'); }, + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + handle = await startIpcServer({ port: 0, host: '127.0.0.1' }); + + const res = await fetch( + `http://127.0.0.1:${handle.port}/api/sessions/${session.sessionId}/board`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ column: 'in_progress', position: 7 }), + }, + ); + + expect(res.status).toBe(500); + expect(await res.json()).toEqual({ ok: false, error: 'forced pre-init failure' }); + expect(sessionStore.getSession(session.sessionId)).toMatchObject({ + queued: true, + queuedPrompt: 'queued board payload', + kanbanColumn: 'backlog', + }); + expect(events).not.toContainEqual(expect.objectContaining({ + type: 'session.update', + body: expect.objectContaining({ sessionId: session.sessionId }), + })); + } finally { + off(); + workerPool.setActiveSessionsRegistry(previousRegistry ?? new Map()); + workerPool.initWorkerPool({ + sessionReply: vi.fn(async () => 'om_reply'), + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 0, + closeSession: vi.fn(), + }); + sessionStore.init(); + config.session.dataDir = previousDataDir; + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); + describe('POST /api/sessions/:sessionId/restart', () => { it('sends a restart IPC message to the live worker', async () => { const send = vi.fn(); @@ -669,7 +794,7 @@ describe('POST /api/sessions/:sessionId/restart', () => { expect(res.status).toBe(200); expect(await res.json()).toMatchObject({ ok: true, sessionId: 's-restart', cliId: 'codex' }); - expect(send).toHaveBeenCalledWith({ type: 'restart' }); + expect(send).toHaveBeenCalledWith({ type: 'restart', reason: 'operator' }); findSpy.mockRestore(); }); @@ -774,6 +899,31 @@ describe('POST /api/sessions/:sessionId/suspend', () => { findSpy.mockRestore(); }); + it('409s before suspension while durable Codex App dispatch ownership is non-empty', async () => { + const ds = { + session: { + sessionId: 's-owned', + cliId: 'codex-app', + codexAppDispatchLedger: [ + { dispatchId: 'd-1', turnId: 't-1', state: 'prepared', content: 'owned' }, + ], + }, + worker: { send: vi.fn(), killed: false }, + adoptedFrom: undefined, + } as any; + const findSpy = vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue(ds); + const suspendSpy = vi.spyOn(workerPool, 'suspendWorker').mockReturnValue(true); + + handle = await startIpcServer({ port: 0, host: '127.0.0.1' }); + const res = await fetch(`http://127.0.0.1:${handle.port}/api/sessions/s-owned/suspend`, { method: 'POST' }); + + expect(res.status).toBe(409); + expect(await res.json()).toMatchObject({ ok: false, error: 'codex_app_dispatch_pending' }); + expect(suspendSpy).not.toHaveBeenCalled(); + findSpy.mockRestore(); + suspendSpy.mockRestore(); + }); + it('rejects adopt/observed sessions (suspending would kill the user pane)', async () => { const suspendSpy = vi.spyOn(workerPool, 'suspendWorker').mockReturnValue(true); const findSpy = vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue({ @@ -1410,6 +1560,196 @@ describe('PUT /api/bot-agent', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('rejects an unsettled Codex App session before config/readIsolation mutation or close', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-agent-pending-ipc-')); + const dataDir = join(dir, 'data'); + const configPath = join(dir, 'bots.json'); + const appId = 'test-agent-pending-app'; + const prevBotsConfig = process.env.BOTS_CONFIG; + const prevDataDir = config.session.dataDir; + try { + process.env.BOTS_CONFIG = configPath; + config.session.dataDir = dataDir; + writeFileSync(configPath, JSON.stringify([{ + larkAppId: appId, + larkAppSecret: 'secret', + cliId: 'codex-app', + model: 'old-model', + readIsolation: true, + }], null, 2)); + loadBotConfigs().forEach((c: any) => registerBot(c)); + sessionStore.init(appId); + const session = sessionStore.createSession('oc_pending', 'om_pending', 'Pending', 'group'); + session.larkAppId = appId; + session.cliId = 'codex-app'; + session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-pending', turnId: 'turn-pending', + state: 'prepared', content: 'prompt', deliverySink: 'lark', + }]; + sessionStore.updateSession(session); + const send = vi.fn(); + const registry = new Map([[sessionKey(session.rootMessageId, appId), { + session, + worker: { killed: false, send }, + workerPort: 1, + workerToken: 'token', + larkAppId: appId, + chatId: session.chatId, + chatType: 'group', + scope: 'thread', + spawnedAt: Date.now(), + cliVersion: 'test', + lastMessageAt: Date.now(), + hasHistory: true, + } as any]]); + workerPool.setActiveSessionsRegistry(registry); + setLarkAppId(appId); + handle = await startIpcServer({ port: 0, host: '127.0.0.1' }); + + const beforeFile = readFileSync(configPath, 'utf8'); + const beforeLive = structuredClone(getBot(appId).config); + const res = await fetch(`http://127.0.0.1:${handle.port}/api/bot-agent`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cliId: 'ttadk-x-codex', model: 'new-model' }), + }); + + expect(res.status).toBe(409); + expect(await res.json()).toEqual({ + ok: false, + error: 'codex_app_dispatch_pending', + blockingSessions: [{ + sessionId: session.sessionId, + cliId: 'codex-app', + reasons: ['codex_app_dispatch'], + }], + }); + expect(readFileSync(configPath, 'utf8')).toBe(beforeFile); + expect(getBot(appId).config).toEqual(beforeLive); + expect(sessionStore.getSession(session.sessionId)).toMatchObject({ + status: 'active', + codexAppDispatchLedger: [{ dispatchId: 'dispatch-pending' }], + }); + expect(registry.has(sessionKey(session.rootMessageId, appId))).toBe(true); + expect(send).not.toHaveBeenCalled(); + } finally { + workerPool.setActiveSessionsRegistry(new Map()); + sessionStore.init(); + config.session.dataDir = prevDataDir; + if (prevBotsConfig === undefined) delete process.env.BOTS_CONFIG; + else process.env.BOTS_CONFIG = prevBotsConfig; + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reports non-Codex pending work with a backend-neutral error and actionable sessions', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-agent-generic-pending-ipc-')); + const dataDir = join(dir, 'data'); + const configPath = join(dir, 'bots.json'); + const appId = 'test-agent-generic-pending-app'; + const prevBotsConfig = process.env.BOTS_CONFIG; + const prevDataDir = config.session.dataDir; + try { + process.env.BOTS_CONFIG = configPath; + config.session.dataDir = dataDir; + writeFileSync(configPath, JSON.stringify([{ + larkAppId: appId, + larkAppSecret: 'secret', + cliId: 'traex', + }], null, 2)); + loadBotConfigs().forEach((c: any) => registerBot(c)); + sessionStore.init(appId); + const session = sessionStore.createSession( + 'oc_generic_pending', + 'om_generic_pending', + 'Generic pending', + 'group', + ); + session.larkAppId = appId; + session.cliId = 'traex'; + session.queued = true; + session.pendingRepoSetup = { mode: 'picker', prompt: 'OPENING_N' }; + sessionStore.updateSession(session); + setLarkAppId(appId); + handle = await startIpcServer({ port: 0, host: '127.0.0.1' }); + + const res = await fetch(`http://127.0.0.1:${handle.port}/api/bot-agent`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cliId: 'codex', model: '' }), + }); + + expect(res.status).toBe(409); + expect(await res.json()).toEqual({ + ok: false, + error: 'session_mutation_pending', + blockingSessions: [{ + sessionId: session.sessionId, + cliId: 'traex', + reasons: ['queued_todo', 'repository_setup'], + }], + }); + expect(JSON.parse(readFileSync(configPath, 'utf8'))[0].cliId).toBe('traex'); + } finally { + workerPool.setActiveSessionsRegistry(new Map()); + sessionStore.init(); + config.session.dataDir = prevDataDir; + if (prevBotsConfig === undefined) delete process.env.BOTS_CONFIG; + else process.env.BOTS_CONFIG = prevBotsConfig; + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps the hot-switch mismatch close after a settled Codex App ledger', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-agent-settled-ipc-')); + const dataDir = join(dir, 'data'); + const configPath = join(dir, 'bots.json'); + const appId = 'test-agent-settled-app'; + const prevBotsConfig = process.env.BOTS_CONFIG; + const prevDataDir = config.session.dataDir; + try { + process.env.BOTS_CONFIG = configPath; + config.session.dataDir = dataDir; + writeFileSync(configPath, JSON.stringify([{ + larkAppId: appId, larkAppSecret: 'secret', cliId: 'codex-app', + }], null, 2)); + loadBotConfigs().forEach((c: any) => registerBot(c)); + sessionStore.init(appId); + const session = sessionStore.createSession('oc_settled', 'om_settled', 'Settled', 'group'); + session.larkAppId = appId; + session.cliId = 'codex-app'; + session.codexAppDispatchLedger = []; + sessionStore.updateSession(session); + const registry = new Map([[sessionKey(session.rootMessageId, appId), { + session, worker: null, workerPort: null, workerToken: null, + larkAppId: appId, chatId: session.chatId, chatType: 'group', scope: 'thread', + spawnedAt: Date.now(), cliVersion: 'test', lastMessageAt: Date.now(), + hasHistory: true, + } as any]]); + workerPool.setActiveSessionsRegistry(registry); + setLarkAppId(appId); + handle = await startIpcServer({ port: 0, host: '127.0.0.1' }); + + const res = await fetch(`http://127.0.0.1:${handle.port}/api/bot-agent`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cliId: 'codex', model: '' }), + }); + + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ ok: true, closedMismatchedSessions: 1 }); + expect(sessionStore.getSession(session.sessionId)?.status).toBe('closed'); + expect(registry.has(sessionKey(session.rootMessageId, appId))).toBe(false); + } finally { + workerPool.setActiveSessionsRegistry(new Map()); + sessionStore.init(); + config.session.dataDir = prevDataDir; + if (prevBotsConfig === undefined) delete process.env.BOTS_CONFIG; + else process.env.BOTS_CONFIG = prevBotsConfig; + rmSync(dir, { recursive: true, force: true }); + } + }); }); describe('PUT /api/bot-riff config safety (finding H)', () => { diff --git a/test/dashboard-sessions-ui.test.ts b/test/dashboard-sessions-ui.test.ts index 1cb3974b3..79c45a661 100644 --- a/test/dashboard-sessions-ui.test.ts +++ b/test/dashboard-sessions-ui.test.ts @@ -6,6 +6,8 @@ import { SessionsKanbanView, type SessionsKanbanCallbacks, type SessionsKanbanSt import { canRestartSession, CLI_FILTER_OPTIONS, + SESSION_STATUS_OPTIONS, + deriveSessionBoardColumn, groupSessionsByTopic, isUnknownChatSession, restartConfirmMessage, @@ -271,6 +273,11 @@ describe('dashboard sessions filters', () => { expect((html.match(/
{ // ─── Imports (must be after mocks) ────────────────────────────────────────── import { __resetAnchorQueues } from '../src/utils/anchor-serializer.js'; -import { __resetEventClaimsForTest, canOperate, canTalk, decideRouting, ensureBotOpenId, isBotMentioned, mentionsAnotherMember, markForwardFollowupsSessionsReady, startLarkEventDispatcher, writeBotInfoFile, type EventHandlers } from '../src/im/lark/event-dispatcher.js'; +import { __resetEventClaimsForTest, canOperate, canTalk, decideRouting, ensureBotOpenId, isBotMentioned, mentionsAnotherMember, markForwardFollowupsSessionsReady, rawMessageIngressAnchor, startLarkEventDispatcher, writeBotInfoFile, type EventHandlers } from '../src/im/lark/event-dispatcher.js'; import { VC_BOT_MEETING_ACTIVITY_EVENT, VC_BOT_MEETING_ENDED_EVENT, @@ -6086,6 +6086,213 @@ describe('im.message.receive_v1 — ack-safe duplicate delivery', () => { await flushEventWork(); }); + it('reserves same-chat arrival order before the first async routing lookup', async () => { + let releaseFirstMode!: (mode: 'group') => void; + const firstMode = new Promise<'group'>(resolve => { releaseFirstMode = resolve; }); + mockGetChatMode.mockReset() + .mockImplementationOnce(async () => firstMode) + .mockResolvedValue('group'); + + const handled: string[] = []; + handlers.handleNewTopic.mockImplementation(async (data: any) => { + handled.push(data.message.message_id); + }); + const event = (messageId: string) => makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: `@BotA ${messageId}` }), + messageId, + chatId: 'chat-ingress-order', + chatType: 'group', + mentions: [{ key: '@_bot_a', name: 'BotA', id: { open_id: MY_OPEN_ID } }], + }); + + capturedHandlers['im.message.receive_v1'](event('msg-ingress-n')); + capturedHandlers['im.message.receive_v1'](event('msg-ingress-n-plus-1')); + await new Promise(resolve => setImmediate(resolve)); + await Promise.resolve(); + + // N is blocked inside decideRouting. N+1 must not even enter that lookup; + // ordering at the later canonical anchor is already too late. + expect(mockGetChatMode).toHaveBeenCalledTimes(1); + expect(handled).toEqual([]); + + releaseFirstMode('group'); + await flushEventWork(); + await flushEventWork(); + expect(handled).toEqual(['msg-ingress-n', 'msg-ingress-n-plus-1']); + }); + + it('orders a topic seed before its thread-shaped reply across raw topology', async () => { + let releaseSeedMode!: (mode: 'topic') => void; + const seedMode = new Promise<'topic'>(resolve => { releaseSeedMode = resolve; }); + mockGetChatMode.mockReset() + .mockImplementationOnce(async () => seedMode) + .mockResolvedValue('topic'); + + const handled: string[] = []; + handlers.handleNewTopic.mockImplementation(async (data: any) => { + handled.push(data.message.message_id); + }); + handlers.handleThreadReply.mockImplementation(async (data: any) => { + handled.push(data.message.message_id); + }); + const mentions = [{ key: '@_bot_a', name: 'BotA', id: { open_id: MY_OPEN_ID } }]; + const seed = makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: '@BotA seed' }), + messageId: 'msg-topic-seed', + chatId: 'chat-topic-ingress', + chatType: 'group', + mentions, + }); + const reply = makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: '@BotA reply' }), + messageId: 'msg-topic-reply', + rootId: 'msg-topic-seed', + threadId: 'omt-topic-thread', + chatId: 'chat-topic-ingress', + chatType: 'group', + mentions, + }); + + capturedHandlers['im.message.receive_v1'](seed); + capturedHandlers['im.message.receive_v1'](reply); + await new Promise(resolve => setImmediate(resolve)); + await Promise.resolve(); + + // A thread-specific raw lane lets the reply bypass the seed here even + // though both later canonicalize to anchor=msg-topic-seed. + expect(handled).toEqual([]); + + releaseSeedMode('topic'); + await flushEventWork(); + await flushEventWork(); + expect(handled).toEqual(['msg-topic-seed', 'msg-topic-reply']); + }); + + it('bounds a reply wait when the earlier seed routing lookup is wedged', async () => { + vi.useFakeTimers(); + try { + capturedHandlers = {}; + __resetAnchorQueues(); + __resetEventClaimsForTest(); + config.daemon.forwardFollowupWaitMs = 25; + let releaseSeedMode!: (mode: 'topic') => void; + const seedMode = new Promise<'topic'>(resolve => { releaseSeedMode = resolve; }); + mockGetChatMode.mockReset() + .mockImplementationOnce(async () => seedMode) + .mockResolvedValue('topic'); + startLarkEventDispatcher(MY_APP_ID, 'secret', handlers); + + const handled: string[] = []; + handlers.handleNewTopic.mockImplementation(async data => { + handled.push(data.message.message_id); + }); + handlers.handleThreadReply.mockImplementation(async data => { + handled.push(data.message.message_id); + }); + const mentions = [{ key: '@_bot_a', name: 'BotA', id: { open_id: MY_OPEN_ID } }]; + const seed = makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: '@BotA seed' }), + messageId: 'msg-wedged-seed', + chatId: 'chat-wedged-seed', + chatType: 'group', + mentions, + }); + const reply = makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: '@BotA reply' }), + messageId: 'msg-wedged-reply', + rootId: 'msg-wedged-seed', + threadId: 'omt-wedged', + chatId: 'chat-wedged-seed', + chatType: 'group', + mentions, + }); + + capturedHandlers['im.message.receive_v1'](seed); + capturedHandlers['im.message.receive_v1'](reply); + await vi.advanceTimersByTimeAsync(5_030); + expect(handled).toContain('msg-wedged-reply'); + + releaseSeedMode('topic'); + await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); + expect(handled).toContain('msg-wedged-seed'); + } finally { + vi.useRealTimers(); + } + }); + + it('releases the chat routing barrier after enqueue so distinct topics still execute concurrently', async () => { + mockGetChatMode.mockReset().mockResolvedValue('topic'); + let releaseFirst!: () => void; + const firstPending = new Promise(resolve => { releaseFirst = resolve; }); + const started: string[] = []; + handlers.handleNewTopic.mockImplementation(async (data: any) => { + started.push(data.message.message_id); + if (data.message.message_id === 'msg-independent-topic-1') await firstPending; + }); + const event = (messageId: string) => makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: `@BotA ${messageId}` }), + messageId, + chatId: 'chat-independent-topics', + chatType: 'group', + mentions: [{ key: '@_bot_a', name: 'BotA', id: { open_id: MY_OPEN_ID } }], + }); + + capturedHandlers['im.message.receive_v1'](event('msg-independent-topic-1')); + capturedHandlers['im.message.receive_v1'](event('msg-independent-topic-2')); + await flushEventWork(); + + expect(started).toEqual(['msg-independent-topic-1', 'msg-independent-topic-2']); + releaseFirst(); + await flushEventWork(); + }); + + it('keeps the canonical session FIFO strict past five seconds', async () => { + vi.useFakeTimers(); + try { + mockGetChatMode.mockReset().mockResolvedValue('group'); + let releaseFirst!: () => void; + const firstPending = new Promise(resolve => { releaseFirst = resolve; }); + const started: string[] = []; + handlers.handleNewTopic.mockImplementation(async (data: any) => { + started.push(data.message.message_id); + if (data.message.message_id === 'msg-canonical-strict-1') await firstPending; + }); + const event = (messageId: string) => makeUserMessageEvent({ + senderOpenId: USER_OPEN_ID, + content: JSON.stringify({ text: `@BotA ${messageId}` }), + messageId, + chatId: 'chat-canonical-strict', + chatType: 'group', + mentions: [{ key: '@_bot_a', name: 'BotA', id: { open_id: MY_OPEN_ID } }], + }); + + capturedHandlers['im.message.receive_v1'](event('msg-canonical-strict-1')); + capturedHandlers['im.message.receive_v1'](event('msg-canonical-strict-2')); + await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); + await Promise.resolve(); + expect(started).toEqual(['msg-canonical-strict-1']); + + await vi.advanceTimersByTimeAsync(5_100); + expect(started).toEqual(['msg-canonical-strict-1']); + + releaseFirst(); + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(started).toEqual(['msg-canonical-strict-1', 'msg-canonical-strict-2']); + } finally { + vi.useRealTimers(); + } + }); + it('dedupes timeout redelivery of the same message_id', async () => { const event = makeUserMessageEvent({ senderOpenId: USER_OPEN_ID, @@ -6125,6 +6332,49 @@ describe('im.message.receive_v1 — ack-safe duplicate delivery', () => { }); }); +describe('rawMessageIngressAnchor', () => { + it('keeps top-level and quote-bubble messages on the same chat lane', () => { + const topLevel = rawMessageIngressAnchor(MY_APP_ID, { chat_id: 'oc_same' }); + const quoteBubble = rawMessageIngressAnchor(MY_APP_ID, { + chat_id: 'oc_same', + root_id: 'om_quoted', + }); + expect(quoteBubble).toBe(topLevel); + }); + + it('uses one routing barrier per chat while isolating bot apps', () => { + const first = rawMessageIngressAnchor(MY_APP_ID, { + chat_id: 'oc_same', + thread_id: 'omt_first', + }); + const second = rawMessageIngressAnchor(MY_APP_ID, { + chat_id: 'oc_same', + thread_id: 'omt_second', + }); + const otherBot = rawMessageIngressAnchor(OTHER_BOT_APP_ID, { + chat_id: 'oc_same', + thread_id: 'omt_first', + }); + expect(second).toBe(first); + expect(otherBot).not.toBe(first); + }); + + it('folds thread-shaped p2p replies together when DM routing is chat-scope', () => { + setupBotState({ p2pMode: 'chat' }); + const first = rawMessageIngressAnchor(MY_APP_ID, { + chat_id: 'oc_dm', + chat_type: 'p2p', + thread_id: 'omt_first', + }); + const second = rawMessageIngressAnchor(MY_APP_ID, { + chat_id: 'oc_dm', + chat_type: 'p2p', + thread_id: 'omt_second', + }); + expect(second).toBe(first); + }); +}); + describe('writeBotInfoFile — multi-daemon merge', () => { beforeEach(() => { mockExistsSync.mockReturnValue(true); diff --git a/test/fixtures/card-action-events.ts b/test/fixtures/card-action-events.ts index a243f278f..38bd22f07 100644 --- a/test/fixtures/card-action-events.ts +++ b/test/fixtures/card-action-events.ts @@ -17,9 +17,21 @@ export function makeRestartEvent(rootId: string, operatorOpenId = 'ou_user') { }; } -export function makeCloseEvent(rootId: string, operatorOpenId = 'ou_user', visibility?: string) { +export function makeCloseEvent( + rootId: string, + operatorOpenId = 'ou_user', + visibility?: string, + sessionId?: string, +) { return { - action: { value: { action: 'close', root_id: rootId, ...(visibility ? { visibility } : {}) } }, + action: { + value: { + action: 'close', + root_id: rootId, + ...(visibility ? { visibility } : {}), + ...(sessionId ? { session_id: sessionId } : {}), + }, + }, operator: { open_id: operatorOpenId }, }; } diff --git a/test/fixtures/fake-codex-app-server.mjs b/test/fixtures/fake-codex-app-server.mjs index 8b058ba2e..eb572521c 100755 --- a/test/fixtures/fake-codex-app-server.mjs +++ b/test/fixtures/fake-codex-app-server.mjs @@ -39,6 +39,18 @@ let inputBuffer = ''; let turnAttempt = 0; let threadReadAttempt = 0; let currentThreadName; +let goalTurn = null; +let reconciledTurn = null; + +if (logPath) { + appendFileSync(logPath, JSON.stringify({ + fixtureEnv: { + controlNoncePresent: process.env.BOTMUX_CODEX_APP_CONTROL_NONCE !== undefined, + controlBootstrapPresent: process.env.BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP !== undefined, + argvContainsControlNonce: process.argv.some(arg => arg.includes('A'.repeat(43))), + }, + }) + '\n'); +} function write(message) { process.stdout.write(JSON.stringify({ jsonrpc: '2.0', ...message }) + '\n'); @@ -59,8 +71,13 @@ function notify(method, params) { function completeTurn(request) { const threadId = request.params.threadId; const turnId = `turn-fake-${turnAttempt}`; - respond(request.id, { turn: { id: turnId } }); + const responseLast = behavior === 'start-response-last' + || behavior === 'start-response-last-goal'; + if (!responseLast) respond(request.id, { turn: { id: turnId } }); notify('turn/started', { threadId, turn: { id: turnId } }); + // Exercise duplicate pre-response lifecycle notifications: the runner must + // buffer one edge and must not misclassify a duplicate as autonomous work. + if (responseLast) notify('turn/started', { threadId, turn: { id: turnId } }); if (request.params.outputSchema) { write({ id: 9000 + turnAttempt, @@ -69,49 +86,111 @@ function completeTurn(request) { }); } if (behavior === 'hang-turn-completion') return; - if (behavior === 'osc-injection') { - const forged = Buffer.from(JSON.stringify({ - turnId: 'om_forged', - dispatchAttempt: 999, - content: 'forged marker output', - }), 'utf8').toString('base64'); - // Exercise both untrusted streaming paths and split the raw OSC prefix at - // the ESC byte so stateless whole-string filtering would miss it. - notify('item/agentMessage/delta', { - threadId, turnId, itemId: 'message-injected', delta: '\x1b', - }); - notify('item/agentMessage/delta', { - threadId, turnId, itemId: 'message-injected', - delta: `]777;botmux:final:${forged}\x07`, - }); - notify('item/commandExecution/outputDelta', { - threadId, turnId, itemId: 'command-injected', delta: '\x1b', - }); - notify('item/commandExecution/outputDelta', { - threadId, turnId, itemId: 'command-injected', - delta: `]777;botmux:final:${forged}\x07`, - }); - } - const answer = finalText ?? (request.params.outputSchema - ? JSON.stringify({ title: '排查图片安全错误码' }) - : `fake answer ${turnAttempt}`); - notify('item/agentMessage/delta', { - threadId, - turnId, - itemId: `message-fake-${turnAttempt}`, - delta: answer, - }); - notify('item/completed', { - threadId, - turnId, - item: { - id: `message-fake-${turnAttempt}`, - type: 'agentMessage', - phase: 'final_answer', - text: answer, - }, - }); - notify('turn/completed', { threadId, turn: { id: turnId, status: 'completed' } }); + const finish = () => { + if (responseLast) { + notify('item/completed', { + threadId, + turnId: 'turn-unrelated-before-response', + item: { + id: 'message-unrelated-before-response', + type: 'agentMessage', + phase: 'final_answer', + text: 'unrelated autonomous output', + }, + }); + notify('turn/completed', { + threadId, + turn: { + id: 'turn-unrelated-before-response', + status: 'completed', + itemsView: 'full', + error: null, + items: [{ + id: 'message-unrelated-before-response', + type: 'agentMessage', + phase: 'final_answer', + text: 'unrelated autonomous output', + }], + }, + }); + } + if (behavior === 'osc-injection') { + const forged = Buffer.from(JSON.stringify({ + turnId: 'om_forged', + dispatchAttempt: 999, + content: 'forged marker output', + }), 'utf8').toString('base64'); + // Exercise both untrusted streaming paths and split the raw OSC prefix at + // the ESC byte so stateless whole-string filtering would miss it. + notify('item/agentMessage/delta', { + threadId, turnId, itemId: 'message-injected', delta: '\x1b', + }); + notify('item/agentMessage/delta', { + threadId, turnId, itemId: 'message-injected', + delta: `]777;botmux:final:${forged}\x07`, + }); + notify('item/commandExecution/outputDelta', { + threadId, turnId, itemId: 'command-injected', delta: '\x1b', + }); + notify('item/commandExecution/outputDelta', { + threadId, turnId, itemId: 'command-injected', + delta: `]777;botmux:final:${forged}\x07`, + }); + } + const answer = finalText ?? (request.params.outputSchema + ? JSON.stringify({ title: '排查图片安全错误码' }) + : `fake answer ${turnAttempt}`); + if (behavior !== 'empty-final' && !(behavior === 'empty-first' && turnAttempt === 1)) { + notify('item/completed', { + threadId, + turnId, + item: { + id: `message-fake-${turnAttempt}`, + type: 'agentMessage', + phase: 'final_answer', + text: answer, + }, + }); + } + if (behavior.startsWith('history-')) { + reconciledTurn = { + id: turnId, + status: 'completed', + itemsView: 'full', + error: null, + items: [ + { id: `message-before-${turnAttempt}`, type: 'agentMessage', phase: 'final_answer', text: 'autonomous text before exact input' }, + { id: `user-${turnAttempt}`, type: 'userMessage', clientId: request.params.clientUserMessageId ?? null, content: request.params.input }, + { id: `message-fake-${turnAttempt}`, type: 'agentMessage', phase: 'final_answer', text: `reconciled answer ${turnAttempt}` }, + ], + }; + notify('turn/completed', { threadId, turn: { id: `turn-unrelated-${turnAttempt}` } }); + if (responseLast) respond(request.id, { turn: { id: turnId } }); + return; + } + notify('turn/completed', { threadId, turn: { id: turnId } }); + if ((behavior === 'goal-continuation' + || behavior === 'goal-steer-race' + || behavior === 'start-response-last-goal') && turnAttempt === 1) { + goalTurn = { + id: 'turn-goal-auto', + threadId, + items: [{ + id: 'message-goal-before-input', + type: 'agentMessage', + phase: 'final_answer', + text: 'autonomous goal text before Lark input', + }], + }; + notify('turn/started', { + threadId, + turn: { id: goalTurn.id, status: 'inProgress', itemsView: 'full', items: goalTurn.items }, + }); + } + if (responseLast) respond(request.id, { turn: { id: turnId } }); + }; + if (behavior === 'delayed-first' && turnAttempt === 1) setTimeout(finish, 300); + else finish(); } function handle(request) { @@ -120,6 +199,7 @@ function handle(request) { if (typeof request.id !== 'number') return; if (request.method === 'initialize') { + if (behavior === 'hang-initialize') return; respond(request.id, { userAgent: 'fake-codex-app-server' }); return; } @@ -128,6 +208,11 @@ function handle(request) { return; } if (request.method === 'thread/resume') { + if (behavior === 'hang-resume') return; + if (behavior === 'resume-not-found') { + reject(request.id, -32001, `thread ${request.params.threadId} not found`); + return; + } respond(request.id, { thread: { id: request.params.threadId } }); return; } @@ -161,6 +246,67 @@ function handle(request) { respond(request.id, {}); return; } + if (request.method === 'thread/turns/list') { + const data = behavior === 'history-no-match' + ? [] + : behavior === 'history-multi-match' && reconciledTurn + ? [reconciledTurn, { ...reconciledTurn, id: `${reconciledTurn.id}-duplicate` }] + : reconciledTurn ? [reconciledTurn] : []; + respond(request.id, { + data, + nextCursor: null, + backwardsCursor: null, + }); + return; + } + if (request.method === 'turn/steer') { + if (!goalTurn || request.params.expectedTurnId !== goalTurn.id) { + reject(request.id, -32000, 'expected turn is not active'); + return; + } + if (behavior === 'goal-steer-race') { + const completedGoal = goalTurn; + goalTurn = null; + notify('turn/completed', { + threadId: completedGoal.threadId, + turn: { + id: completedGoal.id, + status: 'completed', + itemsView: 'full', + error: null, + items: completedGoal.items, + }, + }); + reject(request.id, -32000, 'expected turn is not active'); + return; + } + respond(request.id, { turnId: goalTurn.id }); + const user = { + id: 'user-goal-steer', + type: 'userMessage', + clientId: request.params.clientUserMessageId ?? null, + content: request.params.input, + }; + const answer = { + id: 'message-goal-steer', + type: 'agentMessage', + phase: 'final_answer', + text: 'goal steer answer', + }; + notify('item/completed', { threadId: goalTurn.threadId, turnId: goalTurn.id, item: answer }); + notify('turn/completed', { + threadId: goalTurn.threadId, + turn: { + id: goalTurn.id, + status: 'completed', + itemsView: 'full', + error: null, + items: [...goalTurn.items, user, answer], + }, + }); + goalTurn = null; + return; + } if (request.method !== 'turn/start') { respond(request.id, {}); return; diff --git a/test/herdr-backend.test.ts b/test/herdr-backend.test.ts index 2c4e1728e..9ae4e158f 100644 --- a/test/herdr-backend.test.ts +++ b/test/herdr-backend.test.ts @@ -547,6 +547,21 @@ describe('HerdrBackend.spawn', () => { const be = new HerdrBackend(SESSION, { isReattach: true }); be.spawn('claude', [], { cwd: '/work', cols: 80, rows: 24, env: {} }); expect(herdrCall('agent', 'start', 'botmux')).toBeUndefined(); + expect(be.isReattach).toBe(true); + be.kill(); + }); + + it('reports actual fresh start when a predicted reattach has no reusable agent', () => { + setHerdrResponses([ + { match: a => a[0] === 'session' && a[1] === 'list', reply: () => EXISTING_SESSION_REPLY }, + { match: a => a.includes('agent') && a.includes('get'), reply: () => JSON.stringify({ result: {} }) }, + { match: a => a.includes('agent') && a.includes('start'), reply: () => AGENT_GET_REPLY('fresh-2') }, + { match: a => a.includes('read') && (a.includes('agent') || a.includes('pane')), reply: () => PANE_READ_REPLY('') }, + ]); + const be = new HerdrBackend(SESSION, { isReattach: true }); + be.spawn('claude', [], { cwd: '/work', cols: 80, rows: 24, env: {} }); + expect(herdrCall('agent', 'start', 'botmux')).toBeDefined(); + expect(be.isReattach).toBe(false); be.kill(); }); @@ -898,11 +913,21 @@ describe('HerdrBackend message writing', () => { be.kill(); }); + it('reports a rejected pane command instead of claiming raw input acceptance', () => { + const be = spawnBackend('5-5'); + mockedExecFileSync.mockImplementation(() => { + throw new Error('pane disappeared'); + }); + expect(be.sendText('/goal x')).toBe(false); + expect(be.sendSpecialKeys('Enter')).toBe(false); + be.kill(); + }); + it('write() is a no-op after kill()', () => { const be = spawnBackend('5-5'); be.kill(); mockedExecFileSync.mockClear(); - be.sendText('after-exit'); + expect(be.sendText('after-exit')).toBe(false); const call = herdrCall('pane', 'send-text'); expect(call).toBeUndefined(); }); diff --git a/test/idle-worker-sweeper.test.ts b/test/idle-worker-sweeper.test.ts index 665fb156a..9d6958e64 100644 --- a/test/idle-worker-sweeper.test.ts +++ b/test/idle-worker-sweeper.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('../src/services/session-store.js', () => ({ updateSessionPid: vi.fn(), @@ -16,7 +16,15 @@ vi.mock('../src/utils/logger.js', () => ({ }, })); -import { sweepIdleWorkers, DEFAULT_MAX_LIVE_WORKERS } from '../src/core/idle-worker-sweeper.js'; +import { + sweepIdleWorkers, + sweepIdleWorkersAfterTurnDrain, + DEFAULT_MAX_LIVE_WORKERS, +} from '../src/core/idle-worker-sweeper.js'; +import { + __testOnly_resetBotTurnMutationGates, + withBotTurnAdmission, +} from '../src/core/bot-turn-mutation-gate.js'; function ds(sessionId: string, backendType: string, lastMessageAt: number, worker = {}) { return { @@ -42,6 +50,10 @@ function ds(sessionId: string, backendType: string, lastMessageAt: number, worke const now = 1_000_000; describe('sweepIdleWorkers (per-bot count cap)', () => { + beforeEach(() => { + __testOnly_resetBotTurnMutationGates(); + }); + it('falls back to the default cap (30) when the bot has no explicit value', () => { expect(DEFAULT_MAX_LIVE_WORKERS).toBe(30); // DEFAULT_MAX_LIVE_WORKERS + 2 sessions, oldest first by lastMessageAt. @@ -102,6 +114,24 @@ describe('sweepIdleWorkers (per-bot count cap)', () => { expect(activeSessions.get('f').worker).not.toBe(null); }); + it('skips an idle session with durable Codex App dispatch ownership and suspends the next candidate', () => { + const owned = ds('a', 'tmux', now - 90 * 60_000); + owned.session.codexAppDispatchLedger = [ + { dispatchId: 'd-1', turnId: 't-1', state: 'accepted', content: 'owned' }, + ]; + const activeSessions = new Map([ + ['a', owned], + ['b', ds('b', 'tmux', now - 80 * 60_000)], + ['c', ds('c', 'tmux', now - 70 * 60_000)], + ]); + + const suspended = sweepIdleWorkers(activeSessions, { maxLiveWorkers: 2 }); + + expect(suspended.map(entry => entry.sessionId)).toEqual(['b']); + expect(activeSessions.get('a').worker).not.toBe(null); + expect(activeSessions.get('b').worker).toBe(null); + }); + it('is purely count-based: suspends a recently-active session with NO idle-time threshold', () => { // Both sessions are only a couple minutes idle. The old budget had a 30-min // idle gate that would have suspended nothing here; the new policy caps by @@ -188,4 +218,68 @@ describe('sweepIdleWorkers (per-bot count cap)', () => { expect(suspended).toEqual([]); expect(activeSessions.get('a').worker).not.toBe(null); }); + + it('waits for pre-accept inbound turns before choosing an idle cap victim', async () => { + const appId = 'cli_a'; + const a = ds('a', 'tmux', now - 90 * 60_000); + const b = ds('b', 'tmux', now - 10 * 60_000); + const activeSessions = new Map([['a', a], ['b', b]]); + let releaseAdmission!: () => void; + const pausedBeforeAccept = new Promise(resolve => { releaseAdmission = resolve; }); + let admissionStarted!: () => void; + const started = new Promise(resolve => { admissionStarted = resolve; }); + + // Session A has entered the existing-owner message path but is still + // waiting on sender/reaction work, so its old screen snapshot says idle and + // its durable dispatch ledger is not populated yet. + const inboundA = withBotTurnAdmission(appId, async () => { + admissionStarted(); + await pausedBeforeAccept; + // Durable acceptance occurs before the admission is released. + a.session.codexAppDispatchLedger = [ + { dispatchId: 'd-a', turnId: 't-a', state: 'accepted', content: 'message-a' }, + ]; + }); + await started; + + // Spawning B puts the bot over cap. The cap sweep must drain A's + // admission rather than synchronously suspending A from its stale snapshot. + const sweep = sweepIdleWorkersAfterTurnDrain(appId, activeSessions, { maxLiveWorkers: 1 }); + await Promise.resolve(); + expect(a.worker).not.toBe(null); + + releaseAdmission(); + await inboundA; + const suspended = await sweep; + + expect(suspended.map(entry => entry.sessionId)).toEqual(['b']); + expect(a.worker).not.toBe(null); + expect(b.worker).toBe(null); + }); + + it('skips a sweep after a bounded wait instead of freezing the bot mutation gate', async () => { + const appId = 'cli_wedged'; + const activeSessions = new Map([ + ['a', ds('a', 'tmux', now - 90 * 60_000)], + ['b', ds('b', 'tmux', now - 10 * 60_000)], + ]); + let releaseAdmission!: () => void; + let admissionStarted!: () => void; + const started = new Promise(resolve => { admissionStarted = resolve; }); + const admission = withBotTurnAdmission(appId, async () => { + admissionStarted(); + await new Promise(resolve => { releaseAdmission = resolve; }); + }); + await started; + + await expect(sweepIdleWorkersAfterTurnDrain(appId, activeSessions, { + maxLiveWorkers: 1, + mutationAcquireTimeoutMs: 5, + })).resolves.toEqual([]); + expect(activeSessions.get('a').worker).not.toBe(null); + + releaseAdmission(); + await admission; + await expect(withBotTurnAdmission(appId, async () => 'open')).resolves.toBe('open'); + }); }); diff --git a/test/kill-worker-orphaned-backend.test.ts b/test/kill-worker-orphaned-backend.test.ts index fbdcf124e..c7e017b98 100644 --- a/test/kill-worker-orphaned-backend.test.ts +++ b/test/kill-worker-orphaned-backend.test.ts @@ -18,7 +18,13 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { DaemonSession } from '../src/core/types.js'; -const { tmuxKill, herdrKill, herdrKillAgent, zellijKill, getBotMock } = vi.hoisted(() => ({ +const { + tmuxKill, + herdrKill, + herdrKillAgent, + zellijKill, + getBotMock, +} = vi.hoisted(() => ({ tmuxKill: vi.fn(), herdrKill: vi.fn(), herdrKillAgent: vi.fn(), @@ -42,6 +48,7 @@ vi.mock('../src/adapters/backend/zellij-backend.js', () => ({ vi.mock('../src/bot-registry.js', () => ({ getBot: getBotMock, + getBotBrand: vi.fn(() => 'feishu'), getAllBots: vi.fn(() => []), resolveBrandLabel: vi.fn(() => undefined), })); @@ -58,6 +65,7 @@ vi.mock('../src/im/lark/client.js', () => ({ vi.mock('../src/services/frozen-card-store.js', () => ({ loadFrozenCards: vi.fn(() => new Map()), saveFrozenCards: vi.fn(), + deleteFrozenCards: vi.fn(), })); vi.mock('../src/utils/logger.js', () => ({ @@ -168,4 +176,5 @@ describe('killWorker — with a live worker (unchanged path)', () => { expect(d.worker).toBeNull(); expect(d.managedTurnOrigin).toBeUndefined(); }); + }); diff --git a/test/overview-card.test.ts b/test/overview-card.test.ts index 04bcb0bc7..7934aeeba 100644 --- a/test/overview-card.test.ts +++ b/test/overview-card.test.ts @@ -130,6 +130,12 @@ describe('buildOverviewCard', () => { ])).toEqual({ active: 1, idle: 1, closed: 1 }); }); + it('does not count a stalled turn as idle in the overview', () => { + expect(countSessions([ + sessionRow({ sessionId: 's1', status: 'stalled' }), + ])).toEqual({ active: 1, idle: 0, closed: 0 }); + }); + it('zh overview localizes all module sections and folder buttons', () => { const json = buildOverviewCard( { sessions: [], schedules: [], settings: makeSettings() }, diff --git a/test/pending-repo-journal.test.ts b/test/pending-repo-journal.test.ts new file mode 100644 index 000000000..22516495b --- /dev/null +++ b/test/pending-repo-journal.test.ts @@ -0,0 +1,163 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { DaemonSession } from '../src/core/types.js'; +import type { Session } from '../src/types.js'; + +vi.mock('../src/services/session-store.js', () => ({ + updateSession: vi.fn(), +})); + +import { + persistPendingRepoCardMessageId, + restorePendingRepoRuntime, + stagePendingRepoSetup, +} from '../src/core/pending-repo-journal.js'; +import * as sessionStore from '../src/services/session-store.js'; + +function makeSession(): Session { + return { + sessionId: 'pending-repo-session', + chatId: 'oc_chat', + rootMessageId: 'om_root', + title: 'durable setup', + status: 'active', + createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + }; +} + +function makeDs(): DaemonSession { + return { + session: makeSession(), + worker: null, + workerPort: null, + workerToken: null, + larkAppId: 'app_test', + chatId: 'oc_chat', + chatType: 'group', + scope: 'thread', + spawnedAt: 1, + cliVersion: 'test', + lastMessageAt: 1, + hasHistory: false, + initialStartPending: true, + pendingPrompt: 'OPENING_N', + pendingRawInput: '/goal exact raw', + pendingCodexAppText: 'visible opening', + pendingCodexAppApplicationContext: 'app', + pendingCodexAppMessageContext: 'message', + pendingAttachments: [{ type: 'file', path: '/tmp/spec.md', name: 'spec.md' }], + pendingMentions: [{ key: '@_user_1', name: '晓雪', openId: 'ou_owner' }], + pendingSubstituteTrigger: { + target: { name: 'Reviewer', openId: 'ou_reviewer' }, + observedMention: { name: 'Reviewer' }, + disclosure: 'prefix', + }, + pendingSender: { openId: 'ou_owner', type: 'user', name: '晓雪' }, + } as DaemonSession; +} + +beforeEach(() => { + vi.mocked(sessionStore.updateSession).mockReset(); +}); + +describe('pending repository setup journal', () => { + it('atomically captures the complete opening before a picker/worktree can be published', () => { + const ds = makeDs(); + + stagePendingRepoSetup(ds, { + mode: 'auto_worktree', baseDir: '/repos/base', turnId: 'turn-n', + }); + + expect(ds.session.queued).toBe(true); + expect(ds.session.queuedPrompt).toBe('OPENING_N'); + expect(ds.session.queuedCodexAppText).toBe('visible opening'); + expect(ds.session.queuedCodexAppMessageContext).toBe('message'); + expect(ds.session.pendingRepoSetup).toEqual({ + mode: 'auto_worktree', + prompt: 'OPENING_N', + rawInput: '/goal exact raw', + turnId: 'turn-n', + baseDir: '/repos/base', + codexAppText: 'visible opening', + codexAppApplicationContext: 'app', + codexAppMessageContext: 'message', + attachments: [{ type: 'file', path: '/tmp/spec.md', name: 'spec.md' }], + mentions: [{ key: '@_user_1', name: '晓雪', openId: 'ou_owner' }], + substituteTrigger: { + target: { name: 'Reviewer', openId: 'ou_reviewer' }, + observedMention: { name: 'Reviewer' }, + disclosure: 'prefix', + }, + sender: { openId: 'ou_owner', type: 'user', name: '晓雪' }, + }); + expect(sessionStore.updateSession).toHaveBeenCalledWith(ds.session); + }); + + it('rolls every durable field back when setup persistence fails', () => { + const ds = makeDs(); + const oldSetup = { mode: 'picker' as const, prompt: 'OLD' }; + Object.assign(ds.session, { + queued: false, + queuedPrompt: 'old prompt', + queuedCodexAppText: 'old text', + queuedCodexAppMessageContext: 'old context', + pendingRepoSetup: oldSetup, + }); + vi.mocked(sessionStore.updateSession).mockImplementationOnce(() => { + throw new Error('disk unavailable'); + }); + + expect(() => stagePendingRepoSetup(ds, { mode: 'picker' })).toThrow('disk unavailable'); + expect(ds.session).toMatchObject({ + queued: false, + queuedPrompt: 'old prompt', + queuedCodexAppText: 'old text', + queuedCodexAppMessageContext: 'old context', + pendingRepoSetup: oldSetup, + }); + }); + + it('persists card identity transactionally and reconstructs isolated runtime buffers', () => { + const ds = makeDs(); + stagePendingRepoSetup(ds, { mode: 'picker', turnId: 'turn-n' }); + persistPendingRepoCardMessageId(ds, 'om_picker'); + + const restored = { + ...makeDs(), + session: structuredClone(ds.session), + pendingPrompt: undefined, + pendingRawInput: undefined, + pendingAttachments: undefined, + pendingMentions: undefined, + pendingSubstituteTrigger: undefined, + pendingSender: undefined, + initialStartPending: true, + } as DaemonSession; + expect(restorePendingRepoRuntime(restored)).toBe(true); + expect(restored).toMatchObject({ + pendingRepo: true, + pendingPrompt: 'OPENING_N', + pendingRawInput: '/goal exact raw', + pendingCodexAppText: 'visible opening', + pendingCodexAppApplicationContext: 'app', + pendingCodexAppMessageContext: 'message', + repoCardMessageId: 'om_picker', + initialStartPending: false, + }); + expect(restored.pendingAttachments).toEqual(ds.pendingAttachments); + expect(restored.pendingAttachments).not.toBe(ds.session.pendingRepoSetup?.attachments); + expect(restored.pendingMentions).toEqual(ds.pendingMentions); + expect(restored.pendingMentions).not.toBe(ds.session.pendingRepoSetup?.mentions); + }); + + it('restores the previous card id if its persistence fails', () => { + const ds = makeDs(); + stagePendingRepoSetup(ds, { mode: 'picker' }); + ds.session.pendingRepoSetup!.repoCardMessageId = 'om_old'; + vi.mocked(sessionStore.updateSession).mockImplementationOnce(() => { + throw new Error('card id write failed'); + }); + + expect(() => persistPendingRepoCardMessageId(ds, 'om_new')).toThrow('card id write failed'); + expect(ds.session.pendingRepoSetup?.repoCardMessageId).toBe('om_old'); + }); +}); diff --git a/test/raw-input-followup-atomicity.test.ts b/test/raw-input-followup-atomicity.test.ts index 7ae5f7722..c97758711 100644 --- a/test/raw-input-followup-atomicity.test.ts +++ b/test/raw-input-followup-atomicity.test.ts @@ -18,10 +18,15 @@ * Run: pnpm vitest run test/raw-input-followup-atomicity.test.ts */ import { readFileSync } from 'node:fs'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { + finalizeRawCommandDelivery, + writeRawCommandLine, +} from '../src/core/raw-command-writer.js'; const workerSrc = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf-8'); const poolSrc = readFileSync(new URL('../src/core/worker-pool.ts', import.meta.url), 'utf-8'); +const rawWriterSrc = readFileSync(new URL('../src/core/raw-command-writer.ts', import.meta.url), 'utf-8'); function caseRegion(src: string, marker: string, span = 3000): string { const start = src.indexOf(marker); @@ -74,7 +79,7 @@ describe('worker raw_input delivery', () => { }); it('routes the follow-up through sendToPty (normal busy-queue semantics)', () => { - expect(region).toContain('sendToPty(msg.followUpContent, msg.followUpTurnId, {'); + expect(region).toContain('sendToPty(msg.followUpContent!, msg.followUpTurnId, {'); expect(region).toContain('codexAppInput: msg.followUpCodexAppInput'); }); @@ -95,6 +100,16 @@ describe('worker raw_input delivery', () => { expect(region).not.toContain('if (!isPromptReady)'); expect(region).not.toContain('if (isPromptReady)'); }); + + it('ACKs a durable generic raw opening only after the awaited Enter succeeds', () => { + const sentIdx = region.indexOf('sent = await sendRawCommandLineSerially'); + const ackIdx = region.indexOf("type: 'queued_activation_submitted'", sentIdx); + expect(sentIdx).toBeGreaterThanOrEqual(0); + expect(ackIdx).toBeGreaterThan(sentIdx); + expect(region.slice(sentIdx, ackIdx)).toContain( + 'acknowledgeActivation: !!msg.queuedActivationToken', + ); + }); }); describe('worker command-line write mutex', () => { @@ -110,21 +125,21 @@ describe('worker command-line write mutex', () => { }); describe('worker sendRawCommandLine helper', () => { - const helper = caseRegion(workerSrc, 'async function sendRawCommandLine', 2200); + const helper = caseRegion(rawWriterSrc, 'export async function writeRawCommandLine', 2200); it('generic CLIs: literal text → 200ms beat → Enter in order (slash-picker safe)', () => { const textIdx = helper.indexOf('sendText(content)'); expect(textIdx).toBeGreaterThanOrEqual(0); // Anchor the beat/Enter lookups AFTER the text write so the CoCo branch's own // 200ms beat (which precedes the generic path) can't be mistaken for this one. - const beatIdx = helper.indexOf('setTimeout(r, 200)', textIdx); + const beatIdx = helper.indexOf('delay(beatMs)', textIdx); const enterIdx = helper.indexOf("sendSpecialKeys('Enter')", beatIdx); expect(beatIdx).toBeGreaterThan(textIdx); expect(enterIdx).toBeGreaterThan(beatIdx); }); it('CoCo: types char-by-char (throttled) before a single Enter (paste-coalescing safe)', () => { - const cocoIdx = helper.indexOf("cliId === 'coco'"); + const cocoIdx = helper.indexOf('opts.coco'); expect(cocoIdx, 'CoCo branch present').toBeGreaterThanOrEqual(0); const genericTextIdx = helper.indexOf('sendText(content)'); // The CoCo branch fully precedes the generic one-shot path. @@ -132,23 +147,79 @@ describe('worker sendRawCommandLine helper', () => { // Per-char keystrokes spaced by the throttle — a one-shot write coalesces into // a paste on CoCo, which skips command mode + the slash picker. const charIdx = helper.indexOf('sendText(ch)', cocoIdx); - const throttleIdx = helper.indexOf('COCO_SLASH_TYPE_THROTTLE_MS', cocoIdx); + const throttleIdx = helper.indexOf('opts.cocoThrottleMs', cocoIdx); expect(charIdx).toBeGreaterThan(cocoIdx); expect(charIdx).toBeLessThan(genericTextIdx); expect(throttleIdx).toBeGreaterThan(cocoIdx); // Exactly one Enter, after the beat (a stray 2nd Enter would confirm a /model // selector pick); the branch returns immediately after. const cocoEnterIdx = helper.indexOf("sendSpecialKeys('Enter')", throttleIdx); - const returnIdx = helper.indexOf('return;', throttleIdx); + const returnIdx = helper.indexOf("return sendSpecialKeys('Enter') !== false", throttleIdx); expect(cocoEnterIdx).toBeGreaterThan(throttleIdx); expect(cocoEnterIdx).toBeLessThan(genericTextIdx); - expect(returnIdx).toBeGreaterThan(cocoEnterIdx); + expect(returnIdx).toBeGreaterThan(throttleIdx); + expect(cocoEnterIdx).toBeGreaterThan(returnIdx); expect(returnIdx).toBeLessThan(genericTextIdx); }); }); +describe('raw command backend acceptance', () => { + const immediateDelay = vi.fn(async () => {}); + + it('fails closed when the text write is rejected', async () => { + const sendText = vi.fn(() => false); + const sendSpecialKeys = vi.fn(() => true); + await expect(writeRawCommandLine({ + write: vi.fn(), sendText, sendSpecialKeys, + }, '/goal x', { delay: immediateDelay })).resolves.toBe(false); + expect(sendSpecialKeys).not.toHaveBeenCalled(); + }); + + it('fails closed when Enter is rejected after accepted text', async () => { + const sendText = vi.fn(() => true); + const sendSpecialKeys = vi.fn(() => false); + await expect(writeRawCommandLine({ + write: vi.fn(), sendText, sendSpecialKeys, + }, '/goal x', { delay: immediateDelay })).resolves.toBe(false); + expect(sendSpecialKeys).toHaveBeenCalledWith('Enter'); + }); + + it('fails closed when a PTY-style backend disappears before either write', async () => { + const write = vi.fn(() => false); + await expect(writeRawCommandLine({ write }, '/goal x', { + delay: immediateDelay, + })).resolves.toBe(false); + expect(write).toHaveBeenCalledTimes(1); + }); + + it('does not ACK or enqueue a follower after rejected Enter and retires the durable generation', async () => { + const accepted = await writeRawCommandLine({ + write: vi.fn(), + sendText: vi.fn(() => true), + sendSpecialKeys: vi.fn(() => false), + }, '/goal x', { delay: immediateDelay }); + const onActivationAck = vi.fn(); + const onFollowUp = vi.fn(); + const onDurableFailure = vi.fn(); + + expect(finalizeRawCommandDelivery({ + accepted, + durableActivation: true, + acknowledgeActivation: true, + hasFollowUp: true, + onAccepted: vi.fn(), + onFollowUp, + onActivationAck, + onDurableFailure, + })).toBe(false); + expect(onActivationAck).not.toHaveBeenCalled(); + expect(onFollowUp).not.toHaveBeenCalled(); + expect(onDurableFailure).toHaveBeenCalledOnce(); + }); +}); + describe('daemon prompt_ready dispatch', () => { - const region = caseRegion(poolSrc, "case 'prompt_ready':", 2000); + const region = caseRegion(poolSrc, "case 'prompt_ready':", 5000); it('bundles the follow-up onto the raw_input IPC instead of a second message IPC', () => { expect(region).toContain('followUpContent: followUp?.cliInput'); diff --git a/test/reply-target-fallback.test.ts b/test/reply-target-fallback.test.ts index 3569ee284..e15238006 100644 --- a/test/reply-target-fallback.test.ts +++ b/test/reply-target-fallback.test.ts @@ -13,7 +13,14 @@ * Run: pnpm vitest run test/reply-target-fallback.test.ts */ import { describe, it, expect } from 'vitest'; -import { beginReplyTargetTurn, fallbackTurnId, isSubstituteTurn, pickTurnReplyTarget, resolveSessionReplyTarget } from '../src/core/reply-target.js'; +import { + beginReplyTargetTurn, + fallbackTurnId, + frozenReplyContextForTurn, + isSubstituteTurn, + pickTurnReplyTarget, + resolveSessionReplyTarget, +} from '../src/core/reply-target.js'; import type { DaemonSession } from '../src/core/types.js'; const NOW = new Date().toISOString(); @@ -196,3 +203,31 @@ describe('per-turn replyTargets — queued/concurrent turns keep their own ancho expect(resolveSessionReplyTarget(ds, 'turn-0')).toEqual({ mode: 'plain', chatId: 'oc_chat' }); }); }); + +describe('frozen reply context', () => { + it('keeps turn A root, quote, and sender after mutable state advances to B', () => { + const ds = makeDs() as DaemonSession; + ds.session.quoteTargetId = 'om_a'; + ds.session.quoteTargetSenderOpenId = 'ou_a'; + ds.session.quoteTargetSenderIsBot = false; + beginReplyTargetTurn(ds, 'om_root_a', 'om_a', NOW); + + ds.session.quoteTargetId = 'om_b'; + ds.session.quoteTargetSenderOpenId = 'ou_b'; + ds.session.quoteTargetSenderIsBot = true; + beginReplyTargetTurn(ds, 'om_root_b', 'om_b', NOW); + + expect(frozenReplyContextForTurn(ds, 'om_a')).toEqual({ + target: { mode: 'thread', rootMessageId: 'om_root_a' }, + quoteTargetId: 'om_a', + replyTargetSenderOpenId: 'ou_a', + replyTargetSenderIsBot: false, + }); + expect(frozenReplyContextForTurn(ds, 'om_b')).toEqual({ + target: { mode: 'thread', rootMessageId: 'om_root_b' }, + quoteTargetId: 'om_b', + replyTargetSenderOpenId: 'ou_b', + replyTargetSenderIsBot: true, + }); + }); +}); diff --git a/test/resource-monitor-runtime.test.ts b/test/resource-monitor-runtime.test.ts index b0f4d321e..3423e31b7 100644 --- a/test/resource-monitor-runtime.test.ts +++ b/test/resource-monitor-runtime.test.ts @@ -53,6 +53,7 @@ describe('runtime monitor helpers', () => { expect(sessionRuntimeBucket(session({ sessionId: 's7', status: 'analyzing' }))).toBe('working'); expect(sessionRuntimeBucket(session({ sessionId: 's8', status: 'active' }))).toBe('working'); expect(sessionRuntimeBucket(session({ sessionId: 's9', status: 'dormant' }))).toBe('idle'); + expect(sessionRuntimeBucket(session({ sessionId: 's10', status: 'stalled' }))).toBe('working'); }); it('builds fresh runtime summary with daemon and session pressure', () => { diff --git a/test/restore-zombie-close.test.ts b/test/restore-zombie-close.test.ts index 67ac6e7b4..b702b59ff 100644 --- a/test/restore-zombie-close.test.ts +++ b/test/restore-zombie-close.test.ts @@ -78,12 +78,46 @@ vi.mock('../src/core/worker-pool.js', () => ({ getActiveSessionsRegistry: vi.fn(() => wp.registry ?? undefined), getCurrentCliVersion: vi.fn(() => '1.0.0-test'), restoreUsageLimitRuntimeState: vi.fn(), + withActiveSessionKeyLock: vi.fn(async (_map: Map, _key: string, action: () => any) => action()), setActiveSessionSafe: vi.fn(async (map: Map, key: string, ds: any) => { const prev = map.get(key); if (prev && prev !== ds) { - for (const [k, v] of map) { if (v === prev) { map.delete(k); break; } } + const prevPending = (prev.session?.codexAppDispatchLedger?.length ?? 0) > 0; + const incomingPending = (ds.session?.codexAppDispatchLedger?.length ?? 0) > 0; + if (prevPending && incomingPending) { + return { + accepted: false, + reason: 'both_pending', + keptSessionId: prev.session.sessionId, + preservedIncomingSessionId: ds.session.sessionId, + }; + } + const closeUnregistered = async (loser: any) => { + for (const [k, v] of map) { + if (v === loser) { map.delete(k); break; } + } + const store = await import('../src/services/session-store.js'); + const persisted = store.getSession(loser.session.sessionId); + if (persisted && persisted.status !== 'closed') { + store.closeSession(loser.session.sessionId); + } + }; + if (prevPending) { + await closeUnregistered(ds); + return { + accepted: false, + reason: 'kept_pending_owner', + keptSessionId: prev.session.sessionId, + closedIncomingSessionId: ds.session.sessionId, + }; + } + await closeUnregistered(prev); } map.set(key, ds); + return { + accepted: true, + ...(prev && prev !== ds ? { closedSessionId: prev.session.sessionId } : {}), + }; }), isRelayableRealSession: (ds: any) => !!ds?.worker || !!ds?.session?.cliId || !!ds?.session?.lastCliInput, @@ -166,7 +200,7 @@ vi.mock('../src/core/session-activity.js', () => ({ markSessionActivity: vi.fn(), })); -import { restoreActiveSessions, closeCliMismatchedSessionsForBot } from '../src/core/session-manager.js'; +import { restoreActiveSessions, closeCliMismatchedSessionsForBot, resumeSession } from '../src/core/session-manager.js'; import { TmuxBackend } from '../src/adapters/backend/tmux-backend.js'; import { HerdrBackend } from '../src/adapters/backend/herdr-backend.js'; import { forkWorker, closeSession } from '../src/core/worker-pool.js'; @@ -295,6 +329,34 @@ describe('restoreActiveSessions — persistent-backend zombie-close decision', ( expect(forkWorker).not.toHaveBeenCalled(); }); + it('CLI mismatch on restore preserves and reattaches an unsettled Codex App ledger', async () => { + probe.result = 'exists'; + server.state = 'running'; + const s = makeActivePersistentSession('om_cli_mismatch_pending'); + s.cliId = 'codex-app'; + s.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-pending', + turnId: 'turn-pending', + state: 'prepared', + content: 'prompt', + deliverySink: 'lark', + }]; + sessionStore.updateSession(s); + const map = new Map(); + wp.registry = map; + + await restoreActiveSessions(map); + + expect(closeSession).not.toHaveBeenCalled(); + expect(sessionStore.getSession(s.sessionId)).toMatchObject({ + status: 'active', + codexAppDispatchLedger: [{ dispatchId: 'dispatch-pending' }], + }); + const restored = map.get(sessionKey('om_cli_mismatch_pending', 'app_test')); + expect(restored).toBeDefined(); + expect(forkWorker).toHaveBeenCalledWith(restored, '', true); + }); + it('wrapper mismatch on restore (same cliId) → closes the active record', async () => { // 'aiden x claude' and bare claude-code share cliId='claude-code' but are // distinct launch choices (selectionKeyForBot keys on cliId+wrapperCli). @@ -516,6 +578,85 @@ describe('restoreActiveSessions — persistent-backend zombie-close decision', ( expect(forkWorker).toHaveBeenCalledWith(restored, '', true); expect(sessionStore.getSession(s.sessionId)?.lastCodexAppInput).toEqual(expected); }); + + it('isolates a same-anchor pending collision without aborting startup and preserves both rows', async () => { + probe.result = 'exists'; + bot.cliId = 'codex-app'; + const first = makeActivePersistentSession('om_pending_collision'); + first.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-first', + turnId: 'turn-first', + state: 'prepared', + content: 'first owned output', + deliverySink: 'lark', + }]; + sessionStore.updateSession(first); + const second = makeActivePersistentSession('om_pending_collision'); + second.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-second', + turnId: 'turn-second', + state: 'prepared', + content: 'second owned output', + deliverySink: 'lark', + }]; + sessionStore.updateSession(second); + const map = new Map(); + wp.registry = map; + + await expect(restoreActiveSessions(map)).resolves.toBeUndefined(); + + expect(sessionStore.getSession(first.sessionId)?.status).toBe('active'); + expect(sessionStore.getSession(second.sessionId)?.status).toBe('active'); + expect(closeSession).not.toHaveBeenCalled(); + const canonical = map.get(sessionKey('om_pending_collision', 'app_test'))!; + expect(canonical.session.sessionId).toBe(first.sessionId); + expect(forkWorker).toHaveBeenCalledTimes(1); + expect(forkWorker).toHaveBeenCalledWith(canonical, '', true); + }); +}); + +describe('resumeSession — disk-only legacy anchor collision', () => { + it('refuses a legacy unscoped real owner instead of ghosting it', async () => { + const target = makeActivePersistentSession('om_resume_legacy_conflict'); + sessionStore.closeSession(target.sessionId); + const legacyOwner = makeActivePersistentSession('om_resume_legacy_conflict'); + legacyOwner.larkAppId = undefined; + legacyOwner.lastCliInput = 'existing conversation'; + sessionStore.updateSession(legacyOwner); + const map = new Map(); + wp.registry = map; + + const result = await resumeSession(target.sessionId, map); + + expect(result).toEqual({ + ok: false, + error: 'anchor_occupied', + activeSessionId: legacyOwner.sessionId, + }); + expect(sessionStore.getSession(target.sessionId)?.status).toBe('closed'); + expect(sessionStore.getSession(legacyOwner.sessionId)?.status).toBe('active'); + expect(map.size).toBe(0); + }); + + it('closes a disk-only legacy scratch before reactivating the requested session', async () => { + const target = makeActivePersistentSession('om_resume_legacy_scratch'); + sessionStore.closeSession(target.sessionId); + const legacyScratch = makeActivePersistentSession('om_resume_legacy_scratch'); + legacyScratch.larkAppId = undefined; + legacyScratch.cliId = undefined; + legacyScratch.lastCliInput = undefined; + sessionStore.updateSession(legacyScratch); + const map = new Map(); + wp.registry = map; + + const result = await resumeSession(target.sessionId, map); + + expect(result.ok).toBe(true); + expect(sessionStore.getSession(legacyScratch.sessionId)?.status).toBe('closed'); + expect(sessionStore.getSession(target.sessionId)?.status).toBe('active'); + expect(map.get(sessionKey('om_resume_legacy_scratch', 'app_test'))?.session.sessionId) + .toBe(target.sessionId); + }); }); // ─── Runtime hot-switch sweep (closeCliMismatchedSessionsForBot) ───────────── diff --git a/test/runner-input.test.ts b/test/runner-input.test.ts index 272e39062..eaf19410b 100644 --- a/test/runner-input.test.ts +++ b/test/runner-input.test.ts @@ -36,12 +36,13 @@ function fakeTmuxPty(opts: { failTextAt?: number; failEnter?: boolean } = {}) { } /** Fake raw-PTY handle (no tmux send methods): exercises the write() fallback. */ -function fakeRawPty(opts: { throwOnWrite?: boolean } = {}) { +function fakeRawPty(opts: { throwOnWrite?: boolean; rejectWrite?: boolean } = {}) { const writes: string[] = []; const pty: PtyHandle = { write(data: string) { if (opts.throwOnWrite) throw new Error('pty gone'); writes.push(data); + return opts.rejectWrite ? false : undefined; }, }; return { pty, writes }; @@ -123,7 +124,7 @@ describe('writeRunnerInput — tmux mode', () => { const res = await writeRunnerInput(pty, MARKER, big); - expect(res).toEqual({ submitted: true }); + expect(res).toEqual({ submitted: true, submissionDisposition: 'submitted' }); expect(textChunks.length).toBeGreaterThan(1); for (const c of textChunks) expect(c.length).toBeLessThanOrEqual(RUNNER_INPUT_CHUNK_BYTES); // pre-flush Enter + submit Enter on the happy path. @@ -177,7 +178,7 @@ describe('writeRunnerInput — tmux mode', () => { const res = await writeRunnerInput(pty, MARKER, big); - expect(res).toEqual({ submitted: false }); + expect(res).toEqual({ submitted: false, submissionDisposition: 'dirty_unknown' }); // Chunks 0 and 1 landed; chunk 2 failed and we bailed. expect(textChunks).toHaveLength(2); // pre-flush Enter + flush-on-failure Enter — the partial line gets terminated. @@ -187,11 +188,37 @@ describe('writeRunnerInput — tmux mode', () => { it('pre-flush gate: when every Enter is dropped, bail submitted:false WITHOUT writing any chunk', async () => { const { pty, textChunks } = fakeTmuxPty({ failEnter: true }); const res = await writeRunnerInput(pty, MARKER, 'short message'); - expect(res).toEqual({ submitted: false }); + expect(res).toEqual({ submitted: false, submissionDisposition: 'untouched' }); // The pre-flush Enter never lands, so we must not write the control line // onto a possibly-dirty buffer. expect(textChunks).toHaveLength(0); }); + + it('treats a false last-chunk result as dirty even when bytes landed and cleanup submits it', async () => { + const runner = makeRunnerSim(); + const content = 'last chunk ambiguity ' + 'x'.repeat(2_000); + const chunkCount = chunkAscii( + MARKER + encodeRunnerInput(content), + RUNNER_INPUT_CHUNK_BYTES, + ).length; + let textCall = 0; + const pty: PtyHandle = { + write() { throw new Error('unused'); }, + sendText(text: string) { + runner.feed(text); + return textCall++ === chunkCount - 1 ? false : true; + }, + sendSpecialKeys() { + runner.feed('\r'); + return true; + }, + }; + + const result = await writeRunnerInput(pty, MARKER, content); + + expect(result).toEqual({ submitted: false, submissionDisposition: 'dirty_unknown' }); + expect(runner.enqueued).toEqual([content]); + }); }); describe('writeRunnerInput — raw PTY fallback', () => { @@ -199,14 +226,21 @@ describe('writeRunnerInput — raw PTY fallback', () => { const content = 'fallback path'; const { pty, writes } = fakeRawPty(); const res = await writeRunnerInput(pty, MARKER, content); - expect(res).toEqual({ submitted: true }); + expect(res).toEqual({ submitted: true, submissionDisposition: 'submitted' }); expect(writes).toEqual([MARKER + encodeRunnerInput(content) + '\r']); }); it('reports submitted:false when the raw write throws (pane gone)', async () => { const { pty } = fakeRawPty({ throwOnWrite: true }); const res = await writeRunnerInput(pty, MARKER, 'x'); - expect(res).toEqual({ submitted: false }); + expect(res).toEqual({ submitted: false, submissionDisposition: 'dirty_unknown' }); + }); + + it('does not promote a raw PTY false return to submitted:true', async () => { + const { pty, writes } = fakeRawPty({ rejectWrite: true }); + const res = await writeRunnerInput(pty, MARKER, 'rejected'); + expect(res).toEqual({ submitted: false, submissionDisposition: 'dirty_unknown' }); + expect(writes).toHaveLength(1); }); }); @@ -218,7 +252,7 @@ describe('writeRunnerInput — runner buffer hygiene (no cross-message contamina // Message A: drop chunk 2 mid-stream. const big = 'A'.repeat(15_000); const resA = await writeRunnerInput(fakeRunnerPty(runner, { dropTextAt: 2 }), MARKER, big); - expect(resA).toEqual({ submitted: false }); + expect(resA).toEqual({ submitted: false, submissionDisposition: 'dirty_unknown' }); // A never enqueues intact; its partial got flushed (discarded as bad input), // so the runner buffer is empty — nothing left to prepend to the next line. expect(runner.enqueued).not.toContain(big); @@ -226,7 +260,7 @@ describe('writeRunnerInput — runner buffer hygiene (no cross-message contamina // Message B through the SAME runner: must arrive intact, not merged with A. const resB = await writeRunnerInput(fakeRunnerPty(runner), MARKER, 'clean message B'); - expect(resB).toEqual({ submitted: true }); + expect(resB).toEqual({ submitted: true, submissionDisposition: 'submitted' }); expect(runner.enqueued).toContain('clean message B'); }); @@ -266,7 +300,7 @@ describe('writeRunnerInput — runner buffer hygiene (no cross-message contamina dropText.add(aChunks.length - 1); dropEnter.add(1); dropEnter.add(2); dropEnter.add(3); const a = await writeRunnerInput(pty, MARKER, aContent); - expect(a).toEqual({ submitted: false }); + expect(a).toEqual({ submitted: false, submissionDisposition: 'dirty_unknown' }); expect(runner.buf).not.toBe(''); // A's partial is stuck in the buffer // Message B: drop ALL of B's pre-flush Enter retries (enters 4,5,6). The gate @@ -274,7 +308,7 @@ describe('writeRunnerInput — runner buffer hygiene (no cross-message contamina dropEnter.add(4); dropEnter.add(5); dropEnter.add(6); const textBefore = textIdx; const b = await writeRunnerInput(pty, MARKER, 'message B'); - expect(b).toEqual({ submitted: false }); // NOT a false success + expect(b).toEqual({ submitted: false, submissionDisposition: 'untouched' }); // NOT a false success expect(textIdx).toBe(textBefore); // zero chunks written for B expect(runner.enqueued).not.toContain('message B'); expect(runner.bad).not.toContain('shape'); // no merged bad line attributed as submitted @@ -283,7 +317,7 @@ describe('writeRunnerInput — runner buffer hygiene (no cross-message contamina // (discarded as bad), then enqueues intact. const cleanRunnerPty = fakeRunnerPty(runner); // fresh counters, no drops const bb = await writeRunnerInput(cleanRunnerPty, MARKER, 'message B prime'); - expect(bb).toEqual({ submitted: true }); + expect(bb).toEqual({ submitted: true, submissionDisposition: 'submitted' }); expect(runner.enqueued).toContain('message B prime'); expect(runner.enqueued).not.toContain('message B'); }); diff --git a/test/scheduler-silent-execute.test.ts b/test/scheduler-silent-execute.test.ts index e4ed0b457..07a031181 100644 --- a/test/scheduler-silent-execute.test.ts +++ b/test/scheduler-silent-execute.test.ts @@ -69,7 +69,12 @@ vi.mock('../src/core/worker-pool.js', () => ({ restoreUsageLimitRuntimeState: vi.fn(), setActiveSessionSafe: vi.fn(async (map: Map, k: string, ds: any) => { map.set(k, ds); }), getActiveSessionsRegistry: vi.fn(() => null), - isRelayableRealSession: vi.fn(() => false), + withActiveSessionKeyLock: vi.fn(async ( + _map: Map, + _key: string, + action: () => any, + ) => action()), + isRelayableRealSession: vi.fn((ds: DaemonSession) => !!ds.worker && !ds.worker.killed), closeSession: vi.fn(), suspendWorker: vi.fn(), })); diff --git a/test/session-adopt.test.ts b/test/session-adopt.test.ts index 6d37daab9..55fdaa420 100644 --- a/test/session-adopt.test.ts +++ b/test/session-adopt.test.ts @@ -46,6 +46,7 @@ vi.mock('../src/bot-registry.js', () => ({ })), getAllBots: vi.fn(() => []), getBotClient: vi.fn(), + getBotBrand: vi.fn(() => 'feishu'), })); vi.mock('../src/config.js', () => ({ @@ -57,6 +58,7 @@ vi.mock('../src/config.js', () => ({ })); vi.mock('../src/services/session-store.js', () => ({ + getSession: vi.fn(), closeSession: vi.fn(), updateSession: vi.fn(), createSession: vi.fn(), @@ -89,6 +91,7 @@ vi.mock('../src/core/session-manager.js', () => ({ ds.lastUserPrompt = userPrompt; ds.lastCliInput = cliInput; }), + persistStreamCardState: vi.fn(), })); vi.mock('../src/services/frozen-card-store.js', () => ({ @@ -106,7 +109,7 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ // ─── Imports ────────────────────────────────────────────────────────────── import { handleCardAction, type CardHandlerDeps } from '../src/im/lark/card-handler.js'; -import { killWorker, forkWorker } from '../src/core/worker-pool.js'; +import { killWorker, forkWorker, setActiveSessionsRegistry } from '../src/core/worker-pool.js'; import * as sessionStore from '../src/services/session-store.js'; import { deleteMessage } from '../src/im/lark/client.js'; import { getBot } from '../src/bot-registry.js'; @@ -132,7 +135,7 @@ function makeDaemonSession(overrides?: Partial): DaemonSession { pid: null, chatType: 'group', }, - worker: { killed: false, send: vi.fn() } as any, + worker: { killed: false, send: vi.fn(), once: vi.fn() } as any, workerPort: 8080, workerToken: 'tok_secret', larkAppId: APP_ID, @@ -222,18 +225,25 @@ describe('Adopt card actions', () => { const sKey = sessionKey(ROOT_ID, APP_ID); sessions.set(sKey, ds); const deps = makeDeps(sessions); - - await handleCardAction(makeDisconnectEvent(ROOT_ID), deps, APP_ID); - - expect(killWorker).toHaveBeenCalledWith(ds); - expect(sessionStore.closeSession).toHaveBeenCalledWith('uuid-adopt-test'); - expect(sessions.has(sKey)).toBe(false); - expect(deps.sessionReply).toHaveBeenCalledWith( - ROOT_ID, - expect.stringContaining('断开'), - undefined, - APP_ID, - ); + const worker = ds.worker as any; + vi.mocked(sessionStore.getSession).mockReturnValue(ds.session); + setActiveSessionsRegistry(sessions); + + try { + await handleCardAction(makeDisconnectEvent(ROOT_ID), deps, APP_ID); + + expect(worker.send).toHaveBeenCalledWith({ type: 'close' }); + expect(sessionStore.closeSession).toHaveBeenCalledWith('uuid-adopt-test'); + expect(sessions.has(sKey)).toBe(false); + expect(deps.sessionReply).toHaveBeenCalledWith( + ROOT_ID, + expect.stringContaining('断开'), + undefined, + APP_ID, + ); + } finally { + setActiveSessionsRegistry(new Map()); + } }); it('should be a no-op when session does not exist', async () => { diff --git a/test/session-card-model.test.ts b/test/session-card-model.test.ts index b849d6139..221b8e1eb 100644 --- a/test/session-card-model.test.ts +++ b/test/session-card-model.test.ts @@ -53,6 +53,14 @@ describe('session-card-model · composeEntries / statusToDot', () => { const dot = statusToDot('dormant'); expect(dot).toEqual({ tone: 'neutral', pulse: false, label: 'sessions.status.dormant' }); }); + + it('maps stalled Codex App turns to a non-pulsing danger dot', () => { + expect(statusToDot('stalled')).toEqual({ + tone: 'danger', + pulse: false, + label: 'sessions.status.stalled', + }); + }); }); describe('session-card-model · filters', () => { diff --git a/test/session-kanban.test.ts b/test/session-kanban.test.ts index ee23b93ef..22b40e29a 100644 --- a/test/session-kanban.test.ts +++ b/test/session-kanban.test.ts @@ -74,6 +74,7 @@ describe('deriveKanbanColumn', () => { expect(deriveKanbanColumn({ status: 'idle', tuiPromptActive: true })).toBe('in_review'); expect(deriveKanbanColumn({ status: 'idle', agentAttention: { kind: 'x', reason: 'y', at: 1 } })).toBe('in_review'); expect(deriveKanbanColumn({ status: 'limited' })).toBe('in_review'); + expect(deriveKanbanColumn({ status: 'stalled' })).toBe('in_review'); }); it('maps runtime states to default columns', () => { diff --git a/test/session-lifecycle-start.test.ts b/test/session-lifecycle-start.test.ts index 68821457d..dae0e6ff7 100644 --- a/test/session-lifecycle-start.test.ts +++ b/test/session-lifecycle-start.test.ts @@ -132,10 +132,18 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ })); import { __testOnly_resetSessionLifecycleHooks } from '../src/services/session-lifecycle-hooks.js'; -import { forkAdoptWorker, forkWorker, initWorkerPool, sendWorkerInput } from '../src/core/worker-pool.js'; +import { + forkAdoptWorker, + forkWorker, + initWorkerPool, + promoteQueuedActivationTail, + sendWorkerInput, +} from '../src/core/worker-pool.js'; import type { DaemonSession } from '../src/core/types.js'; import * as sessionStore from '../src/services/session-store.js'; import { getBot } from '../src/bot-registry.js'; +import { dashboardEventBus } from '../src/core/dashboard-events.js'; +import { retireCodexAppDispatchAfterBackingMissing } from '../src/utils/codex-app-dispatch-ledger.js'; import { mkdtempSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir, homedir } from 'node:os'; import { join } from 'node:path'; @@ -199,6 +207,7 @@ function defaultBot(overrides: Record = {}) { beforeEach(() => { vi.clearAllMocks(); + vi.mocked(sessionStore.updateSession).mockImplementation(() => undefined); vi.mocked(getBot).mockImplementation(() => defaultBot()); __testOnly_resetSessionLifecycleHooks(); forkMock.mockImplementation(() => makeFakeWorker()); @@ -286,13 +295,56 @@ describe('Codex App clean-input feature gate', () => { ds.session.vcMeetingImTurnOrigins = { om_vc_im: origin }; expect(sendWorkerInput(ds, payload, 'om_vc_im')).toBe(true); - expect(worker.send).toHaveBeenCalledWith({ + expect(worker.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'message', content: payload.content, codexAppInput: { text: 'clean', clientUserMessageId: 'om_vc_im' }, turnId: 'om_vc_im', + codexAppDispatchId: expect.any(String), vcMeetingImTurnOrigin: origin, - }); + })); + }); + + it('freezes non-Lark delivery sinks into every accepted Codex App entry', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const cases = [ + { + sink: 'doc_comment', + prepare: (ds: any, turnId: string) => { + ds.session.docCommentTargets = { + [turnId]: { fileToken: 'doc', fileType: 'docx', commentId: 'comment', turnId }, + }; + }, + }, + { + sink: 'http_wait', + prepare: (ds: any, turnId: string) => { + ds.pendingWaitPromises = new Map([[turnId, { resolve: vi.fn() }]]); + }, + }, + { + sink: 'http_async', + prepare: (ds: any, turnId: string) => { + ds.asyncTriggerResults = new Map([[turnId, { status: 'pending', createdAt: Date.now() }]]); + }, + }, + { + sink: 'suppressed', + prepare: (ds: any, turnId: string) => { + ds.suppressedFinalOutputTurns = new Map([[turnId, 1]]); + }, + }, + ] as const; + + for (const [index, fixture] of cases.entries()) { + const turnId = `turn-sink-${index}`; + const ds = makeDs({ worker: makeFakeWorker() }); + fixture.prepare(ds, turnId); + expect(sendWorkerInput(ds, `payload-${index}`, turnId, { + ...(fixture.sink === 'suppressed' ? { dispatchAttempt: 1 } : {}), + })).toBe(true); + expect(ds.session.codexAppDispatchLedger?.[0]?.deliverySink).toBe(fixture.sink); + } }); it('resolves explicit meeting IM origin while keeping the clean cold-fork sidecar', () => { @@ -348,7 +400,7 @@ describe('Codex App clean-input feature gate', () => { expect(ds.managedTurnOrigin).toBeUndefined(); }); - it('does not lend a previous human turn to a non-empty system prompt', () => { + it('mints a durable identity without borrowing previous-turn routing for a no-id fork', () => { vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app', codexAppCleanInput: true })); const ds = makeDs({ currentReplyTarget: { @@ -371,12 +423,806 @@ describe('Codex App clean-input feature gate', () => { const init = vi.mocked(worker.send).mock.calls[0][0]; expect(init).toEqual(expect.objectContaining({ prompt: payload.content, - promptCodexAppInput: { text: 'clean' }, + promptCodexAppInput: { + text: 'clean', + clientUserMessageId: expect.stringMatching(/^codex-app-dispatch-/), + }, + turnId: expect.stringMatching(/^codex-app-dispatch-/), + codexAppDispatchId: expect.any(String), })); - expect(init.turnId).toBeUndefined(); + expect(init.promptCodexAppInput.clientUserMessageId).toBe(init.turnId); + expect(init.replyTurnId).toBeUndefined(); expect(init.vcMeetingImTurnOrigin).toBeUndefined(); }); + it('mints distinct persisted identities for consecutive live no-id inputs without losing reply routing', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app', codexAppCleanInput: true })); + const worker = makeFakeWorker(); + const ds = makeDs({ + worker, + currentReplyTarget: { + rootMessageId: 'om_root', + turnId: 'om_route', + updatedAt: new Date().toISOString(), + }, + }); + + expect(sendWorkerInput(ds, { content: 'scheduler one', codexAppInput: { text: 'one' } })).toBe(true); + expect(sendWorkerInput(ds, { content: 'scheduler two', codexAppInput: { text: 'two' } })).toBe(true); + + const messages = vi.mocked(worker.send).mock.calls.map(call => call[0]); + expect(messages).toHaveLength(2); + expect(messages[0]).toEqual(expect.objectContaining({ + turnId: expect.stringMatching(/^codex-app-dispatch-/), + replyTurnId: 'om_route', + codexAppDispatchId: expect.any(String), + })); + expect(messages[1]).toEqual(expect.objectContaining({ + turnId: expect.stringMatching(/^codex-app-dispatch-/), + replyTurnId: 'om_route', + codexAppDispatchId: expect.any(String), + })); + expect(messages[1].turnId).not.toBe(messages[0].turnId); + expect(ds.session.codexAppDispatchLedger).toHaveLength(2); + expect(ds.session.codexAppDispatchLedger?.map(entry => entry.replyTurnId)).toEqual([ + 'om_route', + 'om_route', + ]); + }); + + it('copies the inbound turn registry into the durable ledger after mutable reply state advances', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const worker = makeFakeWorker(); + const ds = makeDs({ + worker, + scope: 'chat', + currentReplyTarget: { + rootMessageId: 'om_topic_b', turnId: 'turn-b', updatedAt: new Date().toISOString(), + }, + }); + ds.session.scope = 'chat'; + ds.session.cliId = 'codex-app'; + ds.session.currentReplyTarget = ds.currentReplyTarget; + ds.session.turnReplyContexts = { + 'turn-a': { + target: { mode: 'thread', rootMessageId: 'om_topic_a' }, + quoteTargetId: 'turn-a', + replyTargetSenderOpenId: 'ou_a', + }, + 'turn-b': { target: { mode: 'thread', rootMessageId: 'om_topic_b' } }, + }; + + expect(sendWorkerInput(ds, 'late A input', 'turn-a')).toBe(true); + expect(ds.session.codexAppDispatchLedger?.[0]?.replyTarget) + .toEqual({ mode: 'thread', rootMessageId: 'om_topic_a' }); + expect(ds.session.codexAppDispatchLedger?.[0]).toMatchObject({ + quoteTargetId: 'turn-a', replyTargetSenderOpenId: 'ou_a', + }); + }); + + it('rejects an empty double-fork before mutating or killing a ledger-owned live worker', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const worker = makeFakeWorker(); + const ds = makeDs({ worker }); + ds.session.cliId = 'codex-app'; + ds.session.queued = true; + ds.session.codexAppDispatchLedger = [ + { dispatchId: 'old', turnId: 'turn-old', state: 'prepared', content: 'old' }, + ]; + + forkWorker(ds, '', true); + + expect(forkMock).not.toHaveBeenCalled(); + expect(worker.send).not.toHaveBeenCalled(); + expect(worker.kill).not.toHaveBeenCalled(); + expect(ds.worker).toBe(worker); + expect(ds.session.queued).toBe(true); + expect(sessionStore.updateSession).not.toHaveBeenCalled(); + }); + + it('routes a non-empty double-fork into the existing durable FIFO without replacing its worker', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app', codexAppCleanInput: true })); + const worker = makeFakeWorker(); + const ds = makeDs({ worker }); + ds.session.cliId = 'codex-app'; + ds.session.codexAppDispatchLedger = [ + { dispatchId: 'old', turnId: 'turn-old', state: 'prepared', content: 'old' }, + ]; + + forkWorker(ds, { content: 'next', codexAppInput: { text: 'next clean' } }, { + resume: true, + turnId: 'turn-next', + dispatchAttempt: 2, + }); + + expect(forkMock).not.toHaveBeenCalled(); + expect(worker.kill).not.toHaveBeenCalled(); + expect(worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + content: 'next', + turnId: 'turn-next', + dispatchAttempt: 2, + codexAppDispatchId: expect.any(String), + })); + expect(ds.session.codexAppDispatchLedger?.map(entry => entry.turnId)) + .toEqual(['turn-old', 'turn-next']); + }); + + it('stages a non-Codex double-fork behind a tokened activation without live IPC', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'claude-code' })); + const worker = makeFakeWorker(); + const ds = makeDs({ worker, initialStartPending: true }); + Object.assign(ds.session, { + cliId: 'claude-code', + queuedActivationPending: true, + queuedActivationToken: 'opening-token', + queuedActivationInput: { content: 'OPENING_N' }, + queuedActivationTurnId: 'turn-opening', + }); + + forkWorker(ds, { content: 'FOLLOWER_N1' }, { + resume: true, + turnId: 'turn-follower', + dispatchAttempt: 3, + }); + + expect(forkMock).not.toHaveBeenCalled(); + expect(worker.send).not.toHaveBeenCalled(); + expect(worker.kill).not.toHaveBeenCalled(); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + order: 1, + turnId: 'turn-follower', + dispatchAttempt: 3, + userPrompt: 'FOLLOWER_N1', + cliInput: { content: 'FOLLOWER_N1' }, + }), + ]); + expect(sessionStore.updateSession).toHaveBeenCalledWith(ds.session); + }); + + it('rejects a gated non-Codex double-fork when exact-tail persistence fails', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'claude-code' })); + const worker = makeFakeWorker(); + const ds = makeDs({ worker, initialStartPending: true }); + Object.assign(ds.session, { + cliId: 'claude-code', + queuedActivationPending: true, + queuedActivationToken: 'opening-token', + queuedActivationInput: { content: 'OPENING_N' }, + queuedActivationTurnId: 'turn-opening', + }); + vi.mocked(sessionStore.updateSession).mockImplementationOnce(() => { + throw new Error('tail store unavailable'); + }); + + expect(() => forkWorker(ds, 'FOLLOWER_MUST_NOT_SEND', { turnId: 'turn-follower' })) + .toThrow('tail store unavailable'); + + expect(forkMock).not.toHaveBeenCalled(); + expect(worker.send).not.toHaveBeenCalled(); + expect(worker.kill).not.toHaveBeenCalled(); + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(ds.session.queuedActivationPending).toBe(true); + }); + + it('rolls back an exact tail promotion when its single durable write fails', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'claude-code' })); + const ds = makeDs({ hasHistory: true, initialStartPending: true }); + ds.session.cliId = 'claude-code'; + ds.session.queuedActivationTail = [{ + id: 'tail-promote-1', + order: 1, + userPrompt: 'PROMOTE_ME', + cliInput: { content: 'PROMOTE_ME' }, + turnId: 'turn-promote', + dispatchAttempt: 4, + }]; + ds.session.queuedActivationTailNextOrder = 1; + vi.mocked(sessionStore.updateSession).mockImplementationOnce(() => { + throw new Error('promotion store unavailable'); + }); + + expect(promoteQueuedActivationTail(ds, { send: false })).toBe(false); + + expect(ds.session.queuedActivationPending).toBeUndefined(); + expect(ds.session.queuedActivationToken).toBeUndefined(); + expect(ds.session.queuedActivationInput).toBeUndefined(); + expect(ds.session.queuedActivationTail).toEqual([expect.objectContaining({ + id: 'tail-promote-1', + turnId: 'turn-promote', + dispatchAttempt: 4, + cliInput: { content: 'PROMOTE_ME' }, + })]); + expect(ds.pendingPrompt).toBeUndefined(); + }); + + it('promotes the admitted clean sidecar exactly even after the live config gate flips off', () => { + let cleanInputEnabled = true; + vi.mocked(getBot).mockImplementation(() => defaultBot({ + cliId: 'codex-app', + codexAppCleanInput: cleanInputEnabled, + })); + const worker = makeFakeWorker(); + const ds = makeDs({ worker, initialStartPending: true, hasHistory: true }); + Object.assign(ds.session, { + cliId: 'codex-app', + queuedActivationPending: true, + queuedActivationToken: 'opening-token', + queuedActivationInput: { content: 'OPENING_N' }, + queuedActivationTurnId: 'turn-opening', + }); + const admittedSidecar = { + text: 'FOLLOWER_CLEAN_N1', + additionalContext: { + hidden: { kind: 'application' as const, value: 'exact' }, + }, + }; + + expect(sendWorkerInput(ds, { + content: 'FOLLOWER_LEGACY_N1', + codexAppInput: admittedSidecar, + }, 'turn-follower', { dispatchAttempt: 5 })).toBe(true); + expect(worker.send).not.toHaveBeenCalled(); + expect(ds.session.queuedActivationTail?.[0]?.cliInput.codexAppInput).toEqual({ + ...admittedSidecar, + clientUserMessageId: 'turn-follower', + }); + + // Model the opening ACK, then flip the immediate setting before N+1 is + // promoted. The persisted entry—not current config—is authoritative. + Object.assign(ds.session, { + queuedActivationPending: undefined, + queuedActivationToken: undefined, + queuedActivationInput: undefined, + queuedActivationTurnId: undefined, + queuedActivationDispatchAttempt: undefined, + queuedActivationResume: undefined, + }); + cleanInputEnabled = false; + + expect(promoteQueuedActivationTail(ds)).toBe(true); + + const exactSidecar = { + ...admittedSidecar, + clientUserMessageId: 'turn-follower', + }; + expect(ds.session.queuedActivationInput?.codexAppInput).toEqual(exactSidecar); + expect(ds.session.codexAppDispatchLedger?.at(-1)?.codexAppInput).toEqual(exactSidecar); + expect(worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'message', + turnId: 'turn-follower', + dispatchAttempt: 5, + codexAppInput: exactSidecar, + queuedActivationToken: expect.any(String), + })); + }); + + it('retries the exact pre-init activation sidecar after the live gate flips off', () => { + let cleanInputEnabled = true; + vi.mocked(getBot).mockImplementation(() => defaultBot({ + cliId: 'codex-app', + codexAppCleanInput: cleanInputEnabled, + })); + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex-app', + queued: true, + queuedPrompt: 'OPENING_LEGACY', + queuedCodexAppText: 'OPENING_CLEAN', + }); + const exactSidecar = { + text: 'OPENING_CLEAN', + additionalContext: { + hidden: { kind: 'application' as const, value: 'retry-exact' }, + }, + clientUserMessageId: 'turn-exact-retry', + }; + const failingWorker = makeFakeWorker(); + failingWorker.send = vi.fn(() => { + throw new Error('init IPC rejected before acceptance'); + }); + forkMock.mockReturnValueOnce(failingWorker); + + expect(() => forkWorker(ds, { + content: 'OPENING_LEGACY', + codexAppInput: { + text: exactSidecar.text, + additionalContext: exactSidecar.additionalContext, + }, + }, { turnId: 'turn-exact-retry', dispatchAttempt: 7 })) + .toThrow('init IPC rejected before acceptance'); + + expect(ds.session.queued).toBe(true); + expect(ds.session.queuedActivationInput?.codexAppInput).toEqual(exactSidecar); + expect(ds.session.codexAppDispatchLedger).toEqual([]); + + cleanInputEnabled = false; + const retained = ds.session.queuedActivationInput!; + forkWorker(ds, retained, { + turnId: ds.session.queuedActivationTurnId, + dispatchAttempt: ds.session.queuedActivationDispatchAttempt, + }); + + const retryWorker = forkMock.mock.results.at(-1)!.value; + const retryInit = vi.mocked(retryWorker.send).mock.calls[0]![0]; + expect(retryInit.promptCodexAppInput).toEqual(exactSidecar); + expect(ds.session.queuedActivationInput?.codexAppInput).toEqual(exactSidecar); + expect(ds.session.codexAppDispatchLedger?.at(-1)?.codexAppInput).toEqual(exactSidecar); + }); + + it('rolls back only the new double-fork acceptance when existing-worker IPC fails', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const worker = makeFakeWorker(); + worker.send = vi.fn(() => { throw new Error('ipc failed'); }); + const ds = makeDs({ worker }); + ds.session.cliId = 'codex-app'; + const oldEntry = { dispatchId: 'old', turnId: 'turn-old', state: 'accepted' as const, content: 'old' }; + ds.session.codexAppDispatchLedger = [oldEntry]; + + forkWorker(ds, 'next', { turnId: 'turn-next' }); + + expect(forkMock).not.toHaveBeenCalled(); + expect(worker.kill).not.toHaveBeenCalled(); + expect(ds.worker).toBe(worker); + expect(ds.session.codexAppDispatchLedger).toEqual([oldEntry]); + }); + + it('persists queued dequeue and Codex App acceptance until the exact submission ACK', async () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const oldEntry = { + dispatchId: 'dispatch-existing', + turnId: 'turn-existing', + state: 'accepted' as const, + content: 'existing FIFO item', + }; + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex-app', + queued: true, + queuedPrompt: 'queued opening', + queuedCodexAppText: 'queued clean input', + queuedCodexAppMessageContext: 'queued context', + codexAppDispatchLedger: [oldEntry], + }); + const persisted: Array = []; + vi.mocked(sessionStore.updateSession).mockImplementation(session => { + persisted.push(structuredClone(session)); + }); + + forkWorker(ds, 'queued opening', { turnId: 'turn-queued' }); + + const firstDequeued = persisted.find(snapshot => snapshot.queued === false); + expect(firstDequeued).toMatchObject({ + queued: false, + queuedActivationPending: true, + queuedActivationToken: expect.any(String), + queuedActivationInput: { content: 'queued opening' }, + queuedActivationTurnId: 'turn-queued', + queuedPrompt: 'queued opening', + queuedCodexAppText: 'queued clean input', + queuedCodexAppMessageContext: 'queued context', + codexAppDispatchLedger: [ + oldEntry, + expect.objectContaining({ + turnId: 'turn-queued', + state: 'accepted', + queuedActivationToken: expect.any(String), + }), + ], + }); + expect(persisted).not.toContainEqual(expect.objectContaining({ + queued: false, + codexAppDispatchLedger: [oldEntry], + })); + expect(ds.session).toMatchObject({ + queued: false, + queuedActivationPending: true, + queuedActivationToken: expect.any(String), + queuedActivationInput: { content: 'queued opening' }, + queuedPrompt: 'queued opening', + codexAppDispatchLedger: [ + oldEntry, + expect.objectContaining({ turnId: 'turn-queued', state: 'accepted' }), + ], + }); + const worker = forkMock.mock.results.at(-1)!.value; + const init = vi.mocked(worker.send).mock.calls[0]![0]; + expect(init.queuedActivationToken).toBe(ds.session.queuedActivationToken); + worker.emit('message', { + type: 'queued_activation_submitted', + sessionId: ds.session.sessionId, + activationToken: init.queuedActivationToken, + }); + await vi.waitFor(() => expect(ds.session.queuedActivationPending).toBeUndefined()); + expect(ds.session.queuedActivationPending).toBeUndefined(); + expect(ds.session.queuedPrompt).toBeUndefined(); + expect(ds.session.queuedActivationInput).toBeUndefined(); + }); + + it('recovers a post-init crash journal through the accepted Codex FIFO until its ACK', async () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const accepted = { + dispatchId: 'dispatch-accepted-before-crash', + turnId: 'turn-queued', + state: 'accepted' as const, + content: 'queued opening', + }; + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex-app', + queued: false, + queuedActivationPending: true, + queuedPrompt: 'queued opening', + queuedCodexAppText: 'queued clean input', + queuedCodexAppMessageContext: 'queued context', + codexAppDispatchLedger: [accepted], + }); + const persisted: Array = []; + vi.mocked(sessionStore.updateSession).mockImplementation(session => { + persisted.push(structuredClone(session)); + }); + + forkWorker(ds, '', true); + + const worker = forkMock.mock.results.at(-1)!.value; + const init = vi.mocked(worker.send).mock.calls[0][0]; + expect(init).toMatchObject({ + prompt: '', + resume: true, + queuedActivationToken: expect.any(String), + codexAppRecoveredDispatches: [expect.objectContaining({ + ...accepted, + queuedActivationToken: expect.any(String), + })], + }); + expect(init).not.toHaveProperty('codexAppDispatchId'); + expect(ds.session.codexAppDispatchLedger).toEqual([accepted]); + expect(ds.session.queued).toBe(false); + expect(ds.session.queuedActivationPending).toBe(true); + expect(ds.session.queuedPrompt).toBe('queued opening'); + worker.emit('message', { + type: 'queued_activation_submitted', + sessionId: ds.session.sessionId, + activationToken: init.queuedActivationToken, + }); + await vi.waitFor(() => expect(ds.session.queuedActivationPending).toBeUndefined()); + expect(ds.session.queuedActivationPending).toBeUndefined(); + expect(ds.session.queuedPrompt).toBeUndefined(); + expect(persisted).toContainEqual(expect.objectContaining({ + queued: false, + queuedActivationPending: undefined, + queuedPrompt: undefined, + codexAppDispatchLedger: [accepted], + })); + }); + + it('durably restores a queued payload and preserves the prior FIFO when child fork throws', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ + cliId: 'codex-app', + codexAppCleanInput: true, + })); + const oldEntry = { + dispatchId: 'dispatch-existing', + turnId: 'turn-existing', + state: 'accepted' as const, + content: 'existing FIFO item', + }; + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex-app', + queued: true, + queuedPrompt: 'wrapped opening', + queuedCodexAppText: 'clean opening', + queuedCodexAppMessageContext: 'queued', + codexAppDispatchLedger: [oldEntry], + }); + const persisted: Array = []; + vi.mocked(sessionStore.updateSession).mockImplementation(session => { + persisted.push(structuredClone(session)); + }); + forkMock.mockImplementationOnce(() => { throw new Error('synchronous fork failed'); }); + + expect(() => forkWorker(ds, { + content: 'wrapped opening', + codexAppInput: { text: 'clean opening' }, + }, { turnId: 'turn-queued' })).toThrow('synchronous fork failed'); + + expect(ds.worker).toBeNull(); + expect(ds.session).toMatchObject({ + queued: true, + queuedPrompt: 'wrapped opening', + queuedCodexAppText: 'clean opening', + queuedCodexAppMessageContext: 'queued', + }); + expect(ds.session.codexAppDispatchLedger).toEqual([oldEntry]); + expect(persisted.at(-1)).toMatchObject({ + queued: true, + queuedPrompt: 'wrapped opening', + queuedCodexAppText: 'clean opening', + queuedCodexAppMessageContext: 'queued', + codexAppDispatchLedger: [oldEntry], + }); + }); + + it('fences the child and atomically restores queued/FIFO state when init IPC throws', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const oldEntry = { + dispatchId: 'dispatch-existing', + turnId: 'turn-existing', + state: 'prepared' as const, + content: 'existing FIFO item', + }; + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex-app', + queued: true, + queuedPrompt: 'queued opening', + queuedCodexAppText: 'queued clean input', + queuedCodexAppMessageContext: 'queued context', + codexAppDispatchLedger: [oldEntry], + }); + const persisted: Array = []; + vi.mocked(sessionStore.updateSession).mockImplementation(session => { + persisted.push(structuredClone(session)); + }); + const worker = makeFakeWorker(); + worker.send = vi.fn(() => { throw new Error('synchronous init IPC failed'); }); + forkMock.mockReturnValueOnce(worker); + + expect(() => forkWorker(ds, 'queued opening', { turnId: 'turn-queued' })) + .toThrow('synchronous init IPC failed'); + + expect(worker.kill).toHaveBeenCalledOnce(); + expect(ds.worker).toBeNull(); + expect(ds.session).toMatchObject({ + queued: true, + queuedPrompt: 'queued opening', + queuedCodexAppText: 'queued clean input', + queuedCodexAppMessageContext: 'queued context', + }); + expect(ds.session.codexAppDispatchLedger).toEqual([oldEntry]); + expect(persisted.at(-1)).toMatchObject({ + queued: true, + queuedPrompt: 'queued opening', + codexAppDispatchLedger: [oldEntry], + }); + }); + + it('retains the exact Codex activation journal after init IPC until submission is proved', async () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex-app', + queued: true, + queuedPrompt: 'queued opening', + queuedCodexAppText: 'queued clean input', + codexAppDispatchLedger: [], + }); + + forkWorker(ds, 'queued opening', { turnId: 'turn-queued' }); + const worker = forkMock.mock.results.at(-1)!.value; + worker.emit('error', new Error('async spawn error after send returned')); + await Promise.resolve(); + + expect(ds.session.queued).toBe(false); + expect(ds.session.queuedActivationPending).toBe(true); + expect(ds.session.queuedActivationToken).toEqual(expect.any(String)); + expect(ds.session.queuedActivationInput).toEqual({ content: 'queued opening' }); + expect(ds.session.queuedPrompt).toBe('queued opening'); + expect(ds.session.codexAppDispatchLedger).toEqual([ + expect.objectContaining({ turnId: 'turn-queued', state: 'accepted' }), + ]); + expect(ds.worker).toBeNull(); + expect(ds.initialStartPending).toBe(false); + expect(worker.kill).toHaveBeenCalledOnce(); + }); + + it.each(['error', 'exit'] as const)( + 'retains the exact non-Codex activation journal when its worker emits %s before ACK', + async event => { + const ds = makeDs(); + Object.assign(ds.session, { + cliId: 'codex', + queued: true, + queuedPrompt: 'BACKLOG_AND_TRIGGER_REPLY', + }); + + forkWorker(ds, { content: 'BACKLOG_AND_TRIGGER_REPLY' }, { + turnId: 'turn-trigger-reply', + dispatchAttempt: 3, + }); + const worker = forkMock.mock.results.at(-1)!.value; + if (event === 'error') worker.emit('error', new Error('pre-ACK worker error')); + else worker.emit('exit', 1, null); + await Promise.resolve(); + + expect(ds.worker).toBeNull(); + expect(ds.initialStartPending).toBe(false); + expect(ds.session).toMatchObject({ + queued: false, + queuedPrompt: 'BACKLOG_AND_TRIGGER_REPLY', + queuedActivationPending: true, + queuedActivationToken: expect.any(String), + queuedActivationInput: { content: 'BACKLOG_AND_TRIGGER_REPLY' }, + queuedActivationTurnId: 'turn-trigger-reply', + queuedActivationDispatchAttempt: 3, + queuedActivationResume: false, + }); + expect(vi.mocked(sessionStore.updateSession).mock.calls.map(call => call[0])) + .toContainEqual(expect.objectContaining({ + queued: false, + queuedActivationPending: true, + queuedActivationInput: { content: 'BACKLOG_AND_TRIGGER_REPLY' }, + })); + }, + ); + + it('keeps the worker authoritative and restores its journal when ACK persistence fails', async () => { + vi.useFakeTimers({ now: 0 }); + try { + const ds = makeDs(); + Object.assign(ds.session, { + queued: true, + queuedPrompt: 'queued opening', + }); + forkWorker(ds, 'queued opening', { turnId: 'turn-queued' }); + const worker = forkMock.mock.results.at(-1)!.value; + const init = vi.mocked(worker.send).mock.calls[0]![0]; + vi.mocked(sessionStore.updateSession).mockImplementationOnce(() => { + throw new Error('ACK journal write failed'); + }); + + worker.emit('message', { + type: 'queued_activation_submitted', + sessionId: ds.session.sessionId, + activationToken: init.queuedActivationToken, + }); + await Promise.resolve(); + + expect(ds.worker).toBe(worker); + expect(worker.kill).not.toHaveBeenCalled(); + expect(ds.hasHistory).toBe(false); + expect(ds.initialStartPending).toBe(true); + expect(ds.session).toMatchObject({ + queued: false, + queuedActivationPending: true, + queuedActivationToken: init.queuedActivationToken, + queuedActivationInput: { content: 'queued opening' }, + queuedPrompt: 'queued opening', + }); + + await vi.advanceTimersByTimeAsync(100); + expect(ds.hasHistory).toBe(true); + expect(ds.initialStartPending).toBe(false); + expect(ds.session.queuedActivationPending).toBeUndefined(); + expect(ds.session.queuedActivationToken).toBeUndefined(); + expect(ds.session.queuedActivationInput).toBeUndefined(); + } finally { + vi.useRealTimers(); + } + }); + + it('never reports retryable activation failure after init IPC accepted the worker', () => { + const enforceLiveSessionCap = vi.fn(() => { + throw new Error('post-send cap projection failed'); + }); + initWorkerPool({ + sessionReply: vi.fn(async () => 'om_reply'), + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + enforceLiveSessionCap, + }); + vi.mocked(sessionStore.updateSessionPid).mockImplementationOnce(() => { + throw new Error('post-send pid persistence failed'); + }); + vi.mocked(dashboardEventBus.publish).mockImplementationOnce(() => { + throw new Error('post-send dashboard projection failed'); + }); + const ds = makeDs(); + Object.assign(ds.session, { + queued: true, + queuedPrompt: 'queued opening', + }); + + expect(() => forkWorker(ds, 'queued opening', { turnId: 'turn-queued' })) + .not.toThrow(); + expect(ds.worker).toBe(forkMock.mock.results.at(-1)!.value); + expect(ds.session).toMatchObject({ + queued: false, + queuedActivationPending: true, + queuedActivationInput: { content: 'queued opening' }, + }); + expect(enforceLiveSessionCap).toHaveBeenCalledOnce(); + }); + + it('retries only an unaccepted ACK follow-up with exponential backoff capped at five seconds', async () => { + vi.useFakeTimers({ now: 0 }); + try { + const callTimes: number[] = []; + const release = vi.fn((session: DaemonSession) => { + callTimes.push(Date.now()); + if (callTimes.length < 9) return false; + session.initialStartPending = false; + return true; + }); + initWorkerPool({ + sessionReply: vi.fn(async () => 'om_reply'), + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + onQueuedActivationSubmitted: release, + }); + const ds = makeDs(); + Object.assign(ds.session, { + queued: true, + queuedPrompt: 'queued opening', + }); + + forkWorker(ds, 'queued opening', { turnId: 'turn-queued' }); + const worker = forkMock.mock.results.at(-1)!.value; + const init = vi.mocked(worker.send).mock.calls[0]![0]; + worker.emit('message', { + type: 'queued_activation_submitted', + sessionId: ds.session.sessionId, + activationToken: init.queuedActivationToken, + }); + await vi.advanceTimersByTimeAsync(0); + for (let i = 0; i < 8; i++) await vi.runOnlyPendingTimersAsync(); + + expect(release).toHaveBeenCalledTimes(9); + expect(callTimes.slice(1).map((time, i) => time - callTimes[i]!)) + .toEqual([100, 200, 400, 800, 1_600, 3_200, 5_000, 5_000]); + expect(ds.initialStartPending).toBe(false); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + it.each(['accepted', 'prepared'] as const)( + 'does not restore crashed %s N beside hub replay N+1 after exact backing-missing retirement', + state => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app' })); + const ds = makeDs(); + ds.session.cliId = 'codex-app'; + ds.session.codexAppDispatchLedger = [{ + dispatchId: 'old-dispatch', turnId: 'delivery', dispatchAttempt: 1, + state, content: 'old N', + }]; + const retired = retireCodexAppDispatchAfterBackingMissing( + ds.session.codexAppDispatchLedger, + 'delivery', + 1, + ); + expect(retired.ok).toBe(true); + if (!retired.ok) return; + ds.session.codexAppDispatchLedger = retired.ledger; + sessionStore.updateSession(ds.session); + + forkWorker(ds, 'hub replay N+1', { + resume: true, + turnId: 'delivery', + dispatchAttempt: 2, + }); + + const worker = forkMock.mock.results.at(-1)!.value; + const init = vi.mocked(worker.send).mock.calls[0][0]; + expect(init.codexAppRecoveredDispatches).toBeUndefined(); + expect(init).toEqual(expect.objectContaining({ + turnId: 'delivery', + dispatchAttempt: 2, + codexAppDispatchId: expect.not.stringMatching(/^old-dispatch$/), + })); + expect(ds.session.codexAppDispatchLedger).toEqual([ + expect.objectContaining({ + turnId: 'delivery', dispatchAttempt: 2, state: 'accepted', + }), + ]); + }, + ); + it('starts an old queued activation without a sidecar and a modern one with exactly one', () => { vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex-app', codexAppCleanInput: true })); const oldPayload = applyQueuedCodexAppLegacyFallback({ @@ -410,18 +1256,20 @@ describe('Codex App clean-input feature gate', () => { ds.session.agentFrozen = true; expect(sendWorkerInput(ds, payload, 'om_1')).toBe(true); - expect(worker.send).toHaveBeenLastCalledWith({ + expect(worker.send).toHaveBeenLastCalledWith(expect.objectContaining({ type: 'message', content: payload.content, codexAppInput: { text: 'clean', clientUserMessageId: 'om_1' }, turnId: 'om_1', - }); + codexAppDispatchId: expect.any(String), + })); bot.config.codexAppCleanInput = undefined; expect(sendWorkerInput(ds, payload, 'om_2')).toBe(true); - expect(worker.send).toHaveBeenLastCalledWith({ + expect(worker.send).toHaveBeenLastCalledWith(expect.objectContaining({ type: 'message', content: payload.content, turnId: 'om_2', - }); + codexAppDispatchId: expect.any(String), + })); }); it('never applies the sidecar to a frozen non-Codex-App session', () => { @@ -1247,12 +2095,13 @@ describe('worker startup failure delivery', () => { content: 'legacy follow-up', codexAppInput: { text: 'clean follow-up' }, }, 'turn-live-clean')).toBe(true); - expect(worker.send).toHaveBeenLastCalledWith({ + expect(worker.send).toHaveBeenLastCalledWith(expect.objectContaining({ type: 'message', content: 'legacy follow-up', codexAppInput: { text: 'clean follow-up', clientUserMessageId: 'turn-live-clean' }, turnId: 'turn-live-clean', - }); + codexAppDispatchId: expect.any(String), + })); worker.emit('message', { type: 'error', message: 'CLI relaunch dependency disappeared', turnId: 'turn-live-clean' }); await Promise.resolve(); diff --git a/test/session-list-liveness.test.ts b/test/session-list-liveness.test.ts index 771feecb9..b067ec0b5 100644 --- a/test/session-list-liveness.test.ts +++ b/test/session-list-liveness.test.ts @@ -16,6 +16,13 @@ describe('botmux list session liveness', () => { )).toBe('prune_real'); }); + it('never auto-prunes an unsettled Codex App owner with no process markers', () => { + expect(sessionListDisposition( + { codexAppDispatchLedger: [{ state: 'prepared' }] }, + { hasPid: false, hasBackingSession: false }, + )).toBe('keep'); + }); + it('keeps live/backed sessions and distinguishes never-started scratch rows', () => { expect(sessionListDisposition({}, { hasPid: true, hasBackingSession: false })).toBe('keep'); expect(sessionListDisposition({}, { hasPid: false, hasBackingSession: true })).toBe('keep'); diff --git a/test/session-manager-auto-recover.test.ts b/test/session-manager-auto-recover.test.ts index c5d703cef..e62e5850c 100644 --- a/test/session-manager-auto-recover.test.ts +++ b/test/session-manager-auto-recover.test.ts @@ -43,7 +43,7 @@ describe('shouldAutoForkOnRestore', () => { describe('staggeredRecoveryFork', () => { const ds = (id: string, worker: unknown = null) => - ({ worker, session: { sessionId: id } } as unknown as DaemonSession); + ({ worker, session: { sessionId: id, status: 'active' } } as unknown as DaemonSession); it('re-forks every queued session', async () => { const forked: string[] = []; @@ -67,6 +67,20 @@ describe('staggeredRecoveryFork', () => { expect(forked).toEqual(['a', 'c']); // 'live' already has a worker — not clobbered }); + it('isolates a synchronous recovery fork failure and continues with later owners', async () => { + const forked: string[] = []; + await expect(staggeredRecoveryFork( + [ds('broken'), ds('healthy')], + current => { + if (current.session.sessionId === 'broken') throw new Error('init IPC rejected'); + forked.push(current.session.sessionId); + }, + 5, + 0, + )).resolves.toBeUndefined(); + expect(forked).toEqual(['healthy']); + }); + it('staggers in batches (delay only kicks in between batches)', async () => { const sessions = Array.from({ length: 5 }, (_, i) => ds(`s${i}`)); const forked: string[] = []; @@ -76,4 +90,25 @@ describe('staggeredRecoveryFork', () => { expect(forked).toHaveLength(5); expect(Date.now() - start).toBeGreaterThanOrEqual(30); }); + + it('rechecks exact ownership after a batch delay and never forks a replaced session', async () => { + const a = ds('a'); + const b = ds('b'); + const owned = new Set([a, b]); + const forked: string[] = []; + setTimeout(() => { + owned.delete(b); + b.session.status = 'closed'; + }, 5); + + await staggeredRecoveryFork( + [a, b], + current => forked.push(current.session.sessionId), + 1, + 20, + current => owned.has(current), + ); + + expect(forked).toEqual(['a']); + }); }); diff --git a/test/session-mutation-guard.test.ts b/test/session-mutation-guard.test.ts new file mode 100644 index 000000000..44b18fcbd --- /dev/null +++ b/test/session-mutation-guard.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; +import type { DaemonSession } from '../src/core/types.js'; +import type { Session } from '../src/types.js'; +import { + hasProtectedSessionMutationOwnership, + protectedSessionMutationReasons, +} from '../src/core/session-mutation-guard.js'; + +function session(overrides: Partial = {}): Session { + return { + sessionId: 'guard-session', + chatId: 'oc_chat', + rootMessageId: 'om_root', + title: 'guard', + status: 'active', + createdAt: new Date('2026-01-01T00:00:00Z').toISOString(), + ...overrides, + }; +} + +describe('hasProtectedSessionMutationOwnership', () => { + it.each([ + ['generic activation head', { queuedActivationPending: true }], + ['durable successor tail', { + queuedActivationTail: [{ + id: 'tail-1', order: 1, userPrompt: 'N+1', + cliInput: { content: 'N+1' }, turnId: 'turn-n-plus-1', + }], + }], + ['pending repository setup', { + pendingRepoSetup: { mode: 'picker', prompt: 'OPENING_N' }, + }], + ['dashboard backlog', { queued: true, queuedPrompt: 'OPENING_N' }], + ['Codex App ledger', { + cliId: 'codex-app', + codexAppDispatchLedger: [{ + dispatchId: 'dispatch-n', turnId: 'turn-n', state: 'accepted', content: 'N', + }], + }], + ] as const)('protects %s', (_name, overrides) => { + expect(hasProtectedSessionMutationOwnership(session(overrides as Partial))).toBe(true); + }); + + it('protects a runtime-only opening claim but leaves a truly idle row mutable', () => { + const ds = { + session: session(), + initialStartPending: true, + } as DaemonSession; + expect(hasProtectedSessionMutationOwnership(ds)).toBe(true); + expect(hasProtectedSessionMutationOwnership(session())).toBe(false); + }); + + it('classifies backend-neutral ownership separately from Codex App dispatches', () => { + expect(protectedSessionMutationReasons(session({ + cliId: 'traex', + queued: true, + pendingRepoSetup: { mode: 'picker', prompt: 'OPENING_N' }, + }))).toEqual(['queued_todo', 'repository_setup']); + expect(protectedSessionMutationReasons(session({ + cliId: 'codex-app', + codexAppDispatchLedger: [{ + dispatchId: 'dispatch-n', turnId: 'turn-n', state: 'accepted', content: 'N', + }], + }))).toEqual(['codex_app_dispatch']); + }); +}); diff --git a/test/session-ready-cli.test.ts b/test/session-ready-cli.test.ts index 8b7061c4d..d63bb3a44 100644 --- a/test/session-ready-cli.test.ts +++ b/test/session-ready-cli.test.ts @@ -65,6 +65,7 @@ describe('botmux session-ready — isolated CLI fallback', () => { writeFileSync( join(relayDir, RELAY_ORIGIN_CAPABILITY_BASENAME), JSON.stringify({ token: capability }), + { mode: 0o600 }, ); let receivedBody = ''; diff --git a/test/session-reply-thread-anchor.test.ts b/test/session-reply-thread-anchor.test.ts index a5b029359..7bca6e624 100644 --- a/test/session-reply-thread-anchor.test.ts +++ b/test/session-reply-thread-anchor.test.ts @@ -118,6 +118,18 @@ describe('sessionReply chat-scope chokepoint — shared fold-back anchoring', () expect(mocks.replyMessage).not.toHaveBeenCalled(); }); + it('honors a daemon-frozen turn-A root after mutable session state advances to turn B', async () => { + seedSharedSession({ rootMessageId: 'om_topic_b', turnId: 'turn-b', updatedAt: NOW }); + await sessionReply(CHAT, 'late A', 'text', APP, 'turn-a', { + replyTarget: { mode: 'thread', rootMessageId: 'om_topic_a' }, + }); + + expect(mocks.replyMessage).toHaveBeenCalledWith( + APP, 'om_topic_a', 'late A', 'text', true, undefined, expect.anything(), + ); + expect(mocks.sendMessage).not.toHaveBeenCalled(); + }); + it('plain chat session (no fold-back anchor) keeps replying flat to the chat top-level', async () => { seedSharedSession(undefined); await sessionReply(CHAT, 'hello', 'text', APP); diff --git a/test/session-resume.test.ts b/test/session-resume.test.ts index 033f22d60..d511fecce 100644 --- a/test/session-resume.test.ts +++ b/test/session-resume.test.ts @@ -54,6 +54,7 @@ vi.mock('../src/core/worker-pool.js', () => ({ killStalePids: vi.fn(), getCurrentCliVersion: vi.fn(() => '1.0.0-test'), restoreUsageLimitRuntimeState: vi.fn(), + withActiveSessionKeyLock: vi.fn(async (_map: Map, _key: string, action: () => any) => action()), // Faithful: mirror the real setActiveSessionSafe — if a DIFFERENT entry // already holds the key, evict it (close) before setting, instead of a // bare overwrite that would mask a lingering occupant. @@ -63,6 +64,7 @@ vi.mock('../src/core/worker-pool.js', () => ({ for (const [k, v] of map) { if (v === prev) { map.delete(k); break; } } } map.set(key, ds); + return { accepted: true }; }), // Real predicate (same logic as production): worker OR persisted CLI markers. isRelayableRealSession: (ds: any) => @@ -384,6 +386,51 @@ describe('resumeSession', () => { // Scratch store row should now be closed. expect(sessionStore.getSession(scratch.sessionId)!.status).toBe('closed'); }); + + it('keeps a fresh first owner that appears while resume awaits scratch cleanup', async () => { + const closed = makeClosedSession({ rootMessageId: 'om_resume_race' }); + const scratch = sessionStore.createSession('oc_chat1', 'om_resume_race', '/relay'); + scratch.larkAppId = 'app_test'; + scratch.scope = 'thread'; + scratch.cliId = undefined as any; + scratch.lastCliInput = undefined as any; + sessionStore.updateSession(scratch); + const map = new Map(); + wp.registry = map; + let releaseCleanup!: () => void; + let cleanupStarted!: () => void; + const paused = new Promise(resolve => { releaseCleanup = resolve; }); + const started = new Promise(resolve => { cleanupStarted = resolve; }); + vi.mocked(closeSession).mockImplementationOnce(async (sid: string) => { + cleanupStarted(); + await paused; + sessionStore.closeSession(sid); + return { ok: true, alreadyClosed: false } as any; + }); + + const resuming = resumeSession(closed.sessionId, map); + await started; + const key = sessionKey('om_resume_race', 'app_test'); + const fresh = { + session: { sessionId: 'fresh-first-owner', status: 'active', queued: false }, + worker: null, + initialStartPending: true, + larkAppId: 'app_test', + chatId: 'oc_chat1', + scope: 'thread', + } as any; + map.set(key, fresh); + releaseCleanup(); + + const result = await resuming; + expect(result).toEqual({ + ok: false, + error: 'anchor_occupied', + activeSessionId: 'fresh-first-owner', + }); + expect(map.get(key)).toBe(fresh); + expect(sessionStore.getSession(closed.sessionId)?.status).toBe('closed'); + }); }); describe('success path', () => { @@ -448,6 +495,51 @@ describe('resumeSession', () => { expect(restored?.session.replyThreadAliases?.om_materialized_root).toBeDefined(); }); + it.each([ + ['pending repo setup', (session: any) => { + session.queued = true; + session.queuedPrompt = 'abandoned picker prompt'; + session.pendingRepoSetup = { mode: 'picker', prompt: 'abandoned picker prompt', repoCardMessageId: 'om_old_picker' }; + }], + ['tokened activation head', (session: any) => { + session.queuedActivationPending = true; + session.queuedActivationToken = 'abandoned-token'; + session.queuedActivationInput = { content: 'abandoned head' }; + session.queuedActivationTurnId = 'abandoned-turn'; + session.queuedActivationDispatchAttempt = 3; + }], + ['activation tail', (session: any) => { + session.queuedActivationTail = [{ + id: 'abandoned-tail', order: 1, userPrompt: 'tail', cliInput: { content: 'abandoned tail' }, turnId: 'tail-turn', + }]; + session.queuedActivationTailNextOrder = 2; + }], + ] as const)('never revives legacy %s when a closed row is resumed', async (_label, injectLegacyState) => { + const closed = makeClosedSession({ rootMessageId: `om_legacy_${_label.replaceAll(' ', '_')}` }); + injectLegacyState(closed); + // Simulate a row written by an older release: closed status plus queued + // ownership that the historical close path did not remove. + sessionStore.updateSession(closed); + const map = new Map(); + + const result = await resumeSession(closed.sessionId, map); + + expect(result.ok).toBe(true); + if (!result.ok) return; + const persisted = sessionStore.getSession(closed.sessionId)!; + expect(persisted.status).toBe('active'); + expect(persisted.queued).toBeUndefined(); + expect(persisted.queuedPrompt).toBeUndefined(); + expect(persisted.pendingRepoSetup).toBeUndefined(); + expect(persisted.queuedActivationPending).toBeUndefined(); + expect(persisted.queuedActivationToken).toBeUndefined(); + expect(persisted.queuedActivationInput).toBeUndefined(); + expect(persisted.queuedActivationTail).toBeUndefined(); + expect(persisted.queuedActivationTailNextOrder).toBeUndefined(); + expect(result.ds.initialStartPending).toBeFalsy(); + expect(result.ds.pendingRepo).toBeFalsy(); + }); + it('restores dedicated VC receivers without collapsing them into the ordinary chat slot', async () => { const make = (title: string, receiver?: { meetingId: string; memberId: string }) => { const s = sessionStore.createSession('oc_listener', 'oc_listener', title, 'group'); diff --git a/test/session-store.test.ts b/test/session-store.test.ts index 7315ec950..e4a0a90a1 100644 --- a/test/session-store.test.ts +++ b/test/session-store.test.ts @@ -61,6 +61,7 @@ import { getSession, listSessions, closeSession, + reactivateClosedSession, updateSession, updateSessionPid, findActiveSessionsByRoot, @@ -388,6 +389,39 @@ describe('closeSession()', () => { }); }); +describe('reactivateClosedSession()', () => { + it('sanitizes queued/setup state left on a legacy closed row', () => { + const session = createSession('chat1', 'root1', 'Legacy Closed Queue'); + closeSession(session.sessionId); + const legacy = getSession(session.sessionId)!; + legacy.queued = true; + legacy.queuedPrompt = 'legacy backlog'; + legacy.pendingRepoSetup = { mode: 'picker', prompt: 'legacy picker' }; + legacy.queuedActivationPending = true; + legacy.queuedActivationToken = 'legacy-token'; + legacy.queuedActivationInput = { content: 'legacy head' }; + legacy.queuedActivationTail = [{ + id: 'legacy-tail', order: 1, userPrompt: 'tail', cliInput: { content: 'legacy tail' }, turnId: 'tail-turn', + }]; + legacy.queuedActivationTailNextOrder = 2; + updateSession(legacy); + + const result = reactivateClosedSession(session.sessionId); + expect(result.ok).toBe(true); + init(); + + const reloaded = getSession(session.sessionId)!; + expect(reloaded.status).toBe('active'); + expect(reloaded.closedAt).toBeUndefined(); + expect(reloaded.queued).toBeUndefined(); + expect(reloaded.pendingRepoSetup).toBeUndefined(); + expect(reloaded.queuedActivationPending).toBeUndefined(); + expect(reloaded.queuedActivationToken).toBeUndefined(); + expect(reloaded.queuedActivationInput).toBeUndefined(); + expect(reloaded.queuedActivationTail).toBeUndefined(); + }); +}); + // ─── updateSession() ───────────────────────────────────────────────────── describe('updateSession()', () => { diff --git a/test/tmux-backend-env.test.ts b/test/tmux-backend-env.test.ts index bf7a5b6b9..69721f998 100644 --- a/test/tmux-backend-env.test.ts +++ b/test/tmux-backend-env.test.ts @@ -133,14 +133,17 @@ describe('buildBotmuxEnvAssignments()', () => { expect(out).not.toContain('PATH=/usr/bin'); }); - it('forwards the worker-owned MCP relay capability into the CLI pane', () => { + it('forwards only a Codex App bootstrap path and strips the retired shared-secret env', () => { + const retiredSharedSecret = 'A'.repeat(43); + const bootstrapPath = '/private/bot-home/control.bootstrap'; const out = buildBotmuxEnvAssignments({ BOTMUX: '1', - BOTMUX_MCP_GATEWAY_SOCKET: '/tmp/botmux-mcp/session/gateway.sock', - BOTMUX_MCP_GATEWAY_REQUIRED: '1', + BOTMUX_CODEX_APP_CONTROL_NONCE: retiredSharedSecret, + BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP: bootstrapPath, }); - expect(out).toContain('BOTMUX_MCP_GATEWAY_SOCKET=/tmp/botmux-mcp/session/gateway.sock'); - expect(out).toContain('BOTMUX_MCP_GATEWAY_REQUIRED=1'); + expect(out).toContain(`BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP=${bootstrapPath}`); + expect(out.join(' ')).not.toContain(retiredSharedSecret); + expect(out.some(value => value.startsWith('BOTMUX_CODEX_APP_CONTROL_NONCE='))).toBe(false); }); it('forwards Hermes profile paths so the pane and transcript reader use the same state DB', () => { diff --git a/test/tmux-reattach-backend.test.ts b/test/tmux-reattach-backend.test.ts index c6348ba3f..c9f3bba63 100644 --- a/test/tmux-reattach-backend.test.ts +++ b/test/tmux-reattach-backend.test.ts @@ -230,4 +230,14 @@ describe('selectSessionBackend', () => { expect(selected.backend.constructor.name).toBe('MockZellijBackend'); expect((selected.backend as any).opts).toEqual({ ownsSession: true, isReattach: false }); }); + + it('marks an existing zellij session as reattach without making it pipe mode', () => { + vi.mocked(ZellijBackend.hasSession).mockReturnValue(true); + + const selected = selectSessionBackend({ sessionId: '9cfa0024-197d-4781-845b-c541dceb8980', backendType: 'zellij' }); + + expect(selected.isZellijMode).toBe(true); + expect(selected.isPipeMode).toBe(false); + expect((selected.backend as any).opts).toEqual({ ownsSession: true, isReattach: true }); + }); }); diff --git a/test/transfer-session.test.ts b/test/transfer-session.test.ts index 6d8264a2c..238fbd382 100644 --- a/test/transfer-session.test.ts +++ b/test/transfer-session.test.ts @@ -16,6 +16,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; vi.mock('../src/services/session-store.js', () => ({ updateSession: vi.fn(), getSession: vi.fn(), + listSessions: vi.fn(() => []), closeSession: vi.fn(), })); @@ -53,6 +54,10 @@ import { dashboardEventBus } from '../src/core/dashboard-events.js'; import { sessionKey } from '../src/core/types.js'; import type { DaemonSession } from '../src/core/types.js'; import type { Session } from '../src/types.js'; +import { + __testOnly_resetBotTurnMutationGates, + withBotTurnAdmission, +} from '../src/core/bot-turn-mutation-gate.js'; function makeDs(overrides: Partial = {}): DaemonSession { const session: Session = { @@ -94,6 +99,13 @@ function makeDs(overrides: Partial = {}): DaemonSession { } as DaemonSession; } +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); + return { promise, resolve, reject }; +} + describe('transferSession', () => { let registry: Map; @@ -112,6 +124,8 @@ describe('transferSession', () => { beforeEach(() => { vi.clearAllMocks(); + __testOnly_resetBotTurnMutationGates(); + vi.mocked(sessionStore.listSessions).mockReturnValue([]); registry = new Map(); setActiveSessionsRegistry(registry); }); @@ -177,6 +191,77 @@ describe('transferSession', () => { expect(adoptDs.chatId).toBe('oc_source'); }); + it('refuses transfer while the durable Codex App dispatch ledger is non-empty', async () => { + const ds = makeDs(); + ds.session.codexAppDispatchLedger = [ + { dispatchId: 'd-1', turnId: 't-1', state: 'prepared', content: 'owned' }, + ]; + registry.set(sessionKey('om_source_root', 'cli_app_test'), ds); + + const result = await callTransfer(ds.session.sessionId, 'oc_target', 'om_target_root'); + + expect(result).toEqual({ ok: false, error: 'codex_app_dispatch_pending' }); + expect(forkWorkerSpy).not.toHaveBeenCalled(); + expect(killWorkerSpy).not.toHaveBeenCalled(); + expect(sessionStore.updateSession).not.toHaveBeenCalled(); + expect(ds.chatId).toBe('oc_source'); + expect(ds.session.rootMessageId).toBe('om_source_root'); + expect(registry.get(sessionKey('om_source_root', 'cli_app_test'))).toBe(ds); + }); + + it('drains a pre-accept turn and rechecks durable ownership before transfer', async () => { + const ds = makeDs(); + registry.set(sessionKey('om_source_root', 'cli_app_test'), ds); + let release!: () => void; + let markStarted!: () => void; + const paused = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { markStarted = resolve; }); + const inbound = withBotTurnAdmission(ds.larkAppId, async () => { + markStarted(); + await paused; + ds.session.codexAppDispatchLedger = [{ + dispatchId: 'd-raced', + turnId: 't-raced', + state: 'accepted', + content: 'accepted before transfer', + }]; + }); + await started; + + const transfer = callTransfer(ds.session.sessionId, 'oc_target', 'om_target_root'); + await Promise.resolve(); + expect(killWorkerSpy).not.toHaveBeenCalled(); + release(); + await inbound; + + expect(await transfer).toEqual({ ok: false, error: 'codex_app_dispatch_pending' }); + expect(killWorkerSpy).not.toHaveBeenCalled(); + expect(forkWorkerSpy).not.toHaveBeenCalled(); + expect(registry.get(sessionKey('om_source_root', ds.larkAppId))).toBe(ds); + }); + + it('does not kill or overwrite a source successor discovered after cleanup awaits', async () => { + const ds = makeDs(); + const sourceKey = sessionKey('om_source_root', 'cli_app_test'); + registry.set(sourceKey, ds); + const successor = makeDs({ + session: { ...ds.session, sessionId: 'source-successor' }, + }); + vi.mocked(sessionStore.listSessions).mockImplementationOnce(() => { + ds.session.status = 'closed'; + registry.set(sourceKey, successor); + return []; + }); + + const result = await callTransfer(ds.session.sessionId, 'oc_target', 'om_target_root'); + + expect(result).toEqual({ ok: false, error: 'session_not_active' }); + expect(registry.get(sourceKey)).toBe(successor); + expect(killWorkerSpy).not.toHaveBeenCalled(); + expect(forkWorkerSpy).not.toHaveBeenCalled(); + expect(ds.chatId).toBe('oc_source'); + }); + it('returns same_anchor when a chat-scope source targets its own chat (chat→chat)', async () => { const ds = makeDs({ scope: 'chat' }); ds.session.scope = 'chat'; @@ -407,6 +492,72 @@ describe('transferSession', () => { expect(registry.get(sessionKey('oc_target', 'cli_app_test'))).toBe(existingDs); }); + it('refuses a disk-only legacy target owner with an unsettled Codex App dispatch', async () => { + const movingDs = makeDs(); + registry.set(sessionKey('om_source_root', 'cli_app_test'), movingDs); + const legacyConflict: Session = { + ...movingDs.session, + sessionId: 'legacy-disk-only-owner', + chatId: 'oc_target', + rootMessageId: 'om_legacy_target', + scope: 'chat', + larkAppId: undefined, + codexAppDispatchLedger: [{ + dispatchId: 'legacy-dispatch', + turnId: 'legacy-turn', + state: 'prepared', + content: 'owned output', + deliverySink: 'lark', + }], + }; + vi.mocked(sessionStore.listSessions).mockReturnValue([legacyConflict]); + + const result = await callTransfer( + movingDs.session.sessionId, + 'oc_target', + 'om_M1_target', + ); + + expect(result).toEqual({ ok: false, error: 'target_chat_has_session' }); + expect(sessionStore.closeSession).not.toHaveBeenCalled(); + expect(killWorkerSpy).not.toHaveBeenCalled(); + expect(forkWorkerSpy).not.toHaveBeenCalled(); + expect(movingDs.chatId).toBe('oc_source'); + expect(registry.get(sessionKey('om_source_root', 'cli_app_test'))).toBe(movingDs); + }); + + it('retires a disk-only legacy scratch before claiming the target anchor', async () => { + const movingDs = makeDs(); + registry.set(sessionKey('om_source_root', 'cli_app_test'), movingDs); + const legacyScratch: Session = { + ...movingDs.session, + sessionId: 'legacy-disk-only-scratch', + chatId: 'oc_target', + rootMessageId: 'om_legacy_scratch', + scope: 'chat', + larkAppId: undefined, + cliId: undefined, + lastCliInput: undefined, + queued: false, + codexAppDispatchLedger: [], + }; + vi.mocked(sessionStore.listSessions).mockReturnValue([legacyScratch]); + vi.mocked(sessionStore.getSession).mockImplementation((sid: string) => + sid === legacyScratch.sessionId ? legacyScratch : undefined, + ); + + const result = await callTransfer( + movingDs.session.sessionId, + 'oc_target', + 'om_M1_target', + ); + + expect(result).toEqual({ ok: true }); + expect(sessionStore.closeSession).toHaveBeenCalledWith(legacyScratch.sessionId); + expect(registry.get(sessionKey('oc_target', 'cli_app_test'))).toBe(movingDs); + expect(forkWorkerSpy).toHaveBeenCalledTimes(1); + }); + it('closes the daemon-command scratch session occupying the target chat slot', async () => { // Regression: a /relay command in the target chat creates a placeholder // session record with `worker: null`. Previously the pre-flight scan @@ -427,6 +578,8 @@ describe('transferSession', () => { rootMessageId: 'om_relay_cmd_msg', scope: 'chat', title: '/relay', + cliId: undefined, + lastCliInput: undefined, }, worker: null, // command-time placeholder, no real worker chatId: 'oc_target', @@ -500,6 +653,28 @@ describe('transferSession', () => { expect(body).toMatch(/"img_key":\s*"old_image_key"/); }); + it('reattaches at the routing commit before awaiting the source-card patch', async () => { + const ds = makeDs(); + registry.set(sessionKey('om_source_root', 'cli_app_test'), ds); + const patch = deferred(); + updateMessageMock.mockImplementationOnce(() => patch.promise); + + const transferring = callTransfer(ds.session.sessionId, 'oc_target', 'om_M1_target'); + await vi.waitFor(() => expect(updateMessageMock).toHaveBeenCalledTimes(1)); + + // The target owner is already runnable before the best-effort Lark PATCH. + // A close that wins during that await must not be followed by a stale fork. + expect(forkWorkerSpy).toHaveBeenCalledTimes(1); + expect(registry.get(sessionKey('oc_target', 'cli_app_test'))).toBe(ds); + registry.delete(sessionKey('oc_target', 'cli_app_test')); + ds.session.status = 'closed'; + + patch.resolve(); + await expect(transferring).resolves.toEqual({ ok: true }); + expect(forkWorkerSpy).toHaveBeenCalledTimes(1); + expect(registry.has(sessionKey('oc_target', 'cli_app_test'))).toBe(false); + }); + it('frozen card renders no extra element when no currentImageKey is set (hidden mode)', async () => { // Sessions in hidden / collapsed display mode never produced a server- // rendered screenshot, so currentImageKey is undefined. We deliberately @@ -640,4 +815,64 @@ describe('setActiveSessionSafe', () => { expect(registry.get(key)).toBe(ds); expect(sessionStore.closeSession).not.toHaveBeenCalled(); }); + + it('keeps an unsettled incumbent and closes a ledger-empty incoming collision', async () => { + const incumbent = makeSimpleDs('pending-incumbent'); + incumbent.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-incumbent', + turnId: 'turn-incumbent', + state: 'prepared', + content: 'owned', + deliverySink: 'lark', + }]; + const incoming = makeSimpleDs('ledger-empty-incoming'); + vi.mocked(sessionStore.getSession).mockImplementation((sid: string) => + sid === incoming.session.sessionId ? incoming.session : undefined, + ); + const key = sessionKey('oc_c', 'cli_app_test'); + registry.set(key, incumbent); + + const result = await setActiveSessionSafe(registry, key, incoming); + + expect(result).toEqual({ + accepted: false, + reason: 'kept_pending_owner', + keptSessionId: incumbent.session.sessionId, + closedIncomingSessionId: incoming.session.sessionId, + }); + expect(registry.get(key)).toBe(incumbent); + expect(sessionStore.closeSession).toHaveBeenCalledWith(incoming.session.sessionId); + }); + + it('fails closed and preserves both rows when both colliding owners are unsettled', async () => { + const incumbent = makeSimpleDs('pending-incumbent'); + incumbent.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-incumbent', + turnId: 'turn-incumbent', + state: 'prepared', + content: 'owned incumbent', + deliverySink: 'lark', + }]; + const incoming = makeSimpleDs('pending-incoming'); + incoming.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-incoming', + turnId: 'turn-incoming', + state: 'prepared', + content: 'owned incoming', + deliverySink: 'lark', + }]; + const key = sessionKey('oc_c', 'cli_app_test'); + registry.set(key, incumbent); + + const result = await setActiveSessionSafe(registry, key, incoming); + + expect(result).toEqual({ + accepted: false, + reason: 'both_pending', + keptSessionId: incumbent.session.sessionId, + preservedIncomingSessionId: incoming.session.sessionId, + }); + expect(registry.get(key)).toBe(incumbent); + expect(sessionStore.closeSession).not.toHaveBeenCalled(); + }); }); diff --git a/test/trigger-session-root-message.test.ts b/test/trigger-session-root-message.test.ts index 9be5629f8..9abc5dd03 100644 --- a/test/trigger-session-root-message.test.ts +++ b/test/trigger-session-root-message.test.ts @@ -29,9 +29,11 @@ vi.mock('../src/services/oncall-store.js', () => ({ const mockCreateSession = vi.fn(); const mockUpdateSession = vi.fn(); +const mockCloseSession = vi.fn(); vi.mock('../src/services/session-store.js', () => ({ createSession: (...args: any[]) => mockCreateSession(...args), updateSession: (...args: any[]) => mockUpdateSession(...args), + closeSession: (...args: any[]) => mockCloseSession(...args), })); vi.mock('../src/services/message-queue.js', () => ({ @@ -39,10 +41,18 @@ vi.mock('../src/services/message-queue.js', () => ({ })); const mockForkWorker = vi.fn(); +const mockQueuedTailAdmission = vi.fn(); +const activeKeyLocks = vi.hoisted(() => ({ + byMap: new WeakMap, Map>>(), +})); vi.mock('../src/core/worker-pool.js', () => ({ forkWorker: (...args: any[]) => mockForkWorker(...args), sendWorkerInput: (ds: any, payload: any, turnId?: string, opts: any = {}) => { if (!ds.worker || ds.worker.killed) return false; + const gated = ds.session.queuedActivationPending === true + || (ds.session.queuedActivationTail?.length ?? 0) > 0 + || (ds.initialStartPending === true && ds.session.queuedActivationInput !== undefined); + if (gated) return mockQueuedTailAdmission(ds, payload, turnId, opts); ds.worker.send({ type: 'message', content: typeof payload === 'string' ? payload : payload.content, @@ -56,10 +66,32 @@ vi.mock('../src/core/worker-pool.js', () => ({ }); return true; }, + hasQueuedActivationAdmissionGate: (ds: any) => ds.session.queuedActivationPending === true + || (ds.session.queuedActivationTail?.length ?? 0) > 0 + || (ds.initialStartPending === true && ds.session.queuedActivationInput !== undefined), getCurrentCliVersion: vi.fn(() => 'test-cli-version'), + withActiveSessionKeyLock: vi.fn(async (map: Map, key: string, action: () => any) => { + let locks = activeKeyLocks.byMap.get(map); + if (!locks) { + locks = new Map(); + activeKeyLocks.byMap.set(map, locks); + } + const previous = locks.get(key) ?? Promise.resolve(); + let release!: () => void; + const hold = new Promise(resolve => { release = resolve; }); + const tail = previous.catch(() => {}).then(() => hold); + locks.set(key, tail); + await previous.catch(() => {}); + try { return await action(); } + finally { + release(); + if (locks.get(key) === tail) locks.delete(key); + } + }), })); const mockRememberLastCliInput = vi.fn(); +const mockGetAvailableBots = vi.fn(async () => []); const mockBuildFollowUpCliInput = vi.fn((prompt: string, _sessionId?: string, opts?: any) => ({ content: `follow:${prompt}`, codexAppInput: opts?.cliId === 'codex-app' && opts?.codexAppText ? { text: opts.codexAppText } : undefined, @@ -74,7 +106,7 @@ vi.mock('../src/core/session-manager.js', () => ({ buildNewTopicPrompt: vi.fn((prompt: string) => `new:${prompt}`), buildNewTopicCliInput: (...args: any[]) => mockBuildNewTopicCliInput(...args), ensureSessionWhiteboard: vi.fn(), - getAvailableBots: vi.fn(async () => []), + getAvailableBots: (...args: any[]) => mockGetAvailableBots(...args), rememberLastCliInput: (...args: any[]) => mockRememberLastCliInput(...args), })); @@ -90,6 +122,7 @@ vi.mock('../src/im/lark/card-handler.js', () => ({ import { buildExternalEventTopicMessage, triggerSessionTurn } from '../src/core/trigger-session.js'; import { sessionKey } from '../src/core/types.js'; +import { withActiveSessionKeyLock } from '../src/core/worker-pool.js'; const APP = 'app1'; const CHAT = 'oc_root_chat'; @@ -136,6 +169,25 @@ describe('triggerSessionTurn rootMessageId target', () => { botOpenId: 'ou_bot', }); mockGetMessageChatId.mockResolvedValue(CHAT); + mockQueuedTailAdmission.mockImplementation((ds: any, payload: any, turnId?: string, opts: any = {}) => { + const order = (ds.session.queuedActivationTailNextOrder ?? 0) + 1; + ds.session.queuedActivationTailNextOrder = order; + ds.session.queuedActivationTail = [ + ...(ds.session.queuedActivationTail ?? []), + { + id: `tail-${order}`, + order, + userPrompt: typeof payload === 'string' ? payload : payload.content, + cliInput: typeof payload === 'string' ? { content: payload } : payload, + turnId: turnId ?? `tail-turn-${order}`, + ...(opts.dispatchAttempt !== undefined + ? { dispatchAttempt: opts.dispatchAttempt } + : {}), + }, + ]; + mockUpdateSession(ds.session); + return true; + }); mockCreateSession.mockImplementation((chatId: string, rootMessageId: string, title: string, chatType: 'group' | 'p2p') => ({ sessionId: 'sess_new', chatId, @@ -224,6 +276,78 @@ describe('triggerSessionTurn rootMessageId target', () => { expect(send).toHaveBeenCalledWith({ type: 'message', content: expect.stringContaining('follow:') }); }); + it.each([ + ['normal opening', undefined], + ['raw text-to-Enter opening', '/goal OPENING_RAW_N'], + ])('durably queues an external trigger behind a live %s activation', async (_label, pendingRawInput) => { + const send = vi.fn(); + const ds = existingDs({ + worker: { killed: false, send } as any, + initialStartPending: true, + ...(pendingRawInput ? { pendingRawInput } : {}), + }); + Object.assign(ds.session, { + cliId: 'claude-code', + queuedActivationPending: true, + queuedActivationToken: 'opening-token', + queuedActivationInput: { content: pendingRawInput ? '' : 'OPENING_N' }, + queuedActivationTurnId: 'turn-opening', + }); + const beforeDispatch = vi.fn(() => ({ dispatchAttempt: 4 })); + + const res = await triggerSessionTurn( + request(), + { larkAppId: APP, activeSessions: new Map([[sessionKey(ROOT, APP), ds]]) }, + { stableTurnId: 'external-follower-n1', beforeDispatch }, + ); + + expect(res).toMatchObject({ + ok: true, + triggerId: 'external-follower-n1', + action: 'queued', + }); + expect(send).not.toHaveBeenCalled(); + expect(mockQueuedTailAdmission).toHaveBeenCalledWith( + ds, + expect.objectContaining({ content: expect.stringContaining('follow:') }), + 'external-follower-n1', + { dispatchAttempt: 4 }, + ); + expect(ds.session.queuedActivationTail).toEqual([ + expect.objectContaining({ + turnId: 'external-follower-n1', + dispatchAttempt: 4, + cliInput: expect.objectContaining({ content: expect.stringContaining('follow:') }), + }), + ]); + expect(mockRememberLastCliInput).toHaveBeenCalled(); + }); + + it('reports failure and does not send or persist input history when gated tail admission fails', async () => { + const send = vi.fn(); + const ds = existingDs({ + worker: { killed: false, send } as any, + initialStartPending: true, + }); + Object.assign(ds.session, { + cliId: 'claude-code', + queuedActivationPending: true, + queuedActivationToken: 'opening-token', + queuedActivationInput: { content: 'OPENING_N' }, + }); + mockQueuedTailAdmission.mockReturnValueOnce(false); + + const res = await triggerSessionTurn(request(), { + larkAppId: APP, + activeSessions: new Map([[sessionKey(ROOT, APP), ds]]), + }); + + expect(res).toMatchObject({ ok: false, errorCode: 'trigger_failed' }); + expect(send).not.toHaveBeenCalled(); + expect(ds.session.queuedActivationTail).toBeUndefined(); + expect(mockRememberLastCliInput).not.toHaveBeenCalled(); + }); + it('uses an internal stable turn id without changing the public trigger schema', async () => { const send = vi.fn(); const ds = existingDs({ worker: { killed: false, send } as any, workerGeneration: 7 }); @@ -334,6 +458,31 @@ describe('triggerSessionTurn rootMessageId target', () => { expect(mockForkWorker).toHaveBeenCalledWith(ds, { content: expect.stringContaining('follow:') }, { resume: true, turnId: expect.stringMatching(/^trg_/) }); }); + it('fails closed for a dormant ledger-only owner instead of reforking over it', async () => { + const ds = existingDs(); + ds.session.cliId = 'codex-app'; + ds.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-owned', + turnId: 'turn-owned', + state: 'prepared', + content: 'durable existing turn', + deliverySink: 'lark', + }]; + + const res = await triggerSessionTurn(request(), { + larkAppId: APP, + activeSessions: new Map([[sessionKey(ROOT, APP), ds]]), + }); + + expect(res).toMatchObject({ + ok: false, + errorCode: 'trigger_failed', + error: expect.stringContaining('durable_owner'), + }); + expect(mockForkWorker).not.toHaveBeenCalled(); + expect(mockRememberLastCliInput).not.toHaveBeenCalled(); + }); + it('preserves the clean split when an external event reforks a stopped Codex App session', async () => { mockGetBot.mockReturnValue({ config: { larkAppId: APP, cliId: 'codex-app', codexAppCleanInput: true, workingDir: '/tmp' }, @@ -379,6 +528,37 @@ describe('triggerSessionTurn rootMessageId target', () => { expect(mockRunAutoWorktreeCommit).toHaveBeenCalledWith(expect.objectContaining({ ds })); }); + it('closes the unpublished trigger row when auto-worktree setup persistence fails', async () => { + mockBotAutoWorktreeEnabled.mockReturnValue(true); + mockUpdateSession + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { throw new Error('setup store unavailable'); }); + const activeSessions = new Map(); + + await expect(triggerSessionTurn(request(), { larkAppId: APP, activeSessions })) + .rejects.toThrow('setup store unavailable'); + + expect(activeSessions.has(sessionKey(ROOT, APP))).toBe(false); + expect(mockCloseSession).toHaveBeenCalledTimes(1); + expect(mockCloseSession).toHaveBeenCalledWith('sess_new'); + expect(mockRunAutoWorktreeCommit).not.toHaveBeenCalled(); + expect(mockForkWorker).not.toHaveBeenCalled(); + }); + + it('never closes an incumbent when the trigger creation path loses the key claim', async () => { + mockBotAutoWorktreeEnabled.mockReturnValue(true); + const send = vi.fn(); + const incumbent = existingDs({ worker: { killed: false, send } as any }); + const activeSessions = new Map([[sessionKey(ROOT, APP), incumbent]]); + + const result = await triggerSessionTurn(request(), { larkAppId: APP, activeSessions }); + + expect(result).toMatchObject({ ok: true, target: { sessionId: incumbent.session.sessionId } }); + expect(activeSessions.get(sessionKey(ROOT, APP))).toBe(incumbent); + expect(mockCloseSession).not.toHaveBeenCalled(); + expect(mockCreateSession).not.toHaveBeenCalled(); + }); + it('passes the clean split into a new Codex App session without worktree staging', async () => { mockGetBot.mockReturnValue({ config: { larkAppId: APP, cliId: 'codex-app', codexAppCleanInput: true, workingDir: '/tmp' }, @@ -506,6 +686,72 @@ describe('triggerSessionTurn rootMessageId target', () => { expect(mockCreateSession).not.toHaveBeenCalled(); }); + it('shares first-owner locking with a paused resume claim and reuses the resume winner', async () => { + const activeSessions = new Map(); + const key = sessionKey(ROOT, APP); + const resumed = existingDs(); + resumed.session.cliId = 'claude-code'; + let resumeEntered!: () => void; + let releaseResume!: () => void; + const entered = new Promise(resolve => { resumeEntered = resolve; }); + const paused = new Promise(resolve => { releaseResume = resolve; }); + + const resumeClaim = withActiveSessionKeyLock(activeSessions, key, async () => { + expect(activeSessions.has(key)).toBe(false); + resumeEntered(); + await paused; + activeSessions.set(key, resumed); + }); + await entered; + + const triggering = triggerSessionTurn(request(), { larkAppId: APP, activeSessions }); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(mockCreateSession).not.toHaveBeenCalled(); + + releaseResume(); + await resumeClaim; + const result = await triggering; + + expect(result).toMatchObject({ ok: true, target: { sessionId: 'sess_existing' } }); + expect(activeSessions.get(key)).toBe(resumed); + expect(mockCreateSession).not.toHaveBeenCalled(); + expect(mockForkWorker).toHaveBeenCalledWith(resumed, expect.anything(), expect.objectContaining({ resume: true })); + }); + + it('keeps the opening reservation and buffers when new-session fork pre-accept throws', async () => { + mockForkWorker.mockImplementationOnce(() => { throw new Error('fork preaccept failed'); }); + const activeSessions = new Map(); + + await expect(triggerSessionTurn(request(), { larkAppId: APP, activeSessions })) + .rejects.toThrow('fork preaccept failed'); + + const ds = activeSessions.get(sessionKey(ROOT, APP)); + expect(ds?.initialStartPending).toBe(true); + expect(ds?.pendingPrompt).toContain(''); + expect(ds?.pendingCodexAppText).toBe('外部事件触发'); + }); + + it('cleans the wait registry but retains the opening reservation when write-ahead throws', async () => { + const activeSessions = new Map(); + const req = request(); + req.options = { waitForFinalOutput: true, timeoutMs: 1000 }; + + const result = await triggerSessionTurn( + req, + { larkAppId: APP, activeSessions }, + { + stableTurnId: 'stable_write_ahead_failure', + beforeDispatch: () => { throw new Error('write-ahead failed'); }, + }, + ); + + expect(result).toMatchObject({ ok: false, errorCode: 'trigger_failed', error: 'write-ahead failed' }); + const ds = activeSessions.get(sessionKey(ROOT, APP)); + expect(ds?.initialStartPending).toBe(true); + expect(ds?.pendingWaitPromises?.size ?? 0).toBe(0); + expect(mockForkWorker).not.toHaveBeenCalled(); + }); + it('requires chatId when rootMessageId is specified', async () => { const activeSessions = new Map(); const res = await triggerSessionTurn(request({ chatId: undefined }), { larkAppId: APP, activeSessions }); diff --git a/test/vc-meeting-daemon-session.test.ts b/test/vc-meeting-daemon-session.test.ts index 205c671fb..75e929ab2 100644 --- a/test/vc-meeting-daemon-session.test.ts +++ b/test/vc-meeting-daemon-session.test.ts @@ -614,22 +614,27 @@ async function waitForConsumerApplyFinalCard(afterIndex = patchedMessages.length } /** 新交互流程：下拉选 agent 只暂存，点"确认"才生效。返回确认后的卡片响应。 */ -async function selectConsumerAgentViaCard(label: string, operatorOpenId = TARGET_OPEN_ID): Promise { - await __vcMeetingAgentTest.handleCardAction({ - operator: { open_id: operatorOpenId }, - action: lastInteractiveCardSelectOption(label), - }, APP_ID); +async function confirmLatestConsumerCard(operatorOpenId = TARGET_OPEN_ID): Promise { const patchIndex = patchedMessages.length; const result = await __vcMeetingAgentTest.handleCardAction({ operator: { open_id: operatorOpenId }, action: { value: lastInteractiveCardButton('确认') }, }, APP_ID); - if (result?.header?.title?.content === '会议处理设置中') { + if (result?.header?.title?.content === '会议处理设置中' + || result?.toast?.content === '会议 agent 选择正在处理中') { return (await waitForConsumerApplyFinalCard(patchIndex)) ?? result; } return result; } +async function selectConsumerAgentViaCard(label: string, operatorOpenId = TARGET_OPEN_ID): Promise { + await __vcMeetingAgentTest.handleCardAction({ + operator: { open_id: operatorOpenId }, + action: lastInteractiveCardSelectOption(label), + }, APP_ID); + return confirmLatestConsumerCard(operatorOpenId); +} + async function selectConsumerProfilesViaCard( profileIds: readonly string[], initialCard?: any, @@ -5654,7 +5659,14 @@ describe('VC meeting daemon session lifecycle', () => { expect(triggerSessionCalls).toHaveLength(0); __vcMeetingAgentTest.setConsumerPendingItemLimitForTest(undefined); - await selectConsumerAgentViaCard('Claude Loopy'); + for (let i = 0; i < 20 + && sentMessages.filter(message => message.msgType === 'interactive').length < 2; + i += 1) await Promise.resolve(); + expect(sentMessages.filter(message => message.msgType === 'interactive')).toHaveLength(2); + // The overflow recovery card already stages the current agent; confirming + // it resumes the same durable member/cursor without starting a second + // dropdown apply in parallel. + await confirmLatestConsumerCard(); await __vcMeetingAgentTest.injectConsumer(APP_ID, 'm_joined_464646464', { force: true }); expect(triggerSessionCalls).toHaveLength(1); expect(triggerSessionCalls[0].req.envelope.payload.entries.filter((entry: any) => entry.kind === 'item')) diff --git a/test/vc-meeting-receiver-recovery-lifecycle.test.ts b/test/vc-meeting-receiver-recovery-lifecycle.test.ts index 1a4e12ec6..642cc1731 100644 --- a/test/vc-meeting-receiver-recovery-lifecycle.test.ts +++ b/test/vc-meeting-receiver-recovery-lifecycle.test.ts @@ -116,6 +116,40 @@ describe('VC meeting receiver boot-recovery lifecycle', () => { expect(recovery.snapshot(key)).toMatchObject({ ready: true, pending: false, timerArmed: false }); }); + it('keeps the boot fence until exact dispatch retirement persists after backing is missing', async () => { + recovery.setBackingMissingProbe(() => true); + let retirementWorks = false; + const retired: Array<[string, string, number]> = []; + recovery.setDispatchRetirement((sessionId, turnId, dispatchAttempt) => { + retired.push([sessionId, turnId, dispatchAttempt]); + return retirementWorks; + }); + const key = recovery.start('sess_retire', 'delivery_retire', 5, { + memberId: 'member_retire', + }); + recovery.finishScheduling(); + + recovery.acknowledge('sess_retire', 'delivery_retire', 5); + expect(recovery.snapshot(key)).toMatchObject({ ready: false, pending: true, timerArmed: true }); + expect(retired).toEqual([['sess_retire', 'delivery_retire', 5]]); + + await vi.advanceTimersByTimeAsync(8_000); + expect(recovery.snapshot(key)).toMatchObject({ ready: false, pending: true, timerArmed: true }); + expect(retired).toEqual([ + ['sess_retire', 'delivery_retire', 5], + ['sess_retire', 'delivery_retire', 5], + ]); + + retirementWorks = true; + await vi.advanceTimersByTimeAsync(5_000); + expect(retired).toEqual([ + ['sess_retire', 'delivery_retire', 5], + ['sess_retire', 'delivery_retire', 5], + ['sess_retire', 'delivery_retire', 5], + ]); + expect(recovery.snapshot(key)).toMatchObject({ ready: true, pending: false, timerArmed: false }); + }); + it('keeps the boot gate pending when reset IPC send throws', () => { const failureLog = daemonSource.indexOf('failed to fence boot-ambiguous receiver'); expect(failureLog).toBeGreaterThanOrEqual(0); diff --git a/test/vc-meeting-runtime-lease-recovery.test.ts b/test/vc-meeting-runtime-lease-recovery.test.ts index 18f350db6..bbbc9d2ee 100644 --- a/test/vc-meeting-runtime-lease-recovery.test.ts +++ b/test/vc-meeting-runtime-lease-recovery.test.ts @@ -86,6 +86,7 @@ function harness(input: { probe?: (backend: 'tmux' | 'herdr' | 'zellij', sessionName: string) => 'exists' | 'missing' | 'unknown'; missingPersistentScope?: FakeSession['testPersistentScope']; backendAvailable?: (backend: 'tmux' | 'herdr' | 'zellij') => boolean; + retireDispatch?: (sessionId: string, turnId: string, dispatchAttempt: number) => boolean; } = {}) { const sessions = new Map((input.sessions ?? []).map(ds => [ds.session.sessionId, ds])); const sent: Array<{ sessionId: string; turnId: string; dispatchAttempt: number }> = []; @@ -94,6 +95,7 @@ function harness(input: { const probes: string[] = []; const warnings: string[] = []; const errors: string[] = []; + const retired: Array<{ sessionId: string; turnId: string; dispatchAttempt: number }> = []; const recovery = createRecovery({ findSession: (sessionId: string) => sessions.get(sessionId), sendExpiry: (ds: FakeSession, message: { turnId: string; dispatchAttempt: number }) => { @@ -117,10 +119,14 @@ function harness(input: { probes.push(`${backend}:${sessionName}`); return input.probe?.(backend, sessionName) ?? 'missing'; }, + retireDispatch: (sessionId: string, turnId: string, dispatchAttempt: number) => { + retired.push({ sessionId, turnId, dispatchAttempt }); + return input.retireDispatch?.(sessionId, turnId, dispatchAttempt) ?? true; + }, warn: (message: string) => warnings.push(message), error: (message: string) => errors.push(message), } as any); - return { recovery, sessions, sent, killed, backingKills, probes, warnings, errors }; + return { recovery, sessions, sent, killed, backingKills, probes, retired, warnings, errors }; } describe('VC meeting runtime lease recovery', () => { @@ -196,6 +202,26 @@ describe('VC meeting runtime lease recovery', () => { expect(h.killed).toEqual(['session_a']); expect(h.backingKills).toEqual(['tmux:bmx-session_']); expect(h.probes).toEqual(['tmux:bmx-session_']); + expect(h.retired).toEqual([{ + sessionId: 'session_a', turnId: 'delivery_a', dispatchAttempt: 1, + }]); + expect(h.recovery.snapshot()).toEqual([]); + }); + + it('keeps a workerless receiver fenced when exact ledger retirement cannot persist', async () => { + let retirementWorks = false; + const h = harness({ + sessions: [fakeSession({ worker: false, persistentScope: 'tmux' })], + retireDispatch: () => retirementWorks, + }); + h.recovery.arm(ref(), 'agent_test'); + + expect(h.recovery.snapshot()).toMatchObject([{ phase: 'blocked', timerArmed: true }]); + expect(h.errors.some(message => message.includes('dispatch retirement failed'))).toBe(true); + + retirementWorks = true; + await vi.advanceTimersByTimeAsync(5_000); + expect(h.retired).toHaveLength(2); expect(h.recovery.snapshot()).toEqual([]); }); diff --git a/test/worker-app-runner-control-wiring.test.ts b/test/worker-app-runner-control-wiring.test.ts index 38ac3530e..cb3dc87ba 100644 --- a/test/worker-app-runner-control-wiring.test.ts +++ b/test/worker-app-runner-control-wiring.test.ts @@ -1,5 +1,9 @@ import { readFileSync } from 'node:fs'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { + CodexAppControlProofDeadline, + codexAppSignedStateReadiness, +} from '../src/utils/codex-app-control.js'; const workerSource = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf8'); @@ -11,10 +15,218 @@ describe('worker app-runner control-channel wiring', () => { expect(workerSource).not.toContain('codexAppOscPending'); }); - it('rejects marker identity mismatches and keeps dispatch authority worker-owned', () => { - expect(workerSource).toContain('if (!identity.ok)'); - expect(workerSource).toContain('payload.dispatchAttempt !== currentBotmuxDispatchAttempt'); - expect(workerSource).toContain('const dispatchAttempt = currentBotmuxDispatchAttempt;'); - expect(workerSource).not.toContain('const dispatchAttempt = payload.dispatchAttempt'); + it('reserves Codex App attribution before writes and settles finals only from the worker FIFO', () => { + const flushStart = workerSource.indexOf('async function flushPending'); + const flushEnd = workerSource.indexOf('function sendToPty', flushStart); + const flush = workerSource.slice(flushStart, flushEnd); + const reserveIdx = flush.indexOf('codexAppTurnDispatchQueue.reserve('); + const writeIdx = flush.indexOf('await cliAdapter.writeStructuredInput('); + expect(reserveIdx).toBeGreaterThan(-1); + expect(writeIdx).toBeGreaterThan(reserveIdx); + expect(flush).toContain("result.submissionDisposition === 'untouched'"); + expect(flush).toContain("result.submissionDisposition === 'flushed_invalid'"); + expect(flush).toContain('input buffer is not provably clean'); + expect(flush).toContain('if (backend && dispatchStillPending) scheduleSubmitFailureNotify('); + expect(flush).toContain('if (backend && dispatchStillPending) {'); + const safeRetryStart = flush.indexOf('const retryQueuedActivation ='); + const retryTransition = flush.indexOf("retryQueuedActivation ? 'retry' : 'cancel'", safeRetryStart); + const requeue = flush.indexOf('requeueUnsubmittedQueuedActivation(item);', retryTransition); + const submittedAck = flush.indexOf("type: 'queued_activation_submitted'", retryTransition); + expect(safeRetryStart).toBeGreaterThan(writeIdx); + expect(retryTransition).toBeGreaterThan(safeRetryStart); + expect(requeue).toBeGreaterThan(retryTransition); + expect(submittedAck).toBeGreaterThan(requeue); + expect(flush.slice(safeRetryStart, submittedAck)).toContain('codexAppSafeNonSubmission'); + expect(flush.slice(safeRetryStart, submittedAck)).not.toContain("type: 'queued_activation_submitted'"); + + const markerStart = workerSource.indexOf('function handleTrustedCodexAppMarker('); + const markerEnd = workerSource.indexOf('function handleAppRunnerOscMarker(', markerStart); + const marker = workerSource.slice(markerStart, markerEnd); + expect(marker).toContain('const settlement = codexAppTurnDispatchQueue.settleFinal(payload, false);'); + expect(marker).toContain('codexAppTurnDispatchQueue.commitExactHead(codexAppDispatchHandle)'); + expect(workerSource).toContain('codexAppControlRecordApplicationGate.run('); + expect(workerSource).toContain('codexAppControlReplayWindow.commit(identity.generation, record.seq);'); + expect(workerSource.indexOf('codexAppControlRecordApplicationGate.run(')) + .toBeLessThan(workerSource.indexOf('codexAppControlReplayWindow.commit(identity.generation, record.seq);')); + const codexSettlementStart = marker.indexOf('const settlement = codexAppTurnDispatchQueue.settleFinal(payload, false);'); + const miraFallbackStart = marker.indexOf('} else {\n // Mira/Mir', codexSettlementStart); + expect(marker.slice(codexSettlementStart, miraFallbackStart)).not.toContain('currentBotmuxTurnId'); + expect(marker.slice(codexSettlementStart, miraFallbackStart)).not.toContain('currentBotmuxDispatchAttempt'); + expect(marker).not.toContain('const dispatchAttempt = payload.dispatchAttempt'); + expect(marker).toContain('empty final settled for botmux turn'); + expect(workerSource).not.toContain('settleLegacyCodexAppEmptyFinal'); + expect(marker).toContain('published idle before the required final transaction'); + expect(marker).toContain('submitted the next turn before the required final transaction'); + }); + + it('ACKs a fresh RPC queued activation only after confirmed turn/start acceptance', () => { + const engageStart = workerSource.indexOf('async function engageCodexRpc('); + const engageEnd = workerSource.indexOf('/** RPC panes have NO terminal input path', engageStart); + const engage = workerSource.slice(engageStart, engageEnd); + const firstTurn = engage.indexOf('await engine.sendFirstTurn('); + const accepted = engage.indexOf("if (first === 'accepted' && cfg.queuedActivationToken)", firstTurn); + const ack = engage.indexOf("type: 'queued_activation_submitted'", accepted); + expect(firstTurn).toBeGreaterThan(-1); + expect(accepted).toBeGreaterThan(firstTurn); + expect(ack).toBeGreaterThan(accepted); + expect(engage.slice(firstTurn, accepted)).toContain("if (first === 'not-sent')"); + }); + + it('restores the durable FIFO but never treats warm signed idle as proof that prepared input was unwritten', () => { + const activateStart = workerSource.indexOf('function activateCodexAppControlConnection('); + const activateEnd = workerSource.indexOf('function handleCodexAppControlLine(', activateStart); + const activate = workerSource.slice(activateStart, activateEnd); + expect(activate).not.toContain('markPromptReady()'); + expect(workerSource).toContain('codexAppTurnDispatchQueue.restore('); + expect(workerSource).not.toContain('requeueUnwrittenRecoveredCodexAppPrefix'); + expect(workerSource).not.toContain("requestCodexAppDispatchTransition('reset'"); + + const markerStart = workerSource.indexOf('async function handleTrustedCodexAppMarker('); + const markerEnd = workerSource.indexOf('function handleAppRunnerOscMarker(', markerStart); + const marker = workerSource.slice(markerStart, markerEnd); + expect(marker).toContain('codexAppTurnDispatchQueue.recoveredPrefix().length > 0'); + expect(marker).toContain('Codex App signed idle cannot prove the recovered prepared frame was never buffered'); + + const stopStart = workerSource.indexOf('function stopCodexAppControlChannel('); + const stopEnd = workerSource.indexOf('function failCodexAppControlGeneration(', stopStart); + const stop = workerSource.slice(stopStart, stopEnd); + expect(stop).toContain('if (!opts.preserveDispatchRecovery) {'); + expect(stop.indexOf('codexAppTurnDispatchQueue.clear();')) + .toBeGreaterThan(stop.indexOf('if (!opts.preserveDispatchRecovery) {')); + + const prepareStart = workerSource.indexOf('async function prepareCodexAppControlGeneration('); + const prepareEnd = workerSource.indexOf('async function rotateCodexAppControlEndpoint(', prepareStart); + expect(workerSource.slice(prepareStart, prepareEnd)) + .toContain('stopCodexAppControlChannel({ preserveDispatchRecovery: true });'); + }); + + it('keeps auth-to-state proof armed and permits type-ahead only after signed runner readiness', () => { + const activateStart = workerSource.indexOf('function activateCodexAppControlConnection('); + const activateEnd = workerSource.indexOf('function handleCodexAppControlLine(', activateStart); + const activate = workerSource.slice(activateStart, activateEnd); + expect(activate).not.toContain('codexAppProofDeadline.clear();'); + expect(activate).toContain('Authenticated Codex App runner did not publish signed state'); + + const markerStart = workerSource.indexOf('async function handleTrustedCodexAppMarker('); + const markerEnd = workerSource.indexOf('function handleAppRunnerOscMarker(', markerStart); + const marker = workerSource.slice(markerStart, markerEnd); + expect(marker.indexOf('codexAppSignedStateObserved = true;')).toBeGreaterThan( + marker.indexOf('if (!state.accepted)'), + ); + expect(marker).toContain('codexAppProofDeadline.clear();'); + expect(marker).toContain("if (readiness === 'invalid')"); + expect(marker).toContain("if (readiness === 'waiting')"); + expect(marker).toContain('codexAppInputReady = true;'); + const invalidStart = marker.indexOf("if (readiness === 'invalid')"); + const waitingStart = marker.indexOf("if (readiness === 'waiting')"); + const applyStart = marker.indexOf('const state = applyTrustedCodexAppStateMarker(', waitingStart); + expect(marker.slice(invalidStart, waitingStart)).toContain('failCodexAppControlGeneration('); + expect(marker.slice(waitingStart, applyStart)).toContain('codexAppProofDeadline.armed'); + expect(marker.slice(waitingStart, applyStart)).not.toContain('codexAppProofDeadline.clear();'); + expect(marker.indexOf('codexAppProofDeadline.clear();')).toBeGreaterThan(applyStart); + + const runtimeGateStart = workerSource.indexOf('function codexAppRuntimeTypeAheadReady()'); + const runtimeGateEnd = workerSource.indexOf('async function flushPending()', runtimeGateStart); + const runtimeGate = workerSource.slice(runtimeGateStart, runtimeGateEnd); + expect(runtimeGate).toContain('codexAppControlProven'); + expect(runtimeGate).toContain('codexAppSignedStateObserved'); + expect(runtimeGate).toContain('codexAppInputReady'); + expect(workerSource).toContain('projectCodexAppControlReadinessStatus(base, {'); + const firstPromptTimeout = workerSource.slice( + workerSource.indexOf('const releaseFirstPromptTimeout'), + workerSource.indexOf('// Riff (and other remote HTTP backends)'), + ); + expect(firstPromptTimeout).toContain( + "if (decideHardTimeoutAction(cliAdapter?.supportsTypeAhead === true) === 'flush')", + ); + expect(firstPromptTimeout).not.toContain('codexAppRuntimeTypeAheadReady()'); + }); + + it('keeps the proof timer armed for acceptingInput:false and clears it only for true', async () => { + vi.useFakeTimers(); + const deadline = new CodexAppControlProofDeadline(); + try { + const falseTimedOut = vi.fn(); + deadline.arm(falseTimedOut, 100); + expect(codexAppSignedStateReadiness({ busy: false, acceptingInput: false })).toBe('waiting'); + await vi.advanceTimersByTimeAsync(100); + expect(falseTimedOut).toHaveBeenCalledTimes(1); + + const missingTimedOut = vi.fn(); + deadline.arm(missingTimedOut, 100); + expect(codexAppSignedStateReadiness({ busy: false })).toBe('invalid'); + await vi.advanceTimersByTimeAsync(100); + expect(missingTimedOut).toHaveBeenCalledTimes(1); + + const readyTimedOut = vi.fn(); + deadline.arm(readyTimedOut, 100); + expect(codexAppSignedStateReadiness({ busy: false, acceptingInput: true })).toBe('ready'); + deadline.clear(); + await vi.advanceTimersByTimeAsync(100); + expect(readyTimedOut).not.toHaveBeenCalled(); + } finally { + deadline.clear(); + vi.useRealTimers(); + } + }); + + it('rejects fresh authentication with recovered prepared ownership before activation or publication', () => { + const activateStart = workerSource.indexOf('function activateCodexAppControlConnection('); + const activateEnd = workerSource.indexOf('async function handleCodexAppControlLine(', activateStart); + const activate = workerSource.slice(activateStart, activateEnd); + const recoveredGuard = activate.indexOf("proofKind === 'fresh runner'"); + const fail = activate.indexOf('failCodexAppControlGeneration(', recoveredGuard); + const persist = activate.indexOf('persistCodexAppControlState(', recoveredGuard); + const accepted = activate.indexOf('encodeCodexAppControlAccepted(', recoveredGuard); + const published = activate.indexOf("type: 'codex_app_generation_active'", recoveredGuard); + + expect(recoveredGuard).toBeGreaterThan(-1); + expect(fail).toBeGreaterThan(recoveredGuard); + expect(persist).toBeGreaterThan(fail); + expect(accepted).toBeGreaterThan(fail); + expect(published).toBeGreaterThan(fail); + }); + + it('fails the worker generation before publishing terminal or exit signals when the real runner exits with prepared ownership', () => { + const callbackStart = workerSource.lastIndexOf('backend.onExit((code, signal) => {'); + const callbackEnd = workerSource.indexOf('backend.onError(', callbackStart); + const callback = workerSource.slice(callbackStart, callbackEnd); + const fatalIdx = callback.indexOf("lastInitConfig?.cliId === 'codex-app' && codexAppControlFatal"); + const fatalReturnIdx = callback.indexOf('return;', fatalIdx); + const preparedIdx = callback.indexOf('const codexAppPreparedAtExit'); + const failIdx = callback.indexOf('failCodexAppControlGeneration(', preparedIdx); + const terminalIdx = callback.indexOf('emitTurnTerminal(', preparedIdx); + const exitIdx = callback.indexOf("send({ type: 'claude_exit'", preparedIdx); + + expect(fatalIdx).toBeGreaterThan(-1); + expect(fatalReturnIdx).toBeGreaterThan(fatalIdx); + expect(preparedIdx).toBeGreaterThan(fatalReturnIdx); + expect(preparedIdx).toBeGreaterThan(-1); + expect(failIdx).toBeGreaterThan(preparedIdx); + expect(terminalIdx).toBeGreaterThan(failIdx); + expect(exitIdx).toBeGreaterThan(failIdx); + }); + + it('destroys incomplete final transactions before cumulative commit or ACK', () => { + const handlerStart = workerSource.indexOf('function handleCodexAppControlLine('); + const handlerEnd = workerSource.indexOf('function acceptCodexAppControlSocket(', handlerStart); + const handler = workerSource.slice(handlerStart, handlerEnd); + const assembleIdx = handler.indexOf('const finalResult = connection.finalAssembler.accept('); + const rejectIdx = handler.indexOf("if (finalResult.status === 'reject')", assembleIdx); + const destroyIdx = handler.indexOf('connection.socket.destroy();', rejectIdx); + const applicationIdx = handler.indexOf('codexAppControlRecordApplicationGate.run(', assembleIdx); + const commitIdx = handler.indexOf('codexAppControlReplayWindow.commit(', assembleIdx); + const ackIdx = handler.indexOf('encodeCodexAppControlAck(', commitIdx); + const semanticRejectIdx = handler.indexOf('if (!applied)', assembleIdx); + + expect(assembleIdx).toBeGreaterThan(-1); + expect(rejectIdx).toBeGreaterThan(assembleIdx); + expect(destroyIdx).toBeGreaterThan(rejectIdx); + expect(semanticRejectIdx).toBeGreaterThan(destroyIdx); + expect(applicationIdx).toBeGreaterThan(assembleIdx); + expect(commitIdx).toBeGreaterThan(semanticRejectIdx); + expect(commitIdx).toBeGreaterThan(destroyIdx); + expect(ackIdx).toBeGreaterThan(commitIdx); + expect(handler).toContain("if (finalResult.status === 'accepted') return;"); }); }); diff --git a/test/worker-codex-app-turn-routing.integration.test.ts b/test/worker-codex-app-turn-routing.integration.test.ts new file mode 100644 index 000000000..1ce6f9efc --- /dev/null +++ b/test/worker-codex-app-turn-routing.integration.test.ts @@ -0,0 +1,598 @@ +import { execFileSync, spawn, type ChildProcess } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import type { DaemonToWorker, WorkerToDaemon } from '../src/types.js'; + +const children = new Set(); +const tempDirs = new Set(); +const tmuxSessions = new Set(); + +async function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise(resolvePromise => { + const timer = setTimeout(() => child.kill('SIGKILL'), 3_000); + child.once('exit', () => { + clearTimeout(timer); + resolvePromise(); + }); + if (child.connected) child.send({ type: 'close' } satisfies DaemonToWorker); + else child.kill('SIGTERM'); + }); +} + +async function hardKillWorkerOnly(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise(resolvePromise => { + child.once('exit', () => resolvePromise()); + child.kill('SIGKILL'); + }); +} + +function waitForChildExit( + child: ChildProcess, + logs: string[], + timeoutMs = 10_000, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve({ code: child.exitCode, signal: child.signalCode }); + } + return new Promise((resolvePromise, rejectPromise) => { + const timer = setTimeout(() => { + child.off('exit', onExit); + rejectPromise(new Error(`worker did not exit after CLI crash\n${logs.join('')}`)); + }, timeoutMs); + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timer); + resolvePromise({ code, signal }); + }; + child.once('exit', onExit); + }); +} + +afterEach(async () => { + await Promise.all([...children].map(stopChild)); + children.clear(); + for (const name of tmuxSessions) { + try { execFileSync('tmux', ['kill-session', '-t', name], { stdio: 'ignore' }); } catch { /* gone */ } + } + tmuxSessions.clear(); + for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true }); + tempDirs.clear(); +}); + +function replacementSessionId(label: string): string { + return `${randomBytes(4).toString('hex')}-${label}-${process.pid}-${Date.now()}`; +} + +function spawnWorker( + root: string, + sessionId: string, + fakeCodex: string, + requestLog: string, + behavior: string, + logs: string[], + messages: WorkerToDaemon[], + onMessage?: (message: WorkerToDaemon, child: ChildProcess) => void, +): ChildProcess { + const child = spawn(process.execPath, ['--import', 'tsx', resolve('src/worker.ts')], { + cwd: resolve('.'), + env: { + ...process.env, + HOME: root, + NODE_ENV: 'test', + NODE_OPTIONS: [process.env.NODE_OPTIONS, '--import=tsx'].filter(Boolean).join(' '), + // tmux launch scripts intentionally sanitize NODE_OPTIONS, so use the + // built JS runner rather than relying on the parent's tsx loader. + BOTMUX_TEST_CODEX_APP_RUNNER_PATH: resolve('dist/codex-app-runner.js'), + SESSION_DATA_DIR: root, + BOTMUX_SESSION_ID: sessionId, + LARK_APP_ID: 'app_worker_replacement', + LARK_APP_SECRET: 'secret', + FAKE_CODEX_LOG: requestLog, + FAKE_CODEX_VERSION: '0.136.0', + FAKE_CODEX_BEHAVIOR: behavior, + }, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }); + children.add(child); + child.stdout?.on('data', chunk => logs.push(chunk.toString())); + child.stderr?.on('data', chunk => logs.push(chunk.toString())); + child.on('message', raw => { + const message = raw as WorkerToDaemon; + messages.push(message); + if (message.type === 'error') logs.push(`[worker-ipc-error] ${message.message}\n`); + if (message.type === 'final_output') { + logs.push(`[worker-ipc-final] turn=${message.turnId} dispatch=${message.codexAppSettlement?.dispatchId ?? '-'} content=${JSON.stringify(message.content)}\n`); + } + onMessage?.(message, child); + }); + return child; +} + +function replacementInit( + sessionId: string, + fakeCodex: string, + prompt: string, + extra: Partial> = {}, +): Extract { + return { + type: 'init', + sessionId, + chatId: 'oc_worker_replacement', + rootMessageId: 'om_worker_replacement_root', + workingDir: resolve('.'), + cliId: 'codex-app', + cliPathOverride: fakeCodex, + backendType: 'tmux', + prompt, + larkAppId: 'app_worker_replacement', + larkAppSecret: 'secret', + ...extra, + }; +} + +function waitFor( + child: ChildProcess, + logs: string[], + predicate: () => boolean, + timeoutMs = 20_000, +): Promise { + if (predicate()) return Promise.resolve(); + return new Promise((resolvePromise, rejectPromise) => { + const poll = setInterval(() => { + if (!predicate()) return; + cleanup(); + resolvePromise(); + }, 20); + const timer = setTimeout(() => { + cleanup(); + rejectPromise(new Error(`worker routing timeout\n${logs.join('')}`)); + }, timeoutMs); + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + rejectPromise(new Error(`worker exited early (${code ?? signal})\n${logs.join('')}`)); + }; + const cleanup = () => { + clearInterval(poll); + clearTimeout(timer); + child.off('exit', onExit); + }; + child.once('exit', onExit); + }); +} + +function readRequests(path: string): Array> { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); +} + +describe('Codex App worker queued-turn attribution', () => { + it('routes and ACKs turn N before N+1 after both inputs were written in one flush', async () => { + const root = mkdtempSync(join(tmpdir(), 'botmux-worker-codex-routing-')); + tempDirs.add(root); + const fakeCodex = join(root, 'fake-codex'); + const requestLog = join(root, 'requests.jsonl'); + copyFileSync(resolve('test/fixtures/fake-codex-app-server.mjs'), fakeCodex); + chmodSync(fakeCodex, 0o755); + + const sessionId = `codex-routing-${process.pid}-${Date.now()}`; + const logs: string[] = []; + const messages: WorkerToDaemon[] = []; + const nodeOptions = [process.env.NODE_OPTIONS, '--import=tsx'].filter(Boolean).join(' '); + const child = spawn(process.execPath, ['--import', 'tsx', resolve('src/worker.ts')], { + cwd: resolve('.'), + env: { + ...process.env, + HOME: root, + NODE_ENV: 'test', + NODE_OPTIONS: nodeOptions, + BOTMUX_TEST_CODEX_APP_RUNNER_PATH: resolve('src/codex-app-runner.ts'), + SESSION_DATA_DIR: root, + BOTMUX_SESSION_ID: sessionId, + LARK_APP_ID: 'app_worker_routing', + LARK_APP_SECRET: 'secret', + FAKE_CODEX_LOG: requestLog, + FAKE_CODEX_VERSION: '0.136.0', + // Hold turn 1 open so flushPending deterministically writes turn 2 and + // overwrites its legacy singleton globals before final 1 arrives. + FAKE_CODEX_BEHAVIOR: 'delayed-first', + }, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }); + children.add(child); + child.stdout?.on('data', chunk => logs.push(chunk.toString())); + child.stderr?.on('data', chunk => logs.push(chunk.toString())); + let followUpSent = false; + child.on('message', raw => { + const message = raw as WorkerToDaemon; + messages.push(message); + if (message.type === 'error') logs.push(`[worker-ipc-error] ${message.message}\n`); + // `ready` means the adapter/backend are installed, while Codex App's + // authenticated runner still has to initialize and render its first ›. + // Queue turn 2 in that deterministic gap. + if (message.type === 'ready' && !followUpSent) { + followUpSent = true; + child.send(followUp); + } + }); + + const init: DaemonToWorker = { + type: 'init', + sessionId, + chatId: 'oc_worker_routing', + rootMessageId: 'om_worker_routing_root', + // Keep Node package resolution anchored at the checkout so the nested + // source runner can load the tsx hook supplied through NODE_OPTIONS. + workingDir: resolve('.'), + cliId: 'codex-app', + cliPathOverride: fakeCodex, + backendType: 'pty', + prompt: 'turn one legacy', + promptCodexAppInput: { + text: 'turn one', + clientUserMessageId: 'om_worker_turn_1', + }, + larkAppId: 'app_worker_routing', + larkAppSecret: 'secret', + turnId: 'om_worker_turn_1', + }; + const followUp: DaemonToWorker = { + type: 'message', + content: 'turn two legacy', + codexAppInput: { + text: 'turn two', + clientUserMessageId: 'om_worker_turn_2', + }, + turnId: 'om_worker_turn_2', + }; + + try { + child.send(init); + await waitFor(child, logs, () => ( + messages.filter(message => message.type === 'final_output').length >= 2 + && messages.filter(message => message.type === 'turn_terminal').length >= 2 + )); + + const finals = messages.filter( + (message): message is Extract => message.type === 'final_output', + ); + expect(finals.map(final => ({ + turnId: final.turnId, + content: final.content, + dispatchAttempt: final.dispatchAttempt, + }))).toEqual([ + { turnId: 'om_worker_turn_1', content: 'fake answer 1', dispatchAttempt: undefined }, + { turnId: 'om_worker_turn_2', content: 'fake answer 2', dispatchAttempt: undefined }, + ]); + const terminals = messages.filter( + (message): message is Extract => message.type === 'turn_terminal', + ); + expect(terminals.map(terminal => ({ + turnId: terminal.turnId, + status: terminal.status, + }))).toEqual([ + { turnId: 'om_worker_turn_1', status: 'completed' }, + { turnId: 'om_worker_turn_2', status: 'completed' }, + ]); + + const turnStarts = readRequests(requestLog).filter(request => request.method === 'turn/start'); + expect(turnStarts.map(request => request.params.clientUserMessageId)).toEqual([ + 'om_worker_turn_1', + 'om_worker_turn_2', + ]); + const joinedLogs = logs.join(''); + const secondWrite = joinedLogs.indexOf('turn two legacy'); + const firstFinalMap = joinedLogs.indexOf('mapped to botmux turn om_worker_tu'); + expect(secondWrite).toBeGreaterThan(-1); + expect(firstFinalMap).toBeGreaterThan(secondWrite); + expect(joinedLogs).not.toContain('rejected final marker'); + } finally { + await stopChild(child); + } + }, 25_000); +}); + +describe('Codex App worker replacement durable handoff', () => { + it('turns a real tmux runner exit with prepared durable ownership into a Node worker exit', async () => { + const root = mkdtempSync(join(tmpdir(), 'botmux-worker-codex-cli-exit-')); + tempDirs.add(root); + const fakeCodex = join(root, 'fake-codex'); + const requestLog = join(root, 'requests.jsonl'); + copyFileSync(resolve('test/fixtures/fake-codex-app-server.mjs'), fakeCodex); + chmodSync(fakeCodex, 0o755); + + const sessionId = replacementSessionId('cli-exit'); + const tmuxSession = `bmx-${sessionId.slice(0, 8)}`; + tmuxSessions.add(tmuxSession); + const entry = { + dispatchId: 'dispatch-cli-exit-1', + turnId: 'turn-cli-exit-1', + dispatchAttempt: 17, + content: 'crash runner after prepared', + codexAppInput: { text: 'crash runner after prepared', clientUserMessageId: 'turn-cli-exit-1' }, + }; + + const logs: string[] = []; + const messages: WorkerToDaemon[] = []; + const prematureAdmissionSignals: WorkerToDaemon[] = []; + const worker = spawnWorker( + root, sessionId, fakeCodex, requestLog, 'success', logs, messages, + (message, child) => { + if (message.type === 'codex_app_dispatch_transition') { + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.requestId, + ok: true, + } satisfies DaemonToWorker); + } + // Withhold settlement durability so the exact dispatch remains + // prepared when the runner process exits. + if (message.type === 'claude_exit' + || (message.type === 'turn_terminal' && message.status === 'ambiguous')) { + // Either edge would let a durable receiver consider N complete and + // admit N+1 before the worker-exit recovery fence is armed. + prematureAdmissionSignals.push(message); + } + }, + ); + worker.send(replacementInit(sessionId, fakeCodex, entry.content, { + env: { FAKE_CODEX_LOG: requestLog, FAKE_CODEX_BEHAVIOR: 'success' }, + turnId: entry.turnId, + dispatchAttempt: entry.dispatchAttempt, + codexAppDispatchId: entry.dispatchId, + promptCodexAppInput: entry.codexAppInput, + })); + + await waitFor(worker, logs, () => messages.some(message => + message.type === 'final_output' + && message.codexAppSettlement?.dispatchId === entry.dispatchId)); + + const workerExit = waitForChildExit(worker, logs); + // Kill the actual runner pane while leaving the Node worker alive. This + // exercises backend.onExit's direct prepared-generation fail-close path, + // unlike SIGKILLing the worker process as the replacement tests below do. + execFileSync('tmux', ['send-keys', '-t', tmuxSession, 'C-c'], { stdio: 'ignore' }); + const exited = await workerExit; + + expect(prematureAdmissionSignals).toEqual([]); + expect(exited).toEqual({ code: null, signal: 'SIGKILL' }); + expect(messages).toContainEqual(expect.objectContaining({ + type: 'error', + message: expect.stringContaining('prepared dispatch'), + turnId: entry.turnId, + dispatchAttempt: entry.dispatchAttempt, + })); + expect(logs.join('')).toContain('worker replacement requires exact recovery'); + expect(readRequests(requestLog).filter(request => request.method === 'turn/start')) + .toHaveLength(1); + }, 35_000); + + it('reattaches a surviving tmux runner and settles its unacked final with an empty replacement prompt', async () => { + const root = mkdtempSync(join(tmpdir(), 'botmux-worker-codex-replace-')); + tempDirs.add(root); + const fakeCodex = join(root, 'fake-codex'); + const requestLog = join(root, 'requests.jsonl'); + copyFileSync(resolve('test/fixtures/fake-codex-app-server.mjs'), fakeCodex); + chmodSync(fakeCodex, 0o755); + + const sessionId = replacementSessionId('replay'); + tmuxSessions.add(`bmx-${sessionId.slice(0, 8)}`); + const entry = { + dispatchId: 'dispatch-replay-1', + turnId: 'turn-replay-1', + state: 'prepared' as const, + content: 'survive worker kill', + codexAppInput: { text: 'survive worker kill', clientUserMessageId: 'turn-replay-1' }, + }; + + const logs1: string[] = []; + const messages1: WorkerToDaemon[] = []; + const worker1 = spawnWorker( + root, sessionId, fakeCodex, requestLog, 'success', logs1, messages1, + (message, child) => { + if (message.type === 'codex_app_dispatch_transition') { + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.requestId, + ok: true, + } satisfies DaemonToWorker); + } + // Intentionally withhold the final settlement ACK. The runner keeps + // final-end unacked while this worker is SIGKILLed. + }, + ); + worker1.send(replacementInit(sessionId, fakeCodex, entry.content, { + env: { FAKE_CODEX_LOG: requestLog, FAKE_CODEX_BEHAVIOR: 'success' }, + turnId: entry.turnId, + codexAppDispatchId: entry.dispatchId, + promptCodexAppInput: entry.codexAppInput, + })); + + await waitFor(worker1, logs1, () => messages1.some(message => + message.type === 'final_output' + && message.codexAppSettlement?.dispatchId === entry.dispatchId)); + await hardKillWorkerOnly(worker1); + + const logs2: string[] = []; + const messages2: WorkerToDaemon[] = []; + const worker2 = spawnWorker( + root, sessionId, fakeCodex, requestLog, 'success', logs2, messages2, + (message, child) => { + if (message.type === 'codex_app_dispatch_transition') { + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.requestId, + ok: true, + } satisfies DaemonToWorker); + } + if (message.type === 'final_output' && message.codexAppSettlement) { + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.codexAppSettlement.requestId, + ok: true, + } satisfies DaemonToWorker); + } + }, + ); + worker2.send(replacementInit(sessionId, fakeCodex, '', { + env: { FAKE_CODEX_LOG: requestLog, FAKE_CODEX_BEHAVIOR: 'success' }, + resume: true, + cliSessionId: 'thread-fake', + codexAppRecoveredDispatches: [entry], + })); + + await waitFor(worker2, logs2, () => messages2.some(message => + message.type === 'turn_terminal' + && message.turnId === entry.turnId + && message.status === 'completed')); + + const replayed = messages2.find((message): message is Extract => + message.type === 'final_output' && message.codexAppSettlement?.dispatchId === entry.dispatchId); + expect(replayed).toMatchObject({ + sessionId, + turnId: entry.turnId, + content: 'fake answer 1', + }); + expect(logs2.join('')).toContain('warm reattach'); + expect(readRequests(requestLog).filter(request => request.method === 'turn/start')).toHaveLength(1); + }, 35_000); + + it('holds N+1 behind replayed empty N until daemon durability ACKs final-end', async () => { + const root = mkdtempSync(join(tmpdir(), 'botmux-worker-codex-empty-replace-')); + tempDirs.add(root); + const fakeCodex = join(root, 'fake-codex'); + const requestLog = join(root, 'requests.jsonl'); + copyFileSync(resolve('test/fixtures/fake-codex-app-server.mjs'), fakeCodex); + chmodSync(fakeCodex, 0o755); + + const sessionId = replacementSessionId('empty'); + tmuxSessions.add(`bmx-${sessionId.slice(0, 8)}`); + const first = { + dispatchId: 'dispatch-empty-1', + turnId: 'turn-empty-1', + state: 'prepared' as const, + content: 'empty first', + codexAppInput: { text: 'empty first', clientUserMessageId: 'turn-empty-1' }, + }; + const second = { + dispatchId: 'dispatch-empty-2', + turnId: 'turn-empty-2', + content: 'second after empty', + codexAppInput: { text: 'second after empty', clientUserMessageId: 'turn-empty-2' }, + }; + + const logs1: string[] = []; + const messages1: WorkerToDaemon[] = []; + const worker1 = spawnWorker( + root, sessionId, fakeCodex, requestLog, 'empty-first', logs1, messages1, + (message, child) => { + if (message.type === 'codex_app_dispatch_transition') { + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.requestId, + ok: true, + } satisfies DaemonToWorker); + } + }, + ); + worker1.send(replacementInit(sessionId, fakeCodex, first.content, { + env: { FAKE_CODEX_LOG: requestLog, FAKE_CODEX_BEHAVIOR: 'empty-first' }, + turnId: first.turnId, + codexAppDispatchId: first.dispatchId, + promptCodexAppInput: first.codexAppInput, + })); + await waitFor(worker1, logs1, () => messages1.some(message => + message.type === 'final_output' + && message.codexAppSettlement?.dispatchId === first.dispatchId + && message.content === '')); + await hardKillWorkerOnly(worker1); + + const logs2: string[] = []; + const messages2: WorkerToDaemon[] = []; + let secondSent = false; + let firstAcked = false; + let secondSubmittedBeforeFirstAck = false; + const order: string[] = []; + const worker2 = spawnWorker( + root, sessionId, fakeCodex, requestLog, 'empty-first', logs2, messages2, + (message, child) => { + if (message.type === 'ready' && !secondSent) { + secondSent = true; + child.send({ + type: 'message', + content: second.content, + codexAppInput: second.codexAppInput, + turnId: second.turnId, + codexAppDispatchId: second.dispatchId, + } satisfies DaemonToWorker); + } + if (message.type === 'codex_app_dispatch_transition') { + if (message.entries[0]?.dispatchId === second.dispatchId) { + secondSubmittedBeforeFirstAck = !firstAcked; + order.push('submit-second'); + } + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.requestId, + ok: true, + } satisfies DaemonToWorker); + } + if (message.type === 'final_output' && message.codexAppSettlement) { + if (message.codexAppSettlement.dispatchId === first.dispatchId) { + order.push('final-first'); + setTimeout(() => { + firstAcked = true; + order.push('ack-first'); + if (child.connected) child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.codexAppSettlement!.requestId, + ok: true, + } satisfies DaemonToWorker); + }, 250); + } else if (message.codexAppSettlement.dispatchId === second.dispatchId) { + order.push('final-second'); + child.send({ + type: 'codex_app_dispatch_persisted', + requestId: message.codexAppSettlement.requestId, + ok: true, + } satisfies DaemonToWorker); + } + } + }, + ); + worker2.send(replacementInit(sessionId, fakeCodex, '', { + env: { FAKE_CODEX_LOG: requestLog, FAKE_CODEX_BEHAVIOR: 'empty-first' }, + resume: true, + cliSessionId: 'thread-fake', + codexAppRecoveredDispatches: [first], + })); + + await waitFor(worker2, logs2, () => messages2.filter(message => + message.type === 'turn_terminal' + && (message.turnId === first.turnId || message.turnId === second.turnId) + && message.status === 'completed').length === 2, 30_000); + + const finals = messages2.filter( + (message): message is Extract => + message.type === 'final_output' && !!message.codexAppSettlement, + ); + expect(finals.map(final => ({ turnId: final.turnId, content: final.content }))).toEqual([ + { turnId: first.turnId, content: '' }, + { turnId: second.turnId, content: 'fake answer 2' }, + ]); + expect(secondSubmittedBeforeFirstAck).toBe(false); + expect(order).toEqual(['final-first', 'ack-first', 'submit-second', 'final-second']); + expect(readRequests(requestLog).filter(request => request.method === 'turn/start')) + .toHaveLength(2); + }, 40_000); +}); diff --git a/test/worker-durable-expiry-order.test.ts b/test/worker-durable-expiry-order.test.ts index b18dc3b0c..678598eaa 100644 --- a/test/worker-durable-expiry-order.test.ts +++ b/test/worker-durable-expiry-order.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it } from 'vitest'; const workerSource = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf8'); describe('worker durable lease expiry ordering', () => { - it('removes exact queued attempt N behind an ordinary current turn before ACKing', () => { + it('durably retires exact queued attempt N behind an ordinary current turn before ACKing', () => { const start = workerSource.indexOf("case 'expire_durable_turn':"); const end = workerSource.indexOf("case 'reset_ambiguous_receiver':", start); expect(start).toBeGreaterThanOrEqual(0); @@ -16,20 +16,26 @@ describe('worker durable lease expiry ordering', () => { const branch = workerSource.slice(start, end); const currentExact = branch.indexOf('const currentExact = durableTurnInFlight'); - const pendingLoop = branch.indexOf('for (let i = pendingMessages.length - 1; i >= 0; i--)'); - const exactTurn = branch.indexOf('item.turnId === msg.turnId', pendingLoop); - const exactAttempt = branch.indexOf('item.dispatchAttempt === msg.dispatchAttempt', pendingLoop); - const remove = branch.indexOf('pendingMessages.splice(i, 1)', pendingLoop); - const pendingAck = branch.indexOf("acknowledge('queued_removed');", remove); + const pendingCount = branch.indexOf('const removedPending = pendingMessages.filter('); + const retire = branch.indexOf('await retireCodexAppDispatchForDurableReplay(', pendingCount); + const pendingAck = branch.indexOf("acknowledge('queued_removed');", retire); const noProof = branch.indexOf('withholding ACK for daemon fencing', pendingAck); expect(currentExact).toBeGreaterThanOrEqual(0); - expect(pendingLoop).toBeGreaterThan(currentExact); - expect(exactTurn).toBeGreaterThan(pendingLoop); - expect(exactAttempt).toBeGreaterThan(exactTurn); - expect(remove).toBeGreaterThan(exactAttempt); - expect(pendingAck).toBeGreaterThan(remove); + expect(pendingCount).toBeGreaterThan(currentExact); + expect(retire).toBeGreaterThan(pendingCount); + expect(pendingAck).toBeGreaterThan(retire); expect(noProof).toBeGreaterThan(pendingAck); + + const retireStart = workerSource.indexOf('async function retireCodexAppDispatchForDurableReplay('); + const retireEnd = workerSource.indexOf('\nfunction ', retireStart + 1); + const retireHelper = workerSource.slice(retireStart, retireEnd); + const persist = retireHelper.indexOf("requestCodexAppDispatchTransition('cancel'"); + const localQueueCancel = retireHelper.indexOf('codexAppTurnDispatchQueue.cancelExact(', persist); + const localPendingRemove = retireHelper.indexOf('pendingMessages.splice(index, 1)', persist); + expect(persist).toBeGreaterThanOrEqual(0); + expect(localQueueCancel).toBeGreaterThan(persist); + expect(localPendingRemove).toBeGreaterThan(persist); }); it('ACKs active exact expiry only after synchronous owned-CLI restart fencing', () => { @@ -37,11 +43,13 @@ describe('worker durable lease expiry ordering', () => { const end = workerSource.indexOf("case 'reset_ambiguous_receiver':", start); const branch = workerSource.slice(start, end); const exactBranch = branch.indexOf('if (currentExact)'); - const restart = branch.indexOf("restartCliProcess('durable lease expiry'", exactBranch); + const retire = branch.indexOf('await retireCodexAppDispatchForDurableReplay(', exactBranch); + const restart = branch.indexOf("restartCliProcess('durable lease expiry'", retire); const ack = branch.indexOf("acknowledge('cli_fenced');", restart); expect(exactBranch).toBeGreaterThanOrEqual(0); - expect(restart).toBeGreaterThan(exactBranch); + expect(retire).toBeGreaterThan(exactBranch); + expect(restart).toBeGreaterThan(retire); expect(ack).toBeGreaterThan(restart); }); diff --git a/test/worker-pipe-initial-screen-order.test.ts b/test/worker-pipe-initial-screen-order.test.ts index cf3170b5c..979ea770a 100644 --- a/test/worker-pipe-initial-screen-order.test.ts +++ b/test/worker-pipe-initial-screen-order.test.ts @@ -34,6 +34,21 @@ describe('worker pipe initial screen ordering', () => { expect(captureIdx).toBeGreaterThan(idleIdx); }); + it('starts Codex App warm liveness only after old-key challenge proof, not from a backend flag', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const activationIdx = source.indexOf('function activateCodexAppControlConnection('); + const proofKindIdx = source.indexOf("const proofKind = identity.generation === codexAppFreshCandidateGeneration", activationIdx); + const beginIdx = source.indexOf('codexAppTurnLiveness.beginReattachObservation();', proofKindIdx); + const pipeGateIdx = source.indexOf('if (isPipeMode && backend && isPersistentBackendReattach)'); + const seedIdx = source.indexOf('seedBackendScreen(`${effectiveBackendType} reattach`, backend);', pipeGateIdx); + + expect(activationIdx).toBeGreaterThan(-1); + expect(proofKindIdx).toBeGreaterThan(activationIdx); + expect(beginIdx).toBeGreaterThan(proofKindIdx); + expect(seedIdx).toBeGreaterThan(pipeGateIdx); + expect(source).not.toContain('shouldBeginCodexAppReattachObservation({'); + }); + it('runs a busy-pattern idle probe after each submitted input', () => { const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); const writeIdx = source.indexOf('result = await cliAdapter.writeStructuredInput(backend, msg, item.codexAppInput);'); @@ -192,6 +207,218 @@ describe('worker pipe initial screen ordering', () => { expect(markReadyIdx).toBeGreaterThan(flushIdx); }); + it('rejects Codex App prompts before proof and while the explicit queue is active', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const markStart = source.indexOf('function markPromptReady'); + const markEnd = source.indexOf('function persistCliSessionId', markStart); + const mark = source.slice(markStart, markEnd); + + const livenessGuardIdx = mark.indexOf( + "if (lastInitConfig?.cliId === 'codex-app' && !codexAppTurnLiveness.notePrompt())", + ); + const proofGuardIdx = mark.indexOf( + "if (lastInitConfig?.cliId === 'codex-app' && !codexAppControlProven)", + ); + const signedIdleGuardIdx = mark.indexOf( + "if (lastInitConfig?.cliId === 'codex-app' && !codexAppReadyAuthority.canPublishPromptReady())", + ); + const readySetIdx = mark.indexOf('isPromptReady = true;'); + const promptReadySendIdx = mark.indexOf("send({ type: 'prompt_ready' });"); + const idleUpdateIdx = mark.indexOf("usageLimitTracker.classify(content, 'idle')"); + + expect(proofGuardIdx).toBeGreaterThan(-1); + expect(proofGuardIdx).toBeLessThan(livenessGuardIdx); + expect(livenessGuardIdx).toBeGreaterThan(-1); + expect(signedIdleGuardIdx).toBeGreaterThan(livenessGuardIdx); + // The explicit runner queue wins before any immediate daemon/card idle + // projection; returning from the guard therefore suppresses both paths. + expect(livenessGuardIdx).toBeLessThan(readySetIdx); + expect(signedIdleGuardIdx).toBeLessThan(readySetIdx); + expect(readySetIdx).toBeLessThan(promptReadySendIdx); + expect(promptReadySendIdx).toBeLessThan(idleUpdateIdx); + }); + + it('lets explicit Codex App activity override a stale idle screen heuristic', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const helperStart = source.indexOf('function codexAppLivenessStatus'); + const helperEnd = source.indexOf('// Per-turn usage-limit state machine', helperStart); + const helper = source.slice(helperStart, helperEnd); + const activityStart = source.indexOf("if (kind === 'activity' && lastInitConfig?.cliId === 'codex-app')"); + const activityEnd = source.indexOf("if (kind === 'final'", activityStart); + const activity = source.slice(activityStart, activityEnd); + + expect(helper).toContain("liveness.active && base === 'idle' ? 'working' : base"); + expect(activity).toContain('applyTrustedCodexAppActivityMarker('); + expect(activity).toContain('if (!activity.accepted)'); + expect(activity).toContain('isPromptReady = false;'); + }); + + it('re-drives a deferred Codex App prompt after a queued submit cancellation', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const flushStart = source.indexOf('async function flushPending'); + const flushEnd = source.indexOf('function sendToPty', flushStart); + const flush = source.slice(flushStart, flushEnd); + + expect(flush.match(/codexAppPromptReplay\.cancelSubmission\(\s*codexAppTurnLiveness,\s*codexAppReadyAuthority,\s*codexAppLivenessHandle,?\s*\)/g)).toHaveLength(2); + expect(flush).toContain('codexAppPromptReplay.consumeAfterFlush(codexAppTurnLiveness)'); + expect(flush.lastIndexOf('markPromptReady();')).toBeGreaterThan(flush.lastIndexOf('isFlushing = false;')); + }); + + it('persists a late-created public candidate before spawn and never trusts Codex App OSC', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const prepareIdx = source.indexOf('await prepareCodexAppControlGeneration(cfg, willReattachPersistent, !!persistentSessionName);'); + const candidateIdx = source.indexOf('prepareFreshCodexAppControlBootstrap(cfg, !!persistentSessionName);'); + const injectIdx = source.indexOf('childEnv[CODEX_APP_CONTROL_BOOTSTRAP_ENV] = codexAppControlBootstrapPathForSpawn;'); + const spawnIdx = source.indexOf('backend.spawn(spawnBin, spawnArgs'); + const finalizeIdx = source.indexOf('finalizeCodexAppControlGeneration(', spawnIdx); + const onDataIdx = source.indexOf('backend.onData(onPtyData);', spawnIdx); + const finalizeStart = source.indexOf('function finalizeCodexAppControlGeneration('); + const finalizeEnd = source.indexOf('function rejectCodexAppControlMarker', finalizeStart); + const finalize = source.slice(finalizeStart, finalizeEnd); + + expect(prepareIdx).toBeGreaterThan(-1); + expect(candidateIdx).toBeGreaterThan(prepareIdx); + expect(injectIdx).toBeGreaterThan(candidateIdx); + expect(spawnIdx).toBeGreaterThan(injectIdx); + expect(finalizeIdx).toBeGreaterThan(spawnIdx); + expect(onDataIdx).toBeGreaterThan(finalizeIdx); + expect(finalize).toContain("codexAppControlProven && codexAppControlStateValue?.status === 'active'"); + expect(source).toContain("const APP_RUNNER_OSC_CLI_IDS = new Set(['mira', 'mir']);"); + expect(source).not.toContain('CODEX_APP_CONTROL_NONCE_ENV'); + expect(source).not.toContain('codexAppControlNonceForSpawn'); + }); + + it('awaits bind and locator publication before every backend spawn path', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const spawnStart = source.indexOf('async function spawnCli('); + const prepareIdx = source.indexOf('await prepareCodexAppControlGeneration(', spawnStart); + const pluginPrepareIdx = source.indexOf('await prepareCliPluginGenerationAndGateway(cfg, cliAdapter)', prepareIdx); + const backendSpawnIdx = source.indexOf('backend.spawn(spawnBin, spawnArgs', prepareIdx); + + expect(spawnStart).toBeGreaterThan(-1); + expect(prepareIdx).toBeGreaterThan(spawnStart); + expect(backendSpawnIdx).toBeGreaterThan(prepareIdx); + expect(source.match(/await spawnCli\(/g)).toHaveLength(3); + expect(source.slice(spawnStart, prepareIdx)).toContain('const spawnGeneration = ++cliSpawnGeneration;'); + expect(source.slice(prepareIdx, backendSpawnIdx)) + .toContain('if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError();'); + expect(source.slice(pluginPrepareIdx, backendSpawnIdx) + .match(/if \(spawnGeneration !== cliSpawnGeneration\) throw new CliSpawnSupersededError\(\);/g)) + .toHaveLength(2); + expect(source).toContain('function killCli(opts: { preservePending?: boolean } = {}): void {\n cliSpawnGeneration++;'); + expect(source.match(/err instanceof CliSpawnSupersededError/g)).toHaveLength(3); + }); + + it('uses hardened locators, random endpoints, and process-lifetime publisher leases', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const prepareStart = source.indexOf('async function prepareCodexAppControlGeneration('); + const prepareEnd = source.indexOf('/** Late-create the only secret-bearing file', prepareStart); + const prepare = source.slice(prepareStart, prepareEnd); + const bootstrapStart = prepareEnd; + const bootstrapEnd = source.indexOf('function finalizeCodexAppControlGeneration(', bootstrapStart); + const bootstrap = source.slice(bootstrapStart, bootstrapEnd); + const acceptStart = source.indexOf('function acceptCodexAppControlSocket('); + const acceptEnd = source.indexOf('function removeStaleCodexAppSocket', acceptStart); + const accept = source.slice(acceptStart, acceptEnd); + const stopStart = source.indexOf('function stopCodexAppControlChannel('); + const stopEnd = source.indexOf('function failCodexAppControlGeneration', stopStart); + const stop = source.slice(stopStart, stopEnd); + + expect(prepare).toContain("process.platform === 'win32' ? codexAppWindowsControlRoot()"); + const leaseIdx = prepare.indexOf('await ensureCodexAppWindowsOwnerLease(cfg.sessionId)'); + const locatorIdx = prepare.indexOf('codexAppControlLocatorPath(controlRoot, cfg.sessionId)'); + expect(leaseIdx).toBeGreaterThan(-1); + expect(locatorIdx).toBeGreaterThan(leaseIdx); + expect(prepare).toContain('else await ensureCodexAppPosixOwnerLease(controlRoot, cfg.sessionId);'); + expect(prepare).toContain('codexAppControlLocatorPath(controlRoot, cfg.sessionId)'); + expect(prepare).toContain('const started = await startCodexAppControlEndpoint(cfg, channelId);'); + expect(source).toContain('await bindThenPublishCodexAppControlLocator({'); + expect(source).toContain('generateCodexAppWindowsPipeEndpoint()'); + expect(source).toContain('generateCodexAppPosixSocketEndpoint(codexAppControlSocketDirectory)'); + expect(bootstrap).toContain("{ kind: 'locator', locatorPath: codexAppControlLocatorPathValue }"); + expect(accept).toContain("rotateCodexAppControlEndpoint('active socket closed')"); + expect(accept).toContain('if (wasActive'); + expect(stop).toContain("if (socketPath && process.platform !== 'win32')"); + expect(stop).not.toContain('unlinkSync(codexAppControlLocatorPathValue)'); + expect(prepare).toContain('Deleting first would'); + const killStart = source.indexOf('function killCli('); + const killEnd = source.indexOf('function cleanup()', killStart); + const cleanupStart = source.indexOf('function cleanup()'); + const cleanupEnd = source.indexOf("process.on('SIGTERM'", cleanupStart); + expect(source.slice(killStart, killEnd)).not.toContain('releaseCodexAppPosixOwnerLease()'); + expect(source.slice(cleanupStart, cleanupEnd)).toContain('releaseCodexAppPosixOwnerLease();'); + }); + + it('prevents retired async endpoints from publishing or failing a newer channel', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const start = source.slice( + source.indexOf('async function startCodexAppControlEndpoint('), + source.indexOf('function installCodexAppControlEndpoint('), + ); + const rotateStart = source.indexOf('function rotateCodexAppControlEndpoint('); + const rotate = source.slice( + rotateStart, + source.indexOf('async function prepareCodexAppControlGeneration(', rotateStart), + ); + + expect(start.indexOf('channelId !== codexAppControlChannelId')).toBeLessThan( + start.indexOf('writeCodexAppControlLocator(locatorPath, locator);'), + ); + expect(start).toContain('if (codexAppControlServer === server)'); + expect(rotate).toContain('if (shouldFailCodexAppControlChannel({'); + expect(rotate).toContain('if (codexAppControlRotation === rotation) codexAppControlRotation = undefined;'); + }); + + it('keeps an unproved endpoint bound until the shared 90-second fail-close deadline', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const acceptStart = source.indexOf('function acceptCodexAppControlSocket('); + const acceptEnd = source.indexOf('function removeStaleCodexAppSocket', acceptStart); + const accept = source.slice(acceptStart, acceptEnd); + const finalizeStart = source.indexOf('function finalizeCodexAppControlGeneration('); + const finalizeEnd = source.indexOf('function rejectCodexAppControlMarker', finalizeStart); + const finalize = source.slice(finalizeStart, finalizeEnd); + + expect(accept).toContain('Rotate only after the authenticated runner closes.'); + expect(accept).toContain('if (wasActive'); + expect(accept).toContain('codexAppProofDeadline.arm(() => {'); + expect(accept).toContain('did not re-authenticate within'); + expect(accept).not.toContain('pre-auth socket closed'); + expect(finalize).toContain('codexAppProofDeadline.arm(() => {'); + }); + + it('shares the 90-second first-prompt cap with bootstrap cleanup and runner proof', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const bootstrapStart = source.indexOf('function prepareFreshCodexAppControlBootstrap('); + const bootstrapEnd = source.indexOf('function finalizeCodexAppControlGeneration(', bootstrapStart); + const bootstrap = source.slice(bootstrapStart, bootstrapEnd); + const proofStart = bootstrapEnd; + const proofEnd = source.indexOf('function rejectCodexAppControlMarker', proofStart); + const proof = source.slice(proofStart, proofEnd); + + expect(source).toContain('const FIRST_PROMPT_HARD_TIMEOUT_MS = CODEX_APP_CONTROL_STARTUP_TIMEOUT_MS;'); + expect(bootstrap).toContain('armCodexAppControlStartupTimeout(cleanupCodexAppControlBootstrap)'); + expect(proof).toContain('codexAppProofDeadline.arm(() => {'); + expect(bootstrap).not.toContain('30_000'); + expect(proof).not.toContain('30_000'); + }); + + it('kills legacy/no-public-key reattach and fail-closes candidate setup before PTY listeners attach', () => { + const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); + const preflightIdx = source.indexOf('shouldColdStartCodexAppReattach({'); + const preflightKillIdx = source.indexOf('killPersistentSession(', preflightIdx); + const prepareIdx = source.indexOf('prepareCodexAppControlGeneration(', preflightIdx); + const finalizeIdx = source.indexOf('finalizeCodexAppControlGeneration(', prepareIdx); + const failureKillIdx = source.indexOf('killPersistentSession(', finalizeIdx); + const onDataIdx = source.indexOf('backend.onData(onPtyData);', finalizeIdx); + + expect(preflightIdx).toBeGreaterThan(-1); + expect(preflightKillIdx).toBeGreaterThan(preflightIdx); + expect(preflightKillIdx).toBeLessThan(prepareIdx); + expect(finalizeIdx).toBeGreaterThan(prepareIdx); + expect(failureKillIdx).toBeGreaterThan(finalizeIdx); + expect(failureKillIdx).toBeLessThan(onDataIdx); + }); + it('honors a true ready signal that arrives AFTER the timeout fallback (slow cold start)', () => { // ReadyGate.receive() is one-shot: once the 45s fallback fires, a later // releaseReadyGate from the real signal is skipped entirely. A CLI whose diff --git a/test/worker-ready-display-mode.test.ts b/test/worker-ready-display-mode.test.ts index 7a83be557..6e85e149b 100644 --- a/test/worker-ready-display-mode.test.ts +++ b/test/worker-ready-display-mode.test.ts @@ -127,6 +127,7 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ // ─── Imports under test ──────────────────────────────────────────────────── import { CARD_POSTING_SENTINEL, initWorkerPool, __testOnly_setupWorkerHandlers } from '../src/core/worker-pool.js'; +import { MessageWithdrawnError } from '../src/im/lark/client.js'; import type { DaemonSession } from '../src/core/types.js'; import { getBot } from '../src/bot-registry.js'; import * as sessionStore from '../src/services/session-store.js'; @@ -309,6 +310,32 @@ describe('Worker ready: set_display_mode re-sync', () => { expect(displayModeCalls).toHaveLength(0); }); + it('preserves worker and pending Codex FIFO when the root is withdrawn during ready POST', async () => { + sessionReplyMock.mockRejectedValueOnce(new MessageWithdrawnError('om_root')); + const fakeWorker = makeFakeWorker(); + const ds = makeDs({ + streamCardPending: true, + streamCardId: undefined, + worker: fakeWorker, + }); + ds.session.codexAppDispatchLedger = [{ + dispatchId: 'dispatch-pending', + turnId: 'turn-pending', + state: 'prepared', + content: 'pending', + }]; + + __testOnly_setupWorkerHandlers(ds, fakeWorker); + fakeWorker.emit('message', { + type: 'ready', port: 9999, token: 'tok_abc', turnId: 'turn-pending', + }); + await flush(); + + expect(closeSessionMock).not.toHaveBeenCalled(); + expect(fakeWorker.kill).not.toHaveBeenCalled(); + expect(ds.session.codexAppDispatchLedger).toHaveLength(1); + }); + it('PATCH path sends set_display_mode when displayMode is screenshot', async () => { const fakeWorker = makeFakeWorker(); // Existing card + streamCardPending=false → PATCH path @@ -540,6 +567,51 @@ describe('Worker ready: set_display_mode re-sync', () => { ); }); + it('replays a restored raw opening with its durable token and releases only on the matching ACK', async () => { + const submitted = vi.fn(async () => true); + initWorkerPool({ + sessionReply: sessionReplyMock, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: closeSessionMock, + onQueuedActivationSubmitted: submitted, + }); + const fakeWorker = makeFakeWorker(); + const ds = makeDs({ + worker: fakeWorker, + pendingRawInput: '/goal RESTORED_RAW_N', + initialStartPending: true, + } as Partial); + Object.assign(ds.session, { + queuedActivationPending: true, + queuedActivationToken: 'raw-activation-token', + queuedActivationTurnId: 'turn-raw-n', + pendingRepoSetup: { + mode: 'picker', prompt: '', rawInput: '/goal RESTORED_RAW_N', turnId: 'turn-raw-n', + }, + }); + + __testOnly_setupWorkerHandlers(ds, fakeWorker); + fakeWorker.emit('message', { type: 'prompt_ready' }); + await flush(); + + expect(fakeWorker.send).toHaveBeenCalledWith({ + type: 'raw_input', + content: '/goal RESTORED_RAW_N', + queuedActivationToken: 'raw-activation-token', + turnId: 'turn-raw-n', + }); + expect(submitted).not.toHaveBeenCalled(); + + fakeWorker.emit('message', { + type: 'queued_activation_submitted', + sessionId: ds.session.sessionId, + activationToken: 'raw-activation-token', + }); + await flush(); + expect(submitted).toHaveBeenCalledWith(ds, 'raw-activation-token'); + }); + it('prompt_ready bundles the buffered follow-up ONTO the raw_input IPC (single atomic message)', async () => { // Two separate IPCs would race inside the worker: its async message // handlers don't serialize, and raw_input awaits 200ms between sendText diff --git a/test/worker-suspend.test.ts b/test/worker-suspend.test.ts index bcbe29287..416c037a2 100644 --- a/test/worker-suspend.test.ts +++ b/test/worker-suspend.test.ts @@ -123,6 +123,30 @@ describe('suspendWorker', () => { expect(ds.managedTurnOrigin).toEqual({ capability: 'cap-pty', turnId: 'om-pty' }); }); + it('refuses suspension while the durable Codex App dispatch ledger is non-empty', () => { + const worker = fakeWorker(); + const ds: any = { + session: { + sessionId: 'sid-owned', + status: 'active', + codexAppDispatchLedger: [ + { dispatchId: 'd-1', turnId: 't-1', state: 'prepared', content: 'owned' }, + ], + }, + initConfig: { backendType: 'tmux' }, + worker, + workerPort: 3456, + workerToken: 'token', + lastScreenStatus: 'idle', + }; + + expect(suspendWorker(ds, 'manual_suspend')).toBe(false); + expect(worker.send).not.toHaveBeenCalled(); + expect(ds.worker).toBe(worker); + expect(ds.workerPort).toBe(3456); + expect(ds.workerToken).toBe('token'); + }); + it.each([ ['missing', null], ['already killed', { killed: true }], diff --git a/test/zellij-observe-backend.test.ts b/test/zellij-observe-backend.test.ts index 06fac6556..b672c4f5e 100644 --- a/test/zellij-observe-backend.test.ts +++ b/test/zellij-observe-backend.test.ts @@ -6,9 +6,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; // transiently disappears and comes back. const calls: string[][] = []; let listPanesResult: () => string = () => JSON.stringify([{ id: 2, is_plugin: false }]); +let failActions = false; vi.mock('node:child_process', () => ({ execFileSync: (bin: string, args: string[]) => { calls.push([bin, ...args]); + if (failActions && args.includes('action')) throw new Error('pane unavailable'); if (args.includes('dump-screen')) return 'line one\nline two\nline three\n'; if (args.includes('list-panes')) return listPanesResult(); return ''; @@ -26,6 +28,7 @@ describe('ZellijObserveBackend input encoding', () => { let be: ZellijObserveBackend; beforeEach(() => { calls.length = 0; + failActions = false; be = new ZellijObserveBackend(S, P, { cliPid: 999 }); }); @@ -44,6 +47,12 @@ describe('ZellijObserveBackend input encoding', () => { expect(actionArgs('write')).toEqual(['write', '--pane-id', P, '3']); }); + it('returns false when the targeted pane action is unavailable', () => { + failActions = true; + expect(be.sendText('/goal x')).toBe(false); + expect(be.sendSpecialKeys('Enter')).toBe(false); + }); + it('pasteText wraps text in bracketed-paste markers', () => { be.pasteText('x'); // captured call = ['zellij','--session',S,'action','write','--pane-id',P,...bytes] From 197b4e8f9c158f19758ee01ac72c1d47984c86a9 Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:37:36 +0800 Subject: [PATCH 2/5] test(codex-app): cover same-admission cap sweep --- test/idle-worker-sweeper.test.ts | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/idle-worker-sweeper.test.ts b/test/idle-worker-sweeper.test.ts index 9d6958e64..08e5fcdaf 100644 --- a/test/idle-worker-sweeper.test.ts +++ b/test/idle-worker-sweeper.test.ts @@ -24,6 +24,7 @@ import { import { __testOnly_resetBotTurnMutationGates, withBotTurnAdmission, + withBotTurnMutation, } from '../src/core/bot-turn-mutation-gate.js'; function ds(sessionId: string, backendType: string, lastMessageAt: number, worker = {}) { @@ -257,6 +258,50 @@ describe('sweepIdleWorkers (per-bot count cap)', () => { expect(b.worker).toBe(null); }); + it('queues a Codex App cap sweep behind a same-admission mutation and preserves its durable owner', async () => { + const appId = 'cli_same_admission'; + const owned = ds('owned', 'tmux', now - 90 * 60_000); + owned.session.codexAppDispatchLedger = [ + { dispatchId: 'd-owned', turnId: 't-owned', state: 'accepted', content: 'owned' }, + ]; + const next = ds('next', 'tmux', now - 80 * 60_000); + const newest = ds('newest', 'tmux', now - 10 * 60_000); + const activeSessions = new Map([ + ['owned', owned], + ['next', next], + ['newest', newest], + ]); + let releaseFirst!: () => void; + const holdFirst = new Promise(resolve => { releaseFirst = resolve; }); + let firstStarted!: () => void; + const started = new Promise(resolve => { firstStarted = resolve; }); + + const admitted = withBotTurnAdmission(appId, async () => { + const firstMutation = withBotTurnMutation(appId, async () => { + firstStarted(); + await holdFirst; + }); + const sweep = sweepIdleWorkersAfterTurnDrain(appId, activeSessions, { + maxLiveWorkers: 2, + mutationAcquireTimeoutMs: 1_000, + }); + const [, suspended] = await Promise.all([firstMutation, sweep]); + return suspended; + }); + + await started; + await Promise.resolve(); + const stayedLiveWhileQueued = next.worker !== null; + releaseFirst(); + expect(stayedLiveWhileQueued).toBe(true); + + await expect(admitted).resolves.toEqual([ + { sessionId: 'next', reason: 'live_worker_cap' }, + ]); + expect(owned.worker).not.toBe(null); + expect(next.worker).toBe(null); + }); + it('skips a sweep after a bounded wait instead of freezing the bot mutation gate', async () => { const appId = 'cli_wedged'; const activeSessions = new Map([ From bcbfa650c6471deed506ba220b2ceeef973cefb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E6=99=97?= Date: Sun, 26 Jul 2026 12:08:02 -0700 Subject: [PATCH 3/5] =?UTF-8?q?fix(codex-app):=20=E4=BF=AE=E9=A6=96?= =?UTF-8?q?=E5=AE=A1=202=20=E4=B8=AA=20P2(Lark=20upload=20=E8=B6=85?= =?UTF-8?q?=E6=97=B6=20+=20restore=20=E5=AD=A4=E5=84=BF=E8=A1=8C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1 Lark upload 超时:#597 给全 bot/全 API 加的 15s SDK 超时对文件/视频 上传过窄(uploadImage/uploadFile 走同一 client 且无重试,~19MB@10Mbps 就 会被截断)。官方 SDK 的 httpInstance 是模块级共享单例、typed 上传方法无 per-request 超时钩子,故给上传单独一个 http 实例(defaultHttpInstance.create + 复制 UA/response-unwrap 拦截器)+ 120s 超时;交互调用仍保持 15s。worker 侧 lark-upload.ts 同款处理。SDK 不再导出 defaultHttpInstance 时 fail-safe 回退到 共享实例(上传退回 15s,不 brick)。 P2-2 restore 孤儿行:restoreActiveSessions 里 activation-tail 提升失败时原本 throw → 落进 per-row 隔离 catch 但未注册 → 该行盘上 active 却不在 activeSessions (IM /close 够不到 + 同 anchor 再来消息会建第二条 active 行,老 tail 永久悬挂)。 改为不 throw:注册成可见的 quarantined owner(未提升 tail 保持 protected 占住 anchor、可 /close、下次激活自然重试提升)。提升失败仅在 send:false 的瞬时持久化 错误时发生,故 quarantine + 重试是自愈的。 测试: - bot-registry:新增上传专用 http 实例断言(交互 15s vs 上传 120s、独立实例、 共享 default 未被污染);FakeClient/mock 补 defaultHttpInstance 镜像真实 SDK - session-resume:新增 restore 瞬时提升失败注册 quarantined owner 回归用例 (变异测试验证:改回 throw 则该用例 FAIL,证明对实现敏感) - pnpm build 绿;P2 相关 8 套件 264 用例全绿;7 个 SDK-mock 套件 147 用例全绿 (defensive 回退生效) Co-Authored-By: Riff --- src/bot-registry.ts | 79 ++++++++++++++++++++++++++++++++++++- src/core/session-manager.ts | 15 ++++++- src/im/lark/client.ts | 6 +-- src/utils/lark-upload.ts | 45 ++++++++++++++++++++- test/bot-registry.test.ts | 33 +++++++++++++++- test/session-resume.test.ts | 48 +++++++++++++++++++++- 6 files changed, 216 insertions(+), 10 deletions(-) diff --git a/src/bot-registry.ts b/src/bot-registry.ts index 9b646350a..267320696 100644 --- a/src/bot-registry.ts +++ b/src/bot-registry.ts @@ -35,12 +35,68 @@ export type { * hold a bot-turn admission or maintenance mutation indefinitely. */ export const LARK_REQUEST_TIMEOUT_MS = 15_000; +/** Media uploads (image/file) ride the same official-SDK path but move real + * bytes: a 30 MB video on a modest uplink legitimately exceeds the interactive + * request bound. They also run in the `botmux send` CLI subprocess, which holds + * no daemon admission/mutation lock, so the interactive timeout's protective + * purpose does not apply to them. Give uploads a far looser ceiling. */ +export const LARK_UPLOAD_TIMEOUT_MS = 120_000; + export function configureLarkClientHttpTimeout(client: unknown): void { const defaults = (client as { httpInstance?: { defaults?: { timeout?: number } } } | null) ?.httpInstance?.defaults; if (defaults) defaults.timeout = LARK_REQUEST_TIMEOUT_MS; } +/** + * A dedicated SDK http instance for media uploads. The official SDK shares ONE + * module-level axios singleton across every `Client` (verified: two clients + * report the same `httpInstance`), and its typed `image.create`/`file.create` + * expose no per-request timeout hook — so the only knob for uploads is a + * separate instance. `defaultHttpInstance.create()` yields an independent axios + * (its own `defaults`, not the shared one); we copy the SDK's own request UA and + * response-unwrap interceptors so upload responses (`res.data` → `image_key`) + * behave identically. Falls back to leaving the client on the shared instance + * if the SDK ever stops exporting `defaultHttpInstance`, so a future SDK bump + * degrades to "uploads keep the interactive timeout" rather than breaking. + */ +let cachedLarkUploadHttpInstance: unknown; +export function larkUploadHttpInstance(): unknown { + if (cachedLarkUploadHttpInstance !== undefined) return cachedLarkUploadHttpInstance; + let base: any; + try { + base = (Lark as unknown as { defaultHttpInstance?: any }).defaultHttpInstance; + } catch { + // A stripped/mocked SDK namespace may throw on accessing an absent export. + base = undefined; + } + if (!base || typeof base.create !== 'function') { + cachedLarkUploadHttpInstance = null; + return cachedLarkUploadHttpInstance; + } + const instance = base.create({ timeout: LARK_UPLOAD_TIMEOUT_MS }); + try { + for (const handler of base.interceptors?.request?.handlers ?? []) { + if (handler) { + instance.interceptors.request.use(handler.fulfilled, handler.rejected, { + synchronous: handler.synchronous, + }); + } + } + for (const handler of base.interceptors?.response?.handlers ?? []) { + if (handler) instance.interceptors.response.use(handler.fulfilled, handler.rejected); + } + } catch { + // A shape change in the SDK's interceptor registry must not brick uploads; + // an instance without the response-unwrap interceptor would misread + // responses, so fall back to the shared instance (interactive timeout). + cachedLarkUploadHttpInstance = null; + return cachedLarkUploadHttpInstance; + } + cachedLarkUploadHttpInstance = instance; + return cachedLarkUploadHttpInstance; +} + export type ChatReplyMode = 'chat' | 'new-topic' | 'shared' | 'chat-topic'; export type ContentTriggerScope = 'topic' | 'regularGroup' | 'both'; export type ContentTriggerMatchType = 'keyword' | 'regex'; @@ -1307,6 +1363,9 @@ export interface BotConfig { export interface BotState { config: BotConfig; client: Lark.Client; + /** Same credentials/domain as `client`, but bound to a dedicated http + * instance with the looser upload timeout. Only media uploads use it. */ + uploadClient: Lark.Client; botOpenId?: string; botName?: string; // Lark app display name (from /bot/v3/info) botAvatarUrl?: string; // Lark app avatar URL (from /bot/v3/info) @@ -1322,6 +1381,7 @@ export function __testOnly_resetBotRegistry(): void { loadedConfigPath = undefined; oncallChatCache = null; brandLabelCache = null; + cachedLarkUploadHttpInstance = undefined; } // Wire the i18n lookup so `localeForBot()` can resolve per-bot locale without @@ -1408,18 +1468,27 @@ const larkLogger = { }; export function registerBot(cfg: BotConfig): BotState { - const client = new Lark.Client({ + const clientParams = { appId: cfg.larkAppId, appSecret: cfg.larkAppSecret, // brand → SDK domain。缺省走 feishu，国际版租户走 larksuite.com。 // 这一行同时修好了所有经由 SDK 的调用（发消息 / 文件 / contact 等）。 domain: sdkDomain(normalizeBrand(cfg.brand)), logger: larkLogger, - }); + }; + const client = new Lark.Client(clientParams); configureLarkClientHttpTimeout(client); + // Media uploads reuse the same credentials/domain but ride a dedicated http + // instance with the looser upload timeout. When the SDK no longer exposes a + // separable instance, fall back to the interactive client (uploads keep 15s). + const uploadHttpInstance = larkUploadHttpInstance(); + const uploadClient = uploadHttpInstance + ? new Lark.Client({ ...clientParams, httpInstance: uploadHttpInstance as any }) + : client; const state: BotState = { config: cfg, client, + uploadClient, resolvedAllowedUsers: [...(cfg.allowedUsers ?? [])], rawAllowedUserResolution: new Map(), }; @@ -1445,6 +1514,12 @@ export function getBotClient(larkAppId: string): Lark.Client { return getBot(larkAppId).client; } +/** Client bound to the looser upload timeout. Use only for media uploads + * (image/file); every other call uses `getBotClient` and its interactive bound. */ +export function getBotUploadClient(larkAppId: string): Lark.Client { + return getBot(larkAppId).uploadClient; +} + /** Owner = bot 首个已授权 open_id，与「缺权限警告私信对象」同口径（见 admin 解析）。 */ export function getOwnerOpenId(larkAppId: string): string | undefined { return bots.get(larkAppId)?.resolvedAllowedUsers.find(u => u.startsWith('ou_')); diff --git a/src/core/session-manager.ts b/src/core/session-manager.ts index 6b9104eda..31295a2e8 100644 --- a/src/core/session-manager.ts +++ b/src/core/session-manager.ts @@ -1660,7 +1660,20 @@ export async function restoreActiveSessions(activeSessions: Map 0 && !promoteQueuedActivationTail(ds, { send: false })) { - throw new Error(`failed to promote durable activation tail for ${session.sessionId}`); + // Promotion here only fails on a transient durable-write error (send:false + // skips the worker-liveness gate, and the caller already checked a non-empty + // tail). Throwing would drop to the isolation catch below WITHOUT registering + // the row — leaving it active-on-disk but invisible to activeSessions, so IM + // `/close` cannot reach it and a later inbound to the same anchor mints a + // second active row while this one's tail dangles. Instead register it as a + // visible quarantined owner: the unpromoted tail keeps protected ownership + // (occupies the anchor, blocks a duplicate, stays closeable), and the normal + // activation path retries the promotion once the durable store recovers. + logger.warn( + `[${session.sessionId.substring(0, 8)}] Deferred durable activation-tail promotion ` + + `(transient persistence failure); registering as a quarantined owner so ` + + `it stays visible/closeable and retries on next activation`, + ); } const anchor = sessionAnchorId(ds); messageQueue.ensureQueue(anchor); diff --git a/src/im/lark/client.ts b/src/im/lark/client.ts index 6e4e80dd6..f86ba03fd 100644 --- a/src/im/lark/client.ts +++ b/src/im/lark/client.ts @@ -2,7 +2,7 @@ import { readFileSync, writeFileSync, createWriteStream, mkdirSync, existsSync } import { dirname, extname, basename, join } from 'node:path'; import { pipeline } from 'node:stream/promises'; import { Client } from '@larksuiteoapi/node-sdk'; -import { getBotClient, getAllBots, getBot, formatLarkError } from '../../bot-registry.js'; +import { getBotClient, getBotUploadClient, getAllBots, getBot, formatLarkError } from '../../bot-registry.js'; import { loadBotConfigs } from '../../bot-registry.js'; import { config } from '../../config.js'; import { emitHookEvent } from '../../services/hook-runner.js'; @@ -957,7 +957,7 @@ const EXT_TO_FILE_TYPE: Record = { }; export async function uploadImage(larkAppId: string, imagePath: string): Promise { - const c = getBotClient(larkAppId); + const c = getBotUploadClient(larkAppId); const buf = readFileSync(imagePath); // SDK returns { image_key } directly (not wrapped in { code, data }) const res = await c.im.v1.image.create({ @@ -970,7 +970,7 @@ export async function uploadImage(larkAppId: string, imagePath: string): Promise } export async function uploadFile(larkAppId: string, filePath: string, opts?: { duration?: number }): Promise { - const c = getBotClient(larkAppId); + const c = getBotUploadClient(larkAppId); const buf = readFileSync(filePath); const ext = extname(filePath).toLowerCase(); const fileType = EXT_TO_FILE_TYPE[ext] ?? 'stream'; diff --git a/src/utils/lark-upload.ts b/src/utils/lark-upload.ts index a331fe69e..2fdeaac51 100644 --- a/src/utils/lark-upload.ts +++ b/src/utils/lark-upload.ts @@ -3,19 +3,60 @@ * Worker doesn't load bot-registry — it gets larkAppId/larkAppSecret/brand from * the daemon's init message (see worker-pool.ts forkWorker / worker.ts init). */ -import { Client, LoggerLevel } from '@larksuiteoapi/node-sdk'; +import { Client, LoggerLevel, defaultHttpInstance } from '@larksuiteoapi/node-sdk'; import { type Brand, sdkDomain } from '../im/lark/lark-hosts.js'; +/** Screenshot/media uploads move real bytes and must not inherit an interactive + * request bound. The worker's client currently has no timeout applied, but pin + * an explicit generous ceiling on a dedicated http instance so this stays true + * even if a future bound is added — mirrors bot-registry's upload client. */ +const UPLOAD_TIMEOUT_MS = 120_000; + +let cachedUploadHttpInstance: any; +function uploadHttpInstance(): any { + if (cachedUploadHttpInstance !== undefined) return cachedUploadHttpInstance; + const base: any = defaultHttpInstance; + if (!base || typeof base.create !== 'function') { + cachedUploadHttpInstance = null; + return cachedUploadHttpInstance; + } + const instance = base.create({ timeout: UPLOAD_TIMEOUT_MS }); + try { + for (const handler of base.interceptors?.request?.handlers ?? []) { + if (handler) { + instance.interceptors.request.use(handler.fulfilled, handler.rejected, { + synchronous: handler.synchronous, + }); + } + } + for (const handler of base.interceptors?.response?.handlers ?? []) { + if (handler) instance.interceptors.response.use(handler.fulfilled, handler.rejected); + } + } catch { + cachedUploadHttpInstance = null; + return cachedUploadHttpInstance; + } + cachedUploadHttpInstance = instance; + return cachedUploadHttpInstance; +} + let cached: { client: any; appId: string; brand: Brand } | null = null; function getClient(appId: string, secret: string, brand: Brand) { // 缓存 key 含 brand：同 appId 不同 brand 不复用打错域的客户端。 if (cached && cached.appId === appId && cached.brand === brand) return cached.client; + const httpInstance = uploadHttpInstance(); cached = { appId, brand, // brand → 域名。Lark bot 截图上传必须打 larksuite.com，否则 image.create 失败。 - client: new Client({ appId, appSecret: secret, domain: sdkDomain(brand), loggerLevel: LoggerLevel.error }), + client: new Client({ + appId, + appSecret: secret, + domain: sdkDomain(brand), + loggerLevel: LoggerLevel.error, + ...(httpInstance ? { httpInstance } : {}), + }), }; return cached.client; } diff --git a/test/bot-registry.test.ts b/test/bot-registry.test.ts index 25c34ea79..769332c54 100644 --- a/test/bot-registry.test.ts +++ b/test/bot-registry.test.ts @@ -10,13 +10,31 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; // Mock @larksuiteoapi/node-sdk — we don't want real Lark connections. // The Client constructor just stores whatever it receives. vi.mock('@larksuiteoapi/node-sdk', () => { + // Mirror the real SDK's separable http instance so the upload-client path is + // exercised (create() → own instance + copyable interceptor registry). + const makeInstance = (): any => ({ + defaults: { timeout: 0 }, + create: (cfg: { timeout?: number }) => { + const inst = makeInstance(); + if (cfg?.timeout !== undefined) inst.defaults.timeout = cfg.timeout; + return inst; + }, + interceptors: { + request: { handlers: [], use(this: any, f: any, r: any) { this.handlers.push({ fulfilled: f, rejected: r }); } }, + response: { handlers: [], use(this: any, f: any, r: any) { this.handlers.push({ fulfilled: f, rejected: r }); } }, + }, + }); + const sharedDefault = makeInstance(); class FakeClient { opts: Record; + httpInstance: any; constructor(opts: Record) { this.opts = opts; + // Real Client: `params.httpInstance || defaultHttpInstance`. + this.httpInstance = (opts?.httpInstance as any) ?? sharedDefault; } } - return { Client: FakeClient }; + return { Client: FakeClient, defaultHttpInstance: sharedDefault }; }); // Mock node:fs so loadBotConfigs doesn't touch real disk. @@ -78,6 +96,19 @@ describe('registerBot', () => { expect(client.httpInstance.defaults.timeout).toBe(mod.LARK_REQUEST_TIMEOUT_MS); }); + it('gives media uploads a dedicated http instance with the looser upload timeout', () => { + const state = mod.registerBot(makeCfg()); + const interactive = state.client as unknown as { httpInstance?: { defaults?: { timeout?: number } } }; + const upload = state.uploadClient as unknown as { httpInstance?: { defaults?: { timeout?: number } } }; + // Interactive client keeps the tight bound; upload client is separate + looser. + expect(interactive.httpInstance?.defaults?.timeout).toBe(mod.LARK_REQUEST_TIMEOUT_MS); + expect(upload.httpInstance?.defaults?.timeout).toBe(mod.LARK_UPLOAD_TIMEOUT_MS); + expect(state.uploadClient).not.toBe(state.client); + expect(mod.getBotUploadClient('app_test_001')).toBe(state.uploadClient); + // The shared SDK default must NOT be mutated to the upload bound. + expect(mod.LARK_UPLOAD_TIMEOUT_MS).toBeGreaterThan(mod.LARK_REQUEST_TIMEOUT_MS); + }); + it('should default the SDK Client domain to feishu when brand is unset', () => { const state = mod.registerBot(makeCfg()); const client = state.client as unknown as { opts: Record }; diff --git a/test/session-resume.test.ts b/test/session-resume.test.ts index d511fecce..3f562ddf2 100644 --- a/test/session-resume.test.ts +++ b/test/session-resume.test.ts @@ -54,6 +54,9 @@ vi.mock('../src/core/worker-pool.js', () => ({ killStalePids: vi.fn(), getCurrentCliVersion: vi.fn(() => '1.0.0-test'), restoreUsageLimitRuntimeState: vi.fn(), + // Default: promotion succeeds. A specific test overrides this to false to + // exercise the restore-time transient-failure quarantine path. + promoteQueuedActivationTail: vi.fn(() => true), withActiveSessionKeyLock: vi.fn(async (_map: Map, _key: string, action: () => any) => action()), // Faithful: mirror the real setActiveSessionSafe — if a DIFFERENT entry // already holds the key, evict it (close) before setting, instead of a @@ -142,7 +145,7 @@ vi.mock('../src/core/session-activity.js', () => ({ })); import { restoreActiveSessions, resumeSession } from '../src/core/session-manager.js'; -import { restoreUsageLimitRuntimeState, closeSession, forkAdoptWorker } from '../src/core/worker-pool.js'; +import { restoreUsageLimitRuntimeState, closeSession, forkAdoptWorker, promoteQueuedActivationTail } from '../src/core/worker-pool.js'; import { TmuxBackend } from '../src/adapters/backend/tmux-backend.js'; import * as sessionStore from '../src/services/session-store.js'; import { sessionKey } from '../src/core/types.js'; @@ -155,6 +158,8 @@ beforeEach(() => { sessionStore.init(); wp.registry = null; vi.mocked(closeSession).mockClear(); + vi.mocked(promoteQueuedActivationTail).mockReset(); + vi.mocked(promoteQueuedActivationTail).mockReturnValue(true); }); afterEach(() => { @@ -783,5 +788,46 @@ describe('resumeSession', () => { expect(r.ds.workingDir).toBe('/srv/app'); expect(r.ds.ownerOpenId).toBe('ou_owner'); }); + + it('registers a visible quarantined owner (not an invisible orphan) when restore-time activation-tail promotion fails transiently', async () => { + // Regression for the P2 fix: a transient durable-write failure during + // restore must NOT throw the row into the isolation catch unregistered. + // Before the fix, the row stayed active-on-disk but absent from the Map, + // so IM `/close` could not reach it and a later inbound to the same anchor + // minted a second active row while this one's tail dangled. + const s = sessionStore.createSession('oc_chatQ', 'om_quarantine', 'Quarantine Topic', 'group'); + s.larkAppId = 'app_test'; + s.workingDir = '/tmp/proj'; + s.cliId = 'codex-app'; + s.scope = 'thread'; + s.status = 'active'; + s.hasHistory = true; + s.queuedActivationTail = [{ + id: 'tail-1', + order: 1, + userPrompt: 'held follow-up', + cliInput: { content: 'held follow-up' }, + turnId: 'turn-held', + }] as any; + s.queuedActivationTailNextOrder = 1; + sessionStore.updateSession(s); + + // Simulate the transient persistence failure inside promotion. + vi.mocked(promoteQueuedActivationTail).mockReturnValue(false); + + const map = new Map(); + await restoreActiveSessions(map); + + // The row is registered (visible + anchor-occupied + closeable), NOT dropped. + const ds = map.get(sessionKey('om_quarantine', 'app_test')); + expect(ds).toBeDefined(); + expect(ds!.session.sessionId).toBe(s.sessionId); + // Its unpromoted tail is retained for a later retry. + expect(ds!.session.queuedActivationTail?.length).toBe(1); + // Promotion was attempted (send:false, worker-null restore path). + expect(vi.mocked(promoteQueuedActivationTail)).toHaveBeenCalled(); + // On-disk row stays active (retained for inspection/retry, not closed away). + expect(sessionStore.getSession(s.sessionId)?.status).toBe('active'); + }); }); }); From 6075451951bcf35d7613c60926a5ca084f6818c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E6=99=97?= Date: Sun, 26 Jul 2026 12:34:52 -0700 Subject: [PATCH 4/5] =?UTF-8?q?fix(codex-app):=20=E6=94=B6=E5=8F=A3=20code?= =?UTF-8?q?x=20=E5=A4=8D=E5=AE=A1=E4=B8=A4=E7=82=B9(upload=20fail-safe=20?= =?UTF-8?q?=E7=9C=9F=E5=9B=9E=E9=80=80=20+=20quarantine=20=E8=87=AA?= =?UTF-8?q?=E6=84=88)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 采纳 codex 二次复审的两个 P2: 1) worker upload fail-safe 失效:lark-upload.ts 用 named import `defaultHttpInstance`,SDK/mock 缺该 export 时在 fail-safe 判断前就抛 (vitest 对缺失 named binding 的访问直接 throw)。改成 `import * as Lark` + 防御式 namespace 访问(try/catch,mirror bot-registry),缺失时得 undefined→回退共享实例而非 brick。test/lark-upload.test.ts 原 3 失败转绿。 2) restore quarantine 不自愈:promotion 首次失败后 initialStartPending 恒 true,tryAcquireInitialStartClaim 因此拒绝认领冷 owner,后续消息只 admitQueuedActivationTail 追加+return,promote 永不重试→session 卡到 /close。修:quarantine 时强制 initialStartPending=false,使下次入站能 claim→fork→forkReservedInitialSession 重新 derive gate 并 releaseQueuedActivationReservation→promoteQueuedActivationTail 排空 tail (新消息按 reservation 排在其后,不错序);持久后端 restore 亦经 toReattach 重 fork。promote 成功路径不受影响(gate 该留则留)。 测试(均变异验证对实现敏感): - lark-upload:补 defaultHttpInstance mock 镜像真实 SDK + 上传实例 120s 断言; ⭐并保留独立的 missing-export 用例(vi.doMock 去掉 export)断言回退到普通 Client 且不抛——不让修 mock 抹掉本 bug 的触发条件(codex 要求) - session-resume:quarantine 用例加断言 initialStartPending===false(自愈 enabler);新增 promote 成功时 initialStartPending 仍 true 的反向用例(防 自愈修法过度反应) - promote 的 retry(false→true)+ FIFO ordering + tokened journal 由既有 session-lifecycle-start 用例覆盖 - build 绿;P2 相关 8 套件 233 用例全绿 注:PR 当前对最新 master(24226c87)CONFLICTING,但冲突文件(cli/daemon/ worker/types/trigger-session/dashboard-ipc/worker-pool)与本次 P2 修复文件 (bot-registry/client/lark-upload/session-manager)不重叠——是 #597 原本 vs #583/#281 的既有撞车,需单独 rebase,与这两个 fix 无关。 Co-Authored-By: Riff --- src/core/session-manager.ts | 21 ++++++++++++-- src/utils/lark-upload.ts | 12 ++++++-- test/lark-upload.test.ts | 55 ++++++++++++++++++++++++++++++++++++- test/session-resume.test.ts | 35 +++++++++++++++++++++++ 4 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/core/session-manager.ts b/src/core/session-manager.ts index 31295a2e8..c5238897c 100644 --- a/src/core/session-manager.ts +++ b/src/core/session-manager.ts @@ -1360,6 +1360,9 @@ export async function restoreActiveSessions(activeSessions: Map { constructed.push(opts); } } - return { Client: FakeClient, LoggerLevel: { error: 0 } }; + // Mirror the real SDK's separable http instance so the upload-timeout path is + // exercised (create() → own instance + copyable interceptor registry) instead + // of only hitting the fail-safe fallback. + const makeInstance = (): any => ({ + defaults: { timeout: 0 }, + create: (cfg: { timeout?: number }) => { + const inst = makeInstance(); + if (cfg?.timeout !== undefined) inst.defaults.timeout = cfg.timeout; + return inst; + }, + interceptors: { + request: { handlers: [], use(this: any, f: any, r: any) { this.handlers.push({ fulfilled: f, rejected: r }); } }, + response: { handlers: [], use(this: any, f: any, r: any) { this.handlers.push({ fulfilled: f, rejected: r }); } }, + }, + }); + return { Client: FakeClient, LoggerLevel: { error: 0 }, defaultHttpInstance: makeInstance() }; }); async function fresh() { @@ -52,4 +67,42 @@ describe('uploadImageBuffer — brand domain', () => { 'https://open.larksuite.com', ]); }); + + it('binds the upload client to a dedicated http instance with the 120s upload timeout', async () => { + const { uploadImageBuffer } = await fresh(); + await uploadImageBuffer('app', 'sec', Buffer.from('x')); + const httpInstance = constructed[0]?.httpInstance as { defaults?: { timeout?: number } } | undefined; + expect(httpInstance).toBeDefined(); + expect(httpInstance?.defaults?.timeout).toBe(120_000); + }); + + it('falls back to the plain Client (no throw) when the SDK omits defaultHttpInstance', async () => { + // Regression for the fail-safe bug codex caught: a stripped/mocked SDK without + // `defaultHttpInstance` must NOT crash the upload path. This deliberately + // re-mocks the module WITHOUT that export so the mock fix above does not erase + // the original bug's trigger condition. + vi.resetModules(); + constructed.length = 0; + vi.doMock('@larksuiteoapi/node-sdk', () => { + class FakeClient { + opts: Record; + im = { v1: { image: { create: async () => ({ image_key: 'img_fallback' }) } } }; + constructor(opts: Record) { + this.opts = opts; + constructed.push(opts); + } + } + return { Client: FakeClient, LoggerLevel: { error: 0 } }; // no defaultHttpInstance + }); + try { + const { uploadImageBuffer } = await import('../src/utils/lark-upload.js'); + const key = await uploadImageBuffer('app', 'sec', Buffer.from('x')); + expect(key).toBe('img_fallback'); + // Fell back to a plain Client: no injected upload httpInstance. + expect(constructed[0]?.httpInstance).toBeUndefined(); + } finally { + vi.doUnmock('@larksuiteoapi/node-sdk'); + vi.resetModules(); + } + }); }); diff --git a/test/session-resume.test.ts b/test/session-resume.test.ts index 3f562ddf2..4ce863d95 100644 --- a/test/session-resume.test.ts +++ b/test/session-resume.test.ts @@ -828,6 +828,41 @@ describe('resumeSession', () => { expect(vi.mocked(promoteQueuedActivationTail)).toHaveBeenCalled(); // On-disk row stays active (retained for inspection/retry, not closed away). expect(sessionStore.getSession(s.sessionId)?.status).toBe('active'); + // SELF-HEAL enabler: initialStartPending must be false so the next inbound + // can claim the cold owner (tryAcquireInitialStartClaim bails when it is + // true) → fork → retry promotion. Left true, the session would wedge on + // the admission gate (append-tail-and-return) forever until /close. + expect(ds!.initialStartPending).toBe(false); + }); + + it('leaves initialStartPending TRUE for a normal (non-quarantine) tail promotion at restore', async () => { + // Guard against the self-heal fix over-reaching: when promotion SUCCEEDS, + // the tokened activation is genuinely in flight, so the gate must stay up. + const s = sessionStore.createSession('oc_chatOk', 'om_ok', 'OK Topic', 'group'); + s.larkAppId = 'app_test'; + s.workingDir = '/tmp/proj'; + s.cliId = 'codex-app'; + s.scope = 'thread'; + s.status = 'active'; + s.hasHistory = true; + s.queuedActivationTail = [{ + id: 'tail-ok', + order: 1, + userPrompt: 'held', + cliInput: { content: 'held' }, + turnId: 'turn-ok', + }] as any; + s.queuedActivationTailNextOrder = 1; + sessionStore.updateSession(s); + + // Promotion succeeds (default mock returns true). + const map = new Map(); + await restoreActiveSessions(map); + + const ds = map.get(sessionKey('om_ok', 'app_test')); + expect(ds).toBeDefined(); + // Gate stays up: a real tokened activation is in flight. + expect(ds!.initialStartPending).toBe(true); }); }); }); From ef47b2214e389010d45c84398693d3894859fad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B3=E6=99=97?= Date: Sun, 26 Jul 2026 12:57:14 -0700 Subject: [PATCH 5/5] =?UTF-8?q?fix(codex-app):=20quarantine=20=E8=87=AA?= =?UTF-8?q?=E6=84=88=E6=94=B9=E4=B8=BA=20fork=20=E8=BE=B9=E7=95=8C?= =?UTF-8?q?=E5=89=8D=E7=BD=AE=20promote-retry(=E9=87=87=E7=BA=B3=20codex?= =?UTF-8?q?=20=E4=B8=89=E5=AE=A1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一版"清 initialStartPending gate"是错的(codex 三审指出+我核实): - toReattach 空 fork 不 promote → 事后 live worker+未提升 tail,入站看到 live worker 跳过 claim、只追加 tail,promote 永不重试; - 冷路径清 gate 后新消息作普通 refork prompt 先发 → 越过老 tail,FIFO 反转。 采纳 codex 硬约束:任何 blank/current-turn fork 前,tail-only quarantine 必须 先 retry 老 head 的 promotion;失败就保持占 owner/gate、跳过 fork,绝不让当前 turn 越过、也绝不留 live-worker + 未提升 tail。 实现: - 新增运行时标记 DaemonSession.quarantinedActivationTailPromotion(restore 时 promote 失败置位,不再清 gate,initialStartPending 保持 true)。 - 抽 `retryQuarantinedActivationTailPromotion(ds)` 共享 helper:非 quarantine→ true(no-op);promote 成功→清标记返 true(可 fork 已提升的 tokened head); promote 仍失败→返 false(调用方跳过 fork,保持 worker:null 占位)。幂等 (promoteQueuedActivationTail 对已 pending 短路 true)。 - 两个 fork 边界前置调用: 1) restore toReattach 空 fork 回调:helper 返 false 则 return 跳过 forkWorker; 2) daemon 入站冷路径:当前 turn 已按 reservation 追加到老 tail 之后,再 retry; 成功→forkReservedInitialSession 冷 fork 已提升的老 head(非当前 turn); 失败→当前 turn 留 tail 等下次,保持 quarantine。 测试(session-resume.test.ts): - restore quarantine 用例改断新契约:initialStartPending 仍 true(gate held)+ quarantinedActivationTailPromotion=true(待 fork 边界 retry),不再断错误的 gate=false;反向用例(promote 成功 gate 仍 true)保留。 - 新增 helper 三用例:成功→true+清标记+promote 以 send:false 调用;失败→false+ 保留标记(调用方必跳 fork);非 quarantine→true 且不碰 promote。⭐失败路径变异 测试验证对实现敏感(fail→true 变异使用例 FAIL)。 - retry(false→true)+FIFO ordering+tokened journal 由既有 lifecycle 用例覆盖。 - build 绿;P2 相关 8 套件 236 用例全绿。 CONFLICTING(master→24226c87,与 #583/#281 既有撞车,不涉本次文件)待 rebase。 Co-Authored-By: Riff --- src/core/session-manager.ts | 48 ++++++++++++++++++++++++++++++-- src/core/types.ts | 7 +++++ src/daemon.ts | 24 ++++++++++++++++ test/session-resume.test.ts | 55 +++++++++++++++++++++++++++++++++---- 4 files changed, 126 insertions(+), 8 deletions(-) diff --git a/src/core/session-manager.ts b/src/core/session-manager.ts index c5238897c..2e761ce5a 100644 --- a/src/core/session-manager.ts +++ b/src/core/session-manager.ts @@ -1300,6 +1300,35 @@ export function shouldAutoForkOnRestore(backendType: BackendType): boolean { return backendType !== 'pty'; } +/** + * Fork-boundary retry for a tail-only quarantine (a restore-time durable + * activation-tail promotion that failed transiently, leaving the tail + * un-promoted with the gate held). MUST be called before any blank / opening + * fork of such a session. + * + * Returns true when it is now safe to fork: either the session was not + * quarantined, or the retry promoted the old head into the tokened journal (so + * a subsequent fork carries the promoted head, never a later turn). Returns + * false when the retry still failed — the caller MUST skip the fork and keep the + * worker:null quarantined owner, so a blank fork never leaves a live worker + * beside an unpromoted tail (which would permanently wedge the admission gate). + * + * Idempotent: `promoteQueuedActivationTail` short-circuits true once the head is + * already pending, and the quarantine flag is cleared on success. + */ +export function retryQuarantinedActivationTailPromotion(ds: DaemonSession): boolean { + if (!ds.quarantinedActivationTailPromotion) return true; + if (!promoteQueuedActivationTail(ds, { send: false })) { + logger.warn( + `[${ds.session.sessionId.substring(0, 8)}] Quarantined activation-tail promotion still failing at ` + + `fork boundary; keeping worker:null quarantined owner (no fork) to avoid a live worker beside an unpromoted tail`, + ); + return false; + } + ds.quarantinedActivationTailPromotion = undefined; + return true; +} + const RECOVERY_FORK_BATCH_SIZE = config.daemon.recoveryForkBatchSize ?? 5; const RECOVERY_FORK_DELAY_MS = config.daemon.recoveryForkDelayMs ?? 250; @@ -1686,11 +1715,20 @@ export async function restoreActiveSessions(activeSessions: Map { + // Quarantined tail-only owner: retry its failed promotion BEFORE the blank + // fork. If the retry still fails, skip forking entirely and keep the + // worker:null owner — a blank fork here would leave a LIVE worker beside an + // unpromoted tail, and the daemon inbound path (seeing a live worker) would + // then only append later turns to the tail, never retrying → permanent wedge. + if (!retryQuarantinedActivationTailPromotion(ds)) return; const recoverExactNonCodex = ds.session.queuedActivationPending && ds.session.cliId !== 'codex-app' && ds.session.queuedActivationInput; diff --git a/src/core/types.ts b/src/core/types.ts index 794ad5a95..2d78bce0d 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -88,6 +88,13 @@ export interface DaemonSession { * reached the first fork yet. Same-anchor turns must buffer into the opening * input instead of reforking worker:null and overtaking it. In-memory only. */ initialStartPending?: boolean; + /** Restore quarantine: a durable activation-tail promotion failed transiently + * during restoreActiveSessions, so the row was registered with its tail + * un-promoted. The next fork boundary (toReattach blank fork / daemon inbound + * refork) must retry the promotion BEFORE forking and skip a blank fork if it + * still fails — never leave a live worker beside an unpromoted tail. Cleared + * once promotion succeeds. In-memory only. */ + quarantinedActivationTailPromotion?: boolean; /** Generation token for the handler that atomically reserved a worker:null * refork. Later same-anchor handlers may prepare concurrently, but only this * owner may cross the fork boundary; followers buffer behind its gate. */ diff --git a/src/daemon.ts b/src/daemon.ts index d98f0c0b7..5d976d985 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -171,6 +171,7 @@ import { ensureTerminalWorkerPort, ensureSessionWhiteboard, closeCliMismatchedSessionsForBot, + retryQuarantinedActivationTailPromotion, } from './core/session-manager.js'; import { triggerSessionTurn } from './core/trigger-session.js'; import { @@ -16929,6 +16930,29 @@ async function handleThreadReplyAdmitted(data: any, ctx: RoutingContext): Promis `[${tag(ds)}] buffered same-anchor turn ${parsed.messageId.substring(0, 12)} ` + 'behind queued activation submission ACK', ); + // Quarantined tail-only owner (restore promotion failed transiently): the + // current turn is now safely appended BEHIND the retained tail via its + // reservation, so retry the old head's promotion. On success, promote the + // exact old head into the tokened journal and cold-fork THAT (never the + // current turn — the invariant is the old head must not be overtaken). On + // failure keep the current turn in the tail and stay quarantined for a later + // retry. Only worker:null owners reach here (a live worker would have skipped + // this gate); the toReattach path handles the eager-fork case. + if (ds.quarantinedActivationTailPromotion && (!ds.worker || ds.worker.killed)) { + // The current turn is now safely appended BEHIND the retained tail (via its + // reservation above), so retry the old head's promotion. On success, the + // old head is in the tokened journal and we cold-fork THAT (never the + // current turn — the old head must not be overtaken). On failure the current + // turn stays parked in the tail and the session stays quarantined. + if (retryQuarantinedActivationTailPromotion(ds)) { + ds.initialStartPending = true; + const availableBots = await getAvailableBots(larkAppId, ctxChatId ?? data?.message?.chat_id); + forkReservedInitialSession(ds, availableBots); + logger.info(`[${tag(ds)}] Quarantined activation-tail promotion recovered on inbound; cold-forked promoted head`); + } else { + logger.warn(`[${tag(ds)}] Quarantined activation-tail promotion still failing on inbound; current turn parked behind old tail, staying quarantined`); + } + } return; } if (ds?.pendingRepo || initialStartPending) { diff --git a/test/session-resume.test.ts b/test/session-resume.test.ts index 4ce863d95..d8f51fee1 100644 --- a/test/session-resume.test.ts +++ b/test/session-resume.test.ts @@ -144,7 +144,7 @@ vi.mock('../src/core/session-activity.js', () => ({ markSessionActivity: vi.fn(), })); -import { restoreActiveSessions, resumeSession } from '../src/core/session-manager.js'; +import { restoreActiveSessions, resumeSession, retryQuarantinedActivationTailPromotion } from '../src/core/session-manager.js'; import { restoreUsageLimitRuntimeState, closeSession, forkAdoptWorker, promoteQueuedActivationTail } from '../src/core/worker-pool.js'; import { TmuxBackend } from '../src/adapters/backend/tmux-backend.js'; import * as sessionStore from '../src/services/session-store.js'; @@ -828,11 +828,13 @@ describe('resumeSession', () => { expect(vi.mocked(promoteQueuedActivationTail)).toHaveBeenCalled(); // On-disk row stays active (retained for inspection/retry, not closed away). expect(sessionStore.getSession(s.sessionId)?.status).toBe('active'); - // SELF-HEAL enabler: initialStartPending must be false so the next inbound - // can claim the cold owner (tryAcquireInitialStartClaim bails when it is - // true) → fork → retry promotion. Left true, the session would wedge on - // the admission gate (append-tail-and-return) forever until /close. - expect(ds!.initialStartPending).toBe(false); + // Gate MUST stay up (initialStartPending true) — the old tail head must not + // be overtaken by a later turn. The retry happens at the next fork boundary + // (toReattach blank fork / daemon inbound refork), not by clearing the gate. + expect(ds!.initialStartPending).toBe(true); + // Marked for fork-boundary retry so a blank fork retries promotion first + // and skips forking if it still fails (never live-worker + unpromoted tail). + expect(ds!.quarantinedActivationTailPromotion).toBe(true); }); it('leaves initialStartPending TRUE for a normal (non-quarantine) tail promotion at restore', async () => { @@ -866,3 +868,44 @@ describe('resumeSession', () => { }); }); }); + +describe('retryQuarantinedActivationTailPromotion (fork-boundary retry)', () => { + beforeEach(() => { + vi.mocked(promoteQueuedActivationTail).mockReset(); + vi.mocked(promoteQueuedActivationTail).mockReturnValue(true); + }); + + function quarantinedDs(): DaemonSession { + return { + session: { + sessionId: 'sid-q', chatId: 'oc', rootMessageId: 'om', status: 'active', + queuedActivationTail: [{ id: 't1', order: 1, userPrompt: 'p', cliInput: { content: 'p' }, turnId: 'turn-1' }], + }, + worker: null, larkAppId: 'app_test', quarantinedActivationTailPromotion: true, + } as unknown as DaemonSession; + } + + it('returns true and clears the flag when the retry promotion SUCCEEDS (safe to fork the promoted head)', () => { + const ds = quarantinedDs(); + vi.mocked(promoteQueuedActivationTail).mockReturnValue(true); + expect(retryQuarantinedActivationTailPromotion(ds)).toBe(true); + expect(ds.quarantinedActivationTailPromotion).toBeUndefined(); + expect(vi.mocked(promoteQueuedActivationTail)).toHaveBeenCalledWith(ds, { send: false }); + }); + + it('returns FALSE and keeps the flag when the retry promotion still FAILS (caller must skip the fork)', () => { + const ds = quarantinedDs(); + vi.mocked(promoteQueuedActivationTail).mockReturnValue(false); + expect(retryQuarantinedActivationTailPromotion(ds)).toBe(false); + // Still quarantined so a later boundary retries; NEVER fork now (avoids a + // live worker beside an unpromoted tail). + expect(ds.quarantinedActivationTailPromotion).toBe(true); + }); + + it('is a no-op (returns true) for a non-quarantined session — never touches promotion', () => { + const ds = quarantinedDs(); + ds.quarantinedActivationTailPromotion = undefined; + expect(retryQuarantinedActivationTailPromotion(ds)).toBe(true); + expect(vi.mocked(promoteQueuedActivationTail)).not.toHaveBeenCalled(); + }); +});