diff --git a/package.json b/package.json index 51907af70..6496caa4d 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "access": "public" }, "scripts": { - "build": "pnpm audit:domains && node scripts/clean-dist.mjs && tsc && cp src/setup/lark-scopes.json dist/setup/ && pnpm dashboard:bundle && chmod +x dist/cli.js && node scripts/audit-dist.mjs", + "build": "pnpm audit:domains && node scripts/clean-dist.mjs && tsc && node scripts/generate-runtime-build-id.mjs && cp src/setup/lark-scopes.json dist/setup/ && pnpm dashboard:bundle && chmod +x dist/cli.js && node scripts/audit-dist.mjs", "audit:domains": "node scripts/audit-public-domains.mjs", "dashboard:bundle": "node scripts/build-dashboard.mjs", "dashboard:watch": "node scripts/build-dashboard.mjs --watch", diff --git a/scripts/generate-runtime-build-id.mjs b/scripts/generate-runtime-build-id.mjs new file mode 100644 index 000000000..5c7c7f1c7 --- /dev/null +++ b/scripts/generate-runtime-build-id.mjs @@ -0,0 +1,15 @@ +#!/usr/bin/env node +import { writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + collectCompiledRuntimeEntries, + computeRuntimeBuildId, +} from '../dist/utils/runtime-build-id.js'; + +const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const sourceRoot = join(projectRoot, 'src'); +const distRoot = join(projectRoot, 'dist'); +const buildId = computeRuntimeBuildId(collectCompiledRuntimeEntries(sourceRoot, distRoot)); +writeFileSync(join(distRoot, '.runtime-build-id'), `${buildId}\n`, 'utf8'); +process.stdout.write(`runtime build id: ${buildId.slice(0, 12)}\n`); diff --git a/src/adapters/cli/codex-app.ts b/src/adapters/cli/codex-app.ts index 4614042ba..a60e71668 100644 --- a/src/adapters/cli/codex-app.ts +++ b/src/adapters/cli/codex-app.ts @@ -64,20 +64,27 @@ export function createCodexAppAdapter(pathOverride?: string): CliAdapter { return null; }, - async writeInput(pty: PtyHandle, content: string) { + async writeInput(pty: PtyHandle, content: string, context) { // Chunked + throttled stdin injection — a single send-keys of the whole // (potentially ~20KB) control line overruns the pane pty input buffer and // gets dropped. See runner-input.ts. - return writeRunnerInput(pty, '::botmux-codex-app:', content); + return writeRunnerInput(pty, '::botmux-codex-app:', content, undefined, context?.turnId); }, - async writeStructuredInput(pty, content, codexAppInput) { + async writeStructuredInput(pty, content, codexAppInput, context) { // The legacy prompt remains in the control payload as a compatibility // fallback. The runner uses the sidecar only on supported app-server // versions and never reverse-parses the XML-ish legacy envelope. - return writeRunnerInput(pty, '::botmux-codex-app:', content, codexAppInput); + return writeRunnerInput( + pty, + '::botmux-codex-app:', + content, + codexAppInput, + context?.turnId, + ); }, + supportsTypeAhead: true, completionPattern: undefined, readyPattern: /›/, systemHints: [], diff --git a/src/adapters/cli/runner-input.ts b/src/adapters/cli/runner-input.ts index 37b03520e..50d66c673 100644 --- a/src/adapters/cli/runner-input.ts +++ b/src/adapters/cli/runner-input.ts @@ -41,10 +41,17 @@ export const RUNNER_INPUT_CHUNK_BYTES = 1024; * pane pty between writes. */ export const RUNNER_INPUT_THROTTLE_MS = 20; -export function encodeRunnerInput(content: string, codexAppInput?: CodexAppTurnInput): string { - const payload = codexAppInput - ? { type: 'message', content, codexAppInput } - : { type: 'message', content }; +export function encodeRunnerInput( + content: string, + codexAppInput?: CodexAppTurnInput, + replyTurnId?: string, +): string { + const payload = { + type: 'message' as const, + content, + ...(codexAppInput ? { codexAppInput } : {}), + ...(replyTurnId ? { replyTurnId } : {}), + }; return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64'); } @@ -84,8 +91,9 @@ export async function writeRunnerInput( markerPrefix: string, content: string, codexAppInput?: CodexAppTurnInput, + replyTurnId?: string, ): Promise<{ submitted: boolean }> { - const line = `${markerPrefix}${encodeRunnerInput(content, codexAppInput)}`; + const line = `${markerPrefix}${encodeRunnerInput(content, codexAppInput, replyTurnId)}`; // Non-tmux fallback (raw PTY): a single write is fine — there's no send-keys // process to time out, and the PTY write isn't bounded the same way. diff --git a/src/adapters/cli/types.ts b/src/adapters/cli/types.ts index dd0a4e399..b8701e3de 100644 --- a/src/adapters/cli/types.ts +++ b/src/adapters/cli/types.ts @@ -30,6 +30,13 @@ export type SubmitRecheckResult = boolean | { cliSessionId?: string; }; +/** Optional per-input correlation metadata. Adapters that do not need it may + * ignore it; runner-based adapters use the immutable botmux/Lark turn id to + * keep protocol ids separate from reply-routing ids. */ +export interface WriteInputContext { + turnId?: string; +} + /** A session discovered on disk that botmux can resume (import) into a topic — * surfaced by `/adopt`'s second filter. Unlike an AdoptableSession (a live * tmux/zellij pane botmux *observes*), this is a stored transcript botmux @@ -190,6 +197,7 @@ export interface CliAdapter { writeInput( pty: PtyHandle, content: string, + context?: WriteInputContext, ): Promise; @@ -29,31 +39,8 @@ interface PendingRequest { method: string; } -interface ActiveTurn { - /** Codex app-server's native turn id. This is used only to correlate - * notifications from the server; botmux routing uses the stable client - * message id carried alongside the queued input. */ - nativeTurnId?: string; - serverStarted: boolean; - startedAtMs: number; - finalText: string; - allAgentText: string; - itemText: Map; - done: Promise; - resolveDone: () => void; -} - -interface QueuedInput { - content: string; - codexAppInput?: CodexAppTurnInput; -} - const output = new RunnerControlWriter(); -function asError(value: unknown): Error { - return value instanceof Error ? value : new Error(String(value)); -} - function parseArgs(argv: string[]): Args { const out: Args = { sessionId: '', @@ -121,8 +108,9 @@ class AppServerClient { private pending = new Map(); private notificationHandlers: Array<(msg: JsonObject) => void> = []; private requestHandlers: Array<(msg: JsonObject) => boolean> = []; + private fatalHandlers: Array<(error: CodexAppTransportError) => void> = []; private lastStderr = ''; - private fatalError?: Error; + private fatalError?: CodexAppTransportError; constructor(private readonly codexBin: string, private readonly cwd: string) { this.child = spawn(codexBin, ['app-server', '--listen', 'stdio://'], { @@ -132,7 +120,7 @@ class AppServerClient { }); this.child.stdout.on('data', chunk => this.onStdout(chunk.toString('utf8'))); - this.child.stdin.on('error', err => this.failAll(new Error(`Codex app-server stdin error: ${err.message}`))); + this.child.stdin.on('error', err => this.failAll(new CodexAppTransportError(`Codex app-server stdin error: ${err.message}`))); this.child.stderr.on('data', chunk => { const text = chunk.toString('utf8'); this.lastStderr = (this.lastStderr + text).slice(-8000); @@ -142,10 +130,10 @@ class AppServerClient { const hint = (err as NodeJS.ErrnoException).code === 'ENOENT' ? '\nHint: install the Codex CLI, or set cliPathOverride to the Codex App bundled binary, for example /Applications/Codex.app/Contents/Resources/codex.' : ''; - this.failAll(new Error(`Failed to start Codex app-server with "${codexBin}": ${err.message}${hint}`)); + this.failAll(new CodexAppTransportError(`Failed to start Codex app-server with "${codexBin}": ${err.message}${hint}`)); }); this.child.on('exit', (code, signal) => { - const err = this.fatalError ?? new Error(`Codex app-server exited (code=${code}, signal=${signal})${this.lastStderr ? `\n${this.lastStderr}` : ''}`); + const err = this.fatalError ?? new CodexAppTransportError(`Codex app-server exited (code=${code}, signal=${signal})${this.lastStderr ? `\n${this.lastStderr}` : ''}`); this.failAll(err); }); } @@ -158,6 +146,11 @@ class AppServerClient { this.requestHandlers.push(handler); } + onFatal(handler: (error: CodexAppTransportError) => void): void { + this.fatalHandlers.push(handler); + if (this.fatalError) handler(this.fatalError); + } + async initialize(): Promise { await this.request('initialize', { clientInfo: { name: 'botmux-codex-app', version: '0.0.0' }, @@ -173,8 +166,8 @@ class AppServerClient { try { this.write({ jsonrpc: '2.0', id, method, params }); } catch (err) { - this.pending.delete(id); - reject(asError(err)); + const message = err instanceof Error ? err.message : String(err); + this.failAll(new CodexAppTransportError(`Codex app-server write failed: ${message}`)); } }); } @@ -199,10 +192,18 @@ class AppServerClient { } private failAll(err: Error): void { - this.fatalError = this.fatalError ?? err; + const firstFailure = this.fatalError === undefined; + this.fatalError = this.fatalError ?? ( + err instanceof CodexAppTransportError + ? err + : new CodexAppTransportError(err.message) + ); const fatal = this.fatalError; for (const pending of this.pending.values()) pending.reject(fatal); this.pending.clear(); + if (firstFailure) { + for (const handler of this.fatalHandlers) handler(fatal); + } } private onStdout(data: string): void { @@ -228,7 +229,7 @@ class AppServerClient { const pending = this.pending.get(msg.id); if (!pending) return; this.pending.delete(msg.id); - if (msg.error) pending.reject(new Error(`${pending.method}: ${JSON.stringify(msg.error)}`)); + if (msg.error) pending.reject(new CodexAppRpcResponseError(pending.method, msg.error)); else pending.resolve(msg.result); return; } @@ -258,14 +259,11 @@ try { const client = new AppServerClient(args.codexBin, args.cwd); let threadId = args.threadId; let threadReady = false; -let activeTurn: ActiveTurn | null = null; -const queue: QueuedInput[] = []; let inputBuffer = ''; -let processing = false; -let cleanInputUnsupported = false; let codexVersionChecked = false; let codexVersion: CodexVersion | undefined; let cleanVersionWarningShown = false; +let controller: CodexAppTurnController; function detectedCodexVersion(): CodexVersion | undefined { if (codexVersionChecked) return codexVersion; @@ -284,20 +282,6 @@ function detectedCodexVersion(): CodexVersion | undefined { return codexVersion; } -function makeTurn(): ActiveTurn { - let resolveDone!: () => void; - const done = new Promise(resolve => { resolveDone = resolve; }); - return { - startedAtMs: Date.now(), - serverStarted: false, - finalText: '', - allAgentText: '', - itemText: new Map(), - done, - resolveDone, - }; -} - function handleServerRequest(msg: JsonObject): boolean { const method = msg.method; if (method === 'item/commandExecution/requestApproval') { @@ -332,59 +316,7 @@ function handleServerRequest(msg: JsonObject): boolean { } function handleNotification(msg: JsonObject): void { - const params = msg.params ?? {}; - if (!activeTurn || params.threadId !== threadId) return; - if (activeTurn.nativeTurnId && params.turnId && params.turnId !== activeTurn.nativeTurnId) return; - - if (msg.method === 'turn/started') { - activeTurn.serverStarted = true; - activeTurn.nativeTurnId = params.turn?.id ?? params.turnId ?? activeTurn.nativeTurnId; - return; - } - - if (msg.method === 'item/started') { - const item = params.item; - if (item?.type === 'commandExecution') { - writeLine(`\n$ ${item.command}`); - } else if (item?.type === 'fileChange') { - writeLine('\n[files changed]'); - } - return; - } - - if (msg.method === 'item/agentMessage/delta') { - const delta = String(params.delta ?? ''); - const itemId = String(params.itemId ?? ''); - activeTurn.itemText.set(itemId, (activeTurn.itemText.get(itemId) ?? '') + delta); - activeTurn.allAgentText += delta; - output.display(delta); - return; - } - - if (msg.method === 'item/commandExecution/outputDelta' || msg.method === 'item/fileChange/outputDelta') { - output.display(String(params.delta ?? '')); - return; - } - - if (msg.method === 'item/completed') { - const item = params.item; - if (item?.type === 'agentMessage') { - if (item.phase === 'final_answer') activeTurn.finalText = String(item.text ?? ''); - else if (!activeTurn.itemText.has(item.id) && item.text) { - activeTurn.allAgentText += String(item.text); - } - } - return; - } - - if (msg.method === 'turn/completed') { - const turn = params.turn; - if (turn?.id && activeTurn.nativeTurnId && turn.id !== activeTurn.nativeTurnId) return; - if (turn?.error?.message && !activeTurn.finalText) { - activeTurn.finalText = `Codex App turn failed: ${turn.error.message}`; - } - activeTurn.resolveDone(); - } + controller?.handleNotification(msg); } async function ensureThread(): Promise { @@ -442,130 +374,81 @@ async function ensureThread(): Promise { return startedThreadId; } -async function runTurn(message: QueuedInput): Promise { - const tid = await ensureThread(); - const turn = makeTurn(); - activeTurn = turn; - const version = message.codexAppInput ? detectedCodexVersion() : undefined; - let built = buildCodexAppTurnStartParams({ - threadId: tid, +function prepareControllerInput( + message: CodexAppRunnerInput, + structuredDisabled: boolean, +): CodexAppPreparedInput { + const version = message.codexAppInput || message.replyTurnId + ? detectedCodexVersion() + : undefined; + const built = buildCodexAppTurnStartParams({ + threadId: threadId ?? '', cwd: args.cwd, legacyContent: message.content, codexAppInput: message.codexAppInput, codexVersion: version, - structuredDisabled: cleanInputUnsupported, + structuredDisabled, }); - if (message.codexAppInput && !built.structured && !cleanInputUnsupported && !cleanVersionWarningShown) { + if ( + message.codexAppInput + && !built.structured + && !structuredDisabled + && !cleanVersionWarningShown + ) { cleanVersionWarningShown = true; const found = version ? `${version.major}.${version.minor}.${version.patch}` : 'unknown'; writeLine(`[codex-app] clean input requires codex >= 0.135.0 (found ${found}); using legacy prompt`); } - for (const path of built.skippedImages) { - writeLine(`[codex-app] skipped unreadable local image: ${path}`); - } - writeLine(); - writeLine('[user]'); - writeLine(built.structured && message.codexAppInput ? message.codexAppInput.text : message.content); - writeLine(); - - let result; - try { - result = await client.request('turn/start', built.params); - } catch (err) { - if (!built.structured || turn.serverStarted || !isCleanInputCapabilityError(err)) throw err; - // The app-server explicitly rejected the experimental field before a turn - // started. Disable structured input for this runner lifetime and retry the - // preserved legacy prompt exactly once. - cleanInputUnsupported = true; - writeLine('[codex-app] clean input unsupported by app-server; retrying this turn with the legacy prompt'); - built = buildCodexAppTurnStartParams({ - threadId: tid, - cwd: args.cwd, - legacyContent: message.content, - codexAppInput: message.codexAppInput, - codexVersion: version, - structuredDisabled: true, - }); - result = await client.request('turn/start', built.params); - } - turn.nativeTurnId = result.turn?.id ?? turn.nativeTurnId; - await turn.done; - - const finalText = (turn.finalText || turn.allAgentText).trim(); - const completedAtMs = Date.now(); - if (finalText) { - // clientUserMessageId is the daemon-frozen botmux/Lark turn identity. The - // app-server generates a different id for the same logical turn; exposing - // that native id as `turnId` breaks daemon wait maps, VC suppression and - // reply routing. When no structured sidecar exists, omit turnId so the - // worker deliberately falls back to its current botmux turn attribution. - const stableTurnId = message.codexAppInput?.clientUserMessageId; - emitMarker('final', { - ...(stableTurnId ? { turnId: stableTurnId } : {}), - ...(turn.nativeTurnId ? { nativeTurnId: turn.nativeTurnId } : {}), - content: finalText, - startedAtMs: turn.startedAtMs, - completedAtMs, - }); - } - writeLine(); - activeTurn = null; + const clientUserMessageId = !structuredDisabled + && message.replyTurnId + && version + && supportsClientUserMessageId(version) + ? message.replyTurnId + : built.params.clientUserMessageId; + return { + input: built.params.input, + ...(built.params.additionalContext + ? { additionalContext: built.params.additionalContext } + : {}), + ...(clientUserMessageId ? { clientUserMessageId } : {}), + visibleText: message.codexAppInput?.text ?? message.content, + structured: built.structured, + skippedImages: built.skippedImages, + }; } -async function drainQueue(): Promise { - if (processing) return; - processing = true; - try { - while (queue.length > 0) { - const next = queue.shift()!; - try { - await runTurn(next); - } catch (err: any) { - const message = `Codex App runner error: ${err?.message ?? err}`; - const completedAtMs = Date.now(); - const stableTurnId = next.codexAppInput?.clientUserMessageId; - const nativeTurnId = activeTurn?.nativeTurnId; - writeLine(message); - emitMarker('final', { - ...(stableTurnId ? { turnId: stableTurnId } : {}), - ...(nativeTurnId ? { nativeTurnId } : {}), - content: message, - startedAtMs: activeTurn?.startedAtMs ?? completedAtMs, - completedAtMs, - }); - activeTurn = null; - } - prompt(); - } - } finally { - processing = false; - } -} +controller = new CodexAppTurnController({ + cwd: args.cwd, + ensureThread, + request: (method, params) => client.request(method, params), + prepareInput: prepareControllerInput, + isStartCapabilityError: isCleanInputCapabilityError, + onTurnInput(_input, prepared) { + writeLine(); + writeLine('[user]'); + writeLine(prepared.visibleText); + writeLine(); + }, + onOutput: text => output.display(text), + onDiagnostic: writeLine, + onLifecycle: event => emitMarker('lifecycle', event), + onFinal: marker => { + emitMarker('final', marker); + writeLine(); + }, + onPrompt: prompt, +}); function enqueueLine(line: string): void { const trimmed = line.trim(); if (!trimmed) return; - if (trimmed.startsWith('::botmux-codex-app:')) { - const encoded = trimmed.slice('::botmux-codex-app:'.length); - try { - const decoded = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); - if (decoded?.type === 'message' && typeof decoded.content === 'string') { - const codexAppInput = isCodexAppTurnInput(decoded.codexAppInput) - ? decoded.codexAppInput - : undefined; - if (decoded.codexAppInput !== undefined && !codexAppInput) { - writeLine('[codex-app] ignored invalid structured input sidecar'); - } - queue.push({ content: decoded.content, codexAppInput }); - void drainQueue(); - } - } catch (err: any) { - writeLine(`[codex-app] bad botmux input: ${err?.message ?? err}`); - } + if (trimmed.startsWith(CODEX_APP_INPUT_PREFIX)) { + const decoded = decodeCodexAppRunnerInput(trimmed); + if (decoded) controller.enqueue(decoded); + else writeLine('[codex-app] bad botmux input'); return; } - queue.push({ content: line }); - void drainQueue(); + controller.enqueue({ type: 'message', content: line }); } function handleInput(data: Buffer): void { @@ -588,6 +471,11 @@ function handleInput(data: Buffer): void { async function main(): Promise { client.onRequest(handleServerRequest); client.onNotification(handleNotification); + client.onFatal(error => { + controller.handleFatal(error); + process.exitCode = 1; + process.stdout.write('', () => process.exit(1)); + }); await client.initialize(); await ensureThread(); writeLine('Codex App connected.'); diff --git a/src/core/command-handler.ts b/src/core/command-handler.ts index d188c1489..80f669e77 100644 --- a/src/core/command-handler.ts +++ b/src/core/command-handler.ts @@ -24,7 +24,7 @@ import { chatAppLink, normalizeBrand } from '../im/lark/lark-hosts.js'; import { claimPairing } from '../services/pairing-store.js'; import { logger } from '../utils/logger.js'; import { scheduleTimeZone } from '../utils/timezone.js'; -import { killWorker, suspendWorker, forkWorker, forkAdoptWorker, adoptSandboxBlocked, getCurrentCliVersion, postFreshStreamingCard, postPrivateSnapshotCard, resolvePrivateCardAudience, deliverEphemeralOrReply, deliverWritableTerminalCardTo } from './worker-pool.js'; +import { killWorker, suspendWorker, forkWorker, forkAdoptWorker, adoptSandboxBlocked, getCurrentCliVersion, postFreshStreamingCard, postPrivateSnapshotCard, resolvePrivateCardAudience, deliverEphemeralOrReply, deliverWritableTerminalCardTo, requestSessionRestart } from './worker-pool.js'; import { expandHome, getSessionWorkingDir, @@ -1343,15 +1343,17 @@ export async function handleCommand( case '/restart': { if (ds) { - if (ds.worker && !ds.worker.killed) { - ds.worker.send({ type: 'restart' } as DaemonToWorker); - const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); - await sessionReply(rootId, t('cmd.restart.in_progress', { cliName }, loc)); - } else { - killWorker(ds); - const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); - await sessionReply(rootId, t('cmd.restart.terminated', { cliName }, loc)); + if (ds.adoptedFrom) { + await sessionReply(rootId, t('card.action.adopt_no_restart', undefined, loc)); + break; } + const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); + requestSessionRestart(ds, { + source: 'slash', + notify: async status => { + await sessionReply(rootId, t(`cmd.restart.${status}`, { cliName }, loc)); + }, + }); logger.info(`[${logTag}] Restart by /restart command`); } else { await sessionReply(rootId, t('cmd.no_active_session', undefined, loc)); diff --git a/src/core/restart-coordinator.ts b/src/core/restart-coordinator.ts new file mode 100644 index 000000000..3206d43e0 --- /dev/null +++ b/src/core/restart-coordinator.ts @@ -0,0 +1,108 @@ +import { randomBytes } from 'node:crypto'; + +export type RestartSource = 'slash' | 'card'; +export type RestartTerminalStatus = 'succeeded' | 'failed' | 'timed_out'; +export type RestartObserverStatus = 'in_progress' | RestartTerminalStatus; + +export interface RestartObserver { + source: RestartSource; + notify(status: RestartObserverStatus): void | Promise; +} + +interface RestartAttempt { + id: string; + observers: RestartObserver[]; + timer: NodeJS.Timeout; +} + +export class RestartCoordinator { + private readonly attempts = new Map(); + private readonly notifications = new WeakMap>(); + + constructor(private readonly options: { + timeoutMs?: number; + createAttemptId?: () => string; + } = {}) {} + + request( + sessionId: string, + observer: RestartObserver, + startPhysicalRestart: (attemptId: string) => void | Promise, + ): { attemptId: string; joined: boolean } { + const existing = this.attempts.get(sessionId); + if (existing) { + existing.observers.push(observer); + this.enqueueNotify(observer, 'in_progress'); + return { attemptId: existing.id, joined: true }; + } + + const attemptId = this.options.createAttemptId?.() ?? randomBytes(12).toString('hex'); + const timer = setTimeout( + () => this.resolve(sessionId, attemptId, 'timed_out'), + this.options.timeoutMs ?? 40_000, + ); + timer.unref?.(); + this.attempts.set(sessionId, { id: attemptId, observers: [observer], timer }); + + // Notifications are deliberately detached: a slow IM request must not + // postpone replacing the process. + this.enqueueNotify(observer, 'in_progress'); + try { + const started = startPhysicalRestart(attemptId); + void Promise.resolve(started).catch(() => { + this.resolve(sessionId, attemptId, 'failed'); + }); + } catch { + this.resolve(sessionId, attemptId, 'failed'); + } + return { attemptId, joined: false }; + } + + resolve(sessionId: string, attemptId: string, status: RestartTerminalStatus): boolean { + const attempt = this.attempts.get(sessionId); + if (!attempt || attempt.id !== attemptId) return false; + this.attempts.delete(sessionId); + clearTimeout(attempt.timer); + for (const observer of attempt.observers) this.enqueueNotify(observer, status); + return true; + } + + failSession(sessionId: string): boolean { + const attempt = this.attempts.get(sessionId); + return attempt ? this.resolve(sessionId, attempt.id, 'failed') : false; + } + + cancelSession(sessionId: string): boolean { + const attempt = this.attempts.get(sessionId); + if (!attempt) return false; + this.attempts.delete(sessionId); + clearTimeout(attempt.timer); + return true; + } + + activeAttemptId(sessionId: string): string | undefined { + return this.attempts.get(sessionId)?.id; + } + + reset(): void { + for (const attempt of this.attempts.values()) clearTimeout(attempt.timer); + this.attempts.clear(); + } + + private async notify(observer: RestartObserver, status: RestartObserverStatus): Promise { + try { + await observer.notify(status); + } catch { + // Best effort; one observer cannot block the restart or other observers. + } + } + + private enqueueNotify(observer: RestartObserver, status: RestartObserverStatus): void { + const previous = this.notifications.get(observer) ?? Promise.resolve(); + const next = previous.then(() => this.notify(observer, status)); + this.notifications.set(observer, next); + void next.finally(() => { + if (this.notifications.get(observer) === next) this.notifications.delete(observer); + }); + } +} diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index b23f5b005..6d5e68f3f 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -37,13 +37,16 @@ import { listDocSubscriptionsForSession, removeDocSubscription } from '../servic import { TmuxBackend } from '../adapters/backend/tmux-backend.js'; import { HerdrBackend } from '../adapters/backend/herdr-backend.js'; import { sandboxEnabled } from '../adapters/backend/sandbox.js'; -import { isSuspendableBackendType, getSessionPersistentBackendType, persistentBackendTargetForSession, killPersistentBackendTarget, managedTargetsForCliChange, resolvePairedSpawnBackendType, resolvePersistentBackendTarget } from './persistent-backend.js'; +import { isSuspendableBackendType, getSessionPersistentBackendType, persistentBackendTargetForSession, killPersistentBackendTarget, probePersistentBackendTarget, managedTargetsForCliChange, resolvePairedSpawnBackendType, resolvePersistentBackendTarget } from './persistent-backend.js'; import { getBot, getAllBots, loadBotConfigs, resolveBrandLabel } from '../bot-registry.js'; +import { RestartCoordinator, type RestartObserver } from './restart-coordinator.js'; +import { runtimeBuildIdentity } from '../utils/runtime-build-id.js'; /** A random id minted once per daemon process (this lifetime). Stamped onto * isolated persistent panes so a suspend→resume reattach (same id) is * distinguishable from a pane surviving a daemon restart (different id). */ const DAEMON_BOOT_ID = randomUUID(); +const restartCoordinator = new RestartCoordinator(); export function getDaemonBootId(): string { return DAEMON_BOOT_ID; @@ -136,6 +139,12 @@ function workerForkEnv(base: NodeJS.ProcessEnv): NodeJS.ProcessEnv { return env; } +/** Fresh workers always start with hidden output, so restore the daemon-owned mode. */ +function syncWorkerDisplayMode(ds: DaemonSession): void { + if (!ds.worker || !ds.displayMode || ds.displayMode === 'hidden') return; + ds.worker.send({ type: 'set_display_mode', mode: ds.displayMode } as DaemonToWorker); +} + // ─── Callbacks set by daemon at startup ───────────────────────────────────── export interface WorkerSessionReplyOptions { @@ -1321,6 +1330,7 @@ export function ensureClaudeFolderTrust(workingDir: string, stateJsonPath: strin // ─── Kill worker ──────────────────────────────────────────────────────────── export function killWorker(ds: DaemonSession): void { + restartCoordinator.cancelSession(ds.session.sessionId); clearUsageLimitState(ds); ds.localProcessAttestation = undefined; // A managed-turn capability belongs to one concrete worker generation. @@ -1358,6 +1368,123 @@ export function killWorker(ds: DaemonSession): void { ds.workerViewToken = null; } +/** + * Whether a worker-less restart must first destroy the session's persistent + * backing pane. Adopt sessions are excluded — botmux never owned the user's + * pane, so killing it would violate the bridge invariant. Pure so the + * adopt-skip decision is unit-testable without spawning a worker. + */ +export function shouldDestroyPaneBeforeRestart( + ds: Pick, +): boolean { + return !ds.initConfig?.adoptMode && !ds.adoptedFrom; +} + +/** + * Destroy a still-alive persistent backing pane (tmux/herdr/zellij) before a + * worker-less restart forks a fresh worker. Without this, a session that lost + * its worker but kept its pane (the normal post-daemon-restart state) would let + * spawnCli REATTACH the surviving CLI instead of relaunching it — the CLI is + * never actually restarted, yet prompt-ready still fires `restart_result: + * succeeded`, so the user sees "restarted" while a wedged CLI stays wedged. + * Killing the pane first forces a genuine physical fresh spawn, making the + * success receipt truthful. + * + * Fail-safe (codex 复审观察): the kill primitives swallow their own failures + * (TmuxBackend.killSession `catch {}`, Herdr `runHerdr` returns false), so a + * plain try/catch here can NEVER observe a failed kill — a wedged tmux server + * could leave the pane alive and the fork would silently reattach. So we PROBE + * after killing and retry once if the pane survives, escalating to a loud warn + * when it still exists. A surviving pane still forks (refusing would strand the + * session with no restart at all, strictly worse than the original bug), but + * the warn makes the rare failure diagnosable instead of a silent false + * success. A fully reattach-proof path (forceFresh signal into spawnCli) is a + * larger, separate change — tracked as a follow-up, not blocking this fix. + * + * Scope: persistent panes only (getSessionPersistentBackendType excludes riff, + * which never reattaches — it always builds a fresh RiffBackend — and whose + * remote task must survive a restart to preserve follow-up lineage). Adopt + * sessions are skipped: botmux never owned the user's pane. + */ +function destroyLivePaneBeforeRestart(ds: DaemonSession): void { + if (!shouldDestroyPaneBeforeRestart(ds)) return; + const target = persistentBackendTargetForSession(ds); + if (!target) return; + + const killOnce = (): void => { + try { + killPersistentBackendTarget(target); + } catch (err) { + // The primitives normally swallow their own errors; this only catches a + // truly unexpected throw (e.g. target resolution). Non-fatal — the probe + // below is the real signal. + logger.warn(`[${tag(ds)}] restart: kill of ${target.backendType} pane threw: ${err}`); + } + }; + + killOnce(); + // Advance a single probe result monotonically through the real retry path so + // an 'unknown' first probe is never mislabelled as a post-retry survivor: + // - 'missing' → gone, fork is genuinely fresh. + // - 'unknown' → probe indeterminate (e.g. tmux server hiccup); do NOT retry + // and do NOT block the restart — pre-fix always forked, stranding is worse. + // - 'exists' → confirmed alive: warn, kill once more, re-probe. Only this + // branch performs (and can report) a retry. + let probe = probePersistentBackendTarget(target); + if (probe === 'exists') { + logger.warn(`[${tag(ds)}] restart: ${target.backendType} pane survived first kill — retrying before refork`); + killOnce(); + probe = probePersistentBackendTarget(target); + } + if (probe === 'exists') { + // The kill genuinely failed (twice). Forking will likely reattach the live + // pane (the very bug this guards), so the eventual `restart_result: + // succeeded` may again be untruthful — but leave a loud, greppable trail. + logger.error( + `[${tag(ds)}] restart: ${target.backendType} pane STILL alive after retry — ` + + 'the refork may reattach instead of relaunching (restart success may be untruthful)', + ); + } else if (probe === 'unknown') { + // Do NOT over-promise a relaunch: an indeterminate probe could still be a + // live pane the fork reattaches. Fork proceeds (stranding is worse) but the + // diagnostic must not claim a fresh relaunch it can't guarantee. + logger.warn( + `[${tag(ds)}] restart: ${target.backendType} kill outcome indeterminate — ` + + 'refork may reattach instead of relaunching', + ); + } else { + logger.info( + `[${tag(ds)}] restart: ${target.backendType} pane missing after kill — CLI will physically relaunch`, + ); + } +} + +/** Join or start one correlated physical restart for a session. */ +export function requestSessionRestart( + ds: DaemonSession, + observer: RestartObserver, +): { attemptId: string; joined: boolean } { + return restartCoordinator.request(ds.session.sessionId, observer, attemptId => { + if (ds.worker && !ds.worker.killed) { + ds.worker.send({ type: 'restart', attemptId } as DaemonToWorker); + return; + } + // No live worker but the persistent pane may still be alive (e.g. after a + // daemon restart). Tear it down first so forkWorker → spawnCli spawns a + // fresh CLI instead of reattaching the old one and falsely reporting a + // successful restart. + destroyLivePaneBeforeRestart(ds); + forkWorker(ds, '', { + resume: ds.hasHistory, + restartAttemptId: attemptId, + }); + }); +} + +export function __testOnly_resetRestartCoordinator(): void { + restartCoordinator.reset(); +} + /** * Tear down a persistent backing session (tmux/herdr/zellij) directly from the * daemon when there is no live worker to do it via the 'close' IPC. The session @@ -1955,6 +2082,7 @@ export function forkWorker( resume?: boolean; turnId?: string; dispatchAttempt?: number; + restartAttemptId?: string; } = false, ): void { // Device enrollment briefly freezes every daemon before the one-way host @@ -1993,12 +2121,14 @@ export function forkWorker( let resume = false; let initTurnId: string | undefined; let initDispatchAttempt: number | undefined; + let restartAttemptId: string | undefined; if (typeof resumeOrTurnId === 'string') { initTurnId = resumeOrTurnId; } else if (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null) { resume = resumeOrTurnId.resume === true; initTurnId = resumeOrTurnId.turnId; initDispatchAttempt = resumeOrTurnId.dispatchAttempt; + restartAttemptId = resumeOrTurnId.restartAttemptId; } else { resume = resumeOrTurnId; } @@ -2238,6 +2368,7 @@ export function forkWorker( promptPayload.codexAppInput, initAttributionTurnId, ); + const runtimeIdentity = runtimeBuildIdentity(); const initMsg: DaemonToWorker = { type: 'init', sessionId: ds.session.sessionId, @@ -2312,6 +2443,11 @@ export function forkWorker( ), pluginBindings: botCfg.plugins, skillPolicy: botCfg.skills, + ...(runtimeIdentity.status === 'known' + ? { runnerBuildId: runtimeIdentity.id } + : {}), + ...(ds.session.runnerBuildId ? { persistedRunnerBuildId: ds.session.runnerBuildId } : {}), + ...(restartAttemptId ? { restartAttemptId } : {}), }; worker.send(initMsg); ds.initConfig = initMsg; @@ -2665,6 +2801,7 @@ function setupWorkerHandlers( // (if any) is left untouched. The next real user turn clears this flag // (rememberLastCliInput) and the normal card flow resumes. if (ds.suppressRecoveryCard) { + syncWorkerDisplayMode(ds); logger.info(`[${t}] Restored session — suppressing recovery streaming card (silent restart)`); break; } @@ -2717,9 +2854,7 @@ function setupWorkerHandlers( } persistStreamCardState(ds); // Re-sync worker's display mode (it starts fresh in 'hidden') - if (ds.worker && ds.displayMode && ds.displayMode !== 'hidden') { - ds.worker.send({ type: 'set_display_mode', mode: ds.displayMode } as DaemonToWorker); - } + syncWorkerDisplayMode(ds); // The restored card is now the active one — withdraw any cards // frozen before the daemon went down so they don't pile up in the // thread on each restart. @@ -2784,9 +2919,7 @@ function setupWorkerHandlers( ds.streamCardPending = false; persistStreamCardState(ds); // Re-sync worker's display mode (it starts fresh in 'hidden') - if (ds.worker && ds.displayMode && ds.displayMode !== 'hidden') { - ds.worker.send({ type: 'set_display_mode', mode: ds.displayMode } as DaemonToWorker); - } + syncWorkerDisplayMode(ds); // New card is live — recall any cards frozen by previous turns. // Done after `streamCardId` is committed so we never delete the old // card without a successor visible to the user. @@ -2901,6 +3034,31 @@ function setupWorkerHandlers( break; } + case 'runner_build_ready': { + const identity = runtimeBuildIdentity(); + if ( + ds.worker === worker + && effectiveCliId === 'codex-app' + && identity.status === 'known' + && msg.runnerBuildId === identity.id + ) { + ds.session.runnerBuildId = msg.runnerBuildId; + sessionStore.updateSession(ds.session); + } else { + logger.warn(`[${t}] Ignored invalid or stale runner_build_ready`); + } + break; + } + + case 'restart_result': { + if (ds.worker !== worker) { + logger.warn(`[${t}] Ignored restart_result from stale worker generation`); + break; + } + restartCoordinator.resolve(ds.session.sessionId, msg.attemptId, msg.status); + break; + } + case 'cli_session_id': { const wasLocalCliOpenReady = isLocalCliOpenReady(ds, { cliId: effectiveCliId }); ds.session.cliSessionId = msg.cliSessionId; @@ -3600,6 +3758,28 @@ function setupWorkerHandlers( break; } + case 'steer_accepted': { + if (ds.worker !== worker) { + logger.warn(`[${t}] Ignored steer_accepted from stale worker generation`); + break; + } + logger.info( + `[${t}] Codex App steer accepted ` + + `appTurn=${msg.appTurnId.slice(0, 12)} replyTurn=${msg.turnId.slice(0, 12)}`, + ); + if (managedAuxUiSuppressed(msg.turnId, undefined)) break; + try { + await scopedReply(tr('worker.steer_accepted', undefined, loc), 'text', msg.turnId); + } catch { + logger.error( + `[${t}] Failed to deliver steer acknowledgement ` + + `appTurn=${msg.appTurnId.slice(0, 12)} replyTurn=${msg.turnId.slice(0, 12)} ` + + 'category=delivery', + ); + } + break; + } + case 'turn_terminal': { if (ds.worker !== worker) { logger.warn(`[${t}] Ignored turn_terminal from stale worker generation`); @@ -3798,6 +3978,7 @@ function setupWorkerHandlers( // A stale takeover worker never clears the replacement — during takeover the // old worker's exit fires AFTER the new worker has been assigned. if (ds.worker === worker) { + restartCoordinator.failSession(ds.session.sessionId); ds.worker = null; ds.workerPort = null; ds.managedTurnOrigin = undefined; diff --git a/src/i18n/en.ts b/src/i18n/en.ts index c955ad042..6ed74a22c 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -1,5 +1,6 @@ /** English translations — mirrors the keys defined in `zh.ts`. */ export const messages: Record = { + 'worker.steer_accepted': 'Got it — guidance accepted', // ─── Card buttons ──────────────────────────────────────────────────────── 'card.btn.open_terminal': '🖥️ Open Web Terminal', 'card.btn.open_writable_terminal': '🖥️ Open Writable Web Terminal', @@ -227,6 +228,9 @@ export const messages: Record = { 'cmd.substitute.owner_only': '⚠️ Only owner/allowedUsers can change substitute mode.', 'cmd.substitute.usage': 'Usage: @me /substitute status | on | off', 'cmd.restart.in_progress': '🔄 Restarting {cliName}…', + 'cmd.restart.succeeded': '✅ {cliName} is ready again.', + 'cmd.restart.failed': '❌ Failed to restart {cliName}.', + 'cmd.restart.timed_out': '⌛ {cliName} restart timed out before becoming ready.', 'cmd.restart.terminated': '{cliName} has been terminated; it will auto-resume on your next message.', 'cmd.cd.usage': 'Usage: /cd \nExample: /cd ~/projects/my-app', 'cmd.cd.switched': 'Working directory switched to {path}. It will resume there on your next message.', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index cc51e5e8b..3093abb0f 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -3,6 +3,7 @@ * "source of truth" dictionary; `en.ts` mirrors the same keys. */ export const messages: Record = { + 'worker.steer_accepted': '收到,引导成功', // ─── Card buttons ──────────────────────────────────────────────────────── 'card.btn.open_terminal': '🖥️ 打开 Web 终端', 'card.btn.open_writable_terminal': '🖥️ 打开可操作 Web 终端', @@ -230,6 +231,9 @@ export const messages: Record = { 'cmd.substitute.owner_only': '⚠️ 只有 owner/allowedUsers 可以修改替身模式开关。', 'cmd.substitute.usage': '用法:@我 /substitute status|on|off', 'cmd.restart.in_progress': '🔄 正在重启 {cliName}...', + 'cmd.restart.succeeded': '✅ {cliName} 已恢复就绪。', + 'cmd.restart.failed': '❌ {cliName} 重启失败。', + 'cmd.restart.timed_out': '⌛ {cliName} 重启超时,尚未恢复就绪。', 'cmd.restart.terminated': '{cliName} 进程已终止,下次发消息时将自动恢复。', 'cmd.cd.usage': '用法:/cd \n例如:/cd ~/projects/my-app', 'cmd.cd.switched': '工作目录已切换到 {path},下次发消息时将在新目录下恢复。', diff --git a/src/im/lark/card-handler.ts b/src/im/lark/card-handler.ts index 536471cdc..39e44af6d 100644 --- a/src/im/lark/card-handler.ts +++ b/src/im/lark/card-handler.ts @@ -69,7 +69,7 @@ import { ttadkConfigModelChoices } from '../../setup/cli-selection.js'; import { logger } from '../../utils/logger.js'; import * as sessionStore from '../../services/session-store.js'; import { loadFrozenCards, saveFrozenCards } from '../../services/frozen-card-store.js'; -import { forkWorker, sendWorkerInput, killWorker, scheduleCardPatch, parkStreamCard, clearUsageLimitState, cardUsageLimit, writableTerminalLinkFor, resolvePrivateCardAudience, deliverWriteLinkCard, deliverEphemeralOrReply, CARD_POSTING_SENTINEL } from '../../core/worker-pool.js'; +import { forkWorker, sendWorkerInput, killWorker, scheduleCardPatch, parkStreamCard, clearUsageLimitState, cardUsageLimit, writableTerminalLinkFor, resolvePrivateCardAudience, deliverWriteLinkCard, deliverEphemeralOrReply, CARD_POSTING_SENTINEL, requestSessionRestart } from '../../core/worker-pool.js'; import { getSessionWorkingDir, buildNewTopicCliInput, getAvailableBots, persistStreamCardState, resumeSession, rememberLastCliInput, ensureSessionWhiteboard } from '../../core/session-manager.js'; import { publishAttentionPatch, announcePendingRepoSession } from '../../core/session-activity.js'; import { fallbackTurnId } from '../../core/reply-target.js'; @@ -1759,22 +1759,22 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe await sessionReply(rootId, t('card.action.adopt_no_restart', undefined, locDs)); return; } - const botCfg = getBot(ds.larkAppId).config; const effectiveCliId = sessionCliId(ds); - if (ds.worker) { - logger.info(`[${tag(ds)}] Restart via card button`); - ds.worker.send({ type: 'restart' } as DaemonToWorker); - const cliName = getCliDisplayName(effectiveCliId); - const restartedMsg = t('card.action.restarted', { cliName }, locDs); - await deliverEphemeralOrReply(ds, operatorOpenId, restartedMsg, 'text', () => sessionReply(rootId, restartedMsg)); - } else { - logger.info(`[${tag(ds)}] Re-forking worker via card button`); - forkWorker(ds, '', ds.hasHistory); - const cliName = getCliDisplayName(effectiveCliId); - const restartedFreshMsg = t('card.action.restarted_fresh', { cliName }, locDs); - await deliverEphemeralOrReply(ds, operatorOpenId, restartedFreshMsg, 'text', () => sessionReply(rootId, restartedFreshMsg)); - // DM card will be sent by the ready handler when worker starts - } + const cliName = getCliDisplayName(effectiveCliId); + logger.info(`[${tag(ds)}] Correlated restart via card button`); + requestSessionRestart(ds, { + source: 'card', + notify: status => { + const content = t(`cmd.restart.${status}`, { cliName }, locDs); + return deliverEphemeralOrReply( + ds, + operatorOpenId, + content, + 'text', + () => sessionReply(rootId, content), + ); + }, + }); } if (actionType === 'close') { diff --git a/src/services/codex-app-runner-protocol.ts b/src/services/codex-app-runner-protocol.ts new file mode 100644 index 000000000..f2fc57b82 --- /dev/null +++ b/src/services/codex-app-runner-protocol.ts @@ -0,0 +1,254 @@ +import { Buffer } from 'node:buffer'; +import type { CodexAppTurnInput } from '../types.js'; +import { isCodexAppTurnInput } from '../adapters/cli/codex-app-turn.js'; + +export const CODEX_APP_INPUT_PREFIX = '::botmux-codex-app:'; + +export interface CodexAppRunnerInput { + type: 'message'; + content: string; + codexAppInput?: CodexAppTurnInput; + /** Immutable botmux/Lark message id used only for reply routing. */ + replyTurnId?: string; +} + +export interface CodexAppFinalMarker { + content: string; + startedAtMs?: number; + completedAtMs?: number; + /** Codex app-server turn id, used for protocol matching and deduplication. */ + appTurnId?: string; + /** botmux/Lark message id, used to select the Feishu reply destination. */ + replyTurnId?: string; + /** Pre-steer Codex App and Mira markers used one id for both domains. */ + legacyTurnId?: string; +} + +interface CodexAppLifecycleBase { + atMs: number; + queueLength?: number; + appTurnId?: string; + replyTurnId?: string; +} + +export type CodexAppLifecycleOperation = 'turn/start' | 'turn/steer' | 'runner'; +export type CodexAppLifecycleCategory = + | 'definite_rejection' + | 'steer_in_flight' + | 'transport' + | 'rpc' + | 'protocol' + | 'runtime'; + +export type CodexAppLifecycleEvent = + | (CodexAppLifecycleBase & { + kind: 'input_queued' | 'turn_start_attempt'; + queueLength: number; + }) + | (CodexAppLifecycleBase & { + kind: 'turn_started' | 'steer_attempt' | 'steer_accepted'; + appTurnId: string; + }) + | (CodexAppLifecycleBase & { + kind: 'steer_rejected_fallback'; + appTurnId: string; + category: 'definite_rejection'; + }) + | (CodexAppLifecycleBase & { + kind: 'completion_race'; + appTurnId: string; + category: 'steer_in_flight'; + }) + | (CodexAppLifecycleBase & { + kind: 'unknown_outcome'; + operation: 'turn/start' | 'turn/steer'; + category: 'transport' | 'rpc' | 'protocol'; + }) + | (CodexAppLifecycleBase & { + kind: 'fatal'; + operation: CodexAppLifecycleOperation; + category: 'transport' | 'rpc' | 'protocol' | 'runtime'; + }); + +export interface AppRunnerFinalIds { + lastUuid: string; + turnId: string; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function optionalNonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function optionalFiniteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function optionalLifecycleId(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 && value.length <= 512 + ? value + : undefined; +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} + +export function decodeCodexAppRunnerInput(line: string): CodexAppRunnerInput | undefined { + const trimmed = line.trim(); + if (!trimmed.startsWith(CODEX_APP_INPUT_PREFIX)) return undefined; + + let value: unknown; + try { + const encoded = trimmed.slice(CODEX_APP_INPUT_PREFIX.length); + value = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); + } catch { + return undefined; + } + if (!isRecord(value) || value.type !== 'message' || typeof value.content !== 'string') { + return undefined; + } + const allowedKeys = new Set(['type', 'content', 'codexAppInput', 'replyTurnId']); + if (Object.keys(value).some(key => !allowedKeys.has(key))) return undefined; + if (value.replyTurnId !== undefined && optionalLifecycleId(value.replyTurnId) === undefined) { + return undefined; + } + if (value.codexAppInput !== undefined && !isCodexAppTurnInput(value.codexAppInput)) { + return undefined; + } + const codexAppInput = isCodexAppTurnInput(value.codexAppInput) + ? value.codexAppInput + : undefined; + const replyTurnId = typeof value.replyTurnId === 'string' + ? value.replyTurnId + : codexAppInput?.clientUserMessageId; + return { + type: 'message', + content: value.content, + ...(codexAppInput ? { codexAppInput } : {}), + ...(replyTurnId ? { replyTurnId } : {}), + }; +} + +export function normalizeAppRunnerFinalMarker(payload: unknown): CodexAppFinalMarker | undefined { + if (!isRecord(payload) || typeof payload.content !== 'string') return undefined; + return { + content: payload.content, + startedAtMs: optionalFiniteNumber(payload.startedAtMs), + completedAtMs: optionalFiniteNumber(payload.completedAtMs), + appTurnId: optionalNonEmptyString(payload.appTurnId), + replyTurnId: optionalNonEmptyString(payload.replyTurnId), + legacyTurnId: optionalNonEmptyString(payload.turnId), + }; +} + +export function normalizeCodexAppLifecycleEvent(payload: unknown): CodexAppLifecycleEvent | undefined { + if (!isRecord(payload) || typeof payload.kind !== 'string') return undefined; + + const allowedKeys = new Set([ + 'kind', + 'atMs', + 'queueLength', + 'appTurnId', + 'replyTurnId', + 'operation', + 'category', + ]); + if (Object.keys(payload).some(key => !allowedKeys.has(key))) return undefined; + if (typeof payload.atMs !== 'number' || !Number.isFinite(payload.atMs) || payload.atMs < 0) { + return undefined; + } + if (payload.queueLength !== undefined && !isNonNegativeInteger(payload.queueLength)) { + return undefined; + } + if (payload.appTurnId !== undefined && optionalLifecycleId(payload.appTurnId) === undefined) { + return undefined; + } + if (payload.replyTurnId !== undefined && optionalLifecycleId(payload.replyTurnId) === undefined) { + return undefined; + } + + const base = { + atMs: payload.atMs, + ...(typeof payload.queueLength === 'number' ? { queueLength: payload.queueLength } : {}), + ...(typeof payload.appTurnId === 'string' ? { appTurnId: payload.appTurnId } : {}), + ...(typeof payload.replyTurnId === 'string' ? { replyTurnId: payload.replyTurnId } : {}), + }; + const appTurnId = optionalLifecycleId(payload.appTurnId); + + switch (payload.kind) { + case 'input_queued': + case 'turn_start_attempt': + if (!isNonNegativeInteger(payload.queueLength) + || payload.appTurnId !== undefined + || payload.operation !== undefined + || payload.category !== undefined) return undefined; + return { ...base, kind: payload.kind, queueLength: payload.queueLength }; + case 'turn_started': + case 'steer_attempt': + case 'steer_accepted': + if (!appTurnId || payload.operation !== undefined || payload.category !== undefined) { + return undefined; + } + return { ...base, kind: payload.kind, appTurnId }; + case 'steer_rejected_fallback': + if (!appTurnId || payload.operation !== undefined || payload.category !== 'definite_rejection') { + return undefined; + } + return { ...base, kind: payload.kind, appTurnId, category: 'definite_rejection' }; + case 'completion_race': + if (!appTurnId || payload.operation !== undefined || payload.category !== 'steer_in_flight') { + return undefined; + } + return { ...base, kind: payload.kind, appTurnId, category: 'steer_in_flight' }; + case 'unknown_outcome': + if ( + (payload.operation !== 'turn/start' && payload.operation !== 'turn/steer') + || (payload.category !== 'transport' + && payload.category !== 'rpc' + && payload.category !== 'protocol') + ) return undefined; + return { + ...base, + kind: payload.kind, + operation: payload.operation, + category: payload.category, + }; + case 'fatal': + if ( + (payload.operation !== 'turn/start' + && payload.operation !== 'turn/steer' + && payload.operation !== 'runner') + || (payload.category !== 'transport' + && payload.category !== 'rpc' + && payload.category !== 'protocol' + && payload.category !== 'runtime') + ) return undefined; + return { + ...base, + kind: payload.kind, + operation: payload.operation, + category: payload.category, + }; + default: + return undefined; + } +} + +export function projectAppRunnerFinalIds( + marker: CodexAppFinalMarker, + fallbackTurnId: string | undefined, + generatedFallbackId: string, +): AppRunnerFinalIds { + if (marker.appTurnId) { + return { + lastUuid: marker.appTurnId, + turnId: marker.replyTurnId ?? fallbackTurnId ?? marker.appTurnId, + }; + } + const legacyId = marker.legacyTurnId ?? fallbackTurnId ?? generatedFallbackId; + return { lastUuid: legacyId, turnId: legacyId }; +} diff --git a/src/services/codex-app-turn-controller.ts b/src/services/codex-app-turn-controller.ts new file mode 100644 index 000000000..b68b74a85 --- /dev/null +++ b/src/services/codex-app-turn-controller.ts @@ -0,0 +1,613 @@ +import type { + CodexAppFinalMarker, + CodexAppLifecycleCategory, + CodexAppLifecycleEvent, + CodexAppLifecycleOperation, + CodexAppRunnerInput, +} from './codex-app-runner-protocol.js'; +import type { CodexAppTurnInput } from '../types.js'; + +type JsonObject = Record; + +export class CodexAppRpcResponseError extends Error { + readonly code?: number; + readonly data?: unknown; + + constructor(readonly method: string, error: unknown) { + const payload = isRecord(error) ? error : {}; + const detail = typeof payload.message === 'string' + ? payload.message + : JSON.stringify(error); + super(`${method}: ${detail}`); + this.name = 'CodexAppRpcResponseError'; + this.code = typeof payload.code === 'number' ? payload.code : undefined; + this.data = payload.data; + } +} + +export class CodexAppTransportError extends Error { + constructor(message: string) { + super(message); + this.name = 'CodexAppTransportError'; + } +} + +export interface CodexAppPreparedInput { + input: Array>; + additionalContext?: CodexAppTurnInput['additionalContext']; + clientUserMessageId?: string; + visibleText: string; + structured: boolean; + skippedImages?: string[]; +} + +export interface CodexAppTurnControllerDeps { + cwd: string; + ensureThread(): Promise; + request(method: string, params: unknown): Promise; + prepareInput(input: CodexAppRunnerInput, structuredDisabled: boolean): CodexAppPreparedInput; + isStartCapabilityError?(error: unknown): boolean; + onTurnInput?(input: CodexAppRunnerInput, prepared: CodexAppPreparedInput): void; + onOutput?(text: string): void; + onDiagnostic?(message: string): void; + onLifecycle?(event: CodexAppLifecycleEvent): void; + onFinal(marker: CodexAppFinalMarker & { appTurnId: string }): void; + onPrompt?(): void; + now?(): number; +} + +type TurnPhase = 'starting' | 'active' | 'closing'; + +interface ActiveTurn { + phase: TurnPhase; + threadId?: string; + appTurnId?: string; + replyTurnId?: string; + startedAtMs: number; + finalText: string; + allAgentText: string; + itemText: Map; + completionSeen: boolean; + serverStarted: boolean; + steeringClosed: boolean; + steerInFlight?: CodexAppRunnerInput; + turnStartedEmitted: boolean; + completionRaceEmitted: boolean; + finalEmitted: boolean; +} + +function isRecord(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function stringField(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function turnIdFromParams(params: JsonObject): string | undefined { + const turn = isRecord(params.turn) ? params.turn : undefined; + return stringField(turn?.id) ?? stringField(params.turnId); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function isDefiniteSteerRejection(error: CodexAppRpcResponseError): boolean { + // JSON-RPC parse/method/params errors prove the operation was not accepted. + if (error.code === -32600 || error.code === -32601 || error.code === -32602) return true; + let dataText = ''; + try { dataText = JSON.stringify(error.data ?? '').toLowerCase(); } catch { /* untrusted data */ } + const detail = `${error.message} ${dataText}`.toLowerCase(); + return detail.includes('no active turn to steer') + || detail.includes('activeturnnotsteerable') + || detail.includes('active turn not steerable') + || detail.includes('expectedturnid') + || detail.includes('expected turn id') + || detail.includes('cannot steer a review turn') + || detail.includes('cannot steer a compact turn') + || detail.includes('input must not be empty'); +} + +const CLOSED_TURN_LIMIT = 64; + +export class CodexAppTurnController { + private readonly queue: CodexAppRunnerInput[] = []; + private readonly closedTurnIds = new Set(); + private active: ActiveTurn | null = null; + private fatal = false; + private structuredDisabled = false; + private failureSequence = 0; + private driving = false; + private driveAgain = false; + + constructor(private readonly deps: CodexAppTurnControllerDeps) {} + + enqueue(input: CodexAppRunnerInput): void { + if (this.fatal) { + this.emitStandaloneFailure(input, 'Codex App runner is unavailable.'); + return; + } + this.queue.push(input); + this.emitLifecycle({ + kind: 'input_queued', + atMs: this.now(), + queueLength: this.queue.length, + ...(input.replyTurnId ? { replyTurnId: input.replyTurnId } : {}), + }); + this.drive(); + } + + handleNotification(message: unknown): void { + if (!isRecord(message) || typeof message.method !== 'string' || !isRecord(message.params)) return; + const turn = this.active; + if (!turn) return; + + const params = message.params; + const notificationThreadId = stringField(params.threadId); + if (turn.threadId && notificationThreadId && notificationThreadId !== turn.threadId) return; + + const notificationTurnId = turnIdFromParams(params); + if (notificationTurnId && this.closedTurnIds.has(notificationTurnId)) return; + if (turn.appTurnId && notificationTurnId && notificationTurnId !== turn.appTurnId) return; + + if (message.method === 'turn/started') { + turn.serverStarted = true; + if (notificationTurnId && !this.adoptAppTurnId(turn, notificationTurnId)) return; + if (!turn.completionSeen) turn.phase = 'active'; + this.drive(); + return; + } + + if (message.method === 'item/started') { + const item = isRecord(params.item) ? params.item : undefined; + if (item?.type === 'commandExecution') { + this.deps.onOutput?.(`\n$ ${String(item.command ?? '')}\n`); + } else if (item?.type === 'fileChange') { + this.deps.onOutput?.('\n[files changed]\n'); + } + return; + } + + if (message.method === 'item/agentMessage/delta') { + const delta = String(params.delta ?? ''); + const itemId = String(params.itemId ?? ''); + turn.itemText.set(itemId, (turn.itemText.get(itemId) ?? '') + delta); + turn.allAgentText += delta; + this.deps.onOutput?.(delta); + return; + } + + if (message.method === 'item/commandExecution/outputDelta' || message.method === 'item/fileChange/outputDelta') { + this.deps.onOutput?.(String(params.delta ?? '')); + return; + } + + if (message.method === 'item/completed') { + const item = isRecord(params.item) ? params.item : undefined; + if (item?.type === 'agentMessage') { + const text = typeof item.text === 'string' ? item.text : ''; + if (item.phase === 'final_answer') turn.finalText = text; + else if (!turn.itemText.has(String(item.id ?? '')) && text) turn.allAgentText += text; + } + return; + } + + if (message.method === 'turn/completed') { + if (notificationTurnId && !this.adoptAppTurnId(turn, notificationTurnId)) return; + const completed = isRecord(params.turn) ? params.turn : undefined; + const completedError = isRecord(completed?.error) ? completed.error : undefined; + if (typeof completedError?.message === 'string' && !turn.finalText) { + turn.finalText = `Codex App turn failed: ${completedError.message}`; + } + if (turn.steerInFlight && turn.appTurnId && !turn.completionRaceEmitted) { + turn.completionRaceEmitted = true; + this.emitLifecycle({ + kind: 'completion_race', + atMs: this.now(), + appTurnId: turn.appTurnId, + replyTurnId: turn.steerInFlight.replyTurnId, + queueLength: this.queue.length, + category: 'steer_in_flight', + }); + } + turn.completionSeen = true; + turn.phase = 'closing'; + this.drive(); + } + } + + handleFatal( + error: unknown, + category: Extract + = this.errorCategory(error), + ): void { + if (this.fatal) return; + this.fatal = true; + const message = `Codex App runner error: ${errorMessage(error)}`; + const active = this.active; + const replyTurnId = active?.steerInFlight?.replyTurnId ?? active?.replyTurnId; + this.emitLifecycle({ + kind: 'fatal', + atMs: this.now(), + operation: this.activeOperation(active), + category, + ...(active?.appTurnId ? { appTurnId: active.appTurnId } : {}), + ...(replyTurnId ? { replyTurnId } : {}), + queueLength: this.queue.length, + }); + this.active = null; + if (active && !active.finalEmitted) { + if (active.steerInFlight && this.queue[0] === active.steerInFlight) { + this.queue.shift(); + } + this.emitFailure(active, replyTurnId, message); + } + for (const input of this.queue.splice(0)) this.emitStandaloneFailure(input, message); + } + + private drive(): void { + if (this.driving) { + this.driveAgain = true; + return; + } + this.driving = true; + try { + do { + this.driveAgain = false; + if (this.fatal) return; + const turn = this.active; + if (!turn) { + const next = this.queue.shift(); + if (!next) return; + this.startTurn(next); + return; + } + if (turn.completionSeen && !turn.steerInFlight) { + this.finalizeTurn(turn); + continue; + } + if ( + turn.phase === 'active' + && turn.appTurnId + && !turn.steeringClosed + && !turn.steerInFlight + && this.queue.length > 0 + ) { + this.steerQueueHead(turn); + } + return; + } while (this.driveAgain); + } finally { + this.driving = false; + if (this.driveAgain) this.drive(); + } + } + + private startTurn(input: CodexAppRunnerInput): void { + const turn: ActiveTurn = { + phase: 'starting', + replyTurnId: input.replyTurnId, + startedAtMs: this.now(), + finalText: '', + allAgentText: '', + itemText: new Map(), + completionSeen: false, + serverStarted: false, + steeringClosed: false, + turnStartedEmitted: false, + completionRaceEmitted: false, + finalEmitted: false, + }; + this.active = turn; + this.emitLifecycle({ + kind: 'turn_start_attempt', + atMs: this.now(), + queueLength: this.queue.length, + ...(input.replyTurnId ? { replyTurnId: input.replyTurnId } : {}), + }); + void this.beginTurnStart(turn, input); + } + + private async beginTurnStart(turn: ActiveTurn, input: CodexAppRunnerInput): Promise { + try { + const threadId = await this.deps.ensureThread(); + if (this.active !== turn || this.fatal) return; + turn.threadId = threadId; + let prepared = this.deps.prepareInput(input, this.structuredDisabled); + this.reportPreparedInput(prepared); + this.deps.onTurnInput?.(input, prepared); + + let result: unknown; + try { + result = await this.deps.request('turn/start', this.turnStartParams(threadId, prepared)); + } catch (error) { + if ( + prepared.structured + && !turn.serverStarted + && this.deps.isStartCapabilityError?.(error) + ) { + this.structuredDisabled = true; + this.deps.onDiagnostic?.( + '[codex-app] structured input unsupported; retrying this turn with the legacy prompt', + ); + prepared = this.deps.prepareInput(input, true); + result = await this.deps.request('turn/start', this.turnStartParams(threadId, prepared)); + } else { + throw error; + } + } + + if (this.active !== turn || this.fatal) return; + const resultTurn = isRecord(result) && isRecord(result.turn) ? result.turn : undefined; + const resultTurnId = stringField(resultTurn?.id); + if (resultTurnId && !this.adoptAppTurnId(turn, resultTurnId)) return; + if (!turn.appTurnId) { + this.emitUnknownOutcome(turn, 'turn/start', 'protocol', input.replyTurnId); + this.handleFatal(new CodexAppTransportError( + 'turn/start returned no app-server turn id.', + ), 'protocol'); + return; + } + if (!turn.completionSeen) turn.phase = 'active'; + this.drive(); + } catch (error) { + if (this.active !== turn || this.fatal) return; + if (error instanceof CodexAppTransportError || turn.serverStarted) { + this.emitUnknownOutcome( + turn, + 'turn/start', + error instanceof CodexAppRpcResponseError ? 'rpc' : 'transport', + input.replyTurnId, + ); + this.handleFatal(error); + return; + } + this.active = null; + this.emitFailure( + turn, + turn.replyTurnId, + `Codex App runner error: ${errorMessage(error)}`, + ); + this.afterTurn(); + } + } + + private steerQueueHead(turn: ActiveTurn): void { + const input = this.queue[0]; + if (!input || !turn.threadId || !turn.appTurnId) return; + turn.steerInFlight = input; + void this.performSteer(turn, input); + } + + private async performSteer(turn: ActiveTurn, input: CodexAppRunnerInput): Promise { + const expectedTurnId = turn.appTurnId!; + try { + const prepared = this.deps.prepareInput(input, this.structuredDisabled); + this.reportPreparedInput(prepared); + this.emitLifecycle({ + kind: 'steer_attempt', + atMs: this.now(), + appTurnId: expectedTurnId, + ...(input.replyTurnId ? { replyTurnId: input.replyTurnId } : {}), + queueLength: this.queue.length, + }); + const result = await this.deps.request('turn/steer', { + threadId: turn.threadId, + input: prepared.input, + expectedTurnId, + ...(prepared.clientUserMessageId + ? { clientUserMessageId: prepared.clientUserMessageId } + : {}), + ...(prepared.additionalContext + ? { additionalContext: prepared.additionalContext } + : {}), + }); + if (this.active !== turn || this.fatal) return; + const responseTurnId = isRecord(result) ? stringField(result.turnId) : undefined; + if (responseTurnId !== expectedTurnId) { + this.emitUnknownOutcome(turn, 'turn/steer', 'protocol', input.replyTurnId); + this.handleFatal(new CodexAppTransportError( + 'turn/steer returned an unexpected turn id.', + ), 'protocol'); + return; + } + if (this.queue[0] === input) this.queue.shift(); + turn.replyTurnId = input.replyTurnId ?? turn.replyTurnId; + this.deps.onTurnInput?.(input, prepared); + this.emitLifecycle({ + kind: 'steer_accepted', + atMs: this.now(), + appTurnId: expectedTurnId, + ...(input.replyTurnId ? { replyTurnId: input.replyTurnId } : {}), + queueLength: this.queue.length, + }); + } catch (error) { + if (this.active !== turn || this.fatal) return; + if ( + error instanceof CodexAppRpcResponseError + && isDefiniteSteerRejection(error) + ) { + turn.steeringClosed = true; + this.emitLifecycle({ + kind: 'steer_rejected_fallback', + atMs: this.now(), + appTurnId: expectedTurnId, + ...(input.replyTurnId ? { replyTurnId: input.replyTurnId } : {}), + queueLength: this.queue.length, + category: 'definite_rejection', + }); + this.deps.onDiagnostic?.('[codex-app] steer rejected; queued for the next turn'); + return; + } + + this.emitUnknownOutcome( + turn, + 'turn/steer', + error instanceof CodexAppRpcResponseError ? 'rpc' : 'transport', + input.replyTurnId, + ); + this.handleFatal(new CodexAppTransportError( + `steer result unknown (${errorMessage(error)})`, + )); + } finally { + if (this.active === turn) { + turn.steerInFlight = undefined; + this.drive(); + } + } + } + + private turnStartParams(threadId: string, prepared: CodexAppPreparedInput): Record { + return { + threadId, + input: prepared.input, + ...(prepared.clientUserMessageId + ? { clientUserMessageId: prepared.clientUserMessageId } + : {}), + ...(prepared.additionalContext + ? { additionalContext: prepared.additionalContext } + : {}), + cwd: this.deps.cwd, + approvalPolicy: 'never', + sandboxPolicy: { type: 'dangerFullAccess' }, + }; + } + + private reportPreparedInput(prepared: CodexAppPreparedInput): void { + for (const path of prepared.skippedImages ?? []) { + this.deps.onDiagnostic?.(`[codex-app] skipped unreadable local image: ${path}`); + } + } + + private adoptAppTurnId(turn: ActiveTurn, appTurnId: string): boolean { + if (!turn.appTurnId) { + turn.appTurnId = appTurnId; + this.emitTurnStarted(turn); + return true; + } + if (turn.appTurnId === appTurnId) { + this.emitTurnStarted(turn); + return true; + } + this.handleFatal(new CodexAppTransportError( + 'app-server turn id changed unexpectedly.', + ), 'protocol'); + return false; + } + + private emitTurnStarted(turn: ActiveTurn): void { + if (turn.turnStartedEmitted || !turn.appTurnId) return; + turn.turnStartedEmitted = true; + this.emitLifecycle({ + kind: 'turn_started', + atMs: this.now(), + appTurnId: turn.appTurnId, + ...(turn.replyTurnId ? { replyTurnId: turn.replyTurnId } : {}), + queueLength: this.queue.length, + }); + } + + private emitUnknownOutcome( + turn: ActiveTurn, + operation: 'turn/start' | 'turn/steer', + category: 'transport' | 'rpc' | 'protocol', + replyTurnId: string | undefined, + ): void { + this.emitLifecycle({ + kind: 'unknown_outcome', + atMs: this.now(), + operation, + category, + ...(turn.appTurnId ? { appTurnId: turn.appTurnId } : {}), + ...(replyTurnId ? { replyTurnId } : {}), + queueLength: this.queue.length, + }); + } + + private finalizeTurn(turn: ActiveTurn): void { + if (this.active !== turn || turn.finalEmitted) return; + turn.finalEmitted = true; + const completedAtMs = this.now(); + const appTurnId = turn.appTurnId ?? this.nextFailureId(completedAtMs); + this.rememberClosedTurn(appTurnId); + const content = (turn.finalText || turn.allAgentText).trim(); + if (content) { + this.deps.onFinal({ + appTurnId, + replyTurnId: turn.replyTurnId, + content, + startedAtMs: turn.startedAtMs, + completedAtMs, + }); + } + this.active = null; + this.afterTurn(); + } + + private emitFailure(turn: ActiveTurn, replyTurnId: string | undefined, content: string): void { + if (turn.finalEmitted) return; + turn.finalEmitted = true; + const completedAtMs = this.now(); + const appTurnId = turn.appTurnId ?? this.nextFailureId(completedAtMs); + this.rememberClosedTurn(appTurnId); + this.deps.onDiagnostic?.(content); + this.deps.onFinal({ + appTurnId, + replyTurnId, + content, + startedAtMs: turn.startedAtMs, + completedAtMs, + }); + } + + private emitStandaloneFailure(input: CodexAppRunnerInput, content: string): void { + const now = this.now(); + this.deps.onDiagnostic?.(content); + this.deps.onFinal({ + appTurnId: this.nextFailureId(now), + replyTurnId: input.replyTurnId, + content, + startedAtMs: now, + completedAtMs: now, + }); + } + + private afterTurn(): void { + if (this.queue.length === 0) this.deps.onPrompt?.(); + this.drive(); + } + + private rememberClosedTurn(turnId: string): void { + this.closedTurnIds.add(turnId); + if (this.closedTurnIds.size <= CLOSED_TURN_LIMIT) return; + const oldest = this.closedTurnIds.values().next().value; + if (typeof oldest === 'string') this.closedTurnIds.delete(oldest); + } + + private nextFailureId(now: number): string { + return `codex-app-error-${now}-${++this.failureSequence}`; + } + + private now(): number { + return this.deps.now?.() ?? Date.now(); + } + + private emitLifecycle(event: CodexAppLifecycleEvent): void { + this.deps.onLifecycle?.(event); + } + + private activeOperation(active: ActiveTurn | null): CodexAppLifecycleOperation { + if (active?.steerInFlight) return 'turn/steer'; + if (active?.phase === 'starting') return 'turn/start'; + return 'runner'; + } + + private errorCategory( + error: unknown, + ): Extract { + if (error instanceof CodexAppTransportError) return 'transport'; + if (error instanceof CodexAppRpcResponseError) return 'rpc'; + return 'runtime'; + } +} diff --git a/src/services/codex-runner-freshness.ts b/src/services/codex-runner-freshness.ts new file mode 100644 index 000000000..c19f49318 --- /dev/null +++ b/src/services/codex-runner-freshness.ts @@ -0,0 +1,110 @@ +export type CodexRunnerFreshnessState = + | 'current' + | 'stale_waiting_idle' + | 'restarting_fresh' + | 'failed' + | 'unknown'; + +export interface CodexRunnerFreshnessDecision { + state: CodexRunnerFreshnessState; + reason: + | 'not_codex_app' + | 'adopt_session' + | 'fresh_spawn' + | 'build_match' + | 'persisted_build_missing' + | 'build_mismatch' + | 'replacement_reattached' + | 'current_build_unknown'; + persistOnReady: boolean; +} + +export function decideCodexRunnerFreshness(input: { + cliId: string; + adoptMode: boolean; + persistentReattach: boolean; + replacementExpectedFresh?: boolean; + currentBuildId?: string; + persistedBuildId?: string; +}): CodexRunnerFreshnessDecision { + if (input.cliId !== 'codex-app') { + return { state: 'current', reason: 'not_codex_app', persistOnReady: false }; + } + if (input.adoptMode) { + return { state: 'current', reason: 'adopt_session', persistOnReady: false }; + } + if (input.replacementExpectedFresh && input.persistentReattach) { + return { state: 'failed', reason: 'replacement_reattached', persistOnReady: false }; + } + if (!input.currentBuildId) { + return { state: 'unknown', reason: 'current_build_unknown', persistOnReady: false }; + } + if (!input.persistentReattach) { + return { state: 'current', reason: 'fresh_spawn', persistOnReady: true }; + } + if (!input.persistedBuildId) { + return { state: 'stale_waiting_idle', reason: 'persisted_build_missing', persistOnReady: false }; + } + if (input.persistedBuildId !== input.currentBuildId) { + return { state: 'stale_waiting_idle', reason: 'build_mismatch', persistOnReady: false }; + } + return { state: 'current', reason: 'build_match', persistOnReady: false }; +} + +export function shouldHoldCodexRunnerInput(state: CodexRunnerFreshnessState): boolean { + return state === 'stale_waiting_idle' || state === 'restarting_fresh' || state === 'failed'; +} + +export function transitionOnCodexRunnerPrompt(state: CodexRunnerFreshnessState): { + state: CodexRunnerFreshnessState; + action: 'publish_ready' | 'reload' | 'ignore'; +} { + if (state === 'stale_waiting_idle') return { state: 'restarting_fresh', action: 'reload' }; + if (state === 'restarting_fresh') return { state: 'current', action: 'publish_ready' }; + if (state === 'failed') return { state: 'failed', action: 'ignore' }; + return { state, action: 'publish_ready' }; +} + +/** + * Worker-owned input queues whose dequeue boundary is fenced by runner + * freshness. Keeping this small seam outside worker.ts makes the stale-runner + * contract executable without importing the worker process (which installs + * IPC handlers and starts runtime services at module load). + */ +export class CodexRunnerFreshnessInputQueue { + readonly normal: TNormal[] = []; + readonly raw: TRaw[] = []; + + constructor( + private readonly getState: () => CodexRunnerFreshnessState, + private readonly setState: (state: CodexRunnerFreshnessState) => void, + ) {} + + enqueueNormal(input: TNormal): void { + this.normal.push(input); + } + + enqueueRaw(input: TRaw): void { + this.raw.push(input); + } + + takeNormal(): TNormal | undefined { + if (shouldHoldCodexRunnerInput(this.getState())) return undefined; + return this.normal.shift(); + } + + takeRaw(): TRaw | undefined { + if (shouldHoldCodexRunnerInput(this.getState())) return undefined; + return this.raw.shift(); + } + + onPromptReady(): 'publish_ready' | 'reload' | 'ignore' { + const transition = transitionOnCodexRunnerPrompt(this.getState()); + this.setState(transition.state); + return transition.action; + } + + onReplacementFailed(): void { + if (this.getState() === 'restarting_fresh') this.setState('failed'); + } +} diff --git a/src/types.ts b/src/types.ts index e5bb00f37..0668a22ae 100644 --- a/src/types.ts +++ b/src/types.ts @@ -116,6 +116,8 @@ export type StreamStatus = ScreenStatus | 'starting'; export interface Session { sessionId: string; + /** Build fingerprint of the last fresh owned Codex App runner that became ready. */ + runnerBuildId?: string; chatId: string; chatType?: 'group' | 'p2p'; /** Thread-scope: an actual root message id under which all replies thread. @@ -581,7 +583,7 @@ export interface CliTurnPayload { /** Messages sent from Daemon to Worker */ export type DaemonToWorker = - | { type: 'init'; sessionId: string; chatId: string; chatType?: 'group' | 'p2p'; rootMessageId: string; workingDir: string; cliId: string; cliPathOverride?: string; wrapperCli?: string; launchShell?: string; model?: string; disableCliBypass?: boolean; codexRpcInput?: boolean; startupCommands?: string[]; env?: Record; sandbox?: boolean; sandboxPaths?: { readWrite?: string[]; readOnly?: string[]; deny?: string[] }; sandboxHidePaths?: string[]; sandboxReadonlyPaths?: string[]; sandboxNetwork?: boolean; readIsolation?: boolean; readDenyExtraPaths?: string[]; daemonBootId?: string; backendType: BackendType; persistentBackendTarget?: PersistentBackendTarget; backendConfig?: RiffBackendConfig; riffParentTaskId?: string; riffRepoDirs?: string[]; deferredScheduleRun?: Session['deferredScheduleRun']; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; prompt: string; promptCodexAppInput?: CodexAppTurnInput; resume?: boolean; cliSessionId?: string; originalSessionId?: string; ownerOpenId?: string; webPort?: number; larkAppId: string; larkAppSecret: string; brand?: 'feishu' | 'lark'; botName?: string; botOpenId?: string; locale?: 'zh' | 'en'; turnId?: string; dispatchAttempt?: number; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; pluginBindings?: string[]; skillPolicy?: BotSkillPolicy; skillPluginDir?: string; skillReadonlyRoots?: string[]; adoptMode?: boolean; adoptSource?: 'tmux' | 'herdr' | 'zellij'; adoptTmuxTarget?: string; adoptZellijSession?: string; adoptZellijPaneId?: string; adoptHerdrSessionName?: string; adoptHerdrTarget?: string; adoptHerdrPaneId?: string; adoptPaneCols?: number; adoptPaneRows?: number; bridgeJsonlPath?: string; adoptCliPid?: number; adoptCwd?: string; adoptRestoredFromMetadata?: boolean } + | { type: 'init'; sessionId: string; chatId: string; chatType?: 'group' | 'p2p'; rootMessageId: string; workingDir: string; cliId: string; cliPathOverride?: string; wrapperCli?: string; launchShell?: string; model?: string; disableCliBypass?: boolean; codexRpcInput?: boolean; startupCommands?: string[]; env?: Record; sandbox?: boolean; sandboxPaths?: { readWrite?: string[]; readOnly?: string[]; deny?: string[] }; sandboxHidePaths?: string[]; sandboxReadonlyPaths?: string[]; sandboxNetwork?: boolean; readIsolation?: boolean; readDenyExtraPaths?: string[]; daemonBootId?: string; backendType: BackendType; persistentBackendTarget?: PersistentBackendTarget; backendConfig?: RiffBackendConfig; riffParentTaskId?: string; riffRepoDirs?: string[]; deferredScheduleRun?: Session['deferredScheduleRun']; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; prompt: string; promptCodexAppInput?: CodexAppTurnInput; resume?: boolean; cliSessionId?: string; originalSessionId?: string; ownerOpenId?: string; webPort?: number; larkAppId: string; larkAppSecret: string; brand?: 'feishu' | 'lark'; botName?: string; botOpenId?: string; locale?: 'zh' | 'en'; turnId?: string; dispatchAttempt?: number; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; pluginBindings?: string[]; skillPolicy?: BotSkillPolicy; skillPluginDir?: string; skillReadonlyRoots?: string[]; adoptMode?: boolean; adoptSource?: 'tmux' | 'herdr' | 'zellij'; adoptTmuxTarget?: string; adoptZellijSession?: string; adoptZellijPaneId?: string; adoptHerdrSessionName?: string; adoptHerdrTarget?: string; adoptHerdrPaneId?: string; adoptPaneCols?: number; adoptPaneRows?: number; bridgeJsonlPath?: string; adoptCliPid?: number; adoptCwd?: string; adoptRestoredFromMetadata?: boolean; runnerBuildId?: string; persistedRunnerBuildId?: string; restartAttemptId?: string } | { type: 'message'; content: string; codexAppInput?: CodexAppTurnInput; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; turnId?: string; dispatchAttempt?: number; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin } /** Literal slash-command passthrough. `followUpContent` rides along so the * worker enqueues it strictly AFTER the slash command's Enter — two separate @@ -599,8 +601,8 @@ export type DaemonToWorker = /** Kill the CLI and respawn it with --resume. `updateWorkingDir`(可选) * 用于角色切换的 cwd-move respawn:respawn 前把 worker 侧 lastInitConfig * 收敛到新目录,让 CLI 在新 cwd 冷启动(新 CLAUDE.md/记忆索引开场注入) - * 同时 --resume 续回对话上下文。 */ - | { type: 'restart'; updateWorkingDir?: string } + * 同时 --resume 续回对话上下文。`attemptId` 关联手工 restart 的完成回执。 */ + | { type: 'restart'; attemptId?: string; updateWorkingDir?: string } /** Lease watchdog fencing: only the exact still-running durable attempt may * tear down/restart the CLI. A late command after terminal/current-turn * advance is ignored worker-side. */ @@ -655,6 +657,13 @@ export type WorkerToDaemon = | { type: 'native_session_title_generated'; title: string } | { type: 'claude_exit'; code: number | null; signal: string | null; logTail?: string; canParkDiagnostic?: boolean; turnId?: string; dispatchAttempt?: number } | { type: 'prompt_ready' } + | { type: 'runner_build_ready'; runnerBuildId: string } + | { + type: 'restart_result'; + attemptId: string; + status: 'succeeded' | 'failed' | 'timed_out'; + category: 'prompt_ready' | 'spawn_failed' | 'runner_exited' | 'readiness_timeout'; + } /** Worker 已处理 SessionStart 信号并建立 post-hook prompt evidence fence。 * daemon 收到后才结束 `botmux session-ready` HTTP 请求。 */ | { type: 'session_ready_ack'; requestId: string } @@ -668,6 +677,10 @@ export type WorkerToDaemon = | { type: 'tui_keys_delivered'; nonce: number; turnId?: string; dispatchAttempt?: number } | { type: 'screenshot_uploaded'; imageKey: string; status: ScreenStatus; usageLimit?: CliUsageLimitState; turnId?: string; dispatchAttempt?: number } | { type: 'user_notify'; message: string; turnId?: string; dispatchAttempt?: number } + /** A normal success acknowledgement for one app-server accepted steer. + * `appTurnId` is diagnostic/protocol identity; `turnId` is the immutable + * botmux/Lark reply route. This must never enter the attention path. */ + | { type: 'steer_accepted'; appTurnId: string; turnId: string } | { type: 'receiver_reset_ready'; sessionId: string; turnId: string; dispatchAttempt: number } /** Runtime lease recovery ACK. Emitted only after the exact durable attempt * was either removed from the worker queue or its owned CLI was fenced. */ diff --git a/src/utils/input-gate.ts b/src/utils/input-gate.ts index c89474ae7..c038743c1 100644 --- a/src/utils/input-gate.ts +++ b/src/utils/input-gate.ts @@ -27,7 +27,10 @@ export function shouldWriteNow(state: { supportsTypeAhead: boolean; /** True until the CLI has reached its first ready state (boot / re-attach window). */ awaitingFirstPrompt: boolean; + /** A stale Codex App runner must not receive normal or type-ahead input. */ + holdForRunnerReload?: boolean; }): boolean { + if (state.holdForRunnerReload) return false; if (state.isPromptReady || state.isFlushing) return true; // Type-ahead is only safe after the TUI has booted at least once. return state.supportsTypeAhead && !state.awaitingFirstPrompt; diff --git a/src/utils/runtime-build-id.ts b/src/utils/runtime-build-id.ts new file mode 100644 index 000000000..670cfd3d9 --- /dev/null +++ b/src/utils/runtime-build-id.ts @@ -0,0 +1,101 @@ +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export interface RuntimeBuildEntry { + path: string; + content: string | Buffer; +} + +export type RuntimeBuildIdentity = + | { status: 'known'; id: string; source: 'artifact' | 'source' } + | { status: 'unknown'; reason: 'artifact_invalid' | 'source_unavailable' | 'read_failed' }; + +const BUILD_ID_RE = /^[a-f0-9]{64}$/; +let cachedIdentity: RuntimeBuildIdentity | undefined; + +export function computeRuntimeBuildId(entries: readonly RuntimeBuildEntry[]): string { + const hash = createHash('sha256'); + for (const entry of [...entries].sort((a, b) => a.path.localeCompare(b.path))) { + const path = entry.path.split(sep).join('/'); + const content = typeof entry.content === 'string' ? Buffer.from(entry.content) : entry.content; + hash.update(`${Buffer.byteLength(path)}:${path}:${content.byteLength}:`); + hash.update(content); + hash.update('\n'); + } + return hash.digest('hex'); +} + +export function isRuntimeBuildId(value: unknown): value is string { + return typeof value === 'string' && BUILD_ID_RE.test(value); +} + +function collectEntries(root: string, extension: '.ts' | '.js'): RuntimeBuildEntry[] { + const entries: RuntimeBuildEntry[] = []; + const visit = (directory: string): void => { + for (const item of readdirSync(directory, { withFileTypes: true })) { + const absolute = join(directory, item.name); + const rel = relative(root, absolute).split(sep).join('/'); + if (item.isDirectory()) { + if (rel === 'dashboard/web' || rel.startsWith('dashboard/web/')) continue; + visit(absolute); + } else if ( + item.isFile() + && item.name.endsWith(extension) + && !item.name.endsWith('.d.ts') + ) { + entries.push({ path: rel, content: readFileSync(absolute) }); + } + } + }; + visit(root); + return entries; +} + +export function collectCompiledRuntimeEntries( + sourceRoot: string, + distRoot: string, +): RuntimeBuildEntry[] { + return collectEntries(sourceRoot, '.ts').map(entry => { + const path = `${entry.path.slice(0, -3)}.js`; + return { path, content: readFileSync(join(distRoot, ...path.split('/'))) }; + }); +} + +export function resolveRuntimeBuildIdentity(options: { + artifactPath: string; + sourceRoot?: string; +}): RuntimeBuildIdentity { + if (existsSync(options.artifactPath)) { + try { + const id = readFileSync(options.artifactPath, 'utf8').trim(); + return isRuntimeBuildId(id) + ? { status: 'known', id, source: 'artifact' } + : { status: 'unknown', reason: 'artifact_invalid' }; + } catch { + return { status: 'unknown', reason: 'read_failed' }; + } + } + if (!options.sourceRoot || !existsSync(options.sourceRoot)) { + return { status: 'unknown', reason: 'source_unavailable' }; + } + try { + const entries = collectEntries(options.sourceRoot, '.ts'); + return entries.length > 0 + ? { status: 'known', id: computeRuntimeBuildId(entries), source: 'source' } + : { status: 'unknown', reason: 'source_unavailable' }; + } catch { + return { status: 'unknown', reason: 'read_failed' }; + } +} + +export function runtimeBuildIdentity(): RuntimeBuildIdentity { + if (cachedIdentity) return cachedIdentity; + const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); + cachedIdentity = resolveRuntimeBuildIdentity({ + artifactPath: join(projectRoot, 'dist', '.runtime-build-id'), + sourceRoot: join(projectRoot, 'src'), + }); + return cachedIdentity; +} diff --git a/src/worker.ts b/src/worker.ts index 4c7472627..4646220cc 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -45,6 +45,12 @@ import { shouldWaitForPostSessionStartPromptEvidence, shouldWriteNow, } from './utils/input-gate.js'; +import { + decideCodexRunnerFreshness, + CodexRunnerFreshnessInputQueue, + shouldHoldCodexRunnerInput, + type CodexRunnerFreshnessState, +} from './services/codex-runner-freshness.js'; import { canStartInjectionFlush, shouldDeferUserFlush, shouldFlushInjectionsFirst, type PendingInjection } from './core/inject-queue-policy.js'; import { decideRestartFollowup, settleDurableTurnForRestart } from './core/restart-followup-policy.js'; import { stripAnsiForLog, tailChars } from './utils/crash-log.js'; @@ -244,6 +250,11 @@ import { fetchDaemonIpc } from './core/daemon-ipc-auth.js'; import { withCodexAppContext } from './utils/codex-app-context.js'; import { resolveCodexAppFinalTurnIdentity } from './adapters/cli/codex-app-turn.js'; import { RunnerControlDecoder } from './adapters/cli/runner-control-channel.js'; +import { + normalizeAppRunnerFinalMarker, + normalizeCodexAppLifecycleEvent, + projectAppRunnerFinalIds, +} from './services/codex-app-runner-protocol.js'; import { hasMatchingManagedOriginCapability, managedOriginCapabilityPath, @@ -1066,6 +1077,12 @@ const IDLE_PROBE_INTERVAL_MS = 3_500; const IDLE_PROBE_MAX_ATTEMPTS = 24; let busyPatternIdleProbeTimer: ReturnType | null = null; let reattachIdleProbeTimer: ReturnType | null = null; +let codexRunnerFreshness: CodexRunnerFreshnessState = 'current'; +let persistCodexRunnerBuildOnReady = false; +let activeRestartAttemptId: string | undefined; +/** Distinguishes a replacement's synchronous ready signal (Riff) from a late + * idle callback emitted by the backend being torn down. */ +let replacementSpawnInProgress = false; /** The effectiveResume flag used by the most recent spawnCli call. Written * immediately after the two-tier fallback check so late-attach timers * (hermes, cursor, etc.) can read THE SAME semantics the spawn used, @@ -1465,11 +1482,46 @@ async function runStartupCommands(): Promise { idleDetector?.reset(); } -const pendingMessages: PendingCliInput[] = []; +const freshnessInputQueue = new CodexRunnerFreshnessInputQueue< + PendingCliInput, + Extract +>( + () => codexRunnerFreshness, + state => { codexRunnerFreshness = state; }, +); +const pendingMessages = freshnessInputQueue.normal; +/** Correlation ids that this worker actually wrote into the owned Codex App + * runner. Runner lifecycle/final markers may route only to ids in this bounded + * local set; model/user display bytes cannot add entries. */ +const submittedCodexAppReplyTurnIds = new Set(); +const pendingCodexAppSteerAckIds = new Map(); +const acknowledgedCodexAppSteers = new Set(); +const CODEX_APP_CORRELATION_LIMIT = 256; +function rememberBounded(set: Set, value: string): void { + set.delete(value); + set.add(value); + while (set.size > CODEX_APP_CORRELATION_LIMIT) { + const oldest = set.values().next().value; + if (typeof oldest !== 'string') break; + set.delete(oldest); + } +} +function rememberBoundedMap(map: Map, key: string, value: string): void { + map.delete(key); + map.set(key, value); + while (map.size > CODEX_APP_CORRELATION_LIMIT) { + const oldest = map.keys().next().value; + if (typeof oldest !== 'string') break; + map.delete(oldest); + } +} +function shortCorrelationId(value: string | undefined): string { + return value?.slice(0, 12) ?? '-'; +} /** Literal commands that arrived while native /rename owned the TUI or while * an owned CLI restart was fenced. Normal raw_input commands are still * delivered immediately (including while busy). */ -const pendingRawInputs: Array> = []; +const pendingRawInputs = freshnessInputQueue.raw; /** Latest requested canonical session title. Unlike a normal prompt this is an * administrative TUI command: never type-ahead while the agent is busy, never * open a model turn, and latest-wins if several renames arrive before idle. */ @@ -4832,9 +4884,89 @@ function handleCodexAppMarker(body: string): void { return; } - if (kind === 'final' && typeof payload.content === 'string') { - const startedAtMs = typeof payload.startedAtMs === 'number' ? payload.startedAtMs : undefined; - const completedAtMs = typeof payload.completedAtMs === 'number' ? payload.completedAtMs : Date.now(); + if (kind === 'lifecycle') { + const event = normalizeCodexAppLifecycleEvent(payload); + if (!event) { + log(`${cliName()} rejected malformed lifecycle marker`); + return; + } + log( + `${cliName()} lifecycle kind=${event.kind}` + + ` appTurn=${shortCorrelationId(event.appTurnId)}` + + ` replyTurn=${shortCorrelationId(event.replyTurnId)}` + + `${event.queueLength !== undefined ? ` queue=${event.queueLength}` : ''}`, + ); + if (!event.replyTurnId || !submittedCodexAppReplyTurnIds.has(event.replyTurnId)) return; + if (event.kind === 'steer_attempt') { + rememberBoundedMap(pendingCodexAppSteerAckIds, event.replyTurnId, event.appTurnId); + return; + } + const steerKey = `${event.appTurnId}\0${event.replyTurnId}`; + if (event.kind !== 'steer_accepted' + || pendingCodexAppSteerAckIds.get(event.replyTurnId) !== event.appTurnId + || acknowledgedCodexAppSteers.has(steerKey)) return; + pendingCodexAppSteerAckIds.delete(event.replyTurnId); + rememberBounded(acknowledgedCodexAppSteers, steerKey); + send({ + type: 'steer_accepted', + appTurnId: event.appTurnId, + turnId: event.replyTurnId, + }); + return; + } + + if (kind === 'final') { + const marker = normalizeAppRunnerFinalMarker(payload); + if (!marker) { + log(`${cliName()} rejected malformed final marker`); + return; + } + const startedAtMs = marker.startedAtMs; + const completedAtMs = marker.completedAtMs ?? Date.now(); + if (marker.appTurnId) { + const trustedReplyTurnId = marker.replyTurnId + && submittedCodexAppReplyTurnIds.has(marker.replyTurnId) + ? marker.replyTurnId + : undefined; + if (marker.replyTurnId && !trustedReplyTurnId) { + log( + `${cliName()} ignored unsubmitted final reply route ` + + `(replyTurn=${shortCorrelationId(marker.replyTurnId)})`, + ); + } + const identity = projectAppRunnerFinalIds( + { ...marker, replyTurnId: trustedReplyTurnId }, + currentBotmuxTurnId, + `${lastInitConfig?.cliId ?? 'app'}-${Date.now()}`, + ); + if (marker.appTurnId !== identity.turnId) { + log( + `${cliName()} app turn ${shortCorrelationId(marker.appTurnId)} mapped ` + + `to botmux turn ${shortCorrelationId(identity.turnId)}`, + ); + } + const dispatchAttempt = currentBotmuxDispatchAttempt; + if (startedAtMs !== undefined && shouldSuppressBridgeEmit( + { markTimeMs: startedAtMs, isLocal: false, finalText: marker.content }, + completedAtMs + 5_001, + readSendMarkers(), + false, + )) { + log(`${cliName()} final_output suppressed (model already called botmux send)`); + emitTurnTerminal(identity.turnId, 'completed', undefined, dispatchAttempt); + return; + } + send({ + type: 'final_output', + content: marker.content, + lastUuid: identity.lastUuid, + turnId: identity.turnId, + ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), + }); + emitTurnTerminal(identity.turnId, 'completed', undefined, dispatchAttempt); + return; + } + // Codex App keeps the app-server-generated id separately for diagnostics. // Routing must use its stable clientUserMessageId marker; legacy envelopes // intentionally omit it and fall back to the worker's frozen botmux turn. @@ -4868,7 +5000,7 @@ function handleCodexAppMarker(body: string): void { const dispatchAttempt = currentBotmuxDispatchAttempt; if (startedAtMs !== undefined) { const sentByModel = shouldSuppressBridgeEmit( - { markTimeMs: startedAtMs, isLocal: false, finalText: payload.content }, + { markTimeMs: startedAtMs, isLocal: false, finalText: marker.content }, completedAtMs + 5_001, readSendMarkers(), false, @@ -4881,7 +5013,7 @@ function handleCodexAppMarker(body: string): void { } send({ type: 'final_output', - content: payload.content, + content: marker.content, lastUuid: turnId, turnId, ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), @@ -5040,6 +5172,10 @@ function markPromptReadyFromPty(): void { function markPromptReady(): void { if (isPromptReady) return; // guard against duplicate calls + if (cliRestartInProgress && !replacementSpawnInProgress) { + log('Ignoring prompt-ready from backend generation being replaced'); + return; + } stopBusyPatternIdleProbe(); // Ready-gate: a startup selector's ❯ (cjadk et al.) falsely matches // readyPattern → the IdleDetector fires idle while the CLI is NOT actually at @@ -5069,6 +5205,13 @@ function markPromptReady(): void { log('Idle detected during ready-gate settle; deferring prompt-ready until settle completes'); return; } + const freshnessAction = freshnessInputQueue.onPromptReady(); + if (freshnessAction === 'reload') { + log('Stale Codex App runner became idle; replacing it before releasing queued input'); + void restartCliProcess('stale runner reached idle', { immediate: true, preservePending: true }); + return; + } + if (freshnessAction === 'ignore') return; isPromptReady = true; clearSessionRenameInFlight(); // An old backend can still report idle while its async teardown is running. @@ -5098,6 +5241,34 @@ function markPromptReady(): void { renderer?.markNewTurn(); // exclude history replay from streaming card } send({ type: 'prompt_ready' }); + if ( + persistCodexRunnerBuildOnReady + && lastInitConfig?.cliId === 'codex-app' + && lastInitConfig.runnerBuildId + ) { + send({ type: 'runner_build_ready', runnerBuildId: lastInitConfig.runnerBuildId }); + persistCodexRunnerBuildOnReady = false; + } + if (activeRestartAttemptId) { + // Defense in depth: only report a successful restart when a replacement + // backend is actually installed. Every legitimate ready path assigns + // `backend` before firing (spawnCli sets it, then idle/PTY callbacks run); + // a stray callback that reached here with no backend (e.g. a late stale + // generation slipping through a future gate change) must NOT claim success + // and consume the attempt id — leave it for the real replacement or the + // coordinator timeout. + if (backend) { + send({ + type: 'restart_result', + attemptId: activeRestartAttemptId, + status: 'succeeded', + category: 'prompt_ready', + }); + activeRestartAttemptId = undefined; + } else { + log('prompt-ready with no backend installed — deferring restart success receipt'); + } + } // Send immediate idle snapshot so Lark card reflects idle status. // BUT: skip when messages are pending — flushPending() will immediately // make the CLI busy, so the idle state is transient and shouldn't appear @@ -5417,6 +5588,7 @@ async function flushPending(): Promise { // old CLI. Never let a new flush (including one triggered by the old // backend's idle/task-done callback) write across that restart boundary. if (cliRestartInProgress) return; + if (shouldHoldCodexRunnerInput(codexRunnerFreshness)) return; if (isFlushing) return; // while loop in active flush will pick up new messages if (!backend || !cliAdapter) return; if (pendingMessages.length === 0 && pendingRawInputs.length === 0 && pendingSessionRename === null) return; // nothing to flush — keep isPromptReady @@ -5545,7 +5717,8 @@ async function flushPending(): Promise { // rename. Some passthroughs (/clear, /new) can rotate the native session; // applying the canonical title last keeps the resume-picker label aligned. if (rawInputReady && pendingRawInputs.length > 0 && backend) { - const raw = pendingRawInputs.shift()!; + const raw = freshnessInputQueue.takeRaw(); + if (!raw) return; await deliverRawInput(raw); return; } @@ -5573,7 +5746,8 @@ async function flushPending(): Promise { return; } while (pendingMessages.length > 0 && backend && cliAdapter) { - const item = pendingMessages.shift()!; + const item = freshnessInputQueue.takeNormal(); + if (!item) break; const durableWrite = item.dispatchAttempt !== undefined; if (durableWrite) durableTurnInFlight = true; // Track as in-flight until the CLI returns to idle (markPromptReady). @@ -5633,7 +5807,14 @@ async function flushPending(): Promise { log('Refused durable Claude submit: transcript terminal bridge is unavailable'); break; } - log(`Writing to PTY (flush): "${msg.substring(0, 80)}"`); + if (lastInitConfig?.cliId === 'codex-app') { + log( + `Writing Codex App input to PTY (flush): ` + + `replyTurn=${shortCorrelationId(item.turnId)} chars=${msg.length}`, + ); + } else { + log(`Writing to PTY (flush): "${msg.substring(0, 80)}"`); + } // Defense in depth: TmuxPipeBackend's send methods no longer throw on a // dead pane (they fire onExit instead), but writeInput can still throw // for other reasons (fs errors while resolving the JSONL, a future @@ -5651,9 +5832,14 @@ async function flushPending(): Promise { await codexRpcEngine.sendTurn(msg, item.turnId); result = { submitted: true }; } else if (item.codexAppInput && cliAdapter.writeStructuredInput) { - result = await cliAdapter.writeStructuredInput(backend, msg, item.codexAppInput); + result = await cliAdapter.writeStructuredInput( + backend, + msg, + item.codexAppInput, + { turnId: item.turnId }, + ); } else { - result = await cliAdapter.writeInput(backend, msg); + result = await cliAdapter.writeInput(backend, msg, { turnId: item.turnId }); } scheduleBusyPatternIdleProbe(`${cliName()} post-submit`); } catch (err: any) { @@ -5679,6 +5865,11 @@ async function flushPending(): Promise { ); break; } + if (lastInitConfig?.cliId === 'codex-app' + && item.turnId + && result?.submitted !== false) { + rememberBounded(submittedCodexAppReplyTurnIds, item.turnId); + } // Persist any sessionId the adapter observed via authoritative sources // (Claude's pid file, Codex's history). Done independently of submit // outcome — the rotation is real even when the current Enter didn't @@ -5750,7 +5941,7 @@ function sendToPty( // old early-return silently dropped it after receiver had already persisted // DISPATCHED. if (cliRestartInProgress || !backend) { - pendingMessages.push(next); + freshnessInputQueue.enqueueNormal(next); log(`Queued message while CLI backend is restarting (${pendingMessages.length} pending)`); return true; } @@ -5765,12 +5956,13 @@ function sendToPty( isFlushing, supportsTypeAhead, awaitingFirstPrompt, + holdForRunnerReload: shouldHoldCodexRunnerInput(codexRunnerFreshness), }) && cliAdapter.mergeQueuedInput === true; const mergedQueued = shouldMergeQueued && mergeQueuedCliInput(pendingMessages, next); if (mergedQueued) { log(`Merged queued message (${pendingMessages.length} pending): "${content.substring(0, 80)}" — ${cliName()} ${awaitingFirstPrompt ? 'still booting' : 'is busy'}`); } else { - pendingMessages.push(next); + freshnessInputQueue.enqueueNormal(next); } // User-override semantics: a fresh Lark message while a TUI prompt is "active" // takes precedence over the AI-detected prompt. The screen analyzer can be @@ -5800,7 +5992,11 @@ function sendToPty( // delivers queued messages instead. See input-gate.ts; this fixes dispatch's // brief reaching Codex before its first idle and never landing. if (!sessionRenameInFlight && commandLineWritesPending === 0 && shouldWriteNow({ - isPromptReady, isFlushing, supportsTypeAhead, awaitingFirstPrompt, + isPromptReady, + isFlushing, + supportsTypeAhead, + awaitingFirstPrompt, + holdForRunnerReload: shouldHoldCodexRunnerInput(codexRunnerFreshness), })) { if (!mergedQueued) log(`Writing to PTY: "${content.substring(0, 80)}"`); flushPending(); // fire-and-forget async; no-op if already flushing @@ -6704,6 +6900,28 @@ async function spawnCli( } } + const replacementExpectedFresh = codexRunnerFreshness === 'restarting_fresh'; + const freshness = decideCodexRunnerFreshness({ + cliId: cfg.cliId, + adoptMode: cfg.adoptMode === true, + persistentReattach: willReattachPersistent, + replacementExpectedFresh, + currentBuildId: cfg.runnerBuildId, + persistedBuildId: cfg.persistedRunnerBuildId, + }); + codexRunnerFreshness = freshness.state; + persistCodexRunnerBuildOnReady = freshness.persistOnReady; + log(`Codex runner freshness=${freshness.state} reason=${freshness.reason}`); + if (freshness.reason === 'replacement_reattached' && activeRestartAttemptId) { + send({ + type: 'restart_result', + attemptId: activeRestartAttemptId, + status: 'failed', + category: 'spawn_failed', + }); + activeRestartAttemptId = undefined; + } + // The plugin set is stable only for the lifetime of one real CLI process. // A warm worker reattach keeps the existing Gateway and catalog untouched; // every fresh/resumed CLI spawn atomically refreshes both from current Bot config. @@ -8038,6 +8256,15 @@ async function spawnCli( // prompt-ready — without this hook a follow-up arriving mid-task would sit // in pendingMessages forever once the task finishes. backend.onTaskDone?.(() => { + // Generation fence (matches the onAgentStatus above and the onExit below): + // a stale RiffBackend's `fetchAndEmitOutput(...).finally(taskDoneCb)` can + // resolve AFTER destroySession()/kill() during a restart (neither clears + // taskDoneCb nor awaits the in-flight fetch). If that late callback reached + // markPromptReady() while a replacement is spawning, it would ride through + // the global restart gate and emit a premature `restart_result: succeeded`, + // swallowing the replacement's true terminal outcome. Only the current + // backend generation may re-arm prompt-ready. + if (backend !== observedBackend) return; log(`${cliName()} task finished — re-arming prompt-ready for queued follow-ups`); markPromptReady(); }); @@ -8123,6 +8350,16 @@ async function spawnCli( isPromptReady = false; currentBotmuxTurnId = undefined; currentBotmuxDispatchAttempt = undefined; + if (!intentionalRestart && activeRestartAttemptId) { + send({ + type: 'restart_result', + attemptId: activeRestartAttemptId, + status: 'failed', + category: 'runner_exited', + }); + activeRestartAttemptId = undefined; + } + if (!intentionalRestart) freshnessInputQueue.onReplacementFailed(); if (intentionalRestart) { log('Suppressed claude_exit for intentional in-worker restart'); } else { @@ -8254,6 +8491,9 @@ function killCli(opts: { preservePending?: boolean } = {}): void { currentBotmuxTurnId = undefined; currentBotmuxDispatchAttempt = undefined; currentVcMeetingImTurnOrigin = undefined; + submittedCodexAppReplyTurnIds.clear(); + pendingCodexAppSteerAckIds.clear(); + acknowledgedCodexAppSteers.clear(); if (sandboxCleanup) { try { sandboxCleanup(); } catch { /* */ } sandboxCleanup = null; @@ -8293,6 +8533,7 @@ async function restartCliProcess( // of firing idle/task-done callbacks. Inputs accepted in that interval must // remain queued until a replacement backend has been installed. cliRestartInProgress = true; + replacementSpawnInProgress = false; rawInputRestartGate = true; // The Node worker stays alive through this restart, so the daemon will see // neither claude_exit nor worker-exit. Explicitly revoke the old turn's @@ -8353,16 +8594,20 @@ async function restartCliProcess( rpcPluginGenerationPrepared = true; await engageCodexRpc(restartCfg); } + replacementSpawnInProgress = true; await spawnCli(restartCfg, { pluginGenerationPrepared: rpcPluginGenerationPrepared }); await prepareCodexNativeTitleGeneration(restartCfg, codexRpcEngine); + replacementSpawnInProgress = false; if (codexRpcEngine) armRpcStartupDialogDismiss(); } catch (err) { + replacementSpawnInProgress = false; cliRestartInProgress = false; await sendFatalWorkerErrorAndExit(err); return; } } cliRestartInProgress = false; + replacementSpawnInProgress = false; // Follow-up decision (pure, unit-tested in restart-followup-policy.ts): // - cwd-move: a role-switch restart landed after restartCfg was // snapshotted → CLI came up in the old cwd while daemon repinned to @@ -8401,6 +8646,7 @@ async function restartCliProcess( if (isPromptReady) void flushPending(); }, 500); } catch (err) { + replacementSpawnInProgress = false; cliRestartInProgress = false; try { await sendFatalWorkerErrorAndExit(err); @@ -9619,6 +9865,15 @@ async function sendFatalWorkerErrorAndExit( ): Promise { if (fatalWorkerErrorPending) return; fatalWorkerErrorPending = true; + if (activeRestartAttemptId) { + await sendAndFlush({ + type: 'restart_result', + attemptId: activeRestartAttemptId, + status: 'failed', + category: 'spawn_failed', + }); + activeRestartAttemptId = undefined; + } await sendAndFlush({ type: 'error', message: err instanceof Error ? err.message : String(err), @@ -9645,6 +9900,7 @@ process.on('message', async (raw: unknown) => { const initStartedAtMs = Date.now(); if (lastInitConfig) return; // already initialized lastInitConfig = msg; + activeRestartAttemptId = msg.restartAttemptId; sessionId = msg.sessionId; refreshTerminalViewToken(); refreshTerminalWriteToken(); @@ -9998,9 +10254,10 @@ process.on('message', async (raw: unknown) => { // 覆盖"已确认裸 shell 启动失败"两种状态,一并入队。裸 shell 确认失败后 // 这些 pending 不会再被 flush(与 pendingMessages 同款处理),符合预期。 if (cliRestartInProgress || rawInputRestartGate || sessionRenameInFlight + || shouldHoldCodexRunnerInput(codexRunnerFreshness) || injectionFlushing || shouldDeferUserFlush(pendingInjections) || bareShellCheckInProgress || bareShellLaunchBlocked) { - pendingRawInputs.push(msg); + freshnessInputQueue.enqueueRaw(msg); log(`Deferred passthrough slash command until CLI input gate settles: ${msg.content}`); } else { await deliverRawInput(msg); @@ -10059,6 +10316,8 @@ process.on('message', async (raw: unknown) => { log(`Restart request merged into in-flight restart${msg.updateWorkingDir ? ` (workingDir → ${msg.updateWorkingDir})` : ''}`); break; } + activeRestartAttemptId = msg.attemptId; + codexRunnerFreshness = 'restarting_fresh'; // restart 杀死 CLI,在飞的 durable turn 随之死亡。对被杀的那次投递,主动发一个 // 'ambiguous' 终端回执:CLI 被中途杀掉,副作用到底发没发是**真的无法证明** // (故不能报 'cancelled'),交由 daemon 的重试策略即时对账。不发的话 receipt 会 diff --git a/test/card-integration.test.ts b/test/card-integration.test.ts index f5bf3943d..9e1cb6315 100644 --- a/test/card-integration.test.ts +++ b/test/card-integration.test.ts @@ -131,6 +131,10 @@ vi.mock('../src/core/worker-pool.js', async (importOriginal) => { forkWorker: vi.fn(), killWorker: vi.fn(), initWorkerPool: vi.fn(), + requestSessionRestart: vi.fn((_ds: any, observer: any) => { + void observer.notify('in_progress'); + return { attemptId: 'attempt-card', joined: false }; + }), }; }); @@ -163,7 +167,7 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ import { handleCardAction, type CardHandlerDeps } from '../src/im/lark/card-handler.js'; import { scheduleCardPatch } from '../src/core/worker-pool.js'; -import { killWorker, forkWorker } from '../src/core/worker-pool.js'; +import { killWorker, forkWorker, requestSessionRestart } from '../src/core/worker-pool.js'; import { sessionKey } from '../src/core/types.js'; import type { DaemonSession } from '../src/core/types.js'; import { buildStreamingCard } from '../src/im/lark/card-builder.js'; @@ -471,7 +475,7 @@ describe('Card integration: full event flow', () => { await handleCardAction(makeRestartEvent(ROOT_ID), deps, APP_ID); - expect(workerSend).toHaveBeenCalledWith({ type: 'restart' }); + expect(requestSessionRestart).toHaveBeenCalledWith(ds, expect.objectContaining({ source: 'card' })); // The confirmation is delivered ephemeral to the clicker (group chat + an // operator open_id), not as a visible group reply. expect(vi.mocked(clientMod.sendEphemeralCard)).toHaveBeenCalledWith( @@ -488,7 +492,7 @@ describe('Card integration: full event flow', () => { await handleCardAction(makeRestartEvent(ROOT_ID), deps, APP_ID); - expect(forkWorker).toHaveBeenCalledWith(ds, '', false); + expect(requestSessionRestart).toHaveBeenCalledWith(ds, expect.objectContaining({ source: 'card' })); }); it('close should kill worker and remove session', async () => { @@ -788,7 +792,7 @@ describe('Card integration: full event flow', () => { await handleCardAction(makeRestartEvent(ROOT_ID), deps, APP_ID); - expect(workerSend).toHaveBeenCalledWith({ type: 'restart' }); + expect(requestSessionRestart).toHaveBeenCalledWith(ds, expect.objectContaining({ source: 'card' })); expect(vi.mocked(clientMod.sendEphemeralCard)).not.toHaveBeenCalled(); expect(deps.sessionReply).toHaveBeenCalledWith( ROOT_ID, expect.stringContaining('重启'), undefined, APP_ID, diff --git a/test/codex-app-runner-protocol.test.ts b/test/codex-app-runner-protocol.test.ts new file mode 100644 index 000000000..130f2b10e --- /dev/null +++ b/test/codex-app-runner-protocol.test.ts @@ -0,0 +1,152 @@ +import { Buffer } from 'node:buffer'; +import { describe, expect, it } from 'vitest'; +import { encodeRunnerInput } from '../src/adapters/cli/runner-input.js'; +import { + CODEX_APP_INPUT_PREFIX, + decodeCodexAppRunnerInput, + normalizeAppRunnerFinalMarker, + normalizeCodexAppLifecycleEvent, + projectAppRunnerFinalIds, +} from '../src/services/codex-app-runner-protocol.js'; + +function encodedLine(value: unknown): string { + return `${CODEX_APP_INPUT_PREFIX}${Buffer.from(JSON.stringify(value), 'utf8').toString('base64')}`; +} + +describe('Codex App runner input protocol', () => { + it('round-trips structured input and an independent reply turn id', () => { + const line = `${CODEX_APP_INPUT_PREFIX}${encodeRunnerInput( + 'legacy prompt', + { + text: 'clean user text', + additionalContext: { + botmux_sender: { kind: 'untrusted', value: 'Alice' }, + }, + }, + 'om_lark_message', + )}`; + + expect(decodeCodexAppRunnerInput(line)).toEqual({ + type: 'message', + content: 'legacy prompt', + codexAppInput: { + text: 'clean user text', + additionalContext: { + botmux_sender: { kind: 'untrusted', value: 'Alice' }, + }, + }, + replyTurnId: 'om_lark_message', + }); + }); + + it('keeps legacy envelopes without correlation or a sidecar valid', () => { + expect(decodeCodexAppRunnerInput(encodedLine({ + type: 'message', + content: 'legacy', + }))).toEqual({ + type: 'message', + content: 'legacy', + }); + }); + + it('uses the legacy sidecar client id when the top-level correlation field is absent', () => { + expect(decodeCodexAppRunnerInput(encodedLine({ + type: 'message', + content: 'legacy', + codexAppInput: { + text: 'clean', + clientUserMessageId: 'om_legacy_sidecar', + }, + }))).toMatchObject({ + replyTurnId: 'om_legacy_sidecar', + }); + }); + + it.each([ + 'not-a-control-line', + `${CODEX_APP_INPUT_PREFIX}not-json`, + encodedLine({ type: 'other', content: 'x' }), + encodedLine({ type: 'message', content: 'x', replyTurnId: 42 }), + encodedLine({ type: 'message', content: 'x', replyTurnId: '' }), + encodedLine({ type: 'message', content: 'x', hidden: true }), + encodedLine({ type: 'message', content: 'x', codexAppInput: { text: 42 } }), + ])('rejects malformed external input: %s', line => { + expect(decodeCodexAppRunnerInput(line)).toBeUndefined(); + }); +}); + +describe('app runner final marker normalization', () => { + it('keeps app-server and Feishu ids in separate fields', () => { + expect(normalizeAppRunnerFinalMarker({ + appTurnId: 'app-turn-1', + replyTurnId: 'om_follow_up', + content: 'done', + startedAtMs: 10, + completedAtMs: 20, + })).toEqual({ + appTurnId: 'app-turn-1', + replyTurnId: 'om_follow_up', + legacyTurnId: undefined, + content: 'done', + startedAtMs: 10, + completedAtMs: 20, + }); + }); + + it('projects new ids separately while preserving legacy single-id markers', () => { + expect(projectAppRunnerFinalIds({ + appTurnId: 'app-turn-1', + replyTurnId: 'om_follow_up', + content: 'done', + }, 'om_latest', 'generated')).toEqual({ + lastUuid: 'app-turn-1', + turnId: 'om_follow_up', + }); + expect(projectAppRunnerFinalIds({ + legacyTurnId: 'legacy-turn', + content: 'done', + }, 'om_latest', 'generated')).toEqual({ + lastUuid: 'legacy-turn', + turnId: 'legacy-turn', + }); + }); + + it('rejects malformed final markers', () => { + expect(normalizeAppRunnerFinalMarker({ content: 42, appTurnId: 'x' })).toBeUndefined(); + expect(normalizeAppRunnerFinalMarker(null)).toBeUndefined(); + }); +}); + +describe('Codex App lifecycle event normalization', () => { + it('accepts a safe steer acceptance event', () => { + expect(normalizeCodexAppLifecycleEvent({ + kind: 'steer_accepted', + atMs: 123, + appTurnId: 'app-turn-1', + replyTurnId: 'om_follow_up', + queueLength: 0, + })).toEqual({ + kind: 'steer_accepted', + atMs: 123, + appTurnId: 'app-turn-1', + replyTurnId: 'om_follow_up', + queueLength: 0, + }); + }); + + it.each([ + null, + { kind: 'steer_accepted', atMs: 'now', appTurnId: 'app-turn-1' }, + { kind: 'steer_accepted', atMs: 123, appTurnId: '' }, + { kind: 'steer_accepted', atMs: 123, appTurnId: 'app-turn-1', queueLength: -1 }, + { kind: 'not-a-lifecycle-event', atMs: 123 }, + { + kind: 'steer_accepted', + atMs: 123, + appTurnId: 'app-turn-1', + content: 'must never cross the lifecycle boundary', + }, + ])('rejects malformed or content-bearing lifecycle payloads: %j', payload => { + expect(normalizeCodexAppLifecycleEvent(payload)).toBeUndefined(); + }); +}); diff --git a/test/codex-app-runner.integration.test.ts b/test/codex-app-runner.integration.test.ts index d961f20cd..67db42884 100644 --- a/test/codex-app-runner.integration.test.ts +++ b/test/codex-app-runner.integration.test.ts @@ -18,6 +18,7 @@ const RUNNER_PATH = resolve('src/codex-app-runner.ts'); const FAKE_SERVER_FIXTURE = resolve('test/fixtures/fake-codex-app-server.mjs'); const CONTROL_PREFIX = '::botmux-codex-app:'; const FINAL_MARKER = /\x1b\]777;botmux:final:([A-Za-z0-9+/=]+)\x07/; +const LIFECYCLE_MARKER = /\x1b\]777;botmux:lifecycle:([A-Za-z0-9+/=]+)\x07/g; interface Harness { child: ChildProcessWithoutNullStreams; @@ -121,6 +122,13 @@ function decodeFinalMarker(output: string): Record { return JSON.parse(Buffer.from(match[1], 'base64').toString('utf8')); } +function decodeLifecycleMarkers(output: string): Array<{ payload: Record; index: number }> { + return [...output.matchAll(LIFECYCLE_MARKER)].map(match => ({ + payload: JSON.parse(Buffer.from(match[1], 'base64').toString('utf8')), + index: match.index, + })); +} + function readRequests(logPath: string): Array> { if (!existsSync(logPath)) return []; return readFileSync(logPath, 'utf8') @@ -208,8 +216,8 @@ describe('codex-app-runner app-server protocol integration', () => { expect(JSON.stringify(turns[0].params)).not.toContain('legacy prompt'); expect(result.output).toContain(`skipped unreadable local image: ${result.missingImagePath}`); expect(result.final.content).toBe('fake answer 1'); - expect(result.final.turnId).toBe('om_integration_123'); - expect(result.final.nativeTurnId).toBe('turn-fake-1'); + expect(result.final.replyTurnId).toBe('om_integration_123'); + expect(result.final.appTurnId).toBe('turn-fake-1'); }); it('preserves the full legacy prompt on codex < 0.135 even if the server would ignore new fields', async () => { @@ -224,8 +232,8 @@ describe('codex-app-runner app-server protocol integration', () => { expect(result.output).toContain('clean input requires codex >= 0.135.0 (found 0.134.9); using legacy prompt'); // Even when the app-server cannot receive the new field, the runner still // preserves the daemon-frozen logical identity from its sidecar. - expect(result.final.turnId).toBe('om_integration_123'); - expect(result.final.nativeTurnId).toBe('turn-fake-1'); + expect(result.final.replyTurnId).toBe('om_integration_123'); + expect(result.final.appTurnId).toBe('turn-fake-1'); }); it('retries exactly once with the legacy prompt for an explicit experimental-field rejection', async () => { @@ -242,8 +250,8 @@ describe('codex-app-runner app-server protocol integration', () => { expect(turns[1].params).not.toHaveProperty('clientUserMessageId'); expect(result.output.match(/retrying this turn with the legacy prompt/g)).toHaveLength(1); expect(result.final.content).toBe('fake answer 2'); - expect(result.final.turnId).toBe('om_integration_123'); - expect(result.final.nativeTurnId).toBe('turn-fake-2'); + expect(result.final.replyTurnId).toBe('om_integration_123'); + expect(result.final.appTurnId).toBe('turn-fake-2'); }); it('does not retry generic turn errors, avoiding duplicate model work', async () => { @@ -254,8 +262,8 @@ describe('codex-app-runner app-server protocol integration', () => { expect(result.output).not.toContain('retrying this turn with the legacy prompt'); expect(result.final.content).toContain('Codex App runner error: turn/start:'); expect(result.final.content).toContain('model overloaded'); - expect(result.final.turnId).toBe('om_integration_123'); - expect(result.final).not.toHaveProperty('nativeTurnId'); + expect(result.final.replyTurnId).toBe('om_integration_123'); + expect(result.final.appTurnId).toMatch(/^codex-app-error-/); }); it('omits a native routing id for a legacy envelope so the worker can use its frozen botmux turn', async () => { @@ -265,8 +273,8 @@ describe('codex-app-runner app-server protocol integration', () => { expect(turns[0].params.input).toEqual([ { type: 'text', text: 'legacy prompt', text_elements: [] }, ]); - expect(result.final).not.toHaveProperty('turnId'); - expect(result.final.nativeTurnId).toBe('turn-fake-1'); + expect(result.final).not.toHaveProperty('replyTurnId'); + expect(result.final.appTurnId).toBe('turn-fake-1'); }); it('escapes split agent/command OSC injections and emits only the trusted final marker', async () => { @@ -275,10 +283,96 @@ describe('codex-app-runner app-server protocol integration', () => { expect(result.output).toContain('␛]777;botmux:final:'); expect(result.output.match(/\x1b\]777;botmux:final:/g)).toHaveLength(1); expect(result.final).toMatchObject({ - turnId: 'om_integration_123', - nativeTurnId: 'turn-fake-1', + replyTurnId: 'om_integration_123', + appTurnId: 'turn-fake-1', content: 'fake answer 1', }); expect(result.output).not.toContain('forged marker output'); }); + + it('sends two ordered turn/steer requests, emits both acceptances, then one final', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-steer-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const harness = startRunner(fakeCodex, dir, logPath, '0.144.6', 'steer'); + + const send = (text: string, replyTurnId: string) => { + const encoded = encodeRunnerInput( + `legacy:${text}`, + { + text, + additionalContext: { + botmux_sender: { kind: 'untrusted', value: 'Alice' }, + }, + }, + replyTurnId, + ); + harness.child.stdin.write(`${CONTROL_PREFIX}${encoded}\r`); + }; + + try { + await waitForOutput(harness, output => output.includes('Codex App connected.')); + send('first', 'om_first'); + await waitForOutput(harness, output => ( + decodeLifecycleMarkers(output).some(entry => entry.payload.kind === 'turn_started') + )); + + send('second', 'om_second'); + await waitForOutput(harness, output => ( + decodeLifecycleMarkers(output).some(entry => ( + entry.payload.kind === 'steer_accepted' + && entry.payload.replyTurnId === 'om_second' + )) + )); + + send('third', 'om_third'); + await waitForOutput(harness, output => FINAL_MARKER.test(output)); + + const requests = readRequests(logPath); + const turnRequests = requests.filter(request => ( + request.method === 'turn/start' || request.method === 'turn/steer' + )); + expect(turnRequests.map(request => request.method)).toEqual([ + 'turn/start', + 'turn/steer', + 'turn/steer', + ]); + expect(turnRequests[1].params).toMatchObject({ + expectedTurnId: 'turn-fake-1', + clientUserMessageId: 'om_second', + input: [{ type: 'text', text: 'second', text_elements: [] }], + additionalContext: { + botmux_sender: { kind: 'untrusted', value: 'Alice' }, + }, + }); + expect(turnRequests[2].params).toMatchObject({ + expectedTurnId: 'turn-fake-1', + clientUserMessageId: 'om_third', + input: [{ type: 'text', text: 'third', text_elements: [] }], + }); + + const lifecycle = decodeLifecycleMarkers(harness.stdout); + expect(lifecycle.filter(entry => entry.payload.kind === 'steer_accepted').map(entry => ( + entry.payload.replyTurnId + ))).toEqual(['om_second', 'om_third']); + const final = decodeFinalMarker(harness.stdout); + expect(final).toMatchObject({ + appTurnId: 'turn-fake-1', + replyTurnId: 'om_third', + content: 'fake answer 1', + }); + const finalIndex = harness.stdout.search(FINAL_MARKER); + const lastAccepted = lifecycle.find(entry => ( + entry.payload.kind === 'steer_accepted' + && entry.payload.replyTurnId === 'om_third' + )); + expect(lastAccepted?.index).toBeLessThan(finalIndex); + expect(harness.stdout.match(/\x1b\]777;botmux:final:/g)).toHaveLength(1); + } finally { + await stopChild(harness.child); + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/test/codex-app-turn-controller.test.ts b/test/codex-app-turn-controller.test.ts new file mode 100644 index 000000000..e68c1dbba --- /dev/null +++ b/test/codex-app-turn-controller.test.ts @@ -0,0 +1,307 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + CodexAppRpcResponseError, + CodexAppTransportError, + CodexAppTurnController, +} from '../src/services/codex-app-turn-controller.js'; +import type { + CodexAppFinalMarker, + CodexAppLifecycleEvent, + CodexAppRunnerInput, +} from '../src/services/codex-app-runner-protocol.js'; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +async function flushAsync(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +function createHarness() { + const requests: Array<{ + method: string; + params: any; + response: Deferred; + }> = []; + const finals: Array = []; + const diagnostics: string[] = []; + const lifecycle: CodexAppLifecycleEvent[] = []; + const displayed: string[] = []; + let now = 100; + const controller = new CodexAppTurnController({ + cwd: '/repo', + ensureThread: async () => 'thread-1', + request: vi.fn((method: string, params: unknown) => { + const response = deferred(); + requests.push({ method, params, response }); + return response.promise; + }), + prepareInput(input: CodexAppRunnerInput, structuredDisabled: boolean) { + const structured = !!input.codexAppInput && !structuredDisabled; + const text = structured ? input.codexAppInput!.text : input.content; + return { + input: [ + { type: 'text', text, text_elements: [] }, + ...(structured + ? (input.codexAppInput?.localImages ?? []).map(image => ({ + type: 'localImage', + path: image.path, + ...(image.detail ? { detail: image.detail } : {}), + })) + : []), + ], + ...(structured && input.codexAppInput?.additionalContext + ? { additionalContext: input.codexAppInput.additionalContext } + : {}), + ...(input.replyTurnId ? { clientUserMessageId: input.replyTurnId } : {}), + visibleText: input.codexAppInput?.text ?? input.content, + structured, + }; + }, + isStartCapabilityError: error => ( + error instanceof CodexAppRpcResponseError + && /additionalContext/.test(error.message) + ), + onTurnInput: (_input, prepared) => displayed.push(prepared.visibleText), + onFinal: marker => finals.push(marker), + onDiagnostic: message => diagnostics.push(message), + onLifecycle: event => lifecycle.push(event), + now: () => now++, + }); + return { controller, requests, finals, diagnostics, lifecycle, displayed }; +} + +function completeTurn(controller: CodexAppTurnController, turnId: string, text = 'done'): void { + controller.handleNotification({ + method: 'item/completed', + params: { + threadId: 'thread-1', + turnId, + item: { id: 'answer', type: 'agentMessage', phase: 'final_answer', text }, + }, + }); + controller.handleNotification({ + method: 'turn/completed', + params: { threadId: 'thread-1', turn: { id: turnId, status: 'completed' } }, + }); +} + +function input(content: string, replyTurnId: string): CodexAppRunnerInput { + return { + type: 'message', + content: `legacy:${content}`, + codexAppInput: { + text: content, + additionalContext: { + botmux_sender: { kind: 'untrusted', value: 'Alice' }, + }, + localImages: [{ path: '/tmp/image.png', detail: 'high' }], + }, + replyTurnId, + }; +} + +describe('CodexAppTurnController', () => { + it('serializes structured steers and binds one final to the last accepted input', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + await flushAsync(); + + expect(h.requests[0]).toMatchObject({ + method: 'turn/start', + params: { + threadId: 'thread-1', + clientUserMessageId: 'om_first', + input: [ + { type: 'text', text: 'first', text_elements: [] }, + { type: 'localImage', path: '/tmp/image.png', detail: 'high' }, + ], + additionalContext: { + botmux_sender: { kind: 'untrusted', value: 'Alice' }, + }, + }, + }); + + h.requests[0].response.resolve({ turn: { id: 'app-1' } }); + await flushAsync(); + h.controller.enqueue(input('second', 'om_second')); + h.controller.enqueue(input('third', 'om_third')); + + expect(h.requests.map(request => request.method)).toEqual(['turn/start', 'turn/steer']); + expect(h.requests[1].params).toMatchObject({ + expectedTurnId: 'app-1', + clientUserMessageId: 'om_second', + input: [ + { type: 'text', text: 'second', text_elements: [] }, + { type: 'localImage', path: '/tmp/image.png', detail: 'high' }, + ], + }); + + h.requests[1].response.resolve({ turnId: 'app-1' }); + await flushAsync(); + h.requests[2].response.resolve({ turnId: 'app-1' }); + await flushAsync(); + completeTurn(h.controller, 'app-1', 'merged'); + + expect(h.lifecycle.filter(event => event.kind === 'steer_accepted')).toEqual([ + expect.objectContaining({ appTurnId: 'app-1', replyTurnId: 'om_second' }), + expect.objectContaining({ appTurnId: 'app-1', replyTurnId: 'om_third' }), + ]); + expect(h.displayed).toEqual(['first', 'second', 'third']); + expect(h.finals).toEqual([expect.objectContaining({ + appTurnId: 'app-1', + replyTurnId: 'om_third', + content: 'merged', + })]); + }); + + it('can steer after turn/started while turn/start response is pending', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + h.controller.enqueue(input('follow up', 'om_second')); + await flushAsync(); + + h.controller.handleNotification({ + method: 'turn/started', + params: { threadId: 'thread-1', turn: { id: 'app-1', status: 'inProgress' } }, + }); + + expect(h.requests.map(request => request.method)).toEqual(['turn/start', 'turn/steer']); + expect(h.requests[1].params.expectedTurnId).toBe('app-1'); + }); + + it('uses a completion barrier while a steer response is in flight', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + await flushAsync(); + h.requests[0].response.resolve({ turn: { id: 'app-1' } }); + await flushAsync(); + h.controller.enqueue(input('second', 'om_second')); + + completeTurn(h.controller, 'app-1', 'merged'); + expect(h.finals).toHaveLength(0); + expect(h.lifecycle).toContainEqual(expect.objectContaining({ + kind: 'completion_race', + appTurnId: 'app-1', + replyTurnId: 'om_second', + })); + + h.requests[1].response.resolve({ turnId: 'app-1' }); + await flushAsync(); + expect(h.finals).toEqual([expect.objectContaining({ + appTurnId: 'app-1', + replyTurnId: 'om_second', + content: 'merged', + })]); + }); + + it('keeps a definitely rejected steer queued for the next turn', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + await flushAsync(); + h.requests[0].response.resolve({ turn: { id: 'app-1' } }); + await flushAsync(); + h.controller.enqueue(input('next turn', 'om_second')); + + h.requests[1].response.reject(new CodexAppRpcResponseError('turn/steer', { + code: -32600, + message: 'no active turn to steer', + })); + await flushAsync(); + completeTurn(h.controller, 'app-1', 'first done'); + await flushAsync(); + + expect(h.requests.map(request => request.method)).toEqual([ + 'turn/start', + 'turn/steer', + 'turn/start', + ]); + expect(h.lifecycle).toContainEqual(expect.objectContaining({ + kind: 'steer_rejected_fallback', + appTurnId: 'app-1', + replyTurnId: 'om_second', + })); + expect(h.lifecycle.some(event => event.kind === 'steer_accepted')).toBe(false); + expect(h.displayed).toEqual(['first', 'next turn']); + }); + + it('does not replay a steer whose transport outcome is unknown', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + await flushAsync(); + h.requests[0].response.resolve({ turn: { id: 'app-1' } }); + await flushAsync(); + h.controller.enqueue(input('uncertain', 'om_second')); + + h.requests[1].response.reject(new CodexAppTransportError('connection lost')); + await flushAsync(); + + expect(h.requests.map(request => request.method)).toEqual(['turn/start', 'turn/steer']); + expect(h.finals).toEqual([expect.objectContaining({ + appTurnId: 'app-1', + replyTurnId: 'om_second', + content: expect.stringContaining('result unknown'), + })]); + expect(h.lifecycle).toContainEqual(expect.objectContaining({ + kind: 'unknown_outcome', + operation: 'turn/steer', + category: 'transport', + })); + }); + + it('retries a pre-start structured capability rejection once with legacy input', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + await flushAsync(); + + h.requests[0].response.reject(new CodexAppRpcResponseError('turn/start', { + code: -32600, + message: 'unknown field additionalContext', + })); + await flushAsync(); + + expect(h.requests).toHaveLength(2); + expect(h.requests[0].params.input[0].text).toBe('first'); + expect(h.requests[1].params.input).toEqual([ + { type: 'text', text: 'legacy:first', text_elements: [] }, + ]); + expect(h.requests[1].params).not.toHaveProperty('additionalContext'); + expect(h.displayed).toEqual(['first']); + expect(h.diagnostics).toContain( + '[codex-app] structured input unsupported; retrying this turn with the legacy prompt', + ); + + h.requests[1].response.resolve({ turn: { id: 'app-legacy' } }); + await flushAsync(); + completeTurn(h.controller, 'app-legacy'); + expect(h.finals).toHaveLength(1); + }); + + it('ignores duplicate completion notifications after emitting the final', async () => { + const h = createHarness(); + h.controller.enqueue(input('first', 'om_first')); + await flushAsync(); + h.requests[0].response.resolve({ turn: { id: 'app-1' } }); + await flushAsync(); + + completeTurn(h.controller, 'app-1'); + completeTurn(h.controller, 'app-1'); + + expect(h.finals).toHaveLength(1); + }); +}); diff --git a/test/codex-runner-freshness.test.ts b/test/codex-runner-freshness.test.ts new file mode 100644 index 000000000..546299ee6 --- /dev/null +++ b/test/codex-runner-freshness.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + decideCodexRunnerFreshness, + shouldHoldCodexRunnerInput, + transitionOnCodexRunnerPrompt, +} from '../src/services/codex-runner-freshness.js'; + +describe('Codex App runner freshness', () => { + const stale = { + cliId: 'codex-app', + adoptMode: false, + persistentReattach: true, + currentBuildId: 'new', + persistedBuildId: 'old', + }; + + it('holds a stale busy runner until old idle then fresh prompt-ready', () => { + expect(decideCodexRunnerFreshness(stale).state).toBe('stale_waiting_idle'); + expect(shouldHoldCodexRunnerInput('stale_waiting_idle')).toBe(true); + const oldIdle = transitionOnCodexRunnerPrompt('stale_waiting_idle'); + expect(oldIdle).toEqual({ state: 'restarting_fresh', action: 'reload' }); + expect(shouldHoldCodexRunnerInput(oldIdle.state)).toBe(true); + expect(transitionOnCodexRunnerPrompt(oldIdle.state)).toEqual({ + state: 'current', + action: 'publish_ready', + }); + }); + + it('fails safe for unknown identity and excludes adopt/non-app sessions', () => { + expect(decideCodexRunnerFreshness({ ...stale, currentBuildId: undefined }).state).toBe('unknown'); + expect(decideCodexRunnerFreshness({ ...stale, adoptMode: true }).reason).toBe('adopt_session'); + expect(decideCodexRunnerFreshness({ ...stale, cliId: 'codex' }).reason).toBe('not_codex_app'); + }); + + it('fails closed when a requested fresh replacement reattaches the old backend', () => { + expect(decideCodexRunnerFreshness({ + ...stale, + replacementExpectedFresh: true, + })).toEqual({ + state: 'failed', + reason: 'replacement_reattached', + persistOnReady: false, + }); + expect(shouldHoldCodexRunnerInput('failed')).toBe(true); + expect(decideCodexRunnerFreshness({ + ...stale, + currentBuildId: undefined, + replacementExpectedFresh: true, + }).reason).toBe('replacement_reattached'); + }); +}); diff --git a/test/command-handler.test.ts b/test/command-handler.test.ts index 4e03bc9a6..bb8127127 100644 --- a/test/command-handler.test.ts +++ b/test/command-handler.test.ts @@ -275,6 +275,10 @@ vi.mock('../src/core/worker-pool.js', () => ({ forkAdoptWorker: vi.fn(), adoptSandboxBlocked: vi.fn((botCfg, session) => botCfg?.sandbox === true || botCfg?.readIsolation === true || session?.sandbox === true || process.env.BOTMUX_SANDBOX === '1'), getCurrentCliVersion: vi.fn(() => '1.0.42'), + requestSessionRestart: vi.fn((_ds: any, observer: any) => { + void observer.notify('in_progress'); + return { attemptId: 'attempt-test', joined: false }; + }), // /close routes the「会话已关闭」card through this: ephemeral (visible-to-you) // when the chat supports it, else the visible reply fallback. The stub just // invokes the fallback so the existing card-shape assertions (on sessionReply) @@ -464,7 +468,7 @@ import { sessionKey } from '../src/core/types.js'; import { setTerminalProxyPort } from '../src/core/terminal-url.js'; import type { DaemonSession } from '../src/core/types.js'; import type { LarkMessage, Session } from '../src/types.js'; -import { killWorker, suspendWorker, forkWorker, getCurrentCliVersion, deliverEphemeralOrReply, deliverWritableTerminalCardTo } from '../src/core/worker-pool.js'; +import { killWorker, suspendWorker, forkWorker, getCurrentCliVersion, deliverEphemeralOrReply, deliverWritableTerminalCardTo, requestSessionRestart } from '../src/core/worker-pool.js'; import { getOwnerOpenId } from '../src/bot-registry.js'; import { canOperate } from '../src/im/lark/event-dispatcher.js'; import { getSessionWorkingDir, buildNewTopicPrompt, buildNewTopicCliInput, ensureSessionWhiteboard, getAvailableBots } from '../src/core/session-manager.js'; @@ -1469,6 +1473,24 @@ describe('handleCommand', () => { // ─── /restart ─────────────────────────────────────────────────────────── describe('/restart', () => { + it('rejects restart for adopted sessions without creating an attempt', async () => { + const ds = makeDaemonSession({ + adoptedFrom: { source: 'tmux', target: 'shared-pane' } as any, + }); + const deps = makeDeps(ds); + + await handleCommand('/restart', ROOT_ID, makeLarkMessage('/restart'), deps, LARK_APP_ID); + + expect(requestSessionRestart).not.toHaveBeenCalled(); + expect(deps.sessionReply).toHaveBeenCalledWith( + ROOT_ID, + expect.stringContaining('adopt'), + undefined, + LARK_APP_ID, + 'msg_001', + ); + }); + it('should send restart IPC when worker is alive', async () => { const workerSend = vi.fn(); const ds = makeDaemonSession({ @@ -1478,7 +1500,7 @@ describe('handleCommand', () => { await handleCommand('/restart', ROOT_ID, makeLarkMessage('/restart'), deps, LARK_APP_ID); - expect(workerSend).toHaveBeenCalledWith({ type: 'restart' }); + expect(requestSessionRestart).toHaveBeenCalledWith(ds, expect.objectContaining({ source: 'slash' })); expect(deps.sessionReply).toHaveBeenCalledWith( ROOT_ID, expect.stringContaining('正在重启'), @@ -1496,10 +1518,10 @@ describe('handleCommand', () => { await handleCommand('/restart', ROOT_ID, makeLarkMessage('/restart'), deps, LARK_APP_ID); - expect(killWorker).toHaveBeenCalledWith(ds); + expect(requestSessionRestart).toHaveBeenCalledWith(ds, expect.objectContaining({ source: 'slash' })); expect(deps.sessionReply).toHaveBeenCalledWith( ROOT_ID, - expect.stringContaining('进程已终止'), + expect.stringContaining('正在重启'), undefined, LARK_APP_ID, 'msg_001', @@ -1512,10 +1534,10 @@ describe('handleCommand', () => { await handleCommand('/restart', ROOT_ID, makeLarkMessage('/restart'), deps, LARK_APP_ID); - expect(killWorker).toHaveBeenCalledWith(ds); + expect(requestSessionRestart).toHaveBeenCalledWith(ds, expect.objectContaining({ source: 'slash' })); expect(deps.sessionReply).toHaveBeenCalledWith( ROOT_ID, - expect.stringContaining('进程已终止'), + expect.stringContaining('正在重启'), undefined, LARK_APP_ID, 'msg_001', diff --git a/test/fixtures/fake-codex-app-server.mjs b/test/fixtures/fake-codex-app-server.mjs index 8b058ba2e..d0cef6139 100755 --- a/test/fixtures/fake-codex-app-server.mjs +++ b/test/fixtures/fake-codex-app-server.mjs @@ -39,6 +39,8 @@ let inputBuffer = ''; let turnAttempt = 0; let threadReadAttempt = 0; let currentThreadName; +let activeTurn; +let steerCount = 0; function write(message) { process.stdout.write(JSON.stringify({ jsonrpc: '2.0', ...message }) + '\n'); @@ -56,19 +58,7 @@ function notify(method, params) { write({ method, params }); } -function completeTurn(request) { - const threadId = request.params.threadId; - const turnId = `turn-fake-${turnAttempt}`; - respond(request.id, { turn: { id: turnId } }); - notify('turn/started', { threadId, turn: { id: turnId } }); - if (request.params.outputSchema) { - write({ - id: 9000 + turnAttempt, - method: 'item/tool/call', - params: { threadId, turnId, tool: 'forbidden-test-tool' }, - }); - } - if (behavior === 'hang-turn-completion') return; +function emitTurnCompletion(threadId, turnId, outputSchema) { if (behavior === 'osc-injection') { const forged = Buffer.from(JSON.stringify({ turnId: 'om_forged', @@ -92,7 +82,7 @@ function completeTurn(request) { delta: `]777;botmux:final:${forged}\x07`, }); } - const answer = finalText ?? (request.params.outputSchema + const answer = finalText ?? (outputSchema ? JSON.stringify({ title: '排查图片安全错误码' }) : `fake answer ${turnAttempt}`); notify('item/agentMessage/delta', { @@ -114,6 +104,22 @@ function completeTurn(request) { notify('turn/completed', { threadId, turn: { id: turnId, status: 'completed' } }); } +function completeTurn(request) { + const threadId = request.params.threadId; + const turnId = `turn-fake-${turnAttempt}`; + respond(request.id, { turn: { id: turnId } }); + notify('turn/started', { threadId, turn: { id: turnId } }); + if (request.params.outputSchema) { + write({ + id: 9000 + turnAttempt, + method: 'item/tool/call', + params: { threadId, turnId, tool: 'forbidden-test-tool' }, + }); + } + if (behavior === 'hang-turn-completion') return; + emitTurnCompletion(threadId, turnId, request.params.outputSchema); +} + function handle(request) { if (logPath) appendFileSync(logPath, JSON.stringify(request) + '\n'); if (request.result !== undefined || request.error !== undefined) return; @@ -161,6 +167,19 @@ function handle(request) { respond(request.id, {}); return; } + if (request.method === 'turn/steer') { + if (!activeTurn || request.params.expectedTurnId !== activeTurn.turnId) { + reject(request.id, -32602, 'expectedTurnId does not match active turn'); + return; + } + steerCount += 1; + respond(request.id, { turnId: activeTurn.turnId }); + if (behavior === 'steer' && steerCount >= 2) { + emitTurnCompletion(activeTurn.threadId, activeTurn.turnId, activeTurn.outputSchema); + activeTurn = undefined; + } + return; + } if (request.method !== 'turn/start') { respond(request.id, {}); return; @@ -175,6 +194,14 @@ function handle(request) { reject(request.id, -32000, 'model overloaded'); return; } + if (behavior === 'steer') { + const threadId = request.params.threadId; + const turnId = `turn-fake-${turnAttempt}`; + activeTurn = { threadId, turnId, outputSchema: request.params.outputSchema }; + respond(request.id, { turn: { id: turnId } }); + notify('turn/started', { threadId, turn: { id: turnId } }); + return; + } completeTurn(request); } diff --git a/test/input-gate.test.ts b/test/input-gate.test.ts index 4be067091..ae3504254 100644 --- a/test/input-gate.test.ts +++ b/test/input-gate.test.ts @@ -29,6 +29,15 @@ const base = { }; describe('shouldWriteNow', () => { + it('holds stale busy Codex App input even when type-ahead is supported', () => { + expect(shouldWriteNow({ + isPromptReady: false, + isFlushing: false, + supportsTypeAhead: true, + awaitingFirstPrompt: false, + holdForRunnerReload: true, + })).toBe(false); + }); it('writes immediately when the prompt is ready (idle)', () => { expect(shouldWriteNow({ ...base, isPromptReady: true })).toBe(true); }); diff --git a/test/raw-input-followup-atomicity.test.ts b/test/raw-input-followup-atomicity.test.ts index 0f838afec..e3e9c325c 100644 --- a/test/raw-input-followup-atomicity.test.ts +++ b/test/raw-input-followup-atomicity.test.ts @@ -36,7 +36,7 @@ describe('worker raw_input handler', () => { const gateIdx = region.indexOf( 'if (cliRestartInProgress || rawInputRestartGate || sessionRenameInFlight', ); - const queueIdx = region.indexOf('pendingRawInputs.push(msg)'); + const queueIdx = region.indexOf('freshnessInputQueue.enqueueRaw(msg)'); const deliverIdx = region.indexOf('await deliverRawInput(msg)'); expect(gateIdx).toBeGreaterThanOrEqual(0); @@ -55,7 +55,7 @@ describe('worker raw_input handler', () => { // 同在入队条件里。 const gate = region.slice( region.indexOf('if (cliRestartInProgress'), - region.indexOf('pendingRawInputs.push(msg)'), + region.indexOf('freshnessInputQueue.enqueueRaw(msg)'), ); expect(gate).toContain('injectionFlushing'); expect(gate).toContain('shouldDeferUserFlush(pendingInjections)'); @@ -69,7 +69,7 @@ describe('worker raw_input handler', () => { // 覆盖"检查进行中",并用 bareShellLaunchBlocked 覆盖"已确认失败"。 const gate = region.slice( region.indexOf('if (cliRestartInProgress'), - region.indexOf('pendingRawInputs.push(msg)'), + region.indexOf('freshnessInputQueue.enqueueRaw(msg)'), ); expect(gate).toContain('bareShellCheckInProgress'); expect(gate).toContain('bareShellLaunchBlocked'); @@ -197,7 +197,7 @@ describe('post-settle restart fence', () => { const detector = flush.indexOf('if (await detectBareShellLaunch())'); const fence = flush.indexOf('if (cliRestartInProgress) return;', detector); const startup = flush.indexOf('await runStartupCommands()', detector); - const rawShift = flush.indexOf('pendingRawInputs.shift()', detector); + const rawShift = flush.indexOf('freshnessInputQueue.takeRaw()', detector); const writeInput = flush.indexOf('cliAdapter.writeInput(backend', detector); expect(detector).toBeGreaterThanOrEqual(0); expect(fence).toBeGreaterThan(detector); diff --git a/test/restart-coordinator.test.ts b/test/restart-coordinator.test.ts new file mode 100644 index 000000000..404c267ca --- /dev/null +++ b/test/restart-coordinator.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from 'vitest'; +import { RestartCoordinator } from '../src/core/restart-coordinator.js'; + +const flush = async () => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('RestartCoordinator', () => { + it('coalesces physical restart and orders progress before terminal per observer', async () => { + const coordinator = new RestartCoordinator({ createAttemptId: () => 'a1' }); + const start = vi.fn(); + const first = vi.fn(); + const second = vi.fn(); + expect(coordinator.request('s', { source: 'slash', notify: first }, start).joined).toBe(false); + expect(coordinator.request('s', { source: 'card', notify: second }, start).joined).toBe(true); + expect(start).toHaveBeenCalledTimes(1); + expect(coordinator.resolve('s', 'a1', 'succeeded')).toBe(true); + expect(coordinator.resolve('s', 'a1', 'failed')).toBe(false); + await flush(); + expect(first.mock.calls.map(call => call[0])).toEqual(['in_progress', 'succeeded']); + expect(second.mock.calls.map(call => call[0])).toEqual(['in_progress', 'succeeded']); + }); + + it('starts physical restart without waiting for a stuck progress notification', () => { + const coordinator = new RestartCoordinator({ createAttemptId: () => 'a2' }); + const start = vi.fn(); + coordinator.request('s', { + source: 'slash', + notify: () => new Promise(() => {}), + }, start); + expect(start).toHaveBeenCalledWith('a2'); + }); + + it('isolates observers when another observer notification is stuck', async () => { + const coordinator = new RestartCoordinator({ createAttemptId: () => 'a3' }); + const second = vi.fn(); + coordinator.request('s', { + source: 'slash', + notify: () => new Promise(() => {}), + }, vi.fn()); + coordinator.request('s', { source: 'card', notify: second }, vi.fn()); + + expect(coordinator.resolve('s', 'a3', 'succeeded')).toBe(true); + await flush(); + expect(second.mock.calls.map(call => call[0])).toEqual(['in_progress', 'succeeded']); + }); + + it('times out once and ignores late completion', async () => { + vi.useFakeTimers(); + try { + const coordinator = new RestartCoordinator({ + createAttemptId: () => 'a4', + timeoutMs: 10, + }); + const notify = vi.fn(); + coordinator.request('s', { source: 'slash', notify }, vi.fn()); + await vi.advanceTimersByTimeAsync(10); + await flush(); + + expect(notify.mock.calls.map(call => call[0])).toEqual(['in_progress', 'timed_out']); + expect(coordinator.resolve('s', 'a4', 'succeeded')).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it('fails a synchronously rejected physical restart once', async () => { + const coordinator = new RestartCoordinator({ createAttemptId: () => 'a5' }); + const notify = vi.fn(); + coordinator.request('s', { source: 'slash', notify }, () => { + throw new Error('send failed'); + }); + await flush(); + expect(notify.mock.calls.map(call => call[0])).toEqual(['in_progress', 'failed']); + }); +}); diff --git a/test/restart-worker-null-reattach.test.ts b/test/restart-worker-null-reattach.test.ts new file mode 100644 index 000000000..f3852d40b --- /dev/null +++ b/test/restart-worker-null-reattach.test.ts @@ -0,0 +1,267 @@ +/** + * Regression tests for two PR #588 restart blockers (codex 复审 / Claude 复核): + * + * P1 — worker-null "fake restart": requestSessionRestart's no-worker branch + * now forks a fresh worker. For a session that lost its worker but kept a + * live persistent pane (the normal post-daemon-restart state), spawnCli + * would REATTACH the surviving CLI instead of relaunching it, yet still + * emit `restart_result: succeeded`. The fix destroys the live pane before + * forking so the CLI physically relaunches. Adopt sessions are exempt. + * + * P2 — stale Riff `taskDoneCb`: RiffBackend fires `taskDoneCb` from + * `fetchAndEmitOutput(...).finally(...)`, which can resolve AFTER + * destroySession()/kill() (neither clears the callback nor awaits the + * in-flight fetch). The worker's onTaskDone hook must therefore drop a + * callback from a superseded backend generation. This asserts the + * backend-side hazard (callback fires post-kill) and the worker-side + * generation fence wiring. + * + * worker.ts installs process-wide IPC/services at import time, so its wiring is + * pinned by source-structure assertions (same approach as + * worker-restart-race.test.ts / worker-app-runner-control-wiring.test.ts). The + * P1 adopt decision is exercised as a pure predicate. + * + * Run: pnpm vitest run test/restart-worker-null-reattach.test.ts + */ +import { readFileSync } from 'node:fs'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +vi.mock('../src/utils/logger.js', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { RiffBackend } from '../src/adapters/backend/riff-backend.js'; + +const workerSource = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf8'); +const workerPoolSource = readFileSync(new URL('../src/core/worker-pool.ts', import.meta.url), 'utf8'); + +// ─── P1: pure adopt-skip decision ─────────────────────────────────────────── + +describe('P1 shouldDestroyPaneBeforeRestart (pure decision)', () => { + it('is imported from worker-pool without triggering its heavy spawn wiring', async () => { + const mod = await import('../src/core/worker-pool.js'); + expect(typeof mod.shouldDestroyPaneBeforeRestart).toBe('function'); + }); + + it('destroys the pane for an ordinary owned session', async () => { + const { shouldDestroyPaneBeforeRestart } = await import('../src/core/worker-pool.js'); + expect(shouldDestroyPaneBeforeRestart({ initConfig: { adoptMode: false } as any, adoptedFrom: undefined })) + .toBe(true); + expect(shouldDestroyPaneBeforeRestart({ initConfig: undefined, adoptedFrom: undefined })) + .toBe(true); + }); + + it('NEVER destroys an adopted user pane (bridge invariant)', async () => { + const { shouldDestroyPaneBeforeRestart } = await import('../src/core/worker-pool.js'); + // adoptMode frozen on the live init config + expect(shouldDestroyPaneBeforeRestart({ initConfig: { adoptMode: true } as any, adoptedFrom: undefined })) + .toBe(false); + // adoptedFrom stamped on the daemon session (restored adopt session) + expect(shouldDestroyPaneBeforeRestart({ + initConfig: undefined, + adoptedFrom: { source: 'tmux', tmuxTarget: 'dev:1.2', cliId: 'claude-code', cwd: '/tmp' } as any, + })).toBe(false); + }); +}); + +// ─── P1: worker-pool wiring (kill BEFORE fork, no-worker branch only) ───────── + +describe('P1 requestSessionRestart wiring', () => { + it('destroys the live pane before forking in the no-worker branch', () => { + const fn = workerPoolSource.indexOf('export function requestSessionRestart'); + expect(fn).toBeGreaterThanOrEqual(0); + const body = workerPoolSource.slice(fn, workerPoolSource.indexOf('\n}', fn) + 2); + + // Live-worker branch still sends the in-worker restart IPC (unchanged). + expect(body).toContain("ds.worker.send({ type: 'restart', attemptId }"); + + // No-worker branch: pane teardown MUST precede forkWorker. + const destroy = body.indexOf('destroyLivePaneBeforeRestart(ds)'); + const fork = body.indexOf('forkWorker(ds'); + expect(destroy).toBeGreaterThan(0); + expect(fork).toBeGreaterThan(destroy); + }); + + it('destroyLivePaneBeforeRestart kills a resolved persistent target and probe-retries on survival', () => { + const fn = workerPoolSource.indexOf('function destroyLivePaneBeforeRestart'); + expect(fn).toBeGreaterThanOrEqual(0); + const body = workerPoolSource.slice(fn, fn + 2800); + expect(body).toContain('shouldDestroyPaneBeforeRestart(ds)'); + expect(body).toContain('persistentBackendTargetForSession(ds)'); + expect(body).toContain('killPersistentBackendTarget(target)'); + // No target (e.g. PTY/riff/legacy-unstamped) → no-op, never throws. + expect(body).toContain('if (!target) return;'); + + // Fail-safe (codex 复审): the kill primitives swallow their own failures, so + // a bare try/catch can't detect a failed kill. Must PROBE after killing. + expect(body).toContain('probePersistentBackendTarget(target)'); + // A confirmed survivor after retry escalates to a loud error (not silent). + expect(body).toContain('logger.error'); + expect(body).toMatch(/STILL alive after retry/); + + // Monotonic single-probe (codex 复审): the retry re-probe happens ONLY + // inside the `exists` branch, so an 'unknown' first probe is never + // mislabelled as a post-retry survivor. Guard against reintroducing an + // unconditional second probe: exactly two probe calls, the second nested in + // the exists branch. + const probeCalls = body.match(/probePersistentBackendTarget\(target\)/g) ?? []; + expect(probeCalls.length).toBe(2); + const existsBranch = body.indexOf("if (probe === 'exists') {"); + const retryProbe = body.indexOf('probe = probePersistentBackendTarget(target);', existsBranch); + const secondExistsCheck = body.indexOf("if (probe === 'exists') {", existsBranch + 1); + expect(existsBranch).toBeGreaterThan(0); + expect(retryProbe).toBeGreaterThan(existsBranch); + // The re-probe is assigned before the second exists check evaluates it. + expect(secondExistsCheck).toBeGreaterThan(retryProbe); + + // Three-state diagnostic (codex 复审): the terminal log must NOT lump + // 'unknown' in with 'missing' — 'missing' promises a relaunch, 'unknown' + // must only warn that a reattach is still possible, 'exists' errors. This + // keeps the diagnostic from lying (the whole point of the fix). + expect(body).toContain("} else if (probe === 'unknown') {"); + expect(body).toMatch(/kill outcome indeterminate/); + expect(body).toMatch(/pane missing after kill — CLI will physically relaunch/); + // Ordering: error(exists) → warn(unknown) → info(missing). + const errIdx = body.indexOf('logger.error'); + const warnIndeterminate = body.indexOf('kill outcome indeterminate'); + const infoMissing = body.indexOf('pane missing after kill'); + expect(errIdx).toBeGreaterThan(0); + expect(warnIndeterminate).toBeGreaterThan(errIdx); + expect(infoMissing).toBeGreaterThan(warnIndeterminate); + }); + + it('probe survival does NOT block the refork (stranding is worse than the guarded bug)', () => { + // The helper only logs on a surviving pane — it never returns early in a way + // that would skip forkWorker. requestSessionRestart always reaches forkWorker + // after destroyLivePaneBeforeRestart regardless of probe outcome. + const fn = workerPoolSource.indexOf('export function requestSessionRestart'); + const body = workerPoolSource.slice(fn, workerPoolSource.indexOf('\n}', fn) + 2); + const destroy = body.indexOf('destroyLivePaneBeforeRestart(ds)'); + const fork = body.indexOf('forkWorker(ds'); + expect(destroy).toBeGreaterThan(0); + expect(fork).toBeGreaterThan(destroy); + // No conditional/throw between them that could skip the fork. + const between = body.slice(destroy, fork); + expect(between).not.toMatch(/\breturn\b|\bthrow\b|if\s*\(/); + }); +}); + +// ─── P2: RiffBackend stale-callback hazard (behavioral) ────────────────────── + +describe('P2 RiffBackend stale taskDoneCb hazard', () => { + let resolvers: Array<(r: Response) => void>; + let fetchMock: ReturnType; + const flush = () => new Promise((r) => setTimeout(r, 0)); + + function pendingSse(): Response { + const body = new ReadableStream({ start() { /* never pushes */ } }); + return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }); + } + + beforeEach(() => { + resolvers = []; + fetchMock = vi.fn(async (url: string | URL) => { + const u = String(url); + if (u.includes('/api2/task-stream')) return pendingSse(); + // task-detail (final-output fetch): resolve manually so the test controls + // WHEN the .finally(taskDoneCb) runs relative to kill(). + if (u.includes('/api/task-detail')) { + return new Promise((resolve) => { resolvers.push(resolve); }); + } + if (u.includes('/api/task-cancel')) return Response.json({ success: true, data: {} }); + // task-execute: resolve immediately with a running task id + return Response.json({ success: true, data: { id: 'task-1', status: 'running' } }); + }); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => { vi.unstubAllGlobals(); }); + + it('fires taskDoneCb AFTER kill() (proves the worker-side generation fence is required)', async () => { + const be = new RiffBackend({ baseUrl: 'https://riff.test', jwt: 'jwt' } as any, 'session-1'); + let killedWhenDoneFired: boolean | undefined; + be.onTaskDone(() => { killedWhenDoneFired = (be as any).killed === true; }); + be.onExit(() => { /* swallow */ }); + + be.spawn('', [], {} as any); + be.write('do work'); + await flush(); + + // Task completes → RiffBackend launches fetchAndEmitOutput(taskId), whose + // .finally() will call taskDoneCb. The detail fetch is still pending. + (be as any).handleSseEvent('event:done\ndata:{"status":"completed"}', 'task-1'); + await flush(); + expect(killedWhenDoneFired).toBeUndefined(); // not fired yet — fetch pending + + // Detach the stream (worker teardown / restart). kill() does NOT clear the + // pending final-output fetch nor its taskDoneCb. + be.kill(); + expect((be as any).killed).toBe(true); + + // The in-flight detail fetch now resolves → .finally(taskDoneCb) runs. + resolvers.shift()!(Response.json({ success: true, data: { task: {} } })); + await flush(); await flush(); + + // Hazard confirmed: the callback fires while the backend is already killed. + // Without the worker's `backend !== observedBackend` fence this would reach + // markPromptReady() and emit a premature restart success. + expect(killedWhenDoneFired).toBe(true); + }); +}); + +// ─── P2: worker-side generation fence + defensive success receipt (wiring) ─── + +describe('P2 worker onTaskDone generation fence', () => { + it('onTaskDone drops callbacks from a superseded backend generation', () => { + const hook = workerSource.indexOf('backend.onTaskDone?.(()'); + expect(hook).toBeGreaterThanOrEqual(0); + const body = workerSource.slice(hook, hook + 900); + // Same fence pattern as onAgentStatus / onExit siblings. + expect(body).toContain('if (backend !== observedBackend) return;'); + // The generation check must precede the markPromptReady() re-arm. Match the + // actual call (with semicolon) so the prose "markPromptReady()" in the + // rationale comment above the fence isn't mistaken for it. + const fence = body.indexOf('if (backend !== observedBackend) return;'); + const mark = body.indexOf('markPromptReady();'); + expect(mark).toBeGreaterThan(fence); + }); + + it('all three async backend ready/exit callbacks in setupBackendHandlers carry the generation fence', () => { + // Within setupBackendHandlers the herdr onAgentStatus, the riff onTaskDone, + // and the backend onExit all guard on `backend !== observedBackend` so a + // superseded generation can neither re-arm prompt-ready nor tear down the + // replacement. Anchored on the setupBackendHandlers occurrences (the adopt + // observe-paths use a different, non-fenced onExit by design). + const setup = workerSource.indexOf('const observedBackend = backend;'); + expect(setup).toBeGreaterThanOrEqual(0); + const region = workerSource.slice(setup, setup + 6000); + + const agentStatus = region.indexOf('.onAgentStatus((status)'); + const taskDone = region.indexOf('backend.onTaskDone?.(()'); + const onExit = region.indexOf('backend.onExit((code, signal)'); + expect(agentStatus, 'onAgentStatus').toBeGreaterThanOrEqual(0); + expect(taskDone, 'onTaskDone').toBeGreaterThanOrEqual(0); + expect(onExit, 'onExit').toBeGreaterThanOrEqual(0); + + for (const [name, start] of [ + ['onAgentStatus', agentStatus], + ['onTaskDone', taskDone], + ['onExit', onExit], + ] as const) { + expect(region.slice(start, start + 700), name) + .toContain('backend !== observedBackend'); + } + }); + + it('markPromptReady defers the restart success receipt when no backend is installed', () => { + const guard = workerSource.indexOf('if (activeRestartAttemptId) {'); + expect(guard).toBeGreaterThanOrEqual(0); + const body = workerSource.slice(guard, guard + 700); + // Success is only reported behind a live-backend check; otherwise the + // attempt id is preserved for the real replacement / coordinator timeout. + const backendCheck = body.indexOf('if (backend) {'); + const succeeded = body.indexOf("status: 'succeeded'"); + expect(backendCheck).toBeGreaterThan(0); + expect(succeeded).toBeGreaterThan(backendCheck); + }); +}); diff --git a/test/runner-input.test.ts b/test/runner-input.test.ts index 272e39062..4f2ff8e77 100644 --- a/test/runner-input.test.ts +++ b/test/runner-input.test.ts @@ -169,6 +169,13 @@ describe('writeRunnerInput — tmux mode', () => { clientUserMessageId: 'om_1', }, }); + + const correlated = encodeRunnerInput('legacy prompt', undefined, 'om_reply_1'); + expect(JSON.parse(Buffer.from(correlated, 'base64').toString('utf8'))).toEqual({ + type: 'message', + content: 'legacy prompt', + replyTurnId: 'om_reply_1', + }); }); it('reports submitted:false and flushes the partial when a chunk is dropped', async () => { diff --git a/test/runtime-build-id.test.ts b/test/runtime-build-id.test.ts new file mode 100644 index 000000000..11b4d81bb --- /dev/null +++ b/test/runtime-build-id.test.ts @@ -0,0 +1,32 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { computeRuntimeBuildId, resolveRuntimeBuildIdentity } from '../src/utils/runtime-build-id.js'; + +const dirs: string[] = []; +afterEach(() => dirs.splice(0).forEach(path => rmSync(path, { recursive: true, force: true }))); + +describe('runtime build identity', () => { + it('is order independent and content sensitive', () => { + const entries = [{ path: 'b.js', content: 'b' }, { path: 'a.js', content: 'a' }]; + expect(computeRuntimeBuildId(entries)).toBe(computeRuntimeBuildId([...entries].reverse())); + expect(computeRuntimeBuildId(entries)).not.toBe(computeRuntimeBuildId([ + entries[0], + { path: 'a.js', content: 'changed' }, + ])); + }); + + it('accepts only a valid generated artifact', () => { + const root = mkdtempSync(join(tmpdir(), 'runtime-id-')); + dirs.push(root); + const artifactPath = join(root, '.runtime-build-id'); + writeFileSync(artifactPath, `${'a'.repeat(64)}\n`); + expect(resolveRuntimeBuildIdentity({ artifactPath }).status).toBe('known'); + writeFileSync(artifactPath, 'invalid\n'); + expect(resolveRuntimeBuildIdentity({ artifactPath })).toEqual({ + status: 'unknown', + reason: 'artifact_invalid', + }); + }); +}); diff --git a/test/session-lifecycle-hooks.test.ts b/test/session-lifecycle-hooks.test.ts index 772d68862..c0af57692 100644 --- a/test/session-lifecycle-hooks.test.ts +++ b/test/session-lifecycle-hooks.test.ts @@ -315,6 +315,63 @@ describe('worker-pool lifecycle hook integration', () => { })); }); + it('routes accepted steer feedback to its exact turn without raising attention', async () => { + const sessionReply = vi.fn(async () => 'om_reply'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const worker = makeFakeWorker(); + const ds = makeDs({ worker }); + __testOnly_setupWorkerHandlers(ds, worker); + + worker.emit('message', { + type: 'steer_accepted', + appTurnId: 'app-turn-accepted', + turnId: 'om_exact_steer_message', + }); + await flush(); + + expect(sessionReply).toHaveBeenCalledWith( + 'om_root', + '收到,引导成功', + 'text', + 'app_test', + 'om_exact_steer_message', + undefined, + ); + expect(emitHookEventMock).not.toHaveBeenCalledWith( + 'session.requires_attention', + expect.anything(), + ); + }); + + it('ignores accepted steer feedback from a replaced worker generation', async () => { + const sessionReply = vi.fn(async () => 'om_reply'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const worker = makeFakeWorker(); + const replacement = makeFakeWorker(); + const ds = makeDs({ worker: replacement }); + __testOnly_setupWorkerHandlers(ds, worker); + + worker.emit('message', { + type: 'steer_accepted', + appTurnId: 'app-turn-stale', + turnId: 'om_stale', + }); + await flush(); + + expect(sessionReply).not.toHaveBeenCalled(); + expect(emitHookEventMock).not.toHaveBeenCalled(); + }); + it('does not emit lifecycle hooks for receiver TUI, notifications, status, or exit', async () => { const worker = makeFakeWorker(); const ds = makeDs({ worker, lastScreenStatus: 'working' }); diff --git a/test/session-rename-worker.test.ts b/test/session-rename-worker.test.ts index a765d9faf..6fda2a50c 100644 --- a/test/session-rename-worker.test.ts +++ b/test/session-rename-worker.test.ts @@ -161,13 +161,13 @@ describe('worker native session rename queue', () => { // PR #441 起入队条件多了注入围栏(injectionFlushing / barrier),rename 围栏 // 仍必须在场——只钉本测试关心的三个 restart/rename 因子,不钉整行。 expect(rawRegion).toContain('if (cliRestartInProgress || rawInputRestartGate || sessionRenameInFlight'); - expect(rawRegion).toContain('pendingRawInputs.push(msg)'); + expect(rawRegion).toContain('freshnessInputQueue.enqueueRaw(msg)'); expect(rawRegion).toContain('await deliverRawInput(msg)'); const flushStart = workerSource.indexOf('async function flushPending()'); const flushEnd = workerSource.indexOf('\nfunction sendToPty(', flushStart); const flushRegion = workerSource.slice(flushStart, flushEnd); - expect(flushRegion).toContain('pendingRawInputs.shift()'); + expect(flushRegion).toContain('freshnessInputQueue.takeRaw()'); expect(flushRegion).toContain('await deliverRawInput(raw)'); expect(workerSource).toContain('await sendRawCommandLineSerially(targetBackend, msg.content)'); expect(flushRegion.indexOf('await deliverRawInput(raw)')) diff --git a/test/worker-app-runner-control-wiring.test.ts b/test/worker-app-runner-control-wiring.test.ts index 38ac3530e..f397303b0 100644 --- a/test/worker-app-runner-control-wiring.test.ts +++ b/test/worker-app-runner-control-wiring.test.ts @@ -1,5 +1,9 @@ import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; +import { + CodexRunnerFreshnessInputQueue, + type CodexRunnerFreshnessState, +} from '../src/services/codex-runner-freshness.js'; const workerSource = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf8'); @@ -17,4 +21,90 @@ describe('worker app-runner control-channel wiring', () => { expect(workerSource).toContain('const dispatchAttempt = currentBotmuxDispatchAttempt;'); expect(workerSource).not.toContain('const dispatchAttempt = payload.dispatchAttempt'); }); + + it('holds stale-busy normal and raw input across old idle and releases only at fresh prompt-ready', () => { + let state: CodexRunnerFreshnessState = 'stale_waiting_idle'; + const queue = new CodexRunnerFreshnessInputQueue( + () => state, + next => { state = next; }, + ); + const oldRunnerWrites: string[] = []; + const replacementWrites: string[] = []; + // Model freshness queue hold/release semantics; production flushPending + // delivers at most one raw input per invocation before normal inputs. + const flush = (writes: string[]): void => { + const raw = queue.takeRaw(); + if (raw) writes.push(`raw:${raw}`); + let normal: string | undefined; + while ((normal = queue.takeNormal()) !== undefined) writes.push(`normal:${normal}`); + }; + + queue.enqueueNormal('normal-one'); + queue.enqueueRaw('/raw-one'); + flush(oldRunnerWrites); + expect(oldRunnerWrites).toEqual([]); + expect(queue.normal).toEqual(['normal-one']); + expect(queue.raw).toEqual(['/raw-one']); + + // The old busy runner's first idle is consumed as the reload boundary. + expect(queue.onPromptReady()).toBe('reload'); + expect(state).toBe('restarting_fresh'); + queue.enqueueNormal('normal-during-replacement'); + queue.enqueueRaw('/raw-during-replacement'); + flush(oldRunnerWrites); + expect(oldRunnerWrites).toEqual([]); + expect(queue.normal).toEqual(['normal-one', 'normal-during-replacement']); + expect(queue.raw).toEqual(['/raw-one', '/raw-during-replacement']); + + // Only the replacement's prompt-ready makes dequeue possible. + expect(queue.onPromptReady()).toBe('publish_ready'); + expect(state).toBe('current'); + flush(replacementWrites); + expect(replacementWrites).toEqual([ + 'raw:/raw-one', + 'normal:normal-one', + 'normal:normal-during-replacement', + ]); + expect(queue.raw).toEqual(['/raw-during-replacement']); + flush(replacementWrites); + expect(replacementWrites).toEqual([ + 'raw:/raw-one', + 'normal:normal-one', + 'normal:normal-during-replacement', + 'raw:/raw-during-replacement', + ]); + expect(queue.normal).toEqual([]); + expect(queue.raw).toEqual([]); + + // The worker's actual queue transitions must stay wired to this tested + // seam; source loading is intentionally avoided because worker.ts starts + // process-wide IPC and runtime services at module evaluation time. + expect(workerSource).toContain('freshnessInputQueue.enqueueNormal(next)'); + expect(workerSource).toContain('freshnessInputQueue.enqueueRaw(msg)'); + expect(workerSource).toContain('freshnessInputQueue.takeNormal()'); + expect(workerSource).toContain('freshnessInputQueue.takeRaw()'); + expect(workerSource).toContain('freshnessInputQueue.onPromptReady()'); + expect(workerSource).toContain( + "restartCliProcess('stale runner reached idle', { immediate: true, preservePending: true })", + ); + }); + + it('keeps both input kinds held after replacement failure', () => { + let state: CodexRunnerFreshnessState = 'restarting_fresh'; + const queue = new CodexRunnerFreshnessInputQueue( + () => state, + next => { state = next; }, + ); + queue.enqueueNormal('normal-held'); + queue.enqueueRaw('/raw-held'); + + queue.onReplacementFailed(); + expect(queue.onPromptReady()).toBe('ignore'); + expect(state).toBe('failed'); + expect(queue.takeNormal()).toBeUndefined(); + expect(queue.takeRaw()).toBeUndefined(); + expect(queue.normal).toEqual(['normal-held']); + expect(queue.raw).toEqual(['/raw-held']); + expect(workerSource).toContain('freshnessInputQueue.onReplacementFailed()'); + }); }); diff --git a/test/worker-durable-expiry-order.test.ts b/test/worker-durable-expiry-order.test.ts index 125c97ca3..2b28c6054 100644 --- a/test/worker-durable-expiry-order.test.ts +++ b/test/worker-durable-expiry-order.test.ts @@ -116,7 +116,7 @@ describe('worker durable lease expiry ordering', () => { const rawGate = rawInput.indexOf( 'if (cliRestartInProgress || rawInputRestartGate || sessionRenameInFlight', ); - const rawQueue = rawInput.indexOf('pendingRawInputs.push(msg)', rawGate); + const rawQueue = rawInput.indexOf('freshnessInputQueue.enqueueRaw(msg)', rawGate); const rawDeliver = rawInput.indexOf('await deliverRawInput(msg)', rawQueue); expect(rawGate).toBeGreaterThanOrEqual(0); expect(rawQueue).toBeGreaterThan(rawGate); diff --git a/test/worker-pipe-initial-screen-order.test.ts b/test/worker-pipe-initial-screen-order.test.ts index cf3170b5c..4ab2de602 100644 --- a/test/worker-pipe-initial-screen-order.test.ts +++ b/test/worker-pipe-initial-screen-order.test.ts @@ -36,7 +36,7 @@ describe('worker pipe initial screen ordering', () => { it('runs a busy-pattern idle probe after each submitted input', () => { const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); - const writeIdx = source.indexOf('result = await cliAdapter.writeStructuredInput(backend, msg, item.codexAppInput);'); + const writeIdx = source.indexOf('result = await cliAdapter.writeStructuredInput('); const probeIdx = source.indexOf('scheduleBusyPatternIdleProbe(`${cliName()} post-submit`);'); const helperIdx = source.indexOf('function scheduleBusyPatternIdleProbe(source: string): void'); diff --git a/test/worker-ready-display-mode.test.ts b/test/worker-ready-display-mode.test.ts index 7a83be557..4bda8c9f8 100644 --- a/test/worker-ready-display-mode.test.ts +++ b/test/worker-ready-display-mode.test.ts @@ -9,6 +9,7 @@ * 1. POST path + displayMode='screenshot' → worker.send called * 2. POST path + displayMode='hidden' → worker.send NOT called * 3. PATCH path + displayMode='screenshot' → worker.send called (symmetry) + * 4. silent recovery + displayMode='screenshot' → worker.send called without card IO * * Run: pnpm vitest run test/worker-ready-display-mode.test.ts */ @@ -335,6 +336,28 @@ describe('Worker ready: set_display_mode re-sync', () => { ); }); + it('silent recovery restores screenshot mode without touching the streaming card', async () => { + const fakeWorker = makeFakeWorker(); + const ds = makeDs({ + displayMode: 'screenshot', + suppressRecoveryCard: true, + streamCardPending: false, + streamCardId: 'om_existing_card', + worker: fakeWorker, + }); + + __testOnly_setupWorkerHandlers(ds, fakeWorker); + fakeWorker.emit('message', { type: 'ready', port: 9999, token: 'tok_abc' }); + await flush(); + + expect(updateMessageMock).not.toHaveBeenCalled(); + expect(sessionReplyMock).not.toHaveBeenCalled(); + expect(fakeWorker.send).toHaveBeenCalledWith({ + type: 'set_display_mode', + mode: 'screenshot', + }); + }); + it('re-applies readiness when cli_session_id races a restored-card PATCH', async () => { let resolveRestorePatch!: () => void; updateMessageMock.mockImplementationOnce(() => new Promise((resolve) => { diff --git a/test/worker-restart-race.test.ts b/test/worker-restart-race.test.ts index 7b2ab146b..00fe26605 100644 --- a/test/worker-restart-race.test.ts +++ b/test/worker-restart-race.test.ts @@ -57,6 +57,23 @@ describe('worker restart P1 — drain reliable terminal before ambiguous emit', }); describe('worker restart P2 — merge guard + replacement-exit recovery (no spurious re-restart)', () => { + it('retains a coalesced cwd update without replacing the active correlated restart attempt', () => { + const branch = restartCaseBranch(); + const cwdUpdate = branch.indexOf('if (msg.updateWorkingDir && lastInitConfig)'); + const guard = branch.indexOf('if (cliRestartInProgress || tmuxRestartTimer)'); + const attempt = branch.indexOf('activeRestartAttemptId = msg.attemptId;', guard); + const freshness = branch.indexOf("codexRunnerFreshness = 'restarting_fresh';", attempt); + + expect(cwdUpdate).toBeGreaterThanOrEqual(0); + expect(guard).toBeGreaterThan(cwdUpdate); + expect(attempt).toBeGreaterThan(guard); + expect(freshness).toBeGreaterThan(attempt); + // The in-flight branch exits before touching the correlated attempt. This + // lets a role-switch cwd update join a manual restart without stealing its + // eventual prompt-ready result. + expect(branch.slice(guard, attempt)).toContain('break;'); + }); + it('the merge guard plainly breaks — it does NOT arm any "restart requested" flag', () => { const branch = restartCaseBranch(); const guard = branch.indexOf('if (cliRestartInProgress || tmuxRestartTimer)');