From dcf37d4562653443d8692f20116fb4591d357bba Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:26:25 +0800 Subject: [PATCH 1/4] =?UTF-8?q?refactor(core):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=9C=BA=E5=99=A8=E4=BA=BA=E7=BA=A7=20turn=20admission/mutatio?= =?UTF-8?q?n=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/bot-turn-mutation-gate.ts | 296 ++++++++++++++++++++ test/bot-turn-mutation-gate.test.ts | 400 ++++++++++++++++++++++++++++ 2 files changed, 696 insertions(+) create mode 100644 src/core/bot-turn-mutation-gate.ts create mode 100644 test/bot-turn-mutation-gate.test.ts diff --git a/src/core/bot-turn-mutation-gate.ts b/src/core/bot-turn-mutation-gate.ts new file mode 100644 index 000000000..317b706b5 --- /dev/null +++ b/src/core/bot-turn-mutation-gate.ts @@ -0,0 +1,296 @@ +/** + * Per-bot admission/drain gate for mutations that replace every live worker + * generation (currently read-isolation). JavaScript's single thread makes the + * check+increment and close+publish edges atomic, while the waiters cover all + * intervening awaits in inbound/HTTP turn preparation. + */ +import { AsyncLocalStorage } from 'node:async_hooks'; + +type GateState = { + activeAdmissions: number; + mutating: boolean; + openWaiters: Array; + drainWaiters: Array; +}; + +type GateWaiter = { wake: () => void }; + +const gates = new Map(); +type AdmissionLease = { active: boolean; ownerFinished: boolean }; +type MutationLease = { active: boolean }; +const admissionContext = new AsyncLocalStorage>(); +const mutationContext = new AsyncLocalStorage>(); + +function stateFor(larkAppId: string): GateState { + let state = gates.get(larkAppId); + if (!state) { + state = { activeAdmissions: 0, mutating: false, openWaiters: [], drainWaiters: [] }; + gates.set(larkAppId, state); + } + return state; +} + +function waitUntilOpen(state: GateState): Promise { + return state.mutating + ? new Promise(resolve => state.openWaiters.push({ wake: resolve })) + : Promise.resolve(); +} + +function waitUntilDrained(state: GateState): Promise { + return state.activeAdmissions > 0 + ? new Promise(resolve => state.drainWaiters.push({ wake: resolve })) + : Promise.resolve(); +} + +function wakeAll(waiters: GateWaiter[]): void { + const pending = waiters.splice(0); + for (const waiter of pending) waiter.wake(); +} + +/** Cancellable condition wait used only by bounded acquisition. Timeout + * removes the exact waiter, so it cannot resume later and run a stale shutdown + * after its caller already reported refusal. */ +function waitUntilBefore( + waiters: GateWaiter[], + ready: () => boolean, + deadlineMs: number, +): Promise { + if (Date.now() >= deadlineMs) return Promise.resolve(false); + if (ready()) return Promise.resolve(true); + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) return Promise.resolve(false); + return new Promise(resolve => { + let settled = false; + let timer: NodeJS.Timeout; + const waiter: GateWaiter = { + wake: () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(Date.now() < deadlineMs); + }, + }; + waiters.push(waiter); + timer = setTimeout(() => { + if (settled) return; + settled = true; + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + resolve(false); + }, remaining); + }); +} + +export async function withBotTurnAdmission( + larkAppId: string, + action: () => Promise | T, +): Promise { + // A mutation already owns the bot exclusively. Re-entering an admission + // boundary from that mutation must not wait for the gate it owns. + const inheritedMutation = mutationContext.getStore()?.get(larkAppId); + if (inheritedMutation?.active) return action(); + const inherited = admissionContext.getStore(); + // Some admitted event paths delegate to triggerSessionTurn, which is also a + // public admission boundary. Treat that nested call as part of the outer + // lease; otherwise a mutation that closed the gate while the outer handler + // awaited would make the handler wait on itself and deadlock the drain. + const inheritedLease = inherited?.get(larkAppId); + if (inheritedLease?.active) return action(); + const state = stateFor(larkAppId); + // A mutation may finish and another queued mutation may close the gate again + // before this continuation runs, so re-check after every wake. + while (state.mutating) await waitUntilOpen(state); + state.activeAdmissions++; + const lease: AdmissionLease = { active: true, ownerFinished: false }; + try { + const next = new Map(inherited ?? []); + next.set(larkAppId, lease); + return await admissionContext.run(next, action); + } finally { + // AsyncLocalStorage descendants can outlive the awaited action when code + // starts fire-and-forget work. They retain this object, so revoke it before + // decrementing the drain count; a detached continuation must acquire a new + // admission instead of impersonating a still-live nested call. + // An admitted handler may upgrade itself to a mutation. The upgrade + // temporarily releases and later reacquires this exact lease; only the + // still-active owner decrements it here. + lease.ownerFinished = true; + if (lease.active) { + lease.active = false; + state.activeAdmissions--; + if (state.activeAdmissions === 0) { + wakeAll(state.drainWaiters); + } + } + } +} + +export async function withBotTurnMutation( + larkAppId: string, + action: () => Promise | T, +): Promise { + const inheritedMutation = mutationContext.getStore()?.get(larkAppId); + if (inheritedMutation?.active) return action(); + const state = stateFor(larkAppId); + const inheritedAdmission = admissionContext.getStore()?.get(larkAppId); + + // Explicit abandon actions are discovered inside already-admitted message + // and card handlers. Upgrade that admission instead of nesting and + // deadlocking: release only this handler's lease, drain every other turn, + // perform the mutation, then atomically reacquire the outer lease before the + // gate is reopened. The caller may safely finish delivery/logging under its + // original admission after the exclusive state change. + const upgrading = inheritedAdmission?.active === true; + if (upgrading) { + inheritedAdmission.active = false; + state.activeAdmissions--; + if (state.activeAdmissions === 0) { + wakeAll(state.drainWaiters); + } + } + + while (state.mutating) await waitUntilOpen(state); + state.mutating = true; + await waitUntilDrained(state); + const mutationLease: MutationLease = { active: true }; + try { + const next = new Map(mutationContext.getStore() ?? []); + next.set(larkAppId, mutationLease); + return await mutationContext.run(next, action); + } finally { + // Detached descendants retain the AsyncLocalStorage map. Revoke the + // mutable lease before reopening the gate so they cannot impersonate the + // completed mutation later. + mutationLease.active = false; + // Reacquire before waking queued admissions/mutations. This keeps the + // remainder of an upgraded outer handler inside the admission lifetime and + // prevents its finally block from double-decrementing the gate. + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + state.mutating = false; + wakeAll(state.openWaiters); + // Keep the state object stable. Resolved admission waiters resume in later + // promise jobs and still hold this object; deleting it here would let a new + // caller create a second gate and bypass those about-to-reacquire waiters. + // The key space is bounded by configured bot ids. + } +} + +export type BoundedBotTurnMutationResult = + | { acquired: true; value: T } + | { acquired: false; reason: 'timeout' | 'upgrade_conflict' }; + +/** Acquire one exact mutation lease within a single absolute deadline. + * + * Unlike Promise.race around `withBotTurnMutation`, this removes a timed-out + * waiter and rolls back a gate already closed while admissions were draining. + * Therefore `action` can never run after `{ acquired:false }` was returned. + * + * A same-bot admission may upgrade only when no other mutation already owns + * the gate. Returning `upgrade_conflict` leaves that admission untouched; this + * keeps the bounded path safe without resuming an admission concurrently with + * a mutation it had temporarily released for. */ +export async function tryWithBotTurnMutation( + larkAppId: string, + acquireTimeoutMs: number, + action: () => Promise | T, +): Promise> { + const inheritedMutation = mutationContext.getStore()?.get(larkAppId); + if (inheritedMutation?.active) { + return { acquired: true, value: await action() }; + } + const state = stateFor(larkAppId); + const inheritedAdmission = admissionContext.getStore()?.get(larkAppId); + const upgrading = inheritedAdmission?.active === true; + if (upgrading && state.mutating) { + return { acquired: false, reason: 'upgrade_conflict' }; + } + + const deadlineMs = Date.now() + Math.max(0, acquireTimeoutMs); + if (upgrading) { + inheritedAdmission.active = false; + state.activeAdmissions--; + if (state.activeAdmissions === 0) wakeAll(state.drainWaiters); + } else { + while (state.mutating) { + if (!await waitUntilBefore(state.openWaiters, () => !state.mutating, deadlineMs)) { + return { acquired: false, reason: 'timeout' }; + } + } + } + + if (Date.now() >= deadlineMs) { + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + return { acquired: false, reason: 'timeout' }; + } + + state.mutating = true; + const drained = await waitUntilBefore( + state.drainWaiters, + () => state.activeAdmissions === 0, + deadlineMs, + ); + if (!drained || Date.now() >= deadlineMs) { + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + state.mutating = false; + wakeAll(state.openWaiters); + return { acquired: false, reason: 'timeout' }; + } + + const mutationLease: MutationLease = { active: true }; + try { + const next = new Map(mutationContext.getStore() ?? []); + next.set(larkAppId, mutationLease); + if (Date.now() >= deadlineMs) { + return { acquired: false, reason: 'timeout' }; + } + const value = await mutationContext.run(next, action); + return { acquired: true, value }; + } finally { + mutationLease.active = false; + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + state.mutating = false; + wakeAll(state.openWaiters); + } +} + +/** + * Start an independent admission branch from fire-and-forget code. Normal + * same-bot nesting is intentionally reentrant and MUST be awaited by its + * caller; JavaScript cannot infer whether a returned Promise was floated. + * Detached callers clear both contexts and acquire a fresh counted root lease. + */ +export function runDetachedBotTurnAdmission( + larkAppId: string, + action: () => Promise | T, +): Promise { + return admissionContext.run(new Map(), () => + mutationContext.run(new Map(), () => withBotTurnAdmission(larkAppId, action))); +} + +/** Fire-and-forget counterpart for exclusive lifecycle mutations. A detached + * mutation inside an admission waits for that admission to drain; one inside a + * mutation waits for the parent mutation to reopen the gate. Do not await this + * helper from the parent scope whose lease it must wait on. */ +export function runDetachedBotTurnMutation( + larkAppId: string, + action: () => Promise | T, +): Promise { + return admissionContext.run(new Map(), () => + mutationContext.run(new Map(), () => withBotTurnMutation(larkAppId, action))); +} + +export function __testOnly_resetBotTurnMutationGates(): void { + gates.clear(); +} diff --git a/test/bot-turn-mutation-gate.test.ts b/test/bot-turn-mutation-gate.test.ts new file mode 100644 index 000000000..5dff838e9 --- /dev/null +++ b/test/bot-turn-mutation-gate.test.ts @@ -0,0 +1,400 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + __testOnly_resetBotTurnMutationGates, + runDetachedBotTurnAdmission, + runDetachedBotTurnMutation, + tryWithBotTurnMutation, + withBotTurnAdmission, + withBotTurnMutation, +} from '../src/core/bot-turn-mutation-gate.js'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +describe('per-bot turn mutation gate', () => { + afterEach(() => { + vi.useRealTimers(); + __testOnly_resetBotTurnMutationGates(); + }); + + it('drains an in-flight admission and blocks new turns through the mutation', async () => { + const finishFirst = deferred(); + const finishMutation = deferred(); + const firstEntered = vi.fn(); + const mutationEntered = vi.fn(); + const secondEntered = vi.fn(); + + const first = withBotTurnAdmission('app-a', async () => { + firstEntered(); + await finishFirst.promise; + }); + await vi.waitFor(() => expect(firstEntered).toHaveBeenCalledOnce()); + + const mutation = withBotTurnMutation('app-a', async () => { + mutationEntered(); + await finishMutation.promise; + }); + await Promise.resolve(); + expect(mutationEntered).not.toHaveBeenCalled(); + + const second = withBotTurnAdmission('app-a', () => { secondEntered(); }); + await Promise.resolve(); + expect(secondEntered).not.toHaveBeenCalled(); + + finishFirst.resolve(); + await vi.waitFor(() => expect(mutationEntered).toHaveBeenCalledOnce()); + expect(secondEntered).not.toHaveBeenCalled(); + + finishMutation.resolve(); + await Promise.all([first, mutation, second]); + expect(secondEntered).toHaveBeenCalledOnce(); + }); + + it('does not serialize an unrelated bot', async () => { + const finishMutation = deferred(); + const mutation = withBotTurnMutation('app-a', () => finishMutation.promise); + const other = vi.fn(); + await withBotTurnAdmission('app-b', () => other()); + expect(other).toHaveBeenCalledOnce(); + finishMutation.resolve(); + await mutation; + }); + + it('keeps woken waiters on the canonical state for the next mutation', async () => { + const finishFirstMutation = deferred(); + const finishQueuedAdmission = deferred(); + const firstMutation = withBotTurnMutation('app-a', () => finishFirstMutation.promise); + const queuedEntered = vi.fn(); + const queuedAdmission = withBotTurnAdmission('app-a', async () => { + queuedEntered(); + await finishQueuedAdmission.promise; + }); + await Promise.resolve(); + expect(queuedEntered).not.toHaveBeenCalled(); + + finishFirstMutation.resolve(); + await firstMutation; + await vi.waitFor(() => expect(queuedEntered).toHaveBeenCalledOnce()); + + const nextMutationEntered = vi.fn(); + const nextMutation = withBotTurnMutation('app-a', () => nextMutationEntered()); + await Promise.resolve(); + expect(nextMutationEntered).not.toHaveBeenCalled(); + + finishQueuedAdmission.resolve(); + await Promise.all([queuedAdmission, nextMutation]); + expect(nextMutationEntered).toHaveBeenCalledOnce(); + }); + + it('lets a draining admitted handler enter a nested same-bot boundary', async () => { + const enterNested = deferred(); + const outerEntered = vi.fn(); + const nestedEntered = vi.fn(); + const mutationEntered = vi.fn(); + const outer = withBotTurnAdmission('app-a', async () => { + outerEntered(); + await enterNested.promise; + await withBotTurnAdmission('app-a', () => nestedEntered()); + }); + await vi.waitFor(() => expect(outerEntered).toHaveBeenCalledOnce()); + + const mutation = withBotTurnMutation('app-a', () => mutationEntered()); + await Promise.resolve(); + expect(mutationEntered).not.toHaveBeenCalled(); + + enterNested.resolve(); + await Promise.all([outer, mutation]); + expect(nestedEntered).toHaveBeenCalledOnce(); + expect(mutationEntered).toHaveBeenCalledOnce(); + }); + + it('revokes inherited reentrancy for a detached descendant after the outer lease ends', async () => { + const releaseDetached = deferred(); + const finishMutation = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + + await withBotTurnAdmission('app-a', async () => { + detached = (async () => { + await releaseDetached.promise; + await withBotTurnAdmission('app-a', () => detachedEntered()); + })(); + }); + + const mutation = withBotTurnMutation('app-a', () => finishMutation.promise); + await Promise.resolve(); + releaseDetached.resolve(); + await Promise.resolve(); + await Promise.resolve(); + expect(detachedEntered).not.toHaveBeenCalled(); + + finishMutation.resolve(); + await Promise.all([mutation, detached]); + expect(detachedEntered).toHaveBeenCalledOnce(); + }); + + it('upgrades an admitted handler to a mutation without self-deadlock', async () => { + const order: string[] = []; + await withBotTurnAdmission('app-a', async () => { + order.push('admission'); + await withBotTurnMutation('app-a', async () => { + order.push('mutation'); + await withBotTurnAdmission('app-a', () => order.push('nested-admission')); + await withBotTurnMutation('app-a', () => order.push('nested-mutation')); + }); + order.push('after'); + }); + expect(order).toEqual([ + 'admission', + 'mutation', + 'nested-admission', + 'nested-mutation', + 'after', + ]); + }); + + it('upgrades through an awaited nested admission without retaining an ancestor count', async () => { + const events: string[] = []; + await withBotTurnAdmission('app-a', async () => { + await withBotTurnAdmission('app-a', async () => { + await withBotTurnMutation('app-a', () => events.push('mutated')); + }); + events.push('outer-finished'); + }); + expect(events).toEqual(['mutated', 'outer-finished']); + }); + + it('an upgraded mutation drains peer admissions and blocks later turns', async () => { + const letPeerFinish = deferred(); + const letCloseFinish = deferred(); + const startUpgrade = deferred(); + const events: string[] = []; + + const closer = withBotTurnAdmission('app-a', async () => { + events.push('close-admitted'); + await startUpgrade.promise; + await withBotTurnMutation('app-a', async () => { + events.push('close-mutating'); + await letCloseFinish.promise; + }); + events.push('close-after'); + }); + const peer = withBotTurnAdmission('app-a', async () => { + events.push('peer-admitted'); + await letPeerFinish.promise; + events.push('peer-finished'); + }); + await vi.waitFor(() => expect(events).toContain('peer-admitted')); + + startUpgrade.resolve(); + await Promise.resolve(); + expect(events).not.toContain('close-mutating'); + + letPeerFinish.resolve(); + await vi.waitFor(() => expect(events).toContain('close-mutating')); + + const later = withBotTurnAdmission('app-a', () => events.push('later-admitted')); + await Promise.resolve(); + expect(events).not.toContain('later-admitted'); + + letCloseFinish.resolve(); + await Promise.all([closer, peer, later]); + expect(events.indexOf('peer-finished')).toBeLessThan(events.indexOf('close-mutating')); + expect(events.indexOf('close-mutating')).toBeLessThan(events.indexOf('close-after')); + }); + + it('revokes inherited mutation ownership for detached descendants', async () => { + const releaseDetached = deferred(); + const holdNextMutation = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + + await withBotTurnMutation('app-a', async () => { + detached = (async () => { + await releaseDetached.promise; + await withBotTurnAdmission('app-a', () => detachedEntered()); + })(); + }); + + const nextMutation = withBotTurnMutation('app-a', () => holdNextMutation.promise); + await Promise.resolve(); + releaseDetached.resolve(); + await Promise.resolve(); + await Promise.resolve(); + expect(detachedEntered).not.toHaveBeenCalled(); + + holdNextMutation.resolve(); + await Promise.all([nextMutation, detached]); + expect(detachedEntered).toHaveBeenCalledOnce(); + }); + + it('runs a detached mutation only after its admission parent drains', async () => { + const releaseDetachedMutation = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + + await withBotTurnAdmission('app-a', () => { + detached = runDetachedBotTurnMutation('app-a', async () => { + detachedEntered(); + await releaseDetachedMutation.promise; + }); + expect(detachedEntered).not.toHaveBeenCalled(); + }); + await vi.waitFor(() => expect(detachedEntered).toHaveBeenCalledOnce()); + + releaseDetachedMutation.resolve(); + await detached; + + const laterMutation = vi.fn(); + const laterAdmission = vi.fn(); + await withBotTurnMutation('app-a', () => laterMutation()); + await withBotTurnAdmission('app-a', () => laterAdmission()); + expect(laterMutation).toHaveBeenCalledOnce(); + expect(laterAdmission).toHaveBeenCalledOnce(); + }); + + it('counts an explicit detached admission begun before its parent returns', async () => { + const releaseChild = deferred(); + const childEntered = vi.fn(); + let child!: Promise; + + await withBotTurnAdmission('app-a', () => { + child = runDetachedBotTurnAdmission('app-a', async () => { + childEntered(); + await releaseChild.promise; + }); + }); + expect(childEntered).toHaveBeenCalledOnce(); + + const mutationEntered = vi.fn(); + const mutation = withBotTurnMutation('app-a', () => mutationEntered()); + await Promise.resolve(); + expect(mutationEntered).not.toHaveBeenCalled(); + + releaseChild.resolve(); + await Promise.all([child, mutation]); + expect(mutationEntered).toHaveBeenCalledOnce(); + }); + + it('clears mutation context for an explicit detached mutation', async () => { + const releaseChild = deferred(); + const childEntered = vi.fn(); + let child!: Promise; + + const parent = withBotTurnMutation('app-a', () => { + child = runDetachedBotTurnMutation('app-a', async () => { + childEntered(); + await releaseChild.promise; + }); + }); + await parent; + await vi.waitFor(() => expect(childEntered).toHaveBeenCalledOnce()); + const admissionEntered = vi.fn(); + const admission = withBotTurnAdmission('app-a', () => admissionEntered()); + await Promise.resolve(); + expect(admissionEntered).not.toHaveBeenCalled(); + + releaseChild.resolve(); + await Promise.all([child, admission]); + expect(admissionEntered).toHaveBeenCalledOnce(); + }); + + it('keeps an explicit detached admission behind its mutation parent', async () => { + const holdParent = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + const parent = withBotTurnMutation('app-a', async () => { + detached = runDetachedBotTurnAdmission('app-a', () => detachedEntered()); + await Promise.resolve(); + expect(detachedEntered).not.toHaveBeenCalled(); + await holdParent.promise; + }); + await Promise.resolve(); + holdParent.resolve(); + await parent; + await detached; + expect(detachedEntered).toHaveBeenCalledOnce(); + }); + + it('removes a bounded waiter on timeout so its action can never run later', async () => { + const releaseOwner = deferred(); + const ownerEntered = vi.fn(); + const owner = withBotTurnMutation('app-a', async () => { + ownerEntered(); + await releaseOwner.promise; + }); + await vi.waitFor(() => expect(ownerEntered).toHaveBeenCalledOnce()); + + const staleAction = vi.fn(); + await expect(tryWithBotTurnMutation('app-a', 10, staleAction)).resolves.toEqual({ + acquired: false, + reason: 'timeout', + }); + releaseOwner.resolve(); + await owner; + await new Promise(resolve => setTimeout(resolve, 20)); + expect(staleAction).not.toHaveBeenCalled(); + + const later = vi.fn(); + await expect(tryWithBotTurnMutation('app-a', 50, later)).resolves.toEqual({ + acquired: true, + value: undefined, + }); + expect(later).toHaveBeenCalledOnce(); + }); + + it('rejects an overdue wake even when owner release runs before the timer callback', async () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + const releaseOwner = deferred(); + const ownerEntered = vi.fn(); + const owner = withBotTurnMutation('app-a', async () => { + ownerEntered(); + await releaseOwner.promise; + }); + await Promise.resolve(); + expect(ownerEntered).toHaveBeenCalledOnce(); + + const staleAction = vi.fn(); + const bounded = tryWithBotTurnMutation('app-a', 10, staleAction); + await Promise.resolve(); + // Advance wall time without running the overdue timer. Releasing the owner + // wakes the waiter first; the wake path itself must enforce the deadline. + vi.setSystemTime(1_020); + releaseOwner.resolve(); + await owner; + + await expect(bounded).resolves.toEqual({ acquired: false, reason: 'timeout' }); + expect(staleAction).not.toHaveBeenCalled(); + await vi.runAllTimersAsync(); + expect(staleAction).not.toHaveBeenCalled(); + }); + + it('rolls back a closed gate when admissions do not drain before the deadline', async () => { + const releaseAdmission = deferred(); + const heldEntered = vi.fn(); + const held = withBotTurnAdmission('app-a', async () => { + heldEntered(); + await releaseAdmission.promise; + }); + await vi.waitFor(() => expect(heldEntered).toHaveBeenCalledOnce()); + + const mutationAction = vi.fn(); + const bounded = tryWithBotTurnMutation('app-a', 10, mutationAction); + const admittedAfterRollback = vi.fn(); + const queuedAdmission = withBotTurnAdmission('app-a', admittedAfterRollback); + + await expect(bounded).resolves.toEqual({ acquired: false, reason: 'timeout' }); + await queuedAdmission; + expect(admittedAfterRollback).toHaveBeenCalledOnce(); + expect(mutationAction).not.toHaveBeenCalled(); + + releaseAdmission.resolve(); + await held; + await new Promise(resolve => setTimeout(resolve, 20)); + expect(mutationAction).not.toHaveBeenCalled(); + }); +}); From 09b96ef5cec0c13a68c72006c160ff1deaaa4a3e Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:49:21 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(riff):=20=E4=B8=BA=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E4=B8=8E=E5=AE=88=E6=8A=A4=E8=BF=9B=E7=A8=8B=E9=80=80=E5=87=BA?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=A4=E9=98=B6=E6=AE=B5=E5=85=B3=E5=81=9C?= =?UTF-8?q?=E5=9B=B4=E6=A0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/adapters/backend/riff-backend.ts | 334 ++++- src/adapters/backend/types.ts | 42 +- src/core/explicit-session-backing-cleanup.ts | 110 ++ src/core/persistent-backend.ts | 10 +- src/core/riff-shutdown-detach.ts | 1042 ++++++++++++++++ src/core/riff-worker-shutdown-readiness.ts | 31 + src/core/shutdown-budgets.ts | 33 + src/core/types.ts | 21 + src/core/worker-pool.ts | 603 ++++++++- src/daemon.ts | 199 ++- src/services/session-store.ts | 374 +++++- src/types.ts | 33 +- src/worker.ts | 271 +++- test/explicit-session-backing-cleanup.test.ts | 113 ++ test/persistent-backend-type.test.ts | 4 + test/riff-backend.test.ts | 241 +++- test/riff-explicit-close.test.ts | 283 +++++ test/riff-shutdown-detach.test.ts | 1093 +++++++++++++++++ test/riff-worker-shutdown-readiness.test.ts | 37 + test/session-store.test.ts | 38 + test/worker-riff-retirement-protocol.test.ts | 134 ++ 21 files changed, 4910 insertions(+), 136 deletions(-) create mode 100644 src/core/explicit-session-backing-cleanup.ts create mode 100644 src/core/riff-shutdown-detach.ts create mode 100644 src/core/riff-worker-shutdown-readiness.ts create mode 100644 src/core/shutdown-budgets.ts create mode 100644 test/explicit-session-backing-cleanup.test.ts create mode 100644 test/riff-explicit-close.test.ts create mode 100644 test/riff-shutdown-detach.test.ts create mode 100644 test/riff-worker-shutdown-readiness.test.ts create mode 100644 test/worker-riff-retirement-protocol.test.ts diff --git a/src/adapters/backend/riff-backend.ts b/src/adapters/backend/riff-backend.ts index 4eb793a65..56bd85b5e 100644 --- a/src/adapters/backend/riff-backend.ts +++ b/src/adapters/backend/riff-backend.ts @@ -2,7 +2,12 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { execFileSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import type { SessionBackend, SpawnOpts } from './types.js'; +import type { + SessionBackend, + SessionDestroyResult, + SessionShutdownDetachResult, + SpawnOpts, +} from './types.js'; import { logger } from '../../utils/logger.js'; /** @@ -339,6 +344,24 @@ export class RiffBackend implements SessionBackend { private cancelTimeoutMs = 4_000; private createTimeoutMs = 10_000; private destroyDeadlineMs = 20_000; + /** Exact late/current task whose close cancellation failed. Retained across + * the prepare-close handshake so the daemon can persist a retry handle. */ + private closeFailureTaskId: string | null = null; + private closeFailureError: string | null = null; + private closeLateTaskHandled = false; + private closePrepared = false; + private closeAttempt: symbol | null = null; + private destroyInFlight: Promise | null = null; + private cancelInFlight: Promise | null = null; + private abortInFlight: Promise | null = null; + /** Graceful daemon shutdown is a non-cancelling two-phase detach. It fences + * only writes arriving after prepare; writes already appended to writeChain + * still drain so a late child id can be durably handed to the daemon. */ + private shutdownDetaching = false; + private shutdownDetachPrepared = false; + private shutdownDetachAttempt: symbol | null = null; + private shutdownDetachInFlight: Promise | null = null; + private shutdownDetachAbortInFlight: Promise | null = null; /** Serializes write() → createTask/followUp. Without this, a second message * arriving before the first task-execute HTTP returns would see * currentTaskId === null and create a duplicate task. */ @@ -417,8 +440,16 @@ export class RiffBackend implements SessionBackend { // No actual process to spawn. Task creation happens on first write(). } - write(data: string): void { - if (this.killed || this.closing) return; + write(data: string): boolean { + if (this.killed) return false; + if (this.shutdownDetaching) { + logger.warn('[riff] write rejected while graceful shutdown detach is preparing/prepared'); + return false; + } + if (this.closing) { + logger.warn('[riff] write rejected while explicit close is preparing/prepared'); + return false; + } const { text, attachments } = this.extractAttachments(data); @@ -440,6 +471,7 @@ export class RiffBackend implements SessionBackend { .catch((err) => { logger.warn(`[riff] queued write failed: ${err}`); }); + return true; } resize(_cols: number, _rows: number): void { @@ -467,8 +499,8 @@ export class RiffBackend implements SessionBackend { this.exitCb?.(0, null); } - async destroySession(): Promise { - // /close（及 /restart 的替换路径）必须把远端任务真正取消掉——fire-and-forget + async destroySession(): Promise { + // /close 必须把远端任务真正取消掉——fire-and-forget // 在 worker 紧接 process.exit 时大概率发不出去，已关闭话题的远端 agent 会 // 继续拿着注入的凭证发消息。有界 await + 一次重试，失败也明确留痕。 // @@ -476,36 +508,249 @@ export class RiffBackend implements SessionBackend { // currentTaskId 还是 null/旧值，直接 cancel 会漏掉 late task。先立 closing // 门（拒新写 + 令 in-flight 完成后自取消），再有界等 writeChain 沉降，最后 // cancel 沉降后的 current task。 + if (this.shutdownDetaching) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'shutdown_detach_in_progress', + }; + } + if (this.destroyInFlight) return this.destroyInFlight; + if (this.closePrepared) { + return { + ok: true, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + }; + } + const attempt = Symbol('riff-close-prepare'); + this.closeAttempt = attempt; this.closing = true; + this.closeFailureTaskId = null; + this.closeFailureError = null; + this.closeLateTaskHandled = false; // 预算层级（单调覆盖，无内层 race——writeChain 本身有界）： // create/follow-up fetch 10s + late cancel 4s×2 = chain 最坏 18s // own cancel 4s×2 = 8s（与 late 情形互斥：closing 分支不登记 current） - // → destroySession 总 deadline 20s → worker close/restart race 22s + // → destroySession 总 deadline 20s → worker close handshake 22s // → daemon SIGTERM backstop 24s / SIGKILL 29s。 // 对 writeChain 只整体 await：单独给它小窗口会在窗口边缘掐掉链内的 // late cancel（create 于 t≈窗口末返回 → cancel 尚 pending → teardown 提前 // resolve → process.exit 掐断取消）。 - const teardown = (async () => { + const teardown = (async (): Promise => { try { await this.writeChain; } catch { /* writeChain never rejects (caught internally) */ } - if (this.currentTaskId && !this.taskDone) { + if (this.closeAttempt !== attempt) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + if (this.closeFailureTaskId) { + return { + ok: false, + taskId: this.closeFailureTaskId, + error: this.closeFailureError ?? 'late_task_cancel_failed', + }; + } + // A task materialized while closing was already cancelled inside the + // writeChain. Do not then cancel its stale parent lineage as if it were + // still the active execution. + if (!this.closeLateTaskHandled && this.currentTaskId && !this.taskDone) { const id = this.currentTaskId; - try { - await this.cancelTask(id); + const cancelled = await this.cancelTaskWithRetry(id, 'close'); + // abortDestroySession invalidates the exact attempt before waiting for + // an already-issued cancellation. A late successful HTTP response must + // not resurrect that aborted generation as a prepared close. + if (this.closeAttempt !== attempt || !this.closing) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + if (cancelled) { logger.info(`[riff] task ${id} cancelled on close`); - } catch (err) { - try { - await this.cancelTask(id); - logger.info(`[riff] task ${id} cancelled on close (retry)`); - } catch (err2) { - logger.warn(`[riff] task-cancel failed on close (task ${id} may keep running remotely): ${err2}`); - } + } else { + return { + ok: false, + taskId: id, + error: this.closeFailureError ?? 'task_cancel_failed', + }; } } + if (this.closeAttempt !== attempt || !this.closing) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + return { + ok: true, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + }; })(); - await Promise.race([teardown, new Promise((r) => setTimeout(r, this.destroyDeadlineMs))]); - this.kill(); + this.destroyInFlight = Promise.race([ + teardown, + new Promise(resolve => setTimeout(() => resolve({ + ok: false, + ...(this.closeFailureTaskId || this.currentTaskId + ? { taskId: this.closeFailureTaskId ?? this.currentTaskId! } + : {}), + error: 'close_timeout', + }), this.destroyDeadlineMs)), + ]).then(async (result) => { + // Promise.race and the teardown continuation each add a microtask + // boundary. Revalidate the generation immediately before publishing the + // prepared bit so a concurrent abort can never be overwritten. + if (result.ok && (this.closeAttempt !== attempt || !this.closing)) { + result = { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + if (result.ok) { + this.closePrepared = true; + } else { + // A failed prepare is not a terminal close. Restore admission so the + // still-active durable owner can accept a follow-up or a close retry. + await this.abortDestroySession(); + } + return result; + }).finally(() => { + this.destroyInFlight = null; + }); + return this.destroyInFlight; + } + + async abortDestroySession(): Promise { + if (this.killed) return; + if (this.abortInFlight) return this.abortInFlight; + this.closeAttempt = null; + this.closePrepared = false; + const pendingCancel = this.cancelInFlight; + this.abortInFlight = (async () => { + // A close timeout can win Promise.race after task-cancel was already + // issued. Reopening admission before that request settles lets a new + // follow-up race a late successful cancellation of its parent. Keep the + // backend fenced until the exact cancellation attempt reaches terminal. + if (pendingCancel) { + try { await pendingCancel; } catch { /* cancel helper returns boolean */ } + } + if (this.killed || this.closeAttempt !== null || this.closePrepared) return; + this.closing = false; + this.closeFailureTaskId = null; + this.closeFailureError = null; + this.closeLateTaskHandled = false; + logger.info('[riff] explicit close aborted; write admission restored'); + })().finally(() => { + this.abortInFlight = null; + }); + return this.abortInFlight; + } + + commitDestroySession(): void { + // The daemon has durably published the closed row. Keep admission fenced + // until the worker immediately detaches/exits. + this.closePrepared = false; + this.closeAttempt = null; + this.closing = true; + } + + async prepareShutdownDetach(): Promise { + if (this.shutdownDetachInFlight) return this.shutdownDetachInFlight; + if (this.shutdownDetachPrepared) { + return { ok: true, taskId: this.currentTaskId }; + } + if (this.killed) { + return { ok: false, taskId: this.currentTaskId, error: 'backend_killed' }; + } + if (this.closing || this.destroyInFlight || this.closePrepared) { + return { ok: false, taskId: this.currentTaskId, error: 'explicit_close_in_progress' }; + } + + const attempt = Symbol('riff-shutdown-detach'); + this.shutdownDetachAttempt = attempt; + this.shutdownDetaching = true; + // Existing SSE delivery is presentation-only. Stop it now, but do not + // cancel the remote task. Any create/follow-up already accepted before the + // fence remains in writeChain and is allowed to materialize below. + this.abortController?.abort(); + + const drain = (async (): Promise => { + try { await this.writeChain; } + catch { /* writeChain catches its own failures */ } + if (this.killed || this.shutdownDetachAttempt !== attempt || !this.shutdownDetaching) { + return { ok: false, taskId: this.currentTaskId, error: 'shutdown_detach_aborted' }; + } + if (this.closing || this.closePrepared) { + return { ok: false, taskId: this.currentTaskId, error: 'explicit_close_in_progress' }; + } + this.shutdownDetachPrepared = true; + logger.info( + `[riff] graceful shutdown detach prepared` + + `${this.currentTaskId ? ` (task ${this.currentTaskId})` : ' (no task lineage)'}`, + ); + return { ok: true, taskId: this.currentTaskId }; + })(); + this.shutdownDetachInFlight = drain.finally(() => { + this.shutdownDetachInFlight = null; + }); + return this.shutdownDetachInFlight; + } + + async abortShutdownDetach(): Promise { + if (this.killed) { + return { ok: false, taskId: this.currentTaskId, error: 'backend_killed' }; + } + if (this.shutdownDetachAbortInFlight) return this.shutdownDetachAbortInFlight; + const pending = this.shutdownDetachInFlight; + const pendingCancel = this.cancelInFlight; + this.shutdownDetachAttempt = null; + this.shutdownDetachPrepared = false; + this.shutdownDetachAbortInFlight = (async (): Promise => { + // Normally shutdown detach never cancels a remote task. Still wait for + // any exact cancellation already issued by an overlapping explicit close + // before reopening admission, otherwise its late result could invalidate + // a newly accepted follow-up. + await Promise.all([ + pending ? pending.catch(() => undefined) : Promise.resolve(), + pendingCancel ? pendingCancel.catch(() => false) : Promise.resolve(), + ]); + if (this.killed) { + return { ok: false, taskId: this.currentTaskId, error: 'backend_killed' }; + } + if (this.closing || this.shutdownDetachAttempt !== null) { + return { + ok: false, + taskId: this.currentTaskId, + error: this.closing ? 'explicit_close_in_progress' : 'new_shutdown_detach_in_progress', + }; + } + this.shutdownDetaching = false; + // prepare stopped SSE before the persistence ACK. If shutdown is + // aborted, reconnect the exact current task so the still-live owner + // resumes normal output and completion tracking. + if (this.currentTaskId && !this.taskDone) { + this.reconnectAttempts = 0; + void this.streamTask(this.currentTaskId); + } + logger.info('[riff] graceful shutdown detach aborted; write admission restored'); + return { ok: true, taskId: this.currentTaskId }; + })().finally(() => { + this.shutdownDetachAbortInFlight = null; + }); + return this.shutdownDetachAbortInFlight; + } + + commitShutdownDetach(): void { + this.shutdownDetachPrepared = false; + this.shutdownDetachAttempt = null; + // Keep admission fenced until the worker exits immediately after commit. + this.shutdownDetaching = true; } getChildPid(): number | null { @@ -760,8 +1005,8 @@ export class RiffBackend implements SessionBackend { * Post-await adoption gate for a freshly created/followed-up task id. * - closing（/close 竞态窗口）：这个 late task 已经没有会话可服务——立即取消 * （有界+一次重试），绝不 stream/登记，防远端 orphan； - * - killed（detach：重启/休眠）：登记 id 让 daemon 持久化血缘，但不 stream - * （任务合法续跑，重启后 follow-up 接上）； + * - killed / shutdownDetaching（detach）：登记 id 让 daemon 持久化血缘， + * 但不 stream（任务合法续跑，重启后 follow-up 接上）； * - 正常：登记 + 由调用方启动 stream。 */ private async adoptLateTask(taskId: string): Promise { @@ -769,20 +1014,21 @@ export class RiffBackend implements SessionBackend { // 在 writeChain 内 await——destroySession 等 writeChain 沉降时就能把这次 // 取消一起等到（void 触发会在 worker exit 时被掐断）。 logger.info(`[riff] task ${taskId} created during close — cancelling late task`); - try { - await this.cancelTask(taskId); - } catch { - try { - await this.cancelTask(taskId); - } catch (err) { - logger.warn(`[riff] late-task cancel failed (task ${taskId} may keep running remotely): ${err}`); - } - } + this.closeLateTaskHandled = true; + // Preserve the exact newest lineage even when cancellation succeeds. + // If the daemon cannot durably commit the close and sends abort, the + // next follow-up must continue from this child rather than its stale + // parent. Publishing before the cancel also makes a failed cancel + // retryable by the daemon. + this.currentTaskId = taskId; + this.taskIdCb?.(taskId); + const cancelled = await this.cancelTaskWithRetry(taskId, 'late-task close'); + if (!cancelled) this.taskDone = false; return false; } this.currentTaskId = taskId; this.taskIdCb?.(taskId); - if (this.killed) return false; + if (this.killed || this.shutdownDetaching) return false; return true; } @@ -809,6 +1055,32 @@ export class RiffBackend implements SessionBackend { if (!resp.ok) throw new Error(`task-cancel HTTP ${resp.status}`); } + private async cancelTaskWithRetry(taskId: string, context: string): Promise { + const operation = (async (): Promise => { + try { + await this.cancelTask(taskId); + return true; + } catch { + try { + await this.cancelTask(taskId); + logger.info(`[riff] task ${taskId} cancelled on ${context} (retry)`); + return true; + } catch (err) { + this.closeFailureTaskId = taskId; + this.closeFailureError = err instanceof Error ? err.message : String(err); + logger.warn(`[riff] ${context} cancel failed (task ${taskId} may keep running remotely): ${err}`); + return false; + } + } + })(); + this.cancelInFlight = operation; + try { + return await operation; + } finally { + if (this.cancelInFlight === operation) this.cancelInFlight = null; + } + } + private async streamTask(taskId: string): Promise { const url = `${this.config.baseUrl}/api2/task-stream?id=${encodeURIComponent(taskId)}`; const headers: Record = {}; diff --git a/src/adapters/backend/types.ts b/src/adapters/backend/types.ts index c709e470c..996ad8d96 100644 --- a/src/adapters/backend/types.ts +++ b/src/adapters/backend/types.ts @@ -53,7 +53,9 @@ export interface SpawnOpts { export interface SessionBackend { spawn(bin: string, args: string[], opts: SpawnOpts): void; - write(data: string): void; + /** Returns false only when the backend can prove the write was not accepted. + * Legacy implementations may return void on success. */ + write(data: string): void | boolean; resize(cols: number, rows: number): void; onData(cb: (data: string) => void): void; onExit(cb: (code: number | null, signal: string | null) => void): void; @@ -87,8 +89,42 @@ export interface SessionBackend { * so the follow-up lineage survives daemon restarts. `null` clears the * persisted lineage (follow-up failed → next message starts fresh). */ onTaskId?(cb: (taskId: string | null) => void): void; - /** Async-capable teardown: riff awaits the remote task-cancel here. */ - destroySession?(): void | Promise; + /** Async-capable teardown: Riff returns a confirmed cancellation result so + * daemon-driven explicit close can fence late create/follow-up races before + * publishing the durable closed row. Local backends remain synchronous. */ + destroySession?(): void | Promise; + /** Roll back a successful remote prepare when the daemon could not commit + * the durable closed row. The backend must restore write admission without + * discarding the last task lineage. */ + abortDestroySession?(): void | Promise; + /** Finalize a successful prepare after the durable row is closed. */ + commitDestroySession?(): void; + /** Graceful daemon shutdown for a remote-task backend. Fence only NEW + * writes, drain every write accepted before the fence, and return the exact + * final lineage without cancelling it. The daemon persists that lineage + * before telling the worker it may exit. */ + prepareShutdownDetach?(): Promise; + /** Restore admission/streaming when the daemon cannot complete a prepared + * shutdown detach (for example, lineage persistence failed). */ + abortShutdownDetach?(): SessionShutdownDetachResult | Promise; + /** Finalize a shutdown detach after exact lineage persistence has been + * acknowledged by the daemon. Shutdown refusal never cancels remote work. */ + commitShutdownDetach?(): void; +} + +export interface SessionDestroyResult { + ok: boolean; + /** Exact remote task that failed cancellation and must remain retryable. */ + taskId?: string; + error?: string; +} + +export interface SessionShutdownDetachResult { + ok: boolean; + /** Exact final lineage after all pre-fence writes have drained. `null` is + * authoritative and clears any stale durable parent. */ + taskId: string | null; + error?: string; } /** diff --git a/src/core/explicit-session-backing-cleanup.ts b/src/core/explicit-session-backing-cleanup.ts new file mode 100644 index 000000000..d5b560194 --- /dev/null +++ b/src/core/explicit-session-backing-cleanup.ts @@ -0,0 +1,110 @@ +import type { BackendType } from '../adapters/backend/types.js'; +import type { RiffBackendConfig } from '../adapters/backend/riff-backend.js'; +import { cancelRiffTaskById } from '../adapters/backend/riff-backend.js'; +import { + isSuspendableBackendType, + killPersistentSession, + persistentSessionName, + probePersistentSession, +} from './persistent-backend.js'; + +export type ExplicitSessionBackingCleanupResult = + | { ok: true; kind: 'skipped_adopted' | 'no_backing' } + | { ok: true; kind: 'destroyed_persistent'; backendType: 'tmux' | 'herdr' | 'zellij'; name: string } + | { ok: true; kind: 'cancelled_riff'; taskId: string } + | { + ok: false; + kind: 'persistent_destroy_failed' | 'riff_config_missing' | 'riff_cancel_failed'; + backendType: BackendType; + taskId?: string; + error?: string; + }; + +export interface ExplicitSessionBackingCleanupInput { + sessionId: string; + backendType?: BackendType; + riffParentTaskId?: string; + /** Adopted panes/agents are user-owned. Explicit Botmux close only detaches + * its logical row and must never destroy the observed backing resource. */ + adopted?: boolean; + /** Current authoritative config for the row's bot. Required only when the + * row is frozen to Riff and still carries a remote task id. */ + riffConfig?: RiffBackendConfig; +} + +export interface ExplicitSessionBackingCleanupDeps { + cancelRiffTask?: typeof cancelRiffTaskById; + killPersistent?: typeof killPersistentSession; + probePersistent?: typeof probePersistentSession; +} + +/** + * Destroy the Botmux-owned backing resource before an offline/worker-less + * explicit close is published. + * + * This helper never mutates the session record. In particular, callers may + * erase `riffParentTaskId` only after `kind === 'cancelled_riff'`; a failed or + * unconfigurable cancellation therefore retains the durable retry handle. + */ +export async function cleanupExplicitSessionBacking( + input: ExplicitSessionBackingCleanupInput, + deps: ExplicitSessionBackingCleanupDeps = {}, +): Promise { + if (input.adopted) return { ok: true, kind: 'skipped_adopted' }; + + if (input.backendType === 'riff') { + const taskId = input.riffParentTaskId; + if (!taskId) return { ok: true, kind: 'no_backing' }; + if (!input.riffConfig?.baseUrl) { + return { ok: false, kind: 'riff_config_missing', backendType: 'riff', taskId }; + } + const cancel = deps.cancelRiffTask ?? cancelRiffTaskById; + try { + const cancelled = await cancel(input.riffConfig, taskId); + return cancelled + ? { ok: true, kind: 'cancelled_riff', taskId } + : { ok: false, kind: 'riff_cancel_failed', backendType: 'riff', taskId }; + } catch (err) { + return { + ok: false, + kind: 'riff_cancel_failed', + backendType: 'riff', + taskId, + error: err instanceof Error ? err.message : String(err), + }; + } + } + + if (!isSuspendableBackendType(input.backendType)) { + // Explicit pty and legacy unstamped rows have no deterministic persistent + // backing session to destroy. Never guess that an unstamped row was tmux. + return { ok: true, kind: 'no_backing' }; + } + + const name = persistentSessionName(input.backendType, input.sessionId); + const kill = deps.killPersistent ?? killPersistentSession; + const probe = deps.probePersistent ?? probePersistentSession; + try { + kill(input.backendType, name); + // Backend kill helpers are intentionally idempotent and historically + // swallow command errors. Offline explicit abandon needs a stronger + // contract: only publish closed after a post-kill probe confirms absence. + const after = probe(input.backendType, name); + if (after !== 'missing') { + return { + ok: false, + kind: 'persistent_destroy_failed', + backendType: input.backendType, + error: after === 'exists' ? 'backing_session_still_exists' : 'backing_session_state_unknown', + }; + } + return { ok: true, kind: 'destroyed_persistent', backendType: input.backendType, name }; + } catch (err) { + return { + ok: false, + kind: 'persistent_destroy_failed', + backendType: input.backendType, + error: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/src/core/persistent-backend.ts b/src/core/persistent-backend.ts index a91b89330..88c886489 100644 --- a/src/core/persistent-backend.ts +++ b/src/core/persistent-backend.ts @@ -119,12 +119,12 @@ export function resolvePairedSpawnBackendType( * Freezing here stops a live backendType edit from changing how a running session * tears down — e.g. detach-preserving a "herdr" session whose real pane is tmux. */ -export function shutdownBackendDisposition(ds: DaemonSession): 'detach' | 'close' { - // riff：远端任务独立于本地进程存活。daemon shutdown 走 'close' 会经 worker 的 - // destroySession() 取消远端任务——重启不该杀任务（血缘已持久化，重启后 - // follow-up 续上，agent 的 botmux send 照常送达）。detach = 仅 SIGTERM worker。 +export function shutdownBackendDisposition(ds: DaemonSession): 'riff-drain-detach' | 'detach' | 'close' { + // Riff 不能落进普通 detach 的直接 SIGTERM：create/follow-up 最长 10s 才返回 + // task id，而 worker SIGTERM 会立即 exit，丢掉唯一血缘。独立 disposition 迫使 + // daemon 先走 drain → durable ACK → commit 协议；类型检查防止未来回归。 const frozen = ds.initConfig?.backendType ?? ds.session.backendType; - if (frozen === 'riff') return 'detach'; + if (frozen === 'riff') return 'riff-drain-detach'; return getSessionPersistentBackendType(ds) ? 'detach' : 'close'; } diff --git a/src/core/riff-shutdown-detach.ts b/src/core/riff-shutdown-detach.ts new file mode 100644 index 000000000..79bbbb9ef --- /dev/null +++ b/src/core/riff-shutdown-detach.ts @@ -0,0 +1,1042 @@ +import { randomUUID } from 'node:crypto'; +import type { ChildProcess } from 'node:child_process'; +import type { DaemonToWorker, WorkerToDaemon } from '../types.js'; +import * as sessionStore from '../services/session-store.js'; +import { logger } from '../utils/logger.js'; +import type { DaemonSession } from './types.js'; +import { + RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS, + RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS, + RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS, +} from './shutdown-budgets.js'; + +type ShutdownPhase = 'prepare' | 'abort'; + +type ShutdownPhaseResult = { + ok: boolean; + taskId: string | null; + error?: string; +}; + +export type PreparedRiffShutdown = { + ok: true; + fence: 'prepared'; + requestId: string; + taskId: string | null; + /** Runtime lineage sampled before the worker fence. A workerless transaction + * must still own this exact value at persistence time; a live transaction may + * advance only to its drained `taskId`. */ + runtimeTaskIdAtPrepare: string | null; + /** Fresh durable lineage sampled before installing the fence. Phase 2 uses + * it as a lock-protected compare-and-set guard, while also accepting an + * already-idempotent target written by the ordinary task-id callback. */ + durableTaskIdAtPrepare: string | null; + durableOwnerAtPrepare: { + pid: number | null; + larkAppId: string | null; + backendType: string | null; + }; + /** Set only after the exact cross-process fresh read succeeds. Used when a + * prepared worker exits before an all-or-nothing rollback can reach it. */ + lineageVerified: boolean; + /** Exact generation fenced by `requestId`. Null means the active logical + * session was already workerless and only its runtime lineage needs an exact + * durable verification. */ + worker: ChildProcess | null; +}; + +export type RiffShutdownFailure = { + ok: false; + requestId?: string; + taskId: string | null; + error: string; + /** Phase-2 coordinator policy. Ownership ambiguity must stay fenced; a + * plain atomic-write I/O failure may restore the exact prepared worker. */ + rollbackDisposition?: 'abort_safe' | 'retain_fence'; +}; + +/** A prepare refusal that is proven to have happened before the worker/backend + * fence. It must never be sent an abort request. */ +export type UnfencedRiffShutdownRefusal = RiffShutdownFailure & { + fence: 'none'; +}; + +/** A prepare attempt whose exact worker may have installed its backend fence. + * Preparation deliberately does not restore admission inline; the fleet + * coordinator includes this handle in its one concurrent abort wave. */ +export type PossiblyFencedRiffShutdown = RiffShutdownFailure & { + fence: 'possible'; + requestId: string; + worker: ChildProcess; + expectedAbortTaskId?: string | null; +}; + +export type RiffShutdownPrepareResult = + | PreparedRiffShutdown + | UnfencedRiffShutdownRefusal + | PossiblyFencedRiffShutdown; + +export type RiffShutdownPrepareOptions = { + drainTimeoutMs?: number; + abortTimeoutMs?: number; + /** Absolute transaction deadline. A worker is never asked to fence unless + * phase-2 plus the configured admission-restore reserve remain after drain. */ + deadlineMs?: number; + now?: () => number; + /** One projection sampled by prepareRiffFleetForShutdown before any fence. */ + durableSnapshot?: sessionStore.ActiveRiffShutdownSnapshot; +}; + +export type RiffFleetPrepareEntry = { + ds: DaemonSession; + result: RiffShutdownPrepareResult; +}; + +export type FencedRiffShutdownParticipant = + | PreparedRiffShutdown + | PossiblyFencedRiffShutdown; + +export type UniqueDaemonShutdownSessions = + | { ok: true; sessions: DaemonSession[] } + | { ok: false; sessionId: string; error: string }; + +/** The active registry can retain multiple aliases to one exact runtime object + * after transfer/restore. Process that object once, but refuse two distinct + * objects claiming the same durable session id: there is no unique generation + * that shutdown can safely fence or retire. */ +export function collectUniqueDaemonShutdownSessions( + candidates: Iterable, +): UniqueDaemonShutdownSessions { + const seenObjects = new Set(); + const bySessionId = new Map(); + const sessions: DaemonSession[] = []; + for (const ds of candidates) { + if (seenObjects.has(ds)) continue; + seenObjects.add(ds); + const sessionId = ds.session.sessionId; + const existing = bySessionId.get(sessionId); + if (existing && existing !== ds) { + return { + ok: false, + sessionId, + error: `distinct daemon session generations share session id ${sessionId}`, + }; + } + bySessionId.set(sessionId, ds); + sessions.push(ds); + } + return { ok: true, sessions }; +} + +export type RiffShutdownDetachOutcome = + | { + ok: true; + requestId: string; + taskId: string | null; + disposition: 'lineage_persisted'; + worker?: ChildProcess; + } + | RiffShutdownFailure; + +function label(ds: DaemonSession): string { + return ds.session.sessionId.slice(0, 8); +} + +function workerHasExited(worker: ChildProcess): boolean { + return (worker.exitCode !== null && worker.exitCode !== undefined) + || (worker.signalCode !== null && worker.signalCode !== undefined); +} + +function send(worker: ChildProcess, message: DaemonToWorker): boolean { + try { + worker.send(message); + return true; + } catch { + return false; + } +} + +/** Attach the exact-worker response/exit listeners before publishing a phase + * request. Results from another generation/session/request are inert. */ +function requestPhase( + ds: DaemonSession, + worker: ChildProcess, + requestId: string, + phase: ShutdownPhase, + timeoutMs: number, +): Promise { + return new Promise(resolve => { + let settled = false; + const finish = (result: ShutdownPhaseResult): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.removeListener('message', onMessage); + worker.removeListener('exit', onExit); + resolve(result); + }; + const onMessage = (raw: unknown): void => { + const msg = raw as WorkerToDaemon; + if (msg?.type !== 'riff_shutdown_result' + || msg.requestId !== requestId + || msg.phase !== phase) return; + if (ds.worker !== worker) { + finish({ ok: false, taskId: msg.taskId, error: 'stale_worker_generation' }); + return; + } + finish({ + ok: msg.ok, + taskId: msg.taskId, + ...(msg.error ? { error: msg.error } : {}), + }); + }; + const onExit = (): void => finish({ + ok: false, + taskId: null, + error: `worker_exited_during_shutdown_${phase}`, + }); + const timer = setTimeout(() => finish({ + ok: false, + taskId: null, + error: `riff_shutdown_${phase}_timeout`, + }), timeoutMs); + timer.unref?.(); + worker.on('message', onMessage); + worker.once('exit', onExit); + const message: DaemonToWorker = phase === 'prepare' + ? { type: 'riff_shutdown_prepare', requestId } + : { type: 'riff_shutdown_abort', requestId }; + if (!send(worker, message)) { + finish({ ok: false, taskId: null, error: `riff_shutdown_${phase}_send_failed` }); + } + }); +} + +function clearWorkerOwnership(ds: DaemonSession, worker: ChildProcess): void { + if (ds.worker !== worker) return; + ds.worker = null; + ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; + ds.managedTurnOrigin = undefined; + ds.riffShutdownState = undefined; +} + +function clearWorkerlessShutdownState(ds: DaemonSession, requestId: string): void { + if (ds.riffShutdownState?.requestId !== requestId) return; + if (ds.worker) return; + ds.riffShutdownState = undefined; +} + +/** Daemon-owned accepted input has not crossed worker IPC yet. The bot-wide + * mutation lease blocks new admissions, but these older buffers must also be + * empty before a worker can be detached. */ +function daemonInputBlocker(ds: DaemonSession): string | null { + const parts: string[] = []; + if (ds.pendingRepoCommitInFlight || ds.worktreeCreating) parts.push('initial_start=1'); + if (ds.pendingPrompt) parts.push('prompt=1'); + if (ds.pendingRawInput) parts.push('raw=1'); + if (ds.pendingFollowUpInput) parts.push('raw_followup=1'); + if ((ds.pendingFollowUps?.length ?? 0) > 0) { + parts.push(`followups=${ds.pendingFollowUps!.length}`); + } + if (ds.session.queued) parts.push('queued=1'); + return parts.length > 0 ? parts.join(',') : null; +} + +async function abortWorkerPreparation( + ds: DaemonSession, + worker: ChildProcess, + requestId: string, + timeoutMs: number, + lineageVerified = false, + exactTask?: { taskId: string | null }, +): Promise { + if (workerHasExited(worker)) { + if (ds.worker && ds.worker !== worker) { + return { + ok: false, + taskId: ds.riffShutdownState?.taskId ?? ds.session.riffParentTaskId ?? null, + error: 'new_worker_generation', + }; + } + if (lineageVerified) { + if (ds.worker === worker) clearWorkerOwnership(ds, worker); + else clearWorkerlessShutdownState(ds, requestId); + return { ok: true, taskId: ds.session.riffParentTaskId ?? null }; + } + // No backend remains to ACK admission restoration and the drained lineage + // was not durably recovered. Retain a daemon-side fence instead of claiming + // rollback success and permitting a stale-lineage replacement. + if (!ds.riffShutdownState || ds.riffShutdownState.requestId === requestId) { + ds.riffShutdownState = { + phase: 'prepared', + requestId, + taskId: ds.session.riffParentTaskId ?? null, + }; + } + return { + ok: false, + taskId: ds.session.riffParentTaskId ?? null, + error: 'worker_exited_before_admission_restore', + }; + } + if (ds.worker !== worker) { + return { + ok: false, + taskId: ds.riffShutdownState?.taskId ?? ds.session.riffParentTaskId ?? null, + error: 'new_worker_generation', + }; + } + const result = await requestPhase(ds, worker, requestId, 'abort', timeoutMs); + if (result.ok && exactTask && result.taskId !== exactTask.taskId) { + return { + ok: false, + taskId: result.taskId, + error: `abort_task_lineage_mismatch:expected=${exactTask.taskId ?? 'none'},` + + `actual=${result.taskId ?? 'none'}`, + }; + } + if (result.ok && ds.riffShutdownState?.requestId === requestId) { + ds.riffShutdownState = undefined; + } + return result; +} + +function unfencedRefusal( + ds: DaemonSession, + error: string, + requestId?: string, +): UnfencedRiffShutdownRefusal { + return { + ok: false, + fence: 'none', + ...(requestId ? { requestId } : {}), + taskId: ds.session.riffParentTaskId ?? null, + error, + }; +} + +/** Phase 1: fence one exact Riff generation and drain task-id materialization. + * Nothing exits or restores admission here. A failure after the prepare send + * returns an exact possibly-fenced handle so all peers can be restored in one + * concurrent fleet wave rather than serial drain+abort chains. */ +export async function prepareRiffSessionForShutdown( + ds: DaemonSession, + options: RiffShutdownPrepareOptions = {}, +): Promise { + const frozenBackend = ds.initConfig?.backendType ?? ds.session.backendType; + if (frozenBackend !== 'riff') { + return unfencedRefusal(ds, 'not_riff_backend'); + } + if (ds.riffCloseState || ds.riffShutdownState) { + return unfencedRefusal( + ds, + ds.riffCloseState ? 'explicit_close_in_progress' : 'shutdown_detach_in_progress', + ); + } + const daemonBlockerBeforePrepare = daemonInputBlocker(ds); + if (daemonBlockerBeforePrepare) { + return unfencedRefusal(ds, `daemon_inputs_not_drained:${daemonBlockerBeforePrepare}`); + } + + const runtimeTaskIdAtPrepare = ds.session.riffParentTaskId ?? null; + let durableSnapshot = options.durableSnapshot; + if (!durableSnapshot) { + try { + [durableSnapshot] = sessionStore.getActiveRiffShutdownSnapshotsBatch([ + ds.session.sessionId, + ]); + } catch (err) { + return unfencedRefusal( + ds, + `durable_session_read_failed:${err instanceof Error ? err.message : String(err)}`, + ); + } + } + if (!durableSnapshot || durableSnapshot.sessionId !== ds.session.sessionId) { + return unfencedRefusal(ds, 'durable_session_snapshot_mismatch'); + } + const durableTaskIdAtPrepare = durableSnapshot.taskId; + const runtimeOwner = { + pid: ds.session.pid ?? null, + larkAppId: ds.session.larkAppId ?? null, + backendType: ds.session.backendType ?? null, + }; + const durableOwnerAtPrepare = durableSnapshot.owner; + if (durableOwnerAtPrepare.pid !== runtimeOwner.pid + || durableOwnerAtPrepare.larkAppId !== runtimeOwner.larkAppId + || durableOwnerAtPrepare.backendType !== runtimeOwner.backendType) { + return unfencedRefusal( + ds, + `durable_session_owner_mismatch:current=${JSON.stringify(durableOwnerAtPrepare)},` + + `runtime=${JSON.stringify(runtimeOwner)}`, + ); + } + + const now = options.now ?? Date.now; + const abortReserveMs = options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS; + let drainTimeoutMs = options.drainTimeoutMs ?? RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS; + if (options.deadlineMs !== undefined) { + const availableDrainMs = options.deadlineMs + - now() + - RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS + - abortReserveMs; + if (availableDrainMs <= 0) { + return unfencedRefusal(ds, 'insufficient_abort_budget_before_fence'); + } + drainTimeoutMs = Math.min(drainTimeoutMs, availableDrainMs); + } + + const requestId = randomUUID(); + const worker = ds.worker; + if (!worker || workerHasExited(worker)) { + // Retire a previously observed dead handle before installing the + // workerless fence. Any later non-null ds.worker is then unambiguously a + // new generation and cannot be blessed by this transaction. + if (worker) clearWorkerOwnership(ds, worker); + // Workerless rows can still carry a newer runtime-only lineage after a + // failed ordinary riff_task_id save. Fence daemon admission now; phase 2 + // performs an exact owner/lineage CAS and fresh disk verification. + ds.riffShutdownState = { + phase: 'prepared', + requestId, + taskId: ds.session.riffParentTaskId ?? null, + }; + return { + ok: true, + fence: 'prepared', + requestId, + taskId: ds.session.riffParentTaskId ?? null, + runtimeTaskIdAtPrepare, + durableTaskIdAtPrepare, + durableOwnerAtPrepare, + lineageVerified: false, + worker: null, + }; + } + + ds.riffShutdownState = { phase: 'preparing', requestId }; + const prepared = await requestPhase( + ds, + worker, + requestId, + 'prepare', + drainTimeoutMs, + ); + if (!prepared.ok) { + // This exact response proves the worker refused before installing a backend + // fence. Every other failure has ambiguous fence state and requires a + // positive admission-restored ACK. + const unfencedWorkerRefusal = prepared.error === 'explicit_close_in_progress' + || prepared.error === 'not_riff_backend' + || prepared.error === 'riff_shutdown_prepare_send_failed' + || prepared.error?.startsWith('worker_inputs_not_drained:') === true; + if (unfencedWorkerRefusal && ds.riffShutdownState?.requestId === requestId) { + ds.riffShutdownState = undefined; + } + if (unfencedWorkerRefusal) { + return { + ok: false, + fence: 'none', + requestId, + taskId: prepared.taskId, + error: prepared.error ?? 'riff_shutdown_prepare_failed', + }; + } + return { + ok: false, + fence: 'possible', + requestId, + taskId: prepared.taskId, + error: prepared.error ?? 'riff_shutdown_prepare_failed', + worker, + }; + } + if (ds.worker !== worker) { + return { + ok: false, + fence: 'possible', + requestId, + taskId: prepared.taskId, + error: 'stale_worker_generation', + worker, + }; + } + ds.riffShutdownState = { phase: 'prepared', requestId, taskId: prepared.taskId }; + + // Worker events can release a queued-activation journal/tail independently + // of bot-turn admission. Re-sample after drain so commit cannot strand input. + const daemonBlockerAfterPrepare = daemonInputBlocker(ds); + if (daemonBlockerAfterPrepare) { + return { + ok: false, + fence: 'possible', + requestId, + taskId: prepared.taskId, + error: `daemon_inputs_not_drained:${daemonBlockerAfterPrepare}`, + worker, + expectedAbortTaskId: prepared.taskId, + }; + } + + return { + ok: true, + fence: 'prepared', + requestId, + taskId: prepared.taskId, + runtimeTaskIdAtPrepare, + durableTaskIdAtPrepare, + durableOwnerAtPrepare, + lineageVerified: false, + worker, + }; +} + +export type RiffFleetPrepareOptions = Omit< + RiffShutdownPrepareOptions, + 'durableSnapshot' +> & { + snapshotTimeoutMs?: number; +}; + +/** Take one fresh projection for the complete candidate set, then (and only + * then) publish prepare requests concurrently. */ +export async function prepareRiffFleetForShutdown( + candidates: readonly DaemonSession[], + options: RiffFleetPrepareOptions = {}, +): Promise { + if (candidates.length === 0) return []; + const now = options.now ?? Date.now; + const configuredSnapshotTimeout = options.snapshotTimeoutMs + ?? RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS; + const remaining = options.deadlineMs === undefined + ? configuredSnapshotTimeout + : Math.max(0, options.deadlineMs - now()); + if (remaining <= 0) { + return candidates.map(ds => ({ + ds, + result: unfencedRefusal(ds, 'shutdown_deadline_elapsed_before_initial_snapshot'), + })); + } + + let snapshots: sessionStore.ActiveRiffShutdownSnapshot[]; + try { + snapshots = sessionStore.getActiveRiffShutdownSnapshotsBatch( + candidates.map(ds => ds.session.sessionId), + { maxWaitMs: Math.min(configuredSnapshotTimeout, remaining) }, + ); + } catch (error) { + const message = `initial_riff_snapshot_failed:${error instanceof Error + ? error.message + : String(error)}`; + return candidates.map(ds => ({ ds, result: unfencedRefusal(ds, message) })); + } + + const snapshotsBySession = new Map(snapshots.map(snapshot => [snapshot.sessionId, snapshot])); + return Promise.all(candidates.map(async ds => ({ + ds, + result: await prepareRiffSessionForShutdown(ds, { + ...options, + durableSnapshot: snapshotsBySession.get(ds.session.sessionId), + }), + }))); +} + +function validatePreparedRiffShutdownForPersistence( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): { ok: true } | RiffShutdownFailure { + if (ds.riffShutdownState?.requestId !== prepared.requestId + || ds.riffShutdownState.phase !== 'prepared') { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'shutdown_prepare_ownership_lost', + rollbackDisposition: 'retain_fence', + }; + } + const daemonBlocker = daemonInputBlocker(ds); + if (daemonBlocker) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `daemon_inputs_not_drained:${daemonBlocker}`, + rollbackDisposition: 'abort_safe', + }; + } + + if (prepared.worker) { + if (ds.worker !== prepared.worker || workerHasExited(prepared.worker)) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'stale_worker_generation', + rollbackDisposition: 'retain_fence', + }; + } + } else if (ds.worker) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'new_worker_generation', + rollbackDisposition: 'retain_fence', + }; + } + const runtimeOwner = { + pid: ds.session.pid ?? null, + larkAppId: ds.session.larkAppId ?? null, + backendType: ds.session.backendType ?? null, + }; + if (runtimeOwner.pid !== prepared.durableOwnerAtPrepare.pid + || runtimeOwner.larkAppId !== prepared.durableOwnerAtPrepare.larkAppId + || runtimeOwner.backendType !== prepared.durableOwnerAtPrepare.backendType) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `runtime_owner_changed:${JSON.stringify(runtimeOwner)}`, + rollbackDisposition: 'retain_fence', + }; + } + const runtimeTaskId = ds.session.riffParentTaskId ?? null; + const runtimeLineageExpected = prepared.worker + ? runtimeTaskId === prepared.runtimeTaskIdAtPrepare || runtimeTaskId === prepared.taskId + : runtimeTaskId === prepared.runtimeTaskIdAtPrepare; + if (!runtimeLineageExpected) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `runtime_lineage_changed:${runtimeTaskId ?? 'none'}`, + rollbackDisposition: 'retain_fence', + }; + } + return { ok: true }; +} + +export type PreparedRiffFleetEntry = { + ds: DaemonSession; + result: PreparedRiffShutdown; +}; + +export type RiffFleetPersistenceResult = { ok: true } +| (RiffShutdownFailure & { + sessionIds: readonly string[]; + retainFencedSessionIds: readonly string[]; +}); + +/** Phase 2 fleet transaction: validate every runtime generation, then compare + * and publish all durable lineage rows with one lock and one rename. */ +export function persistPreparedRiffShutdownFleet( + entries: readonly PreparedRiffFleetEntry[], + options: { + persistTimeoutMs?: number; + deadlineMs?: number; + now?: () => number; + } = {}, +): RiffFleetPersistenceResult { + if (entries.length === 0) return { ok: true }; + const now = options.now ?? Date.now; + + const validationFailures = entries + .map(({ ds, result }) => ({ + sessionId: ds.session.sessionId, + failure: validatePreparedRiffShutdownForPersistence(ds, result), + })) + .filter((entry): entry is { + sessionId: string; + failure: RiffShutdownFailure; + } => !entry.failure.ok); + if (validationFailures.length > 0) { + const retainFencedSessionIds = validationFailures + .filter(({ failure }) => failure.rollbackDisposition === 'retain_fence') + .map(({ sessionId }) => sessionId); + return { + ok: false, + taskId: null, + error: `Riff fleet persistence preflight failed: ${validationFailures + .map(({ sessionId, failure }) => `${sessionId}:${failure.error}`) + .join(';')}`, + rollbackDisposition: retainFencedSessionIds.length > 0 ? 'retain_fence' : 'abort_safe', + sessionIds: validationFailures.map(({ sessionId }) => sessionId), + retainFencedSessionIds, + }; + } + + const configuredPersistTimeout = options.persistTimeoutMs + ?? RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS; + const remaining = options.deadlineMs === undefined + ? configuredPersistTimeout + : Math.max(0, options.deadlineMs - now()); + if (remaining <= 0) { + return { + ok: false, + taskId: null, + error: 'Riff fleet lineage batch failed (prewrite_io): shutdown deadline elapsed', + rollbackDisposition: 'abort_safe', + sessionIds: entries.map(({ ds }) => ds.session.sessionId), + retainFencedSessionIds: [], + }; + } + + try { + sessionStore.persistActiveRiffLineagesExactBatch(entries.map(({ ds, result }) => ({ + sessionId: ds.session.sessionId, + taskId: result.durableTaskIdAtPrepare, + owner: result.durableOwnerAtPrepare, + targetTaskId: result.taskId, + expectedCurrentTaskIds: [result.durableTaskIdAtPrepare, result.taskId], + })), { + maxWaitMs: Math.min(configuredPersistTimeout, remaining), + }); + } catch (error) { + const batchError = error instanceof sessionStore.RiffLineageBatchError ? error : undefined; + const stage = batchError?.stage ?? 'prewrite_io'; + const retainFencedSessionIds = stage === 'postrename_ambiguity' + ? entries.map(({ ds }) => ds.session.sessionId) + : stage === 'prewrite_ownership' + ? [...(batchError?.sessionIds ?? [])] + : []; + return { + ok: false, + taskId: null, + error: `Riff fleet lineage batch failed (${stage}): ${error instanceof Error + ? error.message + : String(error)}`, + rollbackDisposition: retainFencedSessionIds.length > 0 ? 'retain_fence' : 'abort_safe', + sessionIds: batchError?.sessionIds ?? entries.map(({ ds }) => ds.session.sessionId), + retainFencedSessionIds, + }; + } + + for (const { ds, result } of entries) { + ds.session.riffParentTaskId = result.taskId ?? undefined; + result.lineageVerified = true; + } + if (options.deadlineMs !== undefined && now() >= options.deadlineMs) { + const sessionIds = entries.map(({ ds }) => ds.session.sessionId); + return { + ok: false, + taskId: null, + error: 'shutdown_deadline_elapsed_after_batch_persist', + rollbackDisposition: 'retain_fence', + sessionIds, + retainFencedSessionIds: sessionIds, + }; + } + const lost = entries + .filter(({ ds, result }) => !isPreparedRiffSessionCurrent(ds, result)) + .map(({ ds }) => ds.session.sessionId); + if (lost.length > 0) { + return { + ok: false, + taskId: null, + error: `shutdown_prepare_ownership_lost_after_batch_persist:${lost.join(',')}`, + rollbackDisposition: 'retain_fence', + sessionIds: lost, + retainFencedSessionIds: lost, + }; + } + return { ok: true }; +} + +/** Single-session compatibility path. Fleet shutdown uses the batch function + * above; this API remains for explicit focused operations and older callers. */ +export function persistPreparedRiffShutdown( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): { ok: true } | RiffShutdownFailure { + const validation = validatePreparedRiffShutdownForPersistence(ds, prepared); + if (!validation.ok) return validation; + + try { + sessionStore.persistActiveRiffLineageExact(ds.session.sessionId, prepared.taskId, { + expectedCurrentTaskIds: [prepared.durableTaskIdAtPrepare, prepared.taskId], + expectedOwner: prepared.durableOwnerAtPrepare, + }); + } catch (err) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `lineage_persist_failed:${err instanceof Error ? err.message : String(err)}`, + rollbackDisposition: err instanceof sessionStore.RiffLineageOwnershipError + ? 'retain_fence' + : 'abort_safe', + }; + } + + let fresh: ReturnType; + try { + fresh = sessionStore.getSessionFresh(ds.session.sessionId); + } catch (err) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `fresh_lineage_verification_failed:${err instanceof Error ? err.message : String(err)}`, + rollbackDisposition: 'retain_fence', + }; + } + const freshTaskId = fresh?.riffParentTaskId ?? null; + const freshOwner = fresh + ? { + pid: fresh.pid ?? null, + larkAppId: fresh.larkAppId ?? null, + backendType: fresh.backendType ?? null, + } + : null; + const freshOwnerMatches = freshOwner !== null + && freshOwner.pid === prepared.durableOwnerAtPrepare.pid + && freshOwner.larkAppId === prepared.durableOwnerAtPrepare.larkAppId + && freshOwner.backendType === prepared.durableOwnerAtPrepare.backendType; + if (!fresh + || fresh.status !== 'active' + || freshTaskId !== prepared.taskId + || !freshOwnerMatches) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `fresh_lineage_verification_failed:status=${fresh?.status ?? 'missing'},` + + `task=${freshTaskId ?? 'none'},expected=${prepared.taskId ?? 'none'},` + + `owner=${JSON.stringify(freshOwner)},` + + `expectedOwner=${JSON.stringify(prepared.durableOwnerAtPrepare)}`, + rollbackDisposition: 'retain_fence', + }; + } + ds.session.riffParentTaskId = prepared.taskId ?? undefined; + prepared.lineageVerified = true; + + if (!isPreparedRiffSessionCurrent(ds, prepared)) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'shutdown_prepare_ownership_lost_after_persist', + rollbackDisposition: 'retain_fence', + }; + } + return { ok: true }; +} + +export function isPreparedRiffSessionCurrent( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): boolean { + if (ds.riffShutdownState?.requestId !== prepared.requestId + || ds.riffShutdownState.phase !== 'prepared') return false; + if (prepared.worker) { + return ds.worker === prepared.worker && !workerHasExited(prepared.worker); + } + return ds.worker === null; +} + +/** After verified lineage persistence, an exact prepared worker that exited + * and was cleared by worker-pool can be safely rolled back locally. A new + * worker generation or a different fence remains ownership ambiguity. */ +export function canAbortVerifiedExitedRiffPreparation( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): boolean { + return prepared.lineageVerified + && !!prepared.worker + && workerHasExited(prepared.worker) + && ds.worker === null + && ds.riffShutdownState?.requestId === prepared.requestId; +} + +/** Roll back one prepared participant. State clears only after the exact worker + * ACKs admission restoration; timeout/late ACK remains deliberately fail-closed. */ +export async function abortPreparedRiffShutdown( + ds: DaemonSession, + prepared: PreparedRiffShutdown, + options: { abortTimeoutMs?: number } = {}, +): Promise { + if (ds.riffShutdownState?.requestId !== prepared.requestId) { + return { ok: false, taskId: prepared.taskId, error: 'shutdown_prepare_ownership_lost' }; + } + if (!prepared.worker) { + if (ds.worker) { + return { ok: false, taskId: prepared.taskId, error: 'new_worker_generation' }; + } + clearWorkerlessShutdownState(ds, prepared.requestId); + return { ok: true, taskId: prepared.taskId }; + } + if (workerHasExited(prepared.worker)) { + return abortWorkerPreparation( + ds, + prepared.worker, + prepared.requestId, + options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + prepared.lineageVerified, + { taskId: prepared.taskId }, + ); + } + return abortWorkerPreparation( + ds, + prepared.worker, + prepared.requestId, + options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + prepared.lineageVerified, + { taskId: prepared.taskId }, + ); +} + +async function abortPossiblyFencedRiffShutdown( + ds: DaemonSession, + participant: PossiblyFencedRiffShutdown, + timeoutMs: number, +): Promise { + if (ds.riffShutdownState?.requestId !== participant.requestId) { + return { + ok: false, + taskId: participant.taskId, + error: 'shutdown_prepare_ownership_lost', + }; + } + return abortWorkerPreparation( + ds, + participant.worker, + participant.requestId, + timeoutMs, + false, + Object.prototype.hasOwnProperty.call(participant, 'expectedAbortTaskId') + ? { taskId: participant.expectedAbortTaskId ?? null } + : undefined, + ); +} + +export type RiffFleetAbortEntry = { + ds: DaemonSession; + result: FencedRiffShutdownParticipant; +}; + +export type RiffFleetAbortResult = { + ds: DaemonSession; + participant: FencedRiffShutdownParticipant; + result: ShutdownPhaseResult; +}; + +/** Restore every exact prepared/possibly-fenced generation concurrently. No + * participant can consume another's timeout budget. */ +export async function abortRiffShutdownFleet( + entries: readonly RiffFleetAbortEntry[], + options: { + abortTimeoutMs?: number; + deadlineMs?: number; + now?: () => number; + } = {}, +): Promise { + const now = options.now ?? Date.now; + const configuredTimeout = options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS; + const remaining = options.deadlineMs === undefined + ? configuredTimeout + : Math.max(0, options.deadlineMs - now()); + const timeoutMs = Math.min(configuredTimeout, remaining); + + return Promise.all(entries.map(async ({ ds, result: participant }) => { + if (timeoutMs <= 0) { + return { + ds, + participant, + result: { + ok: false, + taskId: participant.taskId, + error: 'shutdown_deadline_elapsed_before_abort', + }, + }; + } + const result = participant.ok + ? await abortPreparedRiffShutdown(ds, participant, { abortTimeoutMs: timeoutMs }) + : await abortPossiblyFencedRiffShutdown(ds, participant, timeoutMs); + return { ds, participant, result }; + })); +} + +/** Phase 3: synchronous, infallible-after-validation commit. The coordinator + * validates every participant first, then calls this without an intervening + * await, so one session can never refuse after a peer was detached. */ +export function commitPreparedRiffShutdown( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): boolean { + if (!isPreparedRiffSessionCurrent(ds, prepared)) return false; + if (!prepared.worker) { + clearWorkerlessShutdownState(ds, prepared.requestId); + return true; + } + if (!send(prepared.worker, { type: 'riff_shutdown_commit', requestId: prepared.requestId })) { + // Lineage is already durably fresh-verified. Direct retirement is safe even + // if the final IPC channel closed between validation and send. + try { prepared.worker.kill('SIGTERM'); } catch { /* already exited */ } + } + clearWorkerOwnership(ds, prepared.worker); + return true; +} + +/** Single-session compatibility wrapper. Daemon shutdown intentionally uses + * the explicit three-phase API above so a multi-Riff fleet cannot half-commit. */ +export async function detachRiffWorkerForShutdown( + ds: DaemonSession, + options: { drainTimeoutMs?: number; abortTimeoutMs?: number } = {}, +): Promise { + const prepared = await prepareRiffSessionForShutdown(ds, options); + if (!prepared.ok) { + if (prepared.fence === 'none') return prepared; + const [aborted] = await abortRiffShutdownFleet([{ ds, result: prepared }], options); + const abortSuffix = aborted.result.ok + ? '' + : `;admission_restore_failed:${aborted.result.error ?? 'unknown'}`; + logger.error( + `[${label(ds)}] Riff shutdown drain failed (${prepared.error}${abortSuffix}); ` + + (aborted.result.ok + ? 'worker retained with admission restored' + : 'worker retained and admission fence kept fail-closed'), + ); + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: prepared.error + abortSuffix, + }; + } + const persisted = persistPreparedRiffShutdown(ds, prepared); + if (!persisted.ok) { + if (persisted.rollbackDisposition === 'retain_fence') return persisted; + const aborted = await abortPreparedRiffShutdown(ds, prepared, options); + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: persisted.error + + (aborted.ok ? '' : `;admission_restore_failed:${aborted.error ?? 'unknown'}`), + }; + } + if (!commitPreparedRiffShutdown(ds, prepared)) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'shutdown_prepare_ownership_lost_before_commit', + rollbackDisposition: 'retain_fence', + }; + } + logger.info( + `[${label(ds)}] Riff shutdown detach committed after durable lineage ` + + `${prepared.taskId ?? 'none'}`, + ); + return { + ok: true, + requestId: prepared.requestId, + taskId: prepared.taskId, + disposition: 'lineage_persisted', + ...(prepared.worker ? { worker: prepared.worker } : {}), + }; +} diff --git a/src/core/riff-worker-shutdown-readiness.ts b/src/core/riff-worker-shutdown-readiness.ts new file mode 100644 index 000000000..f7101dd19 --- /dev/null +++ b/src/core/riff-worker-shutdown-readiness.ts @@ -0,0 +1,31 @@ +export interface RiffWorkerShutdownInputSnapshot { + /** False while async init can still materialize an opening prompt later. */ + initPromptMaterialized: boolean; + /** A normal prompt has been removed from pendingMessages but has not yet + * crossed the adapter/backend write boundary. */ + isFlushing: boolean; + pendingMessages: number; + pendingRawInputs: number; + pendingSessionRename: boolean; + sessionRenameInFlight: boolean; + /** Raw command text -> Enter sequences that can still append a backend + * write after the shutdown fence is sampled. */ + commandLineWritesPending: number; +} + +/** Describe worker-owned input not proven to be inside RiffBackend.writeChain. */ +export function riffWorkerShutdownInputBlocker( + snapshot: RiffWorkerShutdownInputSnapshot, +): string | null { + const parts: string[] = []; + if (!snapshot.initPromptMaterialized) parts.push('init=materializing'); + if (snapshot.isFlushing) parts.push('flushing=1'); + if (snapshot.pendingMessages > 0) parts.push(`messages=${snapshot.pendingMessages}`); + if (snapshot.pendingRawInputs > 0) parts.push(`raw=${snapshot.pendingRawInputs}`); + if (snapshot.pendingSessionRename) parts.push('rename=1'); + if (snapshot.sessionRenameInFlight) parts.push('rename_inflight=1'); + if (snapshot.commandLineWritesPending > 0) { + parts.push(`command_writes=${snapshot.commandLineWritesPending}`); + } + return parts.length > 0 ? parts.join(',') : null; +} diff --git a/src/core/shutdown-budgets.ts b/src/core/shutdown-budgets.ts new file mode 100644 index 000000000..3501ae0ef --- /dev/null +++ b/src/core/shutdown-budgets.ts @@ -0,0 +1,33 @@ +/** + * Graceful-shutdown budgets are ordered so a bounded Riff create/follow-up can + * drain before the daemon persists its final lineage and asks workers to exit. + * Shutdown never cancels accepted Riff work as a fallback. + */ +export const RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS = 12_000; + +/** Admission restoration can wait for the same bounded 10s create/follow-up + * that prepare was draining. The daemon keeps its retirement fence throughout. */ +export const RIFF_ADMISSION_RESTORE_TIMEOUT_MS = 11_000; + +/** Bounded acquisition of the bot-wide mutation lease. A timed-out waiter is + * removed and can never run after shutdown has already been refused. */ +export const BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS = 1_000; + +/** Initial all-owner snapshot and phase-2 batch CAS each use one short lock. */ +export const RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS = 1_000; +export const RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS = 1_000; + +/** Scheduling/logging slack inside the graceful daemon shutdown budget. */ +export const DAEMON_SHUTDOWN_OVERHEAD_MS = 2_000; +export const DAEMON_WORKER_EXIT_GRACE_MS = 3_000; +export const DAEMON_SHUTDOWN_MAX_MS = + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS + + RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS + + RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS + + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS + + Math.max(RIFF_ADMISSION_RESTORE_TIMEOUT_MS, DAEMON_WORKER_EXIT_GRACE_MS) + + DAEMON_SHUTDOWN_OVERHEAD_MS; + +if (DAEMON_SHUTDOWN_MAX_MS > 28_000) { + throw new Error('complete daemon shutdown budget must remain at or below 28 seconds'); +} diff --git a/src/core/types.ts b/src/core/types.ts index 9fcabbcf8..a08e604b7 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -187,6 +187,19 @@ export interface DaemonSession { * "Web终端" button opens the riff sandbox. In-memory only — re-sent by the * worker on each task. */ riffAccessUrl?: string; + /** Explicit-close transaction: while present, no new Riff input may be + * admitted until cancellation commits or admission restoration is ACKed. */ + riffCloseState?: { + phase: 'preparing' | 'prepared' | 'uncertain'; + requestId: string; + taskId?: string; + }; + /** Graceful-shutdown transaction for the exact Riff worker generation. */ + riffShutdownState?: { + phase: 'preparing' | 'prepared'; + requestId: string; + taskId?: string | null; + }; usageLimit?: CliUsageLimitState; usageLimitRetryTimer?: NodeJS.Timeout; lastUserPrompt?: string; @@ -316,6 +329,14 @@ export interface DaemonSession { }; } +/** A non-null value means this Riff generation is deliberately rejecting new + * input until a close/shutdown commit or an ACKed admission restore. */ +export function riffRetirementAdmissionPhase(ds: DaemonSession): string | null { + if (ds.riffShutdownState) return `shutdown-${ds.riffShutdownState.phase}`; + if (ds.riffCloseState) return `close-${ds.riffCloseState.phase}`; + return null; +} + /** Composite key for activeSessions — allows multiple bots to have independent * sessions anchored on the same id. The first arg is the **routing anchor**: * - thread-scope → rootMessageId diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 133c0878e..2629f2f02 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -22,7 +22,7 @@ import { fallbackTurnId, isSubstituteTurn } from './reply-target.js'; import { updateMessage, deleteMessage, sendEphemeralCard, sendUserMessage, addReaction, removeReaction, getMessageChatId, MessageWithdrawnError } from '../im/lark/client.js'; import { buildStreamingCard, buildPrivateSnapshotCard, buildSessionCard, buildTuiPromptCard, buildTuiPromptResolvedCard, buildRelayedFrozenCard, getCliDisplayName } from '../im/lark/card-builder.js'; import { loadFrozenCards, saveFrozenCards } from '../services/frozen-card-store.js'; -import { hashUrlForLog, cancelRiffTaskById } from '../adapters/backend/riff-backend.js'; +import { hashUrlForLog } from '../adapters/backend/riff-backend.js'; import { logger } from '../utils/logger.js'; import { createCliAdapterSync } from '../adapters/cli/registry.js'; import { traeHome } from '../services/traex-paths.js'; @@ -75,7 +75,14 @@ import { isStructuredBridgeAdoptCli } from '../services/structured-bridge-clis.j import { resolveEffectivePluginIds } from './plugins/effective.js'; import { ensureGatewayEntry } from './plugins/mcp/gateway-installer.js'; import type { CliTurnPayload, CodexAppTurnInput, DaemonToWorker, WorkerToDaemon, Session, DisplayMode } from '../types.js'; -import { activeSessionKey, sessionKey, sessionAnchorId, isDocNativeSession, type DaemonSession } from './types.js'; +import { + activeSessionKey, + sessionKey, + sessionAnchorId, + isDocNativeSession, + riffRetirementAdmissionPhase, + type DaemonSession, +} from './types.js'; import { DONE_REACTION_EMOJI_TYPE } from './pending-response.js'; import { buildTerminalUrl } from './terminal-url.js'; import { prependBotmuxBin } from './botmux-wrapper.js'; @@ -101,6 +108,8 @@ import { } from './session-title.js'; import { acknowledgeSessionReady } from './session-ready-handshake.js'; import { recordDispatchInputCommit } from './dispatch.js'; +import { cleanupExplicitSessionBacking } from './explicit-session-backing-cleanup.js'; +import { RIFF_ADMISSION_RESTORE_TIMEOUT_MS } from './shutdown-budgets.js'; type WindowsForkOptions = ForkOptions & { windowsHide?: boolean }; @@ -227,6 +236,18 @@ function requireCallbacks(): WorkerPoolCallbacks { // linear-scan by sessionId. let activeSessionsRegistry: Map | undefined; +type RiffWorkerCloseResult = { + ok: boolean; + taskId?: string; + error?: string; +}; + +const pendingRiffWorkerCloses = new Map void; +}>(); + export function setActiveSessionsRegistry(m: Map): void { activeSessionsRegistry = m; } @@ -1308,7 +1329,23 @@ export function ensureClaudeFolderTrust(workingDir: string, stateJsonPath: strin // ─── Kill worker ──────────────────────────────────────────────────────────── -export function killWorker(ds: DaemonSession): void { +export function killWorker( + ds: DaemonSession, + opts: { riffCloseCommitRequestId?: string } = {}, +): void { + const closeFrozenType = ds.initConfig?.backendType ?? ds.session.backendType; + if (ds.worker && !ds.worker.killed + && closeFrozenType === 'riff' + && !opts.riffCloseCommitRequestId) { + // A generic synchronous retirement cannot safely detach Riff. An accepted + // create/follow-up may still be waiting for the task id that becomes the + // only durable lineage anchor. + logger.error( + `[${tag(ds)}] Refused unprepared live Riff worker retirement; ` + + 'preserving worker and remote-task lineage', + ); + return; + } clearUsageLimitState(ds); ds.localProcessAttestation = undefined; // A managed-turn capability belongs to one concrete worker generation. @@ -1328,18 +1365,28 @@ export function killWorker(ds: DaemonSession): void { // session would leave an orphaned CLI running in tmux that still replies. // Destroy the backing session directly here so /close always terminates it. destroyOrphanedBackingSession(ds); + if (opts.riffCloseCommitRequestId + && ds.riffCloseState?.requestId === opts.riffCloseCommitRequestId) { + ds.riffCloseState = undefined; + } return; } try { - ds.worker.send({ type: 'close' } as DaemonToWorker); + if (opts.riffCloseCommitRequestId) { + ds.worker.send({ + type: 'close_commit', + requestId: opts.riffCloseCommitRequestId, + } as DaemonToWorker); + } else { + ds.worker.send({ type: 'close' } as DaemonToWorker); + } } catch { /* IPC already closed */ } + if (opts.riffCloseCommitRequestId + && ds.riffCloseState?.requestId === opts.riffCloseCommitRequestId) { + ds.riffCloseState = undefined; + } const w = ds.worker; - // riff：worker close 分支要有界 await 远端 task-cancel（destroySession 5s×2 重试， - // 外层 race 8s）。默认 2s SIGTERM backstop 会在取消发出前掐死进程，已关闭话题 - // 的远端任务照跑——冻结为 riff 的会话放宽到 24s（层级：destroy 20s < worker 22s - // < SIGTERM 24s < SIGKILL 29s；正常路径 worker 自行 exit，不会等满）。 - const closeFrozenType = ds.initConfig?.backendType ?? ds.session.backendType; - armWorkerKillBackstop(w, tag(ds), closeFrozenType === 'riff' ? 24_000 : WORKER_SIGTERM_BACKSTOP_MS); + armWorkerKillBackstop(w, tag(ds)); ds.worker = null; ds.workerPort = null; ds.workerToken = null; @@ -1359,22 +1406,17 @@ export function killWorker(ds: DaemonSession): void { function destroyOrphanedBackingSession(ds: DaemonSession): void { if (ds.initConfig?.adoptMode || ds.adoptedFrom) return; reclaimParkedCrashDiagnostic(ds); - // riff：worker 已死时 /close 仍要取消持久化血缘指向的远端任务——否则已关闭 - // 话题的远端 agent 继续拿着注入凭证发消息。fire-and-forget（内部有界+重试）。 + // Riff cancellation is asynchronous and cannot be made safe from this + // synchronous best-effort helper. The authoritative closeSession path awaits + // cancellation before publishing the closed row and retains lineage on + // failure. const frozenType = ds.initConfig?.backendType ?? ds.session.backendType; if (frozenType === 'riff') { - const taskId = ds.session.riffParentTaskId; - if (taskId) { - try { - const riffCfg = getBot(ds.larkAppId).config.riff; - if (riffCfg?.baseUrl) { - void cancelRiffTaskById(riffCfg, taskId).then((ok) => { - if (ok) logger.info(`[${tag(ds)}] killWorker: orphan riff task ${taskId} cancelled`); - }); - } - } catch { /* bot deregistered — nothing to cancel with */ } - ds.session.riffParentTaskId = undefined; - sessionStore.updateSession(ds.session); + if (ds.session.riffParentTaskId) { + logger.warn( + `[${tag(ds)}] worker-less Riff teardown requires awaited explicit close; ` + + `retaining task ${ds.session.riffParentTaskId} for retry`, + ); } return; } @@ -1388,6 +1430,305 @@ function destroyOrphanedBackingSession(ds: DaemonSession): void { } } +type RiffClosePreparation = + | { ok: true; taskId?: string } + | { + ok: false; + error: + | 'riff_cancel_failed' + | 'riff_config_missing' + | 'riff_task_changed' + | 'riff_worker_close_failed' + | 'riff_row_inconsistent' + | 'riff_durable_close_failed' + | 'riff_close_reconciliation_required' + | 'riff_shutdown_fence_in_progress'; + retryable: true; + taskId?: string; + }; + +async function abortLiveRiffWorkerClose( + ds: DaemonSession, + requestId: string, + opts: { allowAbsentAfterProvenRestore?: boolean } = {}, +): Promise { + if (ds.riffCloseState?.requestId !== requestId) return false; + const worker = ds.worker; + if (!worker || worker.killed) { + if (opts.allowAbsentAfterProvenRestore) { + ds.riffCloseState = undefined; + return true; + } + ds.riffCloseState = { ...ds.riffCloseState, phase: 'uncertain' }; + return false; + } + + const restored = await new Promise<{ ok: boolean; error?: string }>(resolve => { + let settled = false; + const finish = (result: { ok: boolean; error?: string }): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.removeListener?.('message', onMessage); + worker.removeListener?.('exit', onExit); + resolve(result); + }; + const onMessage = (raw: unknown): void => { + const msg = raw as WorkerToDaemon; + if (msg?.type !== 'close_abort_result' || msg.requestId !== requestId) return; + if (ds.worker !== worker) { + finish({ ok: false, error: 'stale_worker_generation' }); + return; + } + finish({ ok: msg.ok, ...(msg.error ? { error: msg.error } : {}) }); + }; + const onExit = (): void => finish({ + ok: false, + error: 'worker_exited_before_close_abort_result', + }); + const timer = setTimeout( + () => finish({ ok: false, error: 'close_abort_result_timeout' }), + RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + ); + timer.unref?.(); + worker.on?.('message', onMessage); + worker.once?.('exit', onExit); + try { + worker.send({ type: 'close_abort', requestId } as DaemonToWorker); + } catch (err) { + finish({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }); + + if (restored.ok && ds.riffCloseState?.requestId === requestId) { + ds.riffCloseState = undefined; + return true; + } + if (opts.allowAbsentAfterProvenRestore + && ds.riffCloseState?.requestId === requestId) { + ds.riffCloseState = undefined; + return true; + } + if (ds.riffCloseState?.requestId === requestId) { + ds.riffCloseState = { ...ds.riffCloseState, phase: 'uncertain' }; + } + logger.warn( + `[${tag(ds)}] Riff close abort was not acknowledged (${restored.error ?? 'unknown'}); ` + + 'retaining admission fence pending explicit lineage reconciliation', + ); + return false; +} + +async function prepareLiveRiffWorkerClose(ds: DaemonSession): Promise { + const worker = ds.worker; + if (!worker || worker.killed) { + return { ok: false, error: 'riff_worker_close_failed', retryable: true }; + } + if (ds.riffCloseState || ds.riffShutdownState) { + return { + ok: false, + error: 'riff_worker_close_failed', + retryable: true, + ...((ds.riffCloseState?.taskId ?? ds.riffShutdownState?.taskId) + ? { taskId: (ds.riffCloseState?.taskId ?? ds.riffShutdownState?.taskId)! } + : {}), + }; + } + const requestId = randomUUID(); + ds.riffCloseState = { + phase: 'preparing', + requestId, + ...(ds.session.riffParentTaskId ? { taskId: ds.session.riffParentTaskId } : {}), + }; + let matchedCloseResult = false; + const result = await new Promise((resolve) => { + let settled = false; + const finish = (value: RiffWorkerCloseResult): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.removeListener?.('exit', onExit); + pendingRiffWorkerCloses.delete(requestId); + resolve(value); + }; + const onExit = (): void => finish({ + ok: false, + error: 'worker_exited_before_close_result', + }); + const timer = setTimeout( + () => finish({ ok: false, error: 'worker_close_result_timeout' }), + 23_000, + ); + timer.unref?.(); + worker.once?.('exit', onExit); + pendingRiffWorkerCloses.set(requestId, { + sessionId: ds.session.sessionId, + worker, + resolve: value => { + matchedCloseResult = true; + finish(value); + }, + }); + try { + worker.send({ type: 'close', requestId } as DaemonToWorker); + } catch (err) { + finish({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }); + + const taskId = result.taskId ?? ds.session.riffParentTaskId; + if (result.taskId) { + ds.session.riffParentTaskId = result.taskId; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + await abortLiveRiffWorkerClose(ds, requestId); + logger.error( + `[${tag(ds)}] Riff close lineage persistence failed; close aborted: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return { + ok: false, + error: 'riff_durable_close_failed', + retryable: true, + taskId: result.taskId, + }; + } + } + + if (!result.ok) { + await abortLiveRiffWorkerClose(ds, requestId, { + allowAbsentAfterProvenRestore: matchedCloseResult, + }); + logger.warn( + `[${tag(ds)}] Riff worker close prepare failed: ${result.error ?? 'unknown'}; ` + + `session remains active${taskId ? ` (task ${taskId})` : ''}`, + ); + return { + ok: false, + error: 'riff_worker_close_failed', + retryable: true, + ...(taskId ? { taskId } : {}), + }; + } + + ds.riffCloseState = { + phase: 'prepared', + requestId, + ...(taskId ? { taskId } : {}), + }; + logger.info(`[${tag(ds)}] Riff worker close prepared and remote task cancellation confirmed`); + return { ok: true, ...(taskId ? { taskId } : {}) }; +} + +/** Await remote cancellation for any Riff owner before its durable row is + * closed. Live workers use prepare/commit; worker-less rows cancel their exact + * persisted task through current authoritative bot config. */ +async function prepareRiffExplicitClose( + ds: DaemonSession | undefined, + stored: Session | undefined, +): Promise { + const session = ds?.session ?? stored; + if (!session) return { ok: true }; + const backendType = ds?.initConfig?.backendType ?? session.backendType; + const taskId = session.riffParentTaskId; + if (ds?.riffShutdownState) { + const fencedTaskId = Object.prototype.hasOwnProperty.call(ds.riffShutdownState, 'taskId') + ? ds.riffShutdownState.taskId + : taskId; + return { + ok: false, + error: 'riff_shutdown_fence_in_progress', + retryable: true, + ...(typeof fencedTaskId === 'string' && fencedTaskId ? { taskId: fencedTaskId } : {}), + }; + } + if (ds?.riffCloseState) { + return { + ok: false, + error: 'riff_close_reconciliation_required', + retryable: true, + ...(ds.riffCloseState.taskId ? { taskId: ds.riffCloseState.taskId } : {}), + }; + } + if (backendType !== 'riff') return { ok: true }; + if (ds?.initConfig?.adoptMode || ds?.adoptedFrom || session.adoptedFrom) return { ok: true }; + + if (ds?.worker && !ds.worker.killed) { + if (!stored || stored.status !== 'active') { + return { + ok: false, + error: 'riff_row_inconsistent', + retryable: true, + ...(taskId ? { taskId } : {}), + }; + } + return prepareLiveRiffWorkerClose(ds); + } + + const durableBefore = sessionStore.getSession(session.sessionId); + if (ds && durableBefore + && ds.session.riffParentTaskId !== durableBefore.riffParentTaskId) { + const authoritativeTaskId = durableBefore.riffParentTaskId ?? ds.session.riffParentTaskId; + logger.warn( + `[${tag(ds)}] explicit close refused before cancellation: runtime/durable Riff lineage differs ` + + `(${ds.session.riffParentTaskId ?? 'none'}/${durableBefore.riffParentTaskId ?? 'none'})`, + ); + return { + ok: false, + error: 'riff_task_changed', + retryable: true, + ...(authoritativeTaskId ? { taskId: authoritativeTaskId } : {}), + }; + } + if (!taskId) return { ok: true }; + + let riffConfig; + const larkAppId = ds?.larkAppId ?? session.larkAppId; + try { if (larkAppId) riffConfig = getBot(larkAppId).config.riff; } catch { /* bot removed */ } + const cleanup = await cleanupExplicitSessionBacking({ + sessionId: session.sessionId, + backendType, + riffParentTaskId: taskId, + riffConfig, + }); + const closeLabel = ds ? tag(ds) : session.sessionId.slice(0, 8); + if (!cleanup.ok) { + logger.warn( + `[${closeLabel}] explicit close refused: ${cleanup.kind}; ` + + `Riff task ${taskId} remains active and retryable`, + ); + return { + ok: false, + error: cleanup.kind === 'riff_config_missing' ? 'riff_config_missing' : 'riff_cancel_failed', + retryable: true, + taskId, + }; + } + if (cleanup.kind !== 'cancelled_riff') return { ok: true }; + + const latest = sessionStore.getSession(session.sessionId); + const latestTaskId = latest?.riffParentTaskId; + const runtimeTaskId = ds?.session.riffParentTaskId; + if (!latest || latest.status !== 'active' || latestTaskId !== cleanup.taskId + || (ds && runtimeTaskId !== cleanup.taskId)) { + logger.warn( + `[${closeLabel}] explicit close cancelled stale Riff task ${cleanup.taskId}, ` + + `but runtime/durable lineage or status changed to ` + + `${runtimeTaskId ?? 'none'}/${latestTaskId ?? 'none'}/${latest?.status ?? 'missing'}; retry required`, + ); + return { + ok: false, + error: 'riff_task_changed', + retryable: true, + taskId: latestTaskId ?? runtimeTaskId ?? cleanup.taskId, + }; + } + + logger.info(`[${closeLabel}] Riff task ${cleanup.taskId} cancellation prepared for explicit close`); + return { ok: true, taskId: cleanup.taskId }; +} + /** * Reclaim a session's parked crash-diagnostic shell (`bmx-diag-`) and its * captured `.ansi` file. The worker normally tears these down itself (killCli / @@ -1493,10 +1834,23 @@ function armWorkerKillBackstop(w: ChildProcess, label: string, sigtermMs: number * Calling this on an unknown sessionId, an already-closed session, or a session * whose worker died asynchronously must still resolve with `{ ok: true }`. */ +export type CloseSessionResult = + | { ok: true; alreadyClosed: boolean } + | ({ + ok: false; + alreadyClosed: false; + } & Exclude); + export async function closeSession( sessionId: string, -): Promise<{ ok: true; alreadyClosed: boolean }> { +): Promise { const ds = findActiveBySessionId(sessionId); + const stored = sessionStore.getSession(sessionId); + const wasOpen = !!stored && stored.status !== 'closed'; + const isOwnedRiffClose = !ds?.initConfig?.adoptMode + && !ds?.adoptedFrom + && !stored?.adoptedFrom + && (ds?.initConfig?.backendType ?? ds?.session.backendType ?? stored?.backendType) === 'riff'; let killedLive = false; // 会话关闭即可回收其崩溃重启计数；否则每个曾崩溃过的 session 会在 daemon // 生命周期内永久占位（restartCounts 此前无任何 delete）。 @@ -1505,40 +1859,43 @@ export async function closeSession( // Usage ledger: flush the final delta before the worker goes away (a // crash/limited turn may never have reached an idle edge). recordUsageForDaemonSession(ds); - killWorker(ds); - // 文档入口清理：会话关闭即删除其绑定。只有旧 - // /subscribe-lark-doc 记录需要调飞书逐文件退订 API； - // /watch-comment 仅依赖应用级评论事件，删本地监听表即可。 - try { - const anchor = sessionAnchorId(ds); - const subs = listDocSubscriptionsForSession(config.session.dataDir, ds.larkAppId, anchor); - for (const sub of subs) { - if (sub.managedBy !== 'watch-comment') { - await unsubscribeDocFile(ds.larkAppId, { fileToken: sub.fileToken, fileType: sub.fileType }); - } - removeDocSubscription(config.session.dataDir, ds.larkAppId, sub.fileToken); - } - if (subs.length) logger.info(`[doc-comment] session ${sessionId.slice(0, 8)} closed → removed ${subs.length} doc binding(s)`); - } catch (err: any) { - logger.warn(`[doc-comment] cleanup on close failed for ${sessionId.slice(0, 8)}: ${err?.message ?? err}`); - } - activeSessionsRegistry?.delete(activeSessionKey(ds)); - killedLive = true; - if (!ds.exitEventEmitted) { - ds.exitEventEmitted = true; - dashboardEventBus.publish({ - type: 'session.exited', - body: { sessionId, reason: 'dashboard_close' }, - }); - emitSessionLifecycleHook(ds, 'session.exit', { reason: 'dashboard_close' }); - } } - // Persistence path — load → mark closed → save (delegated to sessionStore). - const stored = sessionStore.getSession(sessionId); - const wasOpen = !!stored && stored.status !== 'closed'; + // Riff is a remote credential-bearing process. Await cancellation before + // closing its durable row; failures preserve the route, worker, and lineage + // so the operation is retryable. + const prepared = await prepareRiffExplicitClose(ds, stored); + if (!prepared.ok) { + return { ...prepared, alreadyClosed: false }; + } + const preparedRiffRequestId = ds?.riffCloseState?.phase === 'prepared' + ? ds.riffCloseState.requestId + : undefined; + if (wasOpen) { - sessionStore.closeSession(sessionId); + try { + if (isOwnedRiffClose) { + sessionStore.closeSession(sessionId, { clearRiffParentTaskId: true }); + } else { + sessionStore.closeSession(sessionId); + } + } catch (err) { + if (!isOwnedRiffClose) throw err; + if (ds && preparedRiffRequestId) { + await abortLiveRiffWorkerClose(ds, preparedRiffRequestId); + } + logger.error( + `[${sessionId.slice(0, 8)}] Durable session close failed after Riff prepare; ` + + `worker admission restored: ${err instanceof Error ? err.message : String(err)}`, + ); + return { + ok: false, + alreadyClosed: false, + error: 'riff_durable_close_failed', + retryable: true, + ...(prepared.taskId ? { taskId: prepared.taskId } : {}), + }; + } const after = sessionStore.getSession(sessionId); dashboardEventBus.publish({ type: 'session.update', @@ -1553,6 +1910,59 @@ export async function closeSession( }); } + if (ds) { + killWorker(ds, { + ...(preparedRiffRequestId ? { riffCloseCommitRequestId: preparedRiffRequestId } : {}), + }); + // A transferred/restored exact object may remain under more than one alias. + // Remove only identity matches, never a same-key successor. + if (activeSessionsRegistry) { + for (const [registeredKey, candidate] of activeSessionsRegistry) { + if (candidate === ds) activeSessionsRegistry.delete(registeredKey); + } + } + killedLive = true; + if (!ds.exitEventEmitted) { + ds.exitEventEmitted = true; + dashboardEventBus.publish({ + type: 'session.exited', + body: { sessionId, reason: 'dashboard_close' }, + }); + emitSessionLifecycleHook(ds, 'session.exit', { reason: 'dashboard_close' }); + } + + // Remove local delivery bindings synchronously with the authoritative + // close. Provider unsubscribe remains best-effort after routing is gone. + const anchor = sessionAnchorId(ds); + const subs = listDocSubscriptionsForSession(config.session.dataDir, ds.larkAppId, anchor); + for (const sub of subs) { + removeDocSubscription(config.session.dataDir, ds.larkAppId, sub.fileToken); + } + if (subs.length) { + logger.info( + `[doc-comment] session ${sessionId.slice(0, 8)} closed → ` + + `removed ${subs.length} local doc binding(s)`, + ); + } + void (async () => { + try { + for (const sub of subs) { + if (sub.managedBy !== 'watch-comment') { + await unsubscribeDocFile(ds.larkAppId, { + fileToken: sub.fileToken, + fileType: sub.fileType, + }); + } + } + } catch (err: any) { + logger.warn( + `[doc-comment] cleanup on close failed for ${sessionId.slice(0, 8)}: ` + + `${err?.message ?? err}`, + ); + } + })(); + } + // alreadyClosed = nothing happened on either path. const alreadyClosed = !killedLive && !wasOpen; return { ok: true, alreadyClosed }; @@ -1878,6 +2288,25 @@ export function sendWorkerInput( dispatchAttempt?: number; } = {}, ): boolean { + const riffRetirementPhase = riffRetirementAdmissionPhase(ds); + if (riffRetirementPhase) { + logger.warn( + `[${tag(ds)}] Rejected turn ${turnId ?? '?'} while Riff retirement fence is ${riffRetirementPhase}`, + ); + void callbacks?.sessionReply( + sessionAnchorId(ds), + tr('worker.riff_close_in_progress', undefined, localeForBot(ds.larkAppId)), + 'text', + ds.larkAppId, + turnId, + ).catch(err => { + logger.warn( + `[${tag(ds)}] Failed to notify rejected Riff close-race turn: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + }); + return false; + } if (!ds.worker || ds.worker.killed) return false; const normalized = typeof payload === 'string' ? { content: payload } : payload; const codexAppInput = codexAppInputForSession(ds, normalized.codexAppInput, turnId); @@ -2155,6 +2584,22 @@ export function forkWorker( worker.on('error', (err) => { const reason = (err as Error)?.message ?? String(err); logger.error(`[${t}] Worker fork error: ${reason}`); + if (ds.worker === worker) { + const retainExactRetirementGeneration = ds.riffShutdownState !== undefined + || ds.riffCloseState !== undefined; + if (!retainExactRetirementGeneration) { + ds.worker = null; + ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; + ds.managedTurnOrigin = undefined; + ds.riffCloseState = undefined; + } + // The retirement coordinator owns a prepared generation. Keep the exact + // ChildProcess pointer until exit so it can decide whether abort/commit + // was durably verified. + try { worker.kill(); } catch { /* best-effort failed-child fence */ } + } if (startupState.failureNotified) return; startupState.failureNotified = true; emitSessionLifecycleHook(ds, 'session.requires_attention', { @@ -3485,8 +3930,16 @@ function setupWorkerHandlers( if (msg.taskId === null) { // follow-up 血缘断裂：清掉持久化锚点，否则 daemon 重启会复活已判坏的 parent。 if (ds.session.riffParentTaskId) { + const priorTaskId = ds.session.riffParentTaskId; ds.session.riffParentTaskId = undefined; - sessionStore.updateSession(ds.session); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.riffParentTaskId = priorTaskId; + logger.error( + `[${t}] Failed to clear Riff lineage: ${err instanceof Error ? err.message : String(err)}`, + ); + } } break; } @@ -3496,7 +3949,34 @@ function setupWorkerHandlers( // next message continues the riff conversation in the warm sandbox // instead of cold-booting a context-less fresh task (4-5 min). ds.session.riffParentTaskId = msg.taskId; - sessionStore.updateSession(ds.session); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + // Retain the newest runtime lineage. A close_result or later task-id + // event retries persistence; reverting here would make a follow-up + // target the stale parent while the worker owns the new child. + logger.error( + `[${t}] Failed to persist Riff lineage ${msg.taskId}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + break; + } + + case 'close_result': { + const pending = pendingRiffWorkerCloses.get(msg.requestId); + if (!pending + || pending.worker !== worker + || pending.sessionId !== ds.session.sessionId + || ds.worker !== worker) { + logger.warn(`[${t}] Ignored stale/unmatched Riff close result ${msg.requestId}`); + break; + } + pendingRiffWorkerCloses.delete(msg.requestId); + pending.resolve({ + ok: msg.ok, + ...(msg.taskId ? { taskId: msg.taskId } : {}), + ...(msg.error ? { error: msg.error } : {}), + }); break; } @@ -3760,7 +4240,14 @@ function setupWorkerHandlers( if (ds.worker === worker) { ds.worker = null; ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; ds.managedTurnOrigin = undefined; + if (ds.riffCloseState) { + ds.riffCloseState = { ...ds.riffCloseState, phase: 'uncertain' }; + } + // Do not clear riffShutdownState here. Only the shutdown coordinator can + // release a generation after lineage persistence or admission restore. // This worker generation is gone. Invalidate any stuck-warning card it // posted so a late click cannot inject keys into a replacement worker. invalidateStuckWarning(ds, 'worker_exit'); diff --git a/src/daemon.ts b/src/daemon.ts index 9f990594d..40723e126 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -284,6 +284,23 @@ import { buildDocCommentTurnInput, buildDocWatchWarmupTurnInput } from './core/d import { advanceDocCommentCursor, docCommentRepliesAfterCursor, latestDocCommentPollCursor } from './core/doc-comment-poller.js'; import { renderBufferedSenderBlock } from './core/session-manager.js'; import { shutdownBackendDisposition } from './core/persistent-backend.js'; +import { + abortRiffShutdownFleet, + canAbortVerifiedExitedRiffPreparation, + collectUniqueDaemonShutdownSessions, + commitPreparedRiffShutdown, + isPreparedRiffSessionCurrent, + persistPreparedRiffShutdownFleet, + prepareRiffFleetForShutdown, + type FencedRiffShutdownParticipant, + type PreparedRiffShutdown, +} from './core/riff-shutdown-detach.js'; +import { + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS, + DAEMON_SHUTDOWN_MAX_MS, + DAEMON_WORKER_EXIT_GRACE_MS, +} from './core/shutdown-budgets.js'; +import { tryWithBotTurnMutation } from './core/bot-turn-mutation-gate.js'; import { evaluateVcMeetingConsumerIsolation } from './services/vc-meeting-consumer-isolation.js'; import { markSessionActivity, @@ -17521,21 +17538,154 @@ export async function startDaemon(botIndex?: number): Promise { }, 5_000).unref?.(); } - // Graceful shutdown. Sends SIGTERM (or `{type:'close'}` IPC via killWorker) - // to every worker, then waits up to SHUTDOWN_GRACE_MS for them to exit + // Graceful shutdown. Riff owners first run a three-phase non-cancelling drain + // that durably ACKs every exact final task lineage as one fleet transaction. + // No worker exits until every participant is prepared and fresh-verified. + // Ordinary workers then receive SIGTERM / close IPC and the daemon waits up + // to DAEMON_WORKER_EXIT_GRACE_MS for them to exit // before sending SIGKILL to stragglers. Without the wait, daemon // `process.exit(0)` races worker signal delivery — and any worker whose // main thread is in a sync code path (e.g. the bridge fingerprint scan // bug fixed in v2.9.2) loses the signal and survives as a ppid=1 orphan // forever (we'd accumulated 841 such orphans across daemon restarts, // consuming ~65 GB of RAM until manually SIGKILL'd). - const SHUTDOWN_GRACE_MS = 3000; let shuttingDown = false; const shutdown = async () => { if (shuttingDown) return; shuttingDown = true; + const shutdownDeadlineMs = Date.now() + DAEMON_SHUTDOWN_MAX_MS; + const mutationResult = await tryWithBotTurnMutation( + cfg.larkAppId, + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS, + async () => { setSessionLifecycleShutdown(true); logger.info(`Daemon shutting down... (active: ${getActiveCount()})`); + + const initialShutdownFleet = collectUniqueDaemonShutdownSessions(activeSessions.values()); + if (!initialShutdownFleet.ok) { + setSessionLifecycleShutdown(false); + shuttingDown = false; + logger.error(`Daemon remains online: ${initialShutdownFleet.error}`); + return; + } + + // Preflight Riff before stopping services/removing descriptors. A failed + // drain or persistence write aborts every successfully prepared peer, so a + // refused shutdown returns to a live fleet instead of leaving an early + // participant workerless. + const riffCandidates = initialShutdownFleet.sessions + .filter(ds => shutdownBackendDisposition(ds) === 'riff-drain-detach'); + const riffPrepareResults = await prepareRiffFleetForShutdown(riffCandidates, { + deadlineMs: shutdownDeadlineMs, + }); + const riffPrepared = riffPrepareResults.filter( + (entry): entry is { ds: DaemonSession; result: PreparedRiffShutdown } => entry.result.ok, + ); + const riffFenced = riffPrepareResults.filter( + (entry): entry is { ds: DaemonSession; result: FencedRiffShutdownParticipant } => + entry.result.fence !== 'none', + ); + + const abortRiffFleet = async ( + reason: string, + retainFenced: ReadonlySet = new Set(), + ): Promise => { + for (const ds of retainFenced) { + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Riff shutdown participant remains fenced: ` + + 'durable/runtime ownership could not be reconciled safely', + ); + } + const aborts = await abortRiffShutdownFleet(riffFenced + .filter(({ ds }) => !retainFenced.has(ds)) + .map(({ ds, result }) => ({ ds, result })), { + deadlineMs: shutdownDeadlineMs, + }); + for (const { ds, result } of aborts) { + if (result.ok) continue; + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Riff shutdown rollback was not ACKed: ` + + `${result.error ?? 'unknown'}; admission remains fail-closed`, + ); + } + setSessionLifecycleShutdown(false); + shuttingDown = false; + logger.error(`Daemon remains online: ${reason}`); + }; + + const riffPrepareFailures = riffPrepareResults.filter(entry => !entry.result.ok); + if (riffPrepareFailures.length > 0) { + for (const { ds, result } of riffPrepareFailures) { + if (result.ok) continue; + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Daemon shutdown prepare refused: ` + + `${result.error}${result.taskId ? ` (task ${result.taskId})` : ''}`, + ); + } + await abortRiffFleet( + `${riffPrepareFailures.length} Riff owner(s) could not be safely drained`, + ); + return; + } + + const riffPersistence = persistPreparedRiffShutdownFleet(riffPrepared, { + deadlineMs: shutdownDeadlineMs, + }); + if (!riffPersistence.ok) { + const retainIds = new Set(riffPersistence.retainFencedSessionIds); + const retainFenced = new Set(riffPrepared + .filter(({ ds }) => retainIds.has(ds.session.sessionId)) + .map(({ ds }) => ds)); + logger.error(`Daemon shutdown lineage verification refused: ${riffPersistence.error}`); + await abortRiffFleet( + `${riffPersistence.sessionIds.length} Riff lineage row(s) could not be fresh-verified`, + retainFenced, + ); + return; + } + + // Recheck generation ownership while every participant is still fenced and + // daemon services remain live. + const currentShutdownFleet = collectUniqueDaemonShutdownSessions(activeSessions.values()); + if (!currentShutdownFleet.ok) { + const ambiguousSessionId = currentShutdownFleet.sessionId; + await abortRiffFleet( + `Riff ownership became ambiguous after shutdown preflight: ${currentShutdownFleet.error}`, + new Set(riffPrepared + .filter(({ ds }) => ds.session.sessionId === ambiguousSessionId) + .map(({ ds }) => ds)), + ); + return; + } + const currentRiffOwners = currentShutdownFleet.sessions + .filter(ds => shutdownBackendDisposition(ds) === 'riff-drain-detach'); + const riffPreparedOwners = new Set(riffPrepared.map(entry => entry.ds)); + const riffGenerationMismatch = currentRiffOwners.some(ds => !riffPreparedOwners.has(ds)) + || riffPrepared.some(({ ds, result }) => + !currentRiffOwners.includes(ds) || !isPreparedRiffSessionCurrent(ds, result)); + if (riffGenerationMismatch) { + const retainFenced = new Set(riffPrepared + .filter(({ ds, result }) => + (!currentRiffOwners.includes(ds) || !isPreparedRiffSessionCurrent(ds, result)) + && !canAbortVerifiedExitedRiffPreparation(ds, result)) + .map(({ ds }) => ds)); + await abortRiffFleet( + 'Riff ownership changed after shutdown preflight', + retainFenced, + ); + return; + } + + // Validate-all above, then commit-all synchronously with no await or + // callback boundary between peers. + const riffRetiredWorkers: ChildProcess[] = []; + for (const { ds, result } of riffPrepared) { + if (!commitPreparedRiffShutdown(ds, result)) { + throw new Error(`Riff shutdown commit invariant lost for ${ds.session.sessionId}`); + } + if (result.worker) riffRetiredWorkers.push(result.worker); + } + scheduler.stopScheduler(); stopMaintenance(); vcMeetingTerminalReconciler?.stop(); @@ -17569,23 +17719,33 @@ export async function startDaemon(botIndex?: number): Promise { const pendingExits: Array> = []; const survivors: ChildProcess[] = []; - for (const [, ds] of activeSessions) { + const trackWorkerExit = (w: ChildProcess): void => { + if (w.exitCode !== null || w.signalCode !== null) return; + pendingExits.push(new Promise(resolve => { + w.once('exit', () => resolve()); + // Close the check/listener race if exit landed between the first + // status read and once(). + if (w.exitCode !== null || w.signalCode !== null) resolve(); + })); + survivors.push(w); + }; + for (const worker of riffRetiredWorkers) trackWorkerExit(worker); + for (const ds of currentShutdownFleet.sessions) { if (ds.worker && !ds.worker.killed) { logger.info(`Shutting down worker for session ${ds.session.sessionId}`); const w = ds.worker; - // Capture the exit promise BEFORE killWorker nulls ds.worker. - if (w.exitCode === null && w.signalCode === null) { - pendingExits.push(new Promise(resolve => { - w.once('exit', () => resolve()); - })); - survivors.push(w); + const disposition = shutdownBackendDisposition(ds); + if (disposition === 'riff-drain-detach') { + throw new Error(`undrained Riff generation after atomic commit: ${ds.session.sessionId}`); } + // Capture the exit promise BEFORE killWorker nulls ds.worker. + trackWorkerExit(w); // Branch by the session's FROZEN backend (stamped on Session.backendType // at spawn), NOT the bot's live config — a dashboard backendType edit must // not change how a running session is torn down, or we'd e.g. try to // detach-preserve a "herdr" session whose real pane is tmux (freeze-once). // undefined (frozen pty, or unresolvable legacy) → non-persistent → killWorker. - if (shutdownBackendDisposition(ds) === 'detach') { + if (disposition === 'detach') { // Persistent backends (tmux / herdr / zellij): just kill the worker process — // the multiplexer session survives for re-attach. The worker's SIGTERM // handler calls backend.kill(), which only DETACHES. Going through @@ -17605,7 +17765,11 @@ export async function startDaemon(botIndex?: number): Promise { } if (pendingExits.length > 0) { - const timeout = new Promise(resolve => setTimeout(resolve, SHUTDOWN_GRACE_MS)); + const exitGraceMs = Math.min( + DAEMON_WORKER_EXIT_GRACE_MS, + Math.max(0, shutdownDeadlineMs - Date.now()), + ); + const timeout = new Promise(resolve => setTimeout(resolve, exitGraceMs)); await Promise.race([Promise.all(pendingExits), timeout]); let stragglers = 0; for (const w of survivors) { @@ -17615,7 +17779,7 @@ export async function startDaemon(botIndex?: number): Promise { } } if (stragglers > 0) { - logger.warn(`${stragglers}/${survivors.length} worker(s) didn't exit within ${SHUTDOWN_GRACE_MS}ms — SIGKILL'd to prevent ppid=1 orphans.`); + logger.warn(`${stragglers}/${survivors.length} worker(s) didn't exit within ${exitGraceMs}ms — SIGKILL'd to prevent ppid=1 orphans.`); } } @@ -17626,6 +17790,15 @@ export async function startDaemon(botIndex?: number): Promise { removePidFile(); process.exit(0); + }, + ); + if (!mutationResult.acquired) { + shuttingDown = false; + logger.error( + `Daemon remains online: could not acquire exclusive shutdown mutation lease ` + + `(${mutationResult.reason})`, + ); + } }; process.on('SIGTERM', () => { shutdown().catch(err => { logger.error(`shutdown failed: ${err?.message ?? err}`); process.exit(1); }); }); diff --git a/src/services/session-store.ts b/src/services/session-store.ts index e70c649b8..69407c22d 100644 --- a/src/services/session-store.ts +++ b/src/services/session-store.ts @@ -1,4 +1,12 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, readdirSync } from 'node:fs'; +import { + readFileSync, + writeFileSync, + mkdirSync, + existsSync, + renameSync, + readdirSync, + unlinkSync, +} from 'node:fs'; import { join, dirname } from 'node:path'; import { randomUUID } from 'node:crypto'; import { config } from '../config.js'; @@ -6,6 +14,7 @@ import { logger } from '../utils/logger.js'; import { cleanupMaterializedDashboardImages } from '../core/dashboard-images.js'; import { deleteFrozenCards } from './frozen-card-store.js'; import type { Session } from '../types.js'; +import { withFileLockSync } from '../utils/file-lock.js'; let sessions: Map = new Map(); let loaded = false; @@ -20,6 +29,64 @@ export function stripLegacyPendingCardFields(session: Record): for (const f of LEGACY_PENDING_CARD_FIELDS) delete session[f]; } +/** The active row no longer has the lineage/ownership sampled by the caller. */ +export class RiffLineageOwnershipError extends Error { + override readonly name = 'RiffLineageOwnershipError'; +} + +export type RiffDurableOwner = { + pid: number | null; + larkAppId: string | null; + backendType: string | null; +}; + +export type ActiveRiffShutdownSnapshot = { + sessionId: string; + taskId: string | null; + owner: RiffDurableOwner; +}; + +export type ActiveRiffLineageBatchUpdate = ActiveRiffShutdownSnapshot & { + targetTaskId: string | null; + expectedCurrentTaskIds: readonly (string | null)[]; +}; + +export type RiffLineageBatchFailureStage = + | 'prewrite_ownership' + | 'prewrite_io' + | 'postrename_ambiguity'; + +export class RiffLineageBatchError extends Error { + override readonly name = 'RiffLineageBatchError'; + + constructor( + readonly stage: RiffLineageBatchFailureStage, + readonly sessionIds: readonly string[], + message: string, + ) { + super(message); + } +} + +function riffDurableOwner(session: Session): RiffDurableOwner { + return { + pid: session.pid ?? null, + larkAppId: session.larkAppId ?? null, + backendType: session.backendType ?? null, + }; +} + +function riffOwnersEqual(left: RiffDurableOwner, right: RiffDurableOwner): boolean { + return left.pid === right.pid + && left.larkAppId === right.larkAppId + && left.backendType === right.backendType; +} + +let testOnlyAfterRiffBatchRename: (() => void) | undefined; +export function __testOnly_setAfterRiffBatchRename(hook: (() => void) | undefined): void { + testOnlyAfterRiffBatchRename = hook; +} + /** * Initialise session store for a specific bot (multi-daemon mode). * When appId is set, sessions are stored in `sessions-{appId}.json`. @@ -139,6 +206,198 @@ function readExistingSessionsFromDisk(fp: string): { raw: string; parsed: Record } } +function readSessionsProjectionStrict(fp: string): { raw: string; parsed: Record } { + if (!existsSync(fp)) return { raw: '', parsed: {} }; + const raw = readFileSync(fp, 'utf-8'); + const value = JSON.parse(raw) as unknown; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`invalid sessions projection at ${fp}`); + } + return { raw, parsed: value as Record }; +} + +function duplicateIds(ids: readonly string[]): string[] { + const seen = new Set(); + const duplicates = new Set(); + for (const id of ids) { + if (seen.has(id)) duplicates.add(id); + else seen.add(id); + } + return [...duplicates]; +} + +/** + * Sample every active Riff participant from one fresh sessions projection. + * Fleet shutdown takes this snapshot before fencing any worker. + */ +export function getActiveRiffShutdownSnapshotsBatch( + sessionIds: readonly string[], + options: { maxWaitMs?: number } = {}, +): ActiveRiffShutdownSnapshot[] { + if (sessionIds.length === 0) return []; + const duplicates = duplicateIds(sessionIds); + if (duplicates.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + duplicates, + `duplicate Riff shutdown session ids: ${duplicates.join(', ')}`, + ); + } + + ensureDir(); + const fp = getFilePath(); + try { + return withFileLockSync(fp, () => { + const { parsed } = readSessionsProjectionStrict(fp); + const invalid = sessionIds.filter((sessionId) => { + const session = parsed[sessionId]; + return !session || session.status !== 'active'; + }); + if (invalid.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + invalid, + `cannot snapshot non-active Riff sessions: ${invalid.join(', ')}`, + ); + } + return sessionIds.map((sessionId) => { + const session = parsed[sessionId]!; + return { + sessionId, + taskId: session.riffParentTaskId ?? null, + owner: riffDurableOwner(session), + }; + }); + }, { maxWaitMs: options.maxWaitMs }); + } catch (error) { + if (error instanceof RiffLineageBatchError) throw error; + throw new RiffLineageBatchError( + 'prewrite_io', + [...sessionIds], + `failed to snapshot active Riff sessions: ${String(error)}`, + ); + } +} + +/** + * Commit every prepared Riff lineage as one compare-and-set transaction. + * The published projection is read back under the same lock before workers + * are allowed to exit. + */ +export function persistActiveRiffLineagesExactBatch( + updates: readonly ActiveRiffLineageBatchUpdate[], + options: { maxWaitMs?: number } = {}, +): ActiveRiffShutdownSnapshot[] { + if (updates.length === 0) return []; + const sessionIds = updates.map(update => update.sessionId); + const duplicates = duplicateIds(sessionIds); + if (duplicates.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + duplicates, + `duplicate Riff lineage batch session ids: ${duplicates.join(', ')}`, + ); + } + + ensureDir(); + const fp = getFilePath(); + let published = false; + let tmpFp: string | undefined; + try { + return withFileLockSync(fp, () => { + const { raw, parsed } = readSessionsProjectionStrict(fp); + const conflicts: string[] = []; + for (const update of updates) { + const durable = parsed[update.sessionId]; + const durableTaskId = durable?.riffParentTaskId ?? null; + if (!durable + || durable.status !== 'active' + || !update.expectedCurrentTaskIds.some(candidate => candidate === durableTaskId) + || !riffOwnersEqual(riffDurableOwner(durable), update.owner)) { + conflicts.push(update.sessionId); + } + } + if (conflicts.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + conflicts, + `Riff lineage batch compare-and-set failed for: ${conflicts.join(', ')}`, + ); + } + + for (const update of updates) { + const durable = parsed[update.sessionId]!; + const next: Session = { + ...durable, + riffParentTaskId: update.targetTaskId ?? undefined, + }; + stripLegacyPendingCardFields(next as unknown as Record); + parsed[update.sessionId] = next; + } + + const json = JSON.stringify(parsed, null, 2); + if (json !== raw) { + tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, json, 'utf-8'); + renameSync(tmpFp, fp); + tmpFp = undefined; + published = true; + testOnlyAfterRiffBatchRename?.(); + } + + let verifiedProjection: Record; + try { + verifiedProjection = readSessionsProjectionStrict(fp).parsed; + } catch (error) { + throw new RiffLineageBatchError( + published ? 'postrename_ambiguity' : 'prewrite_io', + [...sessionIds], + `failed to read back Riff lineage batch: ${String(error)}`, + ); + } + + const ambiguous = updates.filter((update) => { + const durable = verifiedProjection[update.sessionId]; + return !durable + || durable.status !== 'active' + || (durable.riffParentTaskId ?? null) !== update.targetTaskId + || !riffOwnersEqual(riffDurableOwner(durable), update.owner); + }).map(update => update.sessionId); + if (ambiguous.length > 0) { + throw new RiffLineageBatchError( + published ? 'postrename_ambiguity' : 'prewrite_ownership', + ambiguous, + `Riff lineage batch readback mismatch for: ${ambiguous.join(', ')}`, + ); + } + + const verified = updates.map((update) => ({ + sessionId: update.sessionId, + taskId: update.targetTaskId, + owner: riffDurableOwner(verifiedProjection[update.sessionId]!), + })); + if (loaded) { + for (const update of updates) { + const cached = sessions.get(update.sessionId); + if (cached) cached.riffParentTaskId = update.targetTaskId ?? undefined; + } + } + return verified; + }, { maxWaitMs: options.maxWaitMs }); + } catch (error) { + if (error instanceof RiffLineageBatchError) throw error; + throw new RiffLineageBatchError( + published ? 'postrename_ambiguity' : 'prewrite_io', + [...sessionIds], + `failed to persist Riff lineage batch: ${String(error)}`, + ); + } finally { + if (tmpFp) { + try { unlinkSync(tmpFp); } catch { /* best-effort orphan cleanup */ } + } + } +} + function save(): void { ensureDir(); const fp = getFilePath(); @@ -190,6 +449,21 @@ export function getSession(sessionId: string): Session | undefined { return sessions.get(sessionId) ?? findInOtherFiles(sessionId); } +/** Cross-process fresh read ordered after daemon/CLI writes by the shared lock. */ +export function getSessionFresh(sessionId: string): Session | undefined { + ensureDir(); + const fp = getFilePath(); + return withFileLockSync(fp, () => { + if (!existsSync(fp)) return undefined; + try { + const data = JSON.parse(readFileSync(fp, 'utf-8')) as Record; + return data[sessionId]; + } catch { + return undefined; + } + }); +} + /** * Search all session files for a session not found in the current file. * @@ -215,22 +489,42 @@ function findInOtherFiles(sessionId: string): Session | undefined { return undefined; } -export function closeSession(sessionId: string): void { +export function closeSession( + sessionId: string, + opts: { clearRiffParentTaskId?: boolean } = {}, +): void { load(); const session = sessions.get(sessionId); if (session) { - if (session.larkAppId && session.dashboardAttachments?.length) { + const priorStatus = session.status; + const priorClosedAt = session.closedAt; + const priorRiffParentTaskId = session.riffParentTaskId; + const priorDashboardAttachments = session.dashboardAttachments; + const priorQueuedAttachments = session.queuedAttachments; + session.status = 'closed'; + session.closedAt = new Date().toISOString(); + session.dashboardAttachments = undefined; + session.queuedAttachments = undefined; + // Riff cancellation has already completed before this durable transition. + // Clear its retry handle in the same atomic save as status='closed'. + if (opts.clearRiffParentTaskId) session.riffParentTaskId = undefined; + try { + save(); + } catch (err) { + session.status = priorStatus; + session.closedAt = priorClosedAt; + session.riffParentTaskId = priorRiffParentTaskId; + session.dashboardAttachments = priorDashboardAttachments; + session.queuedAttachments = priorQueuedAttachments; + throw err; + } + if (session.larkAppId && priorDashboardAttachments?.length) { try { - cleanupMaterializedDashboardImages(session.larkAppId, session.dashboardAttachments); - session.dashboardAttachments = undefined; - session.queuedAttachments = undefined; + cleanupMaterializedDashboardImages(session.larkAppId, priorDashboardAttachments); } catch (error: any) { logger.warn(`Failed to clean Dashboard images for session ${sessionId}: ${error?.message ?? error}`); } } - session.status = 'closed'; - session.closedAt = new Date().toISOString(); - save(); deleteFrozenCards(sessionId); logger.info(`Closed session ${sessionId}`); } @@ -251,6 +545,68 @@ export function updateSession(session: Session): void { save(); } +/** + * Persist one exact Riff follow-up lineage for an active durable owner. + * The process cache changes only after the atomic file replacement succeeds. + */ +export function persistActiveRiffLineageExact( + sessionId: string, + taskId: string | null, + options: { + expectedCurrentTaskIds?: readonly (string | null)[]; + expectedOwner?: RiffDurableOwner; + } = {}, +): Session { + load(); + ensureDir(); + const fp = getFilePath(); + return withFileLockSync(fp, () => { + const { raw, parsed } = readExistingSessionsFromDisk(fp); + const durable = parsed[sessionId]; + if (!durable || durable.status !== 'active') { + throw new RiffLineageOwnershipError( + `cannot persist Riff lineage for non-active session ${sessionId}`, + ); + } + const durableTaskId = durable.riffParentTaskId ?? null; + const expected = options.expectedCurrentTaskIds; + if (expected && !expected.some(candidate => candidate === durableTaskId)) { + throw new RiffLineageOwnershipError( + `Riff lineage compare-and-set failed for ${sessionId} ` + + `(current=${durableTaskId ?? 'none'}, expected=${expected.map(id => id ?? 'none').join('|')})`, + ); + } + if (options.expectedOwner && !riffOwnersEqual(riffDurableOwner(durable), options.expectedOwner)) { + throw new RiffLineageOwnershipError( + `Riff owner compare-and-set failed for ${sessionId} ` + + `(current=${JSON.stringify(riffDurableOwner(durable))}, ` + + `expected=${JSON.stringify(options.expectedOwner)})`, + ); + } + + const next: Session = { + ...durable, + riffParentTaskId: taskId ?? undefined, + }; + stripLegacyPendingCardFields(next as unknown as Record); + parsed[sessionId] = next; + const json = JSON.stringify(parsed, null, 2); + if (json !== raw) { + const tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, json, 'utf-8'); + renameSync(tmpFp, fp); + } + + const cached = sessions.get(sessionId); + if (cached) { + cached.riffParentTaskId = taskId ?? undefined; + return cached; + } + sessions.set(sessionId, next); + return next; + }); +} + export function listSessions(): Session[] { load(); return [...sessions.values()]; diff --git a/src/types.ts b/src/types.ts index 074cd9551..89ff2c2c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -582,7 +582,15 @@ export type DaemonToWorker = * as a model turn. Only adapters declaring buildSessionRenameCommand handle * it; all other CLIs ignore it. */ | { type: 'rename_session'; title: string } - | { type: 'close' } + | { type: 'close'; requestId?: string } + | { type: 'close_commit'; requestId: string } + | { type: 'close_abort'; requestId: string } + /** Fence new Riff writes, drain accepted writes, and report exact lineage. */ + | { type: 'riff_shutdown_prepare'; requestId: string } + /** Final lineage is durable; detach the worker generation. */ + | { type: 'riff_shutdown_commit'; requestId: string } + /** Shutdown could not commit; restore Riff write admission. */ + | { type: 'riff_shutdown_abort'; requestId: string } | { type: 'suspend' } | { type: 'restart' } /** Lease watchdog fencing: only the exact still-running durable attempt may @@ -716,4 +724,25 @@ export type WorkerToDaemon = | { type: 'adopt_preamble'; userText: string; assistantText: string; turnId?: string } | { type: 'deferred_topic_materialized'; sessionId: string; turnId: string; rootMessageId: string } | { type: 'riff_access_url'; accessUrl: string; directAccessUrl?: string; turnId?: string; dispatchAttempt?: number } - | { type: 'riff_task_id'; taskId: string | null }; + | { type: 'riff_task_id'; taskId: string | null } + | { + type: 'riff_shutdown_result'; + requestId: string; + phase: 'prepare' | 'abort'; + ok: boolean; + taskId: string | null; + error?: string; + } + | { + type: 'close_abort_result'; + requestId: string; + ok: boolean; + error?: string; + } + | { + type: 'close_result'; + requestId: string; + ok: boolean; + taskId?: string; + error?: string; + }; diff --git a/src/worker.ts b/src/worker.ts index 21c31bcb7..33e8d796f 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -60,6 +60,7 @@ import { terminalReleasesDurableTurn, type PendingCliInput, } from './utils/pending-input-queue.js'; +import { riffWorkerShutdownInputBlocker } from './core/riff-worker-shutdown-readiness.js'; import { ReadyGate, shouldArmReadyGate } from './utils/ready-gate.js'; import { shouldRunStartupCommandsOnSpawn, shouldDeferInitialPromptForStartup } from './core/startup-commands.js'; import { sanitizePerBotEnv } from './core/per-bot-env.js'; @@ -201,7 +202,12 @@ import { DEVICE_AUTHORITY_DIRECTORY, DEVICE_CREDENTIAL_FILE, } from './platform/device-paths.js'; -import type { BackendType, SessionBackend } from './adapters/backend/types.js'; +import type { + BackendType, + SessionBackend, + SessionDestroyResult, + SessionShutdownDetachResult, +} from './adapters/backend/types.js'; import { tmuxEnv, probeTmuxFunctionalWithRetry } from './setup/ensure-tmux.js'; import { tmuxRestartJitterMs } from './core/tmux-recovery.js'; import { IdleDetector } from './utils/idle-detector.js'; @@ -1093,6 +1099,13 @@ let effectiveBackendType: BackendType = 'pty'; * generation. The daemon receives this over private IPC; child-writable PID * marker files remain diagnostics only. */ let currentCliCredentialIsolated = false; +/** Successful Riff close prepare awaiting durable daemon commit. */ +let preparedCloseRequestId: string | null = null; +let closeRequestInFlightId: string | null = null; +let lastAbortedCloseRequestId: string | null = null; +/** Graceful daemon-shutdown detach stays alive until lineage commit. */ +let shutdownDetachRequestId: string | null = null; +let shutdownDetachPhase: 'preparing' | 'prepared' | null = null; /** pty-under-zellij backend (BACKEND_TYPE=zellij). Behaves like the non-tmux * pty path for the worker (renderer screenshots, relay web terminal) but owns * a persistent zellij session that survives daemon restart. */ @@ -1448,6 +1461,8 @@ async function runStartupCommands(): Promise { } const pendingMessages: PendingCliInput[] = []; +/** Async init must materialize its opening input before shutdown may fence. */ +let initPromptMaterialized = false; /** Literal commands that arrived while native /rename owned the TUI or while * an owned CLI restart was fenced. Normal raw_input commands are still * delivered immediately (including while busy). */ @@ -9660,6 +9675,7 @@ process.on('message', async (raw: unknown) => { initialInputCommitted = true; } if (initialInputCommitted) acknowledgeTurnInputCommitted(msg.turnId); + initPromptMaterialized = true; // A backend may become prompt-ready before spawnCli() returns. The // initial prompt is queued only afterwards, so the earlier @@ -9901,6 +9917,10 @@ process.on('message', async (raw: unknown) => { } case 'restart': { + if (effectiveBackendType === 'riff') { + log('Refused Riff generation restart; the existing lineage-owning worker is retained'); + break; + } await restartCliProcess('daemon request', { preservePending: true }); break; } @@ -10136,16 +10156,69 @@ process.on('message', async (raw: unknown) => { case 'close': { log('Close requested'); + if (effectiveBackendType === 'riff') { + if (!msg.requestId) { + log('Refused unsafe request-less Riff close; explicit close requires prepare/commit'); + break; + } + + let result: SessionDestroyResult; + let attemptedPrepare = false; + if (shutdownDetachRequestId) { + result = { + ok: false, + error: `shutdown detach already ${shutdownDetachPhase ?? 'active'} as ${shutdownDetachRequestId}`, + }; + } else if (preparedCloseRequestId) { + result = { + ok: false, + error: `close already prepared as ${preparedCloseRequestId}`, + }; + } else if (closeRequestInFlightId) { + result = { + ok: false, + error: `close prepare already in flight as ${closeRequestInFlightId}`, + }; + } else { + attemptedPrepare = true; + lastAbortedCloseRequestId = null; + closeRequestInFlightId = msg.requestId; + try { + const raw = await backend?.destroySession?.(); + result = raw && typeof raw === 'object' && 'ok' in raw + ? raw as SessionDestroyResult + : { ok: true }; + } catch (err) { + result = { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + if (result.ok) { + preparedCloseRequestId = msg.requestId; + } else if (attemptedPrepare) { + await backend?.abortDestroySession?.(); + lastAbortedCloseRequestId = msg.requestId; + } + if (closeRequestInFlightId === msg.requestId) closeRequestInFlightId = null; + send({ + type: 'close_result', + requestId: msg.requestId, + ok: result.ok, + ...(result.taskId ? { taskId: result.taskId } : {}), + ...(result.error ? { error: result.error } : {}), + }); + if (!result.ok) { + log(`Riff close prepare failed (${result.error ?? 'cancel failed'}); session stays active for retry`); + } + break; + } + stopScreenshotLoop(); - // destroySession kills tmux session permanently; kill() only detaches. - // riff 的 destroySession 是异步远端取消——必须有界 await：紧跟着的 - // process.exit 会掐断未发出的 fetch，让已关闭话题的远端 agent 继续跑。 + // Local close: destroySession kills persistent owned sessions. Riff has + // already been handled above and cannot enter this request-less path. const closeTeardown = backend?.destroySession?.(); if (closeTeardown && typeof (closeTeardown as Promise).then === 'function') { - try { - // 预算层级见 RiffBackend.destroySession（总 deadline 20s）——这里 22s 只作兜底。 - await Promise.race([closeTeardown, new Promise((r) => setTimeout(r, 22_000))]); - } catch { /* logged inside destroySession */ } + try { await Promise.race([closeTeardown, new Promise((r) => setTimeout(r, 22_000))]); } + catch { /* logged by backend */ } } killCli(); // Bridge marker file outlives a single CLI process (we keep it across @@ -10157,7 +10230,184 @@ process.on('message', async (raw: unknown) => { process.exit(0); } + case 'riff_shutdown_prepare': { + if (effectiveBackendType !== 'riff') { + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'prepare', + ok: false, + taskId: null, + error: 'not_riff_backend', + }); + break; + } + let result: SessionShutdownDetachResult; + const inputBlocker = riffWorkerShutdownInputBlocker({ + initPromptMaterialized, + isFlushing, + pendingMessages: pendingMessages.length, + pendingRawInputs: pendingRawInputs.length, + pendingSessionRename: pendingSessionRename !== null, + sessionRenameInFlight, + commandLineWritesPending, + }); + if (preparedCloseRequestId || closeRequestInFlightId) { + result = { ok: false, taskId: null, error: 'explicit_close_in_progress' }; + } else if (shutdownDetachRequestId) { + result = { + ok: false, + taskId: null, + error: `shutdown detach already ${shutdownDetachPhase ?? 'active'} as ${shutdownDetachRequestId}`, + }; + } else if (inputBlocker) { + result = { + ok: false, + taskId: null, + error: `worker_inputs_not_drained:${inputBlocker}`, + }; + } else { + shutdownDetachRequestId = msg.requestId; + shutdownDetachPhase = 'preparing'; + try { + result = await backend?.prepareShutdownDetach?.() + ?? { ok: false, taskId: null, error: 'shutdown_detach_unsupported' }; + } catch (err) { + result = { + ok: false, + taskId: null, + error: err instanceof Error ? err.message : String(err), + }; + } + if (shutdownDetachRequestId === msg.requestId && result.ok) { + shutdownDetachPhase = 'prepared'; + } + } + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'prepare', + ok: result.ok, + taskId: result.taskId, + ...(result.error ? { error: result.error } : {}), + }); + break; + } + + case 'riff_shutdown_commit': { + if (effectiveBackendType !== 'riff' + || shutdownDetachRequestId !== msg.requestId + || shutdownDetachPhase !== 'prepared') { + log(`Ignoring stale Riff shutdown commit ${msg.requestId}`); + break; + } + log(`Riff shutdown detach committed (${msg.requestId})`); + shutdownDetachRequestId = null; + shutdownDetachPhase = null; + backend?.commitShutdownDetach?.(); + intentionalRestartBackend = backend; + stopScreenshotLoop(); + killCli(); + cleanup(); + process.exit(0); + } + + case 'riff_shutdown_abort': { + if (shutdownDetachRequestId !== msg.requestId) { + log(`Ignoring stale Riff shutdown abort ${msg.requestId}`); + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'abort', + ok: false, + taskId: null, + error: 'shutdown_detach_not_active', + }); + break; + } + log(`Riff shutdown detach aborted (${msg.requestId})`); + let result: SessionShutdownDetachResult; + try { + result = await backend?.abortShutdownDetach?.() + ?? { ok: false, taskId: null, error: 'shutdown_abort_unsupported' }; + } catch (err) { + result = { + ok: false, + taskId: null, + error: err instanceof Error ? err.message : String(err), + }; + } + if (result.ok && shutdownDetachRequestId === msg.requestId) { + shutdownDetachRequestId = null; + shutdownDetachPhase = null; + } + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'abort', + ok: result.ok, + taskId: result.taskId, + ...(result.error ? { error: result.error } : {}), + }); + break; + } + + case 'close_commit': { + if (!preparedCloseRequestId || preparedCloseRequestId !== msg.requestId) { + log(`Ignoring stale close_commit ${msg.requestId}`); + break; + } + log(`Close committed (${msg.requestId})`); + preparedCloseRequestId = null; + closeRequestInFlightId = null; + lastAbortedCloseRequestId = null; + backend?.commitDestroySession?.(); + stopScreenshotLoop(); + killCli(); + clearSendMarkers(); + cleanup(); + process.exit(0); + } + + case 'close_abort': { + const alreadyRestored = lastAbortedCloseRequestId === msg.requestId; + if (!alreadyRestored + && preparedCloseRequestId !== msg.requestId + && closeRequestInFlightId !== msg.requestId) { + log(`Ignoring stale close_abort ${msg.requestId}`); + send({ + type: 'close_abort_result', + requestId: msg.requestId, + ok: false, + error: 'close_abort_not_active', + }); + break; + } + log(`Close aborted (${msg.requestId}); Riff admission restored`); + let abortError: string | null = null; + if (!alreadyRestored) { + try { await backend?.abortDestroySession?.(); } + catch (err) { abortError = err instanceof Error ? err.message : String(err); } + } + if (!abortError) { + preparedCloseRequestId = null; + closeRequestInFlightId = null; + lastAbortedCloseRequestId = null; + } + send({ + type: 'close_abort_result', + requestId: msg.requestId, + ok: abortError === null, + ...(abortError ? { error: abortError } : {}), + }); + break; + } + case 'suspend': { + if (effectiveBackendType === 'riff') { + log('Refused unsafe Riff suspend; explicit close requires prepare/commit'); + break; + } log('Suspend requested'); stopScreenshotLoop(); stopBridgeWatcher(); @@ -10176,10 +10426,7 @@ process.on('message', async (raw: unknown) => { // uses to recover sessions after a reboot kills the tmux server). revokeManagedTurnOriginForRestart(); try { - // riff：suspend 语义是「休眠待续」——绝不能 cancel 远端任务（血缘已持久化， - // 恢复时 follow-up 续上）；只断流 detach。 - if (effectiveBackendType === 'riff') backend?.kill(); - else (backend?.destroySession ?? backend?.kill)?.call(backend); + (backend?.destroySession ?? backend?.kill)?.call(backend); } catch { /* best-effort */ } backend = null; isPromptReady = false; diff --git a/test/explicit-session-backing-cleanup.test.ts b/test/explicit-session-backing-cleanup.test.ts new file mode 100644 index 000000000..eb15f436a --- /dev/null +++ b/test/explicit-session-backing-cleanup.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from 'vitest'; +import { cleanupExplicitSessionBacking } from '../src/core/explicit-session-backing-cleanup.js'; + +const SID = 'abcd1234-1111-2222-3333-444444444444'; + +describe('offline explicit session backing cleanup', () => { + for (const backendType of ['tmux', 'herdr', 'zellij'] as const) { + it(`confirms ${backendType} is absent before reporting cleanup success`, async () => { + const killPersistent = vi.fn(); + const probePersistent = vi.fn(() => 'missing' as const); + const result = await cleanupExplicitSessionBacking( + { sessionId: SID, backendType }, + { killPersistent, probePersistent }, + ); + + expect(result).toEqual({ + ok: true, + kind: 'destroyed_persistent', + backendType, + name: 'bmx-abcd1234', + }); + expect(killPersistent).toHaveBeenCalledWith(backendType, 'bmx-abcd1234'); + expect(probePersistent).toHaveBeenCalledWith(backendType, 'bmx-abcd1234'); + }); + } + + it('does not report success when a persistent backend still exists after kill', async () => { + const result = await cleanupExplicitSessionBacking( + { sessionId: SID, backendType: 'tmux' }, + { + killPersistent: vi.fn(), + probePersistent: vi.fn(() => 'exists'), + }, + ); + expect(result).toMatchObject({ + ok: false, + kind: 'persistent_destroy_failed', + error: 'backing_session_still_exists', + }); + }); + + it('cancels Riff using the supplied authoritative bot config and returns the exact task id', async () => { + const cancelRiffTask = vi.fn(async () => true); + const result = await cleanupExplicitSessionBacking( + { + sessionId: SID, + backendType: 'riff', + riffParentTaskId: 'riff-task-1', + riffConfig: { baseUrl: 'https://riff.example', jwt: 'token' }, + }, + { cancelRiffTask }, + ); + expect(cancelRiffTask).toHaveBeenCalledWith( + { baseUrl: 'https://riff.example', jwt: 'token' }, + 'riff-task-1', + ); + expect(result).toEqual({ ok: true, kind: 'cancelled_riff', taskId: 'riff-task-1' }); + }); + + it('fails closed on Riff cancel failure so the caller can preserve its task id', async () => { + const record = { + sessionId: SID, + backendType: 'riff' as const, + riffParentTaskId: 'riff-task-retry', + riffConfig: { baseUrl: 'https://riff.example' }, + }; + const result = await cleanupExplicitSessionBacking(record, { + cancelRiffTask: vi.fn(async () => false), + }); + expect(result).toEqual({ + ok: false, + kind: 'riff_cancel_failed', + backendType: 'riff', + taskId: 'riff-task-retry', + }); + expect(record.riffParentTaskId).toBe('riff-task-retry'); + }); + + it('fails closed when current Riff config is unavailable', async () => { + expect(await cleanupExplicitSessionBacking({ + sessionId: SID, + backendType: 'riff', + riffParentTaskId: 'riff-task-no-config', + })).toEqual({ + ok: false, + kind: 'riff_config_missing', + backendType: 'riff', + taskId: 'riff-task-no-config', + }); + }); + + it('never destroys an adopted user-owned pane', async () => { + const killPersistent = vi.fn(); + const probePersistent = vi.fn(); + expect(await cleanupExplicitSessionBacking( + { sessionId: SID, backendType: 'tmux', adopted: true }, + { killPersistent, probePersistent: probePersistent as any }, + )).toEqual({ ok: true, kind: 'skipped_adopted' }); + expect(killPersistent).not.toHaveBeenCalled(); + expect(probePersistent).not.toHaveBeenCalled(); + }); + + it('does not guess a persistent backend for an unstamped legacy row', async () => { + const killPersistent = vi.fn(); + const probePersistent = vi.fn(); + expect(await cleanupExplicitSessionBacking( + { sessionId: SID }, + { killPersistent, probePersistent: probePersistent as any }, + )).toEqual({ ok: true, kind: 'no_backing' }); + expect(killPersistent).not.toHaveBeenCalled(); + expect(probePersistent).not.toHaveBeenCalled(); + }); +}); diff --git a/test/persistent-backend-type.test.ts b/test/persistent-backend-type.test.ts index 7535a95df..065b95b43 100644 --- a/test/persistent-backend-type.test.ts +++ b/test/persistent-backend-type.test.ts @@ -162,4 +162,8 @@ describe('shutdownBackendDisposition (shutdown freeze-once)', () => { bot.backendType = 'herdr'; expect(shutdownBackendDisposition(ds({ sessionBackend: 'pty' }))).toBe('close'); }); + it('routes a frozen Riff worker through drain + durable lineage ACK, never direct SIGTERM', () => { + bot.backendType = 'pty'; + expect(shutdownBackendDisposition(ds({ sessionBackend: 'riff' }))).toBe('riff-drain-detach'); + }); }); diff --git a/test/riff-backend.test.ts b/test/riff-backend.test.ts index 26494d483..d1ddec915 100644 --- a/test/riff-backend.test.ts +++ b/test/riff-backend.test.ts @@ -170,6 +170,105 @@ describe('RiffBackend', () => { }); }); + describe('graceful shutdown detach drain', () => { + it('fences new writes, waits for a slow create, returns its exact id, and never cancels or streams it', async () => { + const be = makeBackend({ injectStatusLines: false }); + const ids: Array = []; + be.onTaskId(id => ids.push(id)); + be.spawn('', [], {} as any); + be.write('opening turn'); + await flush(); + + let settled = false; + const prepare = be.prepareShutdownDetach().then(result => { + settled = true; + return result; + }); + await flush(); + expect(settled).toBe(false); + + // This write arrived after the shutdown fence and must not extend the + // drain or create a new remote task. + be.write('must be rejected'); + resolvers.shift()!(taskResponse('task-shutdown-late')); + const result = await prepare; + + expect(result).toEqual({ ok: true, taskId: 'task-shutdown-late' }); + expect(ids).toEqual(['task-shutdown-late']); + expect(calls.filter(c => c.url.includes('/api/task-execute')).length).toBe(1); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(0); + expect(calls.filter(c => c.url.includes('/api/task-cancel')).length).toBe(0); + expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); + }); + + it('drains two writes accepted before the fence and reports the newest child lineage', async () => { + const be = makeBackend({ injectStatusLines: false }); + const ids: Array = []; + be.onTaskId(id => ids.push(id)); + be.write('first accepted write'); + be.write('second accepted write'); + await flush(); + + const prepare = be.prepareShutdownDetach(); + resolvers.shift()!(taskResponse('task-drain-1')); + await flush(); + const follow = calls.find(c => c.url.includes('/api/task-follow-up')); + expect(follow).toBeDefined(); + expect(JSON.parse(String(follow!.init?.body)).parentTaskId).toBe('task-drain-1'); + resolvers.shift()!(taskResponse('task-drain-2')); + + await expect(prepare).resolves.toEqual({ ok: true, taskId: 'task-drain-2' }); + expect(ids).toEqual(['task-drain-1', 'task-drain-2']); + expect(calls.filter(c => c.url.includes('/api/task-cancel')).length).toBe(0); + expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); + }); + + it('abort restores admission and reconnects the exact drained task', async () => { + const be = makeBackend({ injectStatusLines: false }); + be.write('opening turn'); + await flush(); + const prepare = be.prepareShutdownDetach(); + resolvers.shift()!(taskResponse('task-abort-resume')); + await expect(prepare).resolves.toEqual({ ok: true, taskId: 'task-abort-resume' }); + expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); + + await be.abortShutdownDetach(); + await flush(); + expect(calls.filter(c => c.url.includes('/api2/task-stream?id=task-abort-resume')).length).toBe(1); + + be.write('follow-up after aborted shutdown'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(1); + }); + + it('abort invalidates a still-draining prepare and restores admission only after it settles', async () => { + const be = makeBackend({ injectStatusLines: false }); + be.write('accepted before shutdown'); + await flush(); + + const prepare = be.prepareShutdownDetach(); + let abortSettled = false; + const abort = be.abortShutdownDetach().then(result => { + abortSettled = true; + return result; + }); + await flush(); + expect(abortSettled).toBe(false); + expect((be as any).shutdownDetaching).toBe(true); + + resolvers.shift()!(taskResponse('task-abort-during-drain')); + await expect(prepare).resolves.toEqual({ + ok: false, + taskId: 'task-abort-during-drain', + error: 'shutdown_detach_aborted', + }); + await expect(abort).resolves.toEqual({ ok: true, taskId: 'task-abort-during-drain' }); + expect((be as any).shutdownDetachPrepared).toBe(false); + expect((be as any).shutdownDetaching).toBe(false); + expect(calls.filter(c => c.url.includes('/api/task-cancel'))).toHaveLength(0); + }); + }); + describe('SSE clean EOF without done (finding C)', () => { it('treats a clean EOF with no done event as a stream failure — session must not stay busy forever', async () => { const be = makeBackend({ injectStatusLines: false }); @@ -515,7 +614,10 @@ describe('RiffBackend', () => { expect(cancels.length).toBeGreaterThanOrEqual(1); expect(JSON.parse(String(cancels[cancels.length - 1]!.init?.body ?? '{}')).id ?? JSON.parse(String(cancels[0]!.init?.body)).id).toBe('task-late'); expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); - expect((be as any).currentTaskId).not.toBe('task-late'); + // The cancelled child remains the exact lineage anchor until durable + // close commit. An abort can therefore continue from it, not its stale + // parent; it is still never streamed while prepare is fenced. + expect((be as any).currentTaskId).toBe('task-late'); }); it('close during follow-up: the late follow-up task is cancelled', async () => { @@ -536,8 +638,141 @@ describe('RiffBackend', () => { await destroyP; const cancelIds = calls.filter(c => c.url.includes('/api/task-cancel')).map(c => JSON.parse(String(c.init?.body)).id); expect(cancelIds).toContain('task-late-2'); - // late follow-up 不得成为 current，也不得开流 - expect((be as any).currentTaskId).not.toBe('task-late-2'); + // Preserve the cancelled child as retry lineage, but never stream it. + expect((be as any).currentTaskId).toBe('task-late-2'); + expect(calls.filter(c => c.url.includes('/api2/task-stream?id=task-late-2')).length).toBe(0); + }); + }); + + describe('close prepare / abort / retry state contract', () => { + it('restores write admission after cancel failure, then allows a close retry', async () => { + const be = makeBackend({ resumeParentTaskId: 'task-parent', injectStatusLines: false }); + let cancelCalls = 0; + fetchMock.mockImplementation(async (url: string | URL, init?: RequestInit) => { + const u = String(url); + calls.push({ url: u, init }); + if (u.includes('/api/task-cancel')) { + cancelCalls++; + return cancelCalls <= 2 + ? new Response('cancel failed', { status: 500 }) + : Response.json({ success: true, data: {} }); + } + if (u.includes('/api/task-follow-up')) return taskResponse('task-after-failure'); + if (u.includes('/api2/task-stream')) return pendingSseResponse(); + if (u.includes('/api/task-detail')) return Response.json({ success: true, data: { task: {} } }); + return taskResponse('unexpected-create'); + }); + + const failed = await be.destroySession(); + expect(failed).toMatchObject({ ok: false, taskId: 'task-parent' }); + expect((be as any).closing).toBe(false); + + be.write('continue after failed close'); + await flush(); await flush(); + const follow = calls.find(c => c.url.includes('/api/task-follow-up')); + expect(follow).toBeDefined(); + expect(JSON.parse(String(follow!.init?.body)).parentTaskId).toBe('task-parent'); + + const retried = await be.destroySession(); + expect(retried).toEqual({ ok: true, taskId: 'task-after-failure' }); + }); + + it('aborts a successful prepare without losing a late-created child lineage', async () => { + const be = makeBackend({ injectStatusLines: false }); + be.write('opening message'); + await flush(); + const preparedP = be.destroySession(); + resolvers.shift()!(taskResponse('task-late-prepared')); + const prepared = await preparedP; + expect(prepared).toEqual({ ok: true, taskId: 'task-late-prepared' }); + expect((be as any).closing).toBe(true); + + await be.abortDestroySession(); + expect((be as any).closing).toBe(false); + be.write('continue after durable commit failure'); + await flush(); + const follow = calls.find(c => c.url.includes('/api/task-follow-up')); + expect(follow).toBeDefined(); + expect(JSON.parse(String(follow!.init?.body)).parentTaskId).toBe('task-late-prepared'); + }); + + it('does not reopen admission when close timeout wins during an in-flight cancel', async () => { + const be = makeBackend({ resumeParentTaskId: 'task-timeout-parent', injectStatusLines: false }); + (be as any).destroyDeadlineMs = 10; + let resolveCancel!: (response: Response) => void; + fetchMock.mockImplementation(async (url: string | URL, init?: RequestInit) => { + const u = String(url); + calls.push({ url: u, init }); + if (u.includes('/api/task-cancel')) { + return new Promise(resolve => { resolveCancel = resolve; }); + } + if (u.includes('/api/task-follow-up')) return taskResponse('task-after-timeout'); + if (u.includes('/api2/task-stream')) return pendingSseResponse(); + if (u.includes('/api/task-detail')) return Response.json({ success: true, data: { task: {} } }); + return taskResponse('unexpected-create'); + }); + + let settled = false; + const closeP = be.destroySession().then(result => { settled = true; return result; }); + await new Promise(resolve => setTimeout(resolve, 25)); + expect(settled).toBe(false); + be.write('must remain fenced'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(0); + + resolveCancel(Response.json({ success: true, data: {} })); + const result = await closeP; + expect(result).toEqual({ ok: false, taskId: 'task-timeout-parent', error: 'close_timeout' }); + expect((be as any).closing).toBe(false); + + be.write('admitted after cancel settled'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(1); + }); + + it('cannot publish a prepared close after an abort races a late successful cancel', async () => { + const be = makeBackend({ resumeParentTaskId: 'task-abort-race', injectStatusLines: false }); + let resolveCancel!: (response: Response) => void; + fetchMock.mockImplementation(async (url: string | URL, init?: RequestInit) => { + const u = String(url); + calls.push({ url: u, init }); + if (u.includes('/api/task-cancel')) { + return new Promise(resolve => { resolveCancel = resolve; }); + } + if (u.includes('/api/task-follow-up')) return taskResponse('task-after-abort-race'); + if (u.includes('/api2/task-stream')) return pendingSseResponse(); + if (u.includes('/api/task-detail')) return Response.json({ success: true, data: { task: {} } }); + return taskResponse('unexpected-create'); + }); + + let closeSettled = false; + const close = be.destroySession().then(result => { + closeSettled = true; + return result; + }); + await flush(); + expect(resolveCancel).toBeTypeOf('function'); + + let abortSettled = false; + const abort = be.abortDestroySession().then(() => { abortSettled = true; }); + await flush(); + expect(closeSettled).toBe(false); + expect(abortSettled).toBe(false); + expect((be as any).closing).toBe(true); + + resolveCancel(Response.json({ success: true, data: {} })); + await expect(close).resolves.toEqual({ + ok: false, + taskId: 'task-abort-race', + error: 'close_aborted', + }); + await abort; + expect((be as any).closePrepared).toBe(false); + expect((be as any).closing).toBe(false); + + be.write('admitted after exact abort'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up'))).toHaveLength(1); }); }); diff --git a/test/riff-explicit-close.test.ts b/test/riff-explicit-close.test.ts new file mode 100644 index 000000000..b65957b4d --- /dev/null +++ b/test/riff-explicit-close.test.ts @@ -0,0 +1,283 @@ +import { EventEmitter } from 'node:events'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { activeSessionKey, type DaemonSession } from '../src/core/types.js'; + +const { getBotMock, cancelRiffTaskMock } = vi.hoisted(() => ({ + getBotMock: vi.fn(), + cancelRiffTaskMock: vi.fn(async () => true), +})); + +vi.mock('../src/bot-registry.js', () => ({ + getBot: getBotMock, + getBotBrand: vi.fn(() => 'feishu'), + getAllBots: vi.fn(() => []), + loadBotConfigs: vi.fn(), + resolveBrandLabel: vi.fn(() => undefined), +})); + +vi.mock('../src/adapters/backend/riff-backend.js', () => ({ + hashUrlForLog: vi.fn(() => 'riffhash'), + cancelRiffTaskById: cancelRiffTaskMock, +})); + +vi.mock('../src/im/lark/client.js', () => ({ + updateMessage: vi.fn(), + deleteMessage: vi.fn(), + sendEphemeralCard: vi.fn(), + sendUserMessage: vi.fn(), + addReaction: vi.fn(), + removeReaction: vi.fn(), + getMessageChatId: vi.fn(), + MessageWithdrawnError: class extends Error {}, +})); + +vi.mock('../src/services/frozen-card-store.js', () => ({ + loadFrozenCards: vi.fn(() => new Map()), + saveFrozenCards: vi.fn(), + deleteFrozenCards: vi.fn(), +})); + +vi.mock('../src/utils/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn() }, +})); + +import { config } from '../src/config.js'; +import { + __testOnly_setupWorkerHandlers, + closeSession, + initWorkerPool, + killWorker, + setActiveSessionsRegistry, +} from '../src/core/worker-pool.js'; +import * as sessionStore from '../src/services/session-store.js'; + +let dataDir: string; +let previousDataDir: string; + +function createFixture(options: { + liveWorker?: boolean; + closeOk?: boolean; + resultTaskId?: string; +} = {}) { + sessionStore.init('app'); + const session = sessionStore.createSession('oc_riff', 'om_riff', 'riff close', 'group'); + session.larkAppId = 'app'; + session.scope = 'chat'; + session.backendType = 'riff'; + session.riffParentTaskId = 'task-riff-123'; + sessionStore.updateSession(session); + + const worker = options.liveWorker ? new EventEmitter() as any : null; + if (worker) { + worker.killed = false; + worker.exitCode = null; + worker.signalCode = null; + worker.kill = vi.fn(); + worker.send = vi.fn((message: any) => { + if (message.type === 'close_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'close_abort_result', + requestId: message.requestId, + ok: true, + })); + return; + } + if (message.type !== 'close' || !message.requestId) return; + queueMicrotask(() => worker.emit('message', { + type: 'close_result', + requestId: message.requestId, + ok: options.closeOk ?? true, + taskId: options.resultTaskId ?? 'task-riff-123', + ...((options.closeOk ?? true) ? {} : { error: 'task-cancel HTTP 500' }), + })); + }); + } + + const ds = { + larkAppId: 'app', + chatId: session.chatId, + chatType: 'group', + scope: 'chat', + worker, + session, + initConfig: { backendType: 'riff' }, + } as unknown as DaemonSession; + if (worker) __testOnly_setupWorkerHandlers(ds, worker); + const registry = new Map([[activeSessionKey(ds), ds]]); + setActiveSessionsRegistry(registry); + return { session, ds, worker, registry }; +} + +beforeEach(() => { + vi.clearAllMocks(); + dataDir = mkdtempSync(join(tmpdir(), 'botmux-riff-close-')); + previousDataDir = config.session.dataDir; + config.session.dataDir = dataDir; + getBotMock.mockReturnValue({ + resolvedAllowedUsers: [], + config: { riff: { baseUrl: 'https://riff.invalid', jwt: 'test' } }, + }); + cancelRiffTaskMock.mockResolvedValue(true); + initWorkerPool({ + sessionReply: vi.fn(async () => 'om_reply'), + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + setActiveSessionsRegistry(new Map()); + config.session.dataDir = previousDataDir; + sessionStore.init(); + rmSync(dataDir, { recursive: true, force: true }); +}); + +describe('Riff explicit close', () => { + it('awaits worker-less cancellation before atomically clearing lineage and closing', async () => { + const fixture = createFixture(); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: true, + alreadyClosed: false, + }); + expect(cancelRiffTaskMock).toHaveBeenCalledWith( + expect.objectContaining({ baseUrl: 'https://riff.invalid' }), + 'task-riff-123', + ); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ status: 'closed' }); + expect(sessionStore.getSession(fixture.session.sessionId)?.riffParentTaskId).toBeUndefined(); + expect(fixture.registry.size).toBe(0); + }); + + it('preserves the active row, route and retry lineage when cancellation fails', async () => { + cancelRiffTaskMock.mockResolvedValue(false); + const fixture = createFixture(); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: false, + alreadyClosed: false, + error: 'riff_cancel_failed', + retryable: true, + taskId: 'task-riff-123', + }); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + expect(fixture.registry.get(activeSessionKey(fixture.ds))).toBe(fixture.ds); + }); + + it('commits a live worker close only after its matching prepare result', async () => { + const fixture = createFixture({ liveWorker: true, resultTaskId: 'task-riff-child' }); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: true, + alreadyClosed: false, + }); + expect(fixture.worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'close', + requestId: expect.any(String), + })); + expect(fixture.worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'close_commit', + requestId: expect.any(String), + })); + expect(cancelRiffTaskMock).not.toHaveBeenCalled(); + expect(sessionStore.getSession(fixture.session.sessionId)?.riffParentTaskId).toBeUndefined(); + }); + + it('aborts a failed live prepare and keeps the session retryable', async () => { + const fixture = createFixture({ liveWorker: true, closeOk: false }); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: false, + alreadyClosed: false, + error: 'riff_worker_close_failed', + retryable: true, + taskId: 'task-riff-123', + }); + expect(fixture.worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'close_abort', + requestId: expect.any(String), + })); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + expect(fixture.ds.riffCloseState).toBeUndefined(); + }); + + it('refuses an unprepared generic retirement of a live Riff worker', () => { + const fixture = createFixture({ liveWorker: true }); + + killWorker(fixture.ds); + + expect(fixture.ds.worker).toBe(fixture.worker); + expect(fixture.worker.send).not.toHaveBeenCalled(); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + }); + + it('refuses explicit close while the daemon shutdown fence owns the Riff worker', async () => { + const fixture = createFixture({ liveWorker: true }); + fixture.ds.riffShutdownState = { + phase: 'preparing', + requestId: 'shutdown-riff', + taskId: 'task-riff-123', + }; + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: false, + alreadyClosed: false, + error: 'riff_shutdown_fence_in_progress', + retryable: true, + taskId: 'task-riff-123', + }); + expect(fixture.worker.send).not.toHaveBeenCalled(); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + }); + + it('keeps the newest runtime lineage when its durable save fails', async () => { + const fixture = createFixture({ liveWorker: true }); + vi.spyOn(sessionStore, 'updateSession').mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + fixture.worker.emit('message', { + type: 'riff_task_id', + taskId: 'task-riff-child', + }); + await new Promise(resolve => setImmediate(resolve)); + + expect(fixture.session.riffParentTaskId).toBe('task-riff-child'); + const durable = JSON.parse(readFileSync(join(dataDir, 'sessions-app.json'), 'utf8')); + expect(durable[fixture.session.sessionId].riffParentTaskId).toBe('task-riff-123'); + }); + + it('restores the prior runtime lineage when clearing its durable row fails', async () => { + const fixture = createFixture({ liveWorker: true }); + vi.spyOn(sessionStore, 'updateSession').mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + fixture.worker.emit('message', { + type: 'riff_task_id', + taskId: null, + }); + await new Promise(resolve => setImmediate(resolve)); + + expect(fixture.session.riffParentTaskId).toBe('task-riff-123'); + const durable = JSON.parse(readFileSync(join(dataDir, 'sessions-app.json'), 'utf8')); + expect(durable[fixture.session.sessionId].riffParentTaskId).toBe('task-riff-123'); + }); +}); diff --git a/test/riff-shutdown-detach.test.ts b/test/riff-shutdown-detach.test.ts new file mode 100644 index 000000000..d31c37627 --- /dev/null +++ b/test/riff-shutdown-detach.test.ts @@ -0,0 +1,1093 @@ +import { EventEmitter } from 'node:events'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { config } from '../src/config.js'; +import type { DaemonSession } from '../src/core/types.js'; +import { + abortRiffShutdownFleet, + abortPreparedRiffShutdown, + canAbortVerifiedExitedRiffPreparation, + collectUniqueDaemonShutdownSessions, + commitPreparedRiffShutdown, + detachRiffWorkerForShutdown, + persistPreparedRiffShutdown, + persistPreparedRiffShutdownFleet, + prepareRiffFleetForShutdown, + prepareRiffSessionForShutdown, + type FencedRiffShutdownParticipant, + type PreparedRiffShutdown, +} from '../src/core/riff-shutdown-detach.js'; +import { sendWorkerInput } from '../src/core/worker-pool.js'; +import * as sessionStore from '../src/services/session-store.js'; + +vi.mock('../src/utils/logger.js', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +type FakeWorker = EventEmitter & { + killed: boolean; + exitCode: number | null; + signalCode: NodeJS.Signals | null; + send: ReturnType; + kill: ReturnType; +}; + +describe('Riff graceful daemon-shutdown detach coordinator', () => { + let dataDir: string; + let previousDataDir: string; + + beforeEach(() => { + dataDir = mkdtempSync(join(tmpdir(), 'botmux-riff-shutdown-')); + previousDataDir = config.session.dataDir; + config.session.dataDir = dataDir; + sessionStore.init('app'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + config.session.dataDir = previousDataDir; + sessionStore.init(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + function fixture( + initialTaskId: string | undefined, + onSend: (worker: FakeWorker, message: any) => void, + ): { ds: DaemonSession; worker: FakeWorker; messages: any[] } { + const session = sessionStore.createSession('oc_riff', 'om_riff', 'riff shutdown', 'group'); + session.larkAppId = 'app'; + session.backendType = 'riff'; + session.riffParentTaskId = initialTaskId; + sessionStore.updateSession(session); + const messages: any[] = []; + const worker = new EventEmitter() as FakeWorker; + worker.killed = false; + worker.exitCode = null; + worker.signalCode = null; + worker.kill = vi.fn(); + worker.send = vi.fn((message: any) => { + messages.push(message); + onSend(worker, message); + }); + const ds = { + larkAppId: 'app', + chatId: session.chatId, + chatType: 'group', + scope: 'chat', + session, + worker, + workerPort: 4100, + workerToken: 'write', + workerViewToken: 'view', + managedTurnOrigin: { capability: 'cap' }, + initConfig: { backendType: 'riff' }, + } as unknown as DaemonSession; + return { ds, worker, messages }; + } + + function asPrepared( + result: Awaited>, + ): PreparedRiffShutdown { + if (!result.ok) throw new Error(`expected prepared Riff shutdown: ${result.error}`); + return result; + } + + function asFenced( + result: Awaited>, + ): FencedRiffShutdownParticipant { + if (result.fence === 'none') throw new Error(`expected fenced Riff shutdown: ${result.error}`); + return result; + } + + it('transactional lineage persistence rolls back its in-memory field when save throws', () => { + const session = sessionStore.createSession('oc_txn', 'om_txn', 'txn', 'group'); + session.backendType = 'riff'; + session.riffParentTaskId = 'task-before'; + sessionStore.updateSession(session); + const blockedDataDir = join(dataDir, 'not-a-directory'); + writeFileSync(blockedDataDir, 'block'); + config.session.dataDir = blockedDataDir; + + expect(() => sessionStore.persistActiveRiffLineageExact(session.sessionId, 'task-after')).toThrow(); + expect(session.riffParentTaskId).toBe('task-before'); + + config.session.dataDir = dataDir; + expect(sessionStore.getSessionFresh(session.sessionId)?.riffParentTaskId).toBe('task-before'); + }); + + it('deduplicates multiple registry aliases to the exact same daemon session object', () => { + const f = fixture('task-parent', () => { /* no IPC */ }); + + expect(collectUniqueDaemonShutdownSessions([f.ds, f.ds, f.ds])).toEqual({ + ok: true, + sessions: [f.ds], + }); + }); + + it('fails closed when distinct daemon session objects claim the same session id', () => { + const f = fixture('task-parent', () => { /* no IPC */ }); + const competing = { + ...f.ds, + session: { ...f.ds.session }, + } as DaemonSession; + + expect(collectUniqueDaemonShutdownSessions([f.ds, competing])).toMatchObject({ + ok: false, + sessionId: f.ds.session.sessionId, + error: expect.stringContaining('distinct daemon session generations'), + }); + }); + + it('retains only the ambiguous session fence and restores an unrelated prepared peer', async () => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } else if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'abort', ok: true, taskId: 'task-second-child', + })); + } + }); + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds]); + const competingFirst = { + ...first.ds, + session: { ...first.ds.session }, + } as DaemonSession; + const current = collectUniqueDaemonShutdownSessions([ + first.ds, + competingFirst, + second.ds, + ]); + if (current.ok) throw new Error('expected ambiguous daemon session id'); + + const restored = await abortRiffShutdownFleet(prepared + .filter(({ ds }) => ds.session.sessionId !== current.sessionId) + .map(({ ds, result }) => ({ ds, result: asFenced(result) }))); + + expect(restored).toHaveLength(1); + expect(restored[0].result.ok).toBe(true); + expect(first.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(first.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(second.ds.riffShutdownState).toBeUndefined(); + expect(second.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + }); + + it('refuses an initial durable-read failure before installing any worker fence', async () => { + vi.spyOn(sessionStore, 'getActiveRiffShutdownSnapshotsBatch').mockImplementation(() => { + throw new Error('lock unavailable'); + }); + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + + await expect(prepareRiffSessionForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-parent', + error: 'durable_session_read_failed:lock unavailable', + }); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.ds.worker).toBe(f.worker); + expect(f.messages).toEqual([]); + }); + + it('takes one all-owner snapshot before publishing any fleet prepare request', async () => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } + }); + first.ds.session.pid = 101; + second.ds.session.pid = 202; + sessionStore.updateSession(first.ds.session); + sessionStore.updateSession(second.ds.session); + const originalSnapshot = sessionStore.getActiveRiffShutdownSnapshotsBatch; + const snapshot = vi.spyOn(sessionStore, 'getActiveRiffShutdownSnapshotsBatch') + .mockImplementation((sessionIds, options) => { + expect(first.messages).toEqual([]); + expect(second.messages).toEqual([]); + return originalSnapshot(sessionIds, options); + }); + + const results = await prepareRiffFleetForShutdown([first.ds, second.ds]); + + expect(snapshot).toHaveBeenCalledTimes(1); + expect(snapshot).toHaveBeenCalledWith([ + first.ds.session.sessionId, + second.ds.session.sessionId, + ], expect.any(Object)); + expect(results.every(entry => entry.result.ok)).toBe(true); + expect(first.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(second.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it('does not inline-abort a timed-out prepare and restores all owners in one concurrent wave', async () => { + let firstAbortRequestId: string | undefined; + let secondAbortRequestId: string | undefined; + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } else if (message.type === 'riff_shutdown_abort') { + firstAbortRequestId = message.requestId; + } + }); + const second = fixture('task-second', (_worker, message) => { + if (message.type === 'riff_shutdown_abort') secondAbortRequestId = message.requestId; + // Deliberately never ACK prepare so its fence state is ambiguous. + }); + first.ds.session.pid = 101; + second.ds.session.pid = 202; + sessionStore.updateSession(first.ds.session); + sessionStore.updateSession(second.ds.session); + + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds], { + drainTimeoutMs: 5, + abortTimeoutMs: 200, + }); + expect(prepared[0].result).toMatchObject({ ok: true, fence: 'prepared' }); + expect(prepared[1].result).toMatchObject({ ok: false, fence: 'possible' }); + expect(first.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(second.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + + const aborting = abortRiffShutdownFleet(prepared.map(({ ds, result }) => ({ + ds, + result: asFenced(result), + })), { abortTimeoutMs: 200 }); + await vi.waitFor(() => { + expect(firstAbortRequestId).toEqual(expect.any(String)); + expect(secondAbortRequestId).toEqual(expect.any(String)); + }); + expect(first.messages.filter(message => message.type === 'riff_shutdown_abort')).toHaveLength(1); + expect(second.messages.filter(message => message.type === 'riff_shutdown_abort')).toHaveLength(1); + + first.worker.emit('message', { + type: 'riff_shutdown_result', requestId: firstAbortRequestId, + phase: 'abort', ok: true, taskId: 'task-first-child', + }); + second.worker.emit('message', { + type: 'riff_shutdown_result', requestId: secondAbortRequestId, + phase: 'abort', ok: true, taskId: 'task-second', + }); + await expect(aborting).resolves.toSatisfy( + (results: Array<{ result: { ok: boolean } }>) => results.every(entry => entry.result.ok), + ); + expect(first.ds.riffShutdownState).toBeUndefined(); + expect(second.ds.riffShutdownState).toBeUndefined(); + + // A prepare ACK arriving after the exact abort wave is inert. + second.worker.emit('message', { + type: 'riff_shutdown_result', requestId: secondAbortRequestId, + phase: 'prepare', ok: true, taskId: 'task-late-ack', + }); + expect(second.ds.riffShutdownState).toBeUndefined(); + expect(second.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + }); + + it.each(['explicit_close_in_progress', 'not_riff_backend'])( + 'classifies exact pre-fence worker refusal %s without sending abort', + async (error) => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: false, taskId: null, error, + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + fence: 'none', + error, + }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.ds.worker).toBe(f.worker); + }, + ); + + it('clears the synthetic daemon fence when prepare IPC throws before it can be queued', async () => { + const f = fixture('task-parent', () => { /* send is replaced below */ }); + f.worker.send.mockImplementation(() => { throw new Error('IPC channel closed'); }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + fence: 'none', + error: 'riff_shutdown_prepare_send_failed', + }); + expect(f.messages).toEqual([]); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('does not misclassify an existing older shutdown fence as a pre-fence refusal', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: false, taskId: null, + error: 'shutdown detach already prepared as older-request', + })); + } + }); + + await expect(prepareRiffSessionForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + fence: 'possible', + error: 'shutdown detach already prepared as older-request', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'preparing' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it('refuses before a worker fence unless phase-2 plus the abort reserve remain', async () => { + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + const now = 10_000; + + await expect(prepareRiffFleetForShutdown([f.ds], { + deadlineMs: now + 1_500, + abortTimeoutMs: 1_000, + now: () => now, + })).resolves.toMatchObject([{ + result: { + ok: false, + fence: 'none', + error: 'insufficient_abort_budget_before_fence', + }, + }]); + expect(f.messages).toEqual([]); + expect(f.ds.riffShutdownState).toBeUndefined(); + }); + + it('persists all prepared owners through one phase-2 batch call', async () => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } + }); + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds]); + const entries = prepared.map(({ ds, result }) => ({ ds, result: asPrepared(result) })); + const batch = vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch'); + + expect(persistPreparedRiffShutdownFleet(entries)).toEqual({ ok: true }); + expect(batch).toHaveBeenCalledTimes(1); + expect(sessionStore.getSessionFresh(first.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-first-child'); + expect(sessionStore.getSessionFresh(second.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-second-child'); + expect(entries.every(({ result }) => result.lineageVerified)).toBe(true); + }); + + it('retains every fence when verified phase-2 persistence returns after the absolute deadline', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + const originalBatch = sessionStore.persistActiveRiffLineagesExactBatch; + let now = 10_000; + const deadlineMs = 10_100; + vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch') + .mockImplementation((updates, options) => { + originalBatch(updates, options); + now = deadlineMs; + }); + + const result = persistPreparedRiffShutdownFleet( + [{ ds: f.ds, result: prepared }], + { deadlineMs, now: () => now }, + ); + + expect(result).toMatchObject({ + ok: false, + error: 'shutdown_deadline_elapsed_after_batch_persist', + rollbackDisposition: 'retain_fence', + sessionIds: [f.ds.session.sessionId], + retainFencedSessionIds: [f.ds.session.sessionId], + }); + expect(prepared.lineageVerified).toBe(true); + expect(f.ds.session.riffParentTaskId).toBe('task-child'); + expect(f.ds.riffShutdownState).toMatchObject({ requestId: prepared.requestId }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it.each([ + ['prewrite_ownership', 'affected'] as const, + ['prewrite_io', 'none'] as const, + ['postrename_ambiguity', 'all'] as const, + ])('maps %s batch failure to the exact retain-fence set', async (stage, expected) => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } + }); + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds]); + const entries = prepared.map(({ ds, result }) => ({ ds, result: asPrepared(result) })); + const affected = stage === 'postrename_ambiguity' + ? [first.ds.session.sessionId, second.ds.session.sessionId] + : [second.ds.session.sessionId]; + vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch').mockImplementation(() => { + throw new sessionStore.RiffLineageBatchError(stage, affected, `forced ${stage}`); + }); + + const result = persistPreparedRiffShutdownFleet(entries); + expect(result).toMatchObject({ ok: false }); + if (result.ok) throw new Error('expected batch failure'); + expect(result.retainFencedSessionIds).toEqual( + expected === 'all' + ? [first.ds.session.sessionId, second.ds.session.sessionId] + : expected === 'affected' + ? [second.ds.session.sessionId] + : [], + ); + }); + + it('retains the exact fence without writing when runtime owner changes after prepare', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + f.ds.session.pid = 999_999; + const batch = vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch'); + + const result = persistPreparedRiffShutdownFleet([{ ds: f.ds, result: prepared }]); + + expect(result).toMatchObject({ + ok: false, + retainFencedSessionIds: [f.ds.session.sessionId], + error: expect.stringContaining('runtime_owner_changed'), + }); + expect(batch).not.toHaveBeenCalled(); + expect(f.ds.riffShutdownState).toMatchObject({ requestId: prepared.requestId }); + }); + + it('persists and fresh-verifies the exact late child before commit', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: 'task-late-child', + })); + } + }); + + const result = await detachRiffWorkerForShutdown(f.ds); + expect(result).toMatchObject({ + ok: true, + taskId: 'task-late-child', + disposition: 'lineage_persisted', + }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_commit', + ]); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-late-child', + }); + expect(f.ds.worker).toBeNull(); + expect(f.worker.kill).not.toHaveBeenCalled(); + }); + + it('treats null as authoritative and clears a stale durable parent before commit', async () => { + const f = fixture('task-stale-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: null, + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: true, + taskId: null, + disposition: 'lineage_persisted', + }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId).toBeUndefined(); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_commit', + ]); + }); + + it('restores the prepared worker without cancellation when durable persistence fails', async () => { + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation(() => { + throw new Error('disk full'); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: 'task-exact-child', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'abort', + ok: true, + taskId: 'task-exact-child', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-exact-child', + error: expect.stringContaining('lineage_persist_failed:disk full'), + }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + // Persistence never advanced, so the accepted remote task and its exact + // worker generation remain live with admission restored. + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId).toBe('task-parent'); + expect(f.ds.worker).toBe(f.worker); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(f.messages.some(message => message.type === 'riff_shutdown_cancel')).toBe(false); + }); + + it('fails closed without commit or signal when persistence fails and abort is not ACKed', async () => { + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation(() => { + throw new Error('disk full'); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: 'task-uncancellable', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'abort', + ok: false, + taskId: 'task-uncancellable', + error: 'admission restore refused', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-uncancellable', + error: expect.stringContaining('admission_restore_failed:admission restore refused'), + }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + expect(f.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + }); + + it('keeps the fence when an abort ACK reports a different task lineage', async () => { + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation(() => { + throw new Error('disk full'); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-prepared-child', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'abort', ok: true, taskId: 'task-unexpected-child', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('abort_task_lineage_mismatch'), + }); + expect(f.ds.worker).toBe(f.worker); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + }); + + it('prepares and persists workerless runtime lineage before allowing commit', async () => { + const f = fixture('task-durable-parent', () => { /* workerless: no IPC */ }); + f.ds.worker = null; + f.ds.workerPort = null; + f.ds.workerToken = null; + f.ds.workerViewToken = null; + // Simulate a prior ordinary riff_task_id save failure: runtime owns the + // child while the durable row still names its parent. + f.ds.session.riffParentTaskId = 'task-runtime-child'; + + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + expect(prepared.worker).toBeNull(); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-durable-parent'); + + expect(persistPreparedRiffShutdown(f.ds, prepared)).toEqual({ ok: true }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-runtime-child'); + expect(commitPreparedRiffShutdown(f.ds, prepared)).toBe(true); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.messages).toEqual([]); + }); + + it('aborts every prepared peer and commits none when one workerless lineage write fails', async () => { + const live = fixture('task-live-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-live-child', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'abort', ok: true, taskId: 'task-live-child', + })); + } + }); + const workerless = fixture('task-workerless-parent', () => { /* no IPC */ }); + workerless.ds.worker = null; + workerless.ds.session.riffParentTaskId = 'task-workerless-runtime-child'; + + const [livePrepared, workerlessPrepared] = await Promise.all([ + prepareRiffSessionForShutdown(live.ds), + prepareRiffSessionForShutdown(workerless.ds), + ]).then(results => results.map(asPrepared)); + const originalPersist = sessionStore.persistActiveRiffLineageExact; + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation( + (sessionId, taskId) => { + if (sessionId === workerless.ds.session.sessionId) throw new Error('target disk full'); + return originalPersist(sessionId, taskId); + }, + ); + + const persistence = [ + persistPreparedRiffShutdown(live.ds, livePrepared!), + persistPreparedRiffShutdown(workerless.ds, workerlessPrepared!), + ]; + expect(persistence[0]).toEqual({ ok: true }); + expect(persistence[1]).toMatchObject({ + ok: false, + error: expect.stringContaining('target disk full'), + }); + + await Promise.all([ + abortPreparedRiffShutdown(live.ds, livePrepared!), + abortPreparedRiffShutdown(workerless.ds, workerlessPrepared!), + ]); + expect(live.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + expect(live.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(live.ds.worker).toBe(live.worker); + expect(live.ds.riffShutdownState).toBeUndefined(); + expect(workerless.ds.riffShutdownState).toBeUndefined(); + expect(sessionStore.getSessionFresh(workerless.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-workerless-parent'); + }); + + it('retains a fail-closed fence when an unverified prepared worker exits before abort', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-unverified-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + f.worker.exitCode = 1; + // worker-pool clears the exact dead child handle but retains this request's + // shutdown fence for the coordinator. + f.ds.worker = null; + + expect(persistPreparedRiffShutdown(f.ds, prepared)).toMatchObject({ + ok: false, + error: 'stale_worker_generation', + }); + await expect(abortPreparedRiffShutdown(f.ds, prepared)).resolves.toMatchObject({ + ok: false, + error: 'worker_exited_before_admission_restore', + }); + expect(f.ds.worker).toBeNull(); + expect(f.ds.riffShutdownState).toMatchObject({ + phase: 'prepared', + requestId: prepared.requestId, + }); + expect(sendWorkerInput(f.ds, 'must stay fenced', 'turn-exited-unverified')).toBe(false); + }); + + it('classifies an unexpected durable lineage as ownership loss and never overwrites it', async () => { + const f = fixture('task-durable-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => { + // Simulate a different process advancing the durable owner after + // prepare, without changing this daemon's runtime generation. + const sessionsPath = join(dataDir, 'sessions-app.json'); + const projection = JSON.parse(readFileSync(sessionsPath, 'utf8')) as Record; + projection[f.ds.session.sessionId]!.riffParentTaskId = 'task-external-owner'; + writeFileSync(sessionsPath, JSON.stringify(projection, null, 2)); + worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-prepared-child', + }); + }); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('compare-and-set failed'), + rollbackDisposition: 'retain_fence', + }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-external-owner'); + expect(f.ds.riffShutdownState).toMatchObject({ + phase: 'prepared', + }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it('retains the fence when durable worker ownership changes with the same lineage', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => { + const sessionsPath = join(dataDir, 'sessions-app.json'); + const projection = JSON.parse(readFileSync(sessionsPath, 'utf8')) as Record; + projection[f.ds.session.sessionId]!.pid = 999_999; + writeFileSync(sessionsPath, JSON.stringify(projection, null, 2)); + worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-prepared-child', + }); + }); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('owner compare-and-set failed'), + rollbackDisposition: 'retain_fence', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)).toMatchObject({ + pid: 999_999, + riffParentTaskId: 'task-parent', + }); + }); + + it('retains the fence when ownership is replaced after CAS with the same task id', async () => { + const originalPersist = sessionStore.persistActiveRiffLineageExact; + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation( + (sessionId, taskId, options) => { + const result = originalPersist(sessionId, taskId, options); + // Exact replacement race: CAS wrote the expected task, then a new + // durable owner published the same task before the separate readback. + const sessionsPath = join(dataDir, 'sessions-app.json'); + const projection = JSON.parse(readFileSync(sessionsPath, 'utf8')) as Record; + projection[sessionId]!.pid = 888_888; + writeFileSync(sessionsPath, JSON.stringify(projection, null, 2)); + return result; + }, + ); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-same-after-replacement', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-same-after-replacement', + error: expect.stringContaining('fresh_lineage_verification_failed:'), + rollbackDisposition: 'retain_fence', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)).toMatchObject({ + pid: 888_888, + riffParentTaskId: 'task-same-after-replacement', + }); + }); + + it('retains the fence when the post-write fresh verification cannot be read', async () => { + const originalFresh = sessionStore.getSessionFresh; + let reads = 0; + vi.spyOn(sessionStore, 'getSessionFresh').mockImplementation(sessionId => { + reads++; + if (reads === 1) throw new Error('lock unavailable'); + return originalFresh(sessionId); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-written-not-verified', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: 'fresh_lineage_verification_failed:lock unavailable', + rollbackDisposition: 'retain_fence', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(originalFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-written-not-verified'); + }); + + it('can release an exited prepared generation only after its lineage was durably verified', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-verified-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + expect(persistPreparedRiffShutdown(f.ds, prepared)).toEqual({ ok: true }); + f.worker.exitCode = 0; + f.ds.worker = null; + + expect(canAbortVerifiedExitedRiffPreparation(f.ds, prepared)).toBe(true); + + await expect(abortPreparedRiffShutdown(f.ds, prepared)).resolves.toEqual({ + ok: true, + taskId: 'task-verified-child', + }); + expect(f.ds.worker).toBeNull(); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-verified-child'); + }); + + it('does not bless a replacement generation when the verified prepared worker exits', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-verified-before-replacement', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + expect(persistPreparedRiffShutdown(f.ds, prepared)).toEqual({ ok: true }); + f.worker.exitCode = 0; + const replacement = new EventEmitter() as FakeWorker; + replacement.killed = false; + replacement.exitCode = null; + replacement.signalCode = null; + replacement.send = vi.fn(); + replacement.kill = vi.fn(); + f.ds.worker = replacement; + + expect(canAbortVerifiedExitedRiffPreparation(f.ds, prepared)).toBe(false); + + await expect(abortPreparedRiffShutdown(f.ds, prepared)).resolves.toMatchObject({ + ok: false, + taskId: 'task-verified-before-replacement', + error: 'new_worker_generation', + }); + expect(f.ds.worker).toBe(replacement); + expect(f.ds.riffShutdownState).toMatchObject({ + phase: 'prepared', + requestId: prepared.requestId, + }); + expect(replacement.send).not.toHaveBeenCalled(); + }); + + it('times out boundedly and restores the worker instead of falling through to generic kill', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'abort', + ok: true, + taskId: 'task-parent', + })); + } + }); + const result = await detachRiffWorkerForShutdown(f.ds, { + drainTimeoutMs: 10, + abortTimeoutMs: 100, + }); + + expect(result).toMatchObject({ ok: false, error: 'riff_shutdown_prepare_timeout' }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('keeps daemon admission fenced until delayed abort restoration is ACKed', async () => { + let abortRequestId: string | undefined; + const f = fixture('task-parent', (_worker, message) => { + if (message.type === 'riff_shutdown_abort') abortRequestId = message.requestId; + }); + const detach = detachRiffWorkerForShutdown(f.ds, { + drainTimeoutMs: 5, + abortTimeoutMs: 200, + }); + await vi.waitFor(() => expect(abortRequestId).toEqual(expect.any(String))); + + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'preparing' }); + expect(sendWorkerInput(f.ds, 'must remain fenced', 'turn-during-abort')).toBe(false); + expect(f.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(f.worker.kill).not.toHaveBeenCalled(); + + f.worker.emit('message', { + type: 'riff_shutdown_result', + requestId: abortRequestId, + phase: 'abort', + ok: true, + taskId: 'task-parent', + }); + await expect(detach).resolves.toMatchObject({ + ok: false, + error: 'riff_shutdown_prepare_timeout', + }); + expect(f.ds.riffShutdownState).toBeUndefined(); + }); + + it('refuses before worker prepare when daemon-owned raw/follow-up input is pending', async () => { + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + f.ds.pendingRawInput = '/goal keep this'; + f.ds.pendingFollowUpInput = { + userPrompt: 'follow-up', + cliInput: 'follow-up', + }; + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('daemon_inputs_not_drained:'), + }); + expect(f.messages).toEqual([]); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('refuses before worker prepare while a durable queued backlog is pending', async () => { + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + f.ds.session.queued = true; + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('queued=1'), + }); + expect(f.messages).toEqual([]); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('retains the fail-closed fence when abort restoration cannot be confirmed', async () => { + const f = fixture('task-parent', () => { /* prepare and abort both held */ }); + const result = await detachRiffWorkerForShutdown(f.ds, { + drainTimeoutMs: 5, + abortTimeoutMs: 5, + }); + + expect(result).toMatchObject({ + ok: false, + error: expect.stringContaining('admission_restore_failed:riff_shutdown_abort_timeout'), + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'preparing' }); + expect(sendWorkerInput(f.ds, 'still blocked', 'turn-fail-closed')).toBe(false); + expect(f.worker.kill).not.toHaveBeenCalled(); + }); +}); diff --git a/test/riff-worker-shutdown-readiness.test.ts b/test/riff-worker-shutdown-readiness.test.ts new file mode 100644 index 000000000..df3c5fda7 --- /dev/null +++ b/test/riff-worker-shutdown-readiness.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { riffWorkerShutdownInputBlocker } from '../src/core/riff-worker-shutdown-readiness.js'; + +const idle = { + initPromptMaterialized: true, + isFlushing: false, + pendingMessages: 0, + pendingRawInputs: 0, + pendingSessionRename: false, + sessionRenameInFlight: false, + commandLineWritesPending: 0, +}; + +describe('Riff worker shutdown input ownership', () => { + it('allows the current backend task itself when no unsent worker input exists', () => { + expect(riffWorkerShutdownInputBlocker(idle)).toBeNull(); + }); + + it('refuses current-task detach when a follow-up is still in pendingMessages', () => { + expect(riffWorkerShutdownInputBlocker({ ...idle, pendingMessages: 1 })) + .toBe('messages=1'); + }); + + it('accounts for every worker-owned pre-backend input surface', () => { + expect(riffWorkerShutdownInputBlocker({ + initPromptMaterialized: false, + isFlushing: true, + pendingMessages: 2, + pendingRawInputs: 1, + pendingSessionRename: true, + sessionRenameInFlight: true, + commandLineWritesPending: 1, + })).toBe( + 'init=materializing,flushing=1,messages=2,raw=1,rename=1,rename_inflight=1,command_writes=1', + ); + }); +}); diff --git a/test/session-store.test.ts b/test/session-store.test.ts index 7315ec950..20aac00b4 100644 --- a/test/session-store.test.ts +++ b/test/session-store.test.ts @@ -361,6 +361,44 @@ describe('closeSession()', () => { expect(reloaded!.closedAt).toBeDefined(); }); + it('clears Riff lineage atomically with the durable closed row', () => { + const session = createSession('chat1', 'root1', 'Close Riff'); + session.backendType = 'riff'; + session.riffParentTaskId = 'riff-task-prepared'; + updateSession(session); + + closeSession(session.sessionId, { clearRiffParentTaskId: true }); + init(); + + expect(getSession(session.sessionId)).toMatchObject({ status: 'closed' }); + expect(getSession(session.sessionId)?.riffParentTaskId).toBeUndefined(); + }); + + it('restores Riff close state in memory when the atomic save fails', () => { + const session = createSession('chat1', 'root1', 'Close Riff Save Failure'); + session.backendType = 'riff'; + session.riffParentTaskId = 'riff-task-retry'; + updateSession(session); + fsControl.failSessionWrite = true; + + expect(() => closeSession( + session.sessionId, + { clearRiffParentTaskId: true }, + )).toThrow(/simulated session repair write failure/); + expect(getSession(session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'riff-task-retry', + }); + expect(mockDeleteFrozenCards).not.toHaveBeenCalled(); + + fsControl.failSessionWrite = false; + init(); + expect(getSession(session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'riff-task-retry', + }); + }); + it('should call deleteFrozenCards with the sessionId', () => { const session = createSession('chat1', 'root1', 'Frozen'); closeSession(session.sessionId); diff --git a/test/worker-riff-retirement-protocol.test.ts b/test/worker-riff-retirement-protocol.test.ts new file mode 100644 index 000000000..671c2fede --- /dev/null +++ b/test/worker-riff-retirement-protocol.test.ts @@ -0,0 +1,134 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; + +const workerSource = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf8'); +const workerPoolSource = readFileSync(new URL('../src/core/worker-pool.ts', import.meta.url), 'utf8'); + +describe('worker Riff retirement protocol', () => { + it('refuses Riff generation restart before the local restart helper can run', () => { + const start = workerSource.indexOf("case 'restart':"); + const end = workerSource.indexOf("case 'expire_durable_turn':", start); + const restart = workerSource.slice(start, end); + + const riffGuard = restart.indexOf("if (effectiveBackendType === 'riff')"); + const refusal = restart.indexOf('Refused Riff generation restart', riffGuard); + const guardBreak = restart.indexOf('break;', refusal); + const replacement = restart.indexOf('await restartCliProcess(', guardBreak); + + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(riffGuard).toBeGreaterThanOrEqual(0); + expect(refusal).toBeGreaterThan(riffGuard); + expect(guardBreak).toBeGreaterThan(refusal); + expect(replacement).toBeGreaterThan(guardBreak); + }); + + it('refuses request-less Riff close before destroy or process exit', () => { + const start = workerSource.indexOf("case 'close':"); + const end = workerSource.indexOf("case 'close_commit':", start); + const close = workerSource.slice(start, end); + + const riffBranch = close.indexOf("if (effectiveBackendType === 'riff')"); + const requestlessGuard = close.indexOf('if (!msg.requestId)', riffBranch); + const refusal = close.indexOf('Refused unsafe request-less Riff close', requestlessGuard); + const guardBreak = close.indexOf('break;', refusal); + const localDestroy = close.lastIndexOf('backend?.destroySession?.()'); + const localExit = close.lastIndexOf('process.exit(0)'); + + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(riffBranch).toBeGreaterThanOrEqual(0); + expect(requestlessGuard).toBeGreaterThan(riffBranch); + expect(refusal).toBeGreaterThan(requestlessGuard); + expect(guardBreak).toBeGreaterThan(refusal); + expect(localDestroy).toBeGreaterThan(guardBreak); + expect(localExit).toBeGreaterThan(localDestroy); + }); + + it('refuses request-less Riff suspend before teardown or process exit', () => { + const start = workerSource.indexOf("case 'suspend':"); + const end = workerSource.indexOf('\n }\n});', start); + const suspend = workerSource.slice(start, end); + + const riffGuard = suspend.indexOf("if (effectiveBackendType === 'riff')"); + const refusal = suspend.indexOf('Refused unsafe Riff suspend', riffGuard); + const guardBreak = suspend.indexOf('break;', refusal); + const localDestroy = suspend.indexOf('(backend?.destroySession ?? backend?.kill)', guardBreak); + const localExit = suspend.indexOf('process.exit(0)', localDestroy); + + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(riffGuard).toBeGreaterThanOrEqual(0); + expect(refusal).toBeGreaterThan(riffGuard); + expect(guardBreak).toBeGreaterThan(refusal); + expect(localDestroy).toBeGreaterThan(guardBreak); + expect(localExit).toBeGreaterThan(localDestroy); + }); + + it('checks every unsent input buffer before fencing the backend or allowing commit', () => { + const prepareStart = workerSource.indexOf("case 'riff_shutdown_prepare':"); + const prepareEnd = workerSource.indexOf("case 'riff_shutdown_commit':", prepareStart); + const prepare = workerSource.slice(prepareStart, prepareEnd); + const readiness = prepare.indexOf('riffWorkerShutdownInputBlocker({'); + const queueCount = prepare.indexOf('pendingMessages: pendingMessages.length', readiness); + const rawCount = prepare.indexOf('pendingRawInputs: pendingRawInputs.length', readiness); + const initFence = prepare.indexOf('initPromptMaterialized', readiness); + const refusal = prepare.indexOf('worker_inputs_not_drained:', readiness); + const backendPrepare = prepare.indexOf('backend?.prepareShutdownDetach?.()', refusal); + + expect(readiness).toBeGreaterThanOrEqual(0); + expect(initFence).toBeGreaterThan(readiness); + expect(queueCount).toBeGreaterThan(readiness); + expect(rawCount).toBeGreaterThan(queueCount); + expect(refusal).toBeGreaterThan(rawCount); + expect(backendPrepare).toBeGreaterThan(refusal); + + const commitStart = workerSource.indexOf("case 'riff_shutdown_commit':", prepareEnd); + const commitEnd = workerSource.indexOf("case 'riff_shutdown_abort':", commitStart); + const commit = workerSource.slice(commitStart, commitEnd); + expect(commit).toContain("shutdownDetachPhase !== 'prepared'"); + expect(commit.indexOf("shutdownDetachPhase !== 'prepared'")) + .toBeLessThan(commit.indexOf('process.exit(0)')); + }); + + it('has no shutdown cancellation command that can discard accepted Riff work', () => { + expect(workerSource).not.toContain("case 'riff_shutdown_cancel':"); + expect(workerSource).not.toContain('cancelShutdownDetach'); + }); + + it('ACKs shutdown and explicit-close abort only after backend admission restoration', () => { + const shutdownStart = workerSource.indexOf("case 'riff_shutdown_abort':"); + const shutdownEnd = workerSource.indexOf("case 'close_commit':", shutdownStart); + const shutdown = workerSource.slice(shutdownStart, shutdownEnd); + const shutdownRestore = shutdown.indexOf('await backend?.abortShutdownDetach?.()'); + expect(shutdownRestore).toBeGreaterThanOrEqual(0); + expect(shutdown.indexOf("phase: 'abort'", shutdownRestore)) + .toBeGreaterThan(shutdownRestore); + + const closeStart = workerSource.indexOf("case 'close_abort':"); + const closeEnd = workerSource.indexOf("case 'suspend':", closeStart); + const close = workerSource.slice(closeStart, closeEnd); + const closeRestore = close.indexOf('await backend?.abortDestroySession?.()'); + expect(closeRestore).toBeGreaterThanOrEqual(0); + expect(close.indexOf("type: 'close_abort_result'", closeRestore)) + .toBeGreaterThan(closeRestore); + }); + + it('retains close and shutdown generations across worker error and preserves close fence on exit', () => { + const errorStart = workerPoolSource.indexOf("worker.on('error', (err) => {"); + const errorEnd = workerPoolSource.indexOf("worker.stdout?.on('data'", errorStart); + const errorHandler = workerPoolSource.slice(errorStart, errorEnd); + expect(errorStart).toBeGreaterThanOrEqual(0); + expect(errorHandler).toContain('ds.riffShutdownState !== undefined'); + expect(errorHandler).toContain('|| ds.riffCloseState !== undefined'); + expect(errorHandler.indexOf('if (!retainExactRetirementGeneration)')) + .toBeLessThan(errorHandler.indexOf('ds.riffCloseState = undefined')); + + const exitStart = workerPoolSource.indexOf("worker.on('exit', (code, signal) => {"); + const exitEnd = workerPoolSource.indexOf('\n return worker;', exitStart); + const exitHandler = workerPoolSource.slice(exitStart, exitEnd); + expect(exitStart).toBeGreaterThanOrEqual(0); + expect(exitHandler).toContain("phase: 'uncertain'"); + expect(exitHandler).not.toContain('ds.riffCloseState = undefined'); + }); +}); From db4b799cb0f134d719d0369480a0687b5d65be98 Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:15:58 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix(pm2):=20=E6=8B=86=E5=88=86=E4=BB=A3?= =?UTF-8?q?=E9=99=85=E5=AE=89=E5=85=A8=E7=9A=84=E6=9C=BA=E7=BE=A4=E5=81=9C?= =?UTF-8?q?=E5=90=AF=E5=8D=8F=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli.ts | 1609 +++++++++++++---- src/cli/fleet-shutdown.ts | 452 +++++ src/cli/pm2-descriptor-guard.ts | 144 ++ src/cli/pm2-exact-start.ts | 77 + src/cli/pm2-god-admission.ts | 27 + src/cli/pm2-jlist.ts | 123 ++ src/cli/pm2-preflight.ts | 29 + src/cli/pm2-shutdown-capability.ts | 205 +++ src/cli/pm2-start-transaction.ts | 311 ++++ src/cli/supervisor-shutdown-client.ts | 180 ++ src/core/dashboard-ipc-server.ts | 48 + src/core/process-start-identity.ts | 52 + src/core/restart-report.ts | 19 +- src/core/shutdown-budgets.ts | 27 +- src/core/supervisor-shutdown-ipc.ts | 20 + src/core/supervisor-shutdown-protocol.ts | 16 + src/daemon.ts | 43 +- src/services/restart-intent-store.ts | 173 +- src/setup/bots-store.ts | 14 +- test/fleet-shutdown.test.ts | 914 ++++++++++ test/plugin-service-restart-lifecycle.test.ts | 5 +- test/pm2-descriptor-guard.test.ts | 94 + test/pm2-exact-start.test.ts | 92 + test/pm2-god-admission.test.ts | 22 + test/pm2-jlist.test.ts | 69 + test/pm2-preflight.test.ts | 27 + test/pm2-shutdown-capability.test.ts | 99 + test/pm2-start-transaction.test.ts | 276 +++ test/restart-intent-store.test.ts | 75 + test/restart-report.test.ts | 29 +- test/shutdown-supervisor-contract.test.ts | 416 +++++ test/supervisor-shutdown-client.test.ts | 75 + test/supervisor-shutdown-ipc.test.ts | 25 + ...isor-shutdown-loopback.integration.test.ts | 83 + 34 files changed, 5495 insertions(+), 375 deletions(-) create mode 100644 src/cli/fleet-shutdown.ts create mode 100644 src/cli/pm2-descriptor-guard.ts create mode 100644 src/cli/pm2-exact-start.ts create mode 100644 src/cli/pm2-god-admission.ts create mode 100644 src/cli/pm2-jlist.ts create mode 100644 src/cli/pm2-preflight.ts create mode 100644 src/cli/pm2-shutdown-capability.ts create mode 100644 src/cli/pm2-start-transaction.ts create mode 100644 src/cli/supervisor-shutdown-client.ts create mode 100644 src/core/process-start-identity.ts create mode 100644 src/core/supervisor-shutdown-ipc.ts create mode 100644 src/core/supervisor-shutdown-protocol.ts create mode 100644 test/fleet-shutdown.test.ts create mode 100644 test/pm2-descriptor-guard.test.ts create mode 100644 test/pm2-exact-start.test.ts create mode 100644 test/pm2-god-admission.test.ts create mode 100644 test/pm2-jlist.test.ts create mode 100644 test/pm2-preflight.test.ts create mode 100644 test/pm2-shutdown-capability.test.ts create mode 100644 test/pm2-start-transaction.test.ts create mode 100644 test/shutdown-supervisor-contract.test.ts create mode 100644 test/supervisor-shutdown-client.test.ts create mode 100644 test/supervisor-shutdown-ipc.test.ts create mode 100644 test/supervisor-shutdown-loopback.integration.test.ts diff --git a/src/cli.ts b/src/cli.ts index 7e830dfcc..17b111203 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -8,7 +8,8 @@ * botmux setup list|add|configure|edit|remove — scripted (non-TUI) bot management, see `botmux setup help` * botmux start — start daemon and auto plugin services * botmux stop [--with-plugin] — stop daemon (optionally stop auto plugin services) - * botmux restart [--include-pm2] [--with-plugin] — restart daemon, then ensure auto plugin services + * botmux restart [--include-pm2] [--with-plugin] — restart daemon, then ensure auto plugin services; + * --include-pm2 is a zero-live-God admission fence, not authority to signal an existing PM2 God * botmux logs [--lines] — view daemon logs * botmux status — show daemon status * botmux upgrade|update — upgrade to latest version @@ -21,7 +22,7 @@ * botmux whiteboard status|enable|disable|current|list|read|update|write — local project whiteboard */ import { execSync, execFileSync, spawnSync, spawn } from 'node:child_process'; -import { existsSync, mkdirSync, copyFileSync, readFileSync, writeFileSync, renameSync, readdirSync, readlinkSync, appendFileSync, statSync, unlinkSync, rmSync, realpathSync } from 'node:fs'; +import { existsSync, mkdirSync, copyFileSync, readFileSync, writeFileSync, renameSync, readdirSync, appendFileSync, statSync, unlinkSync, rmSync, realpathSync } from 'node:fs'; import { atomicWriteFileSync } from './utils/atomic-write.js'; import { join, dirname, basename, resolve } from 'node:path'; import { homedir } from 'node:os'; @@ -82,13 +83,53 @@ import { scheduleTimeZone } from './utils/timezone.js'; import { expandHomePath, invalidWorkingDirs } from './utils/working-dir.js'; import { firstPositional } from './cli/arg-utils.js'; import { isColdResumeDormant, sessionListDisposition } from './cli/session-list-liveness.js'; +import { readSupervisorProcessStartIdentity } from './core/process-start-identity.js'; +import { + FLEET_DAEMON_EXIT_WAIT_MS, + FLEET_SUCCESSOR_SETTLE_MS, + PM2_DAEMON_KILL_TIMEOUT_MS, + PM2_DAEMON_RESTART_DELAY_MS, +} from './core/shutdown-budgets.js'; +import { + isFleetEntryProvenFreeOfAutorestartTimer, + signalAndAwaitFleet, + type FleetProcessEntry, +} from './cli/fleet-shutdown.js'; +import { + startExactPm2ProcessIds, + type Pm2ExactStartClient, +} from './cli/pm2-exact-start.js'; import { dispatchPrimaryMessage, findStdinAliasAttachment, normalizeInteractiveCardInput, sendFileAttachments, sendVideoAttachments, shouldSendAsPureVideo, validateVideoAttachments } from './cli/send-dispatch.js'; import { dispatchDeferredTopicSend, type DeferredScheduleRunData } from './cli/deferred-topic-send.js'; import { resolveDaemonEnv } from './cli/daemon-lifecycle-env.js'; import { buildPm2SpawnCommand } from './cli/pm2-command.js'; +import { + parseCanonicalPm2Id, + parsePm2JlistOutput, + parsePm2JlistOutputStrict, + parsePm2Integer, +} from './cli/pm2-jlist.js'; +import { assertLinuxPm2GodExecutableUsable } from './cli/pm2-preflight.js'; +import { assertNoUnregisteredLiveDaemonDescriptorsIn } from './cli/pm2-descriptor-guard.js'; +import { DAEMON_GRACEFUL_EXIT_CODE } from './core/supervisor-shutdown-protocol.js'; +import { assertPm2DaemonShutdownCapabilitiesIn } from './cli/pm2-shutdown-capability.js'; +import { assertIncludePm2RestartAdmission } from './cli/pm2-god-admission.js'; +import { + requestAttestedDaemonShutdown, + requestAttestedDaemonShutdownBatch, +} from './cli/supervisor-shutdown-client.js'; +import { + assertDaemonPm2GracefulExitPolicy, + assertConfiguredPm2FleetReady, + assertExactAttestedDaemonSet, + classifyStartBotFleetAdmission, + normalizeRawPm2StopExitCodes, + reconcileLatePm2StartPublication, + runBoundedPm2StartTransaction, +} from './cli/pm2-start-transaction.js'; import { callDashboard, type DashboardEndpoint, type DashboardResult } from './cli/dashboard-endpoint.js'; import { globalInstallUpdateLockTargetIn, installLatestBotmuxSync } from './core/maintenance.js'; -import { withFileLockSync } from './utils/file-lock.js'; +import { withFileLock, withFileLockSync } from './utils/file-lock.js'; import { formatGlobalInstallCommand, resolveGlobalInstallPlan, @@ -133,7 +174,14 @@ import { whiteboardPath, } from './services/whiteboard-store.js'; import { buildBridgeSendMarkerContent } from './services/bridge-fallback-gate.js'; -import { bindRestartLeaseTo, writeManualIntentIfAbsentTo } from './services/restart-intent-store.js'; +import { + bindRestartLeaseTo, + commitRestartIntentAttemptTo, + consumeRestartIntentTo, + removeRestartIntentAttemptTo, + type RestartIntent, + writeRestartAttemptIntentTo, +} from './services/restart-intent-store.js'; import { repairMissingChatScope, stripLegacyPendingCardFields } from './services/session-store.js'; import { evaluateVcMeetingManagedSend, @@ -160,8 +208,6 @@ import { inspectBotmuxPm2Apps, isExactPm2BotActivationReceipt, managedActivationPm2Disposition, - parsePm2JlistOutputStrict as parsePm2JlistOutput, - stopExactPm2Process, type BotmuxPm2Inspection, } from './core/bot-live-control.js'; @@ -200,6 +246,10 @@ const PM2_NAME = 'botmux'; * when those external pm2 installations get moved or removed. */ const PM2_HOME = join(CONFIG_DIR, 'pm2'); +const PM2_FLEET_MUTATION_LOCK_TARGET = join(CONFIG_DIR, 'pm2-fleet-mutation'); +const PM2_START_COMMAND_TIMEOUT_MS = 30_000; +const PM2_START_VERIFY_TIMEOUT_MS = 10_000; +const PM2_START_LATE_PUBLICATION_SETTLE_MS = 10_000; // ─── Helpers ───────────────────────────────────────────────────────────────── @@ -247,54 +297,85 @@ function pm2Env(home: string = PM2_HOME): NodeJS.ProcessEnv { } function listPm2GodDaemonPids(home: string = PM2_HOME): number[] { - if (process.platform !== 'linux') return []; const marker = `God Daemon (${home})`; const pids: number[] = []; - try { - for (const ent of readdirSync('/proc')) { + if (process.platform === 'linux') { + let entries: string[]; + try { entries = readdirSync('/proc'); } + catch (err) { + throw new Error(`cannot inspect /proc for duplicate PM2 Gods: ${err instanceof Error ? err.message : err}`); + } + for (const ent of entries) { if (!/^\d+$/.test(ent)) continue; const pid = parseInt(ent, 10); if (!pid) continue; try { const cmd = readFileSync(`/proc/${pid}/cmdline`, 'utf-8').replace(/\u0000/g, ' ').trim(); if (cmd.includes('PM2 v') && cmd.includes(marker)) pids.push(pid); - } catch { /* ignore unreadable proc entries */ } + } catch { /* another user's or already-exited process */ } + } + return pids.sort((a, b) => a - b); + } + if (process.platform === 'win32') { + const windowsScan = spawnSync('powershell.exe', [ + '-NoProfile', + '-NonInteractive', + '-Command', + "$needle = \"God Daemon ($env:BOTMUX_PM2_SCAN_HOME)\"; " + + "Get-CimInstance Win32_Process | Where-Object { " + + "$_.CommandLine -and $_.CommandLine.Contains('PM2 v') " + + "-and $_.CommandLine.Contains($needle) } | ForEach-Object { $_.ProcessId }", + ], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 4_000, + env: { ...process.env, BOTMUX_PM2_SCAN_HOME: home }, + }); + if (windowsScan.status !== 0 || windowsScan.error) { + throw new Error( + `cannot inspect Windows process table for duplicate PM2 Gods: ` + + `${windowsScan.error?.message ?? String(windowsScan.stderr || `status ${windowsScan.status}`).trim()}`, + ); + } + for (const line of String(windowsScan.stdout).split(/\r?\n/)) { + const pid = parsePm2Integer(line.trim(), { nonNegative: true }); + if (pid && pid > 1) pids.push(pid); } - } catch { /* ignore proc scan failure */ } + return [...new Set(pids)].sort((a, b) => a - b); + } + const ps = spawnSync('ps', ['-axo', 'pid=,command='], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 2_000, + }); + if (ps.status !== 0 || ps.error) { + throw new Error( + `cannot inspect process table for duplicate PM2 Gods: ` + + `${ps.error?.message ?? String(ps.stderr || `status ${ps.status}`).trim()}`, + ); + } + for (const line of String(ps.stdout).split(/\r?\n/)) { + if (!line.includes('PM2 v') || !line.includes(marker)) continue; + const match = line.match(/^\s*(\d+)\s+/); + if (match) pids.push(Number(match[1])); + } return pids.sort((a, b) => a - b); } -function killDuplicatePm2GodDaemons(home: string = PM2_HOME): boolean { +function listSingletonPm2GodDaemonPidsForMutation(home: string = PM2_HOME): number[] { const pids = listPm2GodDaemonPids(home); - if (pids.length <= 1) return false; - - const pidFile = join(home, 'pm2.pid'); - let keepPid = 0; - if (existsSync(pidFile)) { - try { - const parsed = parseInt(readFileSync(pidFile, 'utf-8').trim(), 10); - if (pids.includes(parsed)) keepPid = parsed; - } catch { /* ignore malformed pid file */ } - } - if (!keepPid) keepPid = pids[pids.length - 1]; - - const dupes = pids.filter(pid => pid !== keepPid); - if (dupes.length === 0) return false; - - for (const pid of dupes) { - try { process.kill(pid, 'SIGTERM'); } catch { /* ignore */ } - try { - process.kill(pid, 0); - process.kill(pid, 'SIGKILL'); - } catch { /* ignore */ } - } - - try { - atomicWriteFileSync(pidFile, `${keepPid}\n`); - } catch { /* ignore */ } + if (pids.length <= 1) return pids; + // Never signal a duplicate God automatically. Its SIGTERM handler may + // serially stop/force-kill managed children, including a Riff generation + // whose lineage is not yet durable in the surviving God's registry. + throw new Error( + `refusing PM2 mutation: multiple PM2 God daemons share ${home} ` + + `(pids: ${pids.join(', ')}); no process was signalled`, + ); +} - console.warn(`⚠️ 检测到同一 PM2_HOME (${home}) 下存在多个 PM2 God Daemon，已清理重复实例；保留 pid ${keepPid}，移除: ${dupes.join(', ')}`); - return true; +function assertNoDuplicatePm2GodDaemons(home: string = PM2_HOME): void { + listSingletonPm2GodDaemonPidsForMutation(home); } function runPm2(args: string[], inherit = true, home: string = PM2_HOME, timeoutMs?: number): void { @@ -335,6 +416,70 @@ function pm2Capture(args: string[], home: string = PM2_HOME, timeoutMs = 10_000) return typeof r.stdout === 'string' ? r.stdout : ''; } +async function cmdInternalPm2StartExact(args: string[]): Promise { + const processIds = args.map(value => Number(value)); + try { + const claimedParent = parsePm2Integer(process.env.BOTMUX_PM2_FLEET_LOCK_OWNER_PID, { + nonNegative: true, + }); + if (claimedParent !== process.ppid) { + throw new Error('internal exact PM2 start requires its live parent fleet-lock owner'); + } + const lockPayload = readFileSync(`${PM2_FLEET_MUTATION_LOCK_TARGET}.lock`, 'utf8').trim(); + let lockPid: number | undefined; + try { + const parsed = JSON.parse(lockPayload) as unknown; + lockPid = parsed && typeof parsed === 'object' + ? parsePm2Integer((parsed as Record).pid, { nonNegative: true }) + : undefined; + } catch { + lockPid = parsePm2Integer(lockPayload, { nonNegative: true }); + } + if (lockPid !== process.ppid) { + throw new Error('internal exact PM2 start could not verify the parent fleet lock'); + } + const pm2 = require('pm2') as { Client: Pm2ExactStartClient }; + await startExactPm2ProcessIds(processIds, pm2.Client); + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + } +} + +function runExactPm2Starts( + entries: FleetProcessEntry[], + home: string, + timeoutMs: number, +): void { + const processIds = entries.map(entry => entry.pmId); + if (processIds.some(id => !Number.isInteger(id) || (id as number) < 0)) { + throw new Error('conditional PM2 compensation requires an exact pm_id for every entry'); + } + const boundedTimeoutMs = Math.floor(timeoutMs); + if (boundedTimeoutMs <= 0) { + throw new Error('fleet deadline exhausted before conditional PM2 compensation'); + } + const result = spawnSync( + process.execPath, + [__filename, '__pm2-start-exact', ...processIds.map(String)], + { + stdio: 'pipe', + env: { + ...pm2Env(home), + BOTMUX_PM2_FLEET_LOCK_OWNER_PID: String(process.pid), + }, + timeout: boundedTimeoutMs, + encoding: 'utf8', + }, + ); + if (result.status !== 0) { + const detail = result.error?.message + ?? result.stderr?.trim() + ?? `status ${result.status}`; + throw new Error(`conditional PM2 compensation failed: ${detail}`); + } +} + function loadBotsJson(): any[] { if (existsSync(BOTS_JSON_FILE)) { try { @@ -390,9 +535,11 @@ function ensureUniqueBotProcessNames(bots: any[]): void { } } -function ecosystemConfig(activationAppId?: string): string { +function ecosystemConfig( + bots: any[] = loadBotsJson(), + activationAppId?: string, +): string { const daemonScript = join(PKG_ROOT, 'dist', 'index-daemon.js'); - const bots = loadBotsJson(); ensureUniqueBotProcessNames(bots); const daemonEnv = resolveDaemonEnv( process.env, @@ -408,20 +555,16 @@ function ecosystemConfig(activationAppId?: string): string { cwd: CONFIG_DIR, autorestart: true, max_restarts: 10, - restart_delay: 3000, - // A graceful daemon shutdown exits 0 (SIGTERM/SIGINT → drain → process.exit(0)). - // Tell pm2 that exit 0 is intentional so it does NOT autorestart the daemon - // while `botmux restart` is tearing the fleet down — otherwise pm2 revives - // each daemon (after restart_delay) the instant our parallel SIGTERM drains - // it, and re-deleting those revivals one-by-one re-serializes the teardown - // (~13s of churn for 31 bots). Crashes (non-zero exit / killed by signal) - // are NOT in this list, so genuine crash-autorestart is preserved. - stop_exit_codes: [0], - // pm2's default kill_timeout (1.6s) is SHORTER than the daemon's own - // SHUTDOWN_GRACE_MS (3s), so any daemon pm2 has to signal directly gets - // SIGKILL'd mid-drain → orphaned (ppid=1) workers. Give pm2 headroom past - // the daemon's graceful-drain budget so it never force-kills mid-shutdown. - kill_timeout: 3500, + restart_delay: PM2_DAEMON_RESTART_DELAY_MS, + // PM2 maps signal-only death to exit_code=0 before applying + // stop_exit_codes. Zero cannot be a graceful sentinel: SIGKILL/OOM during + // a prepared Riff drain would otherwise suppress autorestart and look safe + // to delete. Only shutdown()'s fully committed success exits the reserved + // non-zero code. All signal deaths and ordinary failures still restart. + stop_exit_codes: [DAEMON_GRACEFUL_EXIT_CODE], + // Keep the supervisor outside every bounded Riff prepare/commit/refusal + // handshake so PM2 cannot SIGKILL a correct daemon mid-ACK. + kill_timeout: PM2_DAEMON_KILL_TIMEOUT_MS, log_date_format: 'YYYY-MM-DD HH:mm:ss', merge_logs: true, node_args: [ @@ -501,9 +644,9 @@ function ecosystemConfig(activationAppId?: string): string { cwd: PKG_ROOT, autorestart: true, max_restarts: 10, - restart_delay: 3000, - // Same rationale as the bot daemons: don't let pm2 revive on graceful exit-0 - // during a fleet teardown, and don't SIGKILL mid-shutdown. (See baseApp.) + restart_delay: PM2_DAEMON_RESTART_DELAY_MS, + // Dashboard owns no Session/Riff lineage, so signal-only exit-0 has no + // prepare/commit ambiguity. Keep it separate from the daemon sentinel. stop_exit_codes: [0], kill_timeout: 3500, error_file: join(LOG_DIR, 'dashboard-error.log'), @@ -1543,7 +1686,7 @@ async function cmdSetupScripted(argv: string[]): Promise { ); return; } - const live = ensureBotDaemonStarted(bot.larkAppId, { quiet: cmd.json }); + const live = await ensureBotDaemonStarted(bot.larkAppId, { quiet: cmd.json }); const next = live.ok ? 'live' : (live.reason === 'fleet_down' ? 'botmux start' : 'botmux restart'); if (cmd.json) { console.log(JSON.stringify({ @@ -1816,7 +1959,7 @@ async function cmdSetupScripted(argv: string[]): Promise { return; } // daemon 在跑就直接把新 bot 那一个进程拉起来，免整组 botmux restart。 - const live = ensureBotDaemonStarted(bot.larkAppId, { quiet: cmd.json }); + const live = await ensureBotDaemonStarted(bot.larkAppId, { quiet: cmd.json }); const next = live.ok ? 'live' : (live.reason === 'fleet_down' ? 'botmux start' : 'botmux restart'); if (cmd.json) { console.log(JSON.stringify({ @@ -2158,7 +2301,7 @@ async function cmdSetup(): Promise { console.log(`\n✅ 已添加机器人 ${newBot.larkAppId}，共 ${bots.length + 1} 个`); console.log(` 配置文件: ${BOTS_JSON_FILE}`); await finishOpenPlatformSetup(newBot.larkAppId, botBrand(newBot), { reuseOnly: hasSetupWebSession(newBot) }); - printAddBotLiveHint(newBot.larkAppId); + await printAddBotLiveHint(newBot.larkAppId); return; } @@ -2215,7 +2358,7 @@ async function cmdSetup(): Promise { console.log(` 配置文件: ${BOTS_JSON_FILE}`); console.log(` 旧配置已备份: ${ENV_FILE}.bak`); await finishOpenPlatformSetup(newBot.larkAppId, botBrand(newBot), { reuseOnly: hasSetupWebSession(newBot) }); - printAddBotLiveHint(newBot.larkAppId); + await printAddBotLiveHint(newBot.larkAppId); } else { // --- Fresh install --- @@ -2234,38 +2377,19 @@ async function cmdSetup(): Promise { * the daemon, but the error gets buried in pm2 logs and the user sees * silence. * - * Detects two cases and either auto-heals or aborts with a clear message: - * 1. pm2 god daemon's running binary is deleted → auto `pm2 kill` + * Detects two cases and aborts with a clear message: + * 1. pm2 god daemon's running binary is deleted → fail closed; an automatic + * kill could bypass a managed daemon's Riff shutdown protocol * 2. This package is installed under an nvm Node version that no longer * exists on disk → abort with reinstall instructions */ -function preflightNodeSanity(): void { - // Case 1: pm2 god is alive but its Node binary has been deleted. - const pm2PidFile = join(PM2_HOME, 'pm2.pid'); - if (existsSync(pm2PidFile)) { - let pm2Pid = 0; - try { pm2Pid = parseInt(readFileSync(pm2PidFile, 'utf-8').trim(), 10); } catch { /* ignore */ } - if (pm2Pid) { - let pm2Alive = false; - try { process.kill(pm2Pid, 0); pm2Alive = true; } catch { /* not alive */ } - if (pm2Alive && process.platform === 'linux') { - // On Linux, /proc//exe is a symlink to the running executable. - // readlink includes a " (deleted)" suffix when the on-disk file is gone. - try { - const exe = readlinkSync(`/proc/${pm2Pid}/exe`); - const cleanPath = exe.replace(/ \(deleted\)$/, ''); - const exeDeleted = exe.endsWith(' (deleted)') || !existsSync(cleanPath); - if (exeDeleted) { - console.warn(`⚠️ pm2 god daemon (pid ${pm2Pid}) 使用的 Node 二进制已失效: ${cleanPath}`); - console.warn(` 自动杀掉 pm2 god 以便用当前 Node 重启...`); - try { - runPm2(['kill'], false, PM2_HOME, 10_000); - } catch { - try { process.kill(pm2Pid, 'SIGKILL'); } catch { /* ignore */ } - } - } - } catch { /* /proc not readable, skip */ } - } +function preflightNodeSanity(home: string = PM2_HOME): void { + // `pm2.pid` is only a cache. Inspect every God actually enumerated for this + // PM2_HOME and refuse rather than guessing which generation is authoritative. + const actualGodPids = listPm2GodDaemonPids(home); + if (process.platform === 'linux') { + for (const pm2Pid of actualGodPids) { + assertLinuxPm2GodExecutableUsable(pm2Pid); } } @@ -2296,8 +2420,6 @@ async function cmdStart(): Promise { process.exit(1); } ensureConfigDir(); - killDuplicatePm2GodDaemons(); - preflightNodeSanity(); await ensureSystemDependencies(); // 启动前快速校验每个 bot 的凭证. Codex review 边界 #5: 凭证无效是 @@ -2333,9 +2455,69 @@ async function cmdStart(): Promise { } } - cleanupLegacyPm2(); - const cfg = ecosystemConfig(); - runPm2(['start', cfg]); + await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { + await withFileLock(BOTS_JSON_FILE, async () => { + const lockedBots = loadBotsJson(); + if (JSON.stringify(lockedBots) !== JSON.stringify(botsForCheck)) { + throw new Error('[start] bots.json changed during credential preflight; retry with the new configuration'); + } + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + cleanupLegacyPm2(); + const currentProjection = readVerifiedBotmuxPm2Projection('start'); + assertNoUnregisteredLiveDaemonDescriptors('start', currentProjection); + assertCanonicalUniquePm2Rows('start', currentProjection); + const configuredNames = configuredCoreProcessNames(lockedBots); + const cfg = ecosystemConfig(lockedBots); + const liveEntries = currentProjection.filter(isLivePm2Entry); + if (liveEntries.length > 0) { + try { + readAndAssertConfiguredFleetOnline( + 'start-idempotent-ready', + configuredNames, + PM2_HOME, + PM2_START_VERIFY_TIMEOUT_MS, + ); + return; + } catch (error) { + throw new Error( + `[start] refusing PM2 start while a partial/live core fleet exists ` + + `(${liveEntries.map(entry => `${entry.name}:${entry.pid}`).join(', ')}): ` + + `${error instanceof Error ? error.message : String(error)}; ` + + 'use start-bot only for an exact one-missing-bot fleet, or restart', + ); + } + } + const unprovenDormant = currentProjection.filter( + entry => !isFleetEntryProvenFreeOfAutorestartTimer(entry), + ); + if (unprovenDormant.length > 0) { + throw new Error( + `[start] refusing PM2 start: dormant row(s) may still have a restart timer ` + + `(${unprovenDormant.map(entry => `${entry.name}:${entry.status ?? 'unknown'}`).join(', ')})`, + ); + } + runBoundedPm2StartTransaction( + 'start', + PM2_START_COMMAND_TIMEOUT_MS, + PM2_START_VERIFY_TIMEOUT_MS, + { + start: timeoutMs => { + assertBotsConfigSnapshotUnchanged('start', lockedBots); + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + runPm2(['start', cfg], true, PM2_HOME, timeoutMs); + }, + verifyFresh: timeoutMs => readAndAssertConfiguredFleetOnline( + 'start-after-launch', configuredNames, PM2_HOME, timeoutMs, + ), + rollback: () => rollbackPm2StartAttempt( + 'start', currentProjection, configuredNames, + ), + }, + ); + }, { maxWaitMs: 5_000 }); + }, { maxWaitMs: 5_000 }); await reconcilePluginServicesForCli(undefined, { autoOnly: true }); const bots = loadBotsJson(); const count = bots.length || 1; @@ -2381,81 +2563,586 @@ function isBotmuxCoreProcessName(name: string): boolean { return name === PM2_NAME || (name.startsWith(`${PM2_NAME}-`) && !name.startsWith(`${PM2_NAME}-plugin-`)); } -function deleteAllBotmuxProcesses(home: string = PM2_HOME): void { - let entries: Array<{ name: string; pid: number; online: boolean }>; +function isBotmuxDaemonProcessName(name: string): boolean { + return isBotmuxCoreProcessName(name) && name !== 'botmux-dashboard'; +} + +type BotmuxPm2ProcessEntry = FleetProcessEntry; + +function toBotmuxPm2ProcessEntry(app: any): BotmuxPm2ProcessEntry { + const rawStopExitCodes = app?.pm2_env?.stop_exit_codes; + const pmId = parseCanonicalPm2Id(app); + const exitCode = parsePm2Integer(app?.pm2_env?.exit_code); + return { + name: String(app.name), + ...(pmId !== undefined ? { pmId } : {}), + pid: Number(app.pid) || 0, + online: app?.pm2_env?.status === 'online', + status: String(app?.pm2_env?.status ?? 'unknown'), + autorestart: app?.pm2_env?.autorestart, + stopExitCodes: normalizeRawPm2StopExitCodes(rawStopExitCodes), + ...(exitCode !== undefined ? { exitCode } : {}), + }; +} + +function readVerifiedBotmuxPm2Projection( + operation: string, + home: string = PM2_HOME, + timeoutMs = 10_000, +): BotmuxPm2ProcessEntry[] { + try { + return parsePm2JlistOutputStrict(pm2Capture(['jlist'], home, timeoutMs)) + .filter(app => app && isBotmuxCoreProcessName(String(app.name))) + .map(toBotmuxPm2ProcessEntry); + } catch (err) { + throw new Error( + `[${operation}] pm2 jlist failed; refusing an unverified PM2 mutation: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +function assertNoUnregisteredLiveDaemonDescriptors( + operation: string, + projections: BotmuxPm2ProcessEntry[], +): void { + assertNoUnregisteredLiveDaemonDescriptorsIn( + operation, + projections, + join(resolveDataDir(), 'dashboard-daemons'), + ); +} + +function duplicatePm2CoreNames(entries: BotmuxPm2ProcessEntry[]): string[] { + return [...new Set(entries.map(entry => entry.name))] + .filter(name => entries.filter(entry => entry.name === name).length > 1); +} + +function isLivePm2Entry(entry: BotmuxPm2ProcessEntry): boolean { + if (!Number.isInteger(entry.pid) || entry.pid <= 1) return false; + try { process.kill(entry.pid, 0); return true; } catch { return false; } +} + +function configuredCoreProcessNames( + bots: any[] = loadBotsJson(), + activationAppId?: string, +): string[] { + const names: string[] = []; + bots.forEach((bot, index) => { + const appId = typeof bot?.larkAppId === 'string' ? bot.larkAppId : ''; + const starting = bot?.activationStarting; + const committed = bot?.activationCommitted; + const deactivating = bot?.activationDeactivating; + const conflicting = starting !== undefined && committed !== undefined; + const marker = starting ?? committed; + const validMarker = ( + marker + && typeof marker === 'object' + && !Array.isArray(marker) + && marker.appId === appId + && typeof marker.jobId === 'string' + && marker.jobId + ); + if ( + bot?.activationPending === true + || deactivating !== undefined + || conflicting + || ( + (starting !== undefined || committed !== undefined) + && (!validMarker || activationAppId !== appId) + ) + ) return; + names.push(botProcessName(bot, index, PM2_NAME)); + }); + names.push('botmux-dashboard'); + return names; +} + +function assertBotsConfigSnapshotUnchanged(operation: string, snapshot: any[]): void { + if (JSON.stringify(loadBotsJson()) === JSON.stringify(snapshot)) return; + throw new Error(`[${operation}] bots.json generation changed before PM2 start; no launch attempted`); +} + +function readAndAssertConfiguredFleetOnline( + operation: string, + configuredNames: string[], + home: string = PM2_HOME, + timeoutMs: number = PM2_START_VERIFY_TIMEOUT_MS, +): BotmuxPm2ProcessEntry[] { + const deadline = Date.now() + Math.max(1, Math.floor(timeoutMs)); + let lastError: unknown; + while (Date.now() < deadline) { + try { + const remainingMs = Math.max(1, deadline - Date.now()); + const projection = readVerifiedBotmuxPm2Projection(operation, home, remainingMs); + assertNoUnregisteredLiveDaemonDescriptors(operation, projection); + assertConfiguredPm2FleetReady( + operation, + projection, + configuredNames, + pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + readyEntries => { + const daemonEntries = readyEntries.filter(entry => isBotmuxDaemonProcessName(entry.name)); + assertDaemonPm2GracefulExitPolicy( + `${operation}-handler-ready-pm2-policy`, + daemonEntries, + ); + const attested = assertPm2DaemonShutdownCapabilitiesIn( + `${operation}-handler-ready`, + daemonEntries.map(entry => ({ name: entry.name, pid: entry.pid })), + join(resolveDataDir(), 'dashboard-daemons'), + ); + assertExactAttestedDaemonSet( + `${operation}-handler-ready`, + daemonEntries, + attested.map(entry => entry.pid), + pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + ); + for (const target of attested) { + if (readSupervisorProcessStartIdentity(target.pid) !== target.processStartIdentity) { + throw new Error( + `[${operation}-handler-ready] daemon generation changed after capability scan: ` + + `${target.name}/${target.pid}`, + ); + } + } + if (readyEntries + .filter(entry => !isBotmuxDaemonProcessName(entry.name)) + .some(entry => !isLivePm2Entry(entry))) { + throw new Error(`[${operation}] dashboard exited during handler-ready verification`); + } + }, + ); + return projection; + } catch (error) { + lastError = error; + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + sleepSyncMs(Math.min(100, remainingMs)); + } + } + throw new Error( + `[${operation}] configured fleet never reached PM2-online plus handler-ready capability ` + + `within ${timeoutMs}ms: ${lastError instanceof Error ? lastError.message : String(lastError)}`, + ); +} + +function assertCanonicalUniquePm2Rows( + operation: string, + entries: BotmuxPm2ProcessEntry[], +): void { + const duplicateNames = duplicatePm2CoreNames(entries); + const missingIds = entries.filter(entry => + !Number.isSafeInteger(entry.pmId) || (entry.pmId as number) < 0); + const duplicateIds = [...new Set(entries + .map(entry => entry.pmId) + .filter((id): id is number => Number.isSafeInteger(id)))] + .filter(id => entries.filter(entry => entry.pmId === id).length > 1); + const duplicateLivePids = [...new Set(entries + .map(entry => entry.pid) + .filter(pid => Number.isSafeInteger(pid) && pid > 1))] + .filter(pid => entries.filter(entry => entry.pid === pid).length > 1); + if (duplicateNames.length === 0 + && missingIds.length === 0 + && duplicateIds.length === 0 + && duplicateLivePids.length === 0) return; + throw new Error( + `[${operation}] refusing PM2 mutation: canonical registry identity is ambiguous` + + (duplicateNames.length > 0 ? ` (duplicate names: ${duplicateNames.join(', ')})` : '') + + (duplicateIds.length > 0 ? ` (duplicate pm_id: ${duplicateIds.join(', ')})` : '') + + (duplicateLivePids.length > 0 + ? ` (duplicate positive pid: ${duplicateLivePids.join(', ')})` + : '') + + (missingIds.length > 0 + ? ` (missing pm_id: ${missingIds.map(entry => entry.name).join(', ')})` + : ''), + ); +} + +function exactQuiescentRowsForMutation( + operation: string, + originals: BotmuxPm2ProcessEntry[], + fresh: BotmuxPm2ProcessEntry[], +): BotmuxPm2ProcessEntry[] { + const exact: BotmuxPm2ProcessEntry[] = []; + for (const original of originals) { + const rows = fresh.filter(entry => entry.name === original.name); + if (rows.length === 0) continue; + if (rows.length !== 1 || rows[0]!.pmId !== original.pmId) { + throw new Error( + `[${operation}] refusing PM2 mutation: registry row ${original.name} was recreated or duplicated`, + ); + } + if (isLivePm2Entry(rows[0]!)) { + throw new Error( + `[${operation}] refusing PM2 mutation: live generation appeared for ` + + `${original.name}:${rows[0]!.pid}`, + ); + } + if (!isFleetEntryProvenFreeOfAutorestartTimer(rows[0]!)) { + throw new Error( + `[${operation}] refusing PM2 mutation: ${original.name} may still publish a successor`, + ); + } + exact.push(rows[0]!); + } + return exact; +} + +function revalidateExactQuiescentRowBeforeMutation( + operation: string, + original: BotmuxPm2ProcessEntry, + allOriginals: BotmuxPm2ProcessEntry[], + home: string = PM2_HOME, + additionalDescriptorAuthority: BotmuxPm2ProcessEntry[] = [], +): BotmuxPm2ProcessEntry | undefined { + assertNoDuplicatePm2GodDaemons(home); + const fresh = readVerifiedBotmuxPm2Projection(operation, home); + assertNoUnregisteredLiveDaemonDescriptors( + operation, + [...fresh, ...additionalDescriptorAuthority], + ); + assertCanonicalUniquePm2Rows(operation, fresh); + return exactQuiescentRowsForMutation(operation, allOriginals, fresh) + .find(entry => entry.name === original.name); +} + +function signalAndAwaitBotmuxProcesses( + entries: BotmuxPm2ProcessEntry[], + operation: 'restart' | 'stop', + home: string = PM2_HOME, + additionalDescriptorAuthority: BotmuxPm2ProcessEntry[] = [], +): void { + assertCanonicalUniquePm2Rows(operation, entries); + const processNameByPid = new Map(); + const processEntryByPid = new Map(); + const processStartByPid = new Map(); + const rememberProcessIdentity = ( + identityOperation: string, + entry: BotmuxPm2ProcessEntry, + ): void => { + if (entry.pid <= 1) return; + const identity = readSupervisorProcessStartIdentity(entry.pid); + if (!identity) { + if (!isLivePm2Entry(entry)) return; + throw new Error( + `[${identityOperation}] cannot bind ${entry.name}/${entry.pid} to a process-start identity`, + ); + } + processNameByPid.set(entry.pid, entry.name); + processEntryByPid.set(entry.pid, entry); + processStartByPid.set(entry.pid, identity); + }; + for (const entry of entries) rememberProcessIdentity(`${operation}-initial-identity`, entry); + + const assertShutdownCapability = ( + capabilityOperation: string, + targets: BotmuxPm2ProcessEntry[], + ) => { + const daemonTargets = targets.filter(entry => isBotmuxDaemonProcessName(entry.name)); + assertDaemonPm2GracefulExitPolicy(capabilityOperation, daemonTargets); + return assertPm2DaemonShutdownCapabilitiesIn( + capabilityOperation, + daemonTargets.map(entry => ({ name: entry.name, pid: entry.pid })), + join(resolveDataDir(), 'dashboard-daemons'), + ); + }; + assertShutdownCapability( + `${operation}-shutdown-capability-preflight`, + entries.filter(entry => entry.online && entry.pid > 1), + ); + + const list = (timeoutMs: number): BotmuxPm2ProcessEntry[] => { + const projection = parsePm2JlistOutputStrict(pm2Capture( + ['jlist'], + home, + Math.min(10_000, Math.max(1, Math.floor(timeoutMs))), + )) + .filter(app => app && isBotmuxCoreProcessName(String(app.name))) + .map(toBotmuxPm2ProcessEntry); + assertNoUnregisteredLiveDaemonDescriptors( + `${operation}-successor-projection`, + [...projection, ...additionalDescriptorAuthority], + ); + assertCanonicalUniquePm2Rows(`${operation}-successor-projection`, projection); + for (const entry of projection) { + rememberProcessIdentity(`${operation}-successor-identity`, entry); + } + return projection; + }; + + const shutdownRequestFailures: string[] = []; + const recordShutdownFailure = (name: string, pid: number, error: unknown): void => { + shutdownRequestFailures.push( + `${name}/${pid}: ${error instanceof Error ? error.message : String(error)}`, + ); + }; + const signalDashboardResidual = (name: string, pid: number): void => { + const expectedStart = processStartByPid.get(pid); + const currentStart = readSupervisorProcessStartIdentity(pid); + if (!currentStart) return; + if (!expectedStart || currentStart !== expectedStart) { + recordShutdownFailure(name, pid, 'dashboard process generation changed'); + return; + } + try { process.kill(pid, 'SIGTERM'); } + catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== 'ESRCH') { + recordShutdownFailure(name, pid, error); + } + } + }; + try { - const apps = parsePm2JlistOutput(pm2Capture(['jlist'], home)); - entries = (Array.isArray(apps) ? apps : []) + signalAndAwaitFleet(entries, operation, FLEET_DAEMON_EXIT_WAIT_MS, { + signal: pid => { + const name = processNameByPid.get(pid); + if (!name) { + throw new Error(`[${operation}] refusing signal for unmapped PM2 daemon pid ${pid}`); + } + if (isBotmuxDaemonProcessName(name)) { + try { + const successor = processEntryByPid.get(pid); + if (!successor) { + recordShutdownFailure(name, pid, 'successor PM2 policy projection is missing'); + return; + } + const authorized = assertShutdownCapability( + `${operation}-successor-immediately-before-request`, + [successor], + ); + const target = authorized.find(entry => entry.pid === pid); + if (!target) { + recordShutdownFailure(name, pid, 'daemon exited before exact IPC attestation'); + return; + } + requestAttestedDaemonShutdown(target, loadDaemonIpcSecret()); + } catch (error) { + recordShutdownFailure(name, pid, error); + } + return; + } + signalDashboardResidual(name, pid); + }, + signalInitial: targets => { + let authorized; + try { + authorized = assertShutdownCapability( + `${operation}-initial-immediately-before-batch-request`, + targets as BotmuxPm2ProcessEntry[], + ); + } catch (error) { + for (const target of targets.filter(entry => isBotmuxDaemonProcessName(entry.name))) { + recordShutdownFailure(target.name, target.pid, error); + } + return; + } + const expectedDaemonTargets = targets + .filter(entry => isBotmuxDaemonProcessName(entry.name)); + const authorizedPids = new Set(authorized.map(entry => entry.pid)); + for (const target of expectedDaemonTargets) { + if (!authorizedPids.has(target.pid)) { + recordShutdownFailure( + target.name, + target.pid, + 'daemon exited before initial exact IPC attestation', + ); + } + } + let attempts; + try { + attempts = requestAttestedDaemonShutdownBatch(authorized, loadDaemonIpcSecret()); + } catch (error) { + attempts = authorized.map(target => ({ target, ok: false, error: String(error) })); + } + for (const attempt of attempts) { + if (!attempt.ok) { + recordShutdownFailure( + attempt.target.name, + attempt.target.pid, + attempt.error ?? 'supervisor shutdown request refused', + ); + } + } + for (const target of targets.filter(entry => !isBotmuxDaemonProcessName(entry.name))) { + signalDashboardResidual(target.name, target.pid); + } + }, + assertSignalAuthorityComplete: () => { + if (shutdownRequestFailures.length > 0) { + throw new Error(shutdownRequestFailures.join('; ')); + } + }, + isAlive: pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + now: () => Date.now(), + sleep: sleepSyncMs, + startOffline: (offlineEntries, timeoutMs) => { + runExactPm2Starts(offlineEntries, home, Math.min(10_000, timeoutMs)); + }, + list, + successorSettleMs: FLEET_SUCCESSOR_SETTLE_MS, + }); + } catch (error) { + if (shutdownRequestFailures.length === 0) throw error; + throw new Error( + `${error instanceof Error ? error.message : String(error)}; ` + + `shutdown request refusal(s): ${shutdownRequestFailures.join('; ')}`, + ); + } +} + +function rollbackPm2StartAttempt( + operation: string, + before: BotmuxPm2ProcessEntry[], + candidateNames: string[], + home: string = PM2_HOME, +): void { + const candidateSet = new Set(candidateNames); + reconcileLatePm2StartPublication( + operation, + PM2_START_LATE_PUBLICATION_SETTLE_MS, + FLEET_DAEMON_EXIT_WAIT_MS + PM2_START_LATE_PUBLICATION_SETTLE_MS, + { + now: () => Date.now(), + sleep: sleepSyncMs, + reconcileOnce: () => { + assertNoDuplicatePm2GodDaemons(home); + const fresh = readVerifiedBotmuxPm2Projection(`${operation}-rollback-read`, home); + assertNoUnregisteredLiveDaemonDescriptors(`${operation}-rollback-read`, fresh); + assertCanonicalUniquePm2Rows(`${operation}-rollback-read`, fresh); + const attemptedRows = fresh.filter(entry => candidateSet.has(entry.name)); + + for (const row of attemptedRows) { + const priorRows = before.filter(entry => entry.name === row.name); + if (priorRows.length > 1 + || (priorRows.length === 1 && priorRows[0]!.pmId !== row.pmId) + || (priorRows.length === 1 && isLivePm2Entry(priorRows[0]!))) { + throw new Error( + `[${operation}] cannot prove ownership of partial-launch row ${row.name}/${row.pmId}`, + ); + } + } + + const rowsNeedingCompensation = attemptedRows.filter(row => { + const prior = before.find(entry => entry.name === row.name); + if (!prior) return true; + return isLivePm2Entry(row) || !isFleetEntryProvenFreeOfAutorestartTimer(row); + }); + if (rowsNeedingCompensation.length > 0) { + const shutdownRows = rowsNeedingCompensation.map(entry => { + if (isLivePm2Entry(entry)) return { ...entry, online: true }; + if (isFleetEntryProvenFreeOfAutorestartTimer(entry)) return entry; + return { ...entry, online: false, status: 'stopped', autorestart: false }; + }); + signalAndAwaitBotmuxProcesses(shutdownRows, 'stop', home); + for (const original of rowsNeedingCompensation) { + const exact = revalidateExactQuiescentRowBeforeMutation( + `${operation}-rollback-before-mutation`, + original, + rowsNeedingCompensation, + home, + ); + if (!exact) continue; + const existedBefore = before.some(entry => + entry.name === original.name && entry.pmId === original.pmId); + runPm2( + [existedBefore ? 'stop' : 'delete', String(exact.pmId)], + false, + home, + 10_000, + ); + } + return false; + } + + const restored = candidateNames.every(name => { + const prior = before.find(entry => entry.name === name); + const rows = attemptedRows.filter(entry => entry.name === name); + if (!prior) return rows.length === 0; + return rows.length === 1 + && rows[0]!.pmId === prior.pmId + && !isLivePm2Entry(rows[0]!) + && isFleetEntryProvenFreeOfAutorestartTimer(rows[0]!); + }); + if (!restored) { + throw new Error(`[${operation}] rollback could not prove the pre-start registry shape`); + } + return true; + }, + }, + ); +} + +function deleteAllBotmuxProcesses( + home: string = PM2_HOME, + additionalDescriptorAuthority: BotmuxPm2ProcessEntry[] = [], +): void { + assertNoDuplicatePm2GodDaemons(home); + let entries: BotmuxPm2ProcessEntry[]; + try { + entries = parsePm2JlistOutputStrict(pm2Capture(['jlist'], home)) .filter(a => a && isBotmuxCoreProcessName(String(a.name))) - .map(a => ({ name: String(a.name), pid: Number(a.pid) || 0, online: a?.pm2_env?.status === 'online' })); + .map(toBotmuxPm2ProcessEntry); } catch (e) { - console.error(`[restart] pm2 jlist failed (pm2 not running or no apps?): ${e instanceof Error ? e.message : e}`); - return; + throw new Error( + `[restart] pm2 jlist failed; refusing to start a second fleet without a safe shutdown view: ` + + `${e instanceof Error ? e.message : e}`, + ); } + assertNoUnregisteredLiveDaemonDescriptors( + 'restart', + [...entries, ...additionalDescriptorAuthority], + ); + assertCanonicalUniquePm2Rows('restart', entries); if (entries.length === 0) return; const names = entries.map(e => e.name); - // Parallel graceful shutdown. pm2's own delete stops apps one-at-a-time - // (async eachLimit, concurrency 1) and each botmux daemon's drain eats pm2's - // full kill_timeout (~1.6s) → ~N×1.6s serial (~38s for 31 bots). Instead we - // SIGTERM every online daemon AT ONCE so their graceful drains overlap (the - // daemon's SIGTERM handler detaches workers within SHUTDOWN_GRACE_MS), wait - // once for them all to exit, then let pm2 delete reap the now-dead entries - // instantly. Orphan-safe: each daemon runs its FULL graceful drain and we wait - // for real exit before pm2 touches it — avoiding the mid-drain SIGKILL the old - // path forced (pm2 kill_timeout 1.6s < daemon SHUTDOWN_GRACE_MS 3s). - const pids = entries.filter(e => e.online && e.pid > 0).map(e => e.pid); - for (const pid of pids) { - try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ } - } - // Poll until every signalled daemon has exited (bounded). SHUTDOWN_GRACE_MS is - // 3s; give headroom. Exits early the moment the last one dies. - const deadline = Date.now() + 5_000; - let alive = pids.slice(); - while (alive.length > 0 && Date.now() < deadline) { - sleepSyncMs(50); - alive = alive.filter(pid => { try { process.kill(pid, 0); return true; } catch { return false; } }); - } - - // Reap pm2 entries. Processes are already dead → each delete is instant, and - // ONE batched `pm2 delete name1 name2 …` collapses N pm2 CLI cold-boots - // (~315ms each) into one. A revived (autorestart, gated by restart_delay) - // instance is still removed by name. - const batchTimeout = Math.max(15_000, names.length * 2_500); - try { - runPm2(['delete', ...names], false, home, batchTimeout); - return; - } catch (e) { - // pm2's batched delete (async eachLimit) aborts on the first failed name, - // so a mid-batch failure can leave stragglers. Fall back to the resilient - // per-name loop that try/catches each name independently. - console.error(`[restart] batched pm2 delete failed, falling back to per-name: ${e instanceof Error ? e.message : e}`); - } - for (const name of names) { + signalAndAwaitBotmuxProcesses(entries, 'restart', home, additionalDescriptorAuthority); + + const deleteErrors: string[] = []; + for (const entry of entries) { try { - runPm2(['delete', name], false, home, 10_000); + const exact = revalidateExactQuiescentRowBeforeMutation( + 'restart-before-delete', + entry, + entries, + home, + additionalDescriptorAuthority, + ); + if (!exact) continue; + runPm2(['delete', String(exact.pmId)], false, home, 10_000); } catch (e) { - // Don't swallow silently — a failed delete here used to leave the - // restart half-done with no trace. Surface it (the auto-restart - // driver captures stderr to ~/.botmux/logs/maintenance-restart.log). - console.error(`[restart] pm2 delete ${name} failed: ${e instanceof Error ? e.message : e}`); + const message = e instanceof Error ? e.message : String(e); + deleteErrors.push(`${entry.name}: ${message}`); + console.error(`[restart] pm2 delete ${entry.name}/${entry.pmId} failed: ${message}`); } } -} -function killPm2GodDaemon(home: string = PM2_HOME): void { + let remaining: string[]; try { - runPm2(['kill'], true, home, 15_000); - return; - } catch { - // Fall back to direct pid cleanup below. - } - - for (const pid of listPm2GodDaemonPids(home)) { - try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ } + const targetNames = new Set(names); + const freshProjection = parsePm2JlistOutputStrict(pm2Capture(['jlist'], home)) + .filter(app => app && isBotmuxCoreProcessName(String(app.name))) + .map(toBotmuxPm2ProcessEntry); + assertNoUnregisteredLiveDaemonDescriptors( + 'restart-after-delete', + [...freshProjection, ...additionalDescriptorAuthority], + ); + exactQuiescentRowsForMutation('restart-after-delete', entries, freshProjection); + remaining = freshProjection + .filter(entry => targetNames.has(entry.name)) + .map(entry => entry.name); + } catch (err) { + throw new Error( + `[restart] PM2 delete verification failed; refusing to report a clean fleet: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); } - for (const pid of listPm2GodDaemonPids(home)) { - try { process.kill(pid, 0); process.kill(pid, 'SIGKILL'); } catch { /* already gone */ } + if (remaining.length > 0) { + throw new Error( + `[restart] PM2 delete left registry entries: ${[...new Set(remaining)].join(', ')}` + + (deleteErrors.length > 0 ? ` (${deleteErrors.join('; ')})` : ''), + ); } } @@ -2469,37 +3156,92 @@ function killPm2GodDaemon(home: string = PM2_HOME): void { function cleanupLegacyPm2(): boolean { const legacyHome = join(homedir(), '.pm2'); if (legacyHome === PM2_HOME) return false; - const legacyPidFile = join(legacyHome, 'pm2.pid'); - if (!existsSync(legacyPidFile)) return false; - - let legacyPid = 0; - try { legacyPid = parseInt(readFileSync(legacyPidFile, 'utf-8').trim(), 10); } catch { return false; } - if (!legacyPid) return false; - // If the legacy daemon isn't alive anymore there's nothing to clean. - try { process.kill(legacyPid, 0); } catch { return false; } - - deleteAllBotmuxProcesses(legacyHome); + const legacyGodPids = listPm2GodDaemonPids(legacyHome); + if (legacyGodPids.length === 0) return false; + assertNoDuplicatePm2GodDaemons(legacyHome); + preflightNodeSanity(legacyHome); + const currentProjection = readVerifiedBotmuxPm2Projection('legacy-cleanup-authority'); + assertNoDuplicatePm2GodDaemons(legacyHome); + deleteAllBotmuxProcesses(legacyHome, currentProjection); return true; } async function cmdStop(): Promise { const includePluginServices = process.argv.includes('--with-plugin'); - killDuplicatePm2GodDaemons(); - cleanupLegacyPm2(); - let stopped = false; - try { - const output = pm2Capture(['jlist']); - const apps = parsePm2JlistOutput(output); - for (const app of apps) { - if (isBotmuxCoreProcessName(String(app.name))) { - try { runPm2(['stop', app.name]); stopped = true; } catch { /* */ } + ensureConfigDir(); + await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { + assertNoDuplicatePm2GodDaemons(); + cleanupLegacyPm2(); + let entries: BotmuxPm2ProcessEntry[]; + try { + entries = parsePm2JlistOutputStrict(pm2Capture(['jlist'])) + .filter(app => app && isBotmuxCoreProcessName(String(app.name))) + .map(toBotmuxPm2ProcessEntry); + } catch (err) { + throw new Error( + `[stop] pm2 jlist failed; refusing an unverified stop: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + assertNoUnregisteredLiveDaemonDescriptors('stop', entries); + assertCanonicalUniquePm2Rows('stop', entries); + if (entries.length === 0) { + cleanupStaleDaemonDescriptors(); + if (includePluginServices) { + await stopPluginServicesForCli(undefined, { autoOnly: true }); } + console.log('daemon 未在运行。'); + return; } - } catch { /* */ } - // Wipe abandoned dashboard-daemon descriptors left behind by stopped daemons. - cleanupStaleDaemonDescriptors(); - if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true }); - if (!stopped) console.log('daemon 未在运行。'); + signalAndAwaitBotmuxProcesses(entries, 'stop'); + const beforeStop = readVerifiedBotmuxPm2Projection('stop-before-registry-mutation'); + assertNoUnregisteredLiveDaemonDescriptors('stop-before-registry-mutation', beforeStop); + exactQuiescentRowsForMutation('stop-before-registry-mutation', entries, beforeStop); + + const stopErrors: string[] = []; + for (const entry of entries) { + try { + const exact = revalidateExactQuiescentRowBeforeMutation( + 'stop-immediately-before-registry-mutation', + entry, + entries, + ); + if (!exact) continue; + runPm2(['stop', String(exact.pmId)], false, PM2_HOME, 10_000); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + stopErrors.push(`${entry.name}: ${message}`); + console.error(`[stop] pm2 stop ${entry.name} failed: ${message}`); + } + } + let nonStoppedResidual: string[]; + try { + const targetNames = new Set(entries.map(entry => entry.name)); + const freshProjection = parsePm2JlistOutputStrict(pm2Capture(['jlist'])) + .filter(app => app && isBotmuxCoreProcessName(String(app.name))) + .map(toBotmuxPm2ProcessEntry); + assertNoUnregisteredLiveDaemonDescriptors('stop-after-registry-mutation', freshProjection); + exactQuiescentRowsForMutation('stop-after-registry-mutation', entries, freshProjection); + nonStoppedResidual = freshProjection + .filter(entry => targetNames.has(entry.name) && entry.status !== 'stopped') + .map(entry => `${entry.name}:${entry.status ?? 'unknown'}`); + } catch (err) { + throw new Error( + `[stop] PM2 stop verification failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + if (stopErrors.length > 0 || nonStoppedResidual.length > 0) { + throw new Error( + `[stop] PM2 registry mutation incomplete` + + (nonStoppedResidual.length > 0 + ? ` (not stopped: ${[...new Set(nonStoppedResidual)].join(', ')})` + : '') + + (stopErrors.length > 0 ? ` (errors: ${stopErrors.join('; ')})` : ''), + ); + } + cleanupStaleDaemonDescriptors(); + if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true }); + }, { maxWaitMs: 5_000 }); } async function cmdRestart(): Promise { @@ -2521,39 +3263,100 @@ async function cmdRestart(): Promise { process.exit(1); } ensureConfigDir(); - const includePm2 = process.argv.includes('--include-pm2'); - const includePluginServices = process.argv.includes('--with-plugin'); - // Drop a restart-intent breadcrumb so the fresh daemon knows this was an - // intentional restart and DMs the owner a summary. `IfAbsent` preserves a - // richer breadcrumb (update / auto-restart) already written by the - // maintenance timer that spawned this very `botmux restart`. A pm2 - // crash-autorestart bypasses this path → no breadcrumb → silent. - try { - const now = Date.now(); - writeManualIntentIfAbsentTo(resolveDataDir(), now, new Date(now).toISOString()); - } catch { /* breadcrumb is best-effort */ } - killDuplicatePm2GodDaemons(); - preflightNodeSanity(); - await ensureSystemDependencies(); - const cfg = ecosystemConfig(); - cleanupLegacyPm2(); - // Delete all botmux processes (handles both old single-process and new multi-process) - deleteAllBotmuxProcesses(); - if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true }); - if (includePm2) { - killPm2GodDaemon(); - } - // Wipe abandoned dashboard-daemon descriptors left behind by killed daemons. - cleanupStaleDaemonDescriptors(); - runPm2(['start', cfg]); - // Default restart preserves running plugin services, then ensures every auto - // service is online. --with-plugin changes only the pre-restart side above: - // it explicitly stops auto services first, so this ensure becomes a restart. - await reconcilePluginServicesForCli(undefined, { autoOnly: true }); - if (refreshAutostart({ pkgRoot: PKG_ROOT, configDir: CONFIG_DIR, logDir: LOG_DIR })) { - console.log(`autostart unit 已同步到当前 Node/cli.js 路径`); - } - await printDashboardHintWithRetry(); + await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { + const includePm2 = process.argv.includes('--include-pm2'); + const includePluginServices = process.argv.includes('--with-plugin'); + if (includePm2) { + assertIncludePm2RestartAdmission(listPm2GodDaemonPids()); + } + + const restartIntentDir = resolveDataDir(); + let stagedRestartIntent: RestartIntent | null = null; + try { + stagedRestartIntent = consumeRestartIntentTo(restartIntentDir, Date.now()); + } catch { /* intent reporting is best-effort */ } + + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + await ensureSystemDependencies(); + cleanupLegacyPm2(); + deleteAllBotmuxProcesses(); + if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true }); + cleanupStaleDaemonDescriptors(); + + const retiredProjection = readVerifiedBotmuxPm2Projection('restart-start'); + assertNoUnregisteredLiveDaemonDescriptors('restart-start', retiredProjection); + if (retiredProjection.length > 0) { + throw new Error( + `[restart-start] new PM2 core row(s) appeared after verified retirement: ` + + retiredProjection.map(entry => `${entry.name}:${entry.pid}`).join(', '), + ); + } + + await withFileLock(BOTS_JSON_FILE, async () => { + const restartBots = loadBotsJson(); + const cfg = ecosystemConfig(restartBots); + const restartAttemptId = randomBytes(16).toString('hex'); + let restartIntentPrepared = false; + try { + const now = Date.now(); + writeRestartAttemptIntentTo( + restartIntentDir, + stagedRestartIntent ?? { kind: 'manual', at: new Date(now).toISOString() }, + now, + restartAttemptId, + ); + restartIntentPrepared = true; + } catch { /* breadcrumb is best-effort */ } + + try { + const configuredNames = configuredCoreProcessNames(restartBots); + runBoundedPm2StartTransaction( + 'restart-start', + PM2_START_COMMAND_TIMEOUT_MS, + PM2_START_VERIFY_TIMEOUT_MS, + { + start: timeoutMs => { + assertBotsConfigSnapshotUnchanged('restart-start', restartBots); + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + runPm2(['start', cfg], true, PM2_HOME, timeoutMs); + }, + verifyFresh: timeoutMs => readAndAssertConfiguredFleetOnline( + 'restart-after-launch', configuredNames, PM2_HOME, timeoutMs, + ), + rollback: () => rollbackPm2StartAttempt( + 'restart-start', retiredProjection, configuredNames, + ), + }, + ); + } catch (err) { + try { + removeRestartIntentAttemptTo(restartIntentDir, restartAttemptId); + } catch { /* best-effort */ } + throw err; + } + + if (restartIntentPrepared) { + let committed = false; + try { + committed = commitRestartIntentAttemptTo(restartIntentDir, restartAttemptId); + } catch { /* best-effort after verified healthy fleet */ } + if (!committed) { + try { + removeRestartIntentAttemptTo(restartIntentDir, restartAttemptId); + } catch { /* best-effort */ } + console.warn('⚠️ daemon 已完整启动，但重启摘要凭据未能提交；本次不会发送重启摘要。'); + } + } + }, { maxWaitMs: 5_000 }); + + await reconcilePluginServicesForCli(undefined, { autoOnly: true }); + if (refreshAutostart({ pkgRoot: PKG_ROOT, configDir: CONFIG_DIR, logDir: LOG_DIR })) { + console.log(`autostart unit 已同步到当前 Node/cli.js 路径`); + } + await printDashboardHintWithRetry(); + }, { maxWaitMs: 5_000 }); } /** Observe botmux PM2 rows. Unknown state is never represented as absence. */ @@ -2569,34 +3372,78 @@ export type StopBotLiveResult = | { ok: true; state: 'stopped' | 'already-stopped'; processName: string } | { ok: false; reason: 'not_found' | 'pm2_error'; message: string }; -function ensureBotDaemonStopped(appId: string, opts: { quiet?: boolean } = {}): StopBotLiveResult { - const bots = loadBotsJson(); - const index = bots.findIndex(b => b?.larkAppId === appId); - if (index < 0) { - return { ok: false, reason: 'not_found', message: `appId ${appId} 不在 bots.json 中` }; +function retireExactBotmuxProcess( + operation: string, + target: BotmuxPm2ProcessEntry, + fullProjection: BotmuxPm2ProcessEntry[], +): void { + const peers = fullProjection.filter(entry => entry.name !== target.name); + signalAndAwaitBotmuxProcesses([target], 'stop', PM2_HOME, peers); + const exact = revalidateExactQuiescentRowBeforeMutation( + `${operation}-before-delete`, + target, + [target], + PM2_HOME, + peers, + ); + if (exact) { + runPm2(['delete', String(exact.pmId)], false, PM2_HOME, 10_000); } - const processName = botProcessName(bots[index], index, PM2_NAME); - const running = listBotmuxPm2Apps(); - if (!running.ok) { - return { ok: false, reason: 'pm2_error', message: running.message }; + const fresh = readVerifiedBotmuxPm2Projection(`${operation}-after-delete`); + assertNoUnregisteredLiveDaemonDescriptors(`${operation}-after-delete`, fresh); + if (fresh.some(entry => entry.name === target.name)) { + throw new Error(`[${operation}] PM2 row ${target.name} is still present after exact delete`); } - const named = running.apps.filter(app => app.name === processName); - if (named.length > 0 && !named.some(app => isExactPm2BotActivationReceipt(app, processName, index, appId))) { +} + +async function ensureBotDaemonStopped( + appId: string, + _opts: { quiet?: boolean } = {}, +): Promise { + ensureConfigDir(); + try { + return await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => ( + withFileLock(BOTS_JSON_FILE, async () => { + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + cleanupLegacyPm2(); + const bots = loadBotsJson(); + const index = bots.findIndex(b => b?.larkAppId === appId); + if (index < 0) { + return { ok: false, reason: 'not_found', message: `appId ${appId} 不在 bots.json 中` }; + } + const processName = botProcessName(bots[index], index, PM2_NAME); + const inspection = listBotmuxPm2Apps(); + if (!inspection.ok) { + return { ok: false, reason: 'pm2_error', message: inspection.message }; + } + const named = inspection.apps.filter(app => app.name === processName); + if (named.length > 0 && !named.every(app => + isExactPm2BotActivationReceipt(app, processName, index, appId))) { + return { + ok: false, + reason: 'pm2_error', + message: `pm2 process ${processName} does not match bots.json slot ${index} / ${appId}`, + }; + } + const projection = readVerifiedBotmuxPm2Projection('stop-bot'); + assertNoUnregisteredLiveDaemonDescriptors('stop-bot', projection); + assertCanonicalUniquePm2Rows('stop-bot', projection); + const target = projection.find(entry => entry.name === processName); + if (!target) { + return { ok: true, state: 'already-stopped', processName }; + } + retireExactBotmuxProcess('stop-bot', target, projection); + return { ok: true, state: 'stopped', processName }; + }, { maxWaitMs: 5_000 }) + ), { maxWaitMs: 5_000 }); + } catch (err) { return { ok: false, reason: 'pm2_error', - message: `pm2 process ${processName} does not match bots.json slot ${index} / ${appId}`, + message: err instanceof Error ? err.message : String(err), }; } - return stopExactPm2Process( - processName, - listBotmuxPm2Apps, - exactName => { - // Delete only this exact PM2 entry. The later start-bot command recreates - // it from the current ecosystem config after activation is acknowledged. - runPm2(['delete', exactName], !opts.quiet); - }, - ); } /** @@ -2618,115 +3465,183 @@ function ensureBotDaemonStopped(appId: string, opts: { quiet?: boolean } = {}): * NOT start a lone bot; that case belongs to `botmux start`, which brings up the * entire ecosystem (all bots + dashboard). */ -function ensureBotDaemonStarted(appId: string, opts: { quiet?: boolean } = {}): StartBotLiveResult { - const bots = loadBotsJson(); - const index = bots.findIndex(b => b?.larkAppId === appId); - if (index < 0) { - return { ok: false, reason: 'not_found', message: `appId ${appId} 不在 bots.json 中` }; - } - const bot = bots[index]; - if (bot?.activationPending === true) { - return { ok: false, reason: 'not_ready', message: `appId ${appId} is still activation pending` }; - } - const activationStarting = bot?.activationStarting; - const activationCommitted = bot?.activationCommitted; - const activationDeactivating = bot?.activationDeactivating; - if (activationDeactivating !== undefined) { - return { ok: false, reason: 'not_ready', message: `appId ${appId} is still deactivating` }; - } - if (activationStarting !== undefined && activationCommitted !== undefined) { - return { ok: false, reason: 'not_ready', message: `appId ${appId} has conflicting activation markers` }; - } - const activationMarker = activationStarting ?? activationCommitted; - const activationJobId = ( - activationMarker - && typeof activationMarker === 'object' - && !Array.isArray(activationMarker) - && activationMarker.appId === appId - && typeof activationMarker.jobId === 'string' - && activationMarker.jobId - ) - ? activationMarker.jobId - : undefined; - if (activationMarker !== undefined && !activationJobId) { - return { ok: false, reason: 'not_ready', message: `appId ${appId} has an invalid activation marker` }; - } - const processName = botProcessName(bot, index, PM2_NAME); - - const running = listBotmuxPm2Apps(); - if (!running.ok) { - return { ok: false, reason: 'pm2_error', message: running.message }; - } - if (running.apps.length === 0) { - return { ok: false, reason: 'fleet_down', message: 'daemon 未在运行，请先 botmux start' }; - } - const named = running.apps.filter(app => app.name === processName); - if (!activationJobId && named.some(app => ( - isExactPm2BotActivationReceipt(app, processName, index, appId) && app.online - ))) { - return { ok: true, state: 'already-online', processName }; - } - if (activationJobId && named.length > 0) { - const disposition = managedActivationPm2Disposition( - named, - processName, - index, - appId, - activationJobId, - ); - if (disposition === 'acknowledged') { - return { ok: true, state: 'already-online', processName }; - } - if (disposition === 'identity_mismatch') { - return { - ok: false, - reason: 'pm2_error', - message: `pm2 process ${processName} does not match bots.json slot ${index} / ${appId}`, - }; - } - } else if (named.length > 0) { - return { - ok: false, - reason: 'pm2_error', - message: `pm2 process ${processName} does not match bots.json slot ${index} / ${appId}`, - }; - } - if (activationJobId && named.length > 0) { - // A same App/index process without this durable receipt can be an old - // daemon already consuming Feishu traffic. Delete it and prove absence - // before the gated activation daemon is allowed to become the PM2 ACK. - const stopped = stopExactPm2Process( - processName, - listBotmuxPm2Apps, - exactName => runPm2(['delete', exactName], !opts.quiet), - ); - if (!stopped.ok) { - return { ok: false, reason: 'pm2_error', message: stopped.message }; - } - } - - const cfg = ecosystemConfig(appId); +async function ensureBotDaemonStarted( + appId: string, + opts: { quiet?: boolean } = {}, +): Promise { + ensureConfigDir(); try { - // `--only ` filters the ecosystem to just this app, so pm2 starts only - // the new bot's daemon and never restarts the already-online ones. - runPm2(['start', cfg, '--only', processName], !opts.quiet); - } catch (e) { - return { ok: false, reason: 'pm2_error', message: e instanceof Error ? e.message : String(e) }; - } - const acknowledged = listBotmuxPm2Apps(); - if (!acknowledged.ok) { - return { ok: false, reason: 'pm2_error', message: acknowledged.message }; - } - if (!acknowledged.apps.some(app => ( - isExactPm2BotActivationReceipt(app, processName, index, appId, activationJobId) && app.online - ))) { + return await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => ( + withFileLock(BOTS_JSON_FILE, async () => { + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + cleanupLegacyPm2(); + const bots = loadBotsJson(); + const index = bots.findIndex(b => b?.larkAppId === appId); + if (index < 0) { + return { ok: false, reason: 'not_found', message: `appId ${appId} 不在 bots.json 中` }; + } + const bot = bots[index]; + if (bot?.activationPending === true) { + return { ok: false, reason: 'not_ready', message: `appId ${appId} is still activation pending` }; + } + const activationStarting = bot?.activationStarting; + const activationCommitted = bot?.activationCommitted; + const activationDeactivating = bot?.activationDeactivating; + if (activationDeactivating !== undefined) { + return { ok: false, reason: 'not_ready', message: `appId ${appId} is still deactivating` }; + } + if (activationStarting !== undefined && activationCommitted !== undefined) { + return { ok: false, reason: 'not_ready', message: `appId ${appId} has conflicting activation markers` }; + } + const activationMarker = activationStarting ?? activationCommitted; + const activationJobId = ( + activationMarker + && typeof activationMarker === 'object' + && !Array.isArray(activationMarker) + && activationMarker.appId === appId + && typeof activationMarker.jobId === 'string' + && activationMarker.jobId + ) + ? String(activationMarker.jobId) + : undefined; + if (activationMarker !== undefined && !activationJobId) { + return { ok: false, reason: 'not_ready', message: `appId ${appId} has an invalid activation marker` }; + } + + const processName = botProcessName(bot, index, PM2_NAME); + const configuredNames = configuredCoreProcessNames( + bots, + activationJobId ? appId : undefined, + ); + const inspection = listBotmuxPm2Apps(); + if (!inspection.ok) { + return { ok: false, reason: 'pm2_error', message: inspection.message }; + } + let projection = readVerifiedBotmuxPm2Projection('start-bot'); + assertNoUnregisteredLiveDaemonDescriptors('start-bot', projection); + assertCanonicalUniquePm2Rows('start-bot', projection); + + const namedInspection = inspection.apps.filter(app => app.name === processName); + if (activationJobId && namedInspection.length > 0) { + const disposition = managedActivationPm2Disposition( + namedInspection, + processName, + index, + appId, + activationJobId, + ); + if (disposition === 'identity_mismatch') { + return { + ok: false, + reason: 'pm2_error', + message: `pm2 process ${processName} does not match bots.json slot ${index} / ${appId}`, + }; + } + if (disposition === 'acknowledged') { + readAndAssertConfiguredFleetOnline( + 'start-bot-already-online-ready', + configuredNames, + PM2_HOME, + PM2_START_VERIFY_TIMEOUT_MS, + ); + return { ok: true, state: 'already-online', processName }; + } + const target = projection.find(entry => entry.name === processName); + if (!target) { + throw new Error(`[start-bot] PM2 identity view disagrees about ${processName}`); + } + retireExactBotmuxProcess('start-bot-replace', target, projection); + projection = readVerifiedBotmuxPm2Projection('start-bot-after-replace'); + } else if (namedInspection.length > 0) { + if (!namedInspection.every(app => + isExactPm2BotActivationReceipt(app, processName, index, appId))) { + return { + ok: false, + reason: 'pm2_error', + message: `pm2 process ${processName} does not match bots.json slot ${index} / ${appId}`, + }; + } + } + + const admission = classifyStartBotFleetAdmission( + 'start-bot', + projection, + configuredNames, + processName, + pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + ); + if (admission.state === 'already-online') { + readAndAssertConfiguredFleetOnline( + 'start-bot-already-online-ready', + configuredNames, + PM2_HOME, + PM2_START_VERIFY_TIMEOUT_MS, + ); + return { ok: true, state: 'already-online', processName }; + } + if (admission.state === 'fleet-down') { + return { ok: false, reason: 'fleet_down', message: 'daemon 未在运行，请先 botmux start' }; + } + + const beforeStart = projection; + const cfg = ecosystemConfig(bots, activationJobId ? appId : undefined); + runBoundedPm2StartTransaction( + 'start-bot', + PM2_START_COMMAND_TIMEOUT_MS, + PM2_START_VERIFY_TIMEOUT_MS, + { + start: timeoutMs => { + assertBotsConfigSnapshotUnchanged('start-bot', bots); + assertNoDuplicatePm2GodDaemons(); + preflightNodeSanity(); + runPm2( + ['start', cfg, '--only', processName], + !opts.quiet, + PM2_HOME, + timeoutMs, + ); + }, + verifyFresh: timeoutMs => { + const fresh = readAndAssertConfiguredFleetOnline( + 'start-bot-after-launch', + configuredNames, + PM2_HOME, + timeoutMs, + ); + const acknowledged = listBotmuxPm2Apps(); + if (!acknowledged.ok || !acknowledged.apps.some(app => ( + isExactPm2BotActivationReceipt( + app, + processName, + index, + appId, + activationJobId, + ) && app.online + ))) { + throw new Error( + `pm2 start did not acknowledge ${processName} at bots.json slot ${index} / ${appId}`, + ); + } + return fresh; + }, + rollback: () => rollbackPm2StartAttempt( + 'start-bot', + beforeStart, + [processName], + ), + }, + ); + return { ok: true, state: 'started', processName }; + }, { maxWaitMs: 5_000 }) + ), { maxWaitMs: 5_000 }); + } catch (err) { return { ok: false, reason: 'pm2_error', - message: `pm2 start did not acknowledge ${processName} at bots.json slot ${index} / ${appId}`, + message: err instanceof Error ? err.message : String(err), }; } - return { ok: true, state: 'started', processName }; } /** @@ -2745,7 +3660,7 @@ async function cmdStartBot(argv: string[]): Promise { process.exit(1); } ensureConfigDir(); - const r = ensureBotDaemonStarted(appId, { quiet: wantsJson }); + const r = await ensureBotDaemonStarted(appId, { quiet: wantsJson }); if (wantsJson) { console.log(JSON.stringify(r, null, 2)); if (!r.ok) process.exitCode = 1; @@ -2775,7 +3690,7 @@ async function cmdStopBot(argv: string[]): Promise { process.exit(1); } ensureConfigDir(); - const r = ensureBotDaemonStopped(appId, { quiet: wantsJson }); + const r = await ensureBotDaemonStopped(appId, { quiet: wantsJson }); if (wantsJson) { console.log(JSON.stringify(r, null, 2)); if (!r.ok) process.exitCode = 1; @@ -2794,8 +3709,8 @@ async function cmdStopBot(argv: string[]): Promise { /** Print the post-add "next step" line for interactive setup: auto-start the new * bot's own daemon when the fleet is up (no fleet-wide restart), else fall back * to the botmux start / restart hint. */ -function printAddBotLiveHint(appId: string): void { - const live = ensureBotDaemonStarted(appId); +async function printAddBotLiveHint(appId: string): Promise { + const live = await ensureBotDaemonStarted(appId); if (live.ok) { console.log(`✅ 已自动上线（${live.processName}），无需重启其它机器人。\n`); } else if (live.reason === 'fleet_down') { @@ -2872,7 +3787,6 @@ function warnIfLegacyBotmuxAlive(): void { } function cmdLogs(): void { - killDuplicatePm2GodDaemons(); warnIfLegacyBotmuxAlive(); const lines = process.argv.includes('--lines') ? process.argv[process.argv.indexOf('--lines') + 1] || '50' @@ -2914,7 +3828,6 @@ function cmdLogs(): void { } function cmdStatus(): void { - killDuplicatePm2GodDaemons(); warnIfLegacyBotmuxAlive(); runPm2(['status']); } @@ -4688,7 +5601,8 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 默认使用 botmux 内置 Feishu Web QR 登录尝试自动导入权限/redirect/发布版本；可加 --no-open-platform-auto 跳过 start 启动 daemon，并启动 mode=auto 的插件 service stop 停止 daemon（默认不停止插件 service；--with-plugin 显式停止 mode=auto 的插件 service） - restart 重启 daemon（默认不停止插件 service，core 启动后确保 mode=auto 正在运行；--with-plugin 显式先停再启动 auto service；--include-pm2 同时重启 PM2 God） + restart 重启 daemon（默认不停止插件 service，core 启动后确保 mode=auto 正在运行；--with-plugin 显式先停再启动 auto service） + --include-pm2 仅允许“入场时没有 live PM2 God”的干净启动；若已有 live God，整条命令会在 fleet/breadcrumb 零改动处拒绝，且不会信号或重启现存 God logs 查看 daemon 日志（--lines N, --bot <0-based-index|pm2-name|appId>） status 查看 daemon 状态 upgrade 升级到最新版本（别名：update） @@ -9702,6 +10616,9 @@ async function runPluginCommandByName(rawCommand: string, commandArgs: string[]) } switch (command) { + case '__pm2-start-exact': + await cmdInternalPm2StartExact(process.argv.slice(3)); + break; case '--version': case '-v': console.log(getVersion()); break; case 'capabilities': { diff --git a/src/cli/fleet-shutdown.ts b/src/cli/fleet-shutdown.ts new file mode 100644 index 000000000..84df86c6d --- /dev/null +++ b/src/cli/fleet-shutdown.ts @@ -0,0 +1,452 @@ +import { parsePm2Integer } from './pm2-jlist.js'; + +export type FleetProcessEntry = { + name: string; + /** Stable PM2 registry identity. Required for generation-safe compensation. */ + pmId?: number; + pid: number; + online: boolean; + status?: string; + autorestart?: boolean | string; + stopExitCodes?: unknown[]; + exitCode?: number; +}; + +export interface FleetShutdownRuntime { + signal(pid: number): void; + /** Optional one-shot initial dispatch. Used when each daemon owns an exact + * authenticated IPC endpoint: all requests are delivered concurrently inside + * one bounded helper. Individual refusals must return normally and remain + * live so the standard fresh compensation path can restore exited peers. */ + signalInitial?(entries: FleetProcessEntry[]): void; + /** Checked before declaring the quiet fleet successful. A missing exact IPC + * ACK remains a transaction failure even if that process later crashes; the + * helper then enters the same fresh compensation path as a live refuser. */ + assertSignalAuthorityComplete?(): void; + isAlive(pid: number): boolean; + now(): number; + sleep(ms: number): void; + /** Under the caller's cross-process fleet lock, conditionally start only the + * exact PM2 rows confirmed offline, within the remaining `timeoutMs`. */ + startOffline(entries: FleetProcessEntry[], timeoutMs: number): void; + /** Return a fresh PM2 projection, completing or failing within `timeoutMs`. */ + list(timeoutMs: number): FleetProcessEntry[]; + /** Must cover PM2 restart_delay so an old autorestart policy cannot publish + * a successor immediately after the original PID disappears. */ + successorSettleMs?: number; + pollMs?: number; +} + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function rowsForName( + entries: FleetProcessEntry[], + name: string, +): FleetProcessEntry[] { + return entries.filter(entry => entry.name === name); +} + +function isLiveGeneration( + entry: FleetProcessEntry, + runtime: FleetShutdownRuntime, +): boolean { + // OS liveness is authority. PM2 can expose a real launching/stopping PID + // while its status is not yet `online`; such a generation is never quiet. + return entry.pid > 0 && runtime.isAlive(entry.pid); +} + +export function isFleetEntryProvenFreeOfAutorestartTimer(entry: FleetProcessEntry): boolean { + if (entry.status === 'errored') return true; + // PM2 handleExit labels delayed post-exit rows `waiting restart` even when + // stop_exit_codes/autorestart suppresses creation of restart_task. A bare + // `stopped` status is not enough: handleExit can publish STOPPED before a + // zero-delay restart task is installed. Policy fields still do not make + // launching/stopping/online-dead rows quiescent. + if (entry.status !== 'waiting restart' && entry.status !== 'stopped') return false; + if (entry.autorestart === false || entry.autorestart === 'false') return true; + if (!Number.isFinite(entry.exitCode) || !Array.isArray(entry.stopExitCodes)) return false; + return entry.stopExitCodes.some(code => parsePm2Integer(code) === entry.exitCode); +} + +function rawStopCodeMatchesExitCode(code: unknown, exitCode: number): boolean { + return (typeof code === 'number' && Number.isSafeInteger(code) && code === exitCode) + || (typeof code === 'string' && code === String(exitCode)); +} + +/** Being timer-free is sufficient for initial admission and compensation, but + * not proof that a signalled generation completed its shutdown transaction. + * PM2 can mark an abrupt exit `errored` after max_restarts and suppress a + * successor even though exit_code=0 is not the daemon's graceful sentinel. */ +export function isFleetEntryProvenTerminalAfterSignal(entry: FleetProcessEntry): boolean { + if (entry.status !== 'errored' + && entry.status !== 'waiting restart' + && entry.status !== 'stopped') return false; + if (!Number.isSafeInteger(entry.exitCode) || !Array.isArray(entry.stopExitCodes)) return false; + return entry.stopExitCodes.some(code => rawStopCodeMatchesExitCode(code, entry.exitCode!)); +} + +function isProvenInitiallyQuiescent( + entry: FleetProcessEntry, + runtime: FleetShutdownRuntime, +): boolean { + if (isLiveGeneration(entry, runtime)) return false; + return isFleetEntryProvenFreeOfAutorestartTimer(entry); +} + +/** Signal every observed generation in parallel. Success requires a bounded + * quiet window with no online successor for any target name. If one generation + * refuses, restore only names proven truly offline and never touch a live + * refuser or an already-auto-restarted healthy successor. */ +export function signalAndAwaitFleet( + entries: FleetProcessEntry[], + operation: 'restart' | 'stop', + timeoutMs: number, + runtime: FleetShutdownRuntime, +): void { + const duplicateNames = [...new Set(entries.map(entry => entry.name))] + .filter(name => entries.filter(entry => entry.name === name).length > 1); + if (duplicateNames.length > 0) { + throw new Error( + `[${operation}] refusing PM2 mutation: duplicate registry row(s) for singleton botmux name(s): ` + + duplicateNames.join(', '), + ); + } + const uncertainLive = entries.filter(entry => + !entry.online && entry.pid > 0 && runtime.isAlive(entry.pid)); + if (uncertainLive.length > 0) { + throw new Error( + `[${operation}] refusing PM2 mutation: non-online registry rows still have live PID(s): ` + + uncertainLive.map(entry => `${entry.name}:${entry.pid}`).join(', '), + ); + } + const unverifiedDormant = entries.filter(entry => + !isLiveGeneration(entry, runtime) && !isProvenInitiallyQuiescent(entry, runtime)); + if (unverifiedDormant.length > 0) { + throw new Error( + `[${operation}] refusing PM2 mutation: dormant registry row(s) may still restart: ` + + unverifiedDormant.map(entry => `${entry.name}:${entry.status ?? 'unknown'}`).join(', '), + ); + } + const targets = entries.filter(entry => entry.online && entry.pid > 0); + + const deadline = runtime.now() + Math.max(0, timeoutMs); + const remainingMs = (): number => Math.max(0, Math.floor(deadline - runtime.now())); + const listWithinDeadline = ( + purpose: string, + capMs: number = Number.POSITIVE_INFINITY, + ): FleetProcessEntry[] => { + const budgetMs = remainingMs(); + if (budgetMs <= 0) { + throw new Error(`fleet deadline exhausted before ${purpose}`); + } + const projection = runtime.list(Math.max(1, Math.min(budgetMs, Math.floor(capMs)))); + if (remainingMs() <= 0) { + throw new Error(`fleet deadline exhausted during ${purpose}`); + } + return projection; + }; + + // Later PM2 stop/delete mutates every supplied registry name, including + // initially stopped rows. Every one must remain live-PID-free for the same + // successor settle window before that name mutation is safe. + const targetNames = new Set(entries.map(entry => entry.name)); + const signalledPids = new Set(); + const provenTerminalPids = new Set(); + const latestTrackedPidByName = new Map(); + const tracked = new Map(); + const signalGeneration = (entry: FleetProcessEntry): void => { + if (signalledPids.has(entry.pid)) return; + signalledPids.add(entry.pid); + tracked.set(entry.pid, entry); + latestTrackedPidByName.set(entry.name, entry.pid); + // The concrete runtime handles a proven ESRCH. Capability/generation + // authorization failures must escape; swallowing them here could turn a + // rollout fence into a partially signalled fleet. + runtime.signal(entry.pid); + }; + if (runtime.signalInitial) { + for (const target of targets) { + signalledPids.add(target.pid); + tracked.set(target.pid, target); + latestTrackedPidByName.set(target.name, target.pid); + } + runtime.signalInitial(targets); + if (remainingMs() <= 0) { + throw new Error( + `[${operation}] fleet deadline exhausted during bounded initial daemon dispatch; ` + + 'no later fleet action was attempted', + ); + } + } else { + for (const target of targets) { + if (remainingMs() <= 0) { + throw new Error( + `[${operation}] fleet deadline exhausted while signalling initial daemon generations; ` + + 'no later fleet action was attempted', + ); + } + signalGeneration(target); + // A runtime signal implementation is synchronous and may itself consume + // the remaining budget. Never signal the next target or begin polling + // after such a late return. + if (remainingMs() <= 0) { + throw new Error( + `[${operation}] fleet deadline exhausted during initial daemon signalling; ` + + 'no later fleet action was attempted', + ); + } + } + } + + const pollMs = Math.max(1, runtime.pollMs ?? 50); + const successorSettleMs = Math.max(0, runtime.successorSettleMs ?? 3_500); + // A live refuser is decided early enough to leave one explicitly partitioned + // compensation tail. All-dead fleets may continue their normal successor + // settle beyond this point; they need no compensation subprocesses. + const compensationReserveMs = Math.min(6_000, Math.max(5, Math.floor(timeoutMs / 5))); + const liveRefusalDeadline = Math.max(runtime.now(), deadline - compensationReserveMs); + const preProjectionCapMs = Math.max(1, Math.floor(compensationReserveMs / 5)); + const exactStartCapMs = Math.max(1, Math.floor(compensationReserveMs / 2)); + const postProjectionCapMs = Math.max(1, Math.floor(compensationReserveMs / 5)); + const successorProjectionCapMs = preProjectionCapMs; + let quietSince: number | null = null; + let verificationFailure: string | null = null; + let signalAuthorityFailure: string | null = null; + let terminalOutcomeFailure: string | null = null; + + while (runtime.now() < deadline) { + const aliveTracked = [...tracked.values()].filter(entry => runtime.isAlive(entry.pid)); + if (aliveTracked.length > 0) { + quietSince = null; + if (runtime.now() >= liveRefusalDeadline) break; + runtime.sleep(Math.min(pollMs, Math.max(1, liveRefusalDeadline - runtime.now()))); + continue; + } + + let projection: FleetProcessEntry[]; + try { + projection = listWithinDeadline( + 'PM2 successor verification', + successorProjectionCapMs, + ); + } catch (err) { + verificationFailure = `PM2 successor verification failed: ${errorText(err)}`; + break; + } + const successors = projection.filter(entry => + targetNames.has(entry.name) + && isLiveGeneration(entry, runtime) + && !signalledPids.has(entry.pid)); + + // PM2 restart overlimit is timer-free but not necessarily graceful. Before + // following a successor or starting the quiet window, prove the terminal + // outcome of every dead generation that this transaction signalled. A + // live replacement still carries the prior generation's exit_code, so it + // may prove that outcome before receiving its own shutdown request. + let terminalOutcomePending = false; + for (const [pid, trackedEntry] of tracked) { + const isLatestTrackedGeneration = latestTrackedPidByName.get(trackedEntry.name) === pid; + // A predecessor may be cached only after a fresh successor carrying its + // accepted exit_code was observed and then became the newly signalled + // generation. The latest generation must re-prove its terminal row on + // every quiet-window projection; a later missing row is never success. + if (runtime.isAlive(pid) + || (provenTerminalPids.has(pid) && !isLatestTrackedGeneration)) continue; + const sameNameRows = rowsForName(projection, trackedEntry.name); + const exactRows = Number.isInteger(trackedEntry.pmId) + ? sameNameRows.filter(row => row.pmId === trackedEntry.pmId) + : sameNameRows; + if (exactRows.length !== 1) { + terminalOutcomePending = true; + continue; + } + const exactState = exactRows[0]!; + const replacementPublished = exactState.pid > 0 && exactState.pid !== pid; + const liveReplacementPublished = replacementPublished + && isLiveGeneration(exactState, runtime); + const liveReplacementCarriesAcceptedExit = liveReplacementPublished + && Number.isSafeInteger(exactState.exitCode) + && Array.isArray(exactState.stopExitCodes) + && exactState.stopExitCodes.some(code => + rawStopCodeMatchesExitCode(code, exactState.exitCode!)); + if (liveReplacementCarriesAcceptedExit) { + // This cache becomes usable only after signalGeneration records the + // replacement as this name's latest tracked generation. + provenTerminalPids.add(pid); + continue; + } + // A dead different-PID row may already contain that replacement's own + // exit_code, so it can never prove the predecessor's terminal outcome. + if (!replacementPublished && isFleetEntryProvenTerminalAfterSignal(exactState)) continue; + const terminalStatus = exactState.status === 'errored' + || exactState.status === 'waiting restart' + || exactState.status === 'stopped'; + if (terminalStatus || replacementPublished) { + terminalOutcomeFailure = `${trackedEntry.name}/${pid} exited without an accepted ` + + `stop_exit_codes terminal (status=${exactState.status ?? 'unknown'}, ` + + `exit_code=${exactState.exitCode ?? 'missing'})`; + break; + } + terminalOutcomePending = true; + } + if (terminalOutcomeFailure) break; + if (terminalOutcomePending) { + quietSince = null; + if (runtime.now() >= liveRefusalDeadline) break; + runtime.sleep(Math.min(pollMs, Math.max(1, liveRefusalDeadline - runtime.now()))); + continue; + } + + if (successors.length > 0) { + quietSince = null; + if (runtime.now() >= liveRefusalDeadline) { + // This is a positively verified live refuser, not an unreadable fleet. + // Leave the late generation untouched, but retain it in the refusal + // accounting and compensate unrelated exact offline originals. + for (const successor of successors) tracked.set(successor.pid, successor); + break; + } + // A projection can consume its entire subprocess timeout. Never act on + // an observation returned at or beyond the absolute fleet deadline. + if (remainingMs() <= 0) { + verificationFailure = 'fleet deadline exhausted before signalling a PM2 successor'; + break; + } + for (const successor of successors) { + if (remainingMs() <= 0) { + verificationFailure = 'fleet deadline exhausted while signalling PM2 successors'; + break; + } + signalGeneration(successor); + } + if (verificationFailure) break; + continue; + } + + // No live PID is not enough: PM2 may still own a delayed restart_task or a + // launching/stopping transition. Do not begin the quiet window until every + // visible target row proves that no successor timer exists. + const unprovenDormant = projection.filter(entry => + targetNames.has(entry.name) + && !isLiveGeneration(entry, runtime) + && !isFleetEntryProvenFreeOfAutorestartTimer(entry)); + if (unprovenDormant.length > 0) { + quietSince = null; + if (runtime.now() >= liveRefusalDeadline) break; + runtime.sleep(Math.min(pollMs, Math.max(1, liveRefusalDeadline - runtime.now()))); + continue; + } + + const now = runtime.now(); + quietSince ??= now; + if (now - quietSince >= successorSettleMs) { + try { runtime.assertSignalAuthorityComplete?.(); } + catch (error) { + signalAuthorityFailure = errorText(error); + break; + } + return; + } + runtime.sleep(Math.min(pollMs, Math.max(1, deadline - now))); + } + + const liveSignalled = [...tracked.values()].filter(entry => runtime.isAlive(entry.pid)); + const refusal = `[${operation}] ${liveSignalled.length}/${targetNames.size} daemon generation(s) ` + + `refused or could not be verified within ${timeoutMs}ms` + + (signalAuthorityFailure ? ` (signal authority: ${signalAuthorityFailure})` : '') + + (terminalOutcomeFailure + ? ` (post-signal terminal proof: ${terminalOutcomeFailure})` + : ''); + if (verificationFailure) { + throw new Error( + `${refusal}; fleet state is unverified and no compensation was attempted ` + + `(${verificationFailure})`, + ); + } + + // A refusal path must fresh-read BEFORE any mutation. The old original-PID + // projection is not authority: PM2 may already have published a healthy + // successor for a peer that exited. + let beforeCompensation: FleetProcessEntry[]; + try { + beforeCompensation = listWithinDeadline( + 'PM2 verification before compensation', + preProjectionCapMs, + ); + } catch (err) { + throw new Error( + `${refusal}; fleet state is unverified and no compensation was attempted ` + + `(PM2 verification before compensation failed: ${errorText(err)})`, + ); + } + const offlineEntries = targets + .map(target => { + if (runtime.isAlive(target.pid)) return false; + const sameNameRows = rowsForName(beforeCompensation, target.name); + if (sameNameRows.some(state => isLiveGeneration(state, runtime))) return false; + // Compensation is allowed only against the exact original PM2 row. + // If it was deleted/recreated (or the projection omitted pm_id), there + // is no race-free way to recreate it by name. + if (!Number.isInteger(target.pmId)) return false; + if (sameNameRows.length !== 1 || sameNameRows[0]!.pmId !== target.pmId) return false; + const exactState = sameNameRows[0]!; + if (!exactState || !isFleetEntryProvenFreeOfAutorestartTimer(exactState)) return false; + // God.startProcessId does not clear restart_task. Restrict compensation + // to rows whose exit policy proves PM2 could not have scheduled one. + return exactState; + }) + .filter((entry): entry is FleetProcessEntry => !!entry); + const offlineNames = offlineEntries.map(entry => entry.name); + + const compensationErrors: string[] = []; + if (offlineEntries.length > 0) { + try { + const compensationBudgetMs = remainingMs(); + if (compensationBudgetMs <= 0) { + throw new Error('fleet deadline exhausted before compensation'); + } + runtime.startOffline( + offlineEntries, + Math.min(compensationBudgetMs, exactStartCapMs), + ); + if (remainingMs() <= 0) { + throw new Error('fleet deadline exhausted during compensation'); + } + } + catch (err) { compensationErrors.push(errorText(err)); } + } + + let afterCompensation: FleetProcessEntry[] = []; + try { + afterCompensation = listWithinDeadline( + 'PM2 verification after compensation', + postProjectionCapMs, + ); + } + catch (err) { + compensationErrors.push(`PM2 verification after compensation failed: ${errorText(err)}`); + } + const unavailable = targets + .filter(target => { + if (runtime.isAlive(target.pid)) return false; + return !rowsForName(afterCompensation, target.name) + .some(state => isLiveGeneration(state, runtime)); + }) + .map(target => target.name); + + if (unavailable.length > 0 || compensationErrors.length > 0) { + throw new Error( + `${refusal}; fleet is partially stopped` + + (unavailable.length > 0 ? ` (offline: ${unavailable.join(', ')})` : '') + + (compensationErrors.length > 0 + ? ` (restore errors: ${compensationErrors.join('; ')})` + : ''), + ); + } + throw new Error( + `${refusal}; restored ${offlineNames.length} offline PM2 ` + + `entr${offlineNames.length === 1 ? 'y' : 'ies'} and left every live generation untouched`, + ); +} diff --git a/src/cli/pm2-descriptor-guard.ts b/src/cli/pm2-descriptor-guard.ts new file mode 100644 index 000000000..a4b090c2a --- /dev/null +++ b/src/cli/pm2-descriptor-guard.ts @@ -0,0 +1,144 @@ +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { readSupervisorProcessStartIdentity } from '../core/process-start-identity.js'; + +export interface Pm2DescriptorProjectionEntry { + pid: number; +} + +export interface Pm2DescriptorGuardRuntime { + now(): number; + exists(path: string): boolean; + readdir(path: string): string[]; + read(path: string): string; + mtime(path: string): number; + isAlive(pid: number): boolean; + readStartIdentity(pid: number): string | undefined; +} + +const FRESH_MS = 90_000; + +const defaultRuntime: Pm2DescriptorGuardRuntime = { + now: () => Date.now(), + exists: path => existsSync(path), + readdir: path => readdirSync(path), + read: path => readFileSync(path, 'utf8'), + mtime: path => statSync(path).mtimeMs, + isAlive: pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + readStartIdentity: pid => readSupervisorProcessStartIdentity(pid), +}; + +/** Reconcile PM2 registry authority with daemon-owned descriptors before a + * core mutation. Fresh semantic corruption is fail-closed. A stale descriptor + * is not automatically harmless: an event-loop-frozen/orphan daemon can remain + * live after PM2 loses its registry. Ignore a semantically parseable stale + * record only after proving its PID dead or owned by a different process birth. */ +export function assertNoUnregisteredLiveDaemonDescriptorsIn( + operation: string, + projections: Pm2DescriptorProjectionEntry[], + registryDir: string, + runtime: Pm2DescriptorGuardRuntime = defaultRuntime, +): void { + const registeredPids = new Set( + projections.map(entry => entry.pid).filter(pid => Number.isSafeInteger(pid) && pid > 1), + ); + if (!runtime.exists(registryDir)) return; + const now = runtime.now(); + let names: string[]; + try { names = runtime.readdir(registryDir); } + catch (err) { + throw new Error( + `[${operation}] daemon descriptor registry is unreadable; refusing PM2 mutation: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + + const unregistered: Array<{ appId: string; pid: number }> = []; + for (const name of names) { + if (!name.endsWith('.json')) continue; + const path = join(registryDir, name); + let mtimeMs: number; + try { mtimeMs = runtime.mtime(path); } + catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw new Error( + `[${operation}] cannot inspect daemon descriptor ${name}; refusing PM2 mutation: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + const mtimeFresh = now - mtimeMs <= FRESH_MS; + + let value: unknown; + try { value = JSON.parse(runtime.read(path)); } + catch (err) { + if (!mtimeFresh) continue; + throw new Error( + `[${operation}] fresh daemon descriptor ${name} is unreadable or malformed; ` + + `refusing PM2 mutation: ${err instanceof Error ? err.message : String(err)}`, + ); + } + const record = value && typeof value === 'object' && !Array.isArray(value) + ? value as Record + : undefined; + const heartbeat = record?.lastHeartbeat; + const heartbeatValid = typeof heartbeat === 'number' && Number.isFinite(heartbeat); + const heartbeatFresh = heartbeatValid && now - heartbeat <= FRESH_MS; + const pid = record?.pid; + const appId = record?.larkAppId; + if (!mtimeFresh && !heartbeatFresh) { + // Unparseable old debris has no PID authority to reconcile. But once a + // stale record names a canonical PID, liveness + process birth decide; + // age alone must never authorize a second fleet. + if (!record || !Number.isSafeInteger(pid) || (pid as number) <= 1) continue; + if (!runtime.isAlive(pid as number)) continue; + const describedStart = record.processStartIdentity; + if (typeof describedStart !== 'string' || !describedStart) { + throw new Error( + `[${operation}] stale daemon descriptor ${name} still names live PID ${pid} ` + + 'but has no process-start identity; refusing PM2 mutation', + ); + } + const currentStart = runtime.readStartIdentity(pid as number); + if (!currentStart) { + if (!runtime.isAlive(pid as number)) continue; + throw new Error( + `[${operation}] cannot revalidate process-start identity for live stale descriptor ` + + `${name}/${pid}; refusing PM2 mutation`, + ); + } + if (currentStart !== describedStart) continue; + if (typeof appId !== 'string' || !appId.trim()) { + throw new Error( + `[${operation}] stale daemon descriptor ${name} matches live PID ${pid} ` + + 'but has no canonical app id; refusing PM2 mutation', + ); + } + if (!registeredPids.has(pid as number)) { + unregistered.push({ appId: appId.trim(), pid: pid as number }); + } + continue; + } + + if (!record + || !heartbeatValid + || now - heartbeat > FRESH_MS + || !Number.isSafeInteger(pid) + || (pid as number) <= 1 + || typeof appId !== 'string' + || !appId.trim()) { + throw new Error( + `[${operation}] fresh daemon descriptor ${name} has invalid pid/app id/heartbeat; ` + + 'refusing PM2 mutation', + ); + } + if (runtime.isAlive(pid as number) && !registeredPids.has(pid as number)) { + unregistered.push({ appId: appId.trim(), pid: pid as number }); + } + } + if (unregistered.length > 0) { + throw new Error( + `[${operation}] refusing PM2 mutation: daemon descriptor PID(s) are live but absent ` + + `from PM2 registry (${unregistered.map(item => `${item.appId}:${item.pid}`).join(', ')})`, + ); + } +} diff --git a/src/cli/pm2-exact-start.ts b/src/cli/pm2-exact-start.ts new file mode 100644 index 000000000..5a087e008 --- /dev/null +++ b/src/cli/pm2-exact-start.ts @@ -0,0 +1,77 @@ +export interface Pm2ExactStartClient { + launchRPC(callback: (error?: unknown) => void): void; + executeRemote( + method: 'startProcessId', + processId: number, + callback: (error?: unknown) => void, + ): void; + close(callback: (error?: unknown) => void): void; +} + +function errorText(error: unknown): string { + if (error instanceof Error) return error.message; + if (typeof error === 'string') return error; + try { return JSON.stringify(error); } catch { return String(error); } +} + +/** `God.startProcessId` rejects these states before executeApp and therefore + * cannot have killed or restarted a live generation. The final jlist remains + * authoritative about whether the fleet is available. */ +export function isNonMutatingPm2StartRefusal(error: unknown): boolean { + return /(?:\bid unknown\b|process already online|process already started|process with pid \d+ already exists)/i + .test(errorText(error)); +} + +function launchRpc(client: Pm2ExactStartClient): Promise { + return new Promise((resolve, reject) => { + client.launchRPC(error => error ? reject(error) : resolve()); + }); +} + +function closeRpc(client: Pm2ExactStartClient): Promise { + return new Promise(resolve => client.close(() => resolve())); +} + +function startExactProcessId( + client: Pm2ExactStartClient, + processId: number, +): Promise { + return new Promise(resolve => { + client.executeRemote('startProcessId', processId, error => { + if (!error || isNonMutatingPm2StartRefusal(error)) { + resolve(null); + return; + } + resolve(`pm_id ${processId}: ${errorText(error)}`); + }); + }); +} + +/** Connect once and concurrently issue PM2's conditional startProcessId RPC + * for distinct exact registry identities. The caller must hold Botmux's shared + * fleet-operation lock: PM2 checks a row before an async fork and is not itself + * a CAS across two clients. Unlike `pm2 start `, this never routes + * through restartProcessId or intentionally interrupts a live successor. */ +export async function startExactPm2ProcessIds( + processIds: number[], + client: Pm2ExactStartClient, +): Promise { + const uniqueIds = [...new Set(processIds)]; + if (uniqueIds.length !== processIds.length + || uniqueIds.some(id => !Number.isInteger(id) || id < 0)) { + throw new Error('exact PM2 compensation requires unique non-negative pm_id values'); + } + if (uniqueIds.length === 0) return; + + await launchRpc(client); + try { + const failures = (await Promise.all( + uniqueIds.map(id => startExactProcessId(client, id)), + )).filter((failure): failure is string => !!failure); + if (failures.length > 0) { + throw new Error(`conditional PM2 start failed (${failures.join('; ')})`); + } + } finally { + await closeRpc(client); + } +} diff --git a/src/cli/pm2-god-admission.ts b/src/cli/pm2-god-admission.ts new file mode 100644 index 000000000..ddaa81fca --- /dev/null +++ b/src/cli/pm2-god-admission.ts @@ -0,0 +1,27 @@ +/** + * Node's process.kill is PID-addressed and PM2's `kill` RPC is PM2_HOME/socket + * addressed; neither binds a signal to a PID+birth generation. Therefore a + * live God cannot be safely replaced automatically. This admission check must + * run before any fleet or breadcrumb mutation. + */ +export function assertIncludePm2RestartAdmission(pids: readonly number[]): void { + const canonical = [...new Set(pids)] + .filter(pid => Number.isSafeInteger(pid) && pid > 1) + .sort((a, b) => a - b); + if (canonical.length !== pids.length) { + throw new Error('[restart --include-pm2] PM2 God scan returned invalid/duplicate PIDs'); + } + if (canonical.length === 0) return; + if (canonical.length > 1) { + throw new Error( + `[restart --include-pm2] multiple PM2 God daemons are visible ` + + `(pids: ${canonical.join(', ')}); no process or breadcrumb was changed`, + ); + } + throw new Error( + `[restart --include-pm2] refusing before fleet mutation: live PM2 God pid ${canonical[0]} ` + + 'cannot be signalled with generation-bound authority on this platform; ' + + 'this option does not signal or restart an existing God; ' + + 'no process or breadcrumb was changed', + ); +} diff --git a/src/cli/pm2-jlist.ts b/src/cli/pm2-jlist.ts new file mode 100644 index 000000000..ce9f90218 --- /dev/null +++ b/src/cli/pm2-jlist.ts @@ -0,0 +1,123 @@ +type ParsedProjection = + | { kind: 'array'; value: any[] } + | { kind: 'non-array' } + | { kind: 'malformed' }; + +export function parsePm2Integer( + value: unknown, + options: { nonNegative?: boolean } = {}, +): number | undefined { + if (value === null || value === undefined) return undefined; + if (typeof value === 'string' && !/^-?\d+$/.test(value.trim())) return undefined; + if (typeof value !== 'number' && typeof value !== 'string') return undefined; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) return undefined; + if (options.nonNegative && parsed < 0) return undefined; + return parsed; +} + +/** PM2 RPC addresses the canonical top-level pm_id. A nested pm2_env.pm_id can + * be a stale serialized copy and must never resurrect an absent/null identity. */ +export function parseCanonicalPm2Id(app: unknown): number | undefined { + if (!app || typeof app !== 'object' || Array.isArray(app)) return undefined; + return parsePm2Integer((app as Record).pm_id, { nonNegative: true }); +} + +function parseProjection(output: string): ParsedProjection { + try { + const parsed = JSON.parse(output); + return Array.isArray(parsed) + ? { kind: 'array', value: parsed } + : { kind: 'non-array' }; + } catch { + // PM2 can prefix stdout with informational `[PM2]` lines. Search backward + // for the final valid JSON array without accepting arbitrary JSON values. + for ( + let start = output.lastIndexOf('['); + start >= 0; + start = start === 0 ? -1 : output.lastIndexOf('[', start - 1) + ) { + try { + const parsed = JSON.parse(output.slice(start).trim()); + if (Array.isArray(parsed)) return { kind: 'array', value: parsed }; + } catch { /* try an earlier '[' */ } + } + return { kind: 'malformed' }; + } +} + +function isPlainRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +/** Validate the parts of a PM2 registry row that every mutating caller relies + * on. A syntactically valid JSON array is not authority when one of its rows + * would later be silently coerced to name="undefined", pid=0, or an absent + * canonical pm_id. */ +function assertSemanticPm2Rows(rows: any[]): void { + const ids = new Map(); + const positivePids = new Map(); + for (let index = 0; index < rows.length; index++) { + const row = rows[index] as unknown; + if (!isPlainRecord(row)) { + throw new Error(`pm2 jlist row ${index} is not an object`); + } + const name = row.name; + if (typeof name !== 'string' || !name.trim()) { + throw new Error(`pm2 jlist row ${index} has no non-empty name`); + } + const pmId = parseCanonicalPm2Id(row); + if (pmId === undefined) { + throw new Error(`pm2 jlist row ${index} (${name}) has no canonical non-negative pm_id`); + } + const priorName = ids.get(pmId); + if (priorName !== undefined) { + throw new Error( + `pm2 jlist has duplicate canonical pm_id ${pmId} across ${priorName} and ${name}`, + ); + } + ids.set(pmId, name); + + const pid = parsePm2Integer(row.pid, { nonNegative: true }); + if (pid === undefined) { + throw new Error(`pm2 jlist row ${index} (${name}) has no non-negative pid`); + } + if (pid > 1) { + const priorPidName = positivePids.get(pid); + if (priorPidName !== undefined) { + throw new Error( + `pm2 jlist has duplicate positive pid ${pid} across ${priorPidName} and ${name}`, + ); + } + positivePids.set(pid, name); + } + const env = row.pm2_env; + if (!isPlainRecord(env) + || typeof env.status !== 'string' + || !env.status.trim()) { + throw new Error(`pm2 jlist row ${index} (${name}) has no semantic pm2_env.status`); + } + } +} + +/** Forgiving parser for read-only/status surfaces. */ +export function parsePm2JlistOutput(output: string): any[] { + const parsed = parseProjection(output); + return parsed.kind === 'array' ? parsed.value : []; +} + +/** Shutdown authority must never interpret `{}`, `null`, or malformed output + * as an empty fleet: doing so would allow a later name-based PM2 mutation to + * bypass graceful shutdown of live daemons. */ +export function parsePm2JlistOutputStrict(output: string): any[] { + const parsed = parseProjection(output); + if (parsed.kind === 'array') { + assertSemanticPm2Rows(parsed.value); + return parsed.value; + } + throw new Error( + parsed.kind === 'non-array' + ? 'pm2 jlist returned non-array JSON' + : 'pm2 jlist returned malformed output', + ); +} diff --git a/src/cli/pm2-preflight.ts b/src/cli/pm2-preflight.ts new file mode 100644 index 000000000..678871716 --- /dev/null +++ b/src/cli/pm2-preflight.ts @@ -0,0 +1,29 @@ +import { existsSync, readlinkSync } from 'node:fs'; + +export interface Pm2ExecutableProbeRuntime { + readlink(path: string): string; + exists(path: string): boolean; +} + +/** Fail closed when a live Linux PM2 God is executing a deleted Node binary. + * Failure to inspect /proc remains non-authoritative and is skipped, but a + * successful read is evaluated outside that catch so the safety refusal can + * never be swallowed as an inspection error. */ +export function assertLinuxPm2GodExecutableUsable( + pm2Pid: number, + runtime: Pm2ExecutableProbeRuntime = { + readlink: path => readlinkSync(path), + exists: path => existsSync(path), + }, +): void { + let executable: string; + try { executable = runtime.readlink(`/proc/${pm2Pid}/exe`); } + catch { return; } + + const cleanPath = executable.replace(/ \(deleted\)$/, ''); + if (!executable.endsWith(' (deleted)') && runtime.exists(cleanPath)) return; + throw new Error( + `pm2 god daemon (pid ${pm2Pid}) 使用的 Node 二进制已失效: ${cleanPath}; ` + + '为避免强杀未完成 Riff 交接的 daemon，拒绝自动清理，请先人工核对进程归属', + ); +} diff --git a/src/cli/pm2-shutdown-capability.ts b/src/cli/pm2-shutdown-capability.ts new file mode 100644 index 000000000..5ff11b7b0 --- /dev/null +++ b/src/cli/pm2-shutdown-capability.ts @@ -0,0 +1,205 @@ +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { SUPERVISOR_SHUTDOWN_PROTOCOL } from '../core/supervisor-shutdown-protocol.js'; +import { readSupervisorProcessStartIdentity } from '../core/process-start-identity.js'; + +export interface Pm2ShutdownCapabilityTarget { + name: string; + pid: number; +} + +export interface AttestedPm2DaemonShutdownTarget extends Pm2ShutdownCapabilityTarget { + larkAppId: string; + ipcPort: number; + bootInstanceId: string; + processStartIdentity: string; +} + +export interface Pm2ShutdownCapabilityRuntime { + now(): number; + exists(path: string): boolean; + readdir(path: string): string[]; + read(path: string): string; + mtime(path: string): number; + isAlive(pid: number): boolean; + readStartIdentity(pid: number): string | undefined; +} + +const FRESH_MS = 90_000; + +const defaultRuntime: Pm2ShutdownCapabilityRuntime = { + now: () => Date.now(), + exists: path => existsSync(path), + readdir: path => readdirSync(path), + read: path => readFileSync(path, 'utf8'), + mtime: path => statSync(path).mtimeMs, + isAlive: pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + readStartIdentity: pid => { + // Lazy import is unnecessary here: this pure helper has no daemon state. + // The concrete function is injected below to keep tests deterministic. + return readSupervisorProcessStartIdentity(pid); + }, +}; + +function failure(operation: string, detail: string): Error { + return new Error( + `[${operation}] refusing to signal daemon generation(s): ${detail}. ` + + `The live daemon may predate shutdown protocol ${SUPERVISOR_SHUTDOWN_PROTOCOL}; ` + + 'normal stop/restart intentionally fails closed on this first-upgrade boundary. ' + + 'Confirm every Session/Riff workload is idle before an operator-approved one-time manual bootstrap; ' + + 'automatic update must not be reported as applied until the new handler-ready fleet is verified', + ); +} + +/** + * Require every still-live target daemon PID to own one fresh, daemon-written + * descriptor advertising the exact safe shutdown protocol. This is a rollout + * boundary, not a package-version guess: installing a new CLI does not upgrade + * an already-running old daemon in memory. + */ +export function assertPm2DaemonShutdownCapabilitiesIn( + operation: string, + targets: readonly Pm2ShutdownCapabilityTarget[], + registryDir: string, + runtime: Pm2ShutdownCapabilityRuntime = defaultRuntime, +): AttestedPm2DaemonShutdownTarget[] { + const invalid = targets.filter(target => + !target.name.trim() || !Number.isSafeInteger(target.pid) || target.pid <= 1); + if (invalid.length > 0) { + throw failure(operation, 'shutdown target has no canonical name/live PID'); + } + const duplicatePids = [...new Set(targets.map(target => target.pid))] + .filter(pid => targets.filter(target => target.pid === pid).length > 1); + if (duplicatePids.length > 0) { + throw failure(operation, `multiple shutdown targets share PID(s) ${duplicatePids.join(', ')}`); + } + + // A generation that exited before attestation needs no signal. Any successor + // is discovered from a fresh PM2 projection and must attest independently. + const liveTargets = targets.filter(target => runtime.isAlive(target.pid)); + if (liveTargets.length === 0) return []; + if (!runtime.exists(registryDir)) { + throw failure(operation, 'daemon descriptor registry is missing'); + } + + let names: string[]; + try { names = runtime.readdir(registryDir); } + catch (error) { + throw failure( + operation, + `daemon descriptor registry is unreadable (${error instanceof Error ? error.message : String(error)})`, + ); + } + + const now = runtime.now(); + const targetPids = new Set(liveTargets.map(target => target.pid)); + const descriptors = new Map>(); + for (const name of names) { + if (!name.endsWith('.json')) continue; + const path = join(registryDir, name); + let mtimeMs: number; + try { mtimeMs = runtime.mtime(path); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw failure( + operation, + `cannot inspect daemon descriptor ${name} (${error instanceof Error ? error.message : String(error)})`, + ); + } + const mtimeFresh = now - mtimeMs <= FRESH_MS; + + let value: unknown; + try { value = JSON.parse(runtime.read(path)); } + catch (error) { + if (!mtimeFresh) continue; + throw failure( + operation, + `fresh daemon descriptor ${name} is unreadable or malformed ` + + `(${error instanceof Error ? error.message : String(error)})`, + ); + } + const record = value && typeof value === 'object' && !Array.isArray(value) + ? value as Record + : undefined; + const heartbeat = record?.lastHeartbeat; + const heartbeatFresh = typeof heartbeat === 'number' + && Number.isFinite(heartbeat) + && now - heartbeat <= FRESH_MS; + if (!mtimeFresh && !heartbeatFresh) continue; + + const pid = record?.pid; + if (!Number.isSafeInteger(pid) || (pid as number) <= 1) { + throw failure(operation, `fresh daemon descriptor ${name} has no canonical PID`); + } + if (!targetPids.has(pid as number)) continue; + if (!heartbeatFresh) { + throw failure(operation, `daemon descriptor ${name} has no fresh semantic heartbeat`); + } + if (!runtime.isAlive(pid as number)) continue; + if (descriptors.has(pid as number)) { + throw failure(operation, `multiple fresh descriptors claim live PID ${pid}`); + } + descriptors.set(pid as number, record!); + } + + const authorized: AttestedPm2DaemonShutdownTarget[] = []; + for (const target of liveTargets) { + const descriptor = descriptors.get(target.pid); + if (!descriptor) { + // Re-check process birth: a generation that exited during the registry + // scan needs no signal, but a same-PID successor must not inherit a stale + // capability. + if (!runtime.readStartIdentity(target.pid) && !runtime.isAlive(target.pid)) continue; + throw failure( + operation, + `${target.name}/${target.pid} has no matching fresh daemon descriptor`, + ); + } + if (descriptor.supervisorShutdownProtocol !== SUPERVISOR_SHUTDOWN_PROTOCOL) { + throw failure( + operation, + `${target.name}/${target.pid} does not attest ${SUPERVISOR_SHUTDOWN_PROTOCOL}`, + ); + } + const larkAppId = descriptor.larkAppId; + const ipcPort = descriptor.ipcPort; + const bootInstanceId = descriptor.bootInstanceId; + if (typeof larkAppId !== 'string' || !larkAppId.trim() + || !Number.isSafeInteger(ipcPort) || (ipcPort as number) < 1 || (ipcPort as number) > 65_535 + || typeof bootInstanceId !== 'string' || !bootInstanceId) { + throw failure( + operation, + `${target.name}/${target.pid} descriptor has no canonical app/port/boot identity`, + ); + } + const describedStart = descriptor.processStartIdentity; + if (typeof describedStart !== 'string' || !describedStart) { + throw failure( + operation, + `${target.name}/${target.pid} descriptor has no process-start identity`, + ); + } + const currentStart = runtime.readStartIdentity(target.pid); + if (!currentStart) { + if (!runtime.isAlive(target.pid)) continue; + throw failure( + operation, + `cannot revalidate process-start identity for ${target.name}/${target.pid}`, + ); + } + if (currentStart !== describedStart) { + throw failure( + operation, + `${target.name}/${target.pid} process-start identity does not match its descriptor`, + ); + } + authorized.push({ + ...target, + larkAppId: larkAppId.trim(), + ipcPort: ipcPort as number, + bootInstanceId, + processStartIdentity: describedStart, + }); + } + return authorized; +} diff --git a/src/cli/pm2-start-transaction.ts b/src/cli/pm2-start-transaction.ts new file mode 100644 index 000000000..9d6fbb0c5 --- /dev/null +++ b/src/cli/pm2-start-transaction.ts @@ -0,0 +1,311 @@ +import type { FleetProcessEntry } from './fleet-shutdown.js'; +import { DAEMON_GRACEFUL_EXIT_CODE } from '../core/supervisor-shutdown-protocol.js'; + +/** Preserve PM2's raw stop_exit_codes elements for exact policy validation. + * PM2 applies parseInt to string elements at exit time, so lossy numeric + * projection would hide restart-suppressing extras such as "0foo". */ +export function normalizeRawPm2StopExitCodes(value: unknown): unknown[] { + return Array.isArray(value) ? [...value] : [value]; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function assertUniqueConfiguredNames(names: string[]): void { + const unique = new Set(names); + if (unique.size !== names.length || names.some(name => !name.trim())) { + throw new Error('configured PM2 fleet names must be unique and non-empty'); + } +} + +function assertProjectionIdentities( + operation: string, + entries: FleetProcessEntry[], +): void { + const names = new Set(); + const ids = new Map(); + const positivePids = new Map(); + for (const entry of entries) { + if (names.has(entry.name)) { + throw new Error(`[${operation}] duplicate singleton PM2 row for ${entry.name}`); + } + names.add(entry.name); + if (!Number.isSafeInteger(entry.pmId) || (entry.pmId as number) < 0) { + throw new Error(`[${operation}] ${entry.name} has no canonical pm_id`); + } + const prior = ids.get(entry.pmId as number); + if (prior !== undefined) { + throw new Error( + `[${operation}] duplicate canonical pm_id ${entry.pmId} across ${prior} and ${entry.name}`, + ); + } + ids.set(entry.pmId as number, entry.name); + if (Number.isSafeInteger(entry.pid) && entry.pid > 1) { + const priorPidName = positivePids.get(entry.pid); + if (priorPidName !== undefined) { + throw new Error( + `[${operation}] duplicate positive pid ${entry.pid} across ${priorPidName} and ${entry.name}`, + ); + } + positivePids.set(entry.pid, entry.name); + } + } +} + +function assertNoUnexpectedRows( + operation: string, + entries: FleetProcessEntry[], + configuredNames: string[], +): void { + const configured = new Set(configuredNames); + const unexpected = entries.filter(entry => !configured.has(entry.name)); + if (unexpected.length > 0) { + throw new Error( + `[${operation}] unexpected PM2 core row(s): ${unexpected.map(entry => entry.name).join(', ')}`, + ); + } +} + +function isOnlineAndLive( + entry: FleetProcessEntry, + isAlive: (pid: number) => boolean, +): boolean { + return entry.online + && Number.isSafeInteger(entry.pid) + && entry.pid > 1 + && isAlive(entry.pid); +} + +/** The in-memory shutdown capability is insufficient when PM2 still owns an + * old registry policy. In particular `stop_exit_codes:[0]` suppresses restart + * after PM2 normalizes SIGKILL/OOM to exit_code 0. Require the exact daemon + * policy that makes only shutdown()'s reserved sentinel terminal. */ +export function assertDaemonPm2GracefulExitPolicy( + operation: string, + entries: FleetProcessEntry[], +): void { + const unsafe = entries.filter(entry => { + const codes = entry.stopExitCodes; + const exactSentinel = Array.isArray(codes) + && codes.length === 1 + && (codes[0] === DAEMON_GRACEFUL_EXIT_CODE + || codes[0] === String(DAEMON_GRACEFUL_EXIT_CODE)); + const restartEnabled = entry.autorestart === true || entry.autorestart === 'true'; + return !exactSentinel || !restartEnabled; + }); + if (unsafe.length > 0) { + throw new Error( + `[${operation}] daemon PM2 policy does not prove signal-death autorestart ` + + `(expected autorestart=true and stop_exit_codes=[${DAEMON_GRACEFUL_EXIT_CODE}]; unsafe: ` + + `${unsafe.map(entry => entry.name).join(', ')})`, + ); + } +} + +/** Require one exact, online, OS-live registry row for every configured core + * process and no stale/foreign core rows. This is the postcondition for every + * public fleet start surface. */ +export function assertConfiguredPm2FleetOnline( + operation: string, + entries: FleetProcessEntry[], + configuredNames: string[], + isAlive: (pid: number) => boolean, +): void { + assertUniqueConfiguredNames(configuredNames); + assertProjectionIdentities(operation, entries); + assertNoUnexpectedRows(operation, entries, configuredNames); + + const unavailable = configuredNames.filter(name => { + const row = entries.find(entry => entry.name === name); + return !row || !isOnlineAndLive(row, isAlive); + }); + if (unavailable.length > 0 || entries.length !== configuredNames.length) { + throw new Error( + `[${operation}] configured PM2 fleet is not fully online` + + (unavailable.length > 0 ? ` (unavailable: ${unavailable.join(', ')})` : ''), + ); + } +} + +/** PM2 `online` is published before a daemon's shutdown endpoint/handler-ready + * capability. A public start is complete only after both authorities agree. */ +export function assertConfiguredPm2FleetReady( + operation: string, + entries: TEntry[], + configuredNames: string[], + isAlive: (pid: number) => boolean, + assertDaemonCapabilities: (entries: TEntry[]) => void, +): void { + assertConfiguredPm2FleetOnline(operation, entries, configuredNames, isAlive); + assertDaemonCapabilities(entries); +} + +/** Capability scanners used by shutdown may legitimately omit a target that + * exited during their read. Start verification may not: require an exact + * attested PID set and recheck OS liveness after the capability scan. */ +export function assertExactAttestedDaemonSet( + operation: string, + daemonEntries: FleetProcessEntry[], + attestedPids: readonly number[], + isAlive: (pid: number) => boolean, +): void { + const expected = daemonEntries.map(entry => entry.pid).sort((a, b) => a - b); + const actual = [...attestedPids].sort((a, b) => a - b); + if (expected.length !== actual.length + || expected.some((pid, index) => pid !== actual[index])) { + throw new Error( + `[${operation}] handler-ready capability set is incomplete ` + + `(expected pids: ${expected.join(', ') || 'none'}; attested: ${actual.join(', ') || 'none'})`, + ); + } + const deadAfterAttestation = daemonEntries.filter(entry => !isAlive(entry.pid)); + if (deadAfterAttestation.length > 0) { + throw new Error( + `[${operation}] daemon exited after capability attestation: ` + + deadAfterAttestation.map(entry => `${entry.name}/${entry.pid}`).join(', '), + ); + } +} + +export type StartBotFleetAdmission = + | { state: 'already-online' } + | { state: 'start-eligible' } + | { state: 'fleet-down' }; + +/** `start-bot` is safe only for the append-one-bot case: either the entire + * configured fleet is already online, or exactly the requested bot row is + * absent while every other configured bot and the dashboard are online. */ +export function classifyStartBotFleetAdmission( + operation: string, + entries: FleetProcessEntry[], + configuredNames: string[], + targetName: string, + isAlive: (pid: number) => boolean, +): StartBotFleetAdmission { + assertUniqueConfiguredNames(configuredNames); + if (!configuredNames.includes(targetName)) { + throw new Error(`[${operation}] target ${targetName} is not configured`); + } + assertProjectionIdentities(operation, entries); + assertNoUnexpectedRows(operation, entries, configuredNames); + if (entries.length === 0) return { state: 'fleet-down' }; + + const targetRows = entries.filter(entry => entry.name === targetName); + const unavailablePeers = configuredNames + .filter(name => name !== targetName) + .filter(name => { + const row = entries.find(entry => entry.name === name); + return !row || !isOnlineAndLive(row, isAlive); + }); + if (unavailablePeers.length > 0) { + throw new Error( + `[${operation}] refusing single-bot start because configured peer(s) are unavailable: ` + + unavailablePeers.join(', '), + ); + } + if (targetRows.length === 0) { + if (entries.length !== configuredNames.length - 1) { + throw new Error(`[${operation}] fleet is not the exact one-missing-bot shape`); + } + return { state: 'start-eligible' }; + } + const target = targetRows[0]!; + if (!isOnlineAndLive(target, isAlive)) { + throw new Error( + `[${operation}] refusing start-bot for existing non-live/transitional row ${targetName}`, + ); + } + if (entries.length !== configuredNames.length) { + throw new Error(`[${operation}] fleet is not the exact fully-configured shape`); + } + return { state: 'already-online' }; +} + +export interface Pm2StartTransactionRuntime { + start(timeoutMs: number): void; + /** Must obtain a new projection and validate the complete expected state. */ + verifyFresh(timeoutMs: number): TProjection; + /** Must independently re-read authority before compensating partial launch. */ + rollback(): void; +} + +/** Run one bounded PM2 launch and make the fresh fleet projection—not the CLI + * exit code—the success authority. Any incomplete/unverified launch is + * compensated before the error escapes. */ +export function runBoundedPm2StartTransaction( + operation: string, + startTimeoutMs: number, + verifyTimeoutMs: number, + runtime: Pm2StartTransactionRuntime, +): TProjection { + if (!Number.isFinite(startTimeoutMs) || startTimeoutMs <= 0 + || !Number.isFinite(verifyTimeoutMs) || verifyTimeoutMs <= 0) { + throw new Error(`[${operation}] PM2 start/verification budgets must be positive`); + } + + let startFailure: unknown; + try { runtime.start(Math.floor(startTimeoutMs)); } + catch (error) { startFailure = error; } + + try { + // A timed-out client can race with a God RPC that already completed. A + // complete fresh projection is therefore stronger evidence than the + // launcher exit code and is safe to accept. + return runtime.verifyFresh(Math.floor(verifyTimeoutMs)); + } catch (verifyFailure) { + let rollbackFailure: unknown; + try { runtime.rollback(); } + catch (error) { rollbackFailure = error; } + throw new Error( + `[${operation}] PM2 start transaction did not reach a verified complete fleet` + + (startFailure ? ` (start: ${errorText(startFailure)})` : '') + + ` (verify: ${errorText(verifyFailure)})` + + (rollbackFailure + ? `; partial-launch rollback failed: ${errorText(rollbackFailure)}` + : '; partial launch was rolled back'), + ); + } +} + +export interface LatePm2StartRollbackRuntime { + now(): number; + sleep(ms: number): void; + /** Re-read authority and compensate anything currently published. Return + * true only when this observation exactly matches the pre-start state. */ + reconcileOnce(): boolean; +} + +/** A killed/timed-out PM2 client does not cancel work already queued in God. + * Therefore an empty first rollback projection is not success: require one + * continuous restored window, resetting it whenever a late row appears. */ +export function reconcileLatePm2StartPublication( + operation: string, + settleMs: number, + timeoutMs: number, + runtime: LatePm2StartRollbackRuntime, +): void { + if (!Number.isFinite(settleMs) || settleMs < 0 + || !Number.isFinite(timeoutMs) || timeoutMs <= settleMs) { + throw new Error(`[${operation}] invalid late-publication rollback budgets`); + } + const deadline = runtime.now() + Math.floor(timeoutMs); + let restoredSince: number | undefined; + while (runtime.now() < deadline) { + const restored = runtime.reconcileOnce(); + const now = runtime.now(); + if (now >= deadline) break; + if (!restored) { + restoredSince = undefined; + runtime.sleep(Math.min(100, Math.max(1, deadline - now))); + continue; + } + restoredSince ??= now; + const settledFor = now - restoredSince; + if (settledFor >= settleMs) return; + runtime.sleep(Math.min(100, settleMs - settledFor, Math.max(1, deadline - now))); + } + throw new Error( + `[${operation}] partial-launch rollback remains uncertain after the late-publication settle window`, + ); +} diff --git a/src/cli/supervisor-shutdown-client.ts b/src/cli/supervisor-shutdown-client.ts new file mode 100644 index 000000000..6b22775ee --- /dev/null +++ b/src/cli/supervisor-shutdown-client.ts @@ -0,0 +1,180 @@ +import { spawnSync } from 'node:child_process'; +import { daemonIpcAuthHeaders } from '../core/daemon-ipc-auth.js'; +import { readSupervisorProcessStartIdentity } from '../core/process-start-identity.js'; +import { SUPERVISOR_SHUTDOWN_ROUTE } from '../core/supervisor-shutdown-ipc.js'; +import type { AttestedPm2DaemonShutdownTarget } from './pm2-shutdown-capability.js'; + +export interface SupervisorShutdownHttpResult { + status?: number; + bodyRaw?: string; + error?: string; +} + +export interface SupervisorShutdownHttpInput { + port: number; + path: string; + headers: Record; + bodyRaw: string; + timeoutMs: number; +} + +export interface SupervisorShutdownClientRuntime { + readStartIdentity(pid: number): string | undefined; + postMany(inputs: SupervisorShutdownHttpInput[]): SupervisorShutdownHttpResult[]; +} + +const SYNC_HTTP_SCRIPT = String.raw` +const http = require('node:http'); +const inputs = JSON.parse(process.env.BOTMUX_SUPERVISOR_HTTP_REQUESTS); +function post(input) { + return new Promise((resolve) => { + let settled = false; + const finish = (value) => { if (!settled) { settled = true; resolve(value); } }; + const req = http.request({ + host: '127.0.0.1', port: input.port, path: input.path, method: 'POST', headers: input.headers, + }, (res) => { + const chunks = []; + res.on('data', chunk => chunks.push(Buffer.from(chunk))); + res.on('end', () => finish({ status: res.statusCode || 0, bodyRaw: Buffer.concat(chunks).toString('utf8') })); + }); + req.setTimeout(input.timeoutMs, () => req.destroy(new Error('supervisor shutdown request timed out'))); + req.on('error', err => finish({ error: String(err && err.message || err) })); + req.end(input.bodyRaw); + }); +} +Promise.all(inputs.map(post)).then(results => process.stdout.write(JSON.stringify(results))); +`; + +const defaultRuntime: SupervisorShutdownClientRuntime = { + readStartIdentity: readSupervisorProcessStartIdentity, + postMany(inputs): SupervisorShutdownHttpResult[] { + if (inputs.length === 0) return []; + const timeoutMs = Math.max(...inputs.map(input => input.timeoutMs)); + const result = spawnSync(process.execPath, ['-e', SYNC_HTTP_SCRIPT], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: timeoutMs + 1_000, + env: { + ...process.env, + BOTMUX_SUPERVISOR_HTTP_REQUESTS: JSON.stringify(inputs), + }, + }); + if (result.status !== 0 || result.error) { + throw new Error( + result.error?.message + ?? String(result.stderr || `synchronous HTTP helper exited ${result.status}`).trim(), + ); + } + try { + const parsed = JSON.parse(String(result.stdout)) as SupervisorShutdownHttpResult[]; + if (!Array.isArray(parsed) || parsed.length !== inputs.length) { + throw new Error('invalid helper result'); + } + return parsed; + } catch (error) { + throw new Error(`invalid supervisor shutdown response transport: ${String(error)}`); + } + }, +}; + +export interface SupervisorShutdownAttempt { + target: AttestedPm2DaemonShutdownTarget; + ok: boolean; + error?: string; +} + +function requestInput( + target: AttestedPm2DaemonShutdownTarget, + secret: string, +): SupervisorShutdownHttpInput { + const bodyRaw = JSON.stringify({ + larkAppId: target.larkAppId, + bootInstanceId: target.bootInstanceId, + processStartIdentity: target.processStartIdentity, + }); + const headers = daemonIpcAuthHeaders({ + secret, + port: target.ipcPort, + method: 'POST', + path: SUPERVISOR_SHUTDOWN_ROUTE, + headers: { 'content-type': 'application/json' }, + }); + return { + port: target.ipcPort, + path: SUPERVISOR_SHUTDOWN_ROUTE, + headers: Object.fromEntries(headers.entries()), + bodyRaw, + timeoutMs: 4_000, + }; +} + +function exactAck( + target: AttestedPm2DaemonShutdownTarget, + response: SupervisorShutdownHttpResult, +): boolean { + if (response.status !== 202 || typeof response.bodyRaw !== 'string') return false; + let body: Record | undefined; + try { body = JSON.parse(response.bodyRaw) as Record; } + catch { return false; } + return body.ok === true + && body.accepted === true + && body.larkAppId === target.larkAppId + && body.bootInstanceId === target.bootInstanceId + && body.processStartIdentity === target.processStartIdentity; +} + +/** Dispatch one bounded helper containing concurrent HTTP requests for the + * whole initial fleet. One hung endpoint cannot delay request delivery to its + * peers or consume N times the fleet budget. */ +export function requestAttestedDaemonShutdownBatch( + targets: readonly AttestedPm2DaemonShutdownTarget[], + secret: string, + runtime: SupervisorShutdownClientRuntime = defaultRuntime, +): SupervisorShutdownAttempt[] { + const attempts = targets.map(target => ({ target, ok: false } as SupervisorShutdownAttempt)); + const eligible: Array<{ index: number; target: AttestedPm2DaemonShutdownTarget }> = []; + for (const [index, target] of targets.entries()) { + const currentStart = runtime.readStartIdentity(target.pid); + if (!currentStart) { + attempts[index]!.error = `daemon ${target.name}/${target.pid} exited before shutdown request`; + } else if (currentStart !== target.processStartIdentity) { + attempts[index]!.error = `daemon ${target.name}/${target.pid} process generation changed before shutdown request`; + } else { + eligible.push({ index, target }); + } + } + if (eligible.length === 0) return attempts; + + let responses: SupervisorShutdownHttpResult[]; + try { + responses = runtime.postMany(eligible.map(({ target }) => requestInput(target, secret))); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + for (const { index } of eligible) attempts[index]!.error = message; + return attempts; + } + for (let i = 0; i < eligible.length; i++) { + const { index, target } = eligible[i]!; + const response = responses[i] ?? { error: 'missing helper response' }; + if (exactAck(target, response)) { + attempts[index] = { target, ok: true }; + } else { + attempts[index]!.error = response.error + ?? `daemon ${target.name}/${target.pid} rejected exact supervisor shutdown ` + + `(status ${response.status ?? 'transport-error'})`; + } + } + return attempts; +} + +/** Send a trusted-host request to the exact daemon boot/birth generation. + * The receiving process performs the decisive in-process comparison; a + * successor inheriting the port rejects rather than inheriting authority. */ +export function requestAttestedDaemonShutdown( + target: AttestedPm2DaemonShutdownTarget, + secret: string, + runtime: SupervisorShutdownClientRuntime = defaultRuntime, +): void { + const attempt = requestAttestedDaemonShutdownBatch([target], secret, runtime)[0]!; + if (!attempt.ok) throw new Error(attempt.error ?? `daemon ${target.name}/${target.pid} shutdown failed`); +} diff --git a/src/core/dashboard-ipc-server.ts b/src/core/dashboard-ipc-server.ts index 80df951e3..a923fe0cb 100644 --- a/src/core/dashboard-ipc-server.ts +++ b/src/core/dashboard-ipc-server.ts @@ -92,6 +92,11 @@ import { validateTriggerRequest, type TriggerResponse } from '../services/trigge import { resolveCliSelection, selectionKeyForBot } from '../setup/cli-selection.js'; import { checkCliAvailability } from '../setup/cli-availability.js'; import { enrichHistorySenders, type HistoryBotInfo } from '../dashboard/history-senders.js'; +import { + SUPERVISOR_SHUTDOWN_ROUTE, + isExactSupervisorShutdownRequest, + type SupervisorShutdownIdentity, +} from './supervisor-shutdown-ipc.js'; let exactChatGrantHandler: typeof applyExactChatGrantRequest = applyExactChatGrantRequest; /** Test seam: replace the exact-grant service without touching live Feishu/config state. */ @@ -118,6 +123,16 @@ let botAvatarChanger: ((image: Buffer) => Promise) | null = nu export function setBotAvatarChanger(fn: ((image: Buffer) => Promise) | null): void { botAvatarChanger = fn; } + +type SupervisorShutdownRegistration = SupervisorShutdownIdentity & { + shutdown: () => Promise; +}; +let supervisorShutdownRegistration: SupervisorShutdownRegistration | null = null; +export function setSupervisorShutdownHandler( + registration: SupervisorShutdownRegistration | null, +): void { + supervisorShutdownRegistration = registration; +} import { composeRowFromActive, composeRowFromClosed, @@ -192,6 +207,39 @@ export function jsonRes(res: ServerResponse, status: number, body: unknown): voi res.end(JSON.stringify(body)); } +ipcRoute('POST', SUPERVISOR_SHUTDOWN_ROUTE, async (req, res) => { + // The production server-wide HMAC gate records trusted requests here. Keep + // an explicit route-local check: shutdown is never a bare loopback API. + if (!isTrustedHostIpcRequest(req)) { + return jsonRes(res, 403, { ok: false, error: 'supervisor_shutdown_unauthorized' }); + } + const registration = supervisorShutdownRegistration; + if (!registration) { + return jsonRes(res, 503, { ok: false, error: 'supervisor_shutdown_not_ready' }); + } + let body: unknown; + try { body = await readJsonBody(req); } + catch { return jsonRes(res, 400, { ok: false, error: 'invalid_json' }); } + if (!isExactSupervisorShutdownRequest(registration, body)) { + return jsonRes(res, 409, { ok: false, error: 'supervisor_shutdown_generation_mismatch' }); + } + // ACK means this exact in-memory generation accepted the request; the CLI + // still proves OS/PM2 quiescence. Start after flushing the ACK so a long Riff + // drain cannot turn a valid request into an ambiguous transport timeout. + jsonRes(res, 202, { + ok: true, + accepted: true, + larkAppId: registration.larkAppId, + bootInstanceId: registration.bootInstanceId, + processStartIdentity: registration.processStartIdentity, + }); + setImmediate(() => { + void registration.shutdown().catch(error => { + logger.error(`supervisor shutdown failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }); +}); + export class JsonBodyTooLargeError extends Error { constructor(readonly maxBytes: number) { super(`JSON request body exceeds ${maxBytes} bytes`); diff --git a/src/core/process-start-identity.ts b/src/core/process-start-identity.ts new file mode 100644 index 000000000..161080755 --- /dev/null +++ b/src/core/process-start-identity.ts @@ -0,0 +1,52 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; + +function systemPsBin(): string | undefined { + for (const candidate of ['/usr/bin/ps', '/bin/ps']) { + if (existsSync(candidate)) return candidate; + } + return undefined; +} + +/** Stable process-birth identity used by supervisor mutation authority. */ +export function readSupervisorProcessStartIdentity(pid: number): string | undefined { + if (!Number.isSafeInteger(pid) || pid <= 1) return undefined; + if (process.platform === 'linux') { + try { + const raw = readFileSync(`/proc/${pid}/stat`, 'utf8'); + const closeParen = raw.lastIndexOf(')'); + if (closeParen >= 0) { + const fields = raw.slice(closeParen + 2).trim().split(/\s+/); + if (fields[19]) return fields[19]; + } + } catch { /* exited or unreadable */ } + return undefined; + } + if (process.platform === 'win32') { + try { + const value = execFileSync('powershell.exe', [ + '-NoProfile', + '-NonInteractive', + '-Command', + `$p = Get-CimInstance Win32_Process -Filter \"ProcessId = ${pid}\"; ` + + 'if ($p) { $p.CreationDate.ToUniversalTime().Ticks }', + ], { + encoding: 'utf8', + timeout: 2_000, + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + return value || undefined; + } catch { return undefined; } + } + const ps = systemPsBin(); + if (!ps) return undefined; + try { + const value = execFileSync(ps, ['-o', 'lstart=', '-p', String(pid)], { + encoding: 'utf8', + timeout: 2_000, + stdio: ['ignore', 'pipe', 'ignore'], + env: { PATH: '/usr/bin:/bin', LANG: 'C' }, + }).trim(); + return value || undefined; + } catch { return undefined; } +} diff --git a/src/core/restart-report.ts b/src/core/restart-report.ts index e7c7a7ce7..748ef5fc0 100644 --- a/src/core/restart-report.ts +++ b/src/core/restart-report.ts @@ -8,7 +8,7 @@ */ import { githubAuthHeaders, type GithubAuthResolveOptions } from './github-auth.js'; import type { RestartKind } from '../services/restart-intent-store.js'; -import { consumeRestartIntent } from '../services/restart-intent-store.js'; +import { claimRestartIntentForReport } from '../services/restart-intent-store.js'; import { countActiveSessionsOnDisk } from '../services/session-store.js'; import { botmuxVersion } from '../utils/install-info.js'; import { t, localeForBot, type Locale } from '../i18n/index.js'; @@ -93,6 +93,8 @@ export interface RestartReportWiring { githubAuth?: GithubAuthResolveOptions; now?: number; log?: (msg: string) => void; + wait?: (ms: number) => Promise; + preparedCommitWaitMs?: number; } /** @@ -103,8 +105,19 @@ export interface RestartReportWiring { */ export async function sendRestartReportIfPending(w: RestartReportWiring): Promise { const log = w.log ?? (() => {}); - const intent = consumeRestartIntent(w.now ?? Date.now()); - if (!intent) return; // no breadcrumb → crash/reboot → stay silent + const now = () => w.now ?? Date.now(); + const wait = w.wait ?? (ms => new Promise(resolve => setTimeout(resolve, ms))); + let claim = claimRestartIntentForReport(now()); + let remainingPreparedWaitMs = Math.max(0, w.preparedCommitWaitMs ?? 45_000); + const pollMs = 100; + while (claim.state === 'prepared' && remainingPreparedWaitMs > 0) { + const delayMs = Math.min(pollMs, remainingPreparedWaitMs); + await wait(delayMs); + remainingPreparedWaitMs -= delayMs; + claim = claimRestartIntentForReport(now()); + } + if (claim.state !== 'claimed') return; + const intent = claim.intent; if (!w.ownerOpenId) { log('restart-report: no owner configured — skipping DM'); return; } const locale = localeForBot(w.primaryLarkAppId); diff --git a/src/core/shutdown-budgets.ts b/src/core/shutdown-budgets.ts index 3501ae0ef..33418e82b 100644 --- a/src/core/shutdown-budgets.ts +++ b/src/core/shutdown-budgets.ts @@ -1,7 +1,8 @@ /** - * Graceful-shutdown budgets are ordered so a bounded Riff create/follow-up can - * drain before the daemon persists its final lineage and asks workers to exit. - * Shutdown never cancels accepted Riff work as a fallback. + * Graceful-shutdown budgets are shared with the CLI/PM2 supervisor. Keep the + * ordering monotonic: a Riff create/follow-up fetch is bounded at 10s, + * admission restoration after a refused prepare at 11s, and ordinary worker + * exit at 3s. Shutdown never cancels accepted Riff work as a fallback. */ export const RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS = 12_000; @@ -17,7 +18,7 @@ export const BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS = 1_000; export const RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS = 1_000; export const RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS = 1_000; -/** Scheduling/logging slack inside the graceful daemon shutdown budget. */ +/** Scheduling/logging slack inside the supervisor-visible daemon budget. */ export const DAEMON_SHUTDOWN_OVERHEAD_MS = 2_000; export const DAEMON_WORKER_EXIT_GRACE_MS = 3_000; export const DAEMON_SHUTDOWN_MAX_MS = @@ -27,7 +28,25 @@ export const DAEMON_SHUTDOWN_MAX_MS = + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS + Math.max(RIFF_ADMISSION_RESTORE_TIMEOUT_MS, DAEMON_WORKER_EXIT_GRACE_MS) + DAEMON_SHUTDOWN_OVERHEAD_MS; +export const PM2_DAEMON_KILL_TIMEOUT_MS = 29_000; +export const PM2_DAEMON_RESTART_DELAY_MS = 3_000; +/** A full restart-delay plus projection jitter. The fleet helper must observe + * this quiet window after every signalled generation exits. */ +export const FLEET_SUCCESSOR_SETTLE_MS = PM2_DAEMON_RESTART_DELAY_MS + 500; +export const FLEET_DAEMON_EXIT_WAIT_MS = 35_000; +if (PM2_DAEMON_KILL_TIMEOUT_MS <= DAEMON_SHUTDOWN_MAX_MS) { + throw new Error('PM2 daemon kill timeout must exceed the complete daemon shutdown budget'); +} if (DAEMON_SHUTDOWN_MAX_MS > 28_000) { throw new Error('complete daemon shutdown budget must remain at or below 28 seconds'); } +if (FLEET_DAEMON_EXIT_WAIT_MS <= PM2_DAEMON_KILL_TIMEOUT_MS) { + throw new Error('fleet restart wait must exceed the PM2 daemon kill timeout'); +} +if (FLEET_DAEMON_EXIT_WAIT_MS <= DAEMON_SHUTDOWN_MAX_MS + FLEET_SUCCESSOR_SETTLE_MS) { + throw new Error('fleet restart wait must cover daemon shutdown plus successor quiet window'); +} +if (FLEET_DAEMON_EXIT_WAIT_MS <= PM2_DAEMON_KILL_TIMEOUT_MS + FLEET_SUCCESSOR_SETTLE_MS) { + throw new Error('fleet restart wait must cover PM2 kill timeout plus successor quiet window'); +} diff --git a/src/core/supervisor-shutdown-ipc.ts b/src/core/supervisor-shutdown-ipc.ts new file mode 100644 index 000000000..d00dff066 --- /dev/null +++ b/src/core/supervisor-shutdown-ipc.ts @@ -0,0 +1,20 @@ +export const SUPERVISOR_SHUTDOWN_ROUTE = '/__supervisor-ipc/v1/shutdown'; + +export interface SupervisorShutdownIdentity { + larkAppId: string; + bootInstanceId: string; + processStartIdentity: string; +} + +export interface SupervisorShutdownRequest extends SupervisorShutdownIdentity {} + +export function isExactSupervisorShutdownRequest( + identity: SupervisorShutdownIdentity, + value: unknown, +): value is SupervisorShutdownRequest { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const record = value as Record; + return record.larkAppId === identity.larkAppId + && record.bootInstanceId === identity.bootInstanceId + && record.processStartIdentity === identity.processStartIdentity; +} diff --git a/src/core/supervisor-shutdown-protocol.ts b/src/core/supervisor-shutdown-protocol.ts new file mode 100644 index 000000000..625e88771 --- /dev/null +++ b/src/core/supervisor-shutdown-protocol.ts @@ -0,0 +1,16 @@ +/** + * Descriptor capability required before a supervisor may signal a live daemon. + * Bump this exact value whenever shutdown safety depends on a protocol that an + * already-running older daemon does not implement. + */ +export const SUPERVISOR_SHUTDOWN_PROTOCOL = 'riff-fleet-prepare-persist-commit-exit42-v2' as const; + +/** + * PM2 normalizes signal-only child exits to code 0 (`code || 0`) before it + * evaluates `stop_exit_codes`. Zero therefore cannot prove that the daemon + * completed the protocol above: SIGKILL/OOM may look identical. Only the + * successful end of daemon.shutdown() exits with this reserved non-zero code. + */ +export const DAEMON_GRACEFUL_EXIT_CODE = 42 as const; + +export type SupervisorShutdownProtocol = typeof SUPERVISOR_SHUTDOWN_PROTOCOL; diff --git a/src/daemon.ts b/src/daemon.ts index 40723e126..2ab9042b2 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -28,6 +28,12 @@ import { stopCliRuntimeUpdateMonitor, } from './core/cli-runtime-update.js'; import { sendRestartReportIfPending } from './core/restart-report.js'; +import { + DAEMON_GRACEFUL_EXIT_CODE, + SUPERVISOR_SHUTDOWN_PROTOCOL, + type SupervisorShutdownProtocol, +} from './core/supervisor-shutdown-protocol.js'; +import { readSupervisorProcessStartIdentity } from './core/process-start-identity.js'; import { statSync } from 'node:fs'; import { addReaction, getChatMode, getChatNameAndMode, getMessageChatId, listChatMemberOpenIds, MessageWithdrawnError, replyMessage, resolveAllowedUsersWithMap, sendMessage, sendUserMessage, updateMessage } from './im/lark/client.js'; import { resolveGroupJoinPrompt, waitForAllowedUserInChat } from './core/auto-start.js'; @@ -108,7 +114,7 @@ import { getDaemonBootId, type WorkerSessionReplyOptions, } from './core/worker-pool.js'; -import { AbortDeadlineError, hasExactSafeJsonKeys, ipcRoute, isTrustedHostIpcRequest, JsonBodyTooLargeError, jsonRes, readJsonBody, runWithAbortDeadline, setBotName, setLarkAppId, startIpcServer, setBotRenamer, setBotAvatarChanger } from './core/dashboard-ipc-server.js'; +import { AbortDeadlineError, hasExactSafeJsonKeys, ipcRoute, isTrustedHostIpcRequest, JsonBodyTooLargeError, jsonRes, readJsonBody, runWithAbortDeadline, setBotName, setLarkAppId, startIpcServer, setBotRenamer, setBotAvatarChanger, setSupervisorShutdownHandler } from './core/dashboard-ipc-server.js'; import { setDeviceIsolationDaemonIdentity } from './core/device-isolation-daemon.js'; import { cancelSessionReadyAck, @@ -2951,11 +2957,16 @@ interface DaemonDescriptor { botIndex: number; ipcPort: number; pid: number; + /** Kernel/OS process-birth identity; prevents fresh stale PID reuse. */ + processStartIdentity: string; startedAt: number; /** Public, random audience that changes on every daemon process start. */ bootInstanceId: string; /** Full-envelope Workflow mutation protocol supported by this process. */ workflowIpcProtocol: 'v1'; + /** Exact supervisor protocol this in-memory daemon will execute on signal. + * Absent until the SIGTERM/SIGINT handlers and all captured state are ready. */ + supervisorShutdownProtocol?: SupervisorShutdownProtocol; lastHeartbeat: number; /** * Resolved open_ids from this bot's allowedUsers config (post-email @@ -16944,6 +16955,10 @@ export async function startDaemon(botIndex?: number): Promise { // agent-facing, live-origin-gated endpoints. Internal control endpoints use // a separate daemon-to-daemon credential and never trust this port marker. process.env.BOTMUX_DAEMON_IPC_PORT = String(ipcPort); + const daemonProcessStartIdentity = readSupervisorProcessStartIdentity(process.pid); + if (!daemonProcessStartIdentity) { + throw new Error('cannot bind daemon descriptor to a process-start identity'); + } const desc: DaemonDescriptor = { larkAppId: cfg.larkAppId, botName: cfg.displayName ?? cfg.larkAppId, @@ -16951,6 +16966,7 @@ export async function startDaemon(botIndex?: number): Promise { botIndex: idx, ipcPort, pid: process.pid, + processStartIdentity: daemonProcessStartIdentity, startedAt: Date.now(), bootInstanceId: generateWorkflowDaemonBootInstanceId(), workflowIpcProtocol: 'v1', @@ -17544,7 +17560,8 @@ export async function startDaemon(botIndex?: number): Promise { // Ordinary workers then receive SIGTERM / close IPC and the daemon waits up // to DAEMON_WORKER_EXIT_GRACE_MS for them to exit // before sending SIGKILL to stragglers. Without the wait, daemon - // `process.exit(0)` races worker signal delivery — and any worker whose + // `process.exit(DAEMON_GRACEFUL_EXIT_CODE)` races worker signal delivery — + // and any worker whose // main thread is in a sync code path (e.g. the bridge fingerprint scan // bug fixed in v2.9.2) loses the signal and survives as a ppid=1 orphan // forever (we'd accumulated 841 such orphans across daemon restarts, @@ -17789,7 +17806,10 @@ export async function startDaemon(botIndex?: number): Promise { flushIdentityCacheSync(); removePidFile(); - process.exit(0); + // This reserved non-zero code is the only PM2 stop_exit_codes proof that + // the authenticated prepare/persist/commit shutdown reached its end. + // PM2 maps abrupt signal death to 0, which must remain autorestartable. + process.exit(DAEMON_GRACEFUL_EXIT_CODE); }, ); if (!mutationResult.acquired) { @@ -17803,10 +17823,27 @@ export async function startDaemon(botIndex?: number): Promise { process.on('SIGTERM', () => { shutdown().catch(err => { logger.error(`shutdown failed: ${err?.message ?? err}`); process.exit(1); }); }); process.on('SIGINT', () => { shutdown().catch(err => { logger.error(`shutdown failed: ${err?.message ?? err}`); process.exit(1); }); }); + // Capability publication is the final startup commit for supervisor-driven + // shutdown. The early descriptor intentionally lacks it: a new CLI that + // observes this daemon before both handlers/state closures exist must refuse + // to signal. Atomic rewrite makes the capability visible only afterward. + if (readSupervisorProcessStartIdentity(process.pid) !== desc.processStartIdentity) { + throw new Error('daemon process-start identity changed before shutdown capability commit'); + } + setSupervisorShutdownHandler({ + larkAppId: cfg.larkAppId, + bootInstanceId: desc.bootInstanceId, + processStartIdentity: desc.processStartIdentity, + shutdown, + }); + desc.supervisorShutdownProtocol = SUPERVISOR_SHUTDOWN_PROTOCOL; + desc.lastHeartbeat = Date.now(); + writeDaemonDescriptor(desc); // Best-effort cleanup on plain `exit` (e.g. uncaught fatal). No worker // shutdown here since the process is already on its way out — just remove // the descriptor so the dashboard doesn't see a phantom daemon. process.on('exit', () => { + setSupervisorShutdownHandler(null); clearInterval(descriptorHeartbeat); clearInterval(idleWorkerSweepTimer); clearInterval(docCommentPollTimer); diff --git a/src/services/restart-intent-store.ts b/src/services/restart-intent-store.ts index d453cf632..5ee6f283d 100644 --- a/src/services/restart-intent-store.ts +++ b/src/services/restart-intent-store.ts @@ -14,10 +14,11 @@ import { randomBytes } from 'node:crypto'; import { join } from 'node:path'; import { config } from '../config.js'; import { readProcessStartIdentity } from '../core/session-marker.js'; +import { withFileLockSync } from '../utils/file-lock.js'; export type RestartKind = 'manual' | 'update' | 'rollback'; -export interface RestartIntent { +export interface RestartIntentPayload { kind: RestartKind; /** Present for an update or rollback: the version delta to report. */ oldVersion?: string; @@ -26,6 +27,17 @@ export interface RestartIntent { at: string; } +export interface RestartIntent extends RestartIntentPayload { + attemptId?: string; + attemptState?: 'prepared' | 'committed' | 'aborted'; + deferredIntent?: RestartIntentPayload; +} + +export type RestartIntentReportClaim = + | { state: 'claimed'; intent: RestartIntent } + | { state: 'prepared' } + | { state: 'absent' }; + const FILE = 'restart-intent.json'; const LEASE_FILE = 'restart-lease.json'; @@ -39,7 +51,7 @@ export function restartIntentPathIn(dir: string): string { return join(dir, FILE); } -export function writeRestartIntentTo(dir: string, intent: RestartIntent): void { +function writeRestartIntentUnlocked(dir: string, intent: RestartIntent): void { if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); const path = restartIntentPathIn(dir); const tmp = `${path}.${process.pid}.tmp`; @@ -48,7 +60,37 @@ export function writeRestartIntentTo(dir: string, intent: RestartIntent): void { } export function clearRestartIntentTo(dir: string): void { - try { rmSync(restartIntentPathIn(dir)); } catch { /* absent / best-effort */ } + if (!existsSync(dir)) return; + withFileLockSync(restartIntentPathIn(dir), () => { + try { rmSync(restartIntentPathIn(dir)); } catch { /* absent / best-effort */ } + }); +} + +function payloadOf(intent: RestartIntent): RestartIntentPayload { + return { + kind: intent.kind, + at: intent.at, + ...(intent.oldVersion !== undefined ? { oldVersion: intent.oldVersion } : {}), + ...(intent.newVersion !== undefined ? { newVersion: intent.newVersion } : {}), + }; +} + +export function writeRestartIntentTo(dir: string, intent: RestartIntent): void { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + withFileLockSync(restartIntentPathIn(dir), () => { + const current = readRaw(dir); + const intentAt = Date.parse(intent.at); + const writerNow = Number.isFinite(intentAt) ? intentAt : Date.now(); + if ((current?.attemptState === 'prepared' || current?.attemptState === 'aborted') + && isFresh(current, writerNow)) { + writeRestartIntentUnlocked(dir, { + ...current, + deferredIntent: payloadOf(intent), + }); + return; + } + writeRestartIntentUnlocked(dir, payloadOf(intent)); + }); } function readRaw(dir: string): RestartIntent | null { @@ -152,22 +194,117 @@ export function clearRestartLeaseTo(dir: string, id: string): void { * it fires at most once and never lingers into a later restart. Returns the * intent only when it is fresh. */ export function consumeRestartIntentTo(dir: string, nowMs: number): RestartIntent | null { - const intent = readRaw(dir); - const path = restartIntentPathIn(dir); - if (existsSync(path)) { - try { rmSync(path); } catch { /* best-effort */ } - } - if (!intent) return null; - return isFresh(intent, nowMs) ? intent : null; + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + return withFileLockSync(restartIntentPathIn(dir), () => { + const intent = readRaw(dir); + const path = restartIntentPathIn(dir); + if ((intent?.attemptState === 'prepared' || intent?.attemptState === 'aborted') + && isFresh(intent, nowMs)) { + return null; + } + if (existsSync(path)) { + try { rmSync(path); } catch { /* best-effort */ } + } + if (!intent) return null; + return isFresh(intent, nowMs) ? intent : null; + }); +} + +export function claimRestartIntentForReportTo( + dir: string, + nowMs: number, +): RestartIntentReportClaim { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + return withFileLockSync(restartIntentPathIn(dir), () => { + const intent = readRaw(dir); + const path = restartIntentPathIn(dir); + if (!intent || !isFresh(intent, nowMs)) { + if (existsSync(path)) { + try { rmSync(path); } catch { /* best-effort stale/corrupt cleanup */ } + } + return { state: 'absent' }; + } + if (intent.attemptState === 'prepared') return { state: 'prepared' }; + if (intent.attemptState === 'aborted') return { state: 'absent' }; + if (existsSync(path)) { + try { rmSync(path); } + catch { return { state: 'absent' }; } + } + return { state: 'claimed', intent }; + }); +} + +export function hasPreparedRestartIntentTo(dir: string, nowMs: number): boolean { + if (!existsSync(dir)) return false; + return withFileLockSync(restartIntentPathIn(dir), () => { + const intent = readRaw(dir); + if (intent?.attemptState !== 'prepared') return false; + if (isFresh(intent, nowMs)) return true; + try { rmSync(restartIntentPathIn(dir)); } catch { /* best-effort stale cleanup */ } + return false; + }); } /** Write a `manual` breadcrumb only when no *fresh* breadcrumb already exists — * so a maintenance-written `update` breadcrumb is not clobbered * by the `botmux restart` it spawns. */ export function writeManualIntentIfAbsentTo(dir: string, nowMs: number, atIso: string): void { - const existing = readRaw(dir); - if (existing && isFresh(existing, nowMs)) return; - writeRestartIntentTo(dir, { kind: 'manual', at: atIso }); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + withFileLockSync(restartIntentPathIn(dir), () => { + const existing = readRaw(dir); + if (existing && isFresh(existing, nowMs)) return; + writeRestartIntentUnlocked(dir, { kind: 'manual', at: atIso }); + }); +} + +export function writeRestartAttemptIntentTo( + dir: string, + preferred: RestartIntent, + nowMs: number, + attemptId: string, +): RestartIntent { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + return withFileLockSync(restartIntentPathIn(dir), () => { + const existing = readRaw(dir); + const selected = existing && isFresh(existing, nowMs) + ? ((existing.attemptState === 'prepared' || existing.attemptState === 'aborted') + && existing.deferredIntent + ? existing.deferredIntent + : payloadOf(existing)) + : payloadOf(preferred); + const written: RestartIntent = { ...selected, attemptId, attemptState: 'prepared' }; + writeRestartIntentUnlocked(dir, written); + return written; + }); +} + +export function commitRestartIntentAttemptTo(dir: string, attemptId: string): boolean { + if (!existsSync(dir)) return false; + return withFileLockSync(restartIntentPathIn(dir), () => { + const current = readRaw(dir); + if (current?.attemptId !== attemptId || current.attemptState !== 'prepared') return false; + const selected = current.deferredIntent ?? payloadOf(current); + writeRestartIntentUnlocked(dir, { + ...selected, + attemptId, + attemptState: 'committed', + }); + return true; + }); +} + +export function removeRestartIntentAttemptTo(dir: string, attemptId: string): boolean { + if (!existsSync(dir)) return false; + return withFileLockSync(restartIntentPathIn(dir), () => { + const current = readRaw(dir); + if (current?.attemptId !== attemptId) return false; + writeRestartIntentUnlocked(dir, { + ...(current.deferredIntent ?? payloadOf(current)), + attemptId: `aborted:${attemptId}`, + attemptState: 'aborted', + }); + return true; + }); } // ---- default-dir wrappers (production wiring) ---- @@ -196,6 +333,16 @@ export function clearRestartLease(id: string): void { clearRestartLeaseTo(config.session.dataDir, id); } +export function claimRestartIntentForReport( + nowMs: number = Date.now(), +): RestartIntentReportClaim { + return claimRestartIntentForReportTo(config.session.dataDir, nowMs); +} + +export function hasPreparedRestartIntent(nowMs: number = Date.now()): boolean { + return hasPreparedRestartIntentTo(config.session.dataDir, nowMs); +} + export function writeManualIntentIfAbsent(nowMs: number = Date.now()): void { writeManualIntentIfAbsentTo(config.session.dataDir, nowMs, new Date(nowMs).toISOString()); } diff --git a/src/setup/bots-store.ts b/src/setup/bots-store.ts index 12835d3a1..33746b052 100644 --- a/src/setup/bots-store.ts +++ b/src/setup/bots-store.ts @@ -5,12 +5,18 @@ * - 文件权限 0o600 (只有用户自己能读), secret 不外泄给同机器人其它用户 */ import { writeFileSync, renameSync, existsSync, readFileSync } from 'node:fs'; +import { withFileLockSync } from '../utils/file-lock.js'; export function writeBotsJsonAtomic(botsJsonPath: string, bots: any[]): void { - // 注意: tmp 必须在同一目录下 (同 fs), 否则 rename 可能跨文件系统失败. - const tmp = botsJsonPath + '.tmp'; - writeFileSync(tmp, JSON.stringify(bots, null, 2) + '\n', { mode: 0o600 }); - renameSync(tmp, botsJsonPath); + // PM2 start surfaces hold this same generation lock from snapshot through + // post-start verification/rollback, so the ecosystem and its expected names + // can never be built from different bots.json generations. + withFileLockSync(botsJsonPath, () => { + // 注意: tmp 必须在同一目录下 (同 fs), 否则 rename 可能跨文件系统失败. + const tmp = botsJsonPath + '.tmp'; + writeFileSync(tmp, JSON.stringify(bots, null, 2) + '\n', { mode: 0o600 }); + renameSync(tmp, botsJsonPath); + }); } export function readBotsJsonOrEmpty(botsJsonPath: string): any[] { diff --git a/test/fleet-shutdown.test.ts b/test/fleet-shutdown.test.ts new file mode 100644 index 000000000..eb880b3b0 --- /dev/null +++ b/test/fleet-shutdown.test.ts @@ -0,0 +1,914 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + isFleetEntryProvenFreeOfAutorestartTimer, + isFleetEntryProvenTerminalAfterSignal, + signalAndAwaitFleet, + type FleetProcessEntry, +} from '../src/cli/fleet-shutdown.js'; +import { DAEMON_GRACEFUL_EXIT_CODE } from '../src/core/supervisor-shutdown-protocol.js'; + +const entries: FleetProcessEntry[] = [ + { + name: 'botmux-a', pmId: 1, pid: 101, online: true, + autorestart: true, stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }, + { + name: 'botmux-b', pmId: 2, pid: 202, online: true, + autorestart: true, stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }, + { + name: 'botmux-offline', pmId: 3, pid: 0, online: false, + status: 'stopped', autorestart: false, + }, +]; + +function gracefulTerminalRows(targets: FleetProcessEntry[]): FleetProcessEntry[] { + return targets + .filter(target => target.online && target.pid > 0) + .map(target => ({ + ...target, + pid: 0, + online: false, + status: 'waiting restart', + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + })); +} + +describe('generation-aware fleet graceful shutdown', () => { + it('never treats PM2 signal-death code 0 as the daemon graceful-stop sentinel', () => { + expect(isFleetEntryProvenFreeOfAutorestartTimer({ + name: 'botmux-a', + pid: 0, + online: false, + status: 'waiting restart', + autorestart: true, + exitCode: 0, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + })).toBe(false); + expect(isFleetEntryProvenFreeOfAutorestartTimer({ + name: 'botmux-a', + pid: 0, + online: false, + status: 'waiting restart', + autorestart: true, + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + })).toBe(true); + }); + + it('separates timer-free overlimit admission from post-signal terminal proof', () => { + const abruptOverlimit: FleetProcessEntry = { + name: 'botmux-a', pmId: 1, pid: 0, online: false, + status: 'errored', autorestart: true, exitCode: 0, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }; + expect(isFleetEntryProvenFreeOfAutorestartTimer(abruptOverlimit)).toBe(true); + expect(isFleetEntryProvenTerminalAfterSignal(abruptOverlimit)).toBe(false); + expect(isFleetEntryProvenTerminalAfterSignal({ + ...abruptOverlimit, + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + })).toBe(true); + }); + + it('restores an ACKed daemon whose abrupt exit hits PM2 restart overlimit', () => { + const target = entries[0]!; + const alive = new Set([target.pid]); + let now = 0; + let restored = false; + const signalInitial = vi.fn(() => { alive.delete(target.pid); }); + const startOffline = vi.fn((offlineEntries: FleetProcessEntry[]) => { + expect(offlineEntries).toEqual([ + expect.objectContaining({ name: target.name, status: 'errored', exitCode: 0 }), + ]); + restored = true; + alive.add(303); + }); + + expect(() => signalAndAwaitFleet([target], 'restart', 100, { + signal: vi.fn(), + signalInitial, + assertSignalAuthorityComplete: vi.fn(), + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => restored + ? [{ ...target, pid: 303, online: true }] + : [{ + ...target, + pid: 0, + online: false, + status: 'errored', + exitCode: 0, + }], + successorSettleMs: 10, + })).toThrow(/post-signal terminal proof.*exit_code=0.*restored 1 offline PM2 entry/); + expect(signalInitial).toHaveBeenCalledOnce(); + expect(startOffline).toHaveBeenCalledOnce(); + }); + + it('accepts an errored row only when its exit matches the graceful stop policy', () => { + const target = entries[0]!; + const alive = new Set([target.pid]); + let now = 0; + const authority = vi.fn(); + signalAndAwaitFleet([target], 'stop', 100, { + signal(pid) { alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline: vi.fn(), + list: () => [{ + ...target, + pid: 0, + online: false, + status: 'errored', + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + }], + assertSignalAuthorityComplete: authority, + successorSettleMs: 10, + pollMs: 5, + }); + expect(authority).toHaveBeenCalledOnce(); + }); + + it('does not cache a latest terminal proof across a later missing projection', () => { + const target = entries[0]!; + const alive = new Set([target.pid]); + let now = 0; + let listCalls = 0; + const startOffline = vi.fn(); + + expect(() => signalAndAwaitFleet([target], 'restart', 100, { + signal(pid) { alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => { + listCalls += 1; + if (listCalls === 1) { + return [{ + ...target, + pid: 0, + online: false, + status: 'waiting restart', + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + }]; + } + return []; + }, + successorSettleMs: 10, + pollMs: 5, + })).toThrow(/fleet is partially stopped.*offline: botmux-a/); + expect(listCalls).toBeGreaterThan(1); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('does not use a dead replacement exit code to prove its predecessor', () => { + const target = entries[0]!; + const alive = new Set([target.pid]); + let now = 0; + const startOffline = vi.fn(); + + expect(() => signalAndAwaitFleet([target], 'restart', 100, { + signal(pid) { alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [{ + ...target, + pid: 303, + online: false, + status: 'errored', + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + }], + successorSettleMs: 10, + pollMs: 5, + })).toThrow(/post-signal terminal proof.*botmux-a\/101.*fleet is partially stopped/); + expect(startOffline).toHaveBeenCalledOnce(); + }); + + it('refuses before any signal when a non-online PM2 row still owns a live PID', () => { + const signal = vi.fn(); + const list = vi.fn(() => []); + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet([ + entries[0]!, + { name: 'botmux-transitioning', pmId: 9, pid: 404, online: false, status: 'launching' }, + ], 'restart', 100, { + signal, + isAlive: pid => pid === 101 || pid === 404, + now: () => 0, + sleep: vi.fn(), + startOffline, + list, + })).toThrow(/non-online registry rows still have live PID.*botmux-transitioning:404/); + expect(signal).not.toHaveBeenCalled(); + expect(list).not.toHaveBeenCalled(); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('refuses duplicate singleton PM2 names before collapsing or signalling either PID', () => { + const signal = vi.fn(); + expect(() => signalAndAwaitFleet([ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { name: 'botmux-a', pmId: 4, pid: 404, online: true }, + ], 'stop', 100, { + signal, + isAlive: () => true, + now: () => 0, + sleep: vi.fn(), + startOffline: vi.fn(), + list: vi.fn(() => []), + })).toThrow(/duplicate registry row.*botmux-a/); + expect(signal).not.toHaveBeenCalled(); + }); + + it('does not signal a later initial target after an earlier signal consumes the deadline', () => { + let now = 0; + const signal = vi.fn(() => { now += 10; }); + const list = vi.fn(() => []); + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 5, { + signal, + isAlive: () => true, + now: () => now, + sleep: vi.fn(), + startOffline, + list, + })).toThrow(/deadline exhausted during initial daemon signalling.*no later fleet action/); + expect(signal).toHaveBeenCalledTimes(1); + expect(signal).toHaveBeenCalledWith(101); + expect(list).not.toHaveBeenCalled(); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('propagates a signal authorization failure instead of treating it as already exited', () => { + const list = vi.fn(() => []); + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet([entries[0]!], 'restart', 100, { + signal: () => { throw new Error('old daemon lacks shutdown capability'); }, + isAlive: () => true, + now: () => 0, + sleep: vi.fn(), + startOffline, + list, + })).toThrow(/old daemon lacks shutdown capability/); + expect(list).not.toHaveBeenCalled(); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('signals a live non-online successor and never treats it as quiet', () => { + const alive = new Set([202]); + const signalled: number[] = []; + let now = 0; + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet([entries[1]!], 'restart', 100, { + signal(pid) { + signalled.push(pid); + if (pid === 202) alive.delete(pid); + if (pid === 303) return; // transitional successor refuses + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => { + alive.add(303); + return [{ + name: 'botmux-b', pmId: 2, pid: 303, online: false, status: 'launching', + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }]; + }, + successorSettleMs: 10, + pollMs: 5, + })).toThrow(/daemon generation\(s\).*live generation untouched/); + expect(signalled).toEqual([202, 303]); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('requires an initially dormant row to prove that no restart timer can fire', () => { + const list = vi.fn(() => []); + expect(() => signalAndAwaitFleet([ + { name: 'botmux-waiting', pmId: 7, pid: 0, online: false, status: 'waiting restart' }, + ], 'stop', 100, { + signal: vi.fn(), + isAlive: () => false, + now: () => 0, + sleep: vi.fn(), + startOffline: vi.fn(), + list, + })).toThrow(/dormant registry row.*may still restart/); + expect(list).not.toHaveBeenCalled(); + }); + + it('does not trust a bare stopped status as proof that no restart task exists', () => { + expect(() => signalAndAwaitFleet([ + { name: 'botmux-stopped', pmId: 8, pid: 0, online: false, status: 'stopped' }, + ], 'restart', 100, { + signal: vi.fn(), + isAlive: () => false, + now: () => 0, + sleep: vi.fn(), + startOffline: vi.fn(), + list: vi.fn(() => []), + })).toThrow(/dormant registry row.*may still restart/); + }); + + it('signals the whole online fleet before polling and succeeds after the quiet window', () => { + const order: string[] = []; + const alive = new Set([101, 202]); + let now = 0; + signalAndAwaitFleet(entries, 'restart', 100, { + signal(pid) { order.push(`signal:${pid}`); alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { order.push(`sleep:${ms}`); now += ms; }, + startOffline: names => order.push(`start:${names.join(',')}`), + list: () => gracefulTerminalRows(entries), + successorSettleMs: 10, + pollMs: 5, + }); + + expect(order.slice(0, 2)).toEqual(['signal:101', 'signal:202']); + expect(order).not.toContain(expect.stringMatching(/^start:/)); + expect(now).toBeGreaterThanOrEqual(10); + }); + + it('batch-dispatches every initial endpoint and compensates a peer when one refuses', () => { + const alive = new Set([101, 202]); + let now = 0; + let restored = false; + const signalInitial = vi.fn((targets: FleetProcessEntry[]) => { + expect(targets.map(target => target.pid)).toEqual([101, 202]); + // A's endpoint hangs/refuses; B accepted the concurrent request and exits. + alive.delete(202); + }); + const startOffline = vi.fn((offline: FleetProcessEntry[]) => { + expect(offline.map(entry => entry.name)).toEqual(['botmux-b']); + restored = true; + alive.add(303); + }); + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 100, { + signal: vi.fn(), + signalInitial, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + restored + ? { name: 'botmux-b', pmId: 2, pid: 303, online: true } + : { name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0] }, + ], + successorSettleMs: 10, + })).toThrow(/restored 1 offline PM2 entry.*live generation untouched/); + expect(signalInitial).toHaveBeenCalledOnce(); + expect(startOffline).toHaveBeenCalledOnce(); + }); + + it('compensates and fails when an unacked generation later disappears', () => { + const alive = new Set([101, 202]); + let now = 0; + let restored = false; + const startOffline = vi.fn((offline: FleetProcessEntry[]) => { + expect(offline.map(entry => entry.name)).toEqual(['botmux-a', 'botmux-b']); + restored = true; + alive.add(301); + alive.add(302); + }); + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 100, { + signal: vi.fn(), + signalInitial: () => { alive.clear(); }, + assertSignalAuthorityComplete: () => { + throw new Error('botmux-a exact IPC ACK missing'); + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => restored + ? [ + { name: 'botmux-a', pmId: 1, pid: 301, online: true }, + { name: 'botmux-b', pmId: 2, pid: 302, online: true }, + ] + : [ + { name: 'botmux-a', pmId: 1, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0] }, + { name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0] }, + ], + successorSettleMs: 10, + })).toThrow(/signal authority: botmux-a exact IPC ACK missing.*restored 2 offline PM2 entries/); + expect(startOffline).toHaveBeenCalledOnce(); + }); + + it('never starts the quiet window while a projected row may still publish a successor', () => { + const alive = new Set([101]); + const startOffline = vi.fn(); + let now = 0; + expect(() => signalAndAwaitFleet([entries[0]!], 'stop', 50, { + signal(pid) { alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [{ + name: 'botmux-a', pmId: 1, pid: 0, online: false, + status: 'waiting restart', exitCode: 1, stopExitCodes: [0], + }], + successorSettleMs: 5, + pollMs: 5, + })).toThrow(/fleet is partially stopped.*offline: botmux-a/); + expect(startOffline).not.toHaveBeenCalled(); + expect(now).toBe(0); + }); + + it('detects and gracefully signals a PM2 successor that appears during the quiet window', () => { + const signalled: number[] = []; + const alive = new Set([101, 202]); + let now = 0; + signalAndAwaitFleet(entries, 'restart', 300, { + signal(pid) { + signalled.push(pid); + alive.delete(pid); + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline: vi.fn(), + list: () => { + const terminal = gracefulTerminalRows(entries); + if (now < 50 || signalled.includes(303)) return terminal; + alive.add(303); + return [ + terminal.find(entry => entry.name === 'botmux-a')!, + { + ...entries[1]!, + pid: 303, + online: true, + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + }, + ]; + }, + successorSettleMs: 100, + pollMs: 10, + }); + + expect(signalled).toEqual([101, 202, 303]); + expect(now).toBeGreaterThanOrEqual(150); + }); + + it('does not report success when a discovered successor itself refuses to exit', () => { + const signalled: number[] = []; + const alive = new Set([101, 202, 303]); + let now = 0; + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet([entries[1]!], 'restart', 100, { + signal(pid) { + signalled.push(pid); + if (pid !== 303) alive.delete(pid); + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [{ + name: 'botmux-b', pmId: 2, pid: 303, online: true, + exitCode: DAEMON_GRACEFUL_EXIT_CODE, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }], + successorSettleMs: 10, + pollMs: 5, + })).toThrow(/1\/1 daemon generation\(s\).*live generation untouched/); + expect(signalled).toEqual([202, 303]); + expect(startOffline).not.toHaveBeenCalled(); + expect(alive.has(303)).toBe(true); + }); + + it('fresh-lists first, starts only a truly offline peer, and leaves the refuser untouched', () => { + const alive = new Set([101, 202]); + const projection = new Map(entries.map(entry => [entry.name, { ...entry }])); + let now = 0; + const startOffline = vi.fn((offlineEntries: FleetProcessEntry[]) => { + for (const entry of offlineEntries) { + projection.set(entry.name, { ...entry, pid: 902, online: true }); + alive.add(902); + } + }); + + expect(() => signalAndAwaitFleet(entries, 'stop', 100, { + signal(pid) { + if (pid === 202) { + alive.delete(pid); + projection.set('botmux-b', { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }); + } + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [...projection.values()], + successorSettleMs: 10, + })).toThrow(/restored 1 offline PM2 entry.*live generation untouched/); + + expect(startOffline).toHaveBeenCalledWith( + [expect.objectContaining({ name: 'botmux-b', pmId: 2 })], + expect.any(Number), + ); + expect(startOffline.mock.calls.flatMap(call => call[0]).map(entry => entry.name)) + .not.toContain('botmux-a'); + expect(alive.has(101)).toBe(true); + }); + + it('does not compensate an exact row when a fresh duplicate with the same name exists', () => { + const alive = new Set([101, 202]); + let now = 0; + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, + { + name: 'botmux-b', pmId: 9, pid: 0, online: false, + status: 'waiting restart', exitCode: 1, stopExitCodes: [0], + }, + ], + })).toThrow(/fleet is partially stopped.*offline: botmux-b/); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('leaves a successor first observed at the refusal boundary untouched and restores an unrelated peer', () => { + const alive = new Set([101, 202]); + const signalled: number[] = []; + let now = 0; + let restored = false; + const startOffline = vi.fn((offlineEntries: FleetProcessEntry[]) => { + expect(offlineEntries.map(entry => entry.name)).toEqual(['botmux-b']); + restored = true; + alive.add(909); + }); + const projection = (): FleetProcessEntry[] => [ + ...(now >= 40 + ? [{ name: 'botmux-a', pmId: 1, pid: 808, online: true }] + : [{ + name: 'botmux-a', pmId: 1, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }]), + restored + ? { name: 'botmux-b', pmId: 2, pid: 909, online: true } + : { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, + ]; + + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 50, { + signal(pid) { signalled.push(pid); alive.delete(pid); }, + isAlive(pid) { + if (pid === 808 && now >= 40) return true; + return alive.has(pid); + }, + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: projection, + successorSettleMs: 100, + pollMs: 5, + })).toThrow(/restored 1 offline PM2 entry.*live generation untouched/); + expect(signalled).toEqual([101, 202]); + expect(signalled).not.toContain(808); + expect(startOffline).toHaveBeenCalledOnce(); + }); + + it('does not restart or re-signal a healthy successor discovered during partial refusal', () => { + const alive = new Set([101, 202, 303]); + let now = 0; + const signalled: number[] = []; + const startOffline = vi.fn(); + + expect(() => signalAndAwaitFleet(entries, 'restart', 100, { + signal(pid) { + signalled.push(pid); + if (pid === 202) alive.delete(pid); + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { name: 'botmux-b', pmId: 2, pid: 303, online: true }, + ], + successorSettleMs: 10, + })).toThrow(/left every live generation untouched/); + + expect(signalled).toEqual([101, 202]); + expect(startOffline).not.toHaveBeenCalled(); + expect(alive.has(101)).toBe(true); + expect(alive.has(303)).toBe(true); + }); + + it('fails closed without a PM2 mutation when fresh-list verification fails', () => { + const alive = new Set([101, 202]); + let now = 0; + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet(entries, 'stop', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => { throw new Error('jlist unavailable'); }, + })).toThrow(/no compensation was attempted.*verification before compensation failed.*jlist unavailable/); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('fails closed without compensation when successor verification itself cannot be read', () => { + const alive = new Set([101, 202]); + let now = 0; + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet(entries, 'restart', 100, { + signal(pid) { alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => { throw new Error('projection corrupt'); }, + successorSettleMs: 10, + })).toThrow(/state is unverified and no compensation was attempted.*successor verification failed/); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it('reports a genuinely partial fleet when compensation cannot restore an exited peer', () => { + const alive = new Set([101, 202]); + let now = 0; + expect(() => signalAndAwaitFleet(entries, 'restart', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline() { throw new Error('pm2 unavailable'); }, + list: () => [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, + ], + })).toThrow(/fleet is partially stopped.*botmux-b.*pm2 unavailable/); + }); + + it('compensates a peer whose PM2 row says online but whose PID is dead', () => { + const alive = new Set([101, 202]); + let now = 0; + let projection: FleetProcessEntry[] = [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { + name: 'botmux-b', pmId: 2, pid: 303, online: true, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, // stale: 303 is not alive + ]; + const startOffline = vi.fn((offlineEntries: FleetProcessEntry[]) => { + expect(offlineEntries.map(entry => entry.name)).toEqual(['botmux-b']); + alive.add(404); + projection = [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { name: 'botmux-b', pmId: 2, pid: 404, online: true }, + ]; + }); + + expect(() => signalAndAwaitFleet(entries, 'stop', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => projection, + })).toThrow(/restored 1 offline PM2 entry/); + expect(startOffline).toHaveBeenCalledOnce(); + expect(alive.has(404)).toBe(true); + }); + + it('refuses to claim restoration when PM2 reports an online replacement with a dead PID', () => { + const alive = new Set([101, 202]); + let now = 0; + let projection: FleetProcessEntry[] = [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { + name: 'botmux-b', pmId: 2, pid: 303, online: true, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, // stale before compensation + ]; + const startOffline = vi.fn((offlineEntries: FleetProcessEntry[]) => { + expect(offlineEntries.map(entry => entry.name)).toEqual(['botmux-b']); + // PM2 claims it started 404, but OS liveness never confirms that PID. + projection = [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { name: 'botmux-b', pmId: 2, pid: 404, online: true }, + ]; + }); + + expect(() => signalAndAwaitFleet(entries, 'restart', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => projection, + })).toThrow(/fleet is partially stopped.*offline: botmux-b/); + expect(startOffline).toHaveBeenCalledOnce(); + expect(alive.has(404)).toBe(false); + }); + + it('caps a successor projection at the absolute fleet deadline and never acts on its late result', () => { + const alive = new Set([202]); + const signalled: Array<{ pid: number; at: number }> = []; + const listCalls: Array<{ at: number; budgetMs: number }> = []; + const startOffline = vi.fn(); + let now = 0; + + expect(() => signalAndAwaitFleet([entries[1]!], 'restart', 50, { + signal(pid) { + signalled.push({ pid, at: now }); + alive.delete(pid); + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: budgetMs => { + listCalls.push({ at: now, budgetMs }); + // Even if a non-conforming subprocess returns after its advertised cap, + // no observation at the absolute deadline may trigger another signal. + now = 50; + alive.add(303); + return [{ name: 'botmux-b', pmId: 2, pid: 303, online: true }]; + }, + successorSettleMs: 10, + })).toThrow(/no compensation was attempted.*deadline exhausted during PM2 successor verification/); + + expect(listCalls).toEqual([{ at: 0, budgetMs: 2 }]); + expect(signalled).toEqual([{ pid: 202, at: 0 }]); + expect(startOffline).not.toHaveBeenCalled(); + expect(now).toBe(50); + }); + + it('does not issue another list or signal after a projection consumes the remaining deadline', () => { + const alive = new Set([202]); + const listCalls: Array<{ at: number; budgetMs: number }> = []; + const signalCalls: Array<{ pid: number; at: number }> = []; + const startOffline = vi.fn(); + let now = 0; + + expect(() => signalAndAwaitFleet([entries[1]!], 'stop', 50, { + signal(pid) { signalCalls.push({ pid, at: now }); }, // original refuses + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: budgetMs => { + listCalls.push({ at: now, budgetMs }); + now = 50; + return [{ name: 'botmux-b', pmId: 2, pid: 202, online: true }]; + }, + pollMs: 5, + })).toThrow(/no compensation was attempted.*deadline exhausted during PM2 verification before compensation/); + + expect(listCalls).toEqual([{ at: 40, budgetMs: 2 }]); + expect(signalCalls).toEqual([{ pid: 202, at: 0 }]); + expect(startOffline).not.toHaveBeenCalled(); + expect(now).toBe(50); + }); + + it('partitions one bounded multi-entry compensation inside the absolute fleet deadline', () => { + const targets: FleetProcessEntry[] = [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { name: 'botmux-b', pmId: 2, pid: 202, online: true }, + { name: 'botmux-c', pmId: 3, pid: 303, online: true }, + ]; + const alive = new Set([101, 202, 303]); + const listCalls: Array<{ at: number; budgetMs: number }> = []; + const compensationCalls: Array<{ names: string[]; at: number; budgetMs: number }> = []; + let now = 0; + let greatestNow = 0; + let compensated = false; + const advance = (ms: number) => { + now += ms; + greatestNow = Math.max(greatestNow, now); + }; + + expect(() => signalAndAwaitFleet(targets, 'restart', 50, { + signal(pid) { if (pid !== 101) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep: advance, + startOffline(offlineEntries, budgetMs) { + compensationCalls.push({ + names: offlineEntries.map(entry => entry.name), + at: now, + budgetMs, + }); + // One helper gets one shared tail budget for both ids, never a fresh + // per-name timeout. + advance(budgetMs); + compensated = true; + alive.add(404); + alive.add(505); + }, + list: budgetMs => { + listCalls.push({ at: now, budgetMs }); + return [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + compensated ? { name: 'botmux-b', pmId: 2, pid: 404, online: true } : { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, + compensated ? { name: 'botmux-c', pmId: 3, pid: 505, online: true } : { + name: 'botmux-c', pmId: 3, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, + ]; + }, + })).toThrow(/restored 2 offline PM2 entries.*live generation untouched/); + + expect(compensationCalls).toEqual([{ + names: ['botmux-b', 'botmux-c'], + at: 40, + budgetMs: 5, + }]); + expect(listCalls).toEqual([ + { at: 40, budgetMs: 2 }, + { at: 45, budgetMs: 2 }, + ]); + expect(now).toBe(45); + expect(greatestNow).toBeLessThanOrEqual(50); + }); + + it('does not conditionally start a row whose PM2 policy may still have a restart timer', () => { + const alive = new Set([101, 202]); + let now = 0; + const startOffline = vi.fn(); + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'stop', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 1, stopExitCodes: [0], + }, + ], + })).toThrow(/fleet is partially stopped.*offline: botmux-b/); + expect(startOffline).not.toHaveBeenCalled(); + }); + + it.each([ + { label: 'missing exit_code', exitCode: undefined, stopExitCodes: [0] }, + { label: 'null stop code', exitCode: 0, stopExitCodes: [null] }, + { label: 'empty stop code', exitCode: 0, stopExitCodes: [''] }, + ])('does not coerce malformed timer policy into a safe compensation: $label', ({ + exitCode, + stopExitCodes, + }) => { + const alive = new Set([101, 202]); + const startOffline = vi.fn(); + let now = 0; + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 50, { + signal(pid) { if (pid === 202) alive.delete(pid); }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline, + list: () => [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', + ...(exitCode !== undefined ? { exitCode } : {}), + stopExitCodes, + }, + ], + })).toThrow(/fleet is partially stopped.*offline: botmux-b/); + expect(startOffline).not.toHaveBeenCalled(); + }); +}); diff --git a/test/plugin-service-restart-lifecycle.test.ts b/test/plugin-service-restart-lifecycle.test.ts index 9adc922e0..c567d5b61 100644 --- a/test/plugin-service-restart-lifecycle.test.ts +++ b/test/plugin-service-restart-lifecycle.test.ts @@ -16,14 +16,17 @@ describe('plugin service restart lifecycle', () => { it('preserves auto services by default and always ensures them after core starts', () => { const source = restartFunctionSource(); const stop = 'if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true });'; - const coreStart = "runPm2(['start', cfg]);"; + const transaction = 'runBoundedPm2StartTransaction('; + const coreStart = "runPm2(['start', cfg], true, PM2_HOME, timeoutMs);"; const ensure = 'await reconcilePluginServicesForCli(undefined, { autoOnly: true });'; expect(source).toContain(stop); + expect(source).toContain(transaction); expect(source).toContain(coreStart); expect(source).toContain(ensure); expect(source).not.toContain(`if (includePluginServices) ${ensure}`); expect(source.indexOf(stop)).toBeLessThan(source.indexOf(coreStart)); + expect(source.indexOf(transaction)).toBeLessThan(source.indexOf(coreStart)); expect(source.indexOf(coreStart)).toBeLessThan(source.indexOf(ensure)); }); diff --git a/test/pm2-descriptor-guard.test.ts b/test/pm2-descriptor-guard.test.ts new file mode 100644 index 000000000..8ca9b35f3 --- /dev/null +++ b/test/pm2-descriptor-guard.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + assertNoUnregisteredLiveDaemonDescriptorsIn, + type Pm2DescriptorGuardRuntime, +} from '../src/cli/pm2-descriptor-guard.js'; + +function runtime( + raw: string, + options: { alive?: boolean; mtime?: number; startIdentity?: string } = {}, +): Pm2DescriptorGuardRuntime { + return { + now: () => 100_000, + exists: () => true, + readdir: () => ['app.json'], + read: () => raw, + mtime: () => options.mtime ?? 100_000, + isAlive: () => options.alive ?? true, + readStartIdentity: () => options.startIdentity ?? 'birth-77', + }; +} + +describe('PM2/live-daemon descriptor reconciliation', () => { + it('blocks a mutation when jlist is empty and a fresh descriptor is semantic garbage', () => { + const mutate = vi.fn(); + expect(() => { + assertNoUnregisteredLiveDaemonDescriptorsIn('start', [], '/registry', runtime('{}')); + mutate(); + }).toThrow(/fresh daemon descriptor.*invalid pid\/app id\/heartbeat/); + expect(mutate).not.toHaveBeenCalled(); + }); + + it('blocks a live descriptor PID absent from the PM2 projection', () => { + const raw = JSON.stringify({ larkAppId: 'app', pid: 77, lastHeartbeat: 100_000 }); + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'restart', [], '/registry', runtime(raw), + )).toThrow(/app:77/); + }); + + it('accepts a valid fresh descriptor only when its live PID is registered', () => { + const raw = JSON.stringify({ larkAppId: 'app', pid: 77, lastHeartbeat: 100_000 }); + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'stop', [{ pid: 77 }], '/registry', runtime(raw), + )).not.toThrow(); + }); + + it('ignores stale semantic garbage', () => { + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'start', [], '/registry', runtime('{}', { mtime: 0 }), + )).not.toThrow(); + }); + + it('blocks a stale descriptor whose live PID still has the exact recorded birth', () => { + const raw = JSON.stringify({ + larkAppId: 'app', + pid: 77, + processStartIdentity: 'birth-77', + lastHeartbeat: 0, + }); + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'start', [], '/registry', runtime(raw, { mtime: 0 }), + )).toThrow(/daemon descriptor PID\(s\).*live but absent.*app:77/); + }); + + it('ignores a stale descriptor after proving its recorded PID is dead', () => { + const raw = JSON.stringify({ + larkAppId: 'app', + pid: 77, + processStartIdentity: 'birth-77', + lastHeartbeat: 0, + }); + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'start', [], '/registry', runtime(raw, { alive: false, mtime: 0 }), + )).not.toThrow(); + }); + + it('ignores a stale descriptor after proving the PID belongs to a different birth', () => { + const raw = JSON.stringify({ + larkAppId: 'app', + pid: 77, + processStartIdentity: 'birth-77', + lastHeartbeat: 0, + }); + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'start', [], '/registry', runtime(raw, { mtime: 0, startIdentity: 'birth-successor' }), + )).not.toThrow(); + }); + + it('fails closed when an old-format stale descriptor still names a live PID', () => { + const raw = JSON.stringify({ larkAppId: 'app', pid: 77, lastHeartbeat: 0 }); + expect(() => assertNoUnregisteredLiveDaemonDescriptorsIn( + 'start', [], '/registry', runtime(raw, { mtime: 0 }), + )).toThrow(/stale daemon descriptor.*live PID 77.*no process-start identity/); + }); +}); diff --git a/test/pm2-exact-start.test.ts b/test/pm2-exact-start.test.ts new file mode 100644 index 000000000..7cf322505 --- /dev/null +++ b/test/pm2-exact-start.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + isNonMutatingPm2StartRefusal, + startExactPm2ProcessIds, + type Pm2ExactStartClient, +} from '../src/cli/pm2-exact-start.js'; + +function fakeClient(overrides: Partial = {}): Pm2ExactStartClient { + return { + launchRPC: callback => callback(), + executeRemote: (_method, _id, callback) => callback(), + close: callback => callback(), + ...overrides, + }; +} + +describe('exact PM2 start-if-stopped compensation', () => { + it('uses one RPC connection and dispatches every exact id concurrently', async () => { + const callbacks: Array<(error?: unknown) => void> = []; + const launchRPC = vi.fn((callback: (error?: unknown) => void) => callback()); + const executeRemote = vi.fn(( + _method: 'startProcessId', + _id: number, + callback: (error?: unknown) => void, + ) => { callbacks.push(callback); }); + const close = vi.fn((callback: (error?: unknown) => void) => callback()); + + const pending = startExactPm2ProcessIds( + [1, 2, 3], + fakeClient({ launchRPC, executeRemote, close }), + ); + await Promise.resolve(); + + expect(launchRPC).toHaveBeenCalledOnce(); + expect(executeRemote.mock.calls.map(call => [call[0], call[1]])).toEqual([ + ['startProcessId', 1], + ['startProcessId', 2], + ['startProcessId', 3], + ]); + expect(close).not.toHaveBeenCalled(); + + callbacks.forEach(callback => callback()); + await pending; + expect(close).toHaveBeenCalledOnce(); + }); + + it.each([ + 'process already online', + 'process already started', + 'Process with pid 123 already exists', + '7 id unknown', + ])('treats an atomic non-mutating refusal as a benign no-op: %s', async message => { + expect(isNonMutatingPm2StartRefusal(new Error(message))).toBe(true); + const close = vi.fn((callback: (error?: unknown) => void) => callback()); + await expect(startExactPm2ProcessIds([7], fakeClient({ + executeRemote: (_method, _id, callback) => callback(new Error(message)), + close, + }))).resolves.toBeUndefined(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('aggregates hard start failures and still closes the RPC connection', async () => { + const close = vi.fn((callback: (error?: unknown) => void) => callback()); + await expect(startExactPm2ProcessIds([4, 5], fakeClient({ + executeRemote: (_method, id, callback) => callback( + id === 4 ? new Error('executeApp exploded') : new Error('socket lost'), + ), + close, + }))).rejects.toThrow(/pm_id 4: executeApp exploded; pm_id 5: socket lost/); + expect(close).toHaveBeenCalledOnce(); + }); + + it.each([ + { ids: [1, 1] }, + { ids: [-1] }, + { ids: [1.5] }, + ])('rejects invalid exact id sets before connecting: $ids', async ({ ids }) => { + const launchRPC = vi.fn(); + await expect(startExactPm2ProcessIds(ids, fakeClient({ launchRPC }))) + .rejects.toThrow(/unique non-negative pm_id/); + expect(launchRPC).not.toHaveBeenCalled(); + }); + + it('does not call close when the one RPC connection cannot be opened', async () => { + const close = vi.fn(); + await expect(startExactPm2ProcessIds([1], fakeClient({ + launchRPC: callback => callback(new Error('PM2 unavailable')), + close, + }))).rejects.toThrow(/PM2 unavailable/); + expect(close).not.toHaveBeenCalled(); + }); +}); diff --git a/test/pm2-god-admission.test.ts b/test/pm2-god-admission.test.ts new file mode 100644 index 000000000..2b252ffc0 --- /dev/null +++ b/test/pm2-god-admission.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it, vi } from 'vitest'; +import { assertIncludePm2RestartAdmission } from '../src/cli/pm2-god-admission.js'; + +describe('restart --include-pm2 admission', () => { + it('admits only an initially zero-God state', () => { + expect(() => assertIncludePm2RestartAdmission([])).not.toThrow(); + }); + + it('rejects a live God before any caller mutation', () => { + const mutate = vi.fn(); + expect(() => { + assertIncludePm2RestartAdmission([101]); + mutate(); + }).toThrow(/cannot be signalled with generation-bound authority.*does not signal or restart.*no process or breadcrumb was changed/); + expect(mutate).not.toHaveBeenCalled(); + }); + + it('rejects duplicate Gods without selecting either generation', () => { + expect(() => assertIncludePm2RestartAdmission([101, 202])) + .toThrow(/multiple PM2 God daemons.*no process or breadcrumb was changed/); + }); +}); diff --git a/test/pm2-jlist.test.ts b/test/pm2-jlist.test.ts new file mode 100644 index 000000000..ba5b19675 --- /dev/null +++ b/test/pm2-jlist.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { + parseCanonicalPm2Id, + parsePm2JlistOutput, + parsePm2JlistOutputStrict, + parsePm2Integer, +} from '../src/cli/pm2-jlist.js'; + +describe('PM2 jlist projection parsing', () => { + it.each(['{}', 'null', '"not-a-list"'])('rejects non-array shutdown authority: %s', output => { + expect(() => parsePm2JlistOutputStrict(output)).toThrow(/non-array JSON/); + }); + + it('rejects malformed shutdown authority instead of treating it as an empty fleet', () => { + expect(() => parsePm2JlistOutputStrict('[PM2] unavailable')).toThrow(/malformed output/); + }); + + it('accepts an array after PM2 informational output', () => { + const row = { name: 'botmux', pm_id: 0, pid: 42, pm2_env: { status: 'online' } }; + expect(parsePm2JlistOutputStrict(`[PM2] daemon ready\n${JSON.stringify([row])}`)) + .toEqual([row]); + }); + + it('keeps read-only callers backward-compatible with empty fallback', () => { + expect(parsePm2JlistOutput('{}')).toEqual([]); + }); + + it('never coerces absent PM2 identity or exit-code fields to zero', () => { + expect(parsePm2Integer(null)).toBeUndefined(); + expect(parsePm2Integer(undefined)).toBeUndefined(); + expect(parsePm2Integer('')).toBeUndefined(); + expect(parsePm2Integer('0')).toBe(0); + expect(parsePm2Integer(-1, { nonNegative: true })).toBeUndefined(); + }); + + it('never revives a missing canonical pm_id from nested PM2 environment state', () => { + expect(parseCanonicalPm2Id({ pm_id: null, pm2_env: { pm_id: 7 } })).toBeUndefined(); + expect(parseCanonicalPm2Id({ pm2_env: { pm_id: 7 } })).toBeUndefined(); + expect(parseCanonicalPm2Id({ pm_id: 8, pm2_env: { pm_id: 7 } })).toBe(8); + }); + + it.each([ + ['non-object row', '[null]', /row 0 is not an object/], + ['missing name', '[{"pm_id":0,"pid":1,"pm2_env":{"status":"online"}}]', /non-empty name/], + ['missing canonical id', '[{"name":"botmux","pid":1,"pm2_env":{"status":"online"}}]', /canonical non-negative pm_id/], + ['invalid pid', '[{"name":"botmux","pm_id":0,"pid":null,"pm2_env":{"status":"online"}}]', /non-negative pid/], + ['missing status', '[{"name":"botmux","pm_id":0,"pid":1,"pm2_env":{}}]', /pm2_env.status/], + ])('rejects a syntactically valid but semantically unsafe %s', (_label, output, pattern) => { + expect(() => parsePm2JlistOutputStrict(output)).toThrow(pattern as RegExp); + }); + + it('rejects duplicate canonical pm_id values even when the names differ', () => { + const output = JSON.stringify([ + { name: 'botmux-a', pm_id: 4, pid: 41, pm2_env: { status: 'online' } }, + { name: 'botmux-b', pm_id: 4, pid: 42, pm2_env: { status: 'online' } }, + ]); + expect(() => parsePm2JlistOutputStrict(output)) + .toThrow(/duplicate canonical pm_id 4 across botmux-a and botmux-b/); + }); + + it('rejects duplicate positive PIDs even across daemon and dashboard rows', () => { + const output = JSON.stringify([ + { name: 'botmux-a', pm_id: 4, pid: 41, pm2_env: { status: 'online' } }, + { name: 'botmux-dashboard', pm_id: 5, pid: 41, pm2_env: { status: 'online' } }, + ]); + expect(() => parsePm2JlistOutputStrict(output)) + .toThrow(/duplicate positive pid 41 across botmux-a and botmux-dashboard/); + }); +}); diff --git a/test/pm2-preflight.test.ts b/test/pm2-preflight.test.ts new file mode 100644 index 000000000..674f77139 --- /dev/null +++ b/test/pm2-preflight.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it, vi } from 'vitest'; +import { assertLinuxPm2GodExecutableUsable } from '../src/cli/pm2-preflight.js'; + +describe('PM2 deleted-Node preflight', () => { + it('fails closed when /proc proves the God executable was deleted', () => { + const exists = vi.fn(() => true); + expect(() => assertLinuxPm2GodExecutableUsable(42, { + readlink: () => '/old/node (deleted)', + exists, + })).toThrow(/拒绝自动清理/); + expect(exists).not.toHaveBeenCalled(); + }); + + it('fails closed when the successfully resolved executable no longer exists', () => { + expect(() => assertLinuxPm2GodExecutableUsable(42, { + readlink: () => '/old/node', + exists: () => false, + })).toThrow(/Node 二进制已失效/); + }); + + it('only skips a genuine /proc inspection failure', () => { + expect(() => assertLinuxPm2GodExecutableUsable(42, { + readlink: () => { throw new Error('permission denied'); }, + exists: () => false, + })).not.toThrow(); + }); +}); diff --git a/test/pm2-shutdown-capability.test.ts b/test/pm2-shutdown-capability.test.ts new file mode 100644 index 000000000..2e6385a60 --- /dev/null +++ b/test/pm2-shutdown-capability.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from 'vitest'; +import { SUPERVISOR_SHUTDOWN_PROTOCOL } from '../src/core/supervisor-shutdown-protocol.js'; +import { + assertPm2DaemonShutdownCapabilitiesIn, + type Pm2ShutdownCapabilityRuntime, +} from '../src/cli/pm2-shutdown-capability.js'; + +function descriptor( + pid: number, + protocol: string | null = SUPERVISOR_SHUTDOWN_PROTOCOL, +): string { + return JSON.stringify({ + larkAppId: `app-${pid}`, + ipcPort: 7_900 + pid, + bootInstanceId: `boot-${pid}`, + pid, + processStartIdentity: `birth-${pid}`, + lastHeartbeat: 100_000, + ...(protocol ? { supervisorShutdownProtocol: protocol } : {}), + }); +} + +function runtime( + files: Record, + alive = new Set([101, 202]), +): Pm2ShutdownCapabilityRuntime { + return { + now: () => 100_000, + exists: () => true, + readdir: () => Object.keys(files), + read: path => files[path.split('/').pop()!]!, + mtime: () => 100_000, + isAlive: pid => alive.has(pid), + readStartIdentity: pid => alive.has(pid) ? `birth-${pid}` : undefined, + }; +} + +describe('in-memory daemon shutdown capability rollout fence', () => { + it('accepts one fresh exact-protocol descriptor for every live daemon target', () => { + expect(() => assertPm2DaemonShutdownCapabilitiesIn( + 'restart', + [{ name: 'botmux-a', pid: 101 }, { name: 'botmux-b', pid: 202 }], + '/registry', + runtime({ 'a.json': descriptor(101), 'b.json': descriptor(202) }), + )).not.toThrow(); + }); + + it('blocks an old in-memory daemon lacking the new capability before signal', () => { + const signal = vi.fn(); + expect(() => { + assertPm2DaemonShutdownCapabilitiesIn( + 'restart', + [{ name: 'botmux-a', pid: 101 }], + '/registry', + runtime({ 'a.json': descriptor(101, null) }), + ); + signal(101); + }).toThrow(/does not attest.*first-upgrade boundary.*Session\/Riff workload is idle.*must not be reported as applied/); + expect(signal).not.toHaveBeenCalled(); + }); + + it('fails closed when a live target has no fresh matching descriptor', () => { + expect(() => assertPm2DaemonShutdownCapabilitiesIn( + 'stop', + [{ name: 'botmux-a', pid: 101 }], + '/registry', + runtime({ 'other.json': descriptor(202) }), + )).toThrow(/botmux-a\/101 has no matching fresh daemon descriptor/); + }); + + it('rejects duplicate fresh descriptors for one live PID', () => { + expect(() => assertPm2DaemonShutdownCapabilitiesIn( + 'restart', + [{ name: 'botmux-a', pid: 101 }], + '/registry', + runtime({ 'a.json': descriptor(101), 'copy.json': descriptor(101) }), + )).toThrow(/multiple fresh descriptors claim live PID 101/); + }); + + it('rejects a stale capability when the PID now belongs to a new birth', () => { + const reused = runtime({ 'a.json': descriptor(101) }); + reused.readStartIdentity = () => 'birth-successor'; + expect(() => assertPm2DaemonShutdownCapabilitiesIn( + 'restart-immediately-before-signal', + [{ name: 'botmux-a', pid: 101 }], + '/registry', + reused, + )).toThrow(/process-start identity does not match its descriptor/); + }); + + it('does not require capability from a generation proven already dead', () => { + expect(() => assertPm2DaemonShutdownCapabilitiesIn( + 'restart', + [{ name: 'botmux-a', pid: 101 }], + '/registry', + runtime({}, new Set()), + )).not.toThrow(); + }); +}); diff --git a/test/pm2-start-transaction.test.ts b/test/pm2-start-transaction.test.ts new file mode 100644 index 000000000..f07784c46 --- /dev/null +++ b/test/pm2-start-transaction.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { FleetProcessEntry } from '../src/cli/fleet-shutdown.js'; +import { + assertDaemonPm2GracefulExitPolicy, + assertConfiguredPm2FleetOnline, + assertConfiguredPm2FleetReady, + assertExactAttestedDaemonSet, + classifyStartBotFleetAdmission, + normalizeRawPm2StopExitCodes, + reconcileLatePm2StartPublication, + runBoundedPm2StartTransaction, +} from '../src/cli/pm2-start-transaction.js'; +import { DAEMON_GRACEFUL_EXIT_CODE } from '../src/core/supervisor-shutdown-protocol.js'; + +function row( + name: string, + pmId: number, + options: { pid?: number; online?: boolean } = {}, +): FleetProcessEntry { + return { + name, + pmId, + pid: options.pid ?? pmId + 100, + online: options.online ?? true, + status: options.online === false ? 'stopped' : 'online', + }; +} + +const configured = ['botmux-a', 'botmux-b', 'botmux-dashboard']; +const alive = (pid: number) => pid > 0; + +describe('configured PM2 fleet start authority', () => { + it('requires the new daemon PM2 policy instead of trusting capability alone', () => { + expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ + ...row('botmux-a', 0), + autorestart: true, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }])).not.toThrow(); + expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ + ...row('botmux-a', 0), + autorestart: true, + stopExitCodes: [0], + }])).toThrow(/does not prove signal-death autorestart.*stop_exit_codes=\[42\]/); + expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ + ...row('botmux-a', 0), + autorestart: false, + stopExitCodes: [DAEMON_GRACEFUL_EXIT_CODE], + }])).toThrow(/does not prove signal-death autorestart/); + }); + + it('preserves raw PM2 stop-exit-code elements for exact policy validation', () => { + expect(normalizeRawPm2StopExitCodes([42, '0foo'])).toEqual([42, '0foo']); + expect(normalizeRawPm2StopExitCodes([42, '0x0'])).toEqual([42, '0x0']); + expect(normalizeRawPm2StopExitCodes([42, null])).toEqual([42, null]); + expect(normalizeRawPm2StopExitCodes('42')).toEqual(['42']); + expect(normalizeRawPm2StopExitCodes(null)).toEqual([null]); + }); + + it.each([ + [42, '0foo'], + [42, '0x0'], + [42, null], + ])('rejects restart-suppressing raw stop-exit-code extras: %j', stopExitCodes => { + expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ + ...row('botmux-a', 0), + autorestart: true, + stopExitCodes, + }])).toThrow(/does not prove signal-death autorestart.*stop_exit_codes=\[42\]/); + }); + + it('accepts only the numeric sentinel or its canonical decimal string', () => { + for (const stopExitCodes of [[42], ['42']]) { + expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ + ...row('botmux-a', 0), + autorestart: true, + stopExitCodes, + }])).not.toThrow(); + } + for (const stopExitCodes of [['042'], ['42foo'], ['0x2a']]) { + expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ + ...row('botmux-a', 0), + autorestart: true, + stopExitCodes, + }])).toThrow(/does not prove signal-death autorestart/); + } + }); + + it('accepts only one exact online/live row per configured process', () => { + expect(() => assertConfiguredPm2FleetOnline( + 'start', + [row('botmux-a', 0), row('botmux-b', 1), row('botmux-dashboard', 2)], + configured, + alive, + )).not.toThrow(); + }); + + it('rejects a dead, missing, unexpected, or duplicate-id row', () => { + expect(() => assertConfiguredPm2FleetOnline( + 'start', + [row('botmux-a', 0), row('botmux-b', 1, { pid: 0 }), row('botmux-dashboard', 2)], + configured, + alive, + )).toThrow(/botmux-b/); + expect(() => assertConfiguredPm2FleetOnline( + 'start', + [row('botmux-a', 0), row('botmux-dashboard', 2)], + configured, + alive, + )).toThrow(/botmux-b/); + expect(() => assertConfiguredPm2FleetOnline( + 'start', + [row('botmux-a', 0), row('botmux-b', 1), row('botmux-old', 2)], + configured, + alive, + )).toThrow(/unexpected PM2 core row/); + expect(() => assertConfiguredPm2FleetOnline( + 'start', + [row('botmux-a', 0), row('botmux-b', 0), row('botmux-dashboard', 2)], + configured, + alive, + )).toThrow(/duplicate canonical pm_id 0 across botmux-a and botmux-b/); + expect(() => assertConfiguredPm2FleetOnline( + 'start', + [row('botmux-a', 0, { pid: 777 }), row('botmux-b', 1), + row('botmux-dashboard', 2, { pid: 777 })], + configured, + alive, + )).toThrow(/duplicate positive pid 777 across botmux-a and botmux-dashboard/); + }); + + it.each([ + 'start-idempotent-ready', + 'start-bot-already-online-ready', + ])('does not accept %s PM2-online rows while an old daemon capability is missing', (operation) => { + expect(() => assertConfiguredPm2FleetReady( + operation, + [row('botmux-a', 0), row('botmux-b', 1), row('botmux-dashboard', 2)], + configured, + alive, + () => { throw new Error('botmux-b has no handler-ready shutdown capability'); }, + )).toThrow(/botmux-b has no handler-ready shutdown capability/); + }); + + it('rejects a capability scan that omits a daemon which exited mid-read', () => { + expect(() => assertExactAttestedDaemonSet( + 'restart-after-launch', + [row('botmux-a', 0, { pid: 101 }), row('botmux-b', 1, { pid: 202 })], + [101], + () => true, + )).toThrow(/handler-ready capability set is incomplete.*expected pids: 101, 202.*attested: 101/); + }); +}); + +describe('start-bot exact append-one admission', () => { + it('admits exactly one missing requested bot when all configured peers are live', () => { + expect(classifyStartBotFleetAdmission( + 'start-bot', + [row('botmux-a', 0), row('botmux-dashboard', 2)], + configured, + 'botmux-b', + alive, + )).toEqual({ state: 'start-eligible' }); + }); + + it('is idempotent only for the exact fully-online configured fleet', () => { + expect(classifyStartBotFleetAdmission( + 'start-bot', + [row('botmux-a', 0), row('botmux-b', 1), row('botmux-dashboard', 2)], + configured, + 'botmux-b', + alive, + )).toEqual({ state: 'already-online' }); + }); + + it('rejects dashboard-only and another-missing-bot partial fleets', () => { + expect(() => classifyStartBotFleetAdmission( + 'start-bot', + [row('botmux-dashboard', 2)], + configured, + 'botmux-b', + alive, + )).toThrow(/configured peer.*botmux-a/); + + expect(() => classifyStartBotFleetAdmission( + 'start-bot', + [row('botmux-a', 0), row('botmux-dashboard', 3)], + ['botmux-a', 'botmux-b', 'botmux-c', 'botmux-dashboard'], + 'botmux-c', + alive, + )).toThrow(/configured peer.*botmux-b/); + }); + + it('rejects an existing transitional target instead of routing it through start', () => { + expect(() => classifyStartBotFleetAdmission( + 'start-bot', + [row('botmux-a', 0), row('botmux-b', 1, { online: false }), row('botmux-dashboard', 2)], + configured, + 'botmux-b', + alive, + )).toThrow(/existing non-live\/transitional/); + }); + + it('distinguishes a truly empty fleet from unsafe partial fleets', () => { + expect(classifyStartBotFleetAdmission( + 'start-bot', [], configured, 'botmux-b', alive, + )).toEqual({ state: 'fleet-down' }); + }); +}); + +describe('bounded PM2 start transaction', () => { + it('passes exact budgets and returns only a fresh verified projection', () => { + const order: string[] = []; + const projection = [{ name: 'ready' }]; + const result = runBoundedPm2StartTransaction('start', 30_000, 10_000, { + start: timeout => { order.push(`start:${timeout}`); }, + verifyFresh: timeout => { order.push(`verify:${timeout}`); return projection; }, + rollback: () => { order.push('rollback'); }, + }); + expect(result).toBe(projection); + expect(order).toEqual(['start:30000', 'verify:10000']); + }); + + it('accepts a complete fresh fleet even if the launcher itself timed out', () => { + const rollback = vi.fn(); + expect(runBoundedPm2StartTransaction('start', 30, 10, { + start: () => { throw new Error('ETIMEDOUT'); }, + verifyFresh: () => 'complete-fresh-fleet', + rollback, + })).toBe('complete-fresh-fleet'); + expect(rollback).not.toHaveBeenCalled(); + }); + + it('rolls back a partial launch before exposing verification failure', () => { + const order: string[] = []; + expect(() => runBoundedPm2StartTransaction('restart-start', 30, 10, { + start: () => { order.push('start-partial'); throw new Error('socket closed'); }, + verifyFresh: () => { order.push('verify-fresh'); throw new Error('botmux-b unavailable'); }, + rollback: () => { order.push('rollback-partial'); }, + })).toThrow(/start: socket closed.*verify: botmux-b unavailable.*partial launch was rolled back/); + expect(order).toEqual(['start-partial', 'verify-fresh', 'rollback-partial']); + }); + + it('reports rollback failure without hiding the original start/verify evidence', () => { + expect(() => runBoundedPm2StartTransaction('start-bot', 30, 10, { + start: () => { throw new Error('launch failed'); }, + verifyFresh: () => { throw new Error('target transitional'); }, + rollback: () => { throw new Error('descriptor ambiguous'); }, + })).toThrow(/launch failed.*target transitional.*rollback failed: descriptor ambiguous/); + }); + + it('does not accept an empty first rollback read when a candidate publishes later', () => { + let now = 0; + const observations = [true, false, true, true]; + const reconcileOnce = vi.fn(() => observations.shift() ?? true); + + reconcileLatePm2StartPublication('start-bot', 10, 500, { + now: () => now, + sleep: ms => { now += ms; }, + reconcileOnce, + }); + + // First `true` is the empty projection. The late false observation resets + // the settle clock; only two later restored observations complete it. + expect(reconcileOnce).toHaveBeenCalledTimes(4); + expect(now).toBeGreaterThanOrEqual(120); + }); + + it('returns explicit uncertainty when late publication never settles', () => { + let now = 0; + expect(() => reconcileLatePm2StartPublication('restart-start', 10, 30, { + now: () => now, + sleep: ms => { now += ms; }, + reconcileOnce: () => false, + })).toThrow(/rollback remains uncertain.*late-publication settle window/); + }); +}); diff --git a/test/restart-intent-store.test.ts b/test/restart-intent-store.test.ts index f4e5a2150..1d8316d82 100644 --- a/test/restart-intent-store.test.ts +++ b/test/restart-intent-store.test.ts @@ -11,6 +11,11 @@ import { clearRestartLeaseTo, hasActiveRestartLeaseTo, writeManualIntentIfAbsentTo, + writeRestartAttemptIntentTo, + commitRestartIntentAttemptTo, + claimRestartIntentForReportTo, + hasPreparedRestartIntentTo, + removeRestartIntentAttemptTo, restartIntentPathIn, } from '../src/services/restart-intent-store.js'; @@ -103,4 +108,74 @@ describe('restart-intent store', () => { expect(consumeRestartIntentTo(dir, T0)).toBeNull(); expect(existsSync(restartIntentPathIn(dir))).toBe(false); }); + + it('rolls back only the exact failed start attempt and preserves a newer writer', () => { + writeRestartAttemptIntentTo(dir, { kind: 'manual', at: iso(T0) }, T0, 'attempt-old'); + expect(removeRestartIntentAttemptTo(dir, 'attempt-old')).toBe(true); + expect(consumeRestartIntentTo(dir, T0 + 500)).toBeNull(); + + writeRestartAttemptIntentTo(dir, { kind: 'manual', at: iso(T0) }, T0, 'attempt-old'); + writeRestartIntentTo(dir, { + kind: 'update', oldVersion: '1', newVersion: '2', at: iso(T0 + 1_000), + }); + expect(consumeRestartIntentTo(dir, T0 + 1_500)).toBeNull(); + expect(removeRestartIntentAttemptTo(dir, 'attempt-old')).toBe(true); + expect(consumeRestartIntentTo(dir, T0 + 2_000)).toBeNull(); + expect(hasPreparedRestartIntentTo(dir, T0 + 2_000)).toBe(false); + + writeRestartAttemptIntentTo( + dir, + { kind: 'manual', at: iso(T0 + 2_000) }, + T0 + 2_000, + 'attempt-new', + ); + expect(commitRestartIntentAttemptTo(dir, 'attempt-new')).toBe(true); + expect(consumeRestartIntentTo(dir, T0 + 3_000)).toMatchObject({ + kind: 'update', oldVersion: '1', newVersion: '2', + }); + }); + + it('does not expose a prepared restart until the exact attempt commits', () => { + writeRestartAttemptIntentTo( + dir, + { kind: 'manual', at: iso(T0) }, + T0, + 'attempt-verified', + ); + expect(hasPreparedRestartIntentTo(dir, T0 + 1_000)).toBe(true); + expect(consumeRestartIntentTo(dir, T0 + 1_000)).toBeNull(); + expect(commitRestartIntentAttemptTo(dir, 'wrong-attempt')).toBe(false); + expect(commitRestartIntentAttemptTo(dir, 'attempt-verified')).toBe(true); + expect(consumeRestartIntentTo(dir, T0 + 2_000)).toMatchObject({ + kind: 'manual', + attemptId: 'attempt-verified', + attemptState: 'committed', + }); + }); + + it('atomically claims a commit that lands after a prepared observation', () => { + writeRestartAttemptIntentTo( + dir, + { kind: 'manual', at: iso(T0) }, + T0, + 'attempt-racy-commit', + ); + expect(claimRestartIntentForReportTo(dir, T0 + 1_000)).toEqual({ state: 'prepared' }); + expect(commitRestartIntentAttemptTo(dir, 'attempt-racy-commit')).toBe(true); + expect(claimRestartIntentForReportTo(dir, T0 + 1_001)).toMatchObject({ + state: 'claimed', + intent: { attemptId: 'attempt-racy-commit', attemptState: 'committed' }, + }); + }); + + it('expires an abandoned prepared attempt without ever reporting it', () => { + writeRestartAttemptIntentTo( + dir, + { kind: 'manual', at: iso(T0) }, + T0, + 'attempt-crashed-cli', + ); + expect(consumeRestartIntentTo(dir, T0 + 11 * 60_000)).toBeNull(); + expect(existsSync(restartIntentPathIn(dir))).toBe(false); + }); }); diff --git a/test/restart-report.test.ts b/test/restart-report.test.ts index 34a2ef325..4f1e07fcb 100644 --- a/test/restart-report.test.ts +++ b/test/restart-report.test.ts @@ -4,7 +4,12 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { countActiveSessionsOnDisk } from '../src/services/session-store.js'; import { buildRestartReportText, sendRestartReportIfPending, fetchChangelog } from '../src/core/restart-report.js'; -import { writeRestartIntentTo, restartIntentPathIn } from '../src/services/restart-intent-store.js'; +import { + commitRestartIntentAttemptTo, + restartIntentPathIn, + writeRestartAttemptIntentTo, + writeRestartIntentTo, +} from '../src/services/restart-intent-store.js'; function writeSessions(dir: string, name: string, sessions: Record) { writeFileSync(join(dir, name), JSON.stringify(sessions)); @@ -177,6 +182,28 @@ describe('sendRestartReportIfPending', () => { await sendRestartReportIfPending(w); expect(sent).toHaveLength(1); }); + + it('atomically reclaims a commit that lands after the prepared observation', async () => { + writeRestartAttemptIntentTo( + dir, + { kind: 'manual', at: new Date(T0).toISOString() }, + T0, + 'attempt-full-fleet', + ); + const wait = vi.fn(async () => { + expect(commitRestartIntentAttemptTo(dir, 'attempt-full-fleet')).toBe(true); + }); + const { w, sent } = fakeWiring({ + wait, + preparedCommitWaitMs: 100, + }); + + await sendRestartReportIfPending(w); + + expect(wait).toHaveBeenCalledOnce(); + expect(sent).toHaveLength(1); + expect(existsSync(restartIntentPathIn(dir))).toBe(false); + }); }); describe('fetchChangelog', () => { diff --git a/test/shutdown-supervisor-contract.test.ts b/test/shutdown-supervisor-contract.test.ts new file mode 100644 index 000000000..8830736bc --- /dev/null +++ b/test/shutdown-supervisor-contract.test.ts @@ -0,0 +1,416 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS, + DAEMON_SHUTDOWN_MAX_MS, + DAEMON_SHUTDOWN_OVERHEAD_MS, + DAEMON_WORKER_EXIT_GRACE_MS, + FLEET_DAEMON_EXIT_WAIT_MS, + FLEET_SUCCESSOR_SETTLE_MS, + PM2_DAEMON_KILL_TIMEOUT_MS, + PM2_DAEMON_RESTART_DELAY_MS, + RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS, + RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS, + RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS, +} from '../src/core/shutdown-budgets.js'; +import { DAEMON_GRACEFUL_EXIT_CODE } from '../src/core/supervisor-shutdown-protocol.js'; + +const cli = readFileSync(new URL('../src/cli.ts', import.meta.url), 'utf8'); +const daemon = readFileSync(new URL('../src/daemon.ts', import.meta.url), 'utf8'); +const fleetShutdown = readFileSync(new URL('../src/cli/fleet-shutdown.ts', import.meta.url), 'utf8'); +const ipcServer = readFileSync(new URL('../src/core/dashboard-ipc-server.ts', import.meta.url), 'utf8'); +const pm2Preflight = readFileSync(new URL('../src/cli/pm2-preflight.ts', import.meta.url), 'utf8'); +const botsStore = readFileSync(new URL('../src/setup/bots-store.ts', import.meta.url), 'utf8'); +const bundledPm2God = readFileSync(new URL('../node_modules/pm2/lib/God.js', import.meta.url), 'utf8'); + +describe('graceful shutdown supervisor contract', () => { + it('uses a nonzero daemon-only graceful sentinel because PM2 maps signal death to zero', () => { + expect(DAEMON_GRACEFUL_EXIT_CODE).toBeGreaterThan(0); + expect(DAEMON_GRACEFUL_EXIT_CODE).toBeLessThan(256); + expect(bundledPm2God).toContain('God.handleExit(clu, code || 0, signal);'); + + const ecosystemStart = cli.indexOf('function ecosystemConfig('); + const daemonPolicy = cli.slice(ecosystemStart, cli.indexOf('const apps:', ecosystemStart)); + const dashboardPolicy = cli.slice( + cli.indexOf("name: 'botmux-dashboard'", ecosystemStart), + cli.indexOf('const cfg = { apps };', ecosystemStart), + ); + expect(daemonPolicy).toContain('stop_exit_codes: [DAEMON_GRACEFUL_EXIT_CODE]'); + expect(daemonPolicy).not.toContain('stop_exit_codes: [0]'); + expect(dashboardPolicy).toContain('stop_exit_codes: [0]'); + + const shutdownStart = daemon.indexOf('const shutdown = async () => {'); + const shutdownEnd = daemon.indexOf("process.on('SIGTERM'", shutdownStart); + const shutdown = daemon.slice(shutdownStart, shutdownEnd); + expect(shutdown).toContain('process.exit(DAEMON_GRACEFUL_EXIT_CODE);'); + expect(shutdown).not.toContain('process.exit(0);'); + expect(cli).toContain('assertDaemonPm2GracefulExitPolicy('); + expect(cli).toContain('`${operation}-handler-ready-pm2-policy`'); + + const projectionStart = cli.indexOf('function toBotmuxPm2ProcessEntry('); + const projectionEnd = cli.indexOf('function readVerifiedBotmuxPm2Projection(', projectionStart); + const projection = cli.slice(projectionStart, projectionEnd); + expect(projection).toContain( + 'stopExitCodes: normalizeRawPm2StopExitCodes(rawStopExitCodes)', + ); + expect(projection).not.toContain('.map(code => parsePm2Integer(code))'); + expect(bundledPm2God).toContain( + "stopExitCodes.map((strOrNum) => typeof strOrNum === 'string' ? parseInt(strOrNum, 10) : strOrNum)", + ); + expect(bundledPm2God).toContain('proc.pm2_env.unstable_restarts >= max_restarts'); + expect(bundledPm2God).toContain('if (!stopping && !overlimit)'); + expect(fleetShutdown).toContain('isFleetEntryProvenTerminalAfterSignal(exactState)'); + expect(fleetShutdown).toContain('post-signal terminal proof'); + expect(fleetShutdown).toContain('latestTrackedPidByName.get(trackedEntry.name) === pid'); + expect(fleetShutdown).toContain('a later missing row is never success'); + expect(fleetShutdown).toContain('liveReplacementPublished'); + expect(fleetShutdown).toContain("replacement's own"); + }); + + it('keeps outer supervisor budgets beyond both success and abort-restore paths', () => { + expect(DAEMON_SHUTDOWN_MAX_MS).toBe( + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS + + RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS + + RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS + + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS + + Math.max(RIFF_ADMISSION_RESTORE_TIMEOUT_MS, DAEMON_WORKER_EXIT_GRACE_MS) + + DAEMON_SHUTDOWN_OVERHEAD_MS, + ); + expect(DAEMON_SHUTDOWN_MAX_MS).toBeLessThanOrEqual(28_000); + expect(PM2_DAEMON_KILL_TIMEOUT_MS).toBeGreaterThan(DAEMON_SHUTDOWN_MAX_MS); + expect(FLEET_DAEMON_EXIT_WAIT_MS).toBeGreaterThan(PM2_DAEMON_KILL_TIMEOUT_MS); + expect(FLEET_SUCCESSOR_SETTLE_MS).toBeGreaterThan(PM2_DAEMON_RESTART_DELAY_MS); + expect(FLEET_DAEMON_EXIT_WAIT_MS) + .toBeGreaterThan(DAEMON_SHUTDOWN_MAX_MS + FLEET_SUCCESSOR_SETTLE_MS); + expect(FLEET_DAEMON_EXIT_WAIT_MS) + .toBeGreaterThan(PM2_DAEMON_KILL_TIMEOUT_MS + FLEET_SUCCESSOR_SETTLE_MS); + }); + + it('public stop signals and polls first, never deletes a possibly refusing entry', () => { + const start = cli.indexOf('async function cmdStop()'); + const end = cli.indexOf('async function cmdRestart()', start); + const stop = cli.slice(start, end); + const signal = stop.indexOf("signalAndAwaitBotmuxProcesses(entries, 'stop')"); + const preMutationProjection = stop.indexOf( + "readVerifiedBotmuxPm2Projection('stop-before-registry-mutation')", + signal, + ); + const pm2Stop = stop.indexOf("runPm2(['stop', String(entry.pmId)])", preMutationProjection); + const justInTimeOffset = stop.slice(preMutationProjection).search( + /revalidateExactQuiescentRowBeforeMutation\(\s*'stop-immediately-before-registry-mutation'/, + ); + const justInTime = justInTimeOffset < 0 + ? -1 + : preMutationProjection + justInTimeOffset; + const exactStop = stop.indexOf( + "runPm2(['stop', String(exact.pmId)]", + justInTime, + ); + + expect(signal).toBeGreaterThanOrEqual(0); + expect(preMutationProjection).toBeGreaterThan(signal); + expect(pm2Stop).toBe(-1); + expect(justInTime).toBeGreaterThan(preMutationProjection); + expect(exactStop).toBeGreaterThan(justInTime); + expect(stop).not.toContain("runPm2(['delete'"); + expect(stop).toContain('stopPluginServicesForCli(undefined, { autoOnly: true })'); + expect(stop).toContain('const stopErrors: string[] = []'); + expect(stop.indexOf("pm2Capture(['jlist'])", exactStop)).toBeGreaterThan(exactStop); + expect(stop.indexOf("assertNoUnregisteredLiveDaemonDescriptors('stop-after-registry-mutation'", exactStop)) + .toBeGreaterThan(exactStop); + expect(stop).toContain('PM2 registry mutation incomplete'); + }); + + it('restart waits for the fleet decision before any PM2 delete', () => { + const start = cli.indexOf('function deleteAllBotmuxProcesses('); + const end = cli.indexOf('/**\n * One-time migration', start); + const restart = cli.slice(start, end); + const signal = restart.indexOf("signalAndAwaitBotmuxProcesses(entries, 'restart', home"); + const justInTime = restart.indexOf( + "revalidateExactQuiescentRowBeforeMutation(\n 'restart-before-delete'", + signal, + ); + const remove = restart.indexOf("runPm2(['delete', String(exact.pmId)]", justInTime); + expect(signal).toBeGreaterThanOrEqual(0); + expect(justInTime).toBeGreaterThan(signal); + expect(remove).toBeGreaterThan(justInTime); + expect(restart).not.toContain("runPm2(['delete', ...exactIds]"); + expect(restart.indexOf("pm2Capture(['jlist'], home)", remove)).toBeGreaterThan(remove); + expect(restart.indexOf("assertNoUnregisteredLiveDaemonDescriptors(\n 'restart-after-delete'", remove)) + .toBeGreaterThan(remove); + expect(restart).toContain('PM2 delete left registry entries'); + }); + + it('takes one Riff snapshot, batch-persists, then generation-checks and commits before service stop', () => { + const start = daemon.indexOf('const shutdown = async () => {'); + const stop = daemon.indexOf('scheduler.stopScheduler();', start); + const boundedGate = daemon.indexOf('tryWithBotTurnMutation(', start); + const initialUnique = daemon.indexOf( + 'collectUniqueDaemonShutdownSessions(activeSessions.values())', + boundedGate, + ); + const prepareAll = daemon.indexOf('prepareRiffFleetForShutdown(riffCandidates', initialUnique); + const persistAll = daemon.indexOf('persistPreparedRiffShutdownFleet(riffPrepared', prepareAll); + const currentUnique = daemon.indexOf( + 'collectUniqueDaemonShutdownSessions(activeSessions.values())', + initialUnique + 1, + ); + const secondCheck = daemon.indexOf('const riffGenerationMismatch', persistAll); + const commitAll = daemon.indexOf('commitPreparedRiffShutdown(ds, result)', secondCheck); + const teardownUnique = daemon.indexOf( + 'for (const ds of currentShutdownFleet.sessions)', + commitAll, + ); + expect(boundedGate).toBeGreaterThan(start); + expect(initialUnique).toBeGreaterThan(boundedGate); + expect(prepareAll).toBeGreaterThan(initialUnique); + expect(persistAll).toBeGreaterThan(prepareAll); + expect(currentUnique).toBeGreaterThan(persistAll); + expect(secondCheck).toBeGreaterThan(currentUnique); + expect(commitAll).toBeGreaterThan(secondCheck); + expect(teardownUnique).toBeGreaterThan(commitAll); + expect(stop).toBeGreaterThan(commitAll); + expect(daemon.slice(start, stop)).toContain('abortRiffShutdownFleet('); + expect(daemon.slice(start, stop)).toContain('canAbortVerifiedExitedRiffPreparation('); + }); + + it('publishes shutdown capability only after both signal handlers are installed', () => { + const descStart = daemon.indexOf('const desc: DaemonDescriptor = {'); + const firstDescriptorWrite = daemon.indexOf('writeDaemonDescriptor(desc);', descStart); + const sigtermHandler = daemon.indexOf("process.on('SIGTERM'", firstDescriptorWrite); + const sigintHandler = daemon.indexOf("process.on('SIGINT'", sigtermHandler); + const capabilityCommit = daemon.indexOf( + 'desc.supervisorShutdownProtocol = SUPERVISOR_SHUTDOWN_PROTOCOL;', + sigintHandler, + ); + const ipcHandlerReady = daemon.indexOf('setSupervisorShutdownHandler({', sigintHandler); + const attestedWrite = daemon.indexOf('writeDaemonDescriptor(desc);', capabilityCommit); + + expect(descStart).toBeGreaterThanOrEqual(0); + expect(firstDescriptorWrite).toBeGreaterThan(descStart); + expect(daemon.slice(descStart, firstDescriptorWrite)) + .not.toContain('supervisorShutdownProtocol: SUPERVISOR_SHUTDOWN_PROTOCOL'); + expect(sigtermHandler).toBeGreaterThan(firstDescriptorWrite); + expect(sigintHandler).toBeGreaterThan(sigtermHandler); + expect(ipcHandlerReady).toBeGreaterThan(sigintHandler); + expect(capabilityCommit).toBeGreaterThan(ipcHandlerReady); + expect(attestedWrite).toBeGreaterThan(capabilityCommit); + }); + + it('uses exact-id conditional PM2 start for proven-offline compensation, never public start/restart', () => { + const start = cli.indexOf('startOffline: (offlineEntries, timeoutMs) =>'); + const endOffset = cli.slice(start).search(/\n\s+list,/); + const end = endOffset < 0 ? -1 : start + endOffset; + const compensation = cli.slice(start, end); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(compensation).toContain('runExactPm2Starts(offlineEntries'); + expect(compensation).not.toContain("runPm2(['start'"); + expect(compensation).not.toContain("runPm2(['restart'"); + }); + + it('serializes every core PM2 mutation surface on one async fleet lock', () => { + const regions = [ + ['start', 'async function cmdStart()', '/**\n * Wipe stale dashboard-daemon descriptors'], + ['stop', 'async function cmdStop()', 'async function cmdRestart()'], + ['restart', 'async function cmdRestart()', '/**\n * Bring a SINGLE bot'], + ['start-bot', 'async function ensureBotDaemonStarted(', '/**\n * `botmux start-bot'], + ] as const; + for (const [label, startMarker, endMarker] of regions) { + const start = cli.indexOf(startMarker); + const end = cli.indexOf(endMarker, start); + const region = cli.slice(start, end); + expect(start, label).toBeGreaterThanOrEqual(0); + expect(end, label).toBeGreaterThan(start); + expect(region, label).toContain('withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET'); + expect(region, label).not.toContain('withFileLockSync(PM2_FLEET_MUTATION_LOCK_TARGET'); + } + + const exactHelper = cli.slice( + cli.indexOf('async function cmdInternalPm2StartExact('), + cli.indexOf('function runExactPm2Starts(', cli.indexOf('async function cmdInternalPm2StartExact(')), + ); + expect(exactHelper).toContain('BOTMUX_PM2_FLEET_LOCK_OWNER_PID'); + expect(exactHelper).toContain('PM2_FLEET_MUTATION_LOCK_TARGET}.lock'); + expect(exactHelper).toContain('lockPid !== process.ppid'); + }); + + it('fails closed before PM2 mutation on duplicate Gods, stale preflight, or unregistered descriptors', () => { + const duplicateStart = cli.indexOf('function listSingletonPm2GodDaemonPidsForMutation('); + const duplicateEnd = cli.indexOf('function runPm2(', duplicateStart); + const duplicate = cli.slice(duplicateStart, duplicateEnd); + expect(duplicate).toContain('multiple PM2 God daemons'); + expect(duplicate).not.toContain("process.kill(pid, 'SIGTERM')"); + expect(duplicate).not.toContain("process.kill(pid, 'SIGKILL')"); + + const preflightStart = cli.indexOf('function preflightNodeSanity('); + const preflightEnd = cli.indexOf('async function cmdStart()', preflightStart); + const preflight = cli.slice(preflightStart, preflightEnd); + expect(preflight).toContain('assertLinuxPm2GodExecutableUsable(pm2Pid)'); + expect(preflight).toContain('listPm2GodDaemonPids(home)'); + expect(preflight).not.toContain("join(PM2_HOME, 'pm2.pid')"); + expect(preflight).not.toContain("runPm2(['kill']"); + expect(preflight).not.toContain("'SIGKILL'"); + expect(pm2Preflight).toContain('拒绝自动清理'); + + for (const operation of ['start', 'stop', 'start-bot', 'restart-start']) { + expect(cli).toContain(`assertNoUnregisteredLiveDaemonDescriptors('${operation}'`); + } + }); + + it('publishes manual restart intent only after verified fleet retirement', () => { + const start = cli.indexOf('async function cmdRestart()'); + const end = cli.indexOf('/**\n * Bring a SINGLE bot', start); + const restart = cli.slice(start, end); + const staged = restart.indexOf('consumeRestartIntentTo('); + const preflight = restart.indexOf('assertNoDuplicatePm2GodDaemons()', staged); + const retirement = restart.indexOf('deleteAllBotmuxProcesses()'); + const descriptorCheck = restart.indexOf( + "assertNoUnregisteredLiveDaemonDescriptors('restart-start'", + retirement, + ); + const intent = restart.indexOf('writeRestartAttemptIntentTo(', descriptorCheck); + const transaction = restart.indexOf('runBoundedPm2StartTransaction(', intent); + expect(restart.slice(transaction, transaction + 96)).toContain("'restart-start'"); + const newFleet = restart.indexOf("runPm2(['start', cfg]", transaction); + const verify = restart.indexOf("'restart-after-launch'", newFleet); + const compensate = restart.indexOf('rollbackPm2StartAttempt(', verify); + expect(restart.slice(compensate, compensate + 128)).toContain("'restart-start'"); + const rollback = restart.indexOf('removeRestartIntentAttemptTo(', compensate); + const commit = restart.indexOf('commitRestartIntentAttemptTo(', rollback); + expect(staged).toBeGreaterThanOrEqual(0); + expect(preflight).toBeGreaterThan(staged); + expect(retirement).toBeGreaterThanOrEqual(0); + expect(descriptorCheck).toBeGreaterThan(retirement); + expect(intent).toBeGreaterThan(descriptorCheck); + expect(transaction).toBeGreaterThan(intent); + expect(newFleet).toBeGreaterThan(transaction); + expect(verify).toBeGreaterThan(newFleet); + expect(compensate).toBeGreaterThan(verify); + expect(rollback).toBeGreaterThan(compensate); + expect(commit).toBeGreaterThan(rollback); + }); + + it('bounds and freshly verifies every public PM2 start surface with compensation', () => { + const regions = [ + cli.slice(cli.indexOf('async function cmdStart()'), cli.indexOf('/**\n * Wipe stale dashboard-daemon descriptors')), + cli.slice(cli.indexOf('async function cmdRestart()'), cli.indexOf('/**\n * Bring a SINGLE bot')), + cli.slice(cli.indexOf('async function ensureBotDaemonStarted('), cli.indexOf('/**\n * `botmux start-bot')), + ]; + for (const region of regions) { + expect(region).toContain('runBoundedPm2StartTransaction('); + expect(region).toContain('PM2_START_COMMAND_TIMEOUT_MS'); + expect(region).toContain('readAndAssertConfiguredFleetOnline('); + expect(region).toContain('rollbackPm2StartAttempt('); + expect(region).toContain('timeoutMs'); + } + }); + + it('holds one bots.json generation from ecosystem rendering through verification/rollback', () => { + expect(botsStore).toContain('withFileLockSync(botsJsonPath'); + const regions = [ + cli.slice(cli.indexOf('async function cmdStart()'), cli.indexOf('/**\n * Wipe stale dashboard-daemon descriptors')), + cli.slice(cli.indexOf('async function cmdRestart()'), cli.indexOf('/**\n * Bring a SINGLE bot')), + cli.slice(cli.indexOf('async function ensureBotDaemonStarted('), cli.indexOf('/**\n * `botmux start-bot')), + ]; + for (const region of regions) { + expect(region).toContain('withFileLock(BOTS_JSON_FILE'); + expect(region).toContain('ecosystemConfig('); + expect(region).toContain('configuredCoreProcessNames('); + expect(region).toContain('assertBotsConfigSnapshotUnchanged('); + } + }); + + it('admits start-bot only through the exact configured fleet classifier', () => { + const start = cli.indexOf('async function ensureBotDaemonStarted('); + const end = cli.indexOf('/**\n * `botmux start-bot', start); + const region = cli.slice(start, end); + expect(region).toContain('classifyStartBotFleetAdmission('); + expect(region).toContain("admission.state === 'already-online'"); + const alreadyOnline = region.slice( + region.indexOf("admission.state === 'already-online'"), + region.indexOf("admission.state === 'fleet-down'"), + ); + expect(alreadyOnline).toContain("'start-bot-already-online-ready'"); + expect(alreadyOnline).toContain('readAndAssertConfiguredFleetOnline('); + expect(region).toContain("admission.state === 'fleet-down'"); + expect(region).toContain("'start-bot-after-launch'"); + expect(region).toContain('preflightNodeSanity()'); + }); + + it('requires fresh handler-ready exact-set verification before idempotent start returns', () => { + const start = cli.indexOf('async function cmdStart()'); + const end = cli.indexOf('/**\n * Wipe stale dashboard-daemon descriptors', start); + const region = cli.slice(start, end); + const liveBranch = region.slice( + region.indexOf('if (liveEntries.length > 0)'), + region.indexOf('const unprovenDormant'), + ); + const verify = liveBranch.indexOf('readAndAssertConfiguredFleetOnline('); + const ready = liveBranch.indexOf("'start-idempotent-ready'", verify); + const returns = liveBranch.indexOf('return;', ready); + expect(verify).toBeGreaterThanOrEqual(0); + expect(ready).toBeGreaterThan(verify); + expect(returns).toBeGreaterThan(ready); + expect(liveBranch).not.toContain('assertConfiguredPm2FleetOnline('); + }); + + it('discovers legacy Gods from the process table and rechecks duplicate Gods before mutation', () => { + const start = cli.indexOf('function cleanupLegacyPm2()'); + const end = cli.indexOf('async function cmdStop()', start); + const legacy = cli.slice(start, end); + expect(legacy).toContain('listPm2GodDaemonPids(legacyHome)'); + expect(legacy).not.toContain("join(legacyHome, 'pm2.pid')"); + expect(legacy).toContain('assertNoDuplicatePm2GodDaemons(legacyHome)'); + expect(legacy).toContain('preflightNodeSanity(legacyHome)'); + + expect(cli).not.toContain("runPm2(['kill']"); + }); + + it('rejects include-pm2 before breadcrumb/fleet mutation when a live God exists', () => { + const start = cli.indexOf('async function cmdRestart()'); + const end = cli.indexOf('/**\n * Bring a SINGLE bot', start); + const restart = cli.slice(start, end); + const admission = restart.indexOf( + 'assertIncludePm2RestartAdmission(listPm2GodDaemonPids())', + ); + const consume = restart.indexOf('consumeRestartIntentTo('); + const retire = restart.indexOf('deleteAllBotmuxProcesses()'); + expect(admission).toBeGreaterThanOrEqual(0); + expect(consume).toBeGreaterThan(admission); + expect(retire).toBeGreaterThan(consume); + expect(restart).not.toContain('killPm2GodDaemon'); + expect(cli).toContain('--include-pm2 仅允许“入场时没有 live PM2 God”的干净启动'); + expect(cli).not.toContain('--include-pm2 同时重启 PM2 God'); + }); + + it('attests the whole daemon fleet then uses exact IPC batch/successor requests', () => { + const start = cli.indexOf('function signalAndAwaitBotmuxProcesses('); + const end = cli.indexOf('/** Compensate only rows owned', start); + const helper = cli.slice(start, end); + const preflight = helper.indexOf('`${operation}-shutdown-capability-preflight`'); + const fleet = helper.indexOf('signalAndAwaitFleet(', preflight); + const batch = helper.indexOf('requestAttestedDaemonShutdownBatch(', fleet); + const successor = helper.indexOf('requestAttestedDaemonShutdown(', fleet); + expect(preflight).toBeGreaterThanOrEqual(0); + expect(fleet).toBeGreaterThan(preflight); + expect(batch).toBeGreaterThan(fleet); + expect(successor).toBeGreaterThan(fleet); + expect(helper).toContain('signalInitial: targets =>'); + expect(helper).toContain('processStartByPid'); + expect(cli).toContain("return isBotmuxCoreProcessName(name) && name !== 'botmux-dashboard'"); + }); + + it('keeps supervisor shutdown host-authenticated and exact boot/birth bound', () => { + const route = ipcServer.slice( + ipcServer.indexOf("ipcRoute('POST', SUPERVISOR_SHUTDOWN_ROUTE"), + ipcServer.indexOf('export async function readJsonBody', + ipcServer.indexOf("ipcRoute('POST', SUPERVISOR_SHUTDOWN_ROUTE")), + ); + expect(route).toContain('isTrustedHostIpcRequest(req)'); + expect(route).toContain('isExactSupervisorShutdownRequest(registration, body)'); + expect(route).toContain('jsonRes(res, 202'); + expect(route.indexOf('jsonRes(res, 202')).toBeLessThan(route.indexOf('registration.shutdown()')); + }); +}); diff --git a/test/supervisor-shutdown-client.test.ts b/test/supervisor-shutdown-client.test.ts new file mode 100644 index 000000000..e2eb47721 --- /dev/null +++ b/test/supervisor-shutdown-client.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + requestAttestedDaemonShutdown, + requestAttestedDaemonShutdownBatch, +} from '../src/cli/supervisor-shutdown-client.js'; +import type { AttestedPm2DaemonShutdownTarget } from '../src/cli/pm2-shutdown-capability.js'; + +const target: AttestedPm2DaemonShutdownTarget = { + name: 'botmux-a', + pid: 101, + larkAppId: 'cli_a', + ipcPort: 7901, + bootInstanceId: 'boot-a', + processStartIdentity: 'birth-a', +}; + +describe('attested daemon supervisor shutdown client', () => { + it('accepts only an exact 202 ACK from the target boot/birth', () => { + const postMany = vi.fn(() => [{ + status: 202, + bodyRaw: JSON.stringify({ + ok: true, + accepted: true, + larkAppId: 'cli_a', + bootInstanceId: 'boot-a', + processStartIdentity: 'birth-a', + }), + }]); + expect(() => requestAttestedDaemonShutdown(target, 'secret', { + readStartIdentity: () => 'birth-a', + postMany, + })).not.toThrow(); + expect(postMany).toHaveBeenCalledOnce(); + }); + + it('never contacts a same-PID successor with a different birth', () => { + const postMany = vi.fn(); + expect(() => requestAttestedDaemonShutdown(target, 'secret', { + readStartIdentity: () => 'birth-b', + postMany, + })).toThrow(/process generation changed/); + expect(postMany).not.toHaveBeenCalled(); + }); + + it('fails closed on a generation-mismatch response', () => { + expect(() => requestAttestedDaemonShutdown(target, 'secret', { + readStartIdentity: () => 'birth-a', + postMany: () => [{ + status: 409, + bodyRaw: JSON.stringify({ ok: false, error: 'supervisor_shutdown_generation_mismatch' }), + }], + })).toThrow(/rejected exact supervisor shutdown.*status 409/); + }); + + it('dispatches the initial fleet in one batch so a timeout does not suppress peers', () => { + const peer = { ...target, name: 'botmux-b', pid: 202, larkAppId: 'cli_b', ipcPort: 7902, + bootInstanceId: 'boot-b', processStartIdentity: 'birth-b' }; + const postMany = vi.fn((inputs) => { + expect(inputs).toHaveLength(2); + return [ + { error: 'supervisor shutdown request timed out' }, + { status: 202, bodyRaw: JSON.stringify({ + ok: true, accepted: true, larkAppId: 'cli_b', bootInstanceId: 'boot-b', + processStartIdentity: 'birth-b', + }) }, + ]; + }); + const attempts = requestAttestedDaemonShutdownBatch([target, peer], 'secret', { + readStartIdentity: pid => pid === 101 ? 'birth-a' : 'birth-b', + postMany, + }); + expect(postMany).toHaveBeenCalledOnce(); + expect(attempts.map(attempt => attempt.ok)).toEqual([false, true]); + }); +}); diff --git a/test/supervisor-shutdown-ipc.test.ts b/test/supervisor-shutdown-ipc.test.ts new file mode 100644 index 000000000..bd4bb0362 --- /dev/null +++ b/test/supervisor-shutdown-ipc.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; +import { isExactSupervisorShutdownRequest } from '../src/core/supervisor-shutdown-ipc.js'; + +const identity = { + larkAppId: 'cli_a', + bootInstanceId: 'boot-a', + processStartIdentity: 'birth-a', +}; + +describe('generation-bound supervisor shutdown request', () => { + it('accepts only the exact in-process app/boot/birth tuple', () => { + expect(isExactSupervisorShutdownRequest(identity, { ...identity })).toBe(true); + }); + + it('rejects a successor on the same port or reused PID', () => { + expect(isExactSupervisorShutdownRequest(identity, { + ...identity, + bootInstanceId: 'boot-b', + })).toBe(false); + expect(isExactSupervisorShutdownRequest(identity, { + ...identity, + processStartIdentity: 'birth-b', + })).toBe(false); + }); +}); diff --git a/test/supervisor-shutdown-loopback.integration.test.ts b/test/supervisor-shutdown-loopback.integration.test.ts new file mode 100644 index 000000000..3179a3ab9 --- /dev/null +++ b/test/supervisor-shutdown-loopback.integration.test.ts @@ -0,0 +1,83 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { daemonIpcAuthHeaders } from '../src/core/daemon-ipc-auth.js'; +import { + setIpcAuthSecret, + setSupervisorShutdownHandler, + startIpcServer, + type IpcServerHandle, +} from '../src/core/dashboard-ipc-server.js'; +import { SUPERVISOR_SHUTDOWN_ROUTE } from '../src/core/supervisor-shutdown-ipc.js'; + +const SECRET = 'supervisor-loopback-test-secret'; +const IDENTITY = { + larkAppId: 'cli_loopback', + bootInstanceId: 'boot-loopback', + processStartIdentity: 'birth-loopback', +}; + +let server: IpcServerHandle | null = null; + +afterEach(async () => { + setSupervisorShutdownHandler(null); + setIpcAuthSecret(null); + if (server) await server.close(); + server = null; +}); + +async function signedPost(body: unknown): Promise { + if (!server) throw new Error('test IPC server is not running'); + return fetch(`http://127.0.0.1:${server.port}${SUPERVISOR_SHUTDOWN_ROUTE}`, { + method: 'POST', + headers: daemonIpcAuthHeaders({ + secret: SECRET, + port: server.port, + method: 'POST', + path: SUPERVISOR_SHUTDOWN_ROUTE, + headers: { 'content-type': 'application/json' }, + }), + body: JSON.stringify(body), + }); +} + +describe('supervisor shutdown signed loopback handshake', () => { + it('authenticates the real route and accepts only the registered boot/birth generation', async () => { + setIpcAuthSecret(SECRET); + server = await startIpcServer({ + port: 0, + host: '127.0.0.1', + authRequired: true, + ready: Promise.resolve(), + }); + + const unsigned = await fetch( + `http://127.0.0.1:${server.port}${SUPERVISOR_SHUTDOWN_ROUTE}`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(IDENTITY), + }, + ); + expect(unsigned.status).toBe(401); + + const beforeRegistration = await signedPost(IDENTITY); + expect(beforeRegistration.status).toBe(503); + + const shutdown = vi.fn(async () => {}); + setSupervisorShutdownHandler({ ...IDENTITY, shutdown }); + + const wrongBoot = await signedPost({ ...IDENTITY, bootInstanceId: 'boot-successor' }); + expect(wrongBoot.status).toBe(409); + const wrongBirth = await signedPost({ ...IDENTITY, processStartIdentity: 'birth-successor' }); + expect(wrongBirth.status).toBe(409); + expect(shutdown).not.toHaveBeenCalled(); + + const accepted = await signedPost(IDENTITY); + expect(accepted.status).toBe(202); + expect(await accepted.json()).toEqual({ + ok: true, + accepted: true, + ...IDENTITY, + }); + await vi.waitFor(() => expect(shutdown).toHaveBeenCalledOnce()); + }); +}); From 06b2dc08fb352bfae5227ece938f85e722f9b627 Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:20:18 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix(pm2):=20=E8=A1=A5=E9=BD=90=E9=A6=96?= =?UTF-8?q?=E6=AC=A1=E5=8D=87=E7=BA=A7=E4=B8=8E=E5=90=AF=E5=8A=A8=E9=A2=84?= =?UTF-8?q?=E7=AE=97=E5=9B=B4=E6=A0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli.ts | 159 ++++++++++++++++++++-- src/cli/fleet-shutdown.ts | 40 ++++-- src/cli/pm2-shutdown-capability.ts | 3 +- src/cli/pm2-start-transaction.ts | 4 +- src/core/shutdown-budgets.ts | 2 +- test/fleet-shutdown.test.ts | 41 ++++++ test/pm2-start-transaction.test.ts | 4 +- test/shutdown-supervisor-contract.test.ts | 22 ++- 8 files changed, 244 insertions(+), 31 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 17b111203..5a3f50921 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,8 @@ * botmux stop [--with-plugin] — stop daemon (optionally stop auto plugin services) * botmux restart [--include-pm2] [--with-plugin] — restart daemon, then ensure auto plugin services; * --include-pm2 is a zero-live-God admission fence, not authority to signal an existing PM2 God + * botmux restart --bootstrap-shutdown-protocol --yes — operator-approved one-time retirement + * of a pre-protocol fleet after independently confirming all Session/Riff work is idle * botmux logs [--lines] — view daemon logs * botmux status — show daemon status * botmux upgrade|update — upgrade to latest version @@ -248,9 +250,17 @@ const PM2_NAME = 'botmux'; const PM2_HOME = join(CONFIG_DIR, 'pm2'); const PM2_FLEET_MUTATION_LOCK_TARGET = join(CONFIG_DIR, 'pm2-fleet-mutation'); const PM2_START_COMMAND_TIMEOUT_MS = 30_000; -const PM2_START_VERIFY_TIMEOUT_MS = 10_000; +const PM2_START_VERIFY_MIN_TIMEOUT_MS = 60_000; +const PM2_START_VERIFY_PER_PROCESS_MS = 2_000; const PM2_START_LATE_PUBLICATION_SETTLE_MS = 10_000; +function pm2StartVerifyTimeoutMs(processCount: number): number { + return Math.max( + PM2_START_VERIFY_MIN_TIMEOUT_MS, + Math.max(1, Math.floor(processCount)) * PM2_START_VERIFY_PER_PROCESS_MS, + ); +} + // ─── Helpers ───────────────────────────────────────────────────────────────── function ensureConfigDir(): void { @@ -2468,6 +2478,7 @@ async function cmdStart(): Promise { assertNoUnregisteredLiveDaemonDescriptors('start', currentProjection); assertCanonicalUniquePm2Rows('start', currentProjection); const configuredNames = configuredCoreProcessNames(lockedBots); + const verifyTimeoutMs = pm2StartVerifyTimeoutMs(configuredNames.length); const cfg = ecosystemConfig(lockedBots); const liveEntries = currentProjection.filter(isLivePm2Entry); if (liveEntries.length > 0) { @@ -2476,7 +2487,7 @@ async function cmdStart(): Promise { 'start-idempotent-ready', configuredNames, PM2_HOME, - PM2_START_VERIFY_TIMEOUT_MS, + verifyTimeoutMs, ); return; } catch (error) { @@ -2500,7 +2511,7 @@ async function cmdStart(): Promise { runBoundedPm2StartTransaction( 'start', PM2_START_COMMAND_TIMEOUT_MS, - PM2_START_VERIFY_TIMEOUT_MS, + verifyTimeoutMs, { start: timeoutMs => { assertBotsConfigSnapshotUnchanged('start', lockedBots); @@ -2667,7 +2678,7 @@ function readAndAssertConfiguredFleetOnline( operation: string, configuredNames: string[], home: string = PM2_HOME, - timeoutMs: number = PM2_START_VERIFY_TIMEOUT_MS, + timeoutMs: number = PM2_START_VERIFY_MIN_TIMEOUT_MS, ): BotmuxPm2ProcessEntry[] { const deadline = Date.now() + Math.max(1, Math.floor(timeoutMs)); let lastError: unknown; @@ -3146,6 +3157,90 @@ function deleteAllBotmuxProcesses( } } +/** + * Explicit first-upgrade escape hatch for a live fleet that predates the + * authenticated shutdown protocol. This intentionally bypasses descriptor + * capability attestation, but only behind a named flag plus --yes. Every PM2 + * delete is still bound to the exact name + pm_id + PID + process birth read + * before the first mutation and revalidated immediately before that mutation. + */ +function bootstrapDeleteAllBotmuxProcesses( + operation: 'stop' | 'restart', + home: string = PM2_HOME, +): void { + assertNoDuplicatePm2GodDaemons(home); + const readProjection = (phase: string): BotmuxPm2ProcessEntry[] => { + const apps = parsePm2JlistOutputStrict(pm2Capture(['jlist'], home)); + const projection = (Array.isArray(apps) ? apps : []) + .filter(app => app && isBotmuxCoreProcessName(String(app.name))) + .map(toBotmuxPm2ProcessEntry); + assertCanonicalUniquePm2Rows(`${operation}-bootstrap-${phase}`, projection); + return projection; + }; + + const entries = readProjection('initial'); + const identities = new Map(); + for (const entry of entries) { + if (!Number.isInteger(entry.pmId)) { + throw new Error( + `[${operation}] bootstrap refused: ${entry.name} has no canonical PM2 id`, + ); + } + if (entry.pid > 1 && isLivePm2Entry(entry)) { + const identity = readSupervisorProcessStartIdentity(entry.pid); + if (!identity) { + throw new Error( + `[${operation}] bootstrap refused: cannot bind ${entry.name}/${entry.pid} to a process birth`, + ); + } + identities.set(entry.pmId!, identity); + } + } + + for (const original of entries) { + const fresh = readProjection(`before-delete-${original.pmId}`); + const exact = fresh.filter(entry => entry.pmId === original.pmId); + if (exact.length !== 1 || exact[0]!.name !== original.name) { + throw new Error( + `[${operation}] bootstrap refused: PM2 row ${original.name}/${original.pmId} changed before delete`, + ); + } + const current = exact[0]!; + const expectedBirth = identities.get(original.pmId!); + if (expectedBirth) { + const currentBirth = current.pid > 1 + ? readSupervisorProcessStartIdentity(current.pid) + : undefined; + if (current.pid !== original.pid + || !isLivePm2Entry(current) + || currentBirth !== expectedBirth) { + throw new Error( + `[${operation}] bootstrap refused: process generation changed for ` + + `${original.name}/${original.pmId}`, + ); + } + } else if (isLivePm2Entry(current)) { + throw new Error( + `[${operation}] bootstrap refused: dormant PM2 row ${original.name}/${original.pmId} became live`, + ); + } + runPm2( + ['delete', String(current.pmId)], + false, + home, + FLEET_DAEMON_EXIT_WAIT_MS, + ); + } + + const remaining = readProjection('after-delete'); + if (remaining.length > 0) { + throw new Error( + `[${operation}] bootstrap retirement incomplete: ` + + remaining.map(entry => `${entry.name}/${entry.pmId}`).join(', '), + ); + } +} + /** * One-time migration for users upgrading from versions that used the default * ~/.pm2 directory. Removes any lingering botmux-* processes registered under @@ -3153,25 +3248,46 @@ function deleteAllBotmuxProcesses( * truth. Only touches processes named `botmux` or `botmux-*` — the user's * unrelated pm2 apps are left untouched. No-op on fresh installs. */ -function cleanupLegacyPm2(): boolean { +function cleanupLegacyPm2( + bootstrapOperation?: 'stop' | 'restart', +): boolean { const legacyHome = join(homedir(), '.pm2'); if (legacyHome === PM2_HOME) return false; const legacyGodPids = listPm2GodDaemonPids(legacyHome); if (legacyGodPids.length === 0) return false; assertNoDuplicatePm2GodDaemons(legacyHome); preflightNodeSanity(legacyHome); - const currentProjection = readVerifiedBotmuxPm2Projection('legacy-cleanup-authority'); assertNoDuplicatePm2GodDaemons(legacyHome); - deleteAllBotmuxProcesses(legacyHome, currentProjection); + if (bootstrapOperation) bootstrapDeleteAllBotmuxProcesses(bootstrapOperation, legacyHome); + else { + const currentProjection = readVerifiedBotmuxPm2Projection('legacy-cleanup-authority'); + deleteAllBotmuxProcesses(legacyHome, currentProjection); + } return true; } async function cmdStop(): Promise { const includePluginServices = process.argv.includes('--with-plugin'); + const bootstrapShutdownProtocol = process.argv.includes('--bootstrap-shutdown-protocol'); + const bootstrapConfirmed = process.argv.includes('--yes'); + if (bootstrapShutdownProtocol && !bootstrapConfirmed) { + throw new Error( + '[stop] --bootstrap-shutdown-protocol requires --yes after confirming every Session/Riff workload is idle', + ); + } ensureConfigDir(); await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { assertNoDuplicatePm2GodDaemons(); - cleanupLegacyPm2(); + cleanupLegacyPm2(bootstrapShutdownProtocol ? 'stop' : undefined); + if (bootstrapShutdownProtocol) { + bootstrapDeleteAllBotmuxProcesses('stop'); + cleanupStaleDaemonDescriptors(); + if (includePluginServices) { + await stopPluginServicesForCli(undefined, { autoOnly: true }); + } + console.log('daemon 已通过一次性 shutdown-protocol bootstrap 安全边界停止。'); + return; + } let entries: BotmuxPm2ProcessEntry[]; try { entries = parsePm2JlistOutputStrict(pm2Capture(['jlist'])) @@ -3266,6 +3382,16 @@ async function cmdRestart(): Promise { await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { const includePm2 = process.argv.includes('--include-pm2'); const includePluginServices = process.argv.includes('--with-plugin'); + const bootstrapShutdownProtocol = process.argv.includes('--bootstrap-shutdown-protocol'); + const bootstrapConfirmed = process.argv.includes('--yes'); + if (bootstrapShutdownProtocol && !bootstrapConfirmed) { + throw new Error( + '[restart] --bootstrap-shutdown-protocol requires --yes after confirming every Session/Riff workload is idle', + ); + } + if (bootstrapShutdownProtocol && includePm2) { + throw new Error('[restart] --bootstrap-shutdown-protocol cannot be combined with --include-pm2'); + } if (includePm2) { assertIncludePm2RestartAdmission(listPm2GodDaemonPids()); } @@ -3279,8 +3405,9 @@ async function cmdRestart(): Promise { assertNoDuplicatePm2GodDaemons(); preflightNodeSanity(); await ensureSystemDependencies(); - cleanupLegacyPm2(); - deleteAllBotmuxProcesses(); + cleanupLegacyPm2(bootstrapShutdownProtocol ? 'restart' : undefined); + if (bootstrapShutdownProtocol) bootstrapDeleteAllBotmuxProcesses('restart'); + else deleteAllBotmuxProcesses(); if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true }); cleanupStaleDaemonDescriptors(); @@ -3311,10 +3438,11 @@ async function cmdRestart(): Promise { try { const configuredNames = configuredCoreProcessNames(restartBots); + const verifyTimeoutMs = pm2StartVerifyTimeoutMs(configuredNames.length); runBoundedPm2StartTransaction( 'restart-start', PM2_START_COMMAND_TIMEOUT_MS, - PM2_START_VERIFY_TIMEOUT_MS, + verifyTimeoutMs, { start: timeoutMs => { assertBotsConfigSnapshotUnchanged('restart-start', restartBots); @@ -3514,6 +3642,7 @@ async function ensureBotDaemonStarted( bots, activationJobId ? appId : undefined, ); + const verifyTimeoutMs = pm2StartVerifyTimeoutMs(configuredNames.length); const inspection = listBotmuxPm2Apps(); if (!inspection.ok) { return { ok: false, reason: 'pm2_error', message: inspection.message }; @@ -3543,7 +3672,7 @@ async function ensureBotDaemonStarted( 'start-bot-already-online-ready', configuredNames, PM2_HOME, - PM2_START_VERIFY_TIMEOUT_MS, + verifyTimeoutMs, ); return { ok: true, state: 'already-online', processName }; } @@ -3576,7 +3705,7 @@ async function ensureBotDaemonStarted( 'start-bot-already-online-ready', configuredNames, PM2_HOME, - PM2_START_VERIFY_TIMEOUT_MS, + verifyTimeoutMs, ); return { ok: true, state: 'already-online', processName }; } @@ -3589,7 +3718,7 @@ async function ensureBotDaemonStarted( runBoundedPm2StartTransaction( 'start-bot', PM2_START_COMMAND_TIMEOUT_MS, - PM2_START_VERIFY_TIMEOUT_MS, + verifyTimeoutMs, { start: timeoutMs => { assertBotsConfigSnapshotUnchanged('start-bot', bots); @@ -5603,6 +5732,8 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 stop 停止 daemon（默认不停止插件 service；--with-plugin 显式停止 mode=auto 的插件 service） restart 重启 daemon（默认不停止插件 service，core 启动后确保 mode=auto 正在运行；--with-plugin 显式先停再启动 auto service） --include-pm2 仅允许“入场时没有 live PM2 God”的干净启动；若已有 live God，整条命令会在 fleet/breadcrumb 零改动处拒绝，且不会信号或重启现存 God + 首次升级若旧 daemon 缺少 shutdown protocol：先独立确认所有 Session/Riff 工作均 idle，再一次性运行 + botmux restart --bootstrap-shutdown-protocol --yes；普通 stop/restart 仍保持 fail-closed logs 查看 daemon 日志（--lines N, --bot <0-based-index|pm2-name|appId>） status 查看 daemon 状态 upgrade 升级到最新版本（别名：update） diff --git a/src/cli/fleet-shutdown.ts b/src/cli/fleet-shutdown.ts index 84df86c6d..d6ba8ebc7 100644 --- a/src/cli/fleet-shutdown.ts +++ b/src/cli/fleet-shutdown.ts @@ -137,15 +137,24 @@ export function signalAndAwaitFleet( purpose: string, capMs: number = Number.POSITIVE_INFINITY, ): FleetProcessEntry[] => { - const budgetMs = remainingMs(); - if (budgetMs <= 0) { - throw new Error(`fleet deadline exhausted before ${purpose}`); - } - const projection = runtime.list(Math.max(1, Math.min(budgetMs, Math.floor(capMs)))); - if (remainingMs() <= 0) { - throw new Error(`fleet deadline exhausted during ${purpose}`); + let lastError: unknown; + for (let attempt = 1; attempt <= 2; attempt++) { + const budgetMs = remainingMs(); + if (budgetMs <= 0) { + throw new Error(`fleet deadline exhausted before ${purpose}`); + } + try { + const projection = runtime.list(Math.max(1, Math.min(budgetMs, Math.floor(capMs)))); + if (remainingMs() <= 0) { + throw new Error(`fleet deadline exhausted during ${purpose}`); + } + return projection; + } catch (error) { + lastError = error; + if (remainingMs() <= 0 || attempt === 2) throw error; + } } - return projection; + throw lastError; }; // Later PM2 stop/delete mutates every supplied registry name, including @@ -205,11 +214,18 @@ export function signalAndAwaitFleet( // A live refuser is decided early enough to leave one explicitly partitioned // compensation tail. All-dead fleets may continue their normal successor // settle beyond this point; they need no compensation subprocesses. - const compensationReserveMs = Math.min(6_000, Math.max(5, Math.floor(timeoutMs / 5))); + const productionBudget = timeoutMs >= 10_000; + const compensationReserveMs = productionBudget + ? Math.min(25_000, Math.max(10_000, Math.floor(timeoutMs / 3))) + : Math.min(6_000, Math.max(5, Math.floor(timeoutMs / 5))); const liveRefusalDeadline = Math.max(runtime.now(), deadline - compensationReserveMs); - const preProjectionCapMs = Math.max(1, Math.floor(compensationReserveMs / 5)); - const exactStartCapMs = Math.max(1, Math.floor(compensationReserveMs / 2)); - const postProjectionCapMs = Math.max(1, Math.floor(compensationReserveMs / 5)); + const preProjectionCapMs = productionBudget + ? Math.max(5_000, Math.floor(compensationReserveMs / 4)) + : Math.max(1, Math.floor(compensationReserveMs / 5)); + const exactStartCapMs = productionBudget + ? Math.max(10_000, Math.floor(compensationReserveMs / 2)) + : Math.max(1, Math.floor(compensationReserveMs / 2)); + const postProjectionCapMs = preProjectionCapMs; const successorProjectionCapMs = preProjectionCapMs; let quietSince: number | null = null; let verificationFailure: string | null = null; diff --git a/src/cli/pm2-shutdown-capability.ts b/src/cli/pm2-shutdown-capability.ts index 5ff11b7b0..df0393217 100644 --- a/src/cli/pm2-shutdown-capability.ts +++ b/src/cli/pm2-shutdown-capability.ts @@ -46,7 +46,8 @@ function failure(operation: string, detail: string): Error { `[${operation}] refusing to signal daemon generation(s): ${detail}. ` + `The live daemon may predate shutdown protocol ${SUPERVISOR_SHUTDOWN_PROTOCOL}; ` + 'normal stop/restart intentionally fails closed on this first-upgrade boundary. ' - + 'Confirm every Session/Riff workload is idle before an operator-approved one-time manual bootstrap; ' + + 'After confirming every Session/Riff workload is idle, run ' + + '`botmux restart --bootstrap-shutdown-protocol --yes` for the operator-approved one-time bootstrap; ' + 'automatic update must not be reported as applied until the new handler-ready fleet is verified', ); } diff --git a/src/cli/pm2-start-transaction.ts b/src/cli/pm2-start-transaction.ts index 9d6fbb0c5..6a1694563 100644 --- a/src/cli/pm2-start-transaction.ts +++ b/src/cli/pm2-start-transaction.ts @@ -98,7 +98,9 @@ export function assertDaemonPm2GracefulExitPolicy( throw new Error( `[${operation}] daemon PM2 policy does not prove signal-death autorestart ` + `(expected autorestart=true and stop_exit_codes=[${DAEMON_GRACEFUL_EXIT_CODE}]; unsafe: ` - + `${unsafe.map(entry => entry.name).join(', ')})`, + + `${unsafe.map(entry => entry.name).join(', ')}). ` + + 'For a one-time pre-protocol upgrade, first independently confirm every Session/Riff ' + + 'workload is idle, then run: botmux restart --bootstrap-shutdown-protocol --yes', ); } } diff --git a/src/core/shutdown-budgets.ts b/src/core/shutdown-budgets.ts index 33418e82b..f7b6b3c01 100644 --- a/src/core/shutdown-budgets.ts +++ b/src/core/shutdown-budgets.ts @@ -33,7 +33,7 @@ export const PM2_DAEMON_RESTART_DELAY_MS = 3_000; /** A full restart-delay plus projection jitter. The fleet helper must observe * this quiet window after every signalled generation exits. */ export const FLEET_SUCCESSOR_SETTLE_MS = PM2_DAEMON_RESTART_DELAY_MS + 500; -export const FLEET_DAEMON_EXIT_WAIT_MS = 35_000; +export const FLEET_DAEMON_EXIT_WAIT_MS = 60_000; if (PM2_DAEMON_KILL_TIMEOUT_MS <= DAEMON_SHUTDOWN_MAX_MS) { throw new Error('PM2 daemon kill timeout must exceed the complete daemon shutdown budget'); diff --git a/test/fleet-shutdown.test.ts b/test/fleet-shutdown.test.ts index eb880b3b0..780107ea0 100644 --- a/test/fleet-shutdown.test.ts +++ b/test/fleet-shutdown.test.ts @@ -861,6 +861,47 @@ describe('generation-aware fleet graceful shutdown', () => { expect(greatestNow).toBeLessThanOrEqual(50); }); + it('reserves production-size projection budgets and retries one transient jlist failure', () => { + const alive = new Set([101, 202]); + let now = 0; + let compensated = false; + const listBudgets: number[] = []; + let listCalls = 0; + + expect(() => signalAndAwaitFleet(entries.slice(0, 2), 'restart', 60_000, { + signal(pid) { + if (pid === 202) alive.delete(pid); + }, + isAlive: pid => alive.has(pid), + now: () => now, + sleep(ms) { now += ms; }, + startOffline(offline, budgetMs) { + expect(offline.map(entry => entry.name)).toEqual(['botmux-b']); + expect(budgetMs).toBeGreaterThanOrEqual(10_000); + compensated = true; + alive.add(303); + }, + list: budgetMs => { + listBudgets.push(budgetMs); + listCalls += 1; + if (listCalls === 1) throw new Error('transient jlist timeout'); + return [ + { name: 'botmux-a', pmId: 1, pid: 101, online: true }, + compensated + ? { name: 'botmux-b', pmId: 2, pid: 303, online: true } + : { + name: 'botmux-b', pmId: 2, pid: 0, online: false, + status: 'waiting restart', exitCode: 0, stopExitCodes: [0], + }, + ]; + }, + pollMs: 5_000, + })).toThrow(/restored 1 offline PM2 entry.*live generation untouched/); + + expect(listCalls).toBe(3); + expect(listBudgets.every(budget => budget >= 5_000)).toBe(true); + }); + it('does not conditionally start a row whose PM2 policy may still have a restart timer', () => { const alive = new Set([101, 202]); let now = 0; diff --git a/test/pm2-start-transaction.test.ts b/test/pm2-start-transaction.test.ts index f07784c46..14cec2080 100644 --- a/test/pm2-start-transaction.test.ts +++ b/test/pm2-start-transaction.test.ts @@ -40,7 +40,9 @@ describe('configured PM2 fleet start authority', () => { ...row('botmux-a', 0), autorestart: true, stopExitCodes: [0], - }])).toThrow(/does not prove signal-death autorestart.*stop_exit_codes=\[42\]/); + }])).toThrow( + /does not prove signal-death autorestart.*botmux restart --bootstrap-shutdown-protocol --yes/, + ); expect(() => assertDaemonPm2GracefulExitPolicy('start-idempotent-ready', [{ ...row('botmux-a', 0), autorestart: false, diff --git a/test/shutdown-supervisor-contract.test.ts b/test/shutdown-supervisor-contract.test.ts index 8830736bc..82fbffb4d 100644 --- a/test/shutdown-supervisor-contract.test.ts +++ b/test/shutdown-supervisor-contract.test.ts @@ -293,6 +293,8 @@ describe('graceful shutdown supervisor contract', () => { }); it('bounds and freshly verifies every public PM2 start surface with compensation', () => { + expect(cli).toContain('const PM2_START_VERIFY_MIN_TIMEOUT_MS = 60_000;'); + expect(cli).toContain('pm2StartVerifyTimeoutMs(configuredNames.length)'); const regions = [ cli.slice(cli.indexOf('async function cmdStart()'), cli.indexOf('/**\n * Wipe stale dashboard-daemon descriptors')), cli.slice(cli.indexOf('async function cmdRestart()'), cli.indexOf('/**\n * Bring a SINGLE bot')), @@ -357,7 +359,7 @@ describe('graceful shutdown supervisor contract', () => { }); it('discovers legacy Gods from the process table and rechecks duplicate Gods before mutation', () => { - const start = cli.indexOf('function cleanupLegacyPm2()'); + const start = cli.indexOf('function cleanupLegacyPm2('); const end = cli.indexOf('async function cmdStop()', start); const legacy = cli.slice(start, end); expect(legacy).toContain('listPm2GodDaemonPids(legacyHome)'); @@ -368,6 +370,24 @@ describe('graceful shutdown supervisor contract', () => { expect(cli).not.toContain("runPm2(['kill']"); }); + it('exposes an explicit double-confirmed first-upgrade bootstrap without weakening normal shutdown', () => { + const bootstrapStart = cli.indexOf('function bootstrapDeleteAllBotmuxProcesses('); + const bootstrapEnd = cli.indexOf('/**\n * One-time migration', bootstrapStart); + const bootstrap = cli.slice(bootstrapStart, bootstrapEnd); + expect(bootstrap).toContain('readSupervisorProcessStartIdentity(entry.pid)'); + expect(bootstrap).toContain('current.pid !== original.pid'); + expect(bootstrap).toMatch(/runPm2\(\s*\['delete', String\(current\.pmId\)\]/); + + const restartStart = cli.indexOf('async function cmdRestart()'); + const restartEnd = cli.indexOf('/**\n * Bring a SINGLE bot', restartStart); + const restart = cli.slice(restartStart, restartEnd); + expect(restart).toContain("process.argv.includes('--bootstrap-shutdown-protocol')"); + expect(restart).toContain("process.argv.includes('--yes')"); + expect(restart).toContain("bootstrapDeleteAllBotmuxProcesses('restart')"); + expect(restart).toContain('else deleteAllBotmuxProcesses()'); + expect(cli).toContain('botmux restart --bootstrap-shutdown-protocol --yes'); + }); + it('rejects include-pm2 before breadcrumb/fleet mutation when a live God exists', () => { const start = cli.indexOf('async function cmdRestart()'); const end = cli.indexOf('/**\n * Bring a SINGLE bot', start);