From 6519aed76d105af396c267fcc4b0c67cb193e174 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 08:49:40 +0000 Subject: [PATCH] Add backend E2E story tests: dashboard user stories over a fake agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every major dashboard flow now has an end-to-end test on the backend alone: the stories call the same telefunctions the browser invokes, against the real daemon runtime (createProjectRuntime), real registry state, and real spawned run processes — with the deterministic --fake driver in the agent seat, so the whole suite runs offline and adds ~2s of wall clock. No browser involved. Covered stories: add/register a project and the sidebar reads; start a session and follow its live feed to the archived row; two concurrent sessions in their own worktrees (#736); publish a finished session's branch to a real bare origin (#799); answer a parked question from the questions hub (#304/#1455); live chat becoming the next agent turn and its committed conversation (#714/#908); rearming the handoff mid-run (#1102); stop -> retained worktree -> remove -> delete (#737/#1032); tickets list/detail and cross-project pages (#697/#1144); queueing a ticket and a drain run claiming it (#1164/#1117); dashboard-written preferences reaching a resumed run's argv (#858/#1467); and the usage panel + auto-PM sweep surface (#533/#1210). The harness (src/e2e/harness.ts) wires the Telefunc request context the way runDaemon does, isolates $XDG_CONFIG_HOME per test process, spawns runs through fake-agent-bin.ts (the real CLI with --fake appended), and records each spawn's argv — the only observable place the launcher's toggle-to-flag contract can be asserted, since the spawn is detached. Writing the stories surfaced two real races in the finished-session seam (documented in src/e2e/spec.md): acting on a session the moment its meta flips done collides with teardown's own commits, and a live event tail whose file is retired misses the final events when the fs.watch signal is lost. The stories model the same healing the dashboard uses (wait for retirement; swap to the archived replay). src/e2e is excluded from the publish build; it compiles only into dist-test with the rest of the suite. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XKihCTfcazhDgt4dhUmfM9 --- .../the-framework/src/e2e/fake-agent-bin.ts | 32 ++ packages/the-framework/src/e2e/harness.ts | 285 ++++++++++++++++++ packages/the-framework/src/e2e/spec.md | 22 ++ .../e2e/story-projects-and-settings.test.ts | 145 +++++++++ .../src/e2e/story-session-lifecycle.test.ts | 177 +++++++++++ .../src/e2e/story-steering-and-gates.test.ts | 164 ++++++++++ .../src/e2e/story-tickets-and-queue.test.ts | 133 ++++++++ packages/the-framework/tsconfig.build.json | 2 +- 8 files changed, 959 insertions(+), 1 deletion(-) create mode 100644 packages/the-framework/src/e2e/fake-agent-bin.ts create mode 100644 packages/the-framework/src/e2e/harness.ts create mode 100644 packages/the-framework/src/e2e/spec.md create mode 100644 packages/the-framework/src/e2e/story-projects-and-settings.test.ts create mode 100644 packages/the-framework/src/e2e/story-session-lifecycle.test.ts create mode 100644 packages/the-framework/src/e2e/story-steering-and-gates.test.ts create mode 100644 packages/the-framework/src/e2e/story-tickets-and-queue.test.ts diff --git a/packages/the-framework/src/e2e/fake-agent-bin.ts b/packages/the-framework/src/e2e/fake-agent-bin.ts new file mode 100644 index 00000000..cf3bddc0 --- /dev/null +++ b/packages/the-framework/src/e2e/fake-agent-bin.ts @@ -0,0 +1,32 @@ +// The CLI entry the E2E harness hands to createProjectRuntime as `binPath`. +// +// The daemon spawns a run as `node `; this entry forwards that argv to the real +// CLI with `--fake` appended, so the spawned child executes the complete production run lifecycle +// (worktree cwd, run store, events.jsonl, control watcher, gates, teardown) with the deterministic +// offline FakeDriver where a real coding agent would be. `FRAMEWORK_FAKE_AWAIT` still scripts a +// gate turn, which is how a story parks a run on a question. +// +// When `$FRAMEWORK_E2E_ARGV_FILE` is set, the argv is appended there as one JSON line per spawn — +// the only way a story can assert that a dashboard toggle actually became the run flag it maps to, +// since the spawn is detached and its argv is otherwise observable nowhere. +import { appendFileSync } from 'node:fs' +import { runCli } from '../cli.js' + +const args = process.argv.slice(2) +const argvFile = process.env.FRAMEWORK_E2E_ARGV_FILE +if (argvFile) { + try { + appendFileSync(argvFile, JSON.stringify(args) + '\n') + } catch { + // recording is diagnostics, never a reason to fail the run + } +} + +runCli([...args, '--fake']) + .then(code => { + process.exitCode = code + }) + .catch((err: unknown) => { + console.error(err) + process.exitCode = 1 + }) diff --git a/packages/the-framework/src/e2e/harness.ts b/packages/the-framework/src/e2e/harness.ts new file mode 100644 index 00000000..0e914669 --- /dev/null +++ b/packages/the-framework/src/e2e/harness.ts @@ -0,0 +1,285 @@ +// The world behind the backend E2E story tests (see spec.md): the daemon's business logic wired +// exactly as `runDaemon` wires it, against throwaway state, with runs spawned through +// `fake-agent-bin.js` so the full production lifecycle executes offline. +import { existsSync, mkdtempSync } from 'node:fs' +import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { execFile } from 'node:child_process' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' +import { provideTelefuncContext } from 'telefunc' +import { createProjectRuntime, type ProjectRuntime } from '../daemon-runtime.js' +import { registryPreferencesStore, projectId } from '../registry.js' +import { registryDiscordCredentialsStore } from '../discord-credentials-store.js' +import { loadRunEvents, resolveRunEventsPath, type RunMeta, type RunStatus } from '../store/index.js' +import { tailEvents } from '../dashboard-rpc/events-tail.js' +import { sendAddProject } from '../dashboard-rpc/projects.telefunc.js' +import { sendStart } from '../dashboard-rpc/control.telefunc.js' +import { onRuns } from '../dashboard-rpc/reads.telefunc.js' +import type { FrameworkEvent } from '../events.js' +import type { StartRunKind, StartRunOptions } from '../dashboard/types.js' +import type { QuotaView } from '../dashboard/quota.js' +import type { AutoPmReport } from '../auto-pm.js' + +// Re-home the process-global config home FIRST: the registry, preferences, and daemon state all +// resolve through $XDG_CONFIG_HOME at call time, and run-tests.mjs gives the whole suite ONE +// shared throwaway home — so without this, story files running as sibling processes would see +// each other's registered projects in every cross-project rollup (onProjects, onQueue, onOverview). +process.env.XDG_CONFIG_HOME = mkdtempSync(join(tmpdir(), 'framework-e2e-config-')) + +const exec = promisify(execFile) + +/** Run `git ` in `cwd`, failing the story loudly on error (a broken fixture is a test bug). */ +export async function git(cwd: string, ...args: string[]): Promise { + const { stdout } = await exec('git', args, { cwd }) + return stdout +} + +/** One registered project inside a {@link StoryWorld}: a real git repo the stories act on. */ +export interface StoryProject { + /** The registry id every dashboard RPC keys by. */ + id: string + /** The repo's checkout path on disk. */ + cwd: string +} + +/** A live tail of one run's event log — the same source `onEvents` streams to the browser. */ +export interface RunTail { + /** Every event seen so far, in arrival order. Poll with {@link waitFor}. */ + events: FrameworkEvent[] + stop(): void +} + +/** + * Everything one story test stands up: the daemon runtime on a temp home, the Telefunc request + * context the daemon would provide, and factories for registered projects. `close()` is the + * whole teardown — it stops spawned runs the way daemon shutdown does, then removes the state. + */ +export interface StoryWorld { + /** The daemon's home workspace (a plain temp dir, not a registered project). */ + home: string + runtime: ProjectRuntime + /** The usage panel's reading (mutable): what `onQuota` serves. */ + quota: { view: QuotaView } + /** The auto-PM panel's stubs (mutable): what `onAutoPm` reports and what a sweep records. */ + autoPm: { report?: AutoPmReport; sweeps: Array<{ drainOnly?: boolean }> } + /** + * Bind one dashboard RPC to this world's request context. The real mount provides the context + * per request; sync-mode telefunc drops a provided context at the next macrotask, so a story + * spanning real IO must re-provide it before every call — which is exactly what this does. + * Every telefunction reads its context synchronously at its top, so provide-then-call holds. + */ + rpc(fn: (...args: A) => R): (...args: A) => R + /** Argv of every run child this world spawned, one entry per spawn, oldest first. */ + spawnedArgv(): Promise + /** + * Create a real git repo (initial commit included) and register it through the same + * `sendAddProject` RPC the dashboard's Add-project dialog calls. + */ + addProject(files?: Record): Promise + /** Start a run through the same `sendStart` RPC the launcher calls; returns the run id. */ + startRun(project: StoryProject, prompt: string, options?: StartRunOptions, kind?: StartRunKind): Promise + /** Poll `onRuns` until the run reports one of `until`, failing after `timeoutMs`. */ + waitRun(project: StoryProject, runId: string, until: RunStatus | RunStatus[], timeoutMs?: number): Promise + /** + * Wait until the daemon's teardown has retired the run's worktree. A run's meta flips to + * `done` before teardown archives the checkout, and acting on the session in that window + * (push, resume) races teardown's own git commits — the same window a user hits by clicking + * Push the instant a session finishes. The stories that act on a finished session wait here + * first, which is also the honest reading of "finished". + */ + waitRetired(project: StoryProject, runId: string, timeoutMs?: number): Promise + /** Follow a run's event log live (replays what is already on disk first). */ + tailRun(project: StoryProject, runId: string): Promise + close(): Promise +} + +/** Poll `read` until it yields a non-undefined value; the failure names `what` went unmet. */ +export async function waitFor( + read: () => T | undefined | Promise, + what: string, + timeoutMs = 30_000, +): Promise { + const deadline = Date.now() + timeoutMs + for (;;) { + const value = await read() + if (value !== undefined) return value + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`) + await new Promise(r => setTimeout(r, 100)) + } +} + +/** + * Set `FRAMEWORK_FAKE_AWAIT` for the Starts inside `fn`, so their fake agent's first turn parks + * on that gate. Env-scoped rather than per-call because the spawned child reads it at boot; the + * finally puts it back before the next story's Starts inherit it. + */ +export async function withFakeAwait(mode: 'choices' | 'multiselect' | 'confirmation', fn: () => Promise): Promise { + process.env.FRAMEWORK_FAKE_AWAIT = mode + try { + return await fn() + } finally { + delete process.env.FRAMEWORK_FAKE_AWAIT + } +} + +/** A minimal passing preflight: E2E runs never probe the real agent CLI (there is none here). */ +const agentReady = async () => ({ ok: true, checks: [] }) + +/** + * Stand up one story world. The Telefunc context mirrors `runDaemon`'s `startDashboard` wiring + * piece for piece — same closures, same registry-backed stores — except where the daemon holds a + * live poller/loop (quota, auto PM), which a story controls through mutable stubs instead. + */ +export async function makeWorld(): Promise { + const home = mkdtempSync(join(tmpdir(), 'framework-e2e-home-')) + const argvFile = join(home, 'spawned-argv.jsonl') + process.env.FRAMEWORK_E2E_ARGV_FILE = argvFile + + const runtime = createProjectRuntime({ + cwd: home, + env: process.env, + binPath: fileURLToPath(new URL('./fake-agent-bin.js', import.meta.url)), + agentPreflight: agentReady, + }) + + const quota = { view: { windows: [] } as QuotaView } + const autoPm: StoryWorld['autoPm'] = { sweeps: [] } + const context = { + startRun: runtime.onStart, + addProject: runtime.onAddProject, + preview: runtime.preview, + eventsSource: runtime.remoteEventsSource, + remote: runtime.remoteRuns, + preferences: registryPreferencesStore(), + discord: registryDiscordCredentialsStore(), + quota: { read: async () => quota.view, stop: () => {} }, + autoPm: () => autoPm.report, + autoPmSweep: async (opts?: { drainOnly?: boolean }) => { + autoPm.sweeps.push(opts ?? {}) + }, + } + + const repos: string[] = [] + const tails: RunTail[] = [] + + const rpc: StoryWorld['rpc'] = fn => { + return (...args) => { + provideTelefuncContext(context as never) + return fn(...args) + } + } + + const world: StoryWorld = { + home, + runtime, + quota, + autoPm, + rpc, + + async spawnedArgv() { + const raw = await readFile(argvFile, 'utf8').catch(() => '') + return raw + .split('\n') + .filter(line => line.trim()) + .map(line => JSON.parse(line) as string[]) + }, + + async addProject(files = {}) { + const cwd = mkdtempSync(join(tmpdir(), 'framework-e2e-repo-')) + repos.push(cwd) + await git(cwd, 'init', '-q', '-b', 'main') + await git(cwd, 'config', 'user.email', 'e2e@test') + await git(cwd, 'config', 'user.name', 'e2e') + const seeded = Object.keys(files).length ? files : { 'README.md': '# story fixture\n' } + for (const [file, text] of Object.entries(seeded)) { + await mkdir(dirname(join(cwd, file)), { recursive: true }) + await writeFile(join(cwd, file), text) + } + await git(cwd, 'add', '-A') + await git(cwd, 'commit', '-q', '-m', 'seed') + const added = await rpc(sendAddProject)(cwd, false) + if (!added.ok) throw new Error(`could not register the fixture repo: ${added.error}`) + return { id: projectId(resolve(cwd)), cwd } + }, + + async startRun(project, prompt, options = {}, kind: StartRunKind = 'prompt') { + const result = await rpc(sendStart)(project.id, prompt, kind, options) + if (!result.ok) throw new Error(`sendStart refused: ${result.error}`) + if (!result.runId) throw new Error('sendStart returned no run id for a worktree project') + return result.runId + }, + + async waitRun(project, runId, until, timeoutMs = 30_000) { + const wanted = Array.isArray(until) ? until : [until] + let last: RunMeta | undefined + return waitFor( + async () => { + const runs = await rpc(onRuns)(project.id) + last = runs.find(run => run.id === runId) + return last && wanted.includes(last.status) ? last : undefined + }, + `run ${runId} to be ${wanted.join('/')} (last seen: ${JSON.stringify(last?.status)})`, + timeoutMs, + ) + }, + + async waitRetired(project, runId, timeoutMs = 30_000) { + const worktree = join(project.cwd, '.the-framework', 'worktrees', runId) + await waitFor( + async () => ((await stat(worktree).catch(() => undefined)) ? undefined : true), + `run ${runId}'s worktree to be retired`, + timeoutMs, + ) + }, + + async tailRun(project, runId) { + const path = await resolveRunEventsPath(project.cwd, runId) + const events: FrameworkEvent[] = [] + const stopLive = tailEvents(path, event => events.push(event)) + // Teardown MOVES the live log into the archive and removes the worktree ~100ms after a + // fast run ends, and a tail whose file vanished delivers nothing ever again — the 1s poll + // backstop can lose the final lines to that window when the fs.watch event goes missing. + // The dashboard heals the same way this does: the session view swaps to the archived + // replay once the row settles. So when the live file disappears, finish the feed from the + // run's archived journal (a superset of everything the live tail saw). + let sawFile = false + const finalize = setInterval(() => { + if (existsSync(path)) { + sawFile = true + return + } + if (!sawFile) return // not written yet — the run is still booting, nothing was moved + clearInterval(finalize) + stopLive() + void loadRunEvents(project.cwd, runId) + .then(archived => { + if (archived && archived.length >= events.length) events.splice(0, events.length, ...archived) + }) + .catch(() => {}) + }, 100) + finalize.unref?.() + const tail = { + events, + stop: () => { + clearInterval(finalize) + stopLive() + }, + } + tails.push(tail) + return tail + }, + + async close() { + for (const tail of tails) tail.stop() + // Same order as daemon shutdown: stop the runs this world spawned, then the previews. + await runtime.suspendRuns(2000).catch(() => 0) + await runtime.dispose().catch(() => {}) + delete process.env.FRAMEWORK_E2E_ARGV_FILE + await rm(home, { recursive: true, force: true }).catch(() => {}) + for (const repo of repos) await rm(repo, { recursive: true, force: true }).catch(() => {}) + }, + } + return world +} diff --git a/packages/the-framework/src/e2e/spec.md b/packages/the-framework/src/e2e/spec.md new file mode 100644 index 00000000..ca68b959 --- /dev/null +++ b/packages/the-framework/src/e2e/spec.md @@ -0,0 +1,22 @@ +Backend end-to-end story tests: each test walks one dashboard user story through the real daemon runtime — real registry, real git repos, real spawned run processes — with the deterministic `--fake` driver in place of a coding agent, so the whole flow runs offline in seconds and no browser is involved. + +## What "end-to-end" means here + +- **The entry points are the dashboard's own RPCs.** Stories call the same telefunctions the browser invokes over `/_telefunc` (`sendStart`, `sendChoice`, `onRuns`, `onQueue`, …), with the request context wired exactly the way `runDaemon` wires it: the runtime's `onStart`/`onAddProject`/`preview` closures, the registry-backed preferences store, and stub quota/auto-PM reporters where the daemon would hold live pollers. +- **Runs are real processes.** `createProjectRuntime` spawns each run detached, exactly as the daemon does; the harness's `binPath` points at `fake-agent-bin.js`, which forwards to the real CLI with `--fake` appended. Everything between the Start click and the archived run row — worktree allocation, the run store, `events.jsonl`, the control watcher, gates, teardown, retention — is the production code path; only the agent turn is scripted. +- **The Telefunc transport hop is out of scope.** Telefunc `Channel`s only pump over a real wire, so live-stream assertions tail the run's `events.jsonl` — the very source `onEvents` wraps. The mount, CSRF/rebinding guards, and channel plumbing have their own tests (`dashboard/server.test.ts`, `dashboard-rpc/stream-channel.test.ts`). + +## Isolation + +`harness.ts` points `$XDG_CONFIG_HOME` at a fresh temp dir per test process, so a story file's registry (projects, preferences, daemon state) can never see — or be seen by — the sibling test files that run concurrently under `scripts/run-tests.mjs`'s shared config home. Every world lives in temp dirs and `close()` kills whatever runs it spawned. + +## Scripting the fake agent + +`FRAMEWORK_FAKE_AWAIT=choices|multiselect|confirmation` (set before a Start, spawned children inherit the env) makes the fake agent's first turn end on that gate, which is how stories park a run on a question deterministically. Without it the fake agent answers every prompt with one scripted build turn and the run ends on its own. + +## Two product races these stories deliberately step around + +Writing the stories surfaced two real windows in the finished-session seam; the harness models the same healing the dashboard relies on, and both are candidates for tightening in the product: + +- **Acting on a run the moment its meta flips `done` races teardown.** The child writes `status: done` and exits; the daemon then archives the history, commits the bookkeeping to the run branch, and retires the worktree. A `sendPushBranch`/resume fired inside that window runs `commitPendingWork` against the same checkout teardown is committing in, and the loser reports "could not commit the work this session left uncommitted" (teardown then retains the worktree it would have removed). A user clicking Push the instant a session finishes can hit the same message; clicking again succeeds. Stories wait for `waitRetired` — the honest reading of "finished" — before publishing or resuming. +- **A live tail whose file is retired goes silent without the final events.** Teardown *moves* `events.jsonl` into the archive; `tailEvents`' fs.watch can miss the last appends under watcher pressure, and its 1s poll then finds the file gone — nothing is ever delivered again, including the `end` the transcript needs. The dashboard heals by swapping to the archived replay (`onRun`) once the row settles; `tailRun` mirrors exactly that swap when the live file disappears. diff --git a/packages/the-framework/src/e2e/story-projects-and-settings.test.ts b/packages/the-framework/src/e2e/story-projects-and-settings.test.ts new file mode 100644 index 00000000..7d3d8cf5 --- /dev/null +++ b/packages/the-framework/src/e2e/story-projects-and-settings.test.ts @@ -0,0 +1,145 @@ +import { strict as assert } from 'node:assert' +import { test } from 'node:test' +import { stat } from 'node:fs/promises' +import { join } from 'node:path' +import { makeWorld, waitFor } from './harness.js' +import { onProjects, sendAddProject } from '../dashboard-rpc/projects.telefunc.js' +import { onGitStatus, onRuns, onDocs } from '../dashboard-rpc/reads.telefunc.js' +import { + onPreferences, + patchPreferences, + onProjectPreferences, + patchProjectPreferences, +} from '../dashboard-rpc/preferences.telefunc.js' +import { onQuota, onAutoPm, sendAutoPmSweep } from '../dashboard-rpc/quota.telefunc.js' +import { sendStart } from '../dashboard-rpc/control.telefunc.js' + +// The projects & settings stories (spec.md): registering a repo, what the sidebar then shows, +// how settings written in the dashboard reach the runs the daemon starts, and the usage panel. + +test('add a project: it is installed, registered, and readable like the sidebar reads it (#396)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + + // Install left its marks in the repo: the activation marker and the seeded committed log. + await stat(join(project.cwd, '.the-framework', 'LOGS.md')) + + // The Projects sidebar shows the repo as an activated project. + const projects = await rpc(onProjects)() + const mine = projects.find(p => p.id === project.id) + assert.equal(mine?.path, project.cwd) + assert.equal(mine?.activated, true) + + // The project header's reads answer: the branch, an empty docs rail, an empty run history. + const status = await rpc(onGitStatus)(project.id) + assert.equal(status?.branch, 'main') + assert.deepEqual(await rpc(onDocs)(project.id), []) + assert.deepEqual(await rpc(onRuns)(project.id), []) + + // Adding the same repo again is a no-op the dialog can explain, not a duplicate. + const again = await rpc(sendAddProject)(project.cwd, false) + assert.deepEqual(again, { ok: true, added: 0, alreadyActivated: 1 }) + assert.equal((await rpc(onProjects)()).filter(p => p.id === project.id).length, 1) + + // A bogus path is refused with a reason, not a crash. + const bogus = await rpc(sendAddProject)(join(world.home, 'nope'), false) + assert.equal(bogus.ok, false) + } finally { + await world.close() + } +}) + +test('unknown projects degrade quietly: reads are empty, writes are refused (#427)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + await world.addProject() + assert.deepEqual(await rpc(onRuns)('no-such-project'), []) + assert.equal(await rpc(onGitStatus)('no-such-project'), null) + const refused = await rpc(sendStart)('no-such-project', 'do something') + assert.equal(refused.ok, false) + } finally { + await world.close() + } +}) + +test('settings written in the dashboard reach the next resumed run (#858/#1467)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + + // The Settings page: patch a global and a per-project preference; both read back. + const patched = await rpc(patchPreferences)({ autopilot: false }) + assert.equal(patched.ok, true) + assert.equal((await rpc(onPreferences)()).autopilot, false) + const projectPatched = await rpc(patchProjectPreferences)(project.id, { model: 'fable-e2e' }) + assert.equal(projectPatched.ok, true) + assert.equal((await rpc(onProjectPreferences)(project.id)).model, 'fable-e2e') + + // First leg: a plain run, finished and fully retired (resuming mid-teardown would race the + // archive of the very history the continuation reopens). + const runId = await world.startRun(project, 'Build the settings page') + await world.waitRun(project, runId, 'done') + await world.waitRetired(project, runId) + + // The composer's Resume sends only its seed (#1467); the daemon overlays the project's + // resolved options, so the model chosen in Settings reaches the continued session's argv. + const resumed = await rpc(sendStart)(project.id, 'Keep going', 'prompt', { continueRunId: runId }) + assert.equal(resumed.ok, true) + await world.waitRun(project, runId, 'done') + await world.waitRetired(project, runId) + const argv = await waitFor(async () => { + const spawns = await world.spawnedArgv() + return spawns.length >= 2 ? spawns[1] : undefined + }, 'the resumed child to be spawned') + const modelFlag = argv.indexOf('--model') + assert.notEqual(modelFlag, -1, `the resumed run carries the project model: ${argv.join(' ')}`) + assert.equal(argv[modelFlag + 1], 'fable-e2e') + assert.ok(argv.includes('--continue-run'), 'the resumed run reopens the same session row') + + // One row throughout: the continuation is the same run, not a second history entry. + const rows = await rpc(onRuns)(project.id) + assert.equal(rows.filter(r => r.id === runId).length, 1) + } finally { + await world.close() + } +}) + +test('the usage panel reads the daemon quota source, and the sweep button fires a sweep (#533/#1210)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + // No reading: the panel must hear "unavailable", never an empty bar that reads as unused. + world.quota.view = { windows: [], unavailable: 'fetch-failed' } + assert.equal((await rpc(onQuota)()).unavailable, 'fetch-failed') + + // With a reading, the windows come through as the poller reported them. + world.quota.view = { + windows: [{ label: 'Current week (all models)', usedPercent: 40 } as never], + readAt: 123, + } + const quota = await rpc(onQuota)() + assert.equal(quota.windows.length, 1) + assert.equal(quota.readAt, 123) + + // The auto-PM line under the toggle: silent before the first sweep, then the report. + assert.equal(await rpc(onAutoPm)(), undefined) + world.autoPm.report = { nextSweepAt: 456, outcomes: [] } + assert.deepEqual(await rpc(onAutoPm)(), { nextSweepAt: 456, outcomes: [] }) + + // The "sweep now" button reaches the daemon's loop and reports the outcomes it recorded. + world.autoPm.report = { + nextSweepAt: 789, + outcomes: [{ projectId: 'p', path: '/p', started: false, message: 'the queue is empty' }], + } + const swept = await rpc(sendAutoPmSweep)({ drainOnly: true }) + assert.deepEqual(world.autoPm.sweeps, [{ drainOnly: true }]) + assert.equal(swept.ok, true) + assert.equal(swept.outcomes?.[0]?.message, 'the queue is empty') + } finally { + await world.close() + } +}) diff --git a/packages/the-framework/src/e2e/story-session-lifecycle.test.ts b/packages/the-framework/src/e2e/story-session-lifecycle.test.ts new file mode 100644 index 00000000..507b1c24 --- /dev/null +++ b/packages/the-framework/src/e2e/story-session-lifecycle.test.ts @@ -0,0 +1,177 @@ +import { strict as assert } from 'node:assert' +import { test } from 'node:test' +import { join } from 'node:path' +import { makeWorld, waitFor, withFakeAwait, git } from './harness.js' +import { + onRun, + onRuns, + onRunHandoff, + onRunWorktree, + onRetainedWorktrees, + onProjectLog, + onActivity, + onRecentRuns, +} from '../dashboard-rpc/reads.telefunc.js' +import { sendChoice, sendPushBranch } from '../dashboard-rpc/control.telefunc.js' + +// The session lifecycle stories (spec.md): what a user sees between clicking Start and reading +// the archived session row — through the same RPCs the dashboard calls, with real spawned run +// processes and the fake driver in the agent seat. + +test('start a session, watch it live, and read the archived row when it ends', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + const runId = await world.startRun(project, 'Add a login page', { autoPushBranch: false, autoOpenPr: false }) + const tail = await world.tailRun(project, runId) + + // The live feed narrates the run the way the session view renders it: the session banner + // first (naming the fake driver, so no real agent is running), then the agent's own turn. + const session = await waitFor( + () => tail.events.find(e => e.kind === 'session'), + 'the session event on the live feed', + ) + assert.equal(session.kind === 'session' && session.fake, true) + assert.equal(session.kind === 'session' && session.driver, 'fake') + + // The spawned child got the exact flags the launcher's unticked handoff boxes map to: the + // spawn is detached, so the recorded argv is the only place this contract is observable. + const argv = (await world.spawnedArgv())[0]! + assert.ok(argv.includes('--no-auto-push-branch'), `argv carries --no-auto-push-branch: ${argv.join(' ')}`) + assert.ok(argv.includes('--no-auto-open-pr'), `argv carries --no-auto-open-pr: ${argv.join(' ')}`) + assert.ok(argv.includes('--run-id') && argv.includes(runId), 'the child was handed its worktree run id') + + // The agent's turn reaches the feed: its tool actions, its reply, the accounted usage, and + // the end marker — the exact sequence the transcript pane draws. + await waitFor(() => tail.events.find(e => e.kind === 'end'), 'the end event') + const kinds = tail.events.map(e => e.kind) + assert.ok(kinds.indexOf('session') < kinds.indexOf('end'), 'session precedes end') + const usage = tail.events.find(e => e.kind === 'usage') + assert.ok(usage && usage.kind === 'usage' && usage.inputTokens > 0, 'usage accounting reached the feed') + const end = tail.events.find(e => e.kind === 'end') + assert.equal(end?.kind === 'end' && end.ok, true) + tail.stop() + + // The sidebar row settles to done, carrying what the list renders: the prompt as the label, + // the branch the work is on, and the driver that ran it. + const meta = await world.waitRun(project, runId, 'done') + assert.equal(meta.intent, 'Add a login page') + assert.equal(meta.branch, `the-framework/run-${runId}`) + assert.equal(meta.driver, 'fake') + + // A cleanly finished session retires its worktree (#737): nothing left to inspect, so the + // checkout is gone from disk, the Remove list is empty, and the run addresses the project root. + await world.waitRetired(project, runId) + assert.deepEqual(await rpc(onRetainedWorktrees)(project.id), []) + const worktree = await rpc(onRunWorktree)(project.id, runId) + assert.equal(worktree?.own, false) + + // The archived history replays the same story the live tail told (#1472 reads the run's own + // journal, not another run's), and the cross-project surfaces list the session. + const replay = await rpc(onRun)(project.id, runId) + assert.ok(replay.some(e => e.kind === 'session') && replay.some(e => e.kind === 'end'), 'replay has the whole journal') + const activity = await rpc(onActivity)() + assert.ok(activity.some(a => a.runId === runId && a.kind === 'finished' && a.status === 'done')) + const recent = await rpc(onRecentRuns)() + assert.ok(recent.some(r => r.projectId === project.id && r.run.id === runId)) + + // The session's record rides its branch, not main: teardown commits the LOGS.md line and the + // conversation to the run branch, so the handoff panel sees a branch of pure bookkeeping + // (#1291 calls that `empty` — nothing publishable) while the committed project log stays a + // projection of main and shows the line only once the branch merges. + const handoff = await rpc(onRunHandoff)(project.id, runId) + assert.equal(handoff?.exists, true) + assert.equal(handoff?.empty, true) + assert.deepEqual(await rpc(onProjectLog)(project.id), []) + } finally { + await world.close() + } +}) + +test('two sessions run concurrently, each in its own worktree (#736)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + // Both runs park on their scripted question, so both are provably alive at the same time — + // the one-working-tree collision #736 removed would have refused the second Start. + const [runA, runB] = await withFakeAwait('choices', async () => { + const a = await world.startRun(project, 'First feature') + const b = await world.startRun(project, 'Second feature') + return [a, b] + }) + assert.notEqual(runA, runB) + + const tailA = await world.tailRun(project, runA) + const tailB = await world.tailRun(project, runB) + const gateA = await waitFor(() => tailA.events.find(e => e.kind === 'choice'), 'run A to park on its gate') + const gateB = await waitFor(() => tailB.events.find(e => e.kind === 'choice'), 'run B to park on its gate') + + // Both rows are live in the sidebar, and each names its own checkout under the project's + // worktrees dir — the user's checkout is neither. + const runs = await rpc(onRuns)(project.id) + assert.equal(runs.filter(r => [runA, runB].includes(r.id) && r.status === 'running').length, 2) + const [wtA, wtB] = [await rpc(onRunWorktree)(project.id, runA), await rpc(onRunWorktree)(project.id, runB)] + assert.equal(wtA?.own, true) + assert.equal(wtB?.own, true) + assert.notEqual(wtA?.path, wtB?.path) + assert.equal(world.runtime.activeRunCount(project.id), 2) + + // Answering each question lets each session finish independently. + for (const [runId, gate] of [ + [runA, gateA], + [runB, gateB], + ] as const) { + assert.equal(gate.kind, 'choice') + if (gate.kind !== 'choice') continue + await rpc(sendChoice)(project.id, gate.id, gate.recommended ?? gate.options[0]!.id, 'user', runId) + const meta = await world.waitRun(project, runId, 'done') + assert.equal(meta.status, 'done') + } + // Eventually, not instantly: the meta flips to done a beat before the daemon reaps the + // child's exit, and until the reap the pid still answers as alive. + await waitFor( + () => (world.runtime.activeRunCount(project.id) === 0 ? true : undefined), + 'the daemon to reap both finished runs', + ) + } finally { + await world.close() + } +}) + +test("publish a finished session: push its branch from the handoff panel (#799)", async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + // A real remote for the story: a bare repo standing in for origin, so the push is a real + // git push and the handoff panel's pushed flag comes from the remote, not from a stub. + const remote = join(world.home, 'origin.git') + await git(world.home, 'init', '-q', '--bare', remote) + await git(project.cwd, 'remote', 'add', 'origin', remote) + + const runId = await world.startRun(project, 'Ship the settings page', { autoPushBranch: false, autoOpenPr: false }) + await world.waitRun(project, runId, 'done') + // Fully finished, not merely `done`: acting before teardown retires the checkout races + // teardown's own commits — the same race a user hits clicking Push the moment a run ends. + await world.waitRetired(project, runId) + + // Before the click: the panel reports the branch, the remote, and that nothing is pushed yet. + const before = await waitFor(async () => (await rpc(onRunHandoff)(project.id, runId)) ?? undefined, 'the handoff read') + assert.equal(before.branch, `the-framework/run-${runId}`) + assert.equal(before.exists, true) + assert.equal(before.hasRemote, true) + assert.equal(before.pushed, false) + + // The user's Push click publishes the branch; the remote now has it and the panel says so. + const pushed = await rpc(sendPushBranch)(project.id, runId) + assert.equal(pushed.ok, true, `push failed: ${'error' in pushed ? pushed.error : ''}`) + const remoteBranches = await git(project.cwd, 'ls-remote', '--heads', 'origin') + assert.ok(remoteBranches.includes(`the-framework/run-${runId}`), 'the run branch is on origin') + const after = await rpc(onRunHandoff)(project.id, runId) + assert.equal(after?.pushed, true) + } finally { + await world.close() + } +}) diff --git a/packages/the-framework/src/e2e/story-steering-and-gates.test.ts b/packages/the-framework/src/e2e/story-steering-and-gates.test.ts new file mode 100644 index 00000000..591258b8 --- /dev/null +++ b/packages/the-framework/src/e2e/story-steering-and-gates.test.ts @@ -0,0 +1,164 @@ +import { strict as assert } from 'node:assert' +import { test } from 'node:test' +import { git, makeWorld, waitFor, withFakeAwait } from './harness.js' +import { onOpenQuestions, onRuns, onRetainedWorktrees, onRunWorktree } from '../dashboard-rpc/reads.telefunc.js' +import { + sendChoice, + sendMessage, + sendStop, + sendSetHandoff, + sendRemoveWorktree, + sendDeleteSession, +} from '../dashboard-rpc/control.telefunc.js' + +// The steering stories (spec.md): everything the user does TO a live session — answer its +// question, chat with it, change its handoff, stop it — flows browser -> RPC -> control.jsonl -> +// the run process, and the observable answer comes back through the run's own event log. + +test('answer a parked session’s question from the questions hub (#304/#1455)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + const runId = await withFakeAwait('choices', () => world.startRun(project, 'Wire up auth')) + const tail = await world.tailRun(project, runId) + + // The run parks: the full gate (title, options, recommendation) reaches the feed, and the + // cross-project questions hub lists it against this session. + const gate = await waitFor(() => tail.events.find(e => e.kind === 'choice'), 'the parked gate') + assert.equal(gate.kind, 'choice') + if (gate.kind !== 'choice') return + assert.ok(gate.options.length >= 2, 'the gate offers options') + assert.ok(gate.recommended, 'the gate names a recommended option') + const question = await waitFor( + async () => (await rpc(onOpenQuestions)()).find(q => q.runId === runId), + 'the questions hub to list the parked run', + ) + assert.equal(question.projectId, project.id) + assert.equal(question.choice.id, gate.id) + assert.deepEqual(question.choice.options.map(o => o.id), gate.options.map(o => o.id)) + + // The user picks the recommended option. The run records who answered and carries on to done. + await rpc(sendChoice)(project.id, gate.id, gate.recommended!, 'user', runId) + const resolved = await waitFor(() => tail.events.find(e => e.kind === 'choice-resolved'), 'the resolution event') + assert.equal(resolved.kind === 'choice-resolved' && resolved.picked, gate.recommended) + assert.equal(resolved.kind === 'choice-resolved' && resolved.by, 'user') + await world.waitRun(project, runId, 'done') + + // Answered means gone from the hub. + assert.equal((await rpc(onOpenQuestions)()).some(q => q.runId === runId), false) + } finally { + await world.close() + } +}) + +test('chat with a live session: a message becomes the next agent turn (#714)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + const runId = await withFakeAwait('choices', () => world.startRun(project, 'Build the dashboard page')) + const tail = await world.tailRun(project, runId) + const gate = await waitFor(() => tail.events.find(e => e.kind === 'choice'), 'the parked gate') + if (gate.kind !== 'choice') return + + // Said while the session is parked, so the queue provably holds it until the gate resolves. + await rpc(sendMessage)(project.id, 'Also add a logout button', runId) + await rpc(sendChoice)(project.id, gate.id, gate.recommended!, 'user', runId) + + // The queued message is drained as its own turn: its text shows up as a driver prompt on the + // feed the transcript renders. + await waitFor( + () => + tail.events.find( + e => e.kind === 'driver' && e.event.type === 'start' && e.event.prompt.includes('Also add a logout button'), + ), + 'the chat message to become an agent turn', + ) + await world.waitRun(project, runId, 'done') + await world.waitRetired(project, runId) + + // What was said survives the session (#908): teardown committed the conversation record to + // the run's branch, so a clone carries the chat and not just the fact a run happened. + const conversation = await git( + project.cwd, + 'show', + `the-framework/run-${runId}:.the-framework/conversations/${runId}.md`, + ) + assert.ok(conversation.includes('Also add a logout button'), 'the committed conversation carries the message') + } finally { + await world.close() + } +}) + +test('rearm the handoff mid-run; the meta a reloaded tab reads follows (#1102)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + const runId = await withFakeAwait('choices', () => world.startRun(project, 'Refactor the config layer')) + const tail = await world.tailRun(project, runId) + const gate = await waitFor(() => tail.events.find(e => e.kind === 'choice'), 'the parked gate') + if (gate.kind !== 'choice') return + + // The boxes start armed (push + PR are the defaults), and unticking them mid-run re-announces + // the armed state — the event is what folds onto the meta a tab opened later reads back. + await rpc(sendSetHandoff)(project.id, runId, false, false) + await waitFor( + () => tail.events.find(e => e.kind === 'handoff-armed' && !e.push && !e.pr), + 'the disarmed announcement', + ) + const meta = await waitFor(async () => { + const run = (await rpc(onRuns)(project.id)).find(r => r.id === runId) + return run?.handoff && !run.handoff.push && !run.handoff.pr ? run : undefined + }, 'the disarmed state to reach the run meta') + assert.equal(meta.handoff?.push, false) + assert.equal(meta.handoff?.pr, false) + + await rpc(sendChoice)(project.id, gate.id, gate.recommended!, 'user', runId) + await world.waitRun(project, runId, 'done') + } finally { + await world.close() + } +}) + +test('stop a session; its checkout is retained for inspection, then removed and deleted (#737/#1032)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject() + const runId = await withFakeAwait('choices', () => world.startRun(project, 'Long experiment')) + const tail = await world.tailRun(project, runId) + await waitFor(() => tail.events.find(e => e.kind === 'choice'), 'the parked gate') + + // Stop while parked: the run ends `stopped`, not failed — the user interrupted it. + await rpc(sendStop)(project.id, runId) + const end = await waitFor(() => tail.events.find(e => e.kind === 'end'), 'the end event') + assert.equal(end.kind === 'end' && end.stopped, true) + tail.stop() + await world.waitRun(project, runId, 'stopped') + + // A stopped session keeps its worktree — that is exactly when the user wants to see what it + // was holding — and the dashboard offers removing it. + const retained = await waitFor(async () => { + const ids = await rpc(onRetainedWorktrees)(project.id) + return ids.includes(runId) ? ids : undefined + }, 'the stopped worktree to be retained') + assert.deepEqual(retained, [runId]) + const worktree = await rpc(onRunWorktree)(project.id, runId) + assert.equal(worktree?.own, true) + + // Remove keeps the session row (history), only the checkout goes. + const removed = await rpc(sendRemoveWorktree)(project.id, runId) + assert.equal(removed.ok, true, `remove failed: ${'error' in removed ? removed.error : ''}`) + assert.deepEqual(await rpc(onRetainedWorktrees)(project.id), []) + assert.ok((await rpc(onRuns)(project.id)).some(r => r.id === runId && r.status === 'stopped')) + + // Delete is the destructive sibling: the row itself disappears from the dashboard. + const deleted = await rpc(sendDeleteSession)(project.id, runId) + assert.equal(deleted.ok, true, `delete failed: ${'error' in deleted ? deleted.error : ''}`) + assert.equal((await rpc(onRuns)(project.id)).some(r => r.id === runId), false) + } finally { + await world.close() + } +}) diff --git a/packages/the-framework/src/e2e/story-tickets-and-queue.test.ts b/packages/the-framework/src/e2e/story-tickets-and-queue.test.ts new file mode 100644 index 00000000..f6278d60 --- /dev/null +++ b/packages/the-framework/src/e2e/story-tickets-and-queue.test.ts @@ -0,0 +1,133 @@ +import { strict as assert } from 'node:assert' +import { test } from 'node:test' +import { makeWorld, waitFor, withFakeAwait } from './harness.js' +import { + onTickets, + onTicket, + onAllTickets, + onHotTickets, + onQueue, + onRuns, +} from '../dashboard-rpc/reads.telefunc.js' +import { sendChoice, sendQueueTicket } from '../dashboard-rpc/control.telefunc.js' +import { presets } from '../preset-catalog.js' + +// The roadmap stories (spec.md): tickets are proposals, the flat TODO queue holds confirmed +// work, and a drain run claims the queue's next entry — the propose -> decide -> work loop the +// Tickets and Queue pages drive. + +const TICKET_FILE = '2026-08-01_login-page.md' +const TICKET = [ + 'priority: 8', + '', + '# Login page', + '', + '## TLDR', + '', + 'Add a login page with session cookies.', + '', +].join('\n') + +test('browse the ticket backlog: list, detail, and the cross-project pages (#697/#1144)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject({ + 'README.md': '# fixture\n', + [`tickets/${TICKET_FILE}`]: TICKET, + 'tickets/2026-08-02_dark-mode.md': '# Dark mode\n\n## TLDR\n\nHonor prefers-color-scheme.\n', + }) + + // The project's Tickets page: every ticket with its parsed row fields. + const tickets = await rpc(onTickets)(project.id) + assert.equal(tickets.length, 2) + const login = tickets.find(t => t.file === TICKET_FILE) + assert.equal(login?.title, 'Login page') + assert.equal(login?.priority, '8') + assert.equal(login?.summary, 'Add a login page with session cookies.') + + // The ticket's own page carries the full text; a sibling/path name is refused. + const detail = await rpc(onTicket)(project.id, TICKET_FILE) + assert.ok(detail?.content.includes('session cookies')) + assert.equal(await rpc(onTicket)(project.id, '../escape.md'), null) + + // The cross-project pages see the same backlog under this project. + const all = await rpc(onAllTickets)() + const mine = all.find(p => p.projectId === project.id) + assert.equal(mine?.tickets.length, 2) + } finally { + await world.close() + } +}) + +test('queue a ticket, then a drain run claims it and the boards show it in progress (#1164/#1117)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject({ + 'README.md': '# fixture\n', + [`tickets/${TICKET_FILE}`]: TICKET, + }) + + // The ticket page's Queue action: the entry lands in the flat backlog, linking back to the + // ticket, and the Queue page counts it as open work. + const queued = await rpc(sendQueueTicket)(project.id, 'Login page', { file: TICKET_FILE, priority: '8' }) + assert.equal(queued.ok, true, `queueing failed: ${queued.error ?? ''}`) + assert.equal(queued.file, 'TODO_AGENTS.md') + const queue = await rpc(onQueue)() + const projectQueue = queue.find(q => q.projectId === project.id) + assert.equal(projectQueue?.open, 1) + assert.ok(projectQueue?.items[0]?.text.includes(`tickets/${TICKET_FILE}`), 'the entry links back to its ticket') + + // The queued ticket shows on the hot-tickets rail. + const hotQueued = await rpc(onHotTickets)() + assert.ok(hotQueued.some(h => h.projectId === project.id && h.ticket.file === TICKET_FILE)) + + // A hand-fired drain resolves the queue's next entry to its ticket (#1117) — the run's meta + // names it while the run is live, which is what flips the boards to "implementing". + const runId = await withFakeAwait('choices', () => world.startRun(project, presets.drainQueue.render())) + const tail = await world.tailRun(project, runId) + const gate = await waitFor(() => tail.events.find(e => e.kind === 'choice'), 'the drain run to park') + + const argv = (await world.spawnedArgv())[0]! + const ticketFlag = argv.indexOf('--ticket') + assert.notEqual(ticketFlag, -1, `the drain child carries --ticket: ${argv.join(' ')}`) + assert.equal(argv[ticketFlag + 1], `tickets/${TICKET_FILE}`) + + const running = await waitFor(async () => { + const run = (await rpc(onRuns)(project.id)).find(r => r.id === runId) + return run?.ticket ? run : undefined + }, 'the run meta to name the claimed ticket') + assert.equal(running.ticket, `tickets/${TICKET_FILE}`) + const hot = await rpc(onHotTickets)() + const implementing = hot.find(h => h.ticket.file === TICKET_FILE && h.runId === runId) + assert.ok(implementing, 'the hot rail links the ticket to the run implementing it') + + if (gate.kind === 'choice') await rpc(sendChoice)(project.id, gate.id, gate.recommended!, 'user', runId) + await world.waitRun(project, runId, 'done') + } finally { + await world.close() + } +}) + +test('any other prompt claims nothing: the queue is only worked by a drain (#1117)', async () => { + const world = await makeWorld() + const rpc = world.rpc + try { + const project = await world.addProject({ + 'README.md': '# fixture\n', + [`tickets/${TICKET_FILE}`]: TICKET, + }) + await rpc(sendQueueTicket)(project.id, 'Login page', { file: TICKET_FILE, priority: '8' }) + + const runId = await world.startRun(project, 'Look into the flaky CI job') + await world.waitRun(project, runId, 'done') + const run = (await rpc(onRuns)(project.id)).find(r => r.id === runId) + assert.equal(run?.ticket, undefined, 'an unrelated prompt must not wear the queued ticket') + // The queue entry is still open: nothing consumed it. + const projectQueue = (await rpc(onQueue)()).find(q => q.projectId === project.id) + assert.equal(projectQueue?.open, 1) + } finally { + await world.close() + } +}) diff --git a/packages/the-framework/tsconfig.build.json b/packages/the-framework/tsconfig.build.json index e5780649..1617aec8 100644 --- a/packages/the-framework/tsconfig.build.json +++ b/packages/the-framework/tsconfig.build.json @@ -2,5 +2,5 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "outDir": "dist", "rootDir": "src" }, "include": ["src"], - "exclude": ["src/**/*.test.ts"] + "exclude": ["src/**/*.test.ts", "src/e2e"] }