Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/finished-session-races.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@gemstack/the-framework': patch
---

Two races in the finished-session seam are closed. Everything that mutates one run's checkout — teardown's archive-commit-retire, the Push/Open-PR commit step, Remove/Delete of the worktree, and a Resume's checkout reuse — now serializes on a per-run lock, so an action clicked the instant a session flips done waits a beat instead of failing with "could not commit the work this session left uncommitted" (and teardown no longer strands a worktree it lost that race to). And the live event feed follows the run's journal across its relocation into the archive: a fixed-path tail whose fs.watch missed the final appends used to go silent without the run's `end`; the tail now re-resolves the journal's home and carries its read offset, so exactly the missed lines arrive, once — in the dashboard's live channel and the device relay alike.
135 changes: 74 additions & 61 deletions packages/the-framework/src/daemon-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import type { EventsSource, PreviewHandlers, RemoteRuns } from './dashboard/tele
import { RelayedRuns, startRemoteRun } from './dashboard/remote-run.js'
import { runBranchFor } from './dashboard/run-handoff.js'
import { dispatchRelayRpc } from './dashboard-rpc/relay-dispatch.js'
import { tailEvents } from './dashboard-rpc/events-tail.js'
import { tailEvents, tailRunEvents } from './dashboard-rpc/events-tail.js'
import { isSafeVia } from './conversations.js'
import { ensureSessionsIgnored, resolveUserDir } from './sessions.js'
import { createPreviewRuntime } from './preview-runtime.js'
Expand All @@ -48,6 +48,7 @@ import { resolveProjectRunOptions } from './daemon-services.js'
import { installProject, enumerateGitRepos } from './install.js'
import { isGitRepo } from './project.js'
import { isCliTimeout } from './cli-exec.js'
import { withRunLock } from './run-locks.js'
import { errorMessage } from './error-message.js'
import { preflight, preflightProblems, type PreflightResult } from './preflight.js'
import { isAgentName, type AgentName } from './agent-names.js'
Expand Down Expand Up @@ -433,26 +434,31 @@ export function createProjectRuntime({ cwd, env, binPath, retryDelayMs, agentPre
* The branch is the session's if the agent named one, else the run-id branch it started on.
* Returns undefined when none of that is possible, so the caller can fall back to a new run.
*/
const continueWorkspace = async (projectCwd: string, runId: string): Promise<{ cwd: string; runId: string } | undefined> => {
try {
const path = worktreePath(projectCwd, runId)
const existing = await stat(path).then(s => s.isDirectory()).catch(() => false)
if (!existing) {
const archived = (await listRuns(projectCwd).catch(() => [])).find(run => run.id === runId)
// The recorded branch first (#1277): an agent that branched itself (#326 allows it) has
// its work there, and re-attaching by the session-name guess would continue the run on a
// branch without its previous commits.
const branch = runBranchFor(archived ?? { id: runId })
await attachWorktree(projectCwd, { runId, branch })
await linkDependencies(projectCwd, path).catch(() => [])
const continueWorkspace = (projectCwd: string, runId: string): Promise<{ cwd: string; runId: string } | undefined> =>
// Under the same run lock as teardown: a Resume clicked off a freshly-`done` run lands here
// while teardown is still archiving the very history this restores — reusing the checkout
// mid-retirement spawned the continuation into a tree about to be removed. Waiting the
// teardown out costs the click a beat and makes the reuse read a settled archive.
withRunLock(worktreePath(projectCwd, runId), async () => {
try {
const path = worktreePath(projectCwd, runId)
const existing = await stat(path).then(s => s.isDirectory()).catch(() => false)
if (!existing) {
const archived = (await listRuns(projectCwd).catch(() => [])).find(run => run.id === runId)
// The recorded branch first (#1277): an agent that branched itself (#326 allows it) has
// its work there, and re-attaching by the session-name guess would continue the run on a
// branch without its previous commits.
const branch = runBranchFor(archived ?? { id: runId })
await attachWorktree(projectCwd, { runId, branch })
await linkDependencies(projectCwd, path).catch(() => [])
}
await restoreArchivedRun(projectCwd, path, runId).catch(() => false)
return { cwd: path, runId }
} catch (err) {
console.log(`[framework] could not continue session ${runId} (${errorMessage(err)}); starting a new one`)
return undefined
}
await restoreArchivedRun(projectCwd, path, runId).catch(() => false)
return { cwd: path, runId }
} catch (err) {
console.log(`[framework] could not continue session ${runId} (${errorMessage(err)}); starting a new one`)
return undefined
}
}
})

/**
* Whether the agent this run picked can actually start (#1326), as one line to show when it
Expand Down Expand Up @@ -535,34 +541,40 @@ export function createProjectRuntime({ cwd, env, binPath, retryDelayMs, agentPre
*/
/** The project half of a preview key from a checkout: the registry id every preview RPC keys by. */
const projectKeyFor = (projectCwd: string): string => projectId(resolve(projectCwd))
const tearDownWorktree = async (projectCwd: string, worktree: string, runId?: string): Promise<void> => {
try {
// A session can be serving its own checkout (#797), and that dev server holds the directory
// it is about to lose. Stop it first, whether or not the worktree ends up removed: the run
// is over, so the preview is serving a tree nothing is working on.
await previews.preview.stop(projectKeyFor(projectCwd), runId)
// Where the work ended up, recorded before the checkout can go (#799). The branch outlives
// the worktree and is the only handle the dashboard has left on a finished session.
const branch = await currentBranch(worktree)
// Filed under the identity this repo commits as, and the ignore rules taught to keep it, so
// the session survives the repo being cleaned (#1179).
const user = await resolveUserDir(projectCwd)
await ensureSessionsIgnored(projectCwd, user).catch(() => false)
const meta = await archiveWorktreeRun(worktree, projectCwd, undefined, branch, user)
if (meta?.status !== 'done') return // failed / stopped / unreadable: keep it for inspection
// A finished run can still be holding an uncommitted edit (#786), and removing the
// checkout would destroy it. Commit it to the run's branch, which outlives the
// worktree; if that cannot be done, keep the checkout rather than take the diff with it.
if (!(await commitPendingWork(worktree))) {
console.log(`[framework] keeping worktree ${worktree}: its uncommitted work could not be committed`)
return
// Under the run lock: a Push/Remove/Resume fired off a freshly-`done` meta lands in the daemon
// while this is mid-archive, and both sides commit in the same checkout. The loser used to
// report "could not commit the work this session left uncommitted" — or worse, this side lost
// and kept a worktree it should have removed. Serialized, whoever runs first commits the whole
// pending state (`add -A`) and the other side finds a clean tree and carries on.
const tearDownWorktree = (projectCwd: string, worktree: string, runId?: string): Promise<void> =>
withRunLock(worktree, async () => {
try {
// A session can be serving its own checkout (#797), and that dev server holds the directory
// it is about to lose. Stop it first, whether or not the worktree ends up removed: the run
// is over, so the preview is serving a tree nothing is working on.
await previews.preview.stop(projectKeyFor(projectCwd), runId)
// Where the work ended up, recorded before the checkout can go (#799). The branch outlives
// the worktree and is the only handle the dashboard has left on a finished session.
const branch = await currentBranch(worktree)
// Filed under the identity this repo commits as, and the ignore rules taught to keep it, so
// the session survives the repo being cleaned (#1179).
const user = await resolveUserDir(projectCwd)
await ensureSessionsIgnored(projectCwd, user).catch(() => false)
const meta = await archiveWorktreeRun(worktree, projectCwd, undefined, branch, user)
if (meta?.status !== 'done') return // failed / stopped / unreadable: keep it for inspection
// A finished run can still be holding an uncommitted edit (#786), and removing the
// checkout would destroy it. Commit it to the run's branch, which outlives the
// worktree; if that cannot be done, keep the checkout rather than take the diff with it.
if (!(await commitPendingWork(worktree))) {
console.log(`[framework] keeping worktree ${worktree}: its uncommitted work could not be committed`)
return
}
await removeWorktree(projectCwd, worktree)
await pruneWorktrees(projectCwd)
} catch {
// A worktree we could not retire is a worktree left on disk, which is the safe direction.
}
await removeWorktree(projectCwd, worktree)
await pruneWorktrees(projectCwd)
} catch {
// A worktree we could not retire is a worktree left on disk, which is the safe direction.
}
}
})

// One more try for a run the API dropped mid-work (#1281): the failure is about the transport,
// not the work, and the continue-run machinery (#762/#923) reopens the retained checkout on its
Expand Down Expand Up @@ -1010,21 +1022,22 @@ export function createProjectRuntime({ cwd, env, binPath, retryDelayMs, agentPre
// else undefined so `onEvents` tails the on-disk log as usual for an ordinary local run.
const remoteEventsSource: EventsSource = (_projectId, runId) => relayedRuns.get(runId)

// Tail a relay-started run's own log (#1067) for the `/_relay/events` endpoint. Resolving the run's
// journal is async, so a stop is returned immediately and the tail attaches once the path is known.
// Tail a relay-started run's own log (#1067) for the `/_relay/events` endpoint. The relocating
// tail, for the same reason as the dashboard's onEvents: teardown moves the journal into the
// archive, and the device's fixed-path tail went silent without the run's final events. The
// initial attach takes whatever the resolver answers (a non-git fallback run's journal IS the
// root one); a relocation refuses the root fallback — there it is another run's feed.
const rootJournal = join(cwd, FRAMEWORK_DIR, EVENTS_FILE)
const tailRelayEvents = (runId: string, onEvent: (event: FrameworkEvent) => void): (() => void) => {
let stop = (): void => {}
let cancelled = false
void resolveRunEventsPath(cwd, runId)
.then(path => {
if (cancelled) return
stop = tailEvents(path, onEvent)
})
.catch(() => {})
return () => {
cancelled = true
stop()
}
let initial = true
return tailRunEvents<FrameworkEvent>(async () => {
const next = await resolveRunEventsPath(cwd, runId)
if (initial) {
initial = false
return next
}
return next === rootJournal ? undefined : next
}, onEvent)
}

const dispose = async (): Promise<void> => {
Expand Down
22 changes: 18 additions & 4 deletions packages/the-framework/src/dashboard-rpc/control.telefunc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import { appendFlatTodoEntry, ticketForPrompt } from '../todo-loop.js'
import { TICKETS_DIR, todoPriorityForTicket } from '../tickets.js'
import { isTicketFile } from '../dashboard/tickets.js'
import { releaseTicketLock } from '../ticket-locks.js'
import { findRun, isSafeRunId, type RunMeta } from '../store/index.js'
import { findRun, isSafeRunId, worktreePath, type RunMeta } from '../store/index.js'
import { withRunLock } from '../run-locks.js'
import { removeProjectWorktree, deleteProjectRun } from '../worktrees.js'
import { commitSessionWork, mergeSessionPr, openSessionPullRequest, pushRunBranch, runBranchFor, type HandoffResult } from '../dashboard/run-handoff.js'
import type { ChoiceBy } from '../events.js'
Expand Down Expand Up @@ -161,7 +162,12 @@ async function withWorktreeRemoval<T>(
const preview = contextPreview()
const cwd = await resolveProjectPath(projectId)
if (!cwd) return { ok: false, error: 'this project has no local path on this server' }
return remove(cwd, { beforeRemove: async id => { await preview?.stop(projectId, id) } })
// Under the run lock: a Remove/Delete clicked the moment a run ends races teardown's own
// archive-commit-remove of the same checkout; serialized, whichever runs second finds the
// state the first one left and acts on that.
return withRunLock(worktreePath(cwd, runId), () =>
remove(cwd, { beforeRemove: async id => { await preview?.stop(projectId, id) } }),
)
}

/**
Expand Down Expand Up @@ -310,7 +316,11 @@ export async function sendPushBranch(projectId: string, runId: string): Promise<
const target = await handoffTargetFor(projectId, runId)
if (!target) return { ok: false, error: 'unknown session' }
const branch = runBranchFor(target.run)
if (!(await commitSessionWork(target.checkout, target.cwd, branch))) {
// The commit step holds the run lock: clicked the moment a session flips `done`, this used
// to commit against the checkout teardown was committing in and lose. Serialized, whichever
// side runs first commits everything pending; the other finds a clean tree — or no checkout
// at all, which commitSessionWork already reads as "the branch is authoritative".
if (!(await withRunLock(target.checkout, () => commitSessionWork(target.checkout, target.cwd, branch)))) {
return { ok: false, error: 'could not commit the work this session left uncommitted' }
}
return pushRunBranch(target.cwd, branch)
Expand All @@ -328,7 +338,11 @@ export async function sendOpenPullRequest(projectId: string, runId: string): Pro
return relayOr(runId, 'sendOpenPullRequest', [projectId, runId], async () => {
const target = await handoffTargetFor(projectId, runId)
if (!target) return { ok: false, error: 'unknown session' }
if (!(await commitSessionWork(target.checkout, target.cwd, runBranchFor(target.run)))) {
// Same run lock as sendPushBranch, for the same click-at-`done` race.
const committed = await withRunLock(target.checkout, () =>
commitSessionWork(target.checkout, target.cwd, runBranchFor(target.run)),
)
if (!committed) {
return { ok: false, error: 'could not commit the work this session left uncommitted' }
}
return openSessionPullRequest(target.cwd, target.run)
Expand Down
98 changes: 97 additions & 1 deletion packages/the-framework/src/dashboard-rpc/events-tail.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { appendFile, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type { FrameworkEvent } from '../events.js'
import { tailEvents } from './events-tail.js'
import { tailEvents, tailRunEvents } from './events-tail.js'

const line = (message: string): string => JSON.stringify({ kind: 'log', message } satisfies FrameworkEvent) + '\n'
const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms))
Expand Down Expand Up @@ -117,3 +117,99 @@ test('tailEvents reports the replay boundary even when the log does not exist ye
await rm(cwd, { recursive: true, force: true })
}
})

// The relocating tail: a run's journal is copied verbatim into the archive at teardown and the
// worktree is removed. tailRunEvents re-resolves the path when the tailed file disappears and
// carries the read offset across the move, so the feed gets exactly the lines the move would
// have swallowed — once, with no replay of what was already delivered.

test('tailRunEvents follows the journal into the archive: missed lines arrive exactly once', async () => {
const cwd = await tmpWorkspace()
const live = join(cwd, 'worktree-events.jsonl')
const archive = join(cwd, 'archived-events.jsonl')
await writeFile(live, line('one') + line('two'))
const seen: string[] = []
let sync = 0
const { rename } = await import('node:fs/promises')
const stop = tailRunEvents<FrameworkEvent>(
async () => ((await import('node:fs')).existsSync(live) ? live : archive),
e => void (e.kind === 'log' && seen.push(e.message)),
() => sync++,
)
try {
await sleep(200)
assert.deepEqual(seen, ['one', 'two'])
// The retirement, compressed: the final lines land and the journal moves in one breath, so
// whether the watcher saw the appends before the move is scheduling luck — exactly the
// window that used to swallow a fast run's `end`. Either way the tail must deliver
// everything, each line once.
await appendFile(live, line('three') + line('four'))
await rename(live, archive)
await sleep(1600) // fs.watch is unreliable on CI; wait out the poll backstop behind it
assert.deepEqual(seen, ['one', 'two', 'three', 'four'])
// A relocation is not a new replay boundary: the marker stays once-per-subscription (#1383).
assert.equal(sync, 1)
} finally {
stop()
await rm(cwd, { recursive: true, force: true })
}
})

test('tailRunEvents does not replay a fully-consumed journal after the move', async () => {
const cwd = await tmpWorkspace()
const live = join(cwd, 'worktree-events.jsonl')
const archive = join(cwd, 'archived-events.jsonl')
await writeFile(live, line('one') + line('two'))
const seen: string[] = []
const { copyFile, rm: rmFile } = await import('node:fs/promises')
const stop = tailRunEvents<FrameworkEvent>(
async () => ((await import('node:fs')).existsSync(live) ? live : archive),
e => void (e.kind === 'log' && seen.push(e.message)),
)
try {
await sleep(200)
assert.deepEqual(seen, ['one', 'two'])
await sleep(20) // let the copy's mtime land visibly later than the seeding read
// The archive path: a same-content copy with a NEWER mtime and the SAME length as what was
// consumed — the exact shape the #567 same-length-rewrite detection resets on. The retarget
// must adopt the copy's mtime instead, or every line would be delivered twice.
await copyFile(live, archive)
await rmFile(live)
await sleep(1600)
assert.deepEqual(seen, ['one', 'two'])
} finally {
stop()
await rm(cwd, { recursive: true, force: true })
}
})

test('tailRunEvents stays put while the resolver has no better answer', async () => {
const cwd = await tmpWorkspace()
const live = join(cwd, 'worktree-events.jsonl')
const archive = join(cwd, 'archived-events.jsonl')
await writeFile(live, line('one'))
const seen: string[] = []
const { copyFile, rm: rmFile } = await import('node:fs/promises')
let archiveVisible = false
const stop = tailRunEvents<FrameworkEvent>(
// The window where the live file is gone but the archive is not resolvable yet: the
// resolver answers undefined (a deleted session resolves like this forever), and the tail
// must idle rather than hop somewhere wrong — then catch up once the archive appears.
async () => ((await import('node:fs')).existsSync(live) ? live : archiveVisible ? archive : undefined),
e => void (e.kind === 'log' && seen.push(e.message)),
)
try {
await sleep(200)
assert.deepEqual(seen, ['one'])
await appendFile(live, line('two'))
await copyFile(live, archive)
await rmFile(live)
await sleep(1300) // a full poll with the resolver still answering undefined
archiveVisible = true
await sleep(1600)
assert.deepEqual(seen, ['one', 'two'])
} finally {
stop()
await rm(cwd, { recursive: true, force: true })
}
})
Loading
Loading