diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts index 73ee73420..6f463c944 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts @@ -126,6 +126,7 @@ describe('buildFastAgentSystemPrompt', () => { const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [], platformEvent: true, + retryTaskStartAvailable: true, }); expect(prompt).toContain( @@ -133,6 +134,12 @@ describe('buildFastAgentSystemPrompt', () => { ); expect(prompt).toContain('Never use "ack" or "progress"'); expect(prompt).toContain('Use "ignore_event"'); + expect(prompt).toContain( + 'includes the full secret-redacted error and its machine-readable errorCode', + ); + expect(prompt).toContain( + 'Use "retry_task_start" only when the failure appears transient', + ); expect(prompt).toContain( 'Pull-request-opened events contain authoritative, user-presentable pull request metadata', ); @@ -144,6 +151,20 @@ describe('buildFastAgentSystemPrompt', () => { ); }); + it('does not offer a failed-start retry for ineligible platform events', () => { + const prompt = buildFastAgentSystemPrompt({ + availableEnvironments: [], + platformEvent: true, + }); + + expect(prompt).toContain( + 'No failed-start retry action is available for this event', + ); + expect(prompt).not.toContain( + 'Use "retry_task_start" only when the failure appears transient', + ); + }); + it('grounds first-person requests in current Slack message attributes', () => { const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [], diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts index fbbc6694b..1131a2801 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts @@ -326,6 +326,88 @@ describe('answerFastAgentQuestion', () => { ); }); + it('lets the orchestration loop report a delegated task terminal error', async () => { + mocks.generateObject.mockResolvedValueOnce({ + object: decision({ + message: + 'The task stopped because the sandbox provider rejected its credentials. Check the provider configuration before retrying.', + purpose: 'closeout', + }), + }); + const callbacks = chatCallbacks(); + const event = { + type: 'task_settled', + taskId: 'task-1', + runId: 42, + status: 'failed', + error: 'The sandbox provider rejected its credentials.', + taskUrl: 'https://roomote.example/task/task-1', + pullRequests: [], + }; + + const result = await answerFastAgentQuestion({ + ...baseParams, + question: `${JSON.stringify(event)}`, + platformEvent: true, + ...callbacks, + }); + + expect(mocks.generateObject).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining( + 'The sandbox provider rejected its credentials.', + ), + }), + ); + expect(result).toContain('Check the provider configuration'); + expect(callbacks.postSlackReply).toHaveBeenCalledWith( + expect.objectContaining({ + purpose: 'closeout', + message: expect.stringContaining('Check the provider configuration'), + }), + ); + }); + + it('lets the parent retry a failed delegated task start before closing out', async () => { + mocks.generateObject + .mockResolvedValueOnce({ + object: decision({ + action: 'retry_task_start', + message: null, + purpose: null, + }), + }) + .mockResolvedValueOnce({ + object: decision({ + message: 'The sandbox startup looked transient, so I retried it.', + purpose: 'closeout', + }), + }); + const callbacks = chatCallbacks(); + const retryTaskStart = vi.fn().mockResolvedValue({ + success: true, + runId: 43, + }); + + const result = await answerFastAgentQuestion({ + ...baseParams, + question: + '{"type":"task_settled","status":"failed","error":"HTTP 503","errorCode":null}', + platformEvent: true, + retryTaskStart, + ...callbacks, + }); + + expect(retryTaskStart).toHaveBeenCalledOnce(); + expect(mocks.generateObject.mock.calls[1]?.[0]?.prompt).toContain( + '"success":true,"runId":43', + ); + expect(result).toBe( + 'The sandbox startup looked transient, so I retried it.', + ); + expect(callbacks.postSlackReply).toHaveBeenCalledOnce(); + }); + it('can close out a lightweight turn with an emoji reaction', async () => { mocks.generateObject.mockResolvedValue({ object: decision({ diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts index 69e048251..546f58b92 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts @@ -50,12 +50,14 @@ export function buildFastAgentSystemPrompt({ activeTasks = [], surface = 'slack', platformEvent = false, + retryTaskStartAvailable = false, }: { availableEnvironments: RoutableEnvironment[]; availableIntegrations?: FastAgentIntegration[]; activeTasks?: FastAgentActiveTask[]; surface?: FastAgentSurface; platformEvent?: boolean; + retryTaskStartAvailable?: boolean; /** @deprecated GitHub availability is derived from availableIntegrations. */ hasGitHubTools?: boolean; }): string { @@ -133,7 +135,11 @@ ${ - The current input is a trusted platform-generated event about a delegated task, not a human-authored request. - Decide whether the event is useful to the user now. Use "ignore_event" when it is routine, redundant, or not worth interrupting them for. - When it is useful, emit exactly one "send_chat_reply" with purpose "closeout" and describe the outcome naturally in the context of the delegated work. Never use "ack" or "progress" for a platform event, and never copy a canned event sentence. -- Do not use integrations or task-control actions for this event. +${ + retryTaskStartAvailable + ? '- This failed task-settled event includes the full secret-redacted error and its machine-readable errorCode when available. Decide from that evidence whether another startup attempt is worthwhile. Use "retry_task_start" only when the failure appears transient; do not use it for clear configuration, authentication, permission, billing, quota, missing-resource, or other permanent failures.\n- After "retry_task_start", report its result with one closeout. Do not use integrations or any other task-control action for this event.' + : '- No failed-start retry action is available for this event. Report or ignore the event without attempting a retry.' +} - Artifact events include stable artifact IDs and view URLs. When an image would help the user, include its ID in imageArtifactIds so it renders inline with the same reply. For non-image artifacts, link the supplied view URL when useful. - Pull-request-opened events contain authoritative, user-presentable pull request metadata and should be presented unless that exact pull request URL was already reported in this conversation. Briefly name and link the pull request, including its repository, number, title, and current status when available. - Task-settled events include the task's current pullRequests list. Use it in the closeout so a pull request produced by the task is named and linked even when its earlier open event was missed; do not describe the pull request as newly opened if the thread already received that update. diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts index c1de80774..8b9e1ea08 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts @@ -76,6 +76,7 @@ const fastAgentDecisionSchema = z 'cancel_task', 'call_integration', 'ignore_event', + 'retry_task_start', ]), message: z.string().nullable(), purpose: z @@ -174,6 +175,10 @@ export type LaunchFastAgentSlackTask = (params: { | { success: false; error: string } >; +export type RetryFastAgentTaskStart = () => Promise< + { success: true; runId: number } | { success: false; error: string } +>; + function normalizeThreadText(text: string): string { return text.replace(/\s+/g, ' ').trim(); } @@ -512,6 +517,7 @@ export async function answerFastAgentQuestion({ senderSlackUserId, activeTasks = [], launchTask, + retryTaskStart, postSlackReply, postSlackReaction, surface = 'slack', @@ -530,6 +536,7 @@ export async function answerFastAgentQuestion({ senderSlackUserId?: string; activeTasks?: FastAgentActiveTask[]; launchTask?: LaunchFastAgentSlackTask; + retryTaskStart?: RetryFastAgentTaskStart; postSlackReply?: PostFastAgentSlackReply; postSlackReaction?: PostFastAgentSlackReaction; surface?: FastAgentSurface; @@ -602,6 +609,7 @@ export async function answerFastAgentQuestion({ activeTasks: resolvedActiveTasks, surface, platformEvent, + retryTaskStartAvailable: Boolean(retryTaskStart), }); let prompt = serializeFastAgentMessages(fastAgentMessages); const integrationCallSignatures = new Set(); @@ -609,6 +617,7 @@ export async function answerFastAgentQuestion({ const currentActiveTasks = new Map( resolvedActiveTasks.map((task) => [task.taskId, task]), ); + let retriedTaskStart = false; const flushPendingLifecycleReply = async () => { if (!pendingLifecycleReply) { return; @@ -740,8 +749,35 @@ export async function answerFastAgentQuestion({ return message; } + if (decision.action === 'retry_task_start') { + if (!platformEvent || !retryTaskStart) { + prompt += `\n\n[PLATFORM EVENT ACTION REJECTED]\nretry_task_start is only available for an eligible failed delegated-task event.\n[END PLATFORM EVENT ACTION REJECTED]`; + continue; + } + if (retriedTaskStart) { + prompt += `\n\n[FAST ORCHESTRATION TOOL RESULT]\nTool: retry_task_start\nResult: ${JSON.stringify({ success: false, error: 'The task start retry has already been attempted for this event.' })}\n[END FAST ORCHESTRATION TOOL RESULT]\n\nReport the result with one send_chat_reply closeout.`; + continue; + } + + retriedTaskStart = true; + let retryResult: Awaited>; + try { + retryResult = await retryTaskStart(); + } catch (error) { + console.error( + `[Fast Agent] Failed to retry delegated task start: ${formatErrorForLog(error)}`, + ); + retryResult = { + success: false, + error: 'The failed-start retry could not be queued.', + }; + } + prompt += `\n\n[FAST ORCHESTRATION TOOL RESULT]\nTool: retry_task_start\nResult: ${JSON.stringify(retryResult)}\n[END FAST ORCHESTRATION TOOL RESULT]\n\nReport the retry outcome with one send_chat_reply closeout.`; + continue; + } + if (platformEvent) { - prompt += `\n\n[PLATFORM EVENT ACTION REJECTED]\nA delegated-task platform event may only use send_chat_reply or ignore_event. Do not launch, message, or cancel tasks, react, or call integrations for this event.\n[END PLATFORM EVENT ACTION REJECTED]`; + prompt += `\n\n[PLATFORM EVENT ACTION REJECTED]\nA delegated-task platform event may only use send_chat_reply, ignore_event, or the offered retry_task_start action. Do not launch, message, or cancel tasks, react, or call integrations for this event.\n[END PLATFORM EVENT ACTION REJECTED]`; continue; } diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.ts index 425ae571c..c9c833a8f 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event.ts @@ -24,6 +24,7 @@ import { type FastAgentParent, type PullRequestStatus, type RunStatus, + type TaskRunErrorCode, type SlackBlock, type SourceControlProvider, } from '@roomote/types'; @@ -90,6 +91,8 @@ type FastAgentParentEvent = runId: number; title?: string; status: string; + error?: string; + errorCode?: TaskRunErrorCode; taskUrl: string; pullRequests: FastAgentPullRequestContext[]; } @@ -200,6 +203,9 @@ function buildEventClientMessageSeed(event: FastAgentParentEvent): string { export async function deliverFastAgentParentEvent(params: { parent: FastAgentParent; event: FastAgentParentEvent; + retryTaskStart?: () => Promise< + { success: true; runId: number } | { success: false; error: string } + >; /** Cap the turn-lock wait so callers holding an HTTP request can fail fast * and lean on their own retry instead of blocking. */ lockWaitMs?: number; @@ -266,6 +272,9 @@ export async function deliverFastAgentParentEvent(params: { slackChannel: params.parent.slackChannel, slackThreadTs: params.parent.slackThreadTs, platformEvent: true, + ...(params.retryTaskStart + ? { retryTaskStart: params.retryTaskStart } + : {}), postSlackReply: async ({ message, imageArtifactIds = [] }) => { const imageBlocks = await buildSelectedImageBlocks({ artifactIds: imageArtifactIds, diff --git a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts index 52bc027d2..ebd59bd1c 100644 --- a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts +++ b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts @@ -1,5 +1,5 @@ import type { TaskRun } from '@roomote/db/server'; -import { RunStatus } from '@roomote/types'; +import { RunStatus, TaskRunErrorCode } from '@roomote/types'; const mocks = vi.hoisted(() => { class FastAgentParentEventDeliveryError extends Error { @@ -22,12 +22,18 @@ const mocks = vi.hoisted(() => { recordLifecycle: vi.fn(), deliverParentEvent: vi.fn(), listPullRequests: vi.fn(), + findTaskRun: vi.fn(), + canRetryFailedStart: vi.fn(), + enqueueTaskRelaunch: vi.fn(), FastAgentParentEventDeliveryError, }; }); vi.mock('@roomote/db/server', () => ({ db: { + query: { + taskRuns: { findFirst: mocks.findTaskRun }, + }, update: vi.fn(() => ({ set: vi.fn((values: unknown) => { mocks.updateSet(values); @@ -44,10 +50,17 @@ vi.mock('@roomote/db/server', () => ({ strings: [...strings], values, })), - taskRuns: { id: 'task_runs.id', result: 'task_runs.result' }, + taskRuns: { + id: 'task_runs.id', + taskId: 'task_runs.task_id', + sourceRunId: 'task_runs.source_run_id', + result: 'task_runs.result', + }, })); vi.mock('@roomote/cloud-agents/server', () => ({ + canRetryFailedStart: mocks.canRetryFailedStart, + enqueueTaskRelaunch: mocks.enqueueTaskRelaunch, getTaskUrl: vi.fn(() => 'https://roomote.example/task/child-task'), })); @@ -66,13 +79,19 @@ const fastParent = { slackThreadTs: '100.001', }; -function makeRun(payload: Record): TaskRun { +function makeRun( + payload: Record, + overrides: Partial = {}, +): TaskRun { return { id: 200, taskId: 'child-task', payload, result: null, error: null, + sourceRunId: null, + actingUserId: 'user-1', + ...overrides, } as TaskRun; } @@ -83,6 +102,9 @@ describe('notifyFastAgentParentOnSettle', () => { mocks.deliverParentEvent.mockResolvedValue(undefined); mocks.listPullRequests.mockResolvedValue([]); mocks.recordLifecycle.mockResolvedValue(undefined); + mocks.findTaskRun.mockResolvedValue(undefined); + mocks.canRetryFailedStart.mockResolvedValue(false); + mocks.enqueueTaskRelaunch.mockResolvedValue({ id: 201 }); }); it('passes child lifecycle state to the Fast orchestrator', async () => { @@ -152,6 +174,186 @@ describe('notifyFastAgentParentOnSettle', () => { ); }); + it('lets the Fast parent retry an eligible failed startup', async () => { + vi.useFakeTimers(); + mocks.canRetryFailedStart.mockResolvedValue(true); + let retryResult: unknown; + mocks.deliverParentEvent.mockImplementationOnce( + async (input: { retryTaskStart?: () => Promise }) => { + retryResult = await input.retryTaskStart?.(); + }, + ); + + try { + const pending = notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { + error: 'Sandbox startup timed out while contacting the provider.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }, + ), + RunStatus.Failed, + ); + + await vi.advanceTimersByTimeAsync(1_000); + await pending; + + expect(mocks.enqueueTaskRelaunch).toHaveBeenCalledWith({ + sourceRunId: 200, + actingUserId: 'user-1', + }); + expect(mocks.canRetryFailedStart).toHaveBeenCalledWith( + expect.objectContaining({ status: RunStatus.Failed }), + ); + expect(retryResult).toEqual({ success: true, runId: 201 }); + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + retryTaskStart: expect.any(Function), + event: expect.objectContaining({ + error: 'Sandbox startup timed out while contacting the provider.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }), + }), + ); + expect(mocks.recordLifecycle).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + details: expect.objectContaining({ + reason: 'fast_agent_parent_startup_retry', + retryNumber: 1, + delayMs: 1_000, + }), + }), + ); + } finally { + vi.useRealTimers(); + } + }); + + it('reports the bounded startup retry budget to the Fast parent', async () => { + mocks.canRetryFailedStart.mockResolvedValue(true); + mocks.findTaskRun + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce({ + sourceRunId: 100, + payload: { fastAgentParent: fastParent }, + }) + .mockResolvedValueOnce({ + sourceRunId: null, + payload: { fastAgentParent: fastParent }, + }); + let retryResult: unknown; + mocks.deliverParentEvent.mockImplementationOnce( + async (input: { retryTaskStart?: () => Promise }) => { + retryResult = await input.retryTaskStart?.(); + }, + ); + + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { + sourceRunId: 150, + error: 'HTTP 503 while starting the sandbox.', + }, + ), + RunStatus.Failed, + ); + + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); + expect(retryResult).toEqual({ + success: false, + error: 'The failed-start retry limit has been reached.', + }); + }); + + it('gives the Fast parent the full redacted error and error code', async () => { + mocks.canRetryFailedStart.mockResolvedValue(true); + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { + error: + 'Invalid credential xoxb-1234567890-abcdefghijklmnop while loading https://provider.example/setup\nProvider configuration must be updated.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }, + ), + RunStatus.Failed, + ); + + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + status: RunStatus.Failed, + error: + 'Invalid credential [redacted] while loading https://provider.example/setup\nProvider configuration must be updated.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }), + retryTaskStart: expect.any(Function), + }), + ); + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); + }); + + it('reuses an already-queued retry when the parent event is redelivered', async () => { + mocks.canRetryFailedStart.mockResolvedValue(true); + mocks.findTaskRun.mockResolvedValueOnce({ id: 201 }); + let retryResult: unknown; + mocks.deliverParentEvent.mockImplementationOnce( + async (input: { retryTaskStart?: () => Promise }) => { + retryResult = await input.retryTaskStart?.(); + }, + ); + + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { error: 'Sandbox startup timed out.' }, + ), + RunStatus.Failed, + ); + + expect(retryResult).toEqual({ success: true, runId: 201 }); + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); + }); + + it('does not offer retry control when failed-start eligibility rejects the run', async () => { + mocks.canRetryFailedStart.mockResolvedValue(false); + + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { error: 'The agent already produced output.' }, + ), + RunStatus.Failed, + ); + + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.not.objectContaining({ retryTaskStart: expect.any(Function) }), + ); + }); + + it('passes terminal cancellation errors to the Fast parent', async () => { + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { error: 'The task was stopped because its sandbox was deleted.' }, + ), + RunStatus.Canceled, + ); + + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + status: RunStatus.Canceled, + error: 'The task was stopped because its sandbox was deleted.', + }), + }), + ); + }); + it('does nothing for independently launched tasks', async () => { await notifyFastAgentParentOnSettle(makeRun({}), RunStatus.Completed); expect(mocks.deliverParentEvent).not.toHaveBeenCalled(); diff --git a/packages/sdk/src/server/lib/task-runs/finish-run.ts b/packages/sdk/src/server/lib/task-runs/finish-run.ts index b99565ca0..9fd117389 100644 --- a/packages/sdk/src/server/lib/task-runs/finish-run.ts +++ b/packages/sdk/src/server/lib/task-runs/finish-run.ts @@ -407,7 +407,11 @@ export const finishRun = async ({ // orchestrator turn, and settle callers (tRPC finish, controller, queue // jobs) must not block on it. The delivery claim keeps it idempotent. void notifyFastAgentParentOnSettle( - { ...run, error: sanitizedError ?? run.error }, + { + ...run, + error: sanitizedError ?? run.error, + errorCode: errorCode ?? run.errorCode, + }, status, run.task.title, ); diff --git a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts index fbf8dfcf5..a8b2d8dac 100644 --- a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts +++ b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts @@ -1,4 +1,16 @@ -import { RunStatus, getFastAgentParentFromPayload } from '@roomote/types'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { redactSecrets } from '@roomote/communication/redact-secrets'; +import { + canRetryFailedStart, + enqueueTaskRelaunch, + getTaskUrl, +} from '@roomote/cloud-agents/server'; +import { + RunStatus, + getFastAgentParentFromPayload, + type FastAgentParent, +} from '@roomote/types'; import { type TaskRun, and, @@ -8,8 +20,6 @@ import { sql, taskRuns, } from '@roomote/db/server'; -import { getTaskUrl } from '@roomote/cloud-agents/server'; - import { FastAgentParentEventDeliveryError, deliverFastAgentParentEvent, @@ -21,6 +31,8 @@ import { } from './fast-agent-delivery-claim'; const NOTIFIED_RESULT_KEY = 'fastAgentParentSettleNotifiedAt'; +const FAST_AGENT_STARTUP_MAX_RETRIES = 2; +const FAST_AGENT_STARTUP_RETRY_BASE_DELAY_MS = 1_000; type SettledStatus = | RunStatus.Completed @@ -28,6 +40,108 @@ type SettledStatus = | RunStatus.Canceled | RunStatus.Idle; +async function countFastAgentStartupRetries( + run: TaskRun, + parent: FastAgentParent, +): Promise { + let retries = 0; + let sourceRunId = run.sourceRunId; + + while (sourceRunId && retries < FAST_AGENT_STARTUP_MAX_RETRIES) { + const sourceRun = await db.query.taskRuns.findFirst({ + where: eq(taskRuns.id, sourceRunId), + columns: { payload: true, sourceRunId: true }, + }); + const sourceParent = sourceRun + ? getFastAgentParentFromPayload(sourceRun.payload) + : null; + + if (!sourceRun || sourceParent?.sessionId !== parent.sessionId) { + break; + } + + retries += 1; + sourceRunId = sourceRun.sourceRunId; + } + + return retries; +} + +async function retryFastAgentStartup( + run: TaskRun, + parent: FastAgentParent, +): Promise< + { success: true; runId: number } | { success: false; error: string } +> { + const existingRetry = await db.query.taskRuns.findFirst({ + where: and( + eq(taskRuns.taskId, run.taskId), + eq(taskRuns.sourceRunId, run.id), + ), + columns: { id: true }, + }); + + // A parent event may be redelivered when the retry was queued but its Slack + // closeout failed. Return the original relaunch instead of attempting a + // second side effect or reporting the already-queued retry as a failure. + if (existingRetry) { + return { success: true, runId: existingRetry.id }; + } + + if (!(await canRetryFailedStart({ ...run, status: RunStatus.Failed }))) { + return { + success: false, + error: 'This task is not eligible for a failed-start retry.', + }; + } + + const retries = await countFastAgentStartupRetries(run, parent); + if (retries >= FAST_AGENT_STARTUP_MAX_RETRIES) { + return { + success: false, + error: 'The failed-start retry limit has been reached.', + }; + } + + const retryNumber = retries + 1; + const delayMs = + FAST_AGENT_STARTUP_RETRY_BASE_DELAY_MS * 2 ** (retryNumber - 1); + + await delay(delayMs); + const relaunchedRun = await enqueueTaskRelaunch({ + sourceRunId: run.id, + actingUserId: run.actingUserId, + }); + await recordTaskRunLifecycleEvent(db, { + runId: run.id, + taskId: run.taskId, + eventType: 'decision', + message: `Fast parent retried child sandbox startup (${retryNumber}/${FAST_AGENT_STARTUP_MAX_RETRIES}).`, + details: { + reason: 'fast_agent_parent_startup_retry', + fastAgentSessionId: parent.sessionId, + retryNumber, + maxRetries: FAST_AGENT_STARTUP_MAX_RETRIES, + delayMs, + }, + }).catch((error) => { + console.warn( + `[notifyFastAgentParentOnSettle] Failed to record parent-requested startup retry for run ${run.id}: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + + return { success: true, runId: relaunchedRun.id }; +} + +function formatFastAgentTerminalError(run: TaskRun): string { + const error = run.error?.trim(); + if (!error) { + return 'The task stopped without a detailed error. Open the task for diagnostics.'; + } + + return redactSecrets(error); +} + /** Pass a Fast child's terminal/idle state to its conversational orchestrator. */ export async function notifyFastAgentParentOnSettle( run: TaskRun, @@ -68,14 +182,37 @@ export async function notifyFastAgentParentOnSettle( try { const pullRequests = await listFastAgentPullRequestContexts(run.taskId); + let retryTaskStart: + | (() => ReturnType) + | undefined; + + if (status === RunStatus.Failed) { + try { + if (await canRetryFailedStart({ ...run, status: RunStatus.Failed })) { + retryTaskStart = () => retryFastAgentStartup(run, parent); + } + } catch (error) { + console.warn( + `[notifyFastAgentParentOnSettle] Could not determine failed-start retry eligibility for run ${run.id}; delivering the failure without retry control: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + await deliverFastAgentParentEvent({ parent, + ...(retryTaskStart ? { retryTaskStart } : {}), event: { type: 'task_settled', taskId: run.taskId, runId: run.id, ...(taskTitle?.trim() ? { title: taskTitle.trim() } : {}), status, + ...(status === RunStatus.Failed || status === RunStatus.Canceled + ? { + error: formatFastAgentTerminalError(run), + ...(run.errorCode ? { errorCode: run.errorCode } : {}), + } + : {}), taskUrl: getTaskUrl({ taskId: run.taskId, utm: { source: 'slack', campaign: 'fast-delegation-settle' },