From 1b80d28adc0f12b0a32ecd56272cdce51db34fc6 Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:58:20 +0000 Subject: [PATCH 1/3] fix: retry transient Fast sandbox starts --- .../__tests__/fast-agent-service.test.ts | 42 +++++ .../src/server/lib/fast-agent-parent-event.ts | 1 + ...notify-fast-agent-parent-on-settle.test.ts | 172 +++++++++++++++++- .../src/server/lib/task-runs/finish-run.ts | 6 +- .../notify-fast-agent-parent-on-settle.ts | 154 +++++++++++++++- 5 files changed, 369 insertions(+), 6 deletions(-) diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts index a75751577..89bbfe7da 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts @@ -325,6 +325,48 @@ describe('answerFastAgentQuestion', () => { ); }); + it('lets the orchestration loop report a delegated task terminal error', async () => { + mocks.generateObject.mockResolvedValueOnce({ + object: decision({ + message: + 'The task stopped because the sandbox provider rejected its credentials. Check the provider configuration before retrying.', + purpose: 'closeout', + }), + }); + const callbacks = chatCallbacks(); + const event = { + type: 'task_settled', + taskId: 'task-1', + runId: 42, + status: 'failed', + error: 'The sandbox provider rejected its credentials.', + taskUrl: 'https://roomote.example/task/task-1', + pullRequests: [], + }; + + const result = await answerFastAgentQuestion({ + ...baseParams, + question: `${JSON.stringify(event)}`, + platformEvent: true, + ...callbacks, + }); + + expect(mocks.generateObject).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining( + 'The sandbox provider rejected its credentials.', + ), + }), + ); + expect(result).toContain('Check the provider configuration'); + expect(callbacks.postSlackReply).toHaveBeenCalledWith( + expect.objectContaining({ + purpose: 'closeout', + message: expect.stringContaining('Check the provider configuration'), + }), + ); + }); + it('can close out a lightweight turn with an emoji reaction', async () => { mocks.generateObject.mockResolvedValue({ object: decision({ diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.ts index d63a1704b..7d426bcb2 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event.ts @@ -90,6 +90,7 @@ type FastAgentParentEvent = runId: number; title?: string; status: string; + error?: string; taskUrl: string; pullRequests: FastAgentPullRequestContext[]; } diff --git a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts index 52bc027d2..4a53ad672 100644 --- a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts +++ b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts @@ -1,5 +1,5 @@ import type { TaskRun } from '@roomote/db/server'; -import { RunStatus } from '@roomote/types'; +import { RunStatus, TaskRunErrorCode } from '@roomote/types'; const mocks = vi.hoisted(() => { class FastAgentParentEventDeliveryError extends Error { @@ -22,12 +22,18 @@ const mocks = vi.hoisted(() => { recordLifecycle: vi.fn(), deliverParentEvent: vi.fn(), listPullRequests: vi.fn(), + findTaskRun: vi.fn(), + canRetryFailedStart: vi.fn(), + enqueueTaskRelaunch: vi.fn(), FastAgentParentEventDeliveryError, }; }); vi.mock('@roomote/db/server', () => ({ db: { + query: { + taskRuns: { findFirst: mocks.findTaskRun }, + }, update: vi.fn(() => ({ set: vi.fn((values: unknown) => { mocks.updateSet(values); @@ -48,6 +54,8 @@ vi.mock('@roomote/db/server', () => ({ })); vi.mock('@roomote/cloud-agents/server', () => ({ + canRetryFailedStart: mocks.canRetryFailedStart, + enqueueTaskRelaunch: mocks.enqueueTaskRelaunch, getTaskUrl: vi.fn(() => 'https://roomote.example/task/child-task'), })); @@ -66,13 +74,19 @@ const fastParent = { slackThreadTs: '100.001', }; -function makeRun(payload: Record): TaskRun { +function makeRun( + payload: Record, + overrides: Partial = {}, +): TaskRun { return { id: 200, taskId: 'child-task', payload, result: null, error: null, + sourceRunId: null, + actingUserId: 'user-1', + ...overrides, } as TaskRun; } @@ -83,6 +97,9 @@ describe('notifyFastAgentParentOnSettle', () => { mocks.deliverParentEvent.mockResolvedValue(undefined); mocks.listPullRequests.mockResolvedValue([]); mocks.recordLifecycle.mockResolvedValue(undefined); + mocks.findTaskRun.mockResolvedValue(undefined); + mocks.canRetryFailedStart.mockResolvedValue(false); + mocks.enqueueTaskRelaunch.mockResolvedValue({ id: 201 }); }); it('passes child lifecycle state to the Fast orchestrator', async () => { @@ -152,6 +169,157 @@ describe('notifyFastAgentParentOnSettle', () => { ); }); + it('retries a transient failed startup before notifying the Fast parent', async () => { + vi.useFakeTimers(); + mocks.canRetryFailedStart.mockResolvedValueOnce(true); + + try { + const pending = notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { + error: 'Sandbox startup timed out while contacting the provider.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }, + ), + RunStatus.Failed, + ); + + await vi.advanceTimersByTimeAsync(1_000); + await pending; + + expect(mocks.enqueueTaskRelaunch).toHaveBeenCalledWith({ + sourceRunId: 200, + actingUserId: 'user-1', + }); + expect(mocks.canRetryFailedStart).toHaveBeenCalledWith( + expect.objectContaining({ status: RunStatus.Failed }), + ); + expect(mocks.deliverParentEvent).not.toHaveBeenCalled(); + expect(mocks.recordLifecycle).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + details: expect.objectContaining({ + reason: 'fast_agent_transient_startup_retry', + retryNumber: 1, + delayMs: 1_000, + }), + }), + ); + } finally { + vi.useRealTimers(); + } + }); + + it('stops retrying after the bounded startup retry budget', async () => { + mocks.canRetryFailedStart.mockResolvedValueOnce(true); + mocks.findTaskRun + .mockResolvedValueOnce({ + sourceRunId: 100, + payload: { fastAgentParent: fastParent }, + }) + .mockResolvedValueOnce({ + sourceRunId: null, + payload: { fastAgentParent: fastParent }, + }); + + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { + sourceRunId: 150, + error: 'HTTP 503 while starting the sandbox.', + }, + ), + RunStatus.Failed, + ); + + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + status: RunStatus.Failed, + error: 'HTTP 503 while starting the sandbox.', + }), + }), + ); + }); + + it('does not retry permanent startup errors and redacts the terminal detail', async () => { + mocks.canRetryFailedStart.mockResolvedValueOnce(true); + + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { + error: + 'Invalid credential xoxb-1234567890-abcdefghijklmnop while loading https://provider.example/setup', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }, + ), + RunStatus.Failed, + ); + + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + status: RunStatus.Failed, + error: 'Invalid credential [redacted] while loading [redacted URL]', + }), + }), + ); + }); + + it('reports the terminal failure when automatic relaunch cannot be queued', async () => { + vi.useFakeTimers(); + mocks.canRetryFailedStart.mockResolvedValueOnce(true); + mocks.enqueueTaskRelaunch.mockRejectedValueOnce(new Error('queue offline')); + + try { + const pending = notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { error: 'Sandbox startup timed out.' }, + ), + RunStatus.Failed, + ); + + await vi.advanceTimersByTimeAsync(1_000); + await pending; + + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + status: RunStatus.Failed, + error: 'Sandbox startup timed out.', + }), + }), + ); + } finally { + vi.useRealTimers(); + } + }); + + it('passes terminal cancellation errors to the Fast parent', async () => { + await notifyFastAgentParentOnSettle( + makeRun( + { fastAgentParent: fastParent }, + { error: 'The task was stopped because its sandbox was deleted.' }, + ), + RunStatus.Canceled, + ); + + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + status: RunStatus.Canceled, + error: 'The task was stopped because its sandbox was deleted.', + }), + }), + ); + }); + it('does nothing for independently launched tasks', async () => { await notifyFastAgentParentOnSettle(makeRun({}), RunStatus.Completed); expect(mocks.deliverParentEvent).not.toHaveBeenCalled(); diff --git a/packages/sdk/src/server/lib/task-runs/finish-run.ts b/packages/sdk/src/server/lib/task-runs/finish-run.ts index b99565ca0..9fd117389 100644 --- a/packages/sdk/src/server/lib/task-runs/finish-run.ts +++ b/packages/sdk/src/server/lib/task-runs/finish-run.ts @@ -407,7 +407,11 @@ export const finishRun = async ({ // orchestrator turn, and settle callers (tRPC finish, controller, queue // jobs) must not block on it. The delivery claim keeps it idempotent. void notifyFastAgentParentOnSettle( - { ...run, error: sanitizedError ?? run.error }, + { + ...run, + error: sanitizedError ?? run.error, + errorCode: errorCode ?? run.errorCode, + }, status, run.task.title, ); diff --git a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts index fbf8dfcf5..042660e57 100644 --- a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts +++ b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts @@ -1,4 +1,18 @@ -import { RunStatus, getFastAgentParentFromPayload } from '@roomote/types'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { redactSecrets } from '@roomote/communication/redact-secrets'; +import { + canRetryFailedStart, + enqueueTaskRelaunch, + getTaskUrl, +} from '@roomote/cloud-agents/server'; +import { + RunStatus, + TaskRunErrorCode, + getFastAgentParentFromPayload, + type FastAgentParent, + type TaskRunErrorCode as TaskRunErrorCodeValue, +} from '@roomote/types'; import { type TaskRun, and, @@ -8,8 +22,6 @@ import { sql, taskRuns, } from '@roomote/db/server'; -import { getTaskUrl } from '@roomote/cloud-agents/server'; - import { FastAgentParentEventDeliveryError, deliverFastAgentParentEvent, @@ -21,6 +33,20 @@ import { } from './fast-agent-delivery-claim'; const NOTIFIED_RESULT_KEY = 'fastAgentParentSettleNotifiedAt'; +const FAST_AGENT_STARTUP_MAX_RETRIES = 2; +const FAST_AGENT_STARTUP_RETRY_BASE_DELAY_MS = 1_000; +const FAST_AGENT_ERROR_MAX_CHARS = 300; + +const TRANSIENT_STARTUP_ERROR_CODES = new Set([ + TaskRunErrorCode.DockerDaemonUnreachable, + TaskRunErrorCode.DockerWorkerStartTimeout, + TaskRunErrorCode.DockerWorkerExitedEarly, + TaskRunErrorCode.DockerWorkerFetchFailed, +]); +const PERMANENT_STARTUP_ERROR_PATTERN = + /\b(?:unauthorized|forbidden|invalid (?:api )?key|invalid credential|authentication failed|permission denied|not configured|configuration|unsupported|not found|missing|required|read[ -]?only|spend limit|quota|billing|address pool exhausted|port is already (?:allocated|in use))\b/i; +const TRANSIENT_STARTUP_ERROR_PATTERN = + /\b(?:timed? out|timeout|temporar(?:y|ily)|unavailable|rate limit|too many requests|connection (?:closed|refused|reset)|network error|socket hang up|fetch failed|econnreset|econnrefused|enotfound|http (?:408|429|5\d\d)|status (?:408|429|5\d\d)|machine unavailable)\b/i; type SettledStatus = | RunStatus.Completed @@ -28,6 +54,112 @@ type SettledStatus = | RunStatus.Canceled | RunStatus.Idle; +function isTransientFastAgentStartupFailure(run: TaskRun): boolean { + const error = run.error?.trim(); + if (error && PERMANENT_STARTUP_ERROR_PATTERN.test(error)) { + return false; + } + + if (run.errorCode) { + return TRANSIENT_STARTUP_ERROR_CODES.has(run.errorCode); + } + + if (!error) { + return false; + } + + return TRANSIENT_STARTUP_ERROR_PATTERN.test(error); +} + +async function countFastAgentStartupRetries( + run: TaskRun, + parent: FastAgentParent, +): Promise { + let retries = 0; + let sourceRunId = run.sourceRunId; + + while (sourceRunId && retries < FAST_AGENT_STARTUP_MAX_RETRIES) { + const sourceRun = await db.query.taskRuns.findFirst({ + where: eq(taskRuns.id, sourceRunId), + columns: { payload: true, sourceRunId: true }, + }); + const sourceParent = sourceRun + ? getFastAgentParentFromPayload(sourceRun.payload) + : null; + + if (!sourceRun || sourceParent?.sessionId !== parent.sessionId) { + break; + } + + retries += 1; + sourceRunId = sourceRun.sourceRunId; + } + + return retries; +} + +async function retryTransientFastAgentStartup( + run: TaskRun, + parent: FastAgentParent, +): Promise { + if ( + !isTransientFastAgentStartupFailure(run) || + !(await canRetryFailedStart({ ...run, status: RunStatus.Failed })) + ) { + return false; + } + + const retries = await countFastAgentStartupRetries(run, parent); + if (retries >= FAST_AGENT_STARTUP_MAX_RETRIES) { + return false; + } + + const retryNumber = retries + 1; + const delayMs = + FAST_AGENT_STARTUP_RETRY_BASE_DELAY_MS * 2 ** (retryNumber - 1); + + await delay(delayMs); + await enqueueTaskRelaunch({ + sourceRunId: run.id, + actingUserId: run.actingUserId, + }); + await recordTaskRunLifecycleEvent(db, { + runId: run.id, + taskId: run.taskId, + eventType: 'decision', + message: `Automatically retried transient Fast child sandbox startup (${retryNumber}/${FAST_AGENT_STARTUP_MAX_RETRIES}).`, + details: { + reason: 'fast_agent_transient_startup_retry', + fastAgentSessionId: parent.sessionId, + retryNumber, + maxRetries: FAST_AGENT_STARTUP_MAX_RETRIES, + delayMs, + }, + }).catch((error) => { + console.warn( + `[notifyFastAgentParentOnSettle] Failed to record automatic startup retry for run ${run.id}: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + + return true; +} + +function formatFastAgentTerminalError(run: TaskRun): string { + const firstLine = run.error?.split(/\r?\n/u, 1)[0]?.trim(); + if (!firstLine) { + return 'The task stopped without a detailed error. Open the task for diagnostics.'; + } + + const safeError = redactSecrets(firstLine).replace( + /https?:\/\/\S+/giu, + '[redacted URL]', + ); + + return safeError.length > FAST_AGENT_ERROR_MAX_CHARS + ? `${safeError.slice(0, FAST_AGENT_ERROR_MAX_CHARS - 1)}…` + : safeError; +} + /** Pass a Fast child's terminal/idle state to its conversational orchestrator. */ export async function notifyFastAgentParentOnSettle( run: TaskRun, @@ -67,6 +199,19 @@ export async function notifyFastAgentParentOnSettle( let delivered = false; try { + if (status === RunStatus.Failed) { + try { + if (await retryTransientFastAgentStartup(run, parent)) { + await markSettled(); + return; + } + } catch (error) { + console.warn( + `[notifyFastAgentParentOnSettle] Automatic startup retry failed for run ${run.id}; reporting the terminal failure: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + const pullRequests = await listFastAgentPullRequestContexts(run.taskId); await deliverFastAgentParentEvent({ parent, @@ -76,6 +221,9 @@ export async function notifyFastAgentParentOnSettle( runId: run.id, ...(taskTitle?.trim() ? { title: taskTitle.trim() } : {}), status, + ...(status === RunStatus.Failed || status === RunStatus.Canceled + ? { error: formatFastAgentTerminalError(run) } + : {}), taskUrl: getTaskUrl({ taskId: run.taskId, utm: { source: 'slack', campaign: 'fast-delegation-settle' }, From 3ae33caa434da1c4a698ecf77808ff3fd556aae9 Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:25:10 +0000 Subject: [PATCH 2/3] fix: let Fast parent choose startup retries --- .../__tests__/fast-agent-prompt.test.ts | 4 + .../__tests__/fast-agent-service.test.ts | 40 ++++++++ .../server/fast-agent/fast-agent-prompt.ts | 3 +- .../server/fast-agent/fast-agent-service.ts | 37 +++++++- .../src/server/lib/fast-agent-parent-event.ts | 8 ++ ...notify-fast-agent-parent-on-settle.test.ts | 83 +++++++--------- .../notify-fast-agent-parent-on-settle.ts | 94 ++++++------------- 7 files changed, 153 insertions(+), 116 deletions(-) diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts index 54af88364..107cead07 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts @@ -126,6 +126,10 @@ describe('buildFastAgentSystemPrompt', () => { ); expect(prompt).toContain('Never use "ack" or "progress"'); expect(prompt).toContain('Use "ignore_event"'); + expect(prompt).toContain( + 'includes the full secret-redacted error and its machine-readable errorCode', + ); + expect(prompt).toContain('Use "retry_task_start" only for a failure'); expect(prompt).toContain( 'Pull-request-opened events contain authoritative, user-presentable pull request metadata', ); diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts index 89bbfe7da..83e281c19 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts @@ -367,6 +367,46 @@ describe('answerFastAgentQuestion', () => { ); }); + it('lets the parent retry a failed delegated task start before closing out', async () => { + mocks.generateObject + .mockResolvedValueOnce({ + object: decision({ + action: 'retry_task_start', + message: null, + purpose: null, + }), + }) + .mockResolvedValueOnce({ + object: decision({ + message: 'The sandbox startup looked transient, so I retried it.', + purpose: 'closeout', + }), + }); + const callbacks = chatCallbacks(); + const retryTaskStart = vi.fn().mockResolvedValue({ + success: true, + runId: 43, + }); + + const result = await answerFastAgentQuestion({ + ...baseParams, + question: + '{"type":"task_settled","status":"failed","error":"HTTP 503","errorCode":null}', + platformEvent: true, + retryTaskStart, + ...callbacks, + }); + + expect(retryTaskStart).toHaveBeenCalledOnce(); + expect(mocks.generateObject.mock.calls[1]?.[0]?.prompt).toContain( + '"success":true,"runId":43', + ); + expect(result).toBe( + 'The sandbox startup looked transient, so I retried it.', + ); + expect(callbacks.postSlackReply).toHaveBeenCalledOnce(); + }); + it('can close out a lightweight turn with an emoji reaction', async () => { mocks.generateObject.mockResolvedValue({ object: decision({ diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts index 909b16e0c..80be34923 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts @@ -116,7 +116,8 @@ ${ - The current input is a trusted platform-generated event about a delegated task, not a human-authored request. - Decide whether the event is useful to the user now. Use "ignore_event" when it is routine, redundant, or not worth interrupting them for. - When it is useful, emit exactly one "send_chat_reply" with purpose "closeout" and describe the outcome naturally in the context of the delegated work. Never use "ack" or "progress" for a platform event, and never copy a canned event sentence. -- Do not use integrations or task-control actions for this event. +- A failed task-settled event includes the full secret-redacted error and its machine-readable errorCode when available. Decide from that evidence whether another startup attempt is worthwhile. Use "retry_task_start" only for a failure that appears transient; do not use it for clear configuration, authentication, permission, billing, quota, missing-resource, or other permanent failures. +- After "retry_task_start", report its result with one closeout. Do not use integrations or any other task-control action for this event. - Artifact events include stable artifact IDs and view URLs. When an image would help the user, include its ID in imageArtifactIds so it renders inline with the same reply. For non-image artifacts, link the supplied view URL when useful. - Pull-request-opened events contain authoritative, user-presentable pull request metadata and should be presented unless that exact pull request URL was already reported in this conversation. Briefly name and link the pull request, including its repository, number, title, and current status when available. - Task-settled events include the task's current pullRequests list. Use it in the closeout so a pull request produced by the task is named and linked even when its earlier open event was missed; do not describe the pull request as newly opened if the thread already received that update. diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts index 4ff84dc40..53c93b5db 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts @@ -75,6 +75,7 @@ const fastAgentDecisionSchema = z 'cancel_task', 'call_integration', 'ignore_event', + 'retry_task_start', ]), message: z.string().nullable(), purpose: z @@ -166,6 +167,10 @@ export type LaunchFastAgentSlackTask = (params: { | { success: false; error: string } >; +export type RetryFastAgentTaskStart = () => Promise< + { success: true; runId: number } | { success: false; error: string } +>; + function normalizeThreadText(text: string): string { return text.replace(/\s+/g, ' ').trim(); } @@ -504,6 +509,7 @@ export async function answerFastAgentQuestion({ senderSlackUserId, activeTaskId = null, launchTask, + retryTaskStart, postSlackReply, postSlackReaction, surface = 'slack', @@ -522,6 +528,7 @@ export async function answerFastAgentQuestion({ senderSlackUserId?: string; activeTaskId?: string | null; launchTask?: LaunchFastAgentSlackTask; + retryTaskStart?: RetryFastAgentTaskStart; postSlackReply?: PostFastAgentSlackReply; postSlackReaction?: PostFastAgentSlackReaction; surface?: FastAgentSurface; @@ -591,6 +598,7 @@ export async function answerFastAgentQuestion({ 'launch_task' | 'send_task_message' | 'cancel_task' >(); let currentActiveTaskId = resolvedActiveTaskId; + let retriedTaskStart = false; const flushPendingLifecycleReply = async () => { if (!pendingLifecycleReply) { return; @@ -722,8 +730,35 @@ export async function answerFastAgentQuestion({ return message; } + if (decision.action === 'retry_task_start') { + if (!platformEvent || !retryTaskStart) { + prompt += `\n\n[PLATFORM EVENT ACTION REJECTED]\nretry_task_start is only available for an eligible failed delegated-task event.\n[END PLATFORM EVENT ACTION REJECTED]`; + continue; + } + if (retriedTaskStart) { + prompt += `\n\n[FAST ORCHESTRATION TOOL RESULT]\nTool: retry_task_start\nResult: ${JSON.stringify({ success: false, error: 'The task start retry has already been attempted for this event.' })}\n[END FAST ORCHESTRATION TOOL RESULT]\n\nReport the result with one send_chat_reply closeout.`; + continue; + } + + retriedTaskStart = true; + let retryResult: Awaited>; + try { + retryResult = await retryTaskStart(); + } catch (error) { + console.error( + `[Fast Agent] Failed to retry delegated task start: ${formatErrorForLog(error)}`, + ); + retryResult = { + success: false, + error: 'The failed-start retry could not be queued.', + }; + } + prompt += `\n\n[FAST ORCHESTRATION TOOL RESULT]\nTool: retry_task_start\nResult: ${JSON.stringify(retryResult)}\n[END FAST ORCHESTRATION TOOL RESULT]\n\nReport the retry outcome with one send_chat_reply closeout.`; + continue; + } + if (platformEvent) { - prompt += `\n\n[PLATFORM EVENT ACTION REJECTED]\nA delegated-task platform event may only use send_chat_reply or ignore_event. Do not launch, message, or cancel tasks, react, or call integrations for this event.\n[END PLATFORM EVENT ACTION REJECTED]`; + prompt += `\n\n[PLATFORM EVENT ACTION REJECTED]\nA delegated-task platform event may only use send_chat_reply, ignore_event, or the offered retry_task_start action. Do not launch, message, or cancel tasks, react, or call integrations for this event.\n[END PLATFORM EVENT ACTION REJECTED]`; continue; } diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.ts index 7d426bcb2..4cb2f7162 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event.ts @@ -24,6 +24,7 @@ import { type FastAgentParent, type PullRequestStatus, type RunStatus, + type TaskRunErrorCode, type SlackBlock, type SourceControlProvider, } from '@roomote/types'; @@ -91,6 +92,7 @@ type FastAgentParentEvent = title?: string; status: string; error?: string; + errorCode?: TaskRunErrorCode; taskUrl: string; pullRequests: FastAgentPullRequestContext[]; } @@ -201,6 +203,9 @@ function buildEventClientMessageSeed(event: FastAgentParentEvent): string { export async function deliverFastAgentParentEvent(params: { parent: FastAgentParent; event: FastAgentParentEvent; + retryTaskStart?: () => Promise< + { success: true; runId: number } | { success: false; error: string } + >; /** Cap the turn-lock wait so callers holding an HTTP request can fail fast * and lean on their own retry instead of blocking. */ lockWaitMs?: number; @@ -269,6 +274,9 @@ export async function deliverFastAgentParentEvent(params: { activeTaskId: params.event.type === 'task_settled' ? null : params.event.taskId, platformEvent: true, + ...(params.retryTaskStart + ? { retryTaskStart: params.retryTaskStart } + : {}), postSlackReply: async ({ message, imageArtifactIds = [] }) => { const imageBlocks = await buildSelectedImageBlocks({ artifactIds: imageArtifactIds, diff --git a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts index 4a53ad672..20d293675 100644 --- a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts +++ b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts @@ -169,9 +169,15 @@ describe('notifyFastAgentParentOnSettle', () => { ); }); - it('retries a transient failed startup before notifying the Fast parent', async () => { + it('lets the Fast parent retry an eligible failed startup', async () => { vi.useFakeTimers(); mocks.canRetryFailedStart.mockResolvedValueOnce(true); + let retryResult: unknown; + mocks.deliverParentEvent.mockImplementationOnce( + async (input: { retryTaskStart?: () => Promise }) => { + retryResult = await input.retryTaskStart?.(); + }, + ); try { const pending = notifyFastAgentParentOnSettle( @@ -195,12 +201,21 @@ describe('notifyFastAgentParentOnSettle', () => { expect(mocks.canRetryFailedStart).toHaveBeenCalledWith( expect.objectContaining({ status: RunStatus.Failed }), ); - expect(mocks.deliverParentEvent).not.toHaveBeenCalled(); + expect(retryResult).toEqual({ success: true, runId: 201 }); + expect(mocks.deliverParentEvent).toHaveBeenCalledWith( + expect.objectContaining({ + retryTaskStart: expect.any(Function), + event: expect.objectContaining({ + error: 'Sandbox startup timed out while contacting the provider.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, + }), + }), + ); expect(mocks.recordLifecycle).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ details: expect.objectContaining({ - reason: 'fast_agent_transient_startup_retry', + reason: 'fast_agent_parent_startup_retry', retryNumber: 1, delayMs: 1_000, }), @@ -211,7 +226,7 @@ describe('notifyFastAgentParentOnSettle', () => { } }); - it('stops retrying after the bounded startup retry budget', async () => { + it('reports the bounded startup retry budget to the Fast parent', async () => { mocks.canRetryFailedStart.mockResolvedValueOnce(true); mocks.findTaskRun .mockResolvedValueOnce({ @@ -222,6 +237,12 @@ describe('notifyFastAgentParentOnSettle', () => { sourceRunId: null, payload: { fastAgentParent: fastParent }, }); + let retryResult: unknown; + mocks.deliverParentEvent.mockImplementationOnce( + async (input: { retryTaskStart?: () => Promise }) => { + retryResult = await input.retryTaskStart?.(); + }, + ); await notifyFastAgentParentOnSettle( makeRun( @@ -235,25 +256,19 @@ describe('notifyFastAgentParentOnSettle', () => { ); expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); - expect(mocks.deliverParentEvent).toHaveBeenCalledWith( - expect.objectContaining({ - event: expect.objectContaining({ - status: RunStatus.Failed, - error: 'HTTP 503 while starting the sandbox.', - }), - }), - ); + expect(retryResult).toEqual({ + success: false, + error: 'The automatic failed-start retry limit has been reached.', + }); }); - it('does not retry permanent startup errors and redacts the terminal detail', async () => { - mocks.canRetryFailedStart.mockResolvedValueOnce(true); - + it('gives the Fast parent the full redacted error and error code', async () => { await notifyFastAgentParentOnSettle( makeRun( { fastAgentParent: fastParent }, { error: - 'Invalid credential xoxb-1234567890-abcdefghijklmnop while loading https://provider.example/setup', + 'Invalid credential xoxb-1234567890-abcdefghijklmnop while loading https://provider.example/setup\nProvider configuration must be updated.', errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, }, ), @@ -265,40 +280,14 @@ describe('notifyFastAgentParentOnSettle', () => { expect.objectContaining({ event: expect.objectContaining({ status: RunStatus.Failed, - error: 'Invalid credential [redacted] while loading [redacted URL]', + error: + 'Invalid credential [redacted] while loading https://provider.example/setup\nProvider configuration must be updated.', + errorCode: TaskRunErrorCode.DockerWorkerStartTimeout, }), + retryTaskStart: expect.any(Function), }), ); - }); - - it('reports the terminal failure when automatic relaunch cannot be queued', async () => { - vi.useFakeTimers(); - mocks.canRetryFailedStart.mockResolvedValueOnce(true); - mocks.enqueueTaskRelaunch.mockRejectedValueOnce(new Error('queue offline')); - - try { - const pending = notifyFastAgentParentOnSettle( - makeRun( - { fastAgentParent: fastParent }, - { error: 'Sandbox startup timed out.' }, - ), - RunStatus.Failed, - ); - - await vi.advanceTimersByTimeAsync(1_000); - await pending; - - expect(mocks.deliverParentEvent).toHaveBeenCalledWith( - expect.objectContaining({ - event: expect.objectContaining({ - status: RunStatus.Failed, - error: 'Sandbox startup timed out.', - }), - }), - ); - } finally { - vi.useRealTimers(); - } + expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); }); it('passes terminal cancellation errors to the Fast parent', async () => { diff --git a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts index 042660e57..ed9aa7eb4 100644 --- a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts +++ b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts @@ -8,10 +8,8 @@ import { } from '@roomote/cloud-agents/server'; import { RunStatus, - TaskRunErrorCode, getFastAgentParentFromPayload, type FastAgentParent, - type TaskRunErrorCode as TaskRunErrorCodeValue, } from '@roomote/types'; import { type TaskRun, @@ -35,18 +33,6 @@ import { const NOTIFIED_RESULT_KEY = 'fastAgentParentSettleNotifiedAt'; const FAST_AGENT_STARTUP_MAX_RETRIES = 2; const FAST_AGENT_STARTUP_RETRY_BASE_DELAY_MS = 1_000; -const FAST_AGENT_ERROR_MAX_CHARS = 300; - -const TRANSIENT_STARTUP_ERROR_CODES = new Set([ - TaskRunErrorCode.DockerDaemonUnreachable, - TaskRunErrorCode.DockerWorkerStartTimeout, - TaskRunErrorCode.DockerWorkerExitedEarly, - TaskRunErrorCode.DockerWorkerFetchFailed, -]); -const PERMANENT_STARTUP_ERROR_PATTERN = - /\b(?:unauthorized|forbidden|invalid (?:api )?key|invalid credential|authentication failed|permission denied|not configured|configuration|unsupported|not found|missing|required|read[ -]?only|spend limit|quota|billing|address pool exhausted|port is already (?:allocated|in use))\b/i; -const TRANSIENT_STARTUP_ERROR_PATTERN = - /\b(?:timed? out|timeout|temporar(?:y|ily)|unavailable|rate limit|too many requests|connection (?:closed|refused|reset)|network error|socket hang up|fetch failed|econnreset|econnrefused|enotfound|http (?:408|429|5\d\d)|status (?:408|429|5\d\d)|machine unavailable)\b/i; type SettledStatus = | RunStatus.Completed @@ -54,23 +40,6 @@ type SettledStatus = | RunStatus.Canceled | RunStatus.Idle; -function isTransientFastAgentStartupFailure(run: TaskRun): boolean { - const error = run.error?.trim(); - if (error && PERMANENT_STARTUP_ERROR_PATTERN.test(error)) { - return false; - } - - if (run.errorCode) { - return TRANSIENT_STARTUP_ERROR_CODES.has(run.errorCode); - } - - if (!error) { - return false; - } - - return TRANSIENT_STARTUP_ERROR_PATTERN.test(error); -} - async function countFastAgentStartupRetries( run: TaskRun, parent: FastAgentParent, @@ -98,20 +67,25 @@ async function countFastAgentStartupRetries( return retries; } -async function retryTransientFastAgentStartup( +async function retryFastAgentStartup( run: TaskRun, parent: FastAgentParent, -): Promise { - if ( - !isTransientFastAgentStartupFailure(run) || - !(await canRetryFailedStart({ ...run, status: RunStatus.Failed })) - ) { - return false; +): Promise< + { success: true; runId: number } | { success: false; error: string } +> { + if (!(await canRetryFailedStart({ ...run, status: RunStatus.Failed }))) { + return { + success: false, + error: 'This task is not eligible for a failed-start retry.', + }; } const retries = await countFastAgentStartupRetries(run, parent); if (retries >= FAST_AGENT_STARTUP_MAX_RETRIES) { - return false; + return { + success: false, + error: 'The automatic failed-start retry limit has been reached.', + }; } const retryNumber = retries + 1; @@ -119,7 +93,7 @@ async function retryTransientFastAgentStartup( FAST_AGENT_STARTUP_RETRY_BASE_DELAY_MS * 2 ** (retryNumber - 1); await delay(delayMs); - await enqueueTaskRelaunch({ + const relaunchedRun = await enqueueTaskRelaunch({ sourceRunId: run.id, actingUserId: run.actingUserId, }); @@ -127,9 +101,9 @@ async function retryTransientFastAgentStartup( runId: run.id, taskId: run.taskId, eventType: 'decision', - message: `Automatically retried transient Fast child sandbox startup (${retryNumber}/${FAST_AGENT_STARTUP_MAX_RETRIES}).`, + message: `Fast parent retried child sandbox startup (${retryNumber}/${FAST_AGENT_STARTUP_MAX_RETRIES}).`, details: { - reason: 'fast_agent_transient_startup_retry', + reason: 'fast_agent_parent_startup_retry', fastAgentSessionId: parent.sessionId, retryNumber, maxRetries: FAST_AGENT_STARTUP_MAX_RETRIES, @@ -141,23 +115,16 @@ async function retryTransientFastAgentStartup( ); }); - return true; + return { success: true, runId: relaunchedRun.id }; } function formatFastAgentTerminalError(run: TaskRun): string { - const firstLine = run.error?.split(/\r?\n/u, 1)[0]?.trim(); - if (!firstLine) { + const error = run.error?.trim(); + if (!error) { return 'The task stopped without a detailed error. Open the task for diagnostics.'; } - const safeError = redactSecrets(firstLine).replace( - /https?:\/\/\S+/giu, - '[redacted URL]', - ); - - return safeError.length > FAST_AGENT_ERROR_MAX_CHARS - ? `${safeError.slice(0, FAST_AGENT_ERROR_MAX_CHARS - 1)}…` - : safeError; + return redactSecrets(error); } /** Pass a Fast child's terminal/idle state to its conversational orchestrator. */ @@ -199,22 +166,12 @@ export async function notifyFastAgentParentOnSettle( let delivered = false; try { - if (status === RunStatus.Failed) { - try { - if (await retryTransientFastAgentStartup(run, parent)) { - await markSettled(); - return; - } - } catch (error) { - console.warn( - `[notifyFastAgentParentOnSettle] Automatic startup retry failed for run ${run.id}; reporting the terminal failure: ${error instanceof Error ? error.message : String(error)}`, - ); - } - } - const pullRequests = await listFastAgentPullRequestContexts(run.taskId); await deliverFastAgentParentEvent({ parent, + ...(status === RunStatus.Failed + ? { retryTaskStart: () => retryFastAgentStartup(run, parent) } + : {}), event: { type: 'task_settled', taskId: run.taskId, @@ -222,7 +179,10 @@ export async function notifyFastAgentParentOnSettle( ...(taskTitle?.trim() ? { title: taskTitle.trim() } : {}), status, ...(status === RunStatus.Failed || status === RunStatus.Canceled - ? { error: formatFastAgentTerminalError(run) } + ? { + error: formatFastAgentTerminalError(run), + ...(run.errorCode ? { errorCode: run.errorCode } : {}), + } : {}), taskUrl: getTaskUrl({ taskId: run.taskId, From 96d7cd45bfbc4881ac2caeff6ad2b7565202a368 Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:29:19 +0000 Subject: [PATCH 3/3] chore: align Fast retry wording --- .../__tests__/notify-fast-agent-parent-on-settle.test.ts | 2 +- .../lib/task-runs/notify-fast-agent-parent-on-settle.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts index 20d293675..f0034dadd 100644 --- a/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts +++ b/packages/sdk/src/server/lib/task-runs/__tests__/notify-fast-agent-parent-on-settle.test.ts @@ -258,7 +258,7 @@ describe('notifyFastAgentParentOnSettle', () => { expect(mocks.enqueueTaskRelaunch).not.toHaveBeenCalled(); expect(retryResult).toEqual({ success: false, - error: 'The automatic failed-start retry limit has been reached.', + error: 'The failed-start retry limit has been reached.', }); }); diff --git a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts index ed9aa7eb4..661fb1d02 100644 --- a/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts +++ b/packages/sdk/src/server/lib/task-runs/notify-fast-agent-parent-on-settle.ts @@ -84,7 +84,7 @@ async function retryFastAgentStartup( if (retries >= FAST_AGENT_STARTUP_MAX_RETRIES) { return { success: false, - error: 'The automatic failed-start retry limit has been reached.', + error: 'The failed-start retry limit has been reached.', }; } @@ -111,7 +111,7 @@ async function retryFastAgentStartup( }, }).catch((error) => { console.warn( - `[notifyFastAgentParentOnSettle] Failed to record automatic startup retry for run ${run.id}: ${error instanceof Error ? error.message : String(error)}`, + `[notifyFastAgentParentOnSettle] Failed to record parent-requested startup retry for run ${run.id}: ${error instanceof Error ? error.message : String(error)}`, ); });