From 582d8e334abce0f5f495689823e19f0291f476ba Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Tue, 21 Jul 2026 07:47:33 -0400 Subject: [PATCH 1/4] fix: recover stalled integration tasks durably --- .agents/skills/integration-webhooks/SKILL.md | 80 ++++-- .changeset/calm-slack-queues.md | 5 + packages/core/src/agent/durable-background.ts | 1 + packages/core/src/deploy/build.spec.ts | 42 +++ packages/core/src/deploy/build.ts | 144 +++++++++- .../core/src/deploy/workspace-deploy.spec.ts | 62 +++++ packages/core/src/deploy/workspace-deploy.ts | 121 ++++++++- .../integration-durable-dispatch-config.ts | 22 ++ .../integration-durable-dispatch.spec.ts | 113 ++++++++ .../integration-durable-dispatch.ts | 183 +++++++++++++ .../pending-tasks-retry-job.spec.ts | 196 ++++++++++++-- .../integrations/pending-tasks-retry-job.ts | 247 ++++++++++++------ .../integrations/pending-tasks-store.spec.ts | 38 ++- .../src/integrations/pending-tasks-store.ts | 90 ++++++- packages/core/src/integrations/plugin.spec.ts | 78 ++++++ packages/core/src/integrations/plugin.ts | 153 +++++++---- .../src/integrations/task-queue-stats.spec.ts | 65 +++++ .../core/src/integrations/task-queue-stats.ts | 83 ++++-- .../src/integrations/webhook-handler.spec.ts | 2 + .../core/src/integrations/webhook-handler.ts | 59 +---- packages/core/src/server/auth.spec.ts | 1 + packages/core/src/server/auth.ts | 6 + ...-07-21-recover-stalled-slack-agent-work.md | 6 + 23 files changed, 1542 insertions(+), 255 deletions(-) create mode 100644 .changeset/calm-slack-queues.md create mode 100644 packages/core/src/integrations/integration-durable-dispatch-config.ts create mode 100644 packages/core/src/integrations/integration-durable-dispatch.spec.ts create mode 100644 packages/core/src/integrations/integration-durable-dispatch.ts create mode 100644 packages/core/src/integrations/task-queue-stats.spec.ts create mode 100644 templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md diff --git a/.agents/skills/integration-webhooks/SKILL.md b/.agents/skills/integration-webhooks/SKILL.md index 8ee5e79014..46ef55d976 100644 --- a/.agents/skills/integration-webhooks/SKILL.md +++ b/.agents/skills/integration-webhooks/SKILL.md @@ -16,10 +16,10 @@ metadata: Integration webhooks (Slack, Telegram, WhatsApp, email, Google Docs, etc.) must **enqueue work to SQL and return 200 immediately**, then process the work in a -**separate fresh function execution** kicked off by a self-fired HTTP POST. A -recurring retry job sweeps anything that gets stuck. This pattern works on every -serverless host (Netlify, Vercel, Cloudflare Workers, Fly, Render, Node) without -relying on platform-specific background-execution features. +**separate fresh function execution** kicked off by a self-fired HTTP POST. SQL +is always the source of truth. Long-lived hosts use an in-process retry loop as +a best-effort safety net; serverless deployments must use an external durable +sweep when recovery cannot depend on a process staying alive. Do not run agent loops inside the webhook handler itself. Do not rely on fire-and-forget `Promise`s after `return`ing from a serverless handler — they get @@ -38,8 +38,9 @@ Past attempts that don't work cross-host: freeze the execution context the moment the response goes out. The promise is silently killed, the user gets no reply, and there's no error in the logs. -- **Netlify Background Functions** — Netlify-only, requires a `-background` - filename suffix, breaks on every other host. +- **A platform background function as the queue** — host-specific execution is + useful as a delivery mechanism, but it cannot replace the SQL row or atomic + claim shared by every host. - **Cloudflare `event.waitUntil()`** — CF Workers only, not portable. - **Vercel Fluid / `after()`** — Vercel-only, gated behind specific runtimes. - **A long-lived in-process queue** — fine on a single Node box, but on @@ -62,8 +63,9 @@ and a recurring job is the safety net. 2. INSERT INTO integration_pending_tasks (status='pending', payload=...) │ - 3. fetch(POST /integrations/_process-task) - — fire-and-forget, NO await on body + 3. dispatch POST /integrations/process-task + — portable self-fire by default + — acknowledged background handoff when enabled │ 4. return 200 to platform ◄───────────┘ @@ -79,7 +81,7 @@ and a recurring job is the safety net. ┌──────────────────────────────────────────────┐ - │ Recurring job (every 60s) — safety net │ + │ Recovery sweep (every 60s) — safety net │ │ Re-fires processor for tasks stuck in │ │ 'pending' or 'processing' beyond timeout. │ │ Caps retries at 3 then marks 'failed'. │ @@ -98,6 +100,7 @@ loop. | `packages/core/src/integrations/webhook-handler.ts` | Verifies signature, parses, enqueues task, fires processor | | `packages/core/src/integrations/pending-tasks-store.ts` | SQL queue: `insertPendingTask`, `claimPendingTask`, `markTaskCompleted`, `markTaskFailed` | | `packages/core/src/integrations/pending-tasks-retry-job.ts` | Recurring retry sweep (`startPendingTasksRetryJob`, `retryStuckPendingTasks`) | +| `packages/core/src/integrations/integration-durable-dispatch.ts` | Default-off acknowledged dispatch, scoped rollout, and outcome recording | | `packages/core/src/integrations/types.ts` | `PlatformAdapter`, `IncomingMessage`, `OutgoingMessage` | | `packages/core/src/integrations/adapters/{slack,telegram,whatsapp,email,google-docs}.ts` | One adapter per platform | @@ -108,7 +111,8 @@ All under `/_agent-native/integrations/`: | Method | Path | Purpose | | ------ | -------------------------- | ------------------------------------------------------------- | | POST | `/:platform/webhook` | Platform pings this. Verifies, enqueues, returns 200 quickly. | -| POST | `/_process-task` | Self-webhook target. Claims a task and runs the agent loop. | +| POST | `/process-task` | Processor target. Claims a task and runs the agent loop. | +| POST | `/retry-stuck-tasks` | Signed, bounded recovery sweep for durable schedulers. | | GET | `/status` | All integrations status (settings UI). | | GET | `/:platform/status` | One platform's status. | | POST | `/:platform/enable` | Enable an integration. | @@ -129,6 +133,9 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( org_id TEXT, status TEXT NOT NULL, -- pending | processing | completed | failed attempts INTEGER NOT NULL DEFAULT 0, + dispatch_attempts INTEGER NOT NULL DEFAULT 0, + last_dispatch_at INTEGER, + last_dispatch_outcome TEXT, error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,10 +150,36 @@ The store layer creates this lazily on first use via `ensureTable()` and uses `claimPendingTask` is the critical concurrency primitive: it atomically flips `pending` → `processing` and increments `attempts`, returning `null` if another -worker beat us to it. Both the initial fire-and-forget call and the retry job +worker beat us to it. Both the initial dispatch and every recovery sweep funnel through the same processor endpoint, and `claimPendingTask` is what prevents the same task from being processed twice. +### Netlify durable recovery + +The portable self-dispatch remains the default. To emit and use Netlify's +acknowledged background worker plus a one-minute scheduled recovery function, +set `AGENT_INTEGRATION_DURABLE_DISPATCH=true` at build and runtime. A production +rollout can be narrowed with the comma-separated +`AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are +`:*`, `:`, and, for adapters that expose +one, `:` (for example `slack:C123`). + +Workspace deploys emit this pair only for the Dispatch control-plane app, which +owns workspace messaging integrations. Standalone templates emit their own +pair when the same flag is enabled. + +The background handoff waits only for Netlify's enqueue acknowledgement, not +for the agent loop. A failed or non-2xx acknowledgement falls back to the +portable processor. The scheduled function calls the HMAC-authenticated, +bounded `/retry-stuck-tasks` route; its atomic compare-and-set update prevents +overlapping sweeps from dispatching the same stale row twice. +Rows acknowledged by the 15-minute background worker use a 16-minute stale +cutoff, while portable synchronous dispatches retain the shorter serverless +cutoff. Apply that lease predicate before `LIMIT` so healthy long runs cannot +hide recoverable work. The background processor writes its lease marker in the +same atomic update that claims the row; best-effort dispatch telemetry is not +trusted for replay safety. + ## Adding a New Platform Adapter 1. **Implement `PlatformAdapter`** in `packages/core/src/integrations/adapters/.ts`: @@ -245,9 +278,10 @@ so a normal long-running reply is safe. ## Cross-Platform Considerations -- **No platform-specific background APIs.** No `waitUntil`, no - `-background.ts` filenames, no Vercel `after()`. The pattern works - identically on every host because it only uses `fetch()` and SQL. +- **Portable correctness, optional host acceleration.** Every host uses SQL, + HMAC-authenticated processor routes, and atomic claims. A host-specific + background function or scheduler may provide a more reliable wake-up, but + disabling it must preserve the portable path. - **No assumed runtime.** The processor endpoint is a normal H3 handler under `/_agent-native/`. It runs wherever the rest of the framework runs. - **No persistent in-memory state.** The dedup map in the webhook handler is @@ -262,11 +296,12 @@ so a normal long-running reply is safe. ## Why Fire-and-Forget on Serverless Is Unreliable -Even though the webhook handler does `fetch(processorUrl, ...)` without -awaiting the response body, that initial dispatch is **not** guaranteed to -complete before the function freezes. In practice it usually does — the TCP -connect + write happens quickly — but the recurring retry job is the safety -net for the cases where: +The portable webhook path gives the outbound processor request a short head +start but does not wait for the response body, so that initial dispatch is +**not** guaranteed to complete before the function freezes. On a long-lived +host, the in-process retry loop is a best-effort safety net. On Netlify with +durable dispatch enabled, the initial handoff is acknowledged and the external +scheduled function supplies the recovery wake-up for cases where: - The serverless platform froze the handler before the outbound `fetch` flushed its bytes. @@ -278,8 +313,9 @@ Tasks stuck in `pending` for >90s or `processing` for >5min get re-fired up to 3 times. After 3 attempts they're marked `failed` permanently so we stop spamming the processor. -**Never assume the initial fire-and-forget succeeded.** Always rely on the -queue + retry job for at-least-once delivery. +**Never assume the initial dispatch succeeded.** Rely on the SQL queue, atomic +claim, and a recovery mechanism whose lifetime is independent of the original +request. An in-process timer alone is not that mechanism on serverless. ## Debugging Checklist @@ -292,7 +328,7 @@ queue + retry job for at-least-once delivery. 4. **Status?** `pending` means the processor never picked it up — check that `_process-task` is reachable from the box itself (the self-fetch must work over the public URL). `processing` for over 5 minutes means the processor - died mid-run — the retry job will pick it up. + died mid-run — a live recovery sweep will pick it up. 5. **Failed?** Check `error_message` and `attempts`. After 3 attempts the row is parked at `failed` and won't be retried. 6. **Reply not delivered?** The processor likely succeeded but diff --git a/.changeset/calm-slack-queues.md b/.changeset/calm-slack-queues.md new file mode 100644 index 0000000000..8bb4f38ec1 --- /dev/null +++ b/.changeset/calm-slack-queues.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Add an opt-in, scoped Netlify background handoff and scheduled recovery sweep for durable messaging-integration tasks. diff --git a/packages/core/src/agent/durable-background.ts b/packages/core/src/agent/durable-background.ts index 76fa89b8f9..6c98d517c4 100644 --- a/packages/core/src/agent/durable-background.ts +++ b/packages/core/src/agent/durable-background.ts @@ -85,6 +85,7 @@ export const AGENT_BACKGROUND_FUNCTION_URL_PATH = `/.netlify/functions/${AGENT_B */ export const AGENT_BACKGROUND_PROCESSOR_FIELD = "__agentNativeProcessor"; export const AGENT_BACKGROUND_PROCESSOR_A2A = "a2a"; +export const AGENT_BACKGROUND_PROCESSOR_INTEGRATION = "integration"; export const AGENT_BACKGROUND_PROCESSOR_ROUTE = "route"; export const AGENT_BACKGROUND_PROCESSOR_ROUTE_FIELD = "__agentNativeProcessorRoute"; diff --git a/packages/core/src/deploy/build.spec.ts b/packages/core/src/deploy/build.spec.ts index 90d5d1d6dc..065b3c94e5 100644 --- a/packages/core/src/deploy/build.spec.ts +++ b/packages/core/src/deploy/build.spec.ts @@ -21,6 +21,7 @@ import { cloudflareWorkerStubAliasArgs, copyDir, emitSingleTemplateNetlifyBackgroundFunction, + emitSingleTemplateNetlifyIntegrationRecoveryFunction, findInstalledFfmpegStaticPackage, findInstalledResvgPackages, generateCloudflarePagesStaticShellFromManifest, @@ -28,6 +29,7 @@ import { generateWorkerEntry, getNodeBuiltinNames, isDurableBackgroundDeployEnabled, + isIntegrationDurableDispatchDeployEnabled, NITRO_RUNTIME_IGNORE_PATTERNS, nitroNoExternalsForPreset, resolveNitroBundledYjsEntry, @@ -1506,6 +1508,46 @@ describe("durable-background Netlify function emit (single-template, flag-gated) ); } + it("keeps integration recovery default-off and recognizes explicit opt-in", () => { + delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + expect(isIntegrationDurableDispatchDeployEnabled()).toBe(false); + process.env.AGENT_INTEGRATION_DURABLE_DISPATCH = "true"; + expect(isIntegrationDurableDispatchDeployEnabled()).toBe(true); + delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + }); + + it("emits a bounded one-minute integration recovery function", async () => { + const cwd = setupNetlifyOutput(); + + emitSingleTemplateNetlifyIntegrationRecoveryFunction(cwd); + + const dest = path.join( + cwd, + ".netlify", + "functions-internal", + "server-integration-recovery", + ); + expect(fs.existsSync(path.join(dest, "main.mjs"))).toBe(true); + expect(fs.existsSync(path.join(dest, "server.mjs"))).toBe(false); + const entry = fs.readFileSync( + path.join(dest, "server-integration-recovery.mjs"), + "utf8", + ); + expect(entry).toContain('schedule: "* * * * *"'); + expect(entry).toContain( + 'const SWEEP_PATH = "/_agent-native/integrations/retry-stuck-tasks"', + ); + expect(entry).toContain('createHmac("sha256", secret)'); + expect(entry).toContain( + "if (!enabled()) return new Response(null, { status: 204 })", + ); + expect(entry).not.toMatch(/^\s*path:/m); + const generated = await import( + `${pathToFileURL(path.join(dest, "server-integration-recovery.mjs")).href}?t=${Date.now()}` + ); + expect(generated.config.schedule).toBe("* * * * *"); + }); + it("is OFF BY DEFAULT (flag unset) so the -background function is NOT emitted", () => { // Default-off (opt-in) matches the runtime gate (isFlagEnabled) — durable is // opt-in until the async worker path is proven live, so the 15-min diff --git a/packages/core/src/deploy/build.ts b/packages/core/src/deploy/build.ts index 7c01be1aa2..6c6ec4dd0f 100644 --- a/packages/core/src/deploy/build.ts +++ b/packages/core/src/deploy/build.ts @@ -24,10 +24,16 @@ import { AGENT_BACKGROUND_FUNCTION_NAME, AGENT_BACKGROUND_PROCESSOR_A2A, AGENT_BACKGROUND_PROCESSOR_FIELD, + AGENT_BACKGROUND_PROCESSOR_INTEGRATION, AGENT_BACKGROUND_PROCESSOR_ROUTE, AGENT_BACKGROUND_PROCESSOR_ROUTE_FIELD, AGENT_CHAT_PROCESS_RUN_PATH, } from "../agent/durable-background.js"; +import { + INTEGRATION_RETRY_SWEEP_PATH, + INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, + isIntegrationDurableDispatchConfigured, +} from "../integrations/integration-durable-dispatch-config.js"; import { normalizeAppBasePath } from "../server/app-base-path.js"; import { DEFAULT_SSR_CDN_CACHE_CONTROL, @@ -2345,6 +2351,13 @@ export function isDurableBackgroundDeployEnabled(): boolean { return v === "1" || v === "true" || v === "yes" || v === "on"; } +export const NETLIFY_INTEGRATION_RECOVERY_FUNCTION_NAME = + "server-integration-recovery"; + +export function isIntegrationDurableDispatchDeployEnabled(): boolean { + return isIntegrationDurableDispatchConfigured(); +} + /** * Single-template Netlify build: emit an async (background) function INSIDE the * scanned functions dir so the chat `_process-run` worker runs on Netlify's @@ -2435,10 +2448,16 @@ export function emitSingleTemplateNetlifyBackgroundFunction( const processRunPath = JSON.stringify(AGENT_CHAT_PROCESS_RUN_PATH); const a2aProcessTaskPath = JSON.stringify("/_agent-native/a2a/_process-task"); + const integrationProcessTaskPath = JSON.stringify( + "/_agent-native/integrations/process-task", + ); const backgroundProcessorField = JSON.stringify( AGENT_BACKGROUND_PROCESSOR_FIELD, ); const backgroundProcessorA2A = JSON.stringify(AGENT_BACKGROUND_PROCESSOR_A2A); + const backgroundProcessorIntegration = JSON.stringify( + AGENT_BACKGROUND_PROCESSOR_INTEGRATION, + ); const backgroundProcessorRoute = JSON.stringify( AGENT_BACKGROUND_PROCESSOR_ROUTE, ); @@ -2457,8 +2476,10 @@ globalThis.__AGENT_NATIVE_BACKGROUND_RUNTIME__ = true; // The framework route the Nitro router dispatches to (the _process-run plugin). const PROCESS_RUN_PATH = ${processRunPath}; const A2A_PROCESS_TASK_PATH = ${a2aProcessTaskPath}; +const INTEGRATION_PROCESS_TASK_PATH = ${integrationProcessTaskPath}; const BACKGROUND_PROCESSOR_FIELD = ${backgroundProcessorField}; const BACKGROUND_PROCESSOR_A2A = ${backgroundProcessorA2A}; +const BACKGROUND_PROCESSOR_INTEGRATION = ${backgroundProcessorIntegration}; const BACKGROUND_PROCESSOR_ROUTE = ${backgroundProcessorRoute}; const BACKGROUND_PROCESSOR_ROUTE_FIELD = ${backgroundProcessorRouteField}; @@ -2469,6 +2490,12 @@ function processorPathFromBody(body) { if (parsed?.[BACKGROUND_PROCESSOR_FIELD] === BACKGROUND_PROCESSOR_A2A) { return A2A_PROCESS_TASK_PATH; } + if ( + parsed?.[BACKGROUND_PROCESSOR_FIELD] === + BACKGROUND_PROCESSOR_INTEGRATION + ) { + return INTEGRATION_PROCESS_TASK_PATH; + } const route = parsed?.[BACKGROUND_PROCESSOR_ROUTE_FIELD]; if ( parsed?.[BACKGROUND_PROCESSOR_FIELD] === BACKGROUND_PROCESSOR_ROUTE && @@ -2557,6 +2584,75 @@ export const config = { ); } +export function emitSingleTemplateNetlifyIntegrationRecoveryFunction( + projectCwd: string, +): void { + const internalDir = path.join(projectCwd, ".netlify", "functions-internal"); + const serverDir = path.join(internalDir, "server"); + if (!fs.existsSync(path.join(serverDir, "main.mjs"))) { + console.warn( + "[build] Integration recovery emit skipped: expected Nitro Netlify function " + + "at .netlify/functions-internal/server/main.mjs was not found.", + ); + return; + } + const functionName = NETLIFY_INTEGRATION_RECOVERY_FUNCTION_NAME; + const dest = path.join(internalDir, functionName); + fs.rmSync(dest, { recursive: true, force: true }); + copyDir(serverDir, dest); + fs.rmSync(path.join(dest, "server.mjs"), { force: true }); + + const entry = `import { createHmac } from "node:crypto"; + +const SWEEP_PATH = ${JSON.stringify(INTEGRATION_RETRY_SWEEP_PATH)}; +const SWEEP_SUBJECT = ${JSON.stringify(INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT)}; + +function enabled() { + const raw = process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + if (!raw) return false; + return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase()); +} + +function token() { + const secret = process.env.A2A_SECRET; + if (!secret) throw new Error("A2A_SECRET is required for integration recovery"); + const timestamp = Date.now(); + const signature = createHmac("sha256", secret) + .update(\`${INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT}:\${timestamp}\`) + .digest("hex"); + return \`\${timestamp}.\${signature}\`; +} + +let cachedHandler; + +export default async function handler(request, context) { + if (!enabled()) return new Response(null, { status: 204 }); + cachedHandler ??= (await import("./main.mjs")).default; + const url = new URL(request.url); + url.pathname = SWEEP_PATH; + const rewritten = new Request(url.toString(), { + method: "POST", + headers: { + Authorization: \`Bearer \${token()}\`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ taskId: SWEEP_SUBJECT }), + }); + return cachedHandler(rewritten, context); +} + +export const config = { + name: "integration pending-task recovery", + generator: "agent-native build", + schedule: "* * * * *", + nodeBundler: "none", + includedFiles: ["**"], + preferStatic: false, +}; +`; + fs.writeFileSync(path.join(dest, `${functionName}.mjs`), entry); +} + /** * Nitro's Netlify preset can emit a harmful fallback rewrite to * `/.netlify/functions/server`. With `config.path: "/*"`, that default URL is @@ -2849,7 +2945,10 @@ export function assertSingleTemplateNetlifyBuildOutput( ); } - if (isDurableBackgroundDeployEnabled()) { + if ( + isDurableBackgroundDeployEnabled() || + isIntegrationDurableDispatchDeployEnabled() + ) { const backgroundDir = path.join( internalDir, AGENT_BACKGROUND_FUNCTION_NAME, @@ -2886,6 +2985,32 @@ export function assertSingleTemplateNetlifyBuildOutput( } } + if (isIntegrationDurableDispatchDeployEnabled()) { + const recoveryEntryPath = path.join( + internalDir, + NETLIFY_INTEGRATION_RECOVERY_FUNCTION_NAME, + `${NETLIFY_INTEGRATION_RECOVERY_FUNCTION_NAME}.mjs`, + ); + if (!fs.existsSync(recoveryEntryPath)) { + failures.push( + `integration durable dispatch is enabled but ${path.relative( + projectCwd, + recoveryEntryPath, + )} was not emitted`, + ); + } else { + const recoveryEntry = fs.readFileSync(recoveryEntryPath, "utf-8"); + if (!/\bschedule\s*:\s*["']\* \* \* \* \*["']/.test(recoveryEntry)) { + failures.push( + `integration recovery entry ${path.relative( + projectCwd, + recoveryEntryPath, + )} is missing the one-minute schedule`, + ); + } + } + } + if (failures.length > 0) { throw new Error( "[deploy] Netlify deploy guard failed; refusing to publish an output " + @@ -3534,7 +3659,11 @@ export default bundle; // bundle, so the chat `_process-run` POST lands on Netlify's async (15-min) // function. When not opted in this is a no-op and the single-function // deploy is byte-for-byte unchanged. - if (preset === "netlify" && isDurableBackgroundDeployEnabled()) { + if ( + preset === "netlify" && + (isDurableBackgroundDeployEnabled() || + isIntegrationDurableDispatchDeployEnabled()) + ) { try { emitSingleTemplateNetlifyBackgroundFunction(cwd); } catch (err) { @@ -3545,6 +3674,17 @@ export default bundle; } } + if (preset === "netlify" && isIntegrationDurableDispatchDeployEnabled()) { + try { + emitSingleTemplateNetlifyIntegrationRecoveryFunction(cwd); + } catch (err) { + console.warn( + "[build] Failed to emit integration recovery Netlify function (non-fatal):", + err instanceof Error ? err.message : err, + ); + } + } + if (preset === "netlify") { writeSingleTemplateNetlifyRedirects(cwd); assertSingleTemplateNetlifyBuildOutput(cwd); diff --git a/packages/core/src/deploy/workspace-deploy.spec.ts b/packages/core/src/deploy/workspace-deploy.spec.ts index dcef3ae57e..6d92eda34b 100644 --- a/packages/core/src/deploy/workspace-deploy.spec.ts +++ b/packages/core/src/deploy/workspace-deploy.spec.ts @@ -19,6 +19,7 @@ let previousDatabaseUrl: string | undefined; let previousUnpooledDatabaseUrl: string | undefined; let previousNetlify: string | undefined; let previousNetlifyLocal: string | undefined; +let previousIntegrationDurableDispatch: string | undefined; let previousNitroPreset: string | undefined; let previousVercel: string | undefined; let previousViteWorkspaceAppsJson: string | undefined; @@ -59,6 +60,8 @@ beforeEach(() => { previousUnpooledDatabaseUrl = process.env.NETLIFY_DATABASE_URL_UNPOOLED; previousNetlify = process.env.NETLIFY; previousNetlifyLocal = process.env.NETLIFY_LOCAL; + previousIntegrationDurableDispatch = + process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; previousNitroPreset = process.env.NITRO_PRESET; previousVercel = process.env.VERCEL; previousViteWorkspaceAppsJson = @@ -90,6 +93,7 @@ beforeEach(() => { delete process.env.NETLIFY_DATABASE_URL_UNPOOLED; delete process.env.NETLIFY; delete process.env.NETLIFY_LOCAL; + delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; delete process.env.NITRO_PRESET; delete process.env.VERCEL; delete process.env.VITE_AGENT_NATIVE_WORKSPACE_APPS_JSON; @@ -117,6 +121,10 @@ afterEach(() => { restoreEnv("NETLIFY_DATABASE_URL_UNPOOLED", previousUnpooledDatabaseUrl); restoreEnv("NETLIFY", previousNetlify); restoreEnv("NETLIFY_LOCAL", previousNetlifyLocal); + restoreEnv( + "AGENT_INTEGRATION_DURABLE_DISPATCH", + previousIntegrationDurableDispatch, + ); restoreEnv("NITRO_PRESET", previousNitroPreset); restoreEnv("VERCEL", previousVercel); restoreEnv( @@ -1189,6 +1197,60 @@ describe("durable-background Netlify function emit (workspace, flag-gated)", () expect(fs.existsSync(backgroundFuncDir("starter"))).toBe(false); }); + it("emits scoped integration background and scheduled recovery functions when opted in", async () => { + process.env.AGENT_CHAT_DURABLE_BACKGROUND = "false"; + process.env.AGENT_INTEGRATION_DURABLE_DISPATCH = "true"; + makeWorkspaceApp(tmpDir, "dispatch"); + makeWorkspaceApp(tmpDir, "starter"); + + await runWorkspaceDeploy({ + workspaceRoot: tmpDir, + args: ["--preset=netlify", "--build-only"], + execFile: execFile as typeof execFileSync, + }); + + const backgroundEntry = fs.readFileSync( + path.join(backgroundFuncDir("dispatch"), "dispatch-agent-background.mjs"), + "utf8", + ); + expect(backgroundEntry).toContain( + 'const BACKGROUND_PROCESSOR_INTEGRATION = "integration"', + ); + expect(backgroundEntry).toContain( + 'const INTEGRATION_PROCESS_TASK_PATH = "/dispatch/_agent-native/integrations/process-task"', + ); + + const recoveryDir = path.join( + tmpDir, + ".netlify", + "functions-internal", + "dispatch-integration-recovery", + ); + const recoveryEntry = fs.readFileSync( + path.join(recoveryDir, "dispatch-integration-recovery.mjs"), + "utf8", + ); + expect(recoveryEntry).toContain('schedule: "* * * * *"'); + expect(recoveryEntry).toContain( + 'const SWEEP_PATH = "/dispatch/_agent-native/integrations/retry-stuck-tasks"', + ); + const generated = await import( + `${pathToFileURL(path.join(recoveryDir, "dispatch-integration-recovery.mjs")).href}?t=${Date.now()}` + ); + expect(generated.config.schedule).toBe("* * * * *"); + expect(fs.existsSync(backgroundFuncDir("starter"))).toBe(false); + expect( + fs.existsSync( + path.join( + tmpDir, + ".netlify", + "functions-internal", + "starter-integration-recovery", + ), + ), + ).toBe(false); + }); + it("emits a per-app -background function BY DEFAULT (flag unset) at its DEFAULT url (no custom path)", async () => { // Default-on: the flag is unset (deleted in beforeEach) and the 15-min // `-background` function MUST still be emitted so the worker gets the real diff --git a/packages/core/src/deploy/workspace-deploy.ts b/packages/core/src/deploy/workspace-deploy.ts index fa2b61188c..3ae0c5a3ea 100644 --- a/packages/core/src/deploy/workspace-deploy.ts +++ b/packages/core/src/deploy/workspace-deploy.ts @@ -21,10 +21,16 @@ import path from "path"; import { AGENT_BACKGROUND_PROCESSOR_A2A, AGENT_BACKGROUND_PROCESSOR_FIELD, + AGENT_BACKGROUND_PROCESSOR_INTEGRATION, AGENT_BACKGROUND_PROCESSOR_ROUTE, AGENT_BACKGROUND_PROCESSOR_ROUTE_FIELD, AGENT_CHAT_PROCESS_RUN_PATH, } from "../agent/durable-background.js"; +import { + INTEGRATION_RETRY_SWEEP_PATH, + INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, + isIntegrationDurableDispatchConfigured, +} from "../integrations/integration-durable-dispatch-config.js"; import { findWorkspaceRoot } from "../scripts/utils.js"; import { DEFAULT_WORKSPACE_APP_AUDIENCE, @@ -677,9 +683,19 @@ function copyNetlifyFunctionIntoWorkspace( // Durable background agent runs (default-ON; opt out with a falsy // AGENT_CHAT_DURABLE_BACKGROUND). Additive ONLY: when explicitly opted out // this emits nothing and the single-function deploy is unchanged. - if (isDurableBackgroundDeployEnabled()) { + const integrationDurableDispatch = + app === "dispatch" && isIntegrationDurableDispatchConfigured(); + if (isDurableBackgroundDeployEnabled() || integrationDurableDispatch) { emitNetlifyBackgroundFunction(workspaceRoot, app, src, workspaceApps); } + if (integrationDurableDispatch) { + emitNetlifyIntegrationRecoveryFunction( + workspaceRoot, + app, + src, + workspaceApps, + ); + } } /** @@ -753,6 +769,7 @@ function emitNetlifyBackgroundFunction( // incoming pathname to `//_agent-native/agent-chat/_process-run`. const processRunPath = `${basePath}${AGENT_CHAT_PROCESS_RUN_PATH}`; const a2aProcessTaskPath = `${basePath}/_agent-native/a2a/_process-task`; + const integrationProcessTaskPath = `${basePath}/_agent-native/integrations/process-task`; const server = `// Mark this isolate as the durable background runtime BEFORE the handler bundle // is imported, so isInBackgroundFunctionRuntime() reliably returns true in this // function (the deployed Lambda name is not guaranteed to end in -background). A @@ -764,8 +781,10 @@ const basePath = ${JSON.stringify(basePath)}; // The base-path-prefixed framework route the Nitro router dispatches to. const PROCESS_RUN_PATH = ${JSON.stringify(processRunPath)}; const A2A_PROCESS_TASK_PATH = ${JSON.stringify(a2aProcessTaskPath)}; +const INTEGRATION_PROCESS_TASK_PATH = ${JSON.stringify(integrationProcessTaskPath)}; const BACKGROUND_PROCESSOR_FIELD = ${JSON.stringify(AGENT_BACKGROUND_PROCESSOR_FIELD)}; const BACKGROUND_PROCESSOR_A2A = ${JSON.stringify(AGENT_BACKGROUND_PROCESSOR_A2A)}; +const BACKGROUND_PROCESSOR_INTEGRATION = ${JSON.stringify(AGENT_BACKGROUND_PROCESSOR_INTEGRATION)}; const BACKGROUND_PROCESSOR_ROUTE = ${JSON.stringify(AGENT_BACKGROUND_PROCESSOR_ROUTE)}; const BACKGROUND_PROCESSOR_ROUTE_FIELD = ${JSON.stringify(AGENT_BACKGROUND_PROCESSOR_ROUTE_FIELD)}; @@ -776,6 +795,12 @@ function processorPathFromBody(body) { if (parsed?.[BACKGROUND_PROCESSOR_FIELD] === BACKGROUND_PROCESSOR_A2A) { return A2A_PROCESS_TASK_PATH; } + if ( + parsed?.[BACKGROUND_PROCESSOR_FIELD] === + BACKGROUND_PROCESSOR_INTEGRATION + ) { + return INTEGRATION_PROCESS_TASK_PATH; + } const route = parsed?.[BACKGROUND_PROCESSOR_ROUTE_FIELD]; if ( parsed?.[BACKGROUND_PROCESSOR_FIELD] === BACKGROUND_PROCESSOR_ROUTE && @@ -862,6 +887,100 @@ export const config = { ); } +function emitNetlifyIntegrationRecoveryFunction( + workspaceRoot: string, + app: string, + srcServerDir: string, + workspaceApps: WorkspaceAppManifestEntry[], +): void { + const functionName = `${app}-integration-recovery`; + const dest = path.join(netlifyFunctionsDir(workspaceRoot), functionName); + fs.rmSync(dest, { recursive: true, force: true }); + copyDir(srcServerDir, dest); + fs.rmSync(path.join(dest, "server.mjs"), { force: true }); + + const basePath = `/${app}`; + const workspaceAppAudience = workspaceAppAudienceForApp(workspaceApps, app); + const workspaceAppRouteAccess = workspaceAppRouteAccessForApp( + workspaceApps, + app, + ); + const sweepPath = `${basePath}${INTEGRATION_RETRY_SWEEP_PATH}`; + const entry = `import { createHmac } from "node:crypto"; + +const basePath = ${JSON.stringify(basePath)}; +const SWEEP_PATH = ${JSON.stringify(sweepPath)}; +const SWEEP_SUBJECT = ${JSON.stringify(INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT)}; + +function setBasePathEnv() { + const processRef = globalThis.process ??= { env: {} }; + processRef.env ??= {}; + Object.assign(processRef.env, { + AGENT_NATIVE_WORKSPACE: "1", + AGENT_NATIVE_WORKSPACE_APP_ID: ${JSON.stringify(app)}, + APP_BASE_PATH: basePath, + AGENT_NATIVE_WORKSPACE_APP_AUDIENCE: ${JSON.stringify(workspaceAppAudience)}, + AGENT_NATIVE_WORKSPACE_APP_PUBLIC_PATHS: ${JSON.stringify(JSON.stringify(workspaceAppRouteAccess.publicPaths))}, + AGENT_NATIVE_WORKSPACE_APP_PROTECTED_PATHS: ${JSON.stringify(JSON.stringify(workspaceAppRouteAccess.protectedPaths))}, + VITE_AGENT_NATIVE_WORKSPACE: "1", + VITE_AGENT_NATIVE_WORKSPACE_APP_ID: ${JSON.stringify(app)}, + VITE_APP_BASE_PATH: basePath, + VITE_AGENT_NATIVE_WORKSPACE_APP_AUDIENCE: ${JSON.stringify(workspaceAppAudience)}, + VITE_AGENT_NATIVE_WORKSPACE_APP_PUBLIC_PATHS: ${JSON.stringify(JSON.stringify(workspaceAppRouteAccess.publicPaths))}, + VITE_AGENT_NATIVE_WORKSPACE_APP_PROTECTED_PATHS: ${JSON.stringify(JSON.stringify(workspaceAppRouteAccess.protectedPaths))}, + VITE_AGENT_NATIVE_WORKSPACE_APPS_JSON: ${JSON.stringify(JSON.stringify(workspaceApps))}, + ${JSON.stringify(WORKSPACE_APPS_ENV_KEY)}: ${JSON.stringify(JSON.stringify(workspaceApps))}, + }); +} + +function enabled() { + const raw = process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + if (!raw) return false; + return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase()); +} + +function token() { + const secret = process.env.A2A_SECRET; + if (!secret) throw new Error("A2A_SECRET is required for integration recovery"); + const timestamp = Date.now(); + const signature = createHmac("sha256", secret) + .update(\`${INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT}:\${timestamp}\`) + .digest("hex"); + return \`\${timestamp}.\${signature}\`; +} + +setBasePathEnv(); +let cachedHandler; + +export default async function handler(request, context) { + setBasePathEnv(); + if (!enabled()) return new Response(null, { status: 204 }); + cachedHandler ??= (await import("./main.mjs")).default; + const url = new URL(request.url); + url.pathname = SWEEP_PATH; + const rewritten = new Request(url.toString(), { + method: "POST", + headers: { + Authorization: \`Bearer \${token()}\`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ taskId: SWEEP_SUBJECT }), + }); + return cachedHandler(rewritten, context); +} + +export const config = { + name: ${JSON.stringify(`${app} integration pending-task recovery`)}, + generator: "agent-native workspace deploy", + schedule: "* * * * *", + nodeBundler: "none", + includedFiles: ["**"], + preferStatic: false, +}; +`; + fs.writeFileSync(path.join(dest, `${functionName}.mjs`), entry); +} + function patchNetlifyFunctionEntry( functionDir: string, app: string, diff --git a/packages/core/src/integrations/integration-durable-dispatch-config.ts b/packages/core/src/integrations/integration-durable-dispatch-config.ts new file mode 100644 index 0000000000..9d1267171e --- /dev/null +++ b/packages/core/src/integrations/integration-durable-dispatch-config.ts @@ -0,0 +1,22 @@ +export const INTEGRATION_DURABLE_DISPATCH_ENV = + "AGENT_INTEGRATION_DURABLE_DISPATCH"; +export const INTEGRATION_DURABLE_DISPATCH_SCOPES_ENV = + "AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES"; +export const INTEGRATION_PROCESS_TASK_PATH = + "/_agent-native/integrations/process-task"; +export const INTEGRATION_RETRY_SWEEP_PATH = + "/_agent-native/integrations/retry-stuck-tasks"; +export const INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT = + "integration-pending-tasks-sweep"; + +export function isIntegrationDurableDispatchConfigured(): boolean { + const value = process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + if (!value) return false; + const normalized = value.trim().toLowerCase(); + return ( + normalized === "1" || + normalized === "true" || + normalized === "yes" || + normalized === "on" + ); +} diff --git a/packages/core/src/integrations/integration-durable-dispatch.spec.ts b/packages/core/src/integrations/integration-durable-dispatch.spec.ts new file mode 100644 index 0000000000..89218b8f21 --- /dev/null +++ b/packages/core/src/integrations/integration-durable-dispatch.spec.ts @@ -0,0 +1,113 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const fireInternalDispatchMock = vi.hoisted(() => vi.fn()); +const recordDispatchAttemptMock = vi.hoisted(() => vi.fn()); + +vi.mock("../server/self-dispatch.js", () => ({ + fireInternalDispatch: fireInternalDispatchMock, +})); + +vi.mock("./pending-tasks-store.js", () => ({ + recordPendingTaskDispatchAttempt: recordDispatchAttemptMock, +})); + +describe("durable integration dispatch", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.unstubAllEnvs(); + fireInternalDispatchMock.mockResolvedValue(undefined); + recordDispatchAttemptMock.mockResolvedValue(undefined); + vi.stubEnv("NETLIFY", "true"); + vi.stubEnv("A2A_SECRET", "test-secret"); + }); + + it("keeps the portable path as the default", async () => { + const { dispatchPendingIntegrationTask } = + await import("./integration-durable-dispatch.js"); + + await expect( + dispatchPendingIntegrationTask({ + taskId: "task-1", + task: { platform: "slack", externalThreadId: "slack:team:C123:1" }, + baseUrl: "https://app.test", + }), + ).resolves.toBe("portable-unconfirmed"); + + expect(fireInternalDispatchMock).toHaveBeenCalledWith( + expect.objectContaining({ + path: "/_agent-native/integrations/process-task", + }), + ); + }); + + it("uses an acknowledged Netlify handoff for an enabled Slack scope", async () => { + vi.stubEnv("AGENT_INTEGRATION_DURABLE_DISPATCH", "true"); + vi.stubEnv("AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES", "slack:C123"); + const { dispatchPendingIntegrationTask } = + await import("./integration-durable-dispatch.js"); + + await expect( + dispatchPendingIntegrationTask({ + taskId: "task-2", + task: { platform: "slack", externalThreadId: "slack:team:C123:1" }, + baseUrl: "https://app.test", + }), + ).resolves.toBe("background-acknowledged"); + + expect(fireInternalDispatchMock).toHaveBeenCalledWith( + expect.objectContaining({ + path: "/.netlify/functions/server-agent-background", + awaitResponse: true, + body: { __agentNativeProcessor: "integration" }, + }), + ); + expect(recordDispatchAttemptMock).toHaveBeenCalledWith( + "task-2", + "background-acknowledged", + ); + }); + + it("does not broaden a scoped rollout", async () => { + vi.stubEnv("AGENT_INTEGRATION_DURABLE_DISPATCH", "true"); + vi.stubEnv("AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES", "slack:C999"); + const { dispatchPendingIntegrationTask } = + await import("./integration-durable-dispatch.js"); + + await dispatchPendingIntegrationTask({ + taskId: "task-3", + task: { platform: "slack", externalThreadId: "slack:team:C123:1" }, + baseUrl: "https://app.test", + }); + + expect(fireInternalDispatchMock).toHaveBeenCalledOnce(); + expect(fireInternalDispatchMock).toHaveBeenCalledWith( + expect.objectContaining({ + path: "/_agent-native/integrations/process-task", + }), + ); + }); + + it("falls back to the portable processor when the background handoff fails", async () => { + vi.stubEnv("AGENT_INTEGRATION_DURABLE_DISPATCH", "true"); + fireInternalDispatchMock + .mockRejectedValueOnce(new Error("background unavailable")) + .mockResolvedValueOnce(undefined); + const { dispatchPendingIntegrationTask } = + await import("./integration-durable-dispatch.js"); + + await expect( + dispatchPendingIntegrationTask({ + taskId: "task-4", + task: { platform: "slack", externalThreadId: "slack:team:C123:1" }, + baseUrl: "https://app.test", + }), + ).resolves.toBe("portable-unconfirmed"); + + expect(fireInternalDispatchMock).toHaveBeenCalledTimes(2); + expect(fireInternalDispatchMock.mock.calls[1]?.[0]).toEqual( + expect.objectContaining({ + path: "/_agent-native/integrations/process-task", + }), + ); + }); +}); diff --git a/packages/core/src/integrations/integration-durable-dispatch.ts b/packages/core/src/integrations/integration-durable-dispatch.ts new file mode 100644 index 0000000000..bd698292f6 --- /dev/null +++ b/packages/core/src/integrations/integration-durable-dispatch.ts @@ -0,0 +1,183 @@ +import { hasConfiguredA2ASecret } from "../a2a/auth-policy.js"; +import { + AGENT_BACKGROUND_PROCESSOR_FIELD, + AGENT_BACKGROUND_PROCESSOR_INTEGRATION, + dispatchPathTargetsNetlifyBackgroundFunction, + resolveDurableBackgroundDispatchPath, +} from "../agent/durable-background.js"; +import { fireInternalDispatch } from "../server/self-dispatch.js"; +import { + INTEGRATION_DURABLE_DISPATCH_ENV, + INTEGRATION_DURABLE_DISPATCH_SCOPES_ENV, + INTEGRATION_PROCESS_TASK_PATH, + INTEGRATION_RETRY_SWEEP_PATH, + INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, + isIntegrationDurableDispatchConfigured, +} from "./integration-durable-dispatch-config.js"; +import { recordPendingTaskDispatchAttempt } from "./pending-tasks-store.js"; + +export { + INTEGRATION_DURABLE_DISPATCH_ENV, + INTEGRATION_DURABLE_DISPATCH_SCOPES_ENV, + INTEGRATION_PROCESS_TASK_PATH, + INTEGRATION_RETRY_SWEEP_PATH, + INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, + isIntegrationDurableDispatchConfigured, +}; + +export type IntegrationDispatchOutcome = + | "background-acknowledged" + | "portable-unconfirmed" + | "failed"; + +export interface IntegrationDispatchTaskScope { + platform: string; + externalThreadId: string; + platformContext?: Record; +} + +export interface IntegrationDurableDispatchScope { + platform: string; + value: string; +} + +export function configuredIntegrationDurableDispatchScopes(): + | IntegrationDurableDispatchScope[] + | null { + const raw = process.env.AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES; + if (!raw?.trim()) return null; + return raw + .split(",") + .map((value) => value.trim()) + .filter(Boolean) + .map((scope) => { + const separator = scope.indexOf(":"); + return { + platform: separator > 0 ? scope.slice(0, separator) : "", + value: separator > 0 ? scope.slice(separator + 1) : "", + }; + }) + .filter((scope) => scope.platform && scope.value); +} + +function taskScopeCandidates(task: IntegrationDispatchTaskScope): string[] { + const candidates = [ + `${task.platform}:*`, + `${task.platform}:${task.externalThreadId}`, + ]; + const explicitScope = task.platformContext?.channelId; + if (typeof explicitScope === "string" && explicitScope.trim()) { + candidates.push(`${task.platform}:${explicitScope.trim()}`); + } else if (task.platform === "slack") { + const channelId = task.externalThreadId.split(":")[2]; + if (channelId) candidates.push(`slack:${channelId}`); + } + return candidates; +} + +export function isIntegrationDurableDispatchEnabledForTask( + task: IntegrationDispatchTaskScope, +): boolean { + if (!isIntegrationDurableDispatchConfigured()) return false; + const scopes = configuredIntegrationDurableDispatchScopes(); + if ( + scopes && + !taskScopeCandidates(task).some((candidate) => + scopes.some((scope) => `${scope.platform}:${scope.value}` === candidate), + ) + ) { + return false; + } + const path = resolveDurableBackgroundDispatchPath( + INTEGRATION_PROCESS_TASK_PATH, + ); + return ( + dispatchPathTargetsNetlifyBackgroundFunction(path) && + hasConfiguredA2ASecret() + ); +} + +async function recordDispatch( + taskId: string, + outcome: IntegrationDispatchOutcome, +): Promise { + try { + await recordPendingTaskDispatchAttempt(taskId, outcome); + } catch (error) { + console.error( + `[integrations] Failed to record dispatch outcome for ${taskId}:`, + error, + ); + } +} + +function logDispatch( + taskId: string, + outcome: IntegrationDispatchOutcome, + startedAt: number, +): void { + console.info("[integrations] pending task dispatch", { + taskId, + outcome, + durationMs: Date.now() - startedAt, + }); +} + +export async function dispatchPendingIntegrationTask(input: { + taskId: string; + task: IntegrationDispatchTaskScope; + event?: unknown; + baseUrl?: string; + portableSettleMs?: number; +}): Promise { + const startedAt = Date.now(); + const durable = isIntegrationDurableDispatchEnabledForTask(input.task); + if (durable) { + const backgroundPath = resolveDurableBackgroundDispatchPath( + INTEGRATION_PROCESS_TASK_PATH, + ); + try { + await fireInternalDispatch({ + event: input.event, + baseUrl: input.baseUrl, + path: backgroundPath, + taskId: input.taskId, + body: { + [AGENT_BACKGROUND_PROCESSOR_FIELD]: + AGENT_BACKGROUND_PROCESSOR_INTEGRATION, + }, + awaitResponse: true, + responseTimeoutMs: 2_000, + }); + await recordDispatch(input.taskId, "background-acknowledged"); + logDispatch(input.taskId, "background-acknowledged", startedAt); + return "background-acknowledged"; + } catch (error) { + console.error( + `[integrations] Background dispatch failed for ${input.taskId}; trying the portable processor:`, + error, + ); + } + } + + try { + await fireInternalDispatch({ + event: input.event, + baseUrl: input.baseUrl, + path: INTEGRATION_PROCESS_TASK_PATH, + taskId: input.taskId, + settleMs: input.portableSettleMs, + }); + await recordDispatch(input.taskId, "portable-unconfirmed"); + logDispatch(input.taskId, "portable-unconfirmed", startedAt); + return "portable-unconfirmed"; + } catch (error) { + console.error( + `[integrations] Portable dispatch failed for ${input.taskId}:`, + error, + ); + await recordDispatch(input.taskId, "failed"); + logDispatch(input.taskId, "failed", startedAt); + return "failed"; + } +} diff --git a/packages/core/src/integrations/pending-tasks-retry-job.spec.ts b/packages/core/src/integrations/pending-tasks-retry-job.spec.ts index 4e3becb5b8..06de104574 100644 --- a/packages/core/src/integrations/pending-tasks-retry-job.spec.ts +++ b/packages/core/src/integrations/pending-tasks-retry-job.spec.ts @@ -1,6 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const executeMock = vi.hoisted(() => vi.fn()); +const dispatchPendingTaskMock = vi.hoisted(() => + vi.fn(async () => "background-acknowledged"), +); +const durableEnabledMock = vi.hoisted(() => vi.fn(() => true)); +const configuredScopesMock = vi.hoisted(() => vi.fn(() => null)); +const ensurePendingTasksTableMock = vi.hoisted(() => vi.fn()); vi.mock("../db/client.js", () => ({ getDbExec: () => ({ execute: executeMock }), @@ -10,6 +16,17 @@ vi.mock("../server/core-routes-plugin.js", () => ({ FRAMEWORK_ROUTE_PREFIX: "/_agent-native", })); +vi.mock("./pending-tasks-store.js", () => ({ + ensurePendingTasksTable: ensurePendingTasksTableMock, + MAX_PENDING_TASK_ATTEMPTS: 3, +})); + +vi.mock("./integration-durable-dispatch.js", () => ({ + configuredIntegrationDurableDispatchScopes: configuredScopesMock, + dispatchPendingIntegrationTask: dispatchPendingTaskMock, + isIntegrationDurableDispatchEnabledForTask: durableEnabledMock, +})); + async function loadRetryJob() { vi.resetModules(); return import("./pending-tasks-retry-job.js"); @@ -19,19 +36,28 @@ describe("pending task retry job", () => { beforeEach(() => { vi.clearAllMocks(); vi.unstubAllEnvs(); - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response("ok", { status: 200 })), - ); + dispatchPendingTaskMock.mockResolvedValue("background-acknowledged"); + durableEnabledMock.mockReturnValue(true); + configuredScopesMock.mockReturnValue(null); + ensurePendingTasksTableMock.mockResolvedValue(undefined); }); it("resets stuck processing tasks to pending and re-fires the processor", async () => { const { retryStuckPendingTasks } = await loadRetryJob(); executeMock .mockResolvedValueOnce({ - rows: [{ id: "task-processing", status: "processing", attempts: 1 }], + rows: [ + { + id: "task-processing", + platform: "slack", + external_thread_id: "slack:team:C123:1", + status: "processing", + attempts: 1, + updated_at: 10, + }, + ], }) - .mockResolvedValueOnce({ rows: [] }); + .mockResolvedValueOnce({ rows: [], rowCount: 1 }); await retryStuckPendingTasks("https://app.test"); @@ -39,25 +65,59 @@ describe("pending task retry job", () => { 2, expect.objectContaining({ sql: expect.stringContaining("AND status = ?"), - args: ["pending", expect.any(Number), "task-processing", "processing"], + args: [ + "pending", + expect.any(Number), + "task-processing", + "processing", + 10, + ], }), ); - expect(fetch).toHaveBeenCalledWith( - "https://app.test/_agent-native/integrations/process-task", + expect(dispatchPendingTaskMock).toHaveBeenCalledWith( expect.objectContaining({ - method: "POST", - body: JSON.stringify({ taskId: "task-processing" }), + taskId: "task-processing", + baseUrl: "https://app.test", }), ); + expect(ensurePendingTasksTableMock).toHaveBeenCalledOnce(); + }); + + it("fails loud when the additive queue schema cannot be ensured", async () => { + ensurePendingTasksTableMock.mockRejectedValueOnce( + new Error("migration lock timeout"), + ); + const { retryStuckPendingTasks } = await loadRetryJob(); + + await expect(retryStuckPendingTasks()).rejects.toThrow( + "migration lock timeout", + ); + expect(executeMock).not.toHaveBeenCalled(); + }); + + it("does not misclassify query failures as a missing queue table", async () => { + executeMock.mockRejectedValueOnce(new Error("permission denied")); + const { retryStuckPendingTasks } = await loadRetryJob(); + + await expect(retryStuckPendingTasks()).rejects.toThrow("permission denied"); }); it("marks tasks failed after the retry cap without re-firing", async () => { const { retryStuckPendingTasks } = await loadRetryJob(); executeMock .mockResolvedValueOnce({ - rows: [{ id: "task-exhausted", status: "pending", attempts: 3 }], + rows: [ + { + id: "task-exhausted", + platform: "slack", + external_thread_id: "slack:team:C123:1", + status: "pending", + attempts: 3, + updated_at: 20, + }, + ], }) - .mockResolvedValueOnce({ rows: [] }); + .mockResolvedValueOnce({ rows: [], rowCount: 1 }); await retryStuckPendingTasks("https://app.test"); @@ -70,6 +130,7 @@ describe("pending task retry job", () => { "Retry job: exceeded 3 attempts", "task-exhausted", "pending", + 20, ], }), ); @@ -79,7 +140,7 @@ describe("pending task retry job", () => { expect((executeMock.mock.calls[1]?.[0] as { sql: string }).sql).toContain( "external_event_key = NULL", ); - expect(fetch).not.toHaveBeenCalled(); + expect(dispatchPendingTaskMock).not.toHaveBeenCalled(); }); it("uses a shorter processing-stuck cutoff on serverless hosts", async () => { @@ -94,7 +155,13 @@ describe("pending task retry job", () => { expect(executeMock).toHaveBeenNthCalledWith( 1, expect.objectContaining({ - args: [Date.now() - 90_000, Date.now() - 90_000, Date.now() - 75_000], + args: [ + Date.now() - 90_000, + Date.now() - 90_000, + Date.now() - 16 * 60_000, + Date.now() - 75_000, + 100, + ], }), ); vi.useRealTimers(); @@ -105,11 +172,25 @@ describe("pending task retry job", () => { executeMock .mockResolvedValueOnce({ rows: [ - { id: "task-stale-pending", status: "pending", attempts: 1 }, - { id: "task-stale-processing", status: "processing", attempts: 3 }, + { + id: "task-stale-pending", + platform: "slack", + external_thread_id: "slack:team:C123:1", + status: "pending", + attempts: 1, + updated_at: 30, + }, + { + id: "task-stale-processing", + platform: "slack", + external_thread_id: "slack:team:C123:1", + status: "processing", + attempts: 3, + updated_at: 40, + }, ], }) - .mockResolvedValue({ rows: [] }); + .mockResolvedValue({ rows: [], rowCount: 0 }); await retryStuckPendingTasks("https://app.test"); @@ -117,7 +198,13 @@ describe("pending task retry job", () => { 2, expect.objectContaining({ sql: expect.stringContaining("AND status = ?"), - args: ["pending", expect.any(Number), "task-stale-pending", "pending"], + args: [ + "pending", + expect.any(Number), + "task-stale-pending", + "pending", + 30, + ], }), ); expect(executeMock).toHaveBeenNthCalledWith( @@ -129,8 +216,79 @@ describe("pending task retry job", () => { "Retry job: exceeded 3 attempts", "task-stale-processing", "processing", + 40, ], }), ); + expect(dispatchPendingTaskMock).not.toHaveBeenCalled(); + }); + + it("pushes durable rollout scopes into the bounded SQL selection", async () => { + configuredScopesMock.mockReturnValue([ + { platform: "slack", value: "C123" }, + ]); + durableEnabledMock.mockImplementation(({ externalThreadId }) => + externalThreadId.includes(":C123:"), + ); + const { retryStuckPendingTasks } = await loadRetryJob(); + executeMock + .mockResolvedValueOnce({ + rows: [ + { + id: "enabled-task", + platform: "slack", + external_thread_id: "A1:T1:C123:1.0", + status: "pending", + attempts: 0, + updated_at: 60, + }, + ], + }) + .mockResolvedValueOnce({ rows: [], rowCount: 1 }); + + const result = await retryStuckPendingTasks({ + webhookBaseUrl: "https://app.test", + durableOnly: true, + limit: 1, + }); + + const select = executeMock.mock.calls[0]?.[0] as { + sql: string; + args: unknown[]; + }; + expect(select.sql).toContain("external_thread_id LIKE ?"); + expect(select.args.slice(-4)).toEqual(["slack", "C123", "%:%:C123:%", 1]); + expect(result).toEqual({ + selected: 1, + dispatched: 1, + markedFailed: 0, + skipped: 0, + dispatchFailed: 0, + }); + expect(dispatchPendingTaskMock).toHaveBeenCalledWith( + expect.objectContaining({ taskId: "enabled-task" }), + ); + }); + + it("does not reclaim a healthy durable background task at the synchronous cutoff", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-21T12:00:00.000Z")); + const { retryStuckPendingTasks } = await loadRetryJob(); + executeMock.mockResolvedValueOnce({ rows: [] }); + + const result = await retryStuckPendingTasks(); + + expect(result.selected).toBe(0); + const select = executeMock.mock.calls[0]?.[0] as { + sql: string; + args: unknown[]; + }; + expect(select.sql).toContain( + "last_dispatch_outcome = 'background-acknowledged'", + ); + expect(select.args).toContain(Date.now() - 16 * 60_000); + expect(executeMock).toHaveBeenCalledOnce(); + expect(dispatchPendingTaskMock).not.toHaveBeenCalled(); + vi.useRealTimers(); }); }); diff --git a/packages/core/src/integrations/pending-tasks-retry-job.ts b/packages/core/src/integrations/pending-tasks-retry-job.ts index 8b5a3860ce..2022ed9d84 100644 --- a/packages/core/src/integrations/pending-tasks-retry-job.ts +++ b/packages/core/src/integrations/pending-tasks-retry-job.ts @@ -1,8 +1,13 @@ import { getDbExec } from "../db/client.js"; -import { withConfiguredAppBasePath } from "../server/app-base-path.js"; -import { FRAMEWORK_ROUTE_PREFIX } from "../server/core-routes-plugin.js"; -import { signInternalToken } from "./internal-token.js"; -import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js"; +import { + configuredIntegrationDurableDispatchScopes, + dispatchPendingIntegrationTask, + isIntegrationDurableDispatchEnabledForTask, +} from "./integration-durable-dispatch.js"; +import { + ensurePendingTasksTable, + MAX_PENDING_TASK_ATTEMPTS, +} from "./pending-tasks-store.js"; /** * Retries stuck integration webhook tasks. @@ -10,12 +15,16 @@ import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js"; * The integration webhook flow enqueues work into `integration_pending_tasks` * (see `pending-tasks-store.ts`) and then fires a self-webhook to the * `/_agent-native/integrations/process-task` endpoint to drain the queue. - * If that fire-and-forget dispatch fails (e.g. transient network blip), the + * If that initial dispatch is lost (e.g. transient network blip), the * row stays in `pending` forever. Likewise, if the processor is killed mid- * processing (function timeout, container shutdown), a row can remain in * `processing` forever. * - * This job runs every 60s and re-fires the processor endpoint for tasks that + * The in-process fallback runs every 60s. Durable deployments also invoke the + * same bounded sweep from an external scheduler so recovery does not depend on + * a serverless process remaining alive. + * + * Each sweep re-fires the processor endpoint for tasks that * look stuck: * - status='pending' AND created_at older than 90s (initial dispatch lost) * - status='processing' AND updated_at older than the host-specific @@ -35,8 +44,9 @@ const PENDING_STUCK_AFTER_MS = 90_000; /** Tasks "processing" longer than this are considered killed mid-flight. */ const DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000; const SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000; +const DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS = 16 * 60 * 1000; /** After this many attempts we give up and mark the task failed */ -const PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`; +const DEFAULT_SWEEP_LIMIT = 100; let retryInterval: ReturnType | null = null; let initialTimer: ReturnType | null = null; @@ -49,8 +59,66 @@ let tableExists: boolean | null = null; interface StuckTaskRow { id: string; + platform: string; + externalThreadId: string; status: string; attempts: number; + updatedAt: number; +} + +export interface PendingTasksSweepResult { + selected: number; + dispatched: number; + markedFailed: number; + skipped: number; + dispatchFailed: number; +} + +export interface PendingTasksSweepOptions { + webhookBaseUrl?: string; + limit?: number; + durableOnly?: boolean; +} + +function affectedRows(result: unknown): number { + return Number( + (result as { rowsAffected?: number }).rowsAffected ?? + (result as { rowCount?: number }).rowCount ?? + 0, + ); +} + +function isMissingPendingTasksTableError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error ?? ""); + return /no such table|relation .* does not exist|undefined_table/i.test( + message, + ); +} + +function durableScopeSql(): { clause: string; args: string[] } { + const scopes = configuredIntegrationDurableDispatchScopes(); + if (scopes === null) return { clause: "", args: [] }; + if (scopes.length === 0) return { clause: " AND 1 = 0", args: [] }; + + const clauses: string[] = []; + const args: string[] = []; + for (const scope of scopes) { + if (scope.value === "*") { + clauses.push("platform = ?"); + args.push(scope.platform); + continue; + } + if (scope.platform === "slack" && /^[A-Za-z0-9]+$/.test(scope.value)) { + clauses.push( + "(platform = ? AND (external_thread_id = ? OR external_thread_id LIKE ?))", + ); + args.push(scope.platform, scope.value, `%:%:${scope.value}:%`); + continue; + } + clauses.push("(platform = ? AND external_thread_id = ?)"); + args.push(scope.platform, scope.value); + } + return { clause: ` AND (${clauses.join(" OR ")})`, args }; } /** @@ -58,39 +126,75 @@ interface StuckTaskRow { * Exported for tests and for manual triggers. */ export async function retryStuckPendingTasks( - webhookBaseUrl?: string, -): Promise { - const baseUrl = webhookBaseUrl ?? activeWebhookBaseUrl; + input?: string | PendingTasksSweepOptions, +): Promise { + const options: PendingTasksSweepOptions = + typeof input === "string" ? { webhookBaseUrl: input } : (input ?? {}); + const baseUrl = options.webhookBaseUrl ?? activeWebhookBaseUrl; + const limit = Math.max( + 1, + Math.min(500, Math.floor(options.limit ?? DEFAULT_SWEEP_LIMIT)), + ); + const scopeSql = options.durableOnly + ? durableScopeSql() + : { clause: "", args: [] as string[] }; + const result: PendingTasksSweepResult = { + selected: 0, + dispatched: 0, + markedFailed: 0, + skipped: 0, + dispatchFailed: 0, + }; + await ensurePendingTasksTable(); const client = getDbExec(); const now = Date.now(); const pendingCutoff = now - PENDING_STUCK_AFTER_MS; const processingCutoff = now - getProcessingStuckAfterMs(); + const durableProcessingCutoff = + now - DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS; let stuckRows: StuckTaskRow[]; try { const { rows } = await client.execute({ sql: ` - SELECT id, status, attempts + SELECT id, platform, external_thread_id, status, attempts, updated_at FROM integration_pending_tasks - WHERE (status = 'pending' AND created_at <= ? AND updated_at <= ?) - OR (status = 'processing' AND updated_at <= ?) + WHERE ((status = 'pending' AND created_at <= ? AND updated_at <= ?) + OR (status = 'processing' AND ( + (last_dispatch_outcome = 'background-acknowledged' + AND updated_at <= ?) + OR ((last_dispatch_outcome IS NULL + OR last_dispatch_outcome <> 'background-acknowledged') + AND updated_at <= ?) + ))) + ${scopeSql.clause} + ORDER BY updated_at ASC + LIMIT ? `, // `updated_at` is initialized to `created_at` on insert, so a genuinely // stuck pending row still matches on the first sweep. The retry path // below touches `updated_at`, which (with this predicate) keeps the row // from being re-selected — and re-firing the processor — on every tick. - args: [pendingCutoff, pendingCutoff, processingCutoff], + args: [ + pendingCutoff, + pendingCutoff, + durableProcessingCutoff, + processingCutoff, + ...scopeSql.args, + limit, + ], }); stuckRows = rows.map((r) => ({ id: r.id as string, + platform: r.platform as string, + externalThreadId: r.external_thread_id as string, status: r.status as string, attempts: Number(r.attempts ?? 0), + updatedAt: Number(r.updated_at ?? 0), })); tableExists = true; } catch (err) { - // Most common case: the table hasn't been created yet because no inbound - // integration webhook has been processed on this deployment. Silently - // no-op until the table appears. + if (!isMissingPendingTasksTableError(err)) throw err; if (tableExists !== false) { tableExists = false; if (process.env.DEBUG) { @@ -99,17 +203,26 @@ export async function retryStuckPendingTasks( ); } } - return; + return result; } - if (stuckRows.length === 0) return; + stuckRows = stuckRows.filter((row) => { + const durable = isIntegrationDurableDispatchEnabledForTask({ + platform: row.platform, + externalThreadId: row.externalThreadId, + }); + if (options.durableOnly && !durable) return false; + return true; + }); + result.selected = stuckRows.length; + if (stuckRows.length === 0) return result; for (const row of stuckRows) { try { // Cap retries — mark failed and move on so the row stops bouncing // between pending and processing forever. if (row.attempts >= MAX_PENDING_TASK_ATTEMPTS) { - await client.execute({ + const update = await client.execute({ sql: ` UPDATE integration_pending_tasks SET status = 'failed', @@ -119,14 +232,21 @@ export async function retryStuckPendingTasks( external_event_key = NULL WHERE id = ? AND status = ? + AND updated_at = ? `, args: [ Date.now(), `Retry job: exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts`, row.id, row.status, + row.updatedAt, ], }); + if (affectedRows(update) === 0) { + result.skipped += 1; + continue; + } + result.markedFailed += 1; console.warn( `[integrations] Pending task ${row.id} exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts — marking failed`, ); @@ -138,24 +258,41 @@ export async function retryStuckPendingTasks( // Without this, processing rows stay stuck forever. // For pending rows, just touch updated_at to avoid re-firing every tick. const newStatus = row.status === "processing" ? "pending" : row.status; - await client.execute({ + const update = await client.execute({ sql: ` UPDATE integration_pending_tasks SET status = ?, updated_at = ? WHERE id = ? AND status = ? + AND updated_at = ? `, - args: [newStatus, Date.now(), row.id, row.status], + args: [newStatus, Date.now(), row.id, row.status, row.updatedAt], }); + if (affectedRows(update) === 0) { + result.skipped += 1; + continue; + } - await refireProcessor(row.id, baseUrl); + const outcome = await dispatchPendingIntegrationTask({ + taskId: row.id, + task: { + platform: row.platform, + externalThreadId: row.externalThreadId, + }, + baseUrl, + portableSettleMs: 1_000, + }); + if (outcome === "failed") result.dispatchFailed += 1; + else result.dispatched += 1; } catch (err) { + result.dispatchFailed += 1; console.error( `[integrations] Failed to retry pending task ${row.id}:`, err, ); } } + return result; } function getProcessingStuckAfterMs(): number { @@ -170,70 +307,6 @@ function getProcessingStuckAfterMs(): number { return DEFAULT_PROCESSING_STUCK_AFTER_MS; } -/** - * Fire-and-forget POST to the processor endpoint for a single task id. - * Mirrors the original dispatch from the webhook handler, including the - * short-lived HMAC bearer token bound to this taskId. - */ -async function refireProcessor( - taskId: string, - webhookBaseUrl: string | undefined, -): Promise { - const baseUrl = - webhookBaseUrl || - process.env.WEBHOOK_BASE_URL || - process.env.APP_URL || - process.env.URL || - process.env.DEPLOY_URL || - `http://localhost:${process.env.PORT || 3000}`; - - const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`; - - // Sign with HMAC if A2A_SECRET is configured. In production we MUST sign — - // an unsigned dispatch in production lets attackers re-trigger any queued - // task with a guessable id (C3 in the webhook security audit). In dev we - // fall back to unsigned so contributors can iterate without configuring - // A2A_SECRET locally. - const headers: Record = { - "Content-Type": "application/json", - }; - try { - headers["Authorization"] = `Bearer ${signInternalToken(taskId)}`; - } catch (err) { - if (process.env.NODE_ENV === "production") { - console.error( - `[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` + - "Set A2A_SECRET to enable signed retry dispatches.", - ); - return; - } - // Dev: proceed unsigned. Log the underlying error path so a malformed - // secret (different from "not set") doesn't fail silently (L5 in the audit). - if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) { - console.error( - `[integrations] signInternalToken failed unexpectedly for ${taskId}:`, - err, - ); - } - } - - // Don't await the body — we just want the request to leave the box. - // A short timeout avoids tying up the retry loop on a hung processor. - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 5_000); - - try { - await fetch(url, { - method: "POST", - headers, - body: JSON.stringify({ taskId }), - signal: controller.signal, - }); - } finally { - clearTimeout(timer); - } -} - /** * Start the periodic retry loop. Safe to call multiple times — second call * is a no-op. diff --git a/packages/core/src/integrations/pending-tasks-store.spec.ts b/packages/core/src/integrations/pending-tasks-store.spec.ts index 8e2d43efcd..30ec6a9b7d 100644 --- a/packages/core/src/integrations/pending-tasks-store.spec.ts +++ b/packages/core/src/integrations/pending-tasks-store.spec.ts @@ -50,7 +50,9 @@ describe("integration pending task store", () => { return { rows: [] }; }); - const task = await claimPendingTask("task-1"); + const task = await claimPendingTask("task-1", { + dispatchOutcome: "background-acknowledged", + }); expect(task?.id).toBe("task-1"); const updateCall = executeMock.mock.calls.find(([query]) => { @@ -68,6 +70,15 @@ describe("integration pending task store", () => { expect((updateCall?.[0] as { sql: string }).sql).toContain( "earlier.created_at < integration_pending_tasks.created_at", ); + expect((updateCall?.[0] as { sql: string }).sql).toContain( + "last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)", + ); + expect((updateCall?.[0] as { args: unknown[] }).args).toEqual([ + "processing", + expect.any(Number), + "background-acknowledged", + "task-1", + ]); }); it("does not claim terminal failed tasks", async () => { @@ -103,6 +114,31 @@ describe("integration pending task store", () => { await expect(claimPendingTask("task-failed")).resolves.toBeNull(); }); + it("does not let fallback telemetry downgrade an active background lease", async () => { + executeMock.mockResolvedValue({ rows: [], rowsAffected: 1 }); + const { recordPendingTaskDispatchAttempt } = await loadStore(); + + await recordPendingTaskDispatchAttempt("task-1", "portable-unconfirmed"); + + const update = executeMock.mock.calls + .map(([query]) => query) + .find( + (query): query is { sql: string; args: unknown[] } => + typeof query !== "string" && + query.sql.includes("dispatch_attempts = dispatch_attempts + 1"), + ); + expect(update?.sql).toContain("WHEN status = 'processing'"); + expect(update?.sql).toContain( + "last_dispatch_outcome = 'background-acknowledged'", + ); + expect(update?.sql).toContain("THEN last_dispatch_outcome"); + expect(update?.args).toEqual([ + expect.any(Number), + "portable-unconfirmed", + "task-1", + ]); + }); + it("dispatches same-millisecond thread tasks by stable id order", async () => { executeMock.mockResolvedValue({ rows: [{ id: "task-a" }] }); const { getNextPendingTaskIdForThread } = await loadStore(); diff --git a/packages/core/src/integrations/pending-tasks-store.ts b/packages/core/src/integrations/pending-tasks-store.ts index 33d2847351..de2ede38f5 100644 --- a/packages/core/src/integrations/pending-tasks-store.ts +++ b/packages/core/src/integrations/pending-tasks-store.ts @@ -33,6 +33,9 @@ async function ensureTable(): Promise { org_id TEXT, status TEXT NOT NULL, attempts ${intType()} NOT NULL DEFAULT 0, + dispatch_attempts ${intType()} NOT NULL DEFAULT 0, + last_dispatch_at ${intType()}, + last_dispatch_outcome TEXT, error_message TEXT, created_at ${intType()} NOT NULL, updated_at ${intType()} NOT NULL, @@ -47,6 +50,21 @@ async function ensureTable(): Promise { "external_event_key", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`, ); + await ensureColumnExists( + "integration_pending_tasks", + "dispatch_attempts", + `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_attempts ${intType()} NOT NULL DEFAULT 0`, + ); + await ensureColumnExists( + "integration_pending_tasks", + "last_dispatch_at", + `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_at ${intType()}`, + ); + await ensureColumnExists( + "integration_pending_tasks", + "last_dispatch_outcome", + `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_outcome TEXT`, + ); await ensureIndexExists( "idx_pending_tasks_status_created", `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`, @@ -69,6 +87,7 @@ async function ensureTable(): Promise { // unique index ensures a duplicate INSERT raises an error we can // catch as "already-enqueued". await ensureExternalEventKey(client); + await ensureDispatchColumns(client); await client.execute( `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`, ); @@ -81,6 +100,10 @@ async function ensureTable(): Promise { return _initPromise; } +export async function ensurePendingTasksTable(): Promise { + await ensureTable(); +} + async function ensureExternalEventKey( client: ReturnType, ): Promise { @@ -107,6 +130,31 @@ async function ensureExternalEventKey( } } +async function ensureDispatchColumns( + client: ReturnType, +): Promise { + const columns = [ + `dispatch_attempts ${intType()} NOT NULL DEFAULT 0`, + `last_dispatch_at ${intType()}`, + "last_dispatch_outcome TEXT", + ]; + for (const column of columns) { + try { + await client.execute( + `ALTER TABLE integration_pending_tasks ADD COLUMN ${column}`, + ); + } catch (err: any) { + if ( + !String(err?.message ?? err) + .toLowerCase() + .includes("duplicate") + ) { + throw err; + } + } + } +} + /** Status values for an integration pending task. */ export type PendingTaskStatus = | "pending" @@ -123,6 +171,9 @@ export interface PendingTask { orgId: string | null; status: PendingTaskStatus; attempts: number; + dispatchAttempts: number; + lastDispatchAt: number | null; + lastDispatchOutcome: string | null; errorMessage: string | null; createdAt: number; updatedAt: number; @@ -164,6 +215,12 @@ function rowToTask(row: Record): PendingTask { orgId: (row.org_id as string | null) ?? null, status: row.status as PendingTaskStatus, attempts: Number(row.attempts ?? 0), + dispatchAttempts: Number(row.dispatch_attempts ?? 0), + lastDispatchAt: + row.last_dispatch_at == null + ? null + : Number(row.last_dispatch_at as number), + lastDispatchOutcome: (row.last_dispatch_outcome as string | null) ?? null, errorMessage: (row.error_message as string | null) ?? null, createdAt: Number(row.created_at ?? 0), updatedAt: Number(row.updated_at ?? 0), @@ -238,7 +295,7 @@ export async function getPendingTask(id: string): Promise { await ensureTable(); const client = getDbExec(); const { rows } = await client.execute({ - sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at + sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, error_message, created_at, updated_at, completed_at FROM integration_pending_tasks WHERE id = ? LIMIT 1`, args: [id], }); @@ -342,6 +399,7 @@ export async function resolveIntegrationSourceContext( */ export async function claimPendingTask( id: string, + options?: { dispatchOutcome?: string }, ): Promise { await ensureTable(); const client = getDbExec(); @@ -352,7 +410,8 @@ export async function claimPendingTask( const result = await client.execute({ sql: isPostgres() ? `UPDATE integration_pending_tasks - SET status = ?, attempts = attempts + 1, updated_at = ? + SET status = ?, attempts = attempts + 1, updated_at = ?, + last_dispatch_outcome = COALESCE(?, last_dispatch_outcome) WHERE id = ? AND status = 'pending' AND NOT EXISTS ( SELECT 1 FROM integration_pending_tasks active @@ -374,9 +433,10 @@ export async function claimPendingTask( ) ) ) - RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at` + RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, error_message, created_at, updated_at, completed_at` : `UPDATE integration_pending_tasks - SET status = ?, attempts = attempts + 1, updated_at = ? + SET status = ?, attempts = attempts + 1, updated_at = ?, + last_dispatch_outcome = COALESCE(?, last_dispatch_outcome) WHERE id = ? AND status = 'pending' AND NOT EXISTS ( SELECT 1 FROM integration_pending_tasks active @@ -398,7 +458,7 @@ export async function claimPendingTask( ) ) )`, - args: ["processing", now, id], + args: ["processing", now, options?.dispatchOutcome ?? null, id], }); const rows = result.rows ?? []; @@ -418,6 +478,26 @@ export async function claimPendingTask( return fetched; } +export async function recordPendingTaskDispatchAttempt( + id: string, + outcome: string, +): Promise { + await ensureTable(); + await getDbExec().execute({ + sql: `UPDATE integration_pending_tasks + SET dispatch_attempts = dispatch_attempts + 1, + last_dispatch_at = ?, + last_dispatch_outcome = CASE + WHEN status = 'processing' + AND last_dispatch_outcome = 'background-acknowledged' + THEN last_dispatch_outcome + ELSE ? + END + WHERE id = ?`, + args: [Date.now(), outcome.slice(0, 80), id], + }); +} + /** Next queued turn for a provider thread after its current task completes. */ export async function getNextPendingTaskIdForThread( platform: string, diff --git a/packages/core/src/integrations/plugin.spec.ts b/packages/core/src/integrations/plugin.spec.ts index 3ec9f23576..a0110f7d86 100644 --- a/packages/core/src/integrations/plugin.spec.ts +++ b/packages/core/src/integrations/plugin.spec.ts @@ -1,3 +1,5 @@ +import { createHmac } from "node:crypto"; + import { afterEach, describe, expect, it, vi } from "vitest"; import { IntegrationIdentityDeclinedError } from "./identity.js"; @@ -36,6 +38,7 @@ const markTaskRetryableMock = vi.hoisted(() => vi.fn()); const markTaskDeliveryRetryableMock = vi.hoisted(() => vi.fn()); const stageTaskDeliveryPayloadMock = vi.hoisted(() => vi.fn()); const insertPendingTaskMock = vi.hoisted(() => vi.fn()); +const retryStuckPendingTasksMock = vi.hoisted(() => vi.fn()); vi.mock("../deploy/route-discovery.js", () => ({ getMissingDefaultPlugins: vi.fn(async () => []), @@ -68,6 +71,7 @@ vi.mock("./config-store.js", () => ({ vi.mock("./pending-tasks-retry-job.js", () => ({ startPendingTasksRetryJob: vi.fn(), + retryStuckPendingTasks: retryStuckPendingTasksMock, })); vi.mock("./google-docs-poller.js", () => ({ @@ -100,6 +104,7 @@ vi.mock("./pending-tasks-store.js", () => ({ markTaskFailed: markTaskFailedMock, markTaskRetryable: markTaskRetryableMock, markTaskDeliveryRetryable: markTaskDeliveryRetryableMock, + recordPendingTaskDispatchAttempt: vi.fn(), stageTaskDeliveryPayload: stageTaskDeliveryPayloadMock, })); @@ -144,6 +149,7 @@ async function dispatch( pathname: string, method = "GET", body?: unknown, + headers?: Record, ) { const url = `https://app.test${pathname}`; const requestBody = body === undefined ? undefined : JSON.stringify(body); @@ -159,6 +165,7 @@ async function dispatch( host: "app.test", "x-forwarded-proto": "https", ...(requestBody ? { "content-type": "application/json" } : {}), + ...headers, }, }), res: { @@ -173,6 +180,7 @@ async function dispatch( host: "app.test", "x-forwarded-proto": "https", ...(requestBody ? { "content-type": "application/json" } : {}), + ...headers, }, }, res: { @@ -241,6 +249,7 @@ describe("integrations plugin routes", () => { vi.unstubAllGlobals(); delete process.env.APP_BASE_PATH; delete process.env.VITE_APP_BASE_PATH; + delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; process.env.NODE_ENV = originalNodeEnv; if (originalA2ASecret === undefined) { delete process.env.A2A_SECRET; @@ -260,6 +269,13 @@ describe("integrations plugin routes", () => { resolveSecretMock.mockReset(); resolveSecretMock.mockReturnValue(null); handleWebhookMock.mockResolvedValue({ status: 200, body: "ok" }); + retryStuckPendingTasksMock.mockResolvedValue({ + selected: 0, + dispatched: 0, + markedFailed: 0, + skipped: 0, + dispatchFailed: 0, + }); resourceGetByPathMock.mockImplementation(async () => null); }); @@ -477,6 +493,68 @@ describe("integrations plugin routes", () => { }); }); + it("rejects unsigned durable recovery sweeps", async () => { + process.env.A2A_SECRET = "test-secret"; + process.env.AGENT_INTEGRATION_DURABLE_DISPATCH = "true"; + const nitroApp = createNitroApp(); + await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + + const result = await dispatch( + nitroApp, + "/_agent-native/integrations/retry-stuck-tasks", + "POST", + { taskId: "integration-pending-tasks-sweep" }, + ); + + expect(result.status).toBe(401); + expect(retryStuckPendingTasksMock).not.toHaveBeenCalled(); + }); + + it("runs a bounded durable-only sweep with a valid internal token", async () => { + process.env.A2A_SECRET = "test-secret"; + process.env.AGENT_INTEGRATION_DURABLE_DISPATCH = "true"; + const subject = "integration-pending-tasks-sweep"; + const timestamp = Date.now(); + const signature = createHmac("sha256", process.env.A2A_SECRET) + .update(`${subject}:${timestamp}`) + .digest("hex"); + const nitroApp = createNitroApp(); + await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + + const result = await dispatch( + nitroApp, + "/_agent-native/integrations/retry-stuck-tasks", + "POST", + { taskId: subject }, + { authorization: `Bearer ${timestamp}.${signature}` }, + ); + + expect(result.status).toBe(200); + expect(retryStuckPendingTasksMock).toHaveBeenCalledWith({ + webhookBaseUrl: "https://app.test", + limit: 20, + durableOnly: true, + }); + }); + + it("records the durable lease as part of the background worker claim", async () => { + process.env.NODE_ENV = "development"; + claimPendingTaskMock.mockResolvedValueOnce(null); + const nitroApp = createNitroApp(); + await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + + await dispatch( + nitroApp, + "/_agent-native/integrations/process-task", + "POST", + { taskId: "background-task", __agentNativeProcessor: "integration" }, + ); + + expect(claimPendingTaskMock).toHaveBeenCalledWith("background-task", { + dispatchOutcome: "background-acknowledged", + }); + }); + it("loads compact owner resources when processing queued integration tasks", async () => { process.env.NODE_ENV = "development"; claimPendingTaskMock.mockResolvedValueOnce({ diff --git a/packages/core/src/integrations/plugin.ts b/packages/core/src/integrations/plugin.ts index d13170b885..4006f56a45 100644 --- a/packages/core/src/integrations/plugin.ts +++ b/packages/core/src/integrations/plugin.ts @@ -9,6 +9,10 @@ import { import { getRequestHeader } from "h3"; import { createRemoteJWKSet, jwtVerify } from "jose"; +import { + AGENT_BACKGROUND_PROCESSOR_FIELD, + AGENT_BACKGROUND_PROCESSOR_INTEGRATION, +} from "../agent/durable-background.js"; import { abortRun } from "../agent/run-manager.js"; import { getOrgContext, resolveOrgIdForEmail } from "../org/context.js"; import { loadResourcesForPrompt } from "../server/agent-chat-plugin.js"; @@ -65,18 +69,22 @@ import { updateIntegrationInstallation, upsertIntegrationInstallation, } from "./installations-store.js"; +import { + dispatchPendingIntegrationTask, + INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, + isIntegrationDurableDispatchConfigured, +} from "./integration-durable-dispatch.js"; import { forgetIntegrationMemory, integrationMemoryActions, listIntegrationMemory, rememberForIntegrationScope, } from "./integration-memory.js"; +import { extractBearerToken, verifyInternalToken } from "./internal-token.js"; import { - extractBearerToken, - signInternalToken, - verifyInternalToken, -} from "./internal-token.js"; -import { startPendingTasksRetryJob } from "./pending-tasks-retry-job.js"; + retryStuckPendingTasks, + startPendingTasksRetryJob, +} from "./pending-tasks-retry-job.js"; import { claimPendingTask, failTaskDeliveryTransition, @@ -731,34 +739,18 @@ export function createIntegrationsPlugin( throw err; } - // The SQL row is the durable source of truth. This best-effort self-call - // only reduces latency; the recurring pending-task sweep retries a row - // if the serverless host freezes this webhook execution immediately. - let token: string | undefined; - try { - token = signInternalToken(taskId); - } catch (err) { - if (process.env.NODE_ENV === "production") throw err; - } - void fetch(`${getBaseUrl(event)}${P}/process-task`, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), + await dispatchPendingIntegrationTask({ + taskId, + task: { + platform: incoming.platform, + externalThreadId: noticeThreadId, + platformContext: incoming.platformContext, }, - body: JSON.stringify({ taskId }), - }).catch((err) => { - console.warn("[integrations] System notice dispatch failed:", err); + event, + baseUrl: getBaseUrl(event), }); } - async function requireSession(event: any): Promise { - const session = await getSession(event).catch(() => null); - if (session?.email) return true; - setResponseStatus(event, 401); - return false; - } - async function requireSessionContext( event: any, ): Promise<{ ownerEmail: string; orgId: string | null } | null> { @@ -903,9 +895,10 @@ export function createIntegrationsPlugin( setResponseStatus(event, 405); return { error: "Method not allowed" }; } - if (!(await requireSession(event))) return { error: "unauthorized" }; + const scope = await requireSessionContext(event); + if (!scope) return { error: "unauthorized" }; try { - return await getTaskQueueStats(); + return await getTaskQueueStats(scope); } catch (err: any) { setResponseStatus(event, 500); return { error: err?.message ?? String(err) }; @@ -1595,10 +1588,53 @@ export function createIntegrationsPlugin( }), ); + // ─── Durable pending-task recovery sweep ───────────────────── + h3.use( + `${P}/retry-stuck-tasks`, + defineEventHandler(async (event) => { + if (getMethod(event) !== "POST") { + setResponseStatus(event, 405); + return { error: "Method not allowed" }; + } + const body = (await readBody(event)) as { + taskId?: string; + [AGENT_BACKGROUND_PROCESSOR_FIELD]?: string; + }; + if (body?.taskId !== INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT) { + setResponseStatus(event, 400); + return { error: "invalid sweep subject" }; + } + if (!process.env.A2A_SECRET) { + setResponseStatus(event, 503); + return { error: "durable integration recovery is not configured" }; + } + const token = extractBearerToken( + getRequestHeader(event, "authorization"), + ); + if ( + !token || + !verifyInternalToken(INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, token) + ) { + setResponseStatus(event, 401); + return { error: "Invalid or expired internal token" }; + } + if (!isIntegrationDurableDispatchConfigured()) { + return { ok: true, disabled: true }; + } + const result = await retryStuckPendingTasks({ + webhookBaseUrl: getBaseUrl(event), + limit: 20, + durableOnly: true, + }); + return { ok: true, ...result }; + }), + ); + // ─── Process pending task (cross-platform task queue) ──────── // POST /_agent-native/integrations/process-task - // Internal endpoint invoked via fire-and-forget self-webhook from the - // public webhook handler. Auth: HMAC bearer signed with A2A_SECRET. + // Internal endpoint invoked from the public webhook handler through either + // the portable self-dispatch path or an acknowledged background handoff. + // Auth: HMAC bearer signed with A2A_SECRET. // Each invocation runs the agent loop in a fresh function execution. h3.use( `${P}/process-task`, @@ -1608,7 +1644,10 @@ export function createIntegrationsPlugin( return { error: "Method not allowed" }; } - const body = (await readBody(event)) as { taskId?: string }; + const body = (await readBody(event)) as { + taskId?: string; + [AGENT_BACKGROUND_PROCESSOR_FIELD]?: string; + }; const taskId = body?.taskId; if (!taskId) { setResponseStatus(event, 400); @@ -1645,7 +1684,12 @@ export function createIntegrationsPlugin( } // Atomic claim: only one invocation gets to process this task - const task = await claimPendingTask(taskId); + const dispatchOutcome = + body[AGENT_BACKGROUND_PROCESSOR_FIELD] === + AGENT_BACKGROUND_PROCESSOR_INTEGRATION + ? "background-acknowledged" + : "portable-unconfirmed"; + const task = await claimPendingTask(taskId, { dispatchOutcome }); if (!task) { setResponseStatus(event, 200); return { ok: true, skipped: "already-claimed-or-missing" }; @@ -1784,19 +1828,14 @@ export function createIntegrationsPlugin( task.externalThreadId, ); if (nextTaskId) { - const nextToken = signInternalToken(nextTaskId); - void fetch(`${getBaseUrl(event)}${P}/process-task`, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...(nextToken ? { Authorization: `Bearer ${nextToken}` } : {}), + await dispatchPendingIntegrationTask({ + taskId: nextTaskId, + task: { + platform: task.platform, + externalThreadId: task.externalThreadId, }, - body: JSON.stringify({ taskId: nextTaskId }), - }).catch((err) => { - console.error( - "[integrations] Failed to dispatch queued thread turn:", - err, - ); + event, + baseUrl: getBaseUrl(event), }); } await processDueA2AContinuations({ @@ -2019,17 +2058,15 @@ export function createIntegrationsPlugin( orgId: control.orgId, externalEventKey: `control:${control.id}`, }); - const processUrl = `${getBaseUrl(event)}${P}/process-task`; - const token = signInternalToken(taskId); - void fetch(processUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), + await dispatchPendingIntegrationTask({ + taskId, + task: { + platform: incoming.platform, + externalThreadId: incoming.externalThreadId, + platformContext: incoming.platformContext, }, - body: JSON.stringify({ taskId }), - }).catch((err) => { - console.error("[slack] Approval dispatch failed:", err); + event, + baseUrl: getBaseUrl(event), }); } catch (err) { console.error("[slack] Interaction handling failed:", err); @@ -2480,6 +2517,8 @@ export function createIntegrationsPlugin( if (parts[0] === "remote") return; // Already handled by the dedicated /process-task route above if (parts[0] === "process-task") return; + // Already handled by the signed durable recovery route above + if (parts[0] === "retry-stuck-tasks") return; // Already handled by the dedicated /process-a2a-continuation route above if (parts[0] === "process-a2a-continuation") return; // These are framework-owned control-plane routes, not integration diff --git a/packages/core/src/integrations/task-queue-stats.spec.ts b/packages/core/src/integrations/task-queue-stats.spec.ts new file mode 100644 index 0000000000..5cf1964b33 --- /dev/null +++ b/packages/core/src/integrations/task-queue-stats.spec.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const executeMock = vi.hoisted(() => vi.fn()); +const ensureTableMock = vi.hoisted(() => vi.fn()); + +vi.mock("../db/client.js", () => ({ + getDbExec: () => ({ execute: executeMock }), +})); + +vi.mock("./pending-tasks-store.js", () => ({ + ensurePendingTasksTable: ensureTableMock, +})); + +describe("integration task queue stats", () => { + beforeEach(() => { + vi.clearAllMocks(); + ensureTableMock.mockResolvedValue(undefined); + }); + + it("returns dispatch diagnostics without selecting payloads or user text", async () => { + executeMock + .mockResolvedValueOnce({ rows: [{ status: "pending", c: 1 }] }) + .mockResolvedValueOnce({ rows: [{ status: "completed", c: 2 }] }) + .mockResolvedValueOnce({ rows: [{ created_at: Date.now() - 5_000 }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ + rows: [ + { + id: "task-1", + platform: "slack", + status: "pending", + attempts: 0, + dispatch_attempts: 2, + last_dispatch_outcome: "portable-unconfirmed", + created_at: Date.now() - 5_000, + }, + ], + }); + const { getTaskQueueStats } = await import("./task-queue-stats.js"); + + const result = await getTaskQueueStats({ + ownerEmail: "alice@example.com", + orgId: "org-a", + }); + + expect(ensureTableMock).toHaveBeenCalledOnce(); + expect(result.recent_tasks[0]).toEqual( + expect.objectContaining({ + id: "task-1", + platform: "slack", + dispatch_attempts: 2, + last_dispatch_outcome: "portable-unconfirmed", + }), + ); + const sql = executeMock.mock.calls + .map(([query]) => (query as { sql: string }).sql) + .join("\n"); + expect(sql).not.toMatch(/\bpayload\b|external_thread_id/i); + for (const [query] of executeMock.mock.calls) { + expect((query as { args: unknown[] }).args).toEqual( + expect.arrayContaining(["alice@example.com", "org-a"]), + ); + } + }); +}); diff --git a/packages/core/src/integrations/task-queue-stats.ts b/packages/core/src/integrations/task-queue-stats.ts index b9125222a2..53274cb50c 100644 --- a/packages/core/src/integrations/task-queue-stats.ts +++ b/packages/core/src/integrations/task-queue-stats.ts @@ -3,11 +3,12 @@ * * Lives in its own file so it stays out of `pending-tasks-store.ts`, which is * actively being edited by the agent that owns the queue itself. These - * helpers only SELECT — they never write — and they degrade gracefully if - * the `integration_pending_tasks` table doesn't exist yet (returning zeroed - * stats instead of throwing). + * Queue reads never expose payloads or user text. The helper first runs the + * additive schema guard so older deployments gain the dispatch diagnostic + * columns before the SELECTs execute. */ import { getDbExec } from "../db/client.js"; +import { ensurePendingTasksTable } from "./pending-tasks-store.js"; export interface RecentFailure { id: string; @@ -23,6 +24,20 @@ export interface TaskQueueStats { failed_last_hour: number; oldest_pending_age_seconds: number; recent_failures: RecentFailure[]; + recent_tasks: Array<{ + id: string; + platform: string; + status: string; + attempts: number; + dispatch_attempts: number; + last_dispatch_outcome: string | null; + age_seconds: number; + }>; +} + +export interface TaskQueueStatsScope { + ownerEmail: string; + orgId: string | null; } const ZERO_STATS: TaskQueueStats = { @@ -32,6 +47,7 @@ const ZERO_STATS: TaskQueueStats = { failed_last_hour: 0, oldest_pending_age_seconds: 0, recent_failures: [], + recent_tasks: [], }; function isMissingTableError(err: unknown): boolean { @@ -44,21 +60,27 @@ function isMissingTableError(err: unknown): boolean { /** * Get a snapshot of the integration task queue health. * - * Returns zeros if the table doesn't exist yet — safe to call before the - * pending-tasks store has initialised the schema. + * Safe to call before the pending-tasks store has initialized the schema. */ -export async function getTaskQueueStats(): Promise { +export async function getTaskQueueStats( + scope: TaskQueueStatsScope, +): Promise { + await ensurePendingTasksTable(); const client = getDbExec(); const now = Date.now(); const oneHourAgo = now - 60 * 60 * 1000; + const scopeSql = `owner_email = ? + AND ((org_id IS NULL AND CAST(? AS TEXT) IS NULL) OR org_id = ?)`; + const scopeArgs = [scope.ownerEmail, scope.orgId, scope.orgId]; try { // Status counts (pending, processing) — only need the live ones. const liveCounts = await client.execute({ sql: `SELECT status, COUNT(*) AS c FROM integration_pending_tasks - WHERE status IN ('pending', 'processing') + WHERE ${scopeSql} + AND status IN ('pending', 'processing') GROUP BY status`, - args: [], + args: scopeArgs, }); let pending = 0; @@ -75,9 +97,10 @@ export async function getTaskQueueStats(): Promise { // miss tasks queued >1h ago that just finished now. const lastHourCounts = await client.execute({ sql: `SELECT status, COUNT(*) AS c FROM integration_pending_tasks - WHERE status IN ('completed', 'failed') AND updated_at >= ? + WHERE ${scopeSql} + AND status IN ('completed', 'failed') AND updated_at >= ? GROUP BY status`, - args: [oneHourAgo], + args: [...scopeArgs, oneHourAgo], }); let completedLastHour = 0; @@ -94,10 +117,11 @@ export async function getTaskQueueStats(): Promise { if (pending > 0) { const oldest = await client.execute({ sql: `SELECT created_at FROM integration_pending_tasks - WHERE status = 'pending' + WHERE ${scopeSql} + AND status = 'pending' ORDER BY created_at ASC LIMIT 1`, - args: [], + args: scopeArgs, }); const oldestRow = oldest.rows[0] as Record | undefined; if (oldestRow) { @@ -113,10 +137,11 @@ export async function getTaskQueueStats(): Promise { // blowing up the response payload. const failures = await client.execute({ sql: `SELECT id, platform, error_message, attempts FROM integration_pending_tasks - WHERE status = 'failed' AND updated_at >= ? + WHERE ${scopeSql} + AND status = 'failed' AND updated_at >= ? ORDER BY updated_at DESC LIMIT 5`, - args: [oneHourAgo], + args: [...scopeArgs, oneHourAgo], }); const recentFailures: RecentFailure[] = ( failures.rows as Array> @@ -127,6 +152,33 @@ export async function getTaskQueueStats(): Promise { attempts: Number(row.attempts ?? 0), })); + const recent = await client.execute({ + sql: `SELECT id, platform, status, attempts, dispatch_attempts, + last_dispatch_outcome, created_at + FROM integration_pending_tasks + WHERE ${scopeSql} + ORDER BY created_at DESC + LIMIT 10`, + args: scopeArgs, + }); + const recentTasks = (recent.rows as Array>).map( + (row) => ({ + id: String(row.id ?? ""), + platform: String(row.platform ?? ""), + status: String(row.status ?? ""), + attempts: Number(row.attempts ?? 0), + dispatch_attempts: Number(row.dispatch_attempts ?? 0), + last_dispatch_outcome: + row.last_dispatch_outcome == null + ? null + : String(row.last_dispatch_outcome), + age_seconds: Math.max( + 0, + Math.floor((now - Number(row.created_at ?? now)) / 1000), + ), + }), + ); + return { pending, processing, @@ -134,10 +186,11 @@ export async function getTaskQueueStats(): Promise { failed_last_hour: failedLastHour, oldest_pending_age_seconds: oldestPendingAgeSeconds, recent_failures: recentFailures, + recent_tasks: recentTasks, }; } catch (err) { if (isMissingTableError(err)) { - return { ...ZERO_STATS, recent_failures: [] }; + return { ...ZERO_STATS, recent_failures: [], recent_tasks: [] }; } throw err; } diff --git a/packages/core/src/integrations/webhook-handler.spec.ts b/packages/core/src/integrations/webhook-handler.spec.ts index ee9aa14430..f28cdd539e 100644 --- a/packages/core/src/integrations/webhook-handler.spec.ts +++ b/packages/core/src/integrations/webhook-handler.spec.ts @@ -10,6 +10,7 @@ import { const insertPendingTaskMock = vi.hoisted(() => vi.fn()); const isDuplicateEventErrorMock = vi.hoisted(() => vi.fn(() => false)); +const recordDispatchAttemptMock = vi.hoisted(() => vi.fn()); const resolveOrgIdForEmailMock = vi.hoisted(() => vi.fn()); const getOwnerApiKeyMock = vi.hoisted(() => vi.fn()); const getOwnerActiveApiKeyMock = vi.hoisted(() => vi.fn()); @@ -19,6 +20,7 @@ const canUseDeployCredentialFallbackForRequestMock = vi.hoisted(() => vi.fn()); vi.mock("./pending-tasks-store.js", () => ({ insertPendingTask: insertPendingTaskMock, isDuplicateEventError: isDuplicateEventErrorMock, + recordPendingTaskDispatchAttempt: recordDispatchAttemptMock, })); vi.mock("../org/context.js", () => ({ diff --git a/packages/core/src/integrations/webhook-handler.ts b/packages/core/src/integrations/webhook-handler.ts index 2d5781690b..25c2b87df7 100644 --- a/packages/core/src/integrations/webhook-handler.ts +++ b/packages/core/src/integrations/webhook-handler.ts @@ -46,7 +46,6 @@ import { updateThreadData } from "../chat-threads/store.js"; import { isLocalDatabase } from "../db/client.js"; import { getOrgA2ASecret, resolveOrgIdForEmail } from "../org/context.js"; import { withConfiguredAppBasePath } from "../server/app-base-path.js"; -import { FRAMEWORK_ROUTE_PREFIX } from "../server/core-routes-plugin.js"; import { canUseDeployCredentialFallbackForRequest, readDeployCredentialEnv, @@ -58,8 +57,8 @@ import { clearIntegrationAwaitingInput, setIntegrationAwaitingInput, } from "./awaiting-input-store.js"; +import { dispatchPendingIntegrationTask } from "./integration-durable-dispatch.js"; import { loadIntegrationMemoryPrompt } from "./integration-memory.js"; -import { signInternalToken } from "./internal-token.js"; import { insertPendingTask, isDuplicateEventError, @@ -307,7 +306,7 @@ export async function resolveIntegrationApiKey( * 2. Verify webhook signature * 3. Parse incoming message (null = ignored event) * 4. Persist task to SQL - * 5. Fire-and-forget POST to /_agent-native/integrations/process-task + * 5. Dispatch the queued task through the configured processor handoff * (a fresh function execution with its own timeout budget) * 6. Return HTTP 200 immediately (within Slack's 3s SLA) * @@ -480,45 +479,6 @@ async function enqueueAndDispatch( }); const baseUrl = resolveBaseUrl(event); - const processUrl = `${baseUrl}${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`; - - // Sign the dispatch with an HMAC token so the processor endpoint can - // verify the request came from us and not the public internet. The - // processor refuses unsigned requests in production (C3 in the webhook - // security audit). In dev, dispatching unsigned is allowed and falls - // through to the SQL atomic claim for double-processing protection. - const headers: Record = { - "Content-Type": "application/json", - }; - try { - headers["Authorization"] = `Bearer ${signInternalToken(taskId)}`; - } catch (err) { - // Distinguish "secret not configured" (the documented dev path) from - // a real signing failure — silently swallowing both made malformed - // secrets fail invisibly (L5 in the audit). - if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) { - console.error( - `[integrations] signInternalToken failed unexpectedly for ${taskId}:`, - err, - ); - } - } - - // Fire-and-forget: do NOT await the full response (the processor's run - // takes minutes — we don't want to block the caller). BUT on Netlify - // Lambda, when we return immediately, the runtime can freeze the function - // before the outbound TCP handshake even starts, which leaves the dispatch - // request stuck waiting for the 60s retry-sweep job. Race the fetch - // against a short timer so the request gets a reasonable chance to leave - // the box; the trade-off is at most a couple seconds of added webhook - // latency, still inside Slack's timeout window. - const dispatchPromise = fetch(processUrl, { - method: "POST", - headers, - body: JSON.stringify({ taskId }), - }).catch((err) => { - console.error("[integrations] Failed to dispatch processor request:", err); - }); const settleWaitMs = options.adapter.capabilities?.deferredWebhookResponse ? Math.min( DEFERRED_RESPONSE_DISPATCH_SETTLE_WAIT_MS, @@ -528,10 +488,17 @@ async function enqueueAndDispatch( ), ) : PROCESSOR_DISPATCH_SETTLE_WAIT_MS; - await Promise.race([ - dispatchPromise, - new Promise((resolve) => setTimeout(resolve, settleWaitMs)), - ]); + await dispatchPendingIntegrationTask({ + taskId, + task: { + platform: incoming.platform, + externalThreadId: incoming.externalThreadId, + platformContext: incoming.platformContext, + }, + event, + baseUrl, + portableSettleMs: settleWaitMs, + }); } /** diff --git a/packages/core/src/server/auth.spec.ts b/packages/core/src/server/auth.spec.ts index 68cc78ab0e..439bfdc60d 100644 --- a/packages/core/src/server/auth.spec.ts +++ b/packages/core/src/server/auth.spec.ts @@ -901,6 +901,7 @@ describe("server/auth", () => { for (const path of [ "/dispatch/_agent-native/integrations/process-task", + "/dispatch/_agent-native/integrations/retry-stuck-tasks", "/dispatch/_agent-native/integrations/process-a2a-continuation", ]) { const event = createMockEvent({ path }); diff --git a/packages/core/src/server/auth.ts b/packages/core/src/server/auth.ts index 322b3612df..55330cd017 100644 --- a/packages/core/src/server/auth.ts +++ b/packages/core/src/server/auth.ts @@ -1740,6 +1740,12 @@ function createAuthGuardFn(): ( return; } + // External durable-recovery scheduler. The route verifies a short-lived + // HMAC token bound to its fixed sweep subject before touching the queue. + if (p === "/_agent-native/integrations/retry-stuck-tasks") { + return; + } + // Internal processor endpoint for deferred A2A continuations created by // integration tasks. It uses the same HMAC internal-token scheme as the // primary integration processor, so it must bypass cookie/session auth. diff --git a/templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md b/templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md new file mode 100644 index 0000000000..6b4f05e87d --- /dev/null +++ b/templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-21 +--- + +Slack agent requests recover automatically when the first background handoff is interrupted. From 993f266ddf19ed07d102b1c2df9c97dcfba22e37 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Tue, 21 Jul 2026 07:54:12 -0400 Subject: [PATCH 2/4] chore: sync integration webhook skills --- .../skills/integration-webhooks/SKILL.md | 80 ++++++++++++++----- .../skills/integration-webhooks/SKILL.md | 80 ++++++++++++++----- .../skills/integration-webhooks/SKILL.md | 80 ++++++++++++++----- 3 files changed, 174 insertions(+), 66 deletions(-) diff --git a/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md b/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md index 8ee5e79014..46ef55d976 100644 --- a/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +++ b/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md @@ -16,10 +16,10 @@ metadata: Integration webhooks (Slack, Telegram, WhatsApp, email, Google Docs, etc.) must **enqueue work to SQL and return 200 immediately**, then process the work in a -**separate fresh function execution** kicked off by a self-fired HTTP POST. A -recurring retry job sweeps anything that gets stuck. This pattern works on every -serverless host (Netlify, Vercel, Cloudflare Workers, Fly, Render, Node) without -relying on platform-specific background-execution features. +**separate fresh function execution** kicked off by a self-fired HTTP POST. SQL +is always the source of truth. Long-lived hosts use an in-process retry loop as +a best-effort safety net; serverless deployments must use an external durable +sweep when recovery cannot depend on a process staying alive. Do not run agent loops inside the webhook handler itself. Do not rely on fire-and-forget `Promise`s after `return`ing from a serverless handler — they get @@ -38,8 +38,9 @@ Past attempts that don't work cross-host: freeze the execution context the moment the response goes out. The promise is silently killed, the user gets no reply, and there's no error in the logs. -- **Netlify Background Functions** — Netlify-only, requires a `-background` - filename suffix, breaks on every other host. +- **A platform background function as the queue** — host-specific execution is + useful as a delivery mechanism, but it cannot replace the SQL row or atomic + claim shared by every host. - **Cloudflare `event.waitUntil()`** — CF Workers only, not portable. - **Vercel Fluid / `after()`** — Vercel-only, gated behind specific runtimes. - **A long-lived in-process queue** — fine on a single Node box, but on @@ -62,8 +63,9 @@ and a recurring job is the safety net. 2. INSERT INTO integration_pending_tasks (status='pending', payload=...) │ - 3. fetch(POST /integrations/_process-task) - — fire-and-forget, NO await on body + 3. dispatch POST /integrations/process-task + — portable self-fire by default + — acknowledged background handoff when enabled │ 4. return 200 to platform ◄───────────┘ @@ -79,7 +81,7 @@ and a recurring job is the safety net. ┌──────────────────────────────────────────────┐ - │ Recurring job (every 60s) — safety net │ + │ Recovery sweep (every 60s) — safety net │ │ Re-fires processor for tasks stuck in │ │ 'pending' or 'processing' beyond timeout. │ │ Caps retries at 3 then marks 'failed'. │ @@ -98,6 +100,7 @@ loop. | `packages/core/src/integrations/webhook-handler.ts` | Verifies signature, parses, enqueues task, fires processor | | `packages/core/src/integrations/pending-tasks-store.ts` | SQL queue: `insertPendingTask`, `claimPendingTask`, `markTaskCompleted`, `markTaskFailed` | | `packages/core/src/integrations/pending-tasks-retry-job.ts` | Recurring retry sweep (`startPendingTasksRetryJob`, `retryStuckPendingTasks`) | +| `packages/core/src/integrations/integration-durable-dispatch.ts` | Default-off acknowledged dispatch, scoped rollout, and outcome recording | | `packages/core/src/integrations/types.ts` | `PlatformAdapter`, `IncomingMessage`, `OutgoingMessage` | | `packages/core/src/integrations/adapters/{slack,telegram,whatsapp,email,google-docs}.ts` | One adapter per platform | @@ -108,7 +111,8 @@ All under `/_agent-native/integrations/`: | Method | Path | Purpose | | ------ | -------------------------- | ------------------------------------------------------------- | | POST | `/:platform/webhook` | Platform pings this. Verifies, enqueues, returns 200 quickly. | -| POST | `/_process-task` | Self-webhook target. Claims a task and runs the agent loop. | +| POST | `/process-task` | Processor target. Claims a task and runs the agent loop. | +| POST | `/retry-stuck-tasks` | Signed, bounded recovery sweep for durable schedulers. | | GET | `/status` | All integrations status (settings UI). | | GET | `/:platform/status` | One platform's status. | | POST | `/:platform/enable` | Enable an integration. | @@ -129,6 +133,9 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( org_id TEXT, status TEXT NOT NULL, -- pending | processing | completed | failed attempts INTEGER NOT NULL DEFAULT 0, + dispatch_attempts INTEGER NOT NULL DEFAULT 0, + last_dispatch_at INTEGER, + last_dispatch_outcome TEXT, error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,10 +150,36 @@ The store layer creates this lazily on first use via `ensureTable()` and uses `claimPendingTask` is the critical concurrency primitive: it atomically flips `pending` → `processing` and increments `attempts`, returning `null` if another -worker beat us to it. Both the initial fire-and-forget call and the retry job +worker beat us to it. Both the initial dispatch and every recovery sweep funnel through the same processor endpoint, and `claimPendingTask` is what prevents the same task from being processed twice. +### Netlify durable recovery + +The portable self-dispatch remains the default. To emit and use Netlify's +acknowledged background worker plus a one-minute scheduled recovery function, +set `AGENT_INTEGRATION_DURABLE_DISPATCH=true` at build and runtime. A production +rollout can be narrowed with the comma-separated +`AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are +`:*`, `:`, and, for adapters that expose +one, `:` (for example `slack:C123`). + +Workspace deploys emit this pair only for the Dispatch control-plane app, which +owns workspace messaging integrations. Standalone templates emit their own +pair when the same flag is enabled. + +The background handoff waits only for Netlify's enqueue acknowledgement, not +for the agent loop. A failed or non-2xx acknowledgement falls back to the +portable processor. The scheduled function calls the HMAC-authenticated, +bounded `/retry-stuck-tasks` route; its atomic compare-and-set update prevents +overlapping sweeps from dispatching the same stale row twice. +Rows acknowledged by the 15-minute background worker use a 16-minute stale +cutoff, while portable synchronous dispatches retain the shorter serverless +cutoff. Apply that lease predicate before `LIMIT` so healthy long runs cannot +hide recoverable work. The background processor writes its lease marker in the +same atomic update that claims the row; best-effort dispatch telemetry is not +trusted for replay safety. + ## Adding a New Platform Adapter 1. **Implement `PlatformAdapter`** in `packages/core/src/integrations/adapters/.ts`: @@ -245,9 +278,10 @@ so a normal long-running reply is safe. ## Cross-Platform Considerations -- **No platform-specific background APIs.** No `waitUntil`, no - `-background.ts` filenames, no Vercel `after()`. The pattern works - identically on every host because it only uses `fetch()` and SQL. +- **Portable correctness, optional host acceleration.** Every host uses SQL, + HMAC-authenticated processor routes, and atomic claims. A host-specific + background function or scheduler may provide a more reliable wake-up, but + disabling it must preserve the portable path. - **No assumed runtime.** The processor endpoint is a normal H3 handler under `/_agent-native/`. It runs wherever the rest of the framework runs. - **No persistent in-memory state.** The dedup map in the webhook handler is @@ -262,11 +296,12 @@ so a normal long-running reply is safe. ## Why Fire-and-Forget on Serverless Is Unreliable -Even though the webhook handler does `fetch(processorUrl, ...)` without -awaiting the response body, that initial dispatch is **not** guaranteed to -complete before the function freezes. In practice it usually does — the TCP -connect + write happens quickly — but the recurring retry job is the safety -net for the cases where: +The portable webhook path gives the outbound processor request a short head +start but does not wait for the response body, so that initial dispatch is +**not** guaranteed to complete before the function freezes. On a long-lived +host, the in-process retry loop is a best-effort safety net. On Netlify with +durable dispatch enabled, the initial handoff is acknowledged and the external +scheduled function supplies the recovery wake-up for cases where: - The serverless platform froze the handler before the outbound `fetch` flushed its bytes. @@ -278,8 +313,9 @@ Tasks stuck in `pending` for >90s or `processing` for >5min get re-fired up to 3 times. After 3 attempts they're marked `failed` permanently so we stop spamming the processor. -**Never assume the initial fire-and-forget succeeded.** Always rely on the -queue + retry job for at-least-once delivery. +**Never assume the initial dispatch succeeded.** Rely on the SQL queue, atomic +claim, and a recovery mechanism whose lifetime is independent of the original +request. An in-process timer alone is not that mechanism on serverless. ## Debugging Checklist @@ -292,7 +328,7 @@ queue + retry job for at-least-once delivery. 4. **Status?** `pending` means the processor never picked it up — check that `_process-task` is reachable from the box itself (the self-fetch must work over the public URL). `processing` for over 5 minutes means the processor - died mid-run — the retry job will pick it up. + died mid-run — a live recovery sweep will pick it up. 5. **Failed?** Check `error_message` and `attempts`. After 3 attempts the row is parked at `failed` and won't be retried. 6. **Reply not delivered?** The processor likely succeeded but diff --git a/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md b/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md index 8ee5e79014..46ef55d976 100644 --- a/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +++ b/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md @@ -16,10 +16,10 @@ metadata: Integration webhooks (Slack, Telegram, WhatsApp, email, Google Docs, etc.) must **enqueue work to SQL and return 200 immediately**, then process the work in a -**separate fresh function execution** kicked off by a self-fired HTTP POST. A -recurring retry job sweeps anything that gets stuck. This pattern works on every -serverless host (Netlify, Vercel, Cloudflare Workers, Fly, Render, Node) without -relying on platform-specific background-execution features. +**separate fresh function execution** kicked off by a self-fired HTTP POST. SQL +is always the source of truth. Long-lived hosts use an in-process retry loop as +a best-effort safety net; serverless deployments must use an external durable +sweep when recovery cannot depend on a process staying alive. Do not run agent loops inside the webhook handler itself. Do not rely on fire-and-forget `Promise`s after `return`ing from a serverless handler — they get @@ -38,8 +38,9 @@ Past attempts that don't work cross-host: freeze the execution context the moment the response goes out. The promise is silently killed, the user gets no reply, and there's no error in the logs. -- **Netlify Background Functions** — Netlify-only, requires a `-background` - filename suffix, breaks on every other host. +- **A platform background function as the queue** — host-specific execution is + useful as a delivery mechanism, but it cannot replace the SQL row or atomic + claim shared by every host. - **Cloudflare `event.waitUntil()`** — CF Workers only, not portable. - **Vercel Fluid / `after()`** — Vercel-only, gated behind specific runtimes. - **A long-lived in-process queue** — fine on a single Node box, but on @@ -62,8 +63,9 @@ and a recurring job is the safety net. 2. INSERT INTO integration_pending_tasks (status='pending', payload=...) │ - 3. fetch(POST /integrations/_process-task) - — fire-and-forget, NO await on body + 3. dispatch POST /integrations/process-task + — portable self-fire by default + — acknowledged background handoff when enabled │ 4. return 200 to platform ◄───────────┘ @@ -79,7 +81,7 @@ and a recurring job is the safety net. ┌──────────────────────────────────────────────┐ - │ Recurring job (every 60s) — safety net │ + │ Recovery sweep (every 60s) — safety net │ │ Re-fires processor for tasks stuck in │ │ 'pending' or 'processing' beyond timeout. │ │ Caps retries at 3 then marks 'failed'. │ @@ -98,6 +100,7 @@ loop. | `packages/core/src/integrations/webhook-handler.ts` | Verifies signature, parses, enqueues task, fires processor | | `packages/core/src/integrations/pending-tasks-store.ts` | SQL queue: `insertPendingTask`, `claimPendingTask`, `markTaskCompleted`, `markTaskFailed` | | `packages/core/src/integrations/pending-tasks-retry-job.ts` | Recurring retry sweep (`startPendingTasksRetryJob`, `retryStuckPendingTasks`) | +| `packages/core/src/integrations/integration-durable-dispatch.ts` | Default-off acknowledged dispatch, scoped rollout, and outcome recording | | `packages/core/src/integrations/types.ts` | `PlatformAdapter`, `IncomingMessage`, `OutgoingMessage` | | `packages/core/src/integrations/adapters/{slack,telegram,whatsapp,email,google-docs}.ts` | One adapter per platform | @@ -108,7 +111,8 @@ All under `/_agent-native/integrations/`: | Method | Path | Purpose | | ------ | -------------------------- | ------------------------------------------------------------- | | POST | `/:platform/webhook` | Platform pings this. Verifies, enqueues, returns 200 quickly. | -| POST | `/_process-task` | Self-webhook target. Claims a task and runs the agent loop. | +| POST | `/process-task` | Processor target. Claims a task and runs the agent loop. | +| POST | `/retry-stuck-tasks` | Signed, bounded recovery sweep for durable schedulers. | | GET | `/status` | All integrations status (settings UI). | | GET | `/:platform/status` | One platform's status. | | POST | `/:platform/enable` | Enable an integration. | @@ -129,6 +133,9 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( org_id TEXT, status TEXT NOT NULL, -- pending | processing | completed | failed attempts INTEGER NOT NULL DEFAULT 0, + dispatch_attempts INTEGER NOT NULL DEFAULT 0, + last_dispatch_at INTEGER, + last_dispatch_outcome TEXT, error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,10 +150,36 @@ The store layer creates this lazily on first use via `ensureTable()` and uses `claimPendingTask` is the critical concurrency primitive: it atomically flips `pending` → `processing` and increments `attempts`, returning `null` if another -worker beat us to it. Both the initial fire-and-forget call and the retry job +worker beat us to it. Both the initial dispatch and every recovery sweep funnel through the same processor endpoint, and `claimPendingTask` is what prevents the same task from being processed twice. +### Netlify durable recovery + +The portable self-dispatch remains the default. To emit and use Netlify's +acknowledged background worker plus a one-minute scheduled recovery function, +set `AGENT_INTEGRATION_DURABLE_DISPATCH=true` at build and runtime. A production +rollout can be narrowed with the comma-separated +`AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are +`:*`, `:`, and, for adapters that expose +one, `:` (for example `slack:C123`). + +Workspace deploys emit this pair only for the Dispatch control-plane app, which +owns workspace messaging integrations. Standalone templates emit their own +pair when the same flag is enabled. + +The background handoff waits only for Netlify's enqueue acknowledgement, not +for the agent loop. A failed or non-2xx acknowledgement falls back to the +portable processor. The scheduled function calls the HMAC-authenticated, +bounded `/retry-stuck-tasks` route; its atomic compare-and-set update prevents +overlapping sweeps from dispatching the same stale row twice. +Rows acknowledged by the 15-minute background worker use a 16-minute stale +cutoff, while portable synchronous dispatches retain the shorter serverless +cutoff. Apply that lease predicate before `LIMIT` so healthy long runs cannot +hide recoverable work. The background processor writes its lease marker in the +same atomic update that claims the row; best-effort dispatch telemetry is not +trusted for replay safety. + ## Adding a New Platform Adapter 1. **Implement `PlatformAdapter`** in `packages/core/src/integrations/adapters/.ts`: @@ -245,9 +278,10 @@ so a normal long-running reply is safe. ## Cross-Platform Considerations -- **No platform-specific background APIs.** No `waitUntil`, no - `-background.ts` filenames, no Vercel `after()`. The pattern works - identically on every host because it only uses `fetch()` and SQL. +- **Portable correctness, optional host acceleration.** Every host uses SQL, + HMAC-authenticated processor routes, and atomic claims. A host-specific + background function or scheduler may provide a more reliable wake-up, but + disabling it must preserve the portable path. - **No assumed runtime.** The processor endpoint is a normal H3 handler under `/_agent-native/`. It runs wherever the rest of the framework runs. - **No persistent in-memory state.** The dedup map in the webhook handler is @@ -262,11 +296,12 @@ so a normal long-running reply is safe. ## Why Fire-and-Forget on Serverless Is Unreliable -Even though the webhook handler does `fetch(processorUrl, ...)` without -awaiting the response body, that initial dispatch is **not** guaranteed to -complete before the function freezes. In practice it usually does — the TCP -connect + write happens quickly — but the recurring retry job is the safety -net for the cases where: +The portable webhook path gives the outbound processor request a short head +start but does not wait for the response body, so that initial dispatch is +**not** guaranteed to complete before the function freezes. On a long-lived +host, the in-process retry loop is a best-effort safety net. On Netlify with +durable dispatch enabled, the initial handoff is acknowledged and the external +scheduled function supplies the recovery wake-up for cases where: - The serverless platform froze the handler before the outbound `fetch` flushed its bytes. @@ -278,8 +313,9 @@ Tasks stuck in `pending` for >90s or `processing` for >5min get re-fired up to 3 times. After 3 attempts they're marked `failed` permanently so we stop spamming the processor. -**Never assume the initial fire-and-forget succeeded.** Always rely on the -queue + retry job for at-least-once delivery. +**Never assume the initial dispatch succeeded.** Rely on the SQL queue, atomic +claim, and a recovery mechanism whose lifetime is independent of the original +request. An in-process timer alone is not that mechanism on serverless. ## Debugging Checklist @@ -292,7 +328,7 @@ queue + retry job for at-least-once delivery. 4. **Status?** `pending` means the processor never picked it up — check that `_process-task` is reachable from the box itself (the self-fetch must work over the public URL). `processing` for over 5 minutes means the processor - died mid-run — the retry job will pick it up. + died mid-run — a live recovery sweep will pick it up. 5. **Failed?** Check `error_message` and `attempts`. After 3 attempts the row is parked at `failed` and won't be retried. 6. **Reply not delivered?** The processor likely succeeded but diff --git a/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md b/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md index 8ee5e79014..46ef55d976 100644 --- a/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +++ b/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md @@ -16,10 +16,10 @@ metadata: Integration webhooks (Slack, Telegram, WhatsApp, email, Google Docs, etc.) must **enqueue work to SQL and return 200 immediately**, then process the work in a -**separate fresh function execution** kicked off by a self-fired HTTP POST. A -recurring retry job sweeps anything that gets stuck. This pattern works on every -serverless host (Netlify, Vercel, Cloudflare Workers, Fly, Render, Node) without -relying on platform-specific background-execution features. +**separate fresh function execution** kicked off by a self-fired HTTP POST. SQL +is always the source of truth. Long-lived hosts use an in-process retry loop as +a best-effort safety net; serverless deployments must use an external durable +sweep when recovery cannot depend on a process staying alive. Do not run agent loops inside the webhook handler itself. Do not rely on fire-and-forget `Promise`s after `return`ing from a serverless handler — they get @@ -38,8 +38,9 @@ Past attempts that don't work cross-host: freeze the execution context the moment the response goes out. The promise is silently killed, the user gets no reply, and there's no error in the logs. -- **Netlify Background Functions** — Netlify-only, requires a `-background` - filename suffix, breaks on every other host. +- **A platform background function as the queue** — host-specific execution is + useful as a delivery mechanism, but it cannot replace the SQL row or atomic + claim shared by every host. - **Cloudflare `event.waitUntil()`** — CF Workers only, not portable. - **Vercel Fluid / `after()`** — Vercel-only, gated behind specific runtimes. - **A long-lived in-process queue** — fine on a single Node box, but on @@ -62,8 +63,9 @@ and a recurring job is the safety net. 2. INSERT INTO integration_pending_tasks (status='pending', payload=...) │ - 3. fetch(POST /integrations/_process-task) - — fire-and-forget, NO await on body + 3. dispatch POST /integrations/process-task + — portable self-fire by default + — acknowledged background handoff when enabled │ 4. return 200 to platform ◄───────────┘ @@ -79,7 +81,7 @@ and a recurring job is the safety net. ┌──────────────────────────────────────────────┐ - │ Recurring job (every 60s) — safety net │ + │ Recovery sweep (every 60s) — safety net │ │ Re-fires processor for tasks stuck in │ │ 'pending' or 'processing' beyond timeout. │ │ Caps retries at 3 then marks 'failed'. │ @@ -98,6 +100,7 @@ loop. | `packages/core/src/integrations/webhook-handler.ts` | Verifies signature, parses, enqueues task, fires processor | | `packages/core/src/integrations/pending-tasks-store.ts` | SQL queue: `insertPendingTask`, `claimPendingTask`, `markTaskCompleted`, `markTaskFailed` | | `packages/core/src/integrations/pending-tasks-retry-job.ts` | Recurring retry sweep (`startPendingTasksRetryJob`, `retryStuckPendingTasks`) | +| `packages/core/src/integrations/integration-durable-dispatch.ts` | Default-off acknowledged dispatch, scoped rollout, and outcome recording | | `packages/core/src/integrations/types.ts` | `PlatformAdapter`, `IncomingMessage`, `OutgoingMessage` | | `packages/core/src/integrations/adapters/{slack,telegram,whatsapp,email,google-docs}.ts` | One adapter per platform | @@ -108,7 +111,8 @@ All under `/_agent-native/integrations/`: | Method | Path | Purpose | | ------ | -------------------------- | ------------------------------------------------------------- | | POST | `/:platform/webhook` | Platform pings this. Verifies, enqueues, returns 200 quickly. | -| POST | `/_process-task` | Self-webhook target. Claims a task and runs the agent loop. | +| POST | `/process-task` | Processor target. Claims a task and runs the agent loop. | +| POST | `/retry-stuck-tasks` | Signed, bounded recovery sweep for durable schedulers. | | GET | `/status` | All integrations status (settings UI). | | GET | `/:platform/status` | One platform's status. | | POST | `/:platform/enable` | Enable an integration. | @@ -129,6 +133,9 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( org_id TEXT, status TEXT NOT NULL, -- pending | processing | completed | failed attempts INTEGER NOT NULL DEFAULT 0, + dispatch_attempts INTEGER NOT NULL DEFAULT 0, + last_dispatch_at INTEGER, + last_dispatch_outcome TEXT, error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,10 +150,36 @@ The store layer creates this lazily on first use via `ensureTable()` and uses `claimPendingTask` is the critical concurrency primitive: it atomically flips `pending` → `processing` and increments `attempts`, returning `null` if another -worker beat us to it. Both the initial fire-and-forget call and the retry job +worker beat us to it. Both the initial dispatch and every recovery sweep funnel through the same processor endpoint, and `claimPendingTask` is what prevents the same task from being processed twice. +### Netlify durable recovery + +The portable self-dispatch remains the default. To emit and use Netlify's +acknowledged background worker plus a one-minute scheduled recovery function, +set `AGENT_INTEGRATION_DURABLE_DISPATCH=true` at build and runtime. A production +rollout can be narrowed with the comma-separated +`AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are +`:*`, `:`, and, for adapters that expose +one, `:` (for example `slack:C123`). + +Workspace deploys emit this pair only for the Dispatch control-plane app, which +owns workspace messaging integrations. Standalone templates emit their own +pair when the same flag is enabled. + +The background handoff waits only for Netlify's enqueue acknowledgement, not +for the agent loop. A failed or non-2xx acknowledgement falls back to the +portable processor. The scheduled function calls the HMAC-authenticated, +bounded `/retry-stuck-tasks` route; its atomic compare-and-set update prevents +overlapping sweeps from dispatching the same stale row twice. +Rows acknowledged by the 15-minute background worker use a 16-minute stale +cutoff, while portable synchronous dispatches retain the shorter serverless +cutoff. Apply that lease predicate before `LIMIT` so healthy long runs cannot +hide recoverable work. The background processor writes its lease marker in the +same atomic update that claims the row; best-effort dispatch telemetry is not +trusted for replay safety. + ## Adding a New Platform Adapter 1. **Implement `PlatformAdapter`** in `packages/core/src/integrations/adapters/.ts`: @@ -245,9 +278,10 @@ so a normal long-running reply is safe. ## Cross-Platform Considerations -- **No platform-specific background APIs.** No `waitUntil`, no - `-background.ts` filenames, no Vercel `after()`. The pattern works - identically on every host because it only uses `fetch()` and SQL. +- **Portable correctness, optional host acceleration.** Every host uses SQL, + HMAC-authenticated processor routes, and atomic claims. A host-specific + background function or scheduler may provide a more reliable wake-up, but + disabling it must preserve the portable path. - **No assumed runtime.** The processor endpoint is a normal H3 handler under `/_agent-native/`. It runs wherever the rest of the framework runs. - **No persistent in-memory state.** The dedup map in the webhook handler is @@ -262,11 +296,12 @@ so a normal long-running reply is safe. ## Why Fire-and-Forget on Serverless Is Unreliable -Even though the webhook handler does `fetch(processorUrl, ...)` without -awaiting the response body, that initial dispatch is **not** guaranteed to -complete before the function freezes. In practice it usually does — the TCP -connect + write happens quickly — but the recurring retry job is the safety -net for the cases where: +The portable webhook path gives the outbound processor request a short head +start but does not wait for the response body, so that initial dispatch is +**not** guaranteed to complete before the function freezes. On a long-lived +host, the in-process retry loop is a best-effort safety net. On Netlify with +durable dispatch enabled, the initial handoff is acknowledged and the external +scheduled function supplies the recovery wake-up for cases where: - The serverless platform froze the handler before the outbound `fetch` flushed its bytes. @@ -278,8 +313,9 @@ Tasks stuck in `pending` for >90s or `processing` for >5min get re-fired up to 3 times. After 3 attempts they're marked `failed` permanently so we stop spamming the processor. -**Never assume the initial fire-and-forget succeeded.** Always rely on the -queue + retry job for at-least-once delivery. +**Never assume the initial dispatch succeeded.** Rely on the SQL queue, atomic +claim, and a recovery mechanism whose lifetime is independent of the original +request. An in-process timer alone is not that mechanism on serverless. ## Debugging Checklist @@ -292,7 +328,7 @@ queue + retry job for at-least-once delivery. 4. **Status?** `pending` means the processor never picked it up — check that `_process-task` is reachable from the box itself (the self-fetch must work over the public URL). `processing` for over 5 minutes means the processor - died mid-run — the retry job will pick it up. + died mid-run — a live recovery sweep will pick it up. 5. **Failed?** Check `error_message` and `attempts`. After 3 attempts the row is parked at `failed` and won't be retried. 6. **Reply not delivered?** The processor likely succeeded but From 34abbd40d965203f5348e5066088f2d0910116fa Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:15:27 -0400 Subject: [PATCH 3/4] fix: harden durable recovery configuration --- .agents/skills/integration-webhooks/SKILL.md | 5 ++ packages/core/src/deploy/build.spec.ts | 18 +++++- packages/core/src/deploy/build.ts | 11 ++-- .../core/src/deploy/workspace-deploy.spec.ts | 10 ++++ packages/core/src/deploy/workspace-deploy.ts | 11 ++-- .../integration-durable-dispatch.ts | 13 +++- .../pending-tasks-retry-job.spec.ts | 60 ++++++++++++++++++- .../integrations/pending-tasks-retry-job.ts | 25 ++++++-- .../integrations/pending-tasks-store.spec.ts | 38 ++++++++++-- .../src/integrations/pending-tasks-store.ts | 39 +++++++++--- packages/core/src/integrations/plugin.spec.ts | 47 ++++++++++++++- packages/core/src/integrations/plugin.ts | 22 +++++-- .../core/src/integrations/webhook-handler.ts | 10 +++- .../skills/integration-webhooks/SKILL.md | 5 ++ .../skills/integration-webhooks/SKILL.md | 5 ++ .../skills/integration-webhooks/SKILL.md | 5 ++ 16 files changed, 287 insertions(+), 37 deletions(-) diff --git a/.agents/skills/integration-webhooks/SKILL.md b/.agents/skills/integration-webhooks/SKILL.md index 46ef55d976..da4fbbc9b1 100644 --- a/.agents/skills/integration-webhooks/SKILL.md +++ b/.agents/skills/integration-webhooks/SKILL.md @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( dispatch_attempts INTEGER NOT NULL DEFAULT 0, last_dispatch_at INTEGER, last_dispatch_outcome TEXT, + dispatch_scope TEXT, -- persisted channel scope for recovery error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,6 +144,8 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( ); CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at); +CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope + ON integration_pending_tasks(platform, dispatch_scope); ``` The store layer creates this lazily on first use via `ensureTable()` and uses @@ -163,6 +166,8 @@ rollout can be narrowed with the comma-separated `AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are `:*`, `:`, and, for adapters that expose one, `:` (for example `slack:C123`). +Channel-scoped handoffs persist that scope separately from the provider thread +identity so a later bounded recovery sweep applies the same allowlist decision. Workspace deploys emit this pair only for the Dispatch control-plane app, which owns workspace messaging integrations. Standalone templates emit their own diff --git a/packages/core/src/deploy/build.spec.ts b/packages/core/src/deploy/build.spec.ts index 065b3c94e5..2b8ed6b887 100644 --- a/packages/core/src/deploy/build.spec.ts +++ b/packages/core/src/deploy/build.spec.ts @@ -2,7 +2,7 @@ import fs from "fs"; import path from "path"; import { pathToFileURL } from "url"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { AGENT_CHAT_PROCESS_RUN_PATH } from "../agent/durable-background.js"; import { @@ -1546,6 +1546,22 @@ describe("durable-background Netlify function emit (single-template, flag-gated) `${pathToFileURL(path.join(dest, "server-integration-recovery.mjs")).href}?t=${Date.now()}` ); expect(generated.config.schedule).toBe("* * * * *"); + process.env.AGENT_INTEGRATION_DURABLE_DISPATCH = "true"; + delete process.env.A2A_SECRET; + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const response = await generated.default( + new Request("https://app.test/.netlify/functions/recovery"), + {}, + ); + expect(response.status).toBe(204); + expect(consoleSpy).toHaveBeenCalledWith( + "[integration-recovery] A2A_SECRET is required; sweep skipped", + ); + } finally { + consoleSpy.mockRestore(); + delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + } }); it("is OFF BY DEFAULT (flag unset) so the -background function is NOT emitted", () => { diff --git a/packages/core/src/deploy/build.ts b/packages/core/src/deploy/build.ts index 6c6ec4dd0f..11e74b6f11 100644 --- a/packages/core/src/deploy/build.ts +++ b/packages/core/src/deploy/build.ts @@ -2613,9 +2613,7 @@ function enabled() { return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase()); } -function token() { - const secret = process.env.A2A_SECRET; - if (!secret) throw new Error("A2A_SECRET is required for integration recovery"); +function token(secret) { const timestamp = Date.now(); const signature = createHmac("sha256", secret) .update(\`${INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT}:\${timestamp}\`) @@ -2627,13 +2625,18 @@ let cachedHandler; export default async function handler(request, context) { if (!enabled()) return new Response(null, { status: 204 }); + const secret = process.env.A2A_SECRET; + if (!secret) { + console.error("[integration-recovery] A2A_SECRET is required; sweep skipped"); + return new Response(null, { status: 204 }); + } cachedHandler ??= (await import("./main.mjs")).default; const url = new URL(request.url); url.pathname = SWEEP_PATH; const rewritten = new Request(url.toString(), { method: "POST", headers: { - Authorization: \`Bearer \${token()}\`, + Authorization: \`Bearer \${token(secret)}\`, "Content-Type": "application/json", }, body: JSON.stringify({ taskId: SWEEP_SUBJECT }), diff --git a/packages/core/src/deploy/workspace-deploy.spec.ts b/packages/core/src/deploy/workspace-deploy.spec.ts index 6d92eda34b..ea1e7f67b9 100644 --- a/packages/core/src/deploy/workspace-deploy.spec.ts +++ b/packages/core/src/deploy/workspace-deploy.spec.ts @@ -1238,6 +1238,16 @@ describe("durable-background Netlify function emit (workspace, flag-gated)", () `${pathToFileURL(path.join(recoveryDir, "dispatch-integration-recovery.mjs")).href}?t=${Date.now()}` ); expect(generated.config.schedule).toBe("* * * * *"); + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const response = await generated.default( + new Request("https://app.test/.netlify/functions/recovery"), + {}, + ); + expect(response.status).toBe(204); + expect(consoleSpy).toHaveBeenCalledWith( + "[integration-recovery] A2A_SECRET is required; sweep skipped", + ); + consoleSpy.mockRestore(); expect(fs.existsSync(backgroundFuncDir("starter"))).toBe(false); expect( fs.existsSync( diff --git a/packages/core/src/deploy/workspace-deploy.ts b/packages/core/src/deploy/workspace-deploy.ts index 3ae0c5a3ea..818b32372a 100644 --- a/packages/core/src/deploy/workspace-deploy.ts +++ b/packages/core/src/deploy/workspace-deploy.ts @@ -939,9 +939,7 @@ function enabled() { return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase()); } -function token() { - const secret = process.env.A2A_SECRET; - if (!secret) throw new Error("A2A_SECRET is required for integration recovery"); +function token(secret) { const timestamp = Date.now(); const signature = createHmac("sha256", secret) .update(\`${INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT}:\${timestamp}\`) @@ -955,13 +953,18 @@ let cachedHandler; export default async function handler(request, context) { setBasePathEnv(); if (!enabled()) return new Response(null, { status: 204 }); + const secret = process.env.A2A_SECRET; + if (!secret) { + console.error("[integration-recovery] A2A_SECRET is required; sweep skipped"); + return new Response(null, { status: 204 }); + } cachedHandler ??= (await import("./main.mjs")).default; const url = new URL(request.url); url.pathname = SWEEP_PATH; const rewritten = new Request(url.toString(), { method: "POST", headers: { - Authorization: \`Bearer \${token()}\`, + Authorization: \`Bearer \${token(secret)}\`, "Content-Type": "application/json", }, body: JSON.stringify({ taskId: SWEEP_SUBJECT }), diff --git a/packages/core/src/integrations/integration-durable-dispatch.ts b/packages/core/src/integrations/integration-durable-dispatch.ts index bd698292f6..2fb9d59b70 100644 --- a/packages/core/src/integrations/integration-durable-dispatch.ts +++ b/packages/core/src/integrations/integration-durable-dispatch.ts @@ -41,6 +41,13 @@ export interface IntegrationDurableDispatchScope { value: string; } +export function integrationDispatchScopeValue( + task: IntegrationDispatchTaskScope, +): string | null { + const value = task.platformContext?.channelId; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + export function configuredIntegrationDurableDispatchScopes(): | IntegrationDurableDispatchScope[] | null { @@ -65,9 +72,9 @@ function taskScopeCandidates(task: IntegrationDispatchTaskScope): string[] { `${task.platform}:*`, `${task.platform}:${task.externalThreadId}`, ]; - const explicitScope = task.platformContext?.channelId; - if (typeof explicitScope === "string" && explicitScope.trim()) { - candidates.push(`${task.platform}:${explicitScope.trim()}`); + const explicitScope = integrationDispatchScopeValue(task); + if (explicitScope) { + candidates.push(`${task.platform}:${explicitScope}`); } else if (task.platform === "slack") { const channelId = task.externalThreadId.split(":")[2]; if (channelId) candidates.push(`slack:${channelId}`); diff --git a/packages/core/src/integrations/pending-tasks-retry-job.spec.ts b/packages/core/src/integrations/pending-tasks-retry-job.spec.ts index 06de104574..95973b5873 100644 --- a/packages/core/src/integrations/pending-tasks-retry-job.spec.ts +++ b/packages/core/src/integrations/pending-tasks-retry-job.spec.ts @@ -257,7 +257,13 @@ describe("pending task retry job", () => { args: unknown[]; }; expect(select.sql).toContain("external_thread_id LIKE ?"); - expect(select.args.slice(-4)).toEqual(["slack", "C123", "%:%:C123:%", 1]); + expect(select.args.slice(-5)).toEqual([ + "slack", + "C123", + "C123", + "%:%:C123:%", + 1, + ]); expect(result).toEqual({ selected: 1, dispatched: 1, @@ -270,6 +276,58 @@ describe("pending task retry job", () => { ); }); + it("preserves a non-Slack channel scope through durable recovery", async () => { + configuredScopesMock.mockReturnValue([ + { platform: "microsoft-teams", value: "channel-7" }, + ]); + durableEnabledMock.mockImplementation( + ({ platformContext }) => platformContext?.channelId === "channel-7", + ); + const { retryStuckPendingTasks } = await loadRetryJob(); + executeMock + .mockResolvedValueOnce({ + rows: [ + { + id: "teams-task", + platform: "microsoft-teams", + external_thread_id: "conversation-9", + dispatch_scope: "channel-7", + status: "pending", + attempts: 0, + updated_at: 60, + }, + ], + }) + .mockResolvedValueOnce({ rows: [], rowCount: 1 }); + + const result = await retryStuckPendingTasks({ + webhookBaseUrl: "https://app.test", + durableOnly: true, + limit: 1, + }); + + const select = executeMock.mock.calls[0]?.[0] as { + sql: string; + args: unknown[]; + }; + expect(select.sql).toContain("dispatch_scope = ?"); + expect(select.args.slice(-4)).toEqual([ + "microsoft-teams", + "channel-7", + "channel-7", + 1, + ]); + expect(result.dispatched).toBe(1); + expect(dispatchPendingTaskMock).toHaveBeenCalledWith( + expect.objectContaining({ + taskId: "teams-task", + task: expect.objectContaining({ + platformContext: { channelId: "channel-7" }, + }), + }), + ); + }); + it("does not reclaim a healthy durable background task at the synchronous cutoff", async () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-07-21T12:00:00.000Z")); diff --git a/packages/core/src/integrations/pending-tasks-retry-job.ts b/packages/core/src/integrations/pending-tasks-retry-job.ts index 2022ed9d84..dc7ce9ad50 100644 --- a/packages/core/src/integrations/pending-tasks-retry-job.ts +++ b/packages/core/src/integrations/pending-tasks-retry-job.ts @@ -64,6 +64,7 @@ interface StuckTaskRow { status: string; attempts: number; updatedAt: number; + dispatchScope: string | null; } export interface PendingTasksSweepResult { @@ -110,13 +111,20 @@ function durableScopeSql(): { clause: string; args: string[] } { } if (scope.platform === "slack" && /^[A-Za-z0-9]+$/.test(scope.value)) { clauses.push( - "(platform = ? AND (external_thread_id = ? OR external_thread_id LIKE ?))", + "(platform = ? AND (external_thread_id = ? OR dispatch_scope = ? OR external_thread_id LIKE ?))", + ); + args.push( + scope.platform, + scope.value, + scope.value, + `%:%:${scope.value}:%`, ); - args.push(scope.platform, scope.value, `%:%:${scope.value}:%`); continue; } - clauses.push("(platform = ? AND external_thread_id = ?)"); - args.push(scope.platform, scope.value); + clauses.push( + "(platform = ? AND (external_thread_id = ? OR dispatch_scope = ?))", + ); + args.push(scope.platform, scope.value, scope.value); } return { clause: ` AND (${clauses.join(" OR ")})`, args }; } @@ -157,7 +165,7 @@ export async function retryStuckPendingTasks( try { const { rows } = await client.execute({ sql: ` - SELECT id, platform, external_thread_id, status, attempts, updated_at + SELECT id, platform, external_thread_id, dispatch_scope, status, attempts, updated_at FROM integration_pending_tasks WHERE ((status = 'pending' AND created_at <= ? AND updated_at <= ?) OR (status = 'processing' AND ( @@ -191,6 +199,7 @@ export async function retryStuckPendingTasks( status: r.status as string, attempts: Number(r.attempts ?? 0), updatedAt: Number(r.updated_at ?? 0), + dispatchScope: (r.dispatch_scope as string | null) ?? null, })); tableExists = true; } catch (err) { @@ -210,6 +219,9 @@ export async function retryStuckPendingTasks( const durable = isIntegrationDurableDispatchEnabledForTask({ platform: row.platform, externalThreadId: row.externalThreadId, + platformContext: row.dispatchScope + ? { channelId: row.dispatchScope } + : undefined, }); if (options.durableOnly && !durable) return false; return true; @@ -278,6 +290,9 @@ export async function retryStuckPendingTasks( task: { platform: row.platform, externalThreadId: row.externalThreadId, + platformContext: row.dispatchScope + ? { channelId: row.dispatchScope } + : undefined, }, baseUrl, portableSettleMs: 1_000, diff --git a/packages/core/src/integrations/pending-tasks-store.spec.ts b/packages/core/src/integrations/pending-tasks-store.spec.ts index 30ec6a9b7d..87b32adf16 100644 --- a/packages/core/src/integrations/pending-tasks-store.spec.ts +++ b/packages/core/src/integrations/pending-tasks-store.spec.ts @@ -140,19 +140,23 @@ describe("integration pending task store", () => { }); it("dispatches same-millisecond thread tasks by stable id order", async () => { - executeMock.mockResolvedValue({ rows: [{ id: "task-a" }] }); - const { getNextPendingTaskIdForThread } = await loadStore(); + executeMock.mockResolvedValue({ + rows: [{ id: "task-a", dispatch_scope: "channel-7" }], + }); + const { getNextPendingTaskForThread } = await loadStore(); await expect( - getNextPendingTaskIdForThread("slack", "thread-1"), - ).resolves.toBe("task-a"); + getNextPendingTaskForThread("slack", "thread-1"), + ).resolves.toEqual({ id: "task-a", dispatchScope: "channel-7" }); const select = executeMock.mock.calls .map(([query]) => query) .find( (query): query is { sql: string; args: unknown[] } => typeof query !== "string" && - query.sql.includes("SELECT id FROM integration_pending_tasks"), + query.sql.includes( + "SELECT id, dispatch_scope FROM integration_pending_tasks", + ), ); expect(select?.sql).toContain("ORDER BY created_at ASC, id ASC"); expect(select?.args).toEqual(["slack", "thread-1"]); @@ -240,6 +244,30 @@ describe("integration pending task store", () => { ); }); + it("persists the channel scope used by durable dispatch", async () => { + executeMock.mockResolvedValue({ rows: [], rowsAffected: 1 }); + const { insertPendingTask } = await loadStore(); + + await insertPendingTask({ + id: "task-scoped", + platform: "microsoft-teams", + externalThreadId: "conversation-9", + payload: "{}", + ownerEmail: "member@example.com", + dispatchScope: "channel-7", + }); + + const insert = executeMock.mock.calls + .map(([query]) => query) + .find( + (query): query is { sql: string; args: unknown[] } => + typeof query !== "string" && + query.sql.includes("INSERT INTO integration_pending_tasks"), + ); + expect(insert?.sql).toContain("dispatch_scope"); + expect(insert?.args.at(-1)).toBe("channel-7"); + }); + it("resolves valid Slack provenance from the caller's stored task", async () => { executeMock.mockImplementation(async (query: string | { sql: string }) => { const sql = typeof query === "string" ? query : query.sql; diff --git a/packages/core/src/integrations/pending-tasks-store.ts b/packages/core/src/integrations/pending-tasks-store.ts index de2ede38f5..13c6998465 100644 --- a/packages/core/src/integrations/pending-tasks-store.ts +++ b/packages/core/src/integrations/pending-tasks-store.ts @@ -36,6 +36,7 @@ async function ensureTable(): Promise { dispatch_attempts ${intType()} NOT NULL DEFAULT 0, last_dispatch_at ${intType()}, last_dispatch_outcome TEXT, + dispatch_scope TEXT, error_message TEXT, created_at ${intType()} NOT NULL, updated_at ${intType()} NOT NULL, @@ -65,6 +66,11 @@ async function ensureTable(): Promise { "last_dispatch_outcome", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_outcome TEXT`, ); + await ensureColumnExists( + "integration_pending_tasks", + "dispatch_scope", + `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_scope TEXT`, + ); await ensureIndexExists( "idx_pending_tasks_status_created", `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`, @@ -73,6 +79,10 @@ async function ensureTable(): Promise { "idx_pending_tasks_event_key", `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`, ); + await ensureIndexExists( + "idx_pending_tasks_dispatch_scope", + `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`, + ); return; } @@ -88,6 +98,9 @@ async function ensureTable(): Promise { // catch as "already-enqueued". await ensureExternalEventKey(client); await ensureDispatchColumns(client); + await client.execute( + `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`, + ); await client.execute( `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`, ); @@ -137,6 +150,7 @@ async function ensureDispatchColumns( `dispatch_attempts ${intType()} NOT NULL DEFAULT 0`, `last_dispatch_at ${intType()}`, "last_dispatch_outcome TEXT", + "dispatch_scope TEXT", ]; for (const column of columns) { try { @@ -174,6 +188,7 @@ export interface PendingTask { dispatchAttempts: number; lastDispatchAt: number | null; lastDispatchOutcome: string | null; + dispatchScope: string | null; errorMessage: string | null; createdAt: number; updatedAt: number; @@ -221,6 +236,7 @@ function rowToTask(row: Record): PendingTask { ? null : Number(row.last_dispatch_at as number), lastDispatchOutcome: (row.last_dispatch_outcome as string | null) ?? null, + dispatchScope: (row.dispatch_scope as string | null) ?? null, errorMessage: (row.error_message as string | null) ?? null, createdAt: Number(row.created_at ?? 0), updatedAt: Number(row.updated_at ?? 0), @@ -246,14 +262,15 @@ export async function insertPendingTask(input: { ownerEmail: string; orgId?: string | null; externalEventKey?: string | null; + dispatchScope?: string | null; }): Promise { await ensureTable(); const client = getDbExec(); const now = Date.now(); await client.execute({ sql: `INSERT INTO integration_pending_tasks - (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key, dispatch_scope) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, args: [ input.id, input.platform, @@ -266,6 +283,7 @@ export async function insertPendingTask(input: { now, now, input.externalEventKey ?? null, + input.dispatchScope ?? null, ], }); } @@ -295,7 +313,7 @@ export async function getPendingTask(id: string): Promise { await ensureTable(); const client = getDbExec(); const { rows } = await client.execute({ - sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, error_message, created_at, updated_at, completed_at + sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at FROM integration_pending_tasks WHERE id = ? LIMIT 1`, args: [id], }); @@ -433,7 +451,7 @@ export async function claimPendingTask( ) ) ) - RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, error_message, created_at, updated_at, completed_at` + RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at` : `UPDATE integration_pending_tasks SET status = ?, attempts = attempts + 1, updated_at = ?, last_dispatch_outcome = COALESCE(?, last_dispatch_outcome) @@ -499,18 +517,23 @@ export async function recordPendingTaskDispatchAttempt( } /** Next queued turn for a provider thread after its current task completes. */ -export async function getNextPendingTaskIdForThread( +export async function getNextPendingTaskForThread( platform: string, externalThreadId: string, -): Promise { +): Promise<{ id: string; dispatchScope: string | null } | null> { await ensureTable(); const { rows } = await getDbExec().execute({ - sql: `SELECT id FROM integration_pending_tasks + sql: `SELECT id, dispatch_scope FROM integration_pending_tasks WHERE platform = ? AND external_thread_id = ? AND status = 'pending' ORDER BY created_at ASC, id ASC LIMIT 1`, args: [platform, externalThreadId], }); - return rows[0]?.id ? String(rows[0].id) : null; + return rows[0]?.id + ? { + id: String(rows[0].id), + dispatchScope: (rows[0].dispatch_scope as string | null) ?? null, + } + : null; } /** Mark a task as completed. */ diff --git a/packages/core/src/integrations/plugin.spec.ts b/packages/core/src/integrations/plugin.spec.ts index a0110f7d86..071eb94087 100644 --- a/packages/core/src/integrations/plugin.spec.ts +++ b/packages/core/src/integrations/plugin.spec.ts @@ -39,6 +39,10 @@ const markTaskDeliveryRetryableMock = vi.hoisted(() => vi.fn()); const stageTaskDeliveryPayloadMock = vi.hoisted(() => vi.fn()); const insertPendingTaskMock = vi.hoisted(() => vi.fn()); const retryStuckPendingTasksMock = vi.hoisted(() => vi.fn()); +const getNextPendingTaskForThreadMock = vi.hoisted(() => + vi.fn(async () => null), +); +const dispatchPendingIntegrationTaskMock = vi.hoisted(() => vi.fn()); vi.mock("../deploy/route-discovery.js", () => ({ getMissingDefaultPlugins: vi.fn(async () => []), @@ -74,6 +78,16 @@ vi.mock("./pending-tasks-retry-job.js", () => ({ retryStuckPendingTasks: retryStuckPendingTasksMock, })); +vi.mock("./integration-durable-dispatch.js", async () => { + const actual = await vi.importActual< + typeof import("./integration-durable-dispatch.js") + >("./integration-durable-dispatch.js"); + return { + ...actual, + dispatchPendingIntegrationTask: dispatchPendingIntegrationTaskMock, + }; +}); + vi.mock("./google-docs-poller.js", () => ({ startGoogleDocsPoller: vi.fn(), handlePushNotification: vi.fn(), @@ -97,7 +111,7 @@ vi.mock("./pending-tasks-store.js", () => ({ claimPendingTask: claimPendingTaskMock, failTaskDeliveryTransition: failTaskDeliveryTransitionMock, getPendingTask: vi.fn(), - getNextPendingTaskIdForThread: vi.fn(async () => null), + getNextPendingTaskForThread: getNextPendingTaskForThreadMock, insertPendingTask: insertPendingTaskMock, isDuplicateEventError: vi.fn(() => false), markTaskCompleted: markTaskCompletedMock, @@ -633,6 +647,37 @@ describe("integrations plugin routes", () => { ); }); + it("preserves the queued successor's channel scope during immediate dispatch", async () => { + process.env.NODE_ENV = "development"; + const task = claimedTask(1); + claimPendingTaskMock.mockResolvedValueOnce(task); + getNextPendingTaskForThreadMock.mockResolvedValueOnce({ + id: "next-task", + dispatchScope: "channel-7", + }); + const nitroApp = createNitroApp(); + await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + + const result = await dispatch( + nitroApp, + "/_agent-native/integrations/process-task", + "POST", + { taskId: task.id }, + ); + + expect(result.status).toBe(200); + expect(dispatchPendingIntegrationTaskMock).toHaveBeenCalledWith( + expect.objectContaining({ + taskId: "next-task", + task: { + platform: task.platform, + externalThreadId: task.externalThreadId, + platformContext: { channelId: "channel-7" }, + }, + }), + ); + }); + it("checkpoints a failed reply for delivery-only retry without rerunning the agent", async () => { process.env.NODE_ENV = "development"; const task = claimedTask(1); diff --git a/packages/core/src/integrations/plugin.ts b/packages/core/src/integrations/plugin.ts index 4006f56a45..d46950150e 100644 --- a/packages/core/src/integrations/plugin.ts +++ b/packages/core/src/integrations/plugin.ts @@ -72,6 +72,7 @@ import { import { dispatchPendingIntegrationTask, INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, + integrationDispatchScopeValue, isIntegrationDurableDispatchConfigured, } from "./integration-durable-dispatch.js"; import { @@ -88,7 +89,7 @@ import { import { claimPendingTask, failTaskDeliveryTransition, - getNextPendingTaskIdForThread, + getNextPendingTaskForThread, insertPendingTask, isDuplicateEventError, MAX_PENDING_TASK_ATTEMPTS, @@ -733,6 +734,11 @@ export function createIntegrationsPlugin( externalEventKey: opts?.dedupeKey ? systemNoticeEventKey(opts.dedupeKey, dedupeTtlMs) : undefined, + dispatchScope: integrationDispatchScopeValue({ + platform: incoming.platform, + externalThreadId: noticeThreadId, + platformContext: incoming.platformContext, + }), }); } catch (err) { if (isDuplicateEventError(err)) return; @@ -1823,16 +1829,19 @@ export function createIntegrationsPlugin( return { ok: true, taskId, retrying: "response-delivery" }; } await markTaskCompleted(taskId); - const nextTaskId = await getNextPendingTaskIdForThread( + const nextTask = await getNextPendingTaskForThread( task.platform, task.externalThreadId, ); - if (nextTaskId) { + if (nextTask) { await dispatchPendingIntegrationTask({ - taskId: nextTaskId, + taskId: nextTask.id, task: { platform: task.platform, externalThreadId: task.externalThreadId, + platformContext: nextTask.dispatchScope + ? { channelId: nextTask.dispatchScope } + : undefined, }, event, baseUrl: getBaseUrl(event), @@ -2057,6 +2066,11 @@ export function createIntegrationsPlugin( ownerEmail: control.ownerEmail, orgId: control.orgId, externalEventKey: `control:${control.id}`, + dispatchScope: integrationDispatchScopeValue({ + platform: incoming.platform, + externalThreadId: incoming.externalThreadId, + platformContext: incoming.platformContext, + }), }); await dispatchPendingIntegrationTask({ taskId, diff --git a/packages/core/src/integrations/webhook-handler.ts b/packages/core/src/integrations/webhook-handler.ts index 25c2b87df7..63c4cc03ca 100644 --- a/packages/core/src/integrations/webhook-handler.ts +++ b/packages/core/src/integrations/webhook-handler.ts @@ -57,7 +57,10 @@ import { clearIntegrationAwaitingInput, setIntegrationAwaitingInput, } from "./awaiting-input-store.js"; -import { dispatchPendingIntegrationTask } from "./integration-durable-dispatch.js"; +import { + dispatchPendingIntegrationTask, + integrationDispatchScopeValue, +} from "./integration-durable-dispatch.js"; import { loadIntegrationMemoryPrompt } from "./integration-memory.js"; import { insertPendingTask, @@ -476,6 +479,11 @@ async function enqueueAndDispatch( // platform produce the same key, so the unique index rejects the // second insert (H3 in the webhook security audit). externalEventKey: buildEventDedupKey(incoming), + dispatchScope: integrationDispatchScopeValue({ + platform: incoming.platform, + externalThreadId: incoming.externalThreadId, + platformContext: incoming.platformContext, + }), }); const baseUrl = resolveBaseUrl(event); diff --git a/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md b/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md index 46ef55d976..da4fbbc9b1 100644 --- a/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +++ b/packages/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( dispatch_attempts INTEGER NOT NULL DEFAULT 0, last_dispatch_at INTEGER, last_dispatch_outcome TEXT, + dispatch_scope TEXT, -- persisted channel scope for recovery error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,6 +144,8 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( ); CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at); +CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope + ON integration_pending_tasks(platform, dispatch_scope); ``` The store layer creates this lazily on first use via `ensureTable()` and uses @@ -163,6 +166,8 @@ rollout can be narrowed with the comma-separated `AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are `:*`, `:`, and, for adapters that expose one, `:` (for example `slack:C123`). +Channel-scoped handoffs persist that scope separately from the provider thread +identity so a later bounded recovery sweep applies the same allowlist decision. Workspace deploys emit this pair only for the Dispatch control-plane app, which owns workspace messaging integrations. Standalone templates emit their own diff --git a/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md b/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md index 46ef55d976..da4fbbc9b1 100644 --- a/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +++ b/packages/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( dispatch_attempts INTEGER NOT NULL DEFAULT 0, last_dispatch_at INTEGER, last_dispatch_outcome TEXT, + dispatch_scope TEXT, -- persisted channel scope for recovery error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,6 +144,8 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( ); CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at); +CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope + ON integration_pending_tasks(platform, dispatch_scope); ``` The store layer creates this lazily on first use via `ensureTable()` and uses @@ -163,6 +166,8 @@ rollout can be narrowed with the comma-separated `AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are `:*`, `:`, and, for adapters that expose one, `:` (for example `slack:C123`). +Channel-scoped handoffs persist that scope separately from the provider thread +identity so a later bounded recovery sweep applies the same allowlist decision. Workspace deploys emit this pair only for the Dispatch control-plane app, which owns workspace messaging integrations. Standalone templates emit their own diff --git a/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md b/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md index 46ef55d976..da4fbbc9b1 100644 --- a/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +++ b/packages/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( dispatch_attempts INTEGER NOT NULL DEFAULT 0, last_dispatch_at INTEGER, last_dispatch_outcome TEXT, + dispatch_scope TEXT, -- persisted channel scope for recovery error_message TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -143,6 +144,8 @@ CREATE TABLE IF NOT EXISTS integration_pending_tasks ( ); CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at); +CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope + ON integration_pending_tasks(platform, dispatch_scope); ``` The store layer creates this lazily on first use via `ensureTable()` and uses @@ -163,6 +166,8 @@ rollout can be narrowed with the comma-separated `AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES` allowlist. Supported values are `:*`, `:`, and, for adapters that expose one, `:` (for example `slack:C123`). +Channel-scoped handoffs persist that scope separately from the provider thread +identity so a later bounded recovery sweep applies the same allowlist decision. Workspace deploys emit this pair only for the Dispatch control-plane app, which owns workspace messaging integrations. Standalone templates emit their own From 0044d799422578ba88dee203984fceb6f52f2b9d Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:14:43 -0400 Subject: [PATCH 4/4] Stabilize A2A poll deadline test --- packages/core/src/a2a/client.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/a2a/client.spec.ts b/packages/core/src/a2a/client.spec.ts index 821b5755db..d4ecc7bf92 100644 --- a/packages/core/src/a2a/client.spec.ts +++ b/packages/core/src/a2a/client.spec.ts @@ -197,7 +197,7 @@ describe("A2AClient", () => { JSON.parse(String(init.body)).method === "tasks/get", )?.[1]?.signal, ).toBeInstanceOf(AbortSignal); - }); + }, 30_000); it("recovers after one task-status request exceeds the per-request timeout", async () => { vi.useFakeTimers();