improvement(copilot): retry docs fetches and grep docs directories in parallel

Two robustness upgrades to the docs corpus. Page fetches from docs.sim.ai
now retry transient failures (5xx, 429, network, timeout) with jittered
backoff over three 3s attempts instead of a single 10s attempt, so a
momentary stall recovers in seconds instead of failing the tool call.
And grep now accepts a docs directory path: it fans out to every manifest
page under the directory with bounded concurrency and runs one multi-file
grep, replacing the single-page restriction that forced agents into
per-page call sweeps. Pages the site no longer serves are skipped; an
unreachable page fails the whole grep so a partial result is never
mistaken for "not documented".

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: j15z

apps/sim/lib/copilot/docs/docs-corpus.test.ts

@@ -2,15 +2,21 @@
  * @vitest-environment node
  */
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/utils/helpers', () => ({
+  sleep: vi.fn(() => Promise.resolve()),
+}))
+
 import {
   couldMatchDocsScope,
   DocsCorpusError,
   globDocs,
-  grepDocsPage,
+  grepDocs,
   isDocsPath,
   readDocsPage,
 } from '@/lib/copilot/docs/docs-corpus'
 import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest'
+import type { GrepMatch } from '@/lib/copilot/vfs/operations'
 
 const SAMPLE_PAGE = DOCS_MANIFEST.find((path) => path === 'workflows/blocks/agent.mdx')
 
@@ -98,33 +104,52 @@ describe('readDocsPage', () => {
     expect(fetchMock).not.toHaveBeenCalled()
   })
 
-  it('surfaces a docs-site outage as a retryable error', async () => {
+  it('surfaces a docs-site outage as a retryable error after exhausting retries', async () => {
     fetchMock.mockResolvedValue({ ok: false, status: 502, text: async () => '' })
-    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledTimes(3)
   })
 
   it('treats a network failure as retryable', async () => {
     fetchMock.mockRejectedValue(new Error('socket hang up'))
-    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledTimes(3)
+  })
+
+  it('recovers when a transient failure clears on retry', async () => {
+    fetchMock
+      .mockRejectedValueOnce(new Error('socket hang up'))
+      .mockResolvedValue({ ok: true, status: 200, text: async () => '# Agent\n\nbody' })
+
+    const page = await readDocsPage(`docs/${SAMPLE_PAGE}`)
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 })
   })
 
-  it('reports a page the site no longer serves as permanent, not retryable', async () => {
+  it('reports a page the site no longer serves as permanent, without retrying', async () => {
     fetchMock.mockResolvedValue({ ok: false, status: 404, text: async () => '' })
     const error = await readDocsPage(`docs/${SAMPLE_PAGE}`).catch((e) => e)
     expect(error).toBeInstanceOf(DocsCorpusError)
     expect(error.message).toMatch(/does not serve it/)
     expect(error.message).toMatch(/retrying will not help/)
-    expect(error.message).not.toMatch(/temporarily unavailable/)
+    expect(error.message).not.toMatch(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledOnce()
   })
 
   it('still treats 429 as retryable rather than permanent', async () => {
     fetchMock.mockResolvedValue({ ok: false, status: 429, text: async () => '' })
-    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledTimes(3)
   })
 })
 
-describe('grepDocsPage', () => {
+describe('grepDocs', () => {
   const fetchMock = vi.fn()
+  const SECTION_DIR = 'docs/workflows/blocks'
+  const SECTION_PAGES = DOCS_MANIFEST.filter((path) => path.startsWith('workflows/blocks/')).map(
+    (path) => `docs/${path}`
+  )
 
   beforeEach(() => {
     fetchMock.mockReset()
@@ -135,24 +160,66 @@ describe('grepDocsPage', () => {
     vi.unstubAllGlobals()
   })
 
-  it('greps exactly one page', async () => {
+  it('greps exactly one page for a page path', async () => {
     fetchMock.mockResolvedValue({
       ok: true,
       status: 200,
       text: async () => 'intro line\nsystemPrompt matters\ntail',
     })
 
-    const matches = await grepDocsPage(`docs/${SAMPLE_PAGE}`, 'systemPrompt')
+    const matches = await grepDocs(`docs/${SAMPLE_PAGE}`, 'systemPrompt')
 
     expect(fetchMock).toHaveBeenCalledOnce()
     expect(matches).toEqual([
       { path: `docs/${SAMPLE_PAGE}`, line: 2, content: 'systemPrompt matters' },
     ])
   })
 
-  it('refuses a multi-page scope so one grep is never hundreds of fetches', async () => {
-    await expect(grepDocsPage('docs/', 'cron')).rejects.toThrow(/single page/)
-    await expect(grepDocsPage('docs/workflows', 'cron')).rejects.toThrow(/single page/)
+  it('greps a directory by fetching every page under it', async () => {
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => 'intro\ncron marker line\ntail',
+    })
+    expect(SECTION_PAGES.length).toBeGreaterThan(1)
+
+    const matches = (await grepDocs(SECTION_DIR, 'cron marker', {
+      maxResults: 10_000,
+    })) as GrepMatch[]
+
+    expect(fetchMock).toHaveBeenCalledTimes(SECTION_PAGES.length)
+    expect(matches.map((match) => match.path)).toEqual(SECTION_PAGES)
+  })
+
+  it('skips pages the site no longer serves instead of failing the directory grep', async () => {
+    const missingUrl = `https://docs.sim.ai/${SECTION_PAGES[0].slice('docs/'.length)}`
+    fetchMock.mockImplementation(async (url: string) =>
+      url === missingUrl
+        ? { ok: false, status: 404, text: async () => '' }
+        : { ok: true, status: 200, text: async () => 'cron marker line' }
+    )
+
+    const matches = (await grepDocs(SECTION_DIR, 'cron marker', {
+      maxResults: 10_000,
+    })) as GrepMatch[]
+
+    expect(matches.map((match) => match.path)).toEqual(SECTION_PAGES.slice(1))
+  })
+
+  it('fails the whole directory grep when a page cannot be reached', async () => {
+    fetchMock.mockImplementation(async (url: string) =>
+      url.endsWith(`/${SAMPLE_PAGE}`)
+        ? { ok: false, status: 502, text: async () => '' }
+        : { ok: true, status: 200, text: async () => 'cron marker line' }
+    )
+
+    await expect(grepDocs(SECTION_DIR, 'cron marker')).rejects.toThrow(/Retry shortly/)
+  })
+
+  it('rejects a path that is neither a page nor a directory without fetching', async () => {
+    await expect(grepDocs('docs/not-a-real-page.mdx', 'cron')).rejects.toThrow(
+      /not a docs page or directory/
+    )
     expect(fetchMock).not.toHaveBeenCalled()
   })
 })

apps/sim/lib/copilot/docs/docs-corpus.ts

@@ -1,9 +1,12 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { backoffWithJitter } from '@sim/utils/retry'
 import { foldDocsIndexPath } from '@/lib/copilot/docs/docs-path'
 import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest'
 import type { GrepCountEntry, GrepMatch, GrepOptions } from '@/lib/copilot/vfs/operations'
-import { glob as globPaths, grepReadResult } from '@/lib/copilot/vfs/operations'
+import { glob as globPaths, grep, grepReadResult } from '@/lib/copilot/vfs/operations'
+import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
 
 const logger = createLogger('DocsCorpus')
 
@@ -13,7 +16,12 @@ const DOCS_BASE_URL = 'https://docs.sim.ai'
 /** VFS prefix the docs corpus is mounted at. */
 const DOCS_PREFIX = 'docs/'
 
-const FETCH_TIMEOUT_MS = 10_000
+/** Per-attempt budget — the site is CDN-cached and normally answers in well under a second. */
+const FETCH_ATTEMPT_TIMEOUT_MS = 3_000
+const FETCH_MAX_ATTEMPTS = 3
+
+/** Parallel page fetches for a directory-scoped grep. */
+const GREP_FETCH_CONCURRENCY = 8
 
 /**
  * Thrown for expected, user-facing docs-corpus conditions (unknown page,
@@ -108,8 +116,9 @@ export interface DocsPage {
  * Fetch one docs page's raw markdown from the live site. The manifest path IS
  * the URL path (`docs/workflows/blocks/agent.mdx` →
  * `https://docs.sim.ai/workflows/blocks/agent.mdx`, which the docs app rewrites
- * to its raw-markdown route), so no mapping table is needed. Returns null when
- * the page is not in the manifest or the site does not serve it.
+ * to its raw-markdown route), so no mapping table is needed. Transient failures
+ * (5xx, 429, network error, timeout) are retried with jittered backoff before
+ * being reported as unavailable.
  */
 type DocsFetchResult =
   | { outcome: 'ok'; content: string }
@@ -118,13 +127,10 @@ type DocsFetchResult =
   /** Transient: 5xx, 429, network error, or timeout. */
   | { outcome: 'unavailable' }
 
-async function fetchDocsPage(path: string): Promise<DocsFetchResult> {
-  const key = normalize(path)
-  if (!docsKeyView.has(key)) return { outcome: 'missing' }
-  const url = `${DOCS_BASE_URL}/${key.slice(DOCS_PREFIX.length)}`
+async function fetchDocsPageOnce(url: string): Promise<DocsFetchResult> {
   try {
     const response = await fetch(url, {
-      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      signal: AbortSignal.timeout(FETCH_ATTEMPT_TIMEOUT_MS),
       headers: { Accept: 'text/markdown, text/plain' },
     })
     if (!response.ok) {
@@ -139,6 +145,17 @@ async function fetchDocsPage(path: string): Promise<DocsFetchResult> {
   }
 }
 
+async function fetchDocsPage(path: string): Promise<DocsFetchResult> {
+  const key = normalize(path)
+  if (!docsKeyView.has(key)) return { outcome: 'missing' }
+  const url = `${DOCS_BASE_URL}/${key.slice(DOCS_PREFIX.length)}`
+  for (let attempt = 1; ; attempt++) {
+    const result = await fetchDocsPageOnce(url)
+    if (result.outcome !== 'unavailable' || attempt >= FETCH_MAX_ATTEMPTS) return result
+    await sleep(backoffWithJitter(attempt, null))
+  }
+}
+
 /**
  * Read one docs page. Throws {@link DocsCorpusError} for the expected user-facing
  * conditions (directory path, unknown page, site unreachable) so the handler can
@@ -163,28 +180,55 @@ export async function readDocsPage(path: string): Promise<DocsPage> {
   }
   if (result.outcome === 'unavailable') {
     throw new DocsCorpusError(
-      `Could not load ${key} from ${DOCS_BASE_URL} — the docs site is temporarily unavailable. Retry shortly.`
+      `Could not load ${key} from ${DOCS_BASE_URL} — the docs site could not be reached. Retry shortly.`
     )
   }
   return { content: result.content, totalLines: result.content.split('\n').length }
 }
 
 /**
- * Grep ONE docs page, mirroring how grep over `files/` works: each page is a
- * separate fetch from the docs site, so a multi-page grep would mean hundreds of
- * requests. A path that is not a single page throws.
+ * Grep the docs corpus. A single page greps just that page. A directory path
+ * (`docs`, `docs/files`) fans out to every manifest page under it: pages are
+ * fetched in parallel and searched as one multi-file grep, so results follow
+ * manifest order and `maxResults` applies across pages. Pages the site no
+ * longer serves are skipped; a page that cannot be reached after retries fails
+ * the whole grep, because a silent partial result would misread as "not
+ * documented".
  */
-export async function grepDocsPage(
+export async function grepDocs(
   path: string,
   pattern: string,
   options?: GrepOptions
 ): Promise<GrepMatch[] | string[] | GrepCountEntry[]> {
   const key = normalize(path)
-  if (!docsKeyView.has(key)) {
+  if (docsKeyView.has(key)) {
+    const page = await readDocsPage(key)
+    return grepReadResult(key, page, pattern, key, options)
+  }
+  if (!isDocsDir(key)) {
+    throw new DocsCorpusError(
+      `"${path}" is not a docs page or directory. Use glob("docs/**") to list the docs corpus.`
+    )
+  }
+  const dir = `${key}/`
+  const pages = [...docsKeyView.keys()].filter((pageKey) => pageKey.startsWith(dir))
+  let unreachable = 0
+  const results = await mapWithConcurrency(pages, GREP_FETCH_CONCURRENCY, async (pageKey) => {
+    // Once any page is unreachable the grep is going to fail — skip the
+    // remaining fetches instead of hammering a site that is not answering.
+    if (unreachable > 0) return null
+    const result = await fetchDocsPage(pageKey)
+    if (result.outcome === 'unavailable') unreachable++
+    return result
+  })
+  if (unreachable > 0) {
     throw new DocsCorpusError(
-      `Grep over the docs corpus must target a single page (e.g. path: "docs/workflows/blocks/agent.mdx"). "${path}" is not a docs page. Use glob("docs/**") to find the exact path, then grep that one page.`
+      `Could not load every page under ${dir} from ${DOCS_BASE_URL} — a partial grep could misread as "not documented". Retry shortly.`
     )
   }
-  const page = await readDocsPage(key)
-  return grepReadResult(key, page, pattern, key, options)
+  const contents = new Map<string, string>()
+  results.forEach((result, index) => {
+    if (result?.outcome === 'ok') contents.set(pages[index], result.content)
+  })
+  return grep(contents, pattern, undefined, options)
 }

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -8,7 +8,7 @@ import {
   couldMatchDocsScope,
   DocsCorpusError,
   globDocs,
-  grepDocsPage,
+  grepDocs,
   isDocsPath,
   readDocsPage,
 } from '@/lib/copilot/docs/docs-corpus'
@@ -203,7 +203,7 @@ export async function executeVfsGrep(
     let result: GrepMatch[] | string[] | GrepCountEntry[]
     let provenanceFile: WorkspaceFileSecretProvenanceIdentity | undefined
     if (rawPath !== undefined && isDocsPath(rawPath)) {
-      result = await grepDocsPage(rawPath, pattern, grepOptions)
+      result = await grepDocs(rawPath, pattern, grepOptions)
     } else if (isChatUploadGrepPath(rawPath)) {
       if (!context.chatId) {
         return { success: false, error: 'No chat context available for uploads/' }

apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts

@@ -27,19 +27,13 @@ describe('search_docs dispatch chain', () => {
   })
 })
 
-/**
- * The retired ids are fully unregistered server-side — no catalog entry, no
- * handler, no alias. Only the client-side chip suppression survives, forever,
- * so historical persisted chats replay without rendering chips for tools that
- * no longer exist (the load_agent_skill precedent).
- */
-describe('retired docs-tool ids', () => {
-  for (const retired of ['search_documentation', 'get_platform_actions']) {
-    it(`${retired} is gone from the catalog and server registry but stays chip-hidden`, () => {
-      expect(TOOL_CATALOG[retired]).toBeUndefined()
-      expect(isKnownTool(retired)).toBe(false)
-      expect(getRegisteredServerToolNames()).not.toContain(retired)
-      expect(getHiddenToolNames().has(retired)).toBe(true)
+describe('removed docs-tool ids', () => {
+  for (const removed of ['search_documentation', 'get_platform_actions']) {
+    it(`${removed} is absent from the catalog, registries, and hidden-tool set`, () => {
+      expect(TOOL_CATALOG[removed]).toBeUndefined()
+      expect(isKnownTool(removed)).toBe(false)
+      expect(getRegisteredServerToolNames()).not.toContain(removed)
+      expect(getHiddenToolNames().has(removed)).toBe(false)
     })
   }
 })