fix(review): harden docs corpus edges — trailing-slash glob, root-index drops, oversized-line reads

Review findings applied from the multi-agent pass on this branch:
- glob("docs/") matched no key and silently returned empty; normalize now
  strips trailing slashes so it resolves like "docs"
- unscoped search_docs no longer returns root-homepage chunks that would
  only be counted against topK and then dropped as stale (the manifest
  deliberately omits index.mdx)
- a docs page whose single line exceeds the inline cap now fails with grep
  guidance instead of returning an over-cap payload as success
- test coverage for the vfs docs routing (glob/read/grep dispatch,
  DocsCorpusError surfacing, truncation paths), the search_docs server
  tool's shortfall notes, the empty-embedding outcome, and the inert
  @docs context

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: j15z

apps/sim/lib/copilot/chat/process-contents.test.ts

@@ -78,6 +78,23 @@ describe('processContextsServer - skill contexts', () => {
   })
 })
 
+describe('processContextsServer - docs contexts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('resolves a tagged docs context to nothing while @docs tagging is disabled', async () => {
+    const result = await processContextsServer(
+      [{ kind: 'docs', label: 'Docs' } as ChatContext],
+      'user-1',
+      'how do loops work @Docs',
+      'ws-1'
+    )
+
+    expect(result).toEqual([])
+  })
+})
+
 describe('processContextsServer - MCP contexts', () => {
   beforeEach(() => {
     vi.clearAllMocks()

apps/sim/lib/copilot/docs/docs-corpus.test.ts

@@ -58,6 +58,11 @@ describe('globDocs', () => {
     expect(globDocs('docs/workflows.mdx')).toEqual(['docs/workflows.mdx'])
     expect(globDocs('docs/workflows/index.mdx')).toEqual([])
   })
+
+  it('treats a trailing-slash pattern like the bare directory instead of matching nothing', () => {
+    expect(globDocs('docs/')).toEqual(['docs'])
+    expect(globDocs('docs/integrations/')).toEqual(['docs/integrations'])
+  })
 })
 
 describe('readDocsPage', () => {

apps/sim/lib/copilot/docs/docs-corpus.ts

@@ -38,7 +38,10 @@ const docsKeyView: Map<string, string> = new Map(
 )
 
 function normalize(path: string): string {
-  return path.trim().replace(/^\/+/, '')
+  // Trailing slashes are stripped so `docs/` addresses the corpus the same way
+  // `docs` does — otherwise a trailing-slash glob pattern matches no key and
+  // silently returns an empty result instead of the corpus listing.
+  return path.trim().replace(/^\/+/, '').replace(/\/+$/, '')
 }
 
 /**

apps/sim/lib/copilot/docs/docs-search.test.ts

@@ -26,6 +26,7 @@ vi.mock('drizzle-orm', () => {
     and: op('and'),
     or: op('or'),
     eq: op('eq'),
+    ne: op('ne'),
     like: op('like'),
     notLike: op('notLike'),
     sql: (strings: TemplateStringsArray) => ({ op: 'sql', text: strings.join('?') }),
@@ -77,6 +78,12 @@ describe('searchDocs path scoping', () => {
     expect(whereText()).toContain('academy/%')
   })
 
+  it('excludes the root homepage when unscoped — its chunks have no live docs/ path', async () => {
+    await searchDocs('cron')
+    expect(whereText()).toContain('"op":"ne"')
+    expect(whereText()).toContain('index.mdx')
+  })
+
   it('scopes a page to both on-disk layouts', async () => {
     await searchDocs('cron', { path: 'docs/workflows/blocks/agent.mdx' })
     const text = whereText()
@@ -171,6 +178,17 @@ describe('searchDocs results', () => {
     expect((await searchDocs('cron')).results).toEqual([])
   })
 
+  it('returns the zero-candidate outcome without querying when the embedding is empty', async () => {
+    mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [] })
+    const outcome = await searchDocs('cron')
+    expect(outcome).toEqual({
+      results: [],
+      candidatesConsidered: 0,
+      droppedBelowThreshold: 0,
+      droppedStale: 0,
+    })
+  })
+
   it('drops chunks below the similarity threshold', async () => {
     mockRows.value = [
       {

apps/sim/lib/copilot/docs/docs-search.ts

@@ -1,7 +1,7 @@
 import { db } from '@sim/db'
 import { docsEmbeddings } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
-import { and, eq, like, notLike, or, sql } from 'drizzle-orm'
+import { and, eq, like, ne, notLike, or, sql } from 'drizzle-orm'
 import { docsPathForSourceDocument, isDocsDir, isDocsPage } from '@/lib/copilot/docs/docs-corpus'
 import { docsSourceCandidates, UNMOUNTED_DOCS_SECTIONS } from '@/lib/copilot/docs/docs-path'
 import { generateSearchEmbedding } from '@/lib/knowledge/embeddings'
@@ -60,12 +60,16 @@ export class DocsSearchScopeError extends Error {
  *
  * An unscoped search excludes every {@link UNMOUNTED_DOCS_SECTIONS} section:
  * they are indexed but not mounted in the VFS, so a hit there would be a chunk
- * the agent cannot then read.
+ * the agent cannot then read. The root homepage (`index.mdx`) is excluded for
+ * the same reason — the manifest generator drops it (its URL is `/`, which
+ * redirects), so its chunks would only ever be counted against topK and then
+ * discarded as stale.
  */
 function scopeCondition(path?: string) {
   const normalized = (path ?? '').trim().replace(/^\/+/, '').replace(/\/+$/, '')
   if (normalized === '' || normalized === 'docs') {
     return and(
+      ne(docsEmbeddings.sourceDocument, 'index.mdx'),
       ...UNMOUNTED_DOCS_SECTIONS.map((section) =>
         notLike(docsEmbeddings.sourceDocument, `${section}/%`)
       )

apps/sim/lib/copilot/tools/handlers/vfs.test.ts

@@ -2,7 +2,7 @@
  * @vitest-environment node
  */
 
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { TOOL_RESULT_MAX_INLINE_CHARS } from '@/lib/copilot/constants'
 
 const { getOrMaterializeVFS } = vi.hoisted(() => ({
@@ -405,3 +405,109 @@ describe('vfs uploads are opt-in (like recently-deleted/)', () => {
     expect(grepChatUpload).toHaveBeenCalledWith('report.json', 'chat-1', 'x', expect.any(Object))
   })
 })
+
+describe('vfs handlers docs corpus routing', () => {
+  const fetchMock = vi.fn()
+  const DOCS_PAGE = 'docs/workflows/blocks/agent.mdx'
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    fetchMock.mockReset()
+    vi.stubGlobal('fetch', fetchMock)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('globs the docs corpus without materializing the workspace VFS', async () => {
+    const result = await executeVfsGlob({ pattern: 'docs/**' }, GREP_CTX)
+
+    expect(result.success).toBe(true)
+    expect((result.output as { files: string[] }).files).toContain(DOCS_PAGE)
+    expect(getOrMaterializeVFS).not.toHaveBeenCalled()
+  })
+
+  it('reads a docs page via the live-site fetch, not the workspace VFS', async () => {
+    fetchMock.mockResolvedValue({ ok: true, status: 200, text: async () => 'line one\nline two' })
+
+    const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX)
+
+    expect(result.success).toBe(true)
+    expect(result.output).toEqual({ content: 'line one\nline two', totalLines: 2 })
+    expect(getOrMaterializeVFS).not.toHaveBeenCalled()
+  })
+
+  it('surfaces DocsCorpusError messages verbatim from read, without fetching', async () => {
+    const unknown = await executeVfsRead({ path: 'docs/not-a-real-page.mdx' }, GREP_CTX)
+    expect(unknown.success).toBe(false)
+    expect(unknown.error).toContain('Docs page not found')
+
+    const dir = await executeVfsRead({ path: 'docs/workflows/blocks' }, GREP_CTX)
+    expect(dir.success).toBe(false)
+    expect(dir.error).toContain('is a directory')
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('greps exactly one docs page and rejects multi-page scopes verbatim', async () => {
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => 'alpha\ncron beta\ngamma',
+    })
+
+    const single = await executeVfsGrep({ pattern: 'cron', path: DOCS_PAGE }, GREP_CTX)
+    expect(single.success).toBe(true)
+
+    const multi = await executeVfsGrep({ pattern: 'cron', path: 'docs/workflows' }, GREP_CTX)
+    expect(multi.success).toBe(false)
+    expect(multi.error).toContain('single page')
+    expect(getOrMaterializeVFS).not.toHaveBeenCalled()
+  })
+
+  it('truncates an oversized multi-line docs page to fit the inline cap', async () => {
+    const line = 'y'.repeat(200)
+    const totalLines = Math.ceil((TOOL_RESULT_MAX_INLINE_CHARS * 2) / (line.length + 1))
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => Array.from({ length: totalLines }, () => line).join('\n'),
+    })
+
+    const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX)
+
+    expect(result.success).toBe(true)
+    const output = result.output as { content: string; totalLines: number }
+    expect(output.totalLines).toBe(totalLines)
+    expect(output.content).toContain('[Page truncated: returned lines 1-')
+    expect(JSON.stringify(output).length).toBeLessThanOrEqual(TOOL_RESULT_MAX_INLINE_CHARS)
+  })
+
+  it('fails a docs page whose single line cannot fit inline instead of returning it oversized', async () => {
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => 'z'.repeat(TOOL_RESULT_MAX_INLINE_CHARS + 1000),
+    })
+
+    const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX)
+
+    expect(result.success).toBe(false)
+    expect(result.error).toContain('Grep this page')
+  })
+
+  it('rejects an explicit window that still overflows instead of truncating it', async () => {
+    const line = 'y'.repeat(200)
+    const totalLines = Math.ceil((TOOL_RESULT_MAX_INLINE_CHARS * 2) / (line.length + 1))
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => Array.from({ length: totalLines }, () => line).join('\n'),
+    })
+
+    const result = await executeVfsRead({ path: DOCS_PAGE, offset: 0, limit: totalLines }, GREP_CTX)
+
+    expect(result.success).toBe(false)
+    expect(result.error).toContain('still too large over the requested window')
+  })
+})

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -90,12 +90,14 @@ function hasModelAttachment(result: unknown): boolean {
 /**
  * Trim an oversized docs page to the largest whole-line prefix that fits the
  * inline budget, preserving the true `totalLines` so the model can page through
- * the rest with offset/limit.
+ * the rest with offset/limit. Returns null when not even one line fits — a
+ * single line longer than the cap — so the caller can fail instead of returning
+ * an over-cap payload as success.
  */
 function truncateDocsPageToInlineCap(page: { content: string; totalLines: number }): {
   output: { content: string; totalLines: number }
   returnedLines: number
-} {
+} | null {
   const lines = page.content.split('\n')
   // Route to ONE more fetch, not two. Telling the model to grep and then read
   // costs two more uncached fetches of a page it already partly has; grep and
@@ -105,17 +107,16 @@ function truncateDocsPageToInlineCap(page: { content: string; totalLines: number
     `\n\n[Page truncated: returned lines 1-${shown} of ${page.totalLines}. To continue, read this path with offset: ${shown}. To jump straight to a section, grep this path INSTEAD of reading it — grep is the same single fetch and returns only matching lines with their numbers.]`
 
   let kept = lines.length
-  let content = page.content
   while (kept > 0) {
-    content = `${lines.slice(0, kept).join('\n')}${notice(kept)}`
+    const content = `${lines.slice(0, kept).join('\n')}${notice(kept)}`
     if (
       serializedResultSize({ content, totalLines: page.totalLines }) <= TOOL_RESULT_MAX_INLINE_CHARS
     ) {
-      break
+      return { output: { content, totalLines: page.totalLines }, returnedLines: kept }
     }
     kept = Math.floor(kept / 2)
   }
-  return { output: { content, totalLines: page.totalLines }, returnedLines: kept }
+  return null
 }
 
 export async function executeVfsGrep(
@@ -317,6 +318,12 @@ export async function executeVfsRead(
           }
         }
         const truncated = truncateDocsPageToInlineCap(page)
+        if (!truncated) {
+          return {
+            success: false,
+            error: `${path} is too large to return inline even truncated. Grep this page for the section you need.`,
+          }
+        }
         logger.debug('vfs_read truncated oversized docs page', {
           path,
           totalLines: page.totalLines,

apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts

@@ -0,0 +1,102 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { DocsSearchOutcome } from '@/lib/copilot/docs/docs-search'
+
+const { mockSearchDocs } = vi.hoisted(() => ({
+  mockSearchDocs: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/docs/docs-search', () => ({
+  searchDocs: mockSearchDocs,
+}))
+
+import { searchDocsServerTool } from '@/lib/copilot/tools/server/docs/search-docs'
+
+function outcome(overrides: Partial<DocsSearchOutcome>): DocsSearchOutcome {
+  return {
+    results: [],
+    candidatesConsidered: 0,
+    droppedBelowThreshold: 0,
+    droppedStale: 0,
+    ...overrides,
+  }
+}
+
+const RESULT = {
+  path: 'docs/agents.mdx',
+  url: 'https://docs.sim.ai/agents',
+  title: 'Agents',
+  content: 'body',
+  similarity: 0.9,
+}
+
+describe('searchDocsServerTool', () => {
+  beforeEach(() => {
+    mockSearchDocs.mockReset()
+  })
+
+  it('forwards query, path, and topK to the search layer', async () => {
+    mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 }))
+
+    const output = await searchDocsServerTool.execute({
+      query: 'how do agents work',
+      path: 'docs/agents.mdx',
+      topK: 7,
+    })
+
+    expect(mockSearchDocs).toHaveBeenCalledWith('how do agents work', {
+      path: 'docs/agents.mdx',
+      topK: 7,
+    })
+    expect(output).toEqual({
+      results: [RESULT],
+      query: 'how do agents work',
+      totalResults: 1,
+    })
+  })
+
+  it('omits the note when nothing was dropped', async () => {
+    mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 }))
+
+    const output = await searchDocsServerTool.execute({ query: 'q' })
+
+    expect(output.note).toBeUndefined()
+  })
+
+  it('explains an empty result set caused by filtering, so it does not read as missing docs', async () => {
+    mockSearchDocs.mockResolvedValue(
+      outcome({ candidatesConsidered: 2, droppedBelowThreshold: 1, droppedStale: 1 })
+    )
+
+    const output = await searchDocsServerTool.execute({ query: 'q' })
+
+    expect(output.note).toContain('does NOT mean the docs lack this topic')
+    expect(output.note).toContain('1 scored too low')
+    expect(output.note).toContain('1 point at pages no longer in the docs')
+  })
+
+  it('notes threshold-only drops on a partial result set', async () => {
+    mockSearchDocs.mockResolvedValue(
+      outcome({ results: [RESULT], candidatesConsidered: 3, droppedBelowThreshold: 2 })
+    )
+
+    const output = await searchDocsServerTool.execute({ query: 'q' })
+
+    expect(output.note).toContain('Returned 1 of 3 candidate(s)')
+    expect(output.note).toContain('2 scored too low')
+    expect(output.note).not.toContain('no longer in the docs')
+  })
+
+  it('notes stale-only drops on a partial result set', async () => {
+    mockSearchDocs.mockResolvedValue(
+      outcome({ results: [RESULT], candidatesConsidered: 2, droppedStale: 1 })
+    )
+
+    const output = await searchDocsServerTool.execute({ query: 'q' })
+
+    expect(output.note).toContain('1 point at pages no longer in the docs')
+    expect(output.note).not.toContain('scored too low')
+  })
+})