improvement(copilot): share the unmounted-docs list, shrink the search default

Two places decide what the docs/ corpus is: the manifest generator (what is
readable) and the vector search's unscoped filter (what is findable). They
each carried their own copy of the excluded-section list. If they drift, a
hit in a section that is indexed but not mounted comes back as a chunk the
agent cannot then read — dropped as stale, silently shrinking the result set.
UNMOUNTED_DOCS_SECTIONS is now the one list both import.

search_docs returns 5 chunks by default instead of 10; raise topK when a pass
genuinely comes back thin.

A truncated docs page now routes to one more fetch instead of two. grep and
read cost the same single uncached fetch of the page, so grep is an
alternative to a read here, never a step after one.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: j15z

apps/sim/lib/copilot/docs/docs-path.ts

@@ -15,6 +15,28 @@
 /** Suffix that marks a section overview page on disk. */
 export const DOCS_INDEX_SUFFIX = '/index.mdx'
 
+/**
+ * Top-level docs sections deliberately left out of the copilot's `docs/` tree.
+ *
+ * Two places must agree on this list or the corpus goes subtly wrong: the
+ * manifest generator (which decides what is readable) and the vector search's
+ * unscoped filter (which decides what is findable). If search still matched an
+ * unmounted section, every hit there would be a chunk the agent cannot then
+ * `read` — dropped as stale, silently shrinking the result set.
+ *
+ * Mounting a section later is not uniform work, so plan per section:
+ * - `academy` is plain mdx under `apps/docs/content/docs/en/academy` and is
+ *   already indexed in `docs_embeddings` — removing it here and regenerating
+ *   the manifest is the whole change.
+ * - `api-reference` is mostly generated from `apps/docs/openapi.json` at build
+ *   time, so its pages have no source mdx for the generator to walk (only the
+ *   four handwritten ones: authentication, getting-started, python, typescript).
+ *   Mounting it properly needs the spec served publicly again — the
+ *   `apps/docs/app/openapi.json` route existed for exactly this and was
+ *   reverted — plus a generator branch that walks the spec's tags.
+ */
+export const UNMOUNTED_DOCS_SECTIONS = ['academy', 'api-reference'] as const
+
 /**
  * Fold an `en`-relative mdx file path onto its public path — the value used as
  * both the `docs/`-relative VFS path and the docs.sim.ai URL path.

apps/sim/lib/copilot/docs/docs-search.test.ts

@@ -241,9 +241,9 @@ describe('searchDocs topK clamping', () => {
     mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] })
   })
 
-  it('defaults to 10 when unspecified', async () => {
+  it('defaults to 5 when unspecified', async () => {
     await searchDocs('cron')
-    expect(capturedLimit.value).toBe(10)
+    expect(capturedLimit.value).toBe(5)
   })
 
   it('caps at 25 — the documented max, which the old tool never enforced', async () => {
@@ -266,10 +266,10 @@ describe('searchDocs topK clamping', () => {
   it('falls back to the default rather than passing NaN to the query', async () => {
     // Math.min/Math.max propagate NaN, so a bare clamp would reach `.limit(NaN)`.
     await searchDocs('cron', { topK: Number.NaN })
-    expect(capturedLimit.value).toBe(10)
+    expect(capturedLimit.value).toBe(5)
     await searchDocs('cron', { topK: 'twelve' as unknown as number })
-    expect(capturedLimit.value).toBe(10)
+    expect(capturedLimit.value).toBe(5)
     await searchDocs('cron', { topK: Number.POSITIVE_INFINITY })
-    expect(capturedLimit.value).toBe(10)
+    expect(capturedLimit.value).toBe(5)
   })
 })

apps/sim/lib/copilot/docs/docs-search.ts

@@ -3,13 +3,13 @@ import { docsEmbeddings } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { and, eq, like, notLike, or, sql } from 'drizzle-orm'
 import { docsPathForSourceDocument, isDocsDir, isDocsPage } from '@/lib/copilot/docs/docs-corpus'
-import { docsSourceCandidates } from '@/lib/copilot/docs/docs-path'
+import { docsSourceCandidates, UNMOUNTED_DOCS_SECTIONS } from '@/lib/copilot/docs/docs-path'
 import { generateSearchEmbedding } from '@/lib/knowledge/embeddings'
 
 const logger = createLogger('DocsSearch')
 
 const SIMILARITY_THRESHOLD = 0.3
-const DEFAULT_TOP_K = 10
+const DEFAULT_TOP_K = 5
 const MAX_TOP_K = 25
 
 export interface DocsSearchResult {
@@ -58,16 +58,17 @@ export class DocsSearchScopeError extends Error {
  * `workflows/index.mdx` (or `workflows.mdx`) on disk. A directory scope covers
  * the whole subtree plus the overview in either layout.
  *
- * Returns undefined for an unscoped search, which excludes `academy/` and
- * `api-reference/`: both are indexed but neither is mounted in the VFS, so a hit
- * there would be a chunk the agent cannot then read.
+ * An unscoped search excludes every {@link UNMOUNTED_DOCS_SECTIONS} section:
+ * they are indexed but not mounted in the VFS, so a hit there would be a chunk
+ * the agent cannot then read.
  */
 function scopeCondition(path?: string) {
   const normalized = (path ?? '').trim().replace(/^\/+/, '').replace(/\/+$/, '')
   if (normalized === '' || normalized === 'docs') {
     return and(
-      notLike(docsEmbeddings.sourceDocument, 'academy/%'),
-      notLike(docsEmbeddings.sourceDocument, 'api-reference/%')
+      ...UNMOUNTED_DOCS_SECTIONS.map((section) =>
+        notLike(docsEmbeddings.sourceDocument, `${section}/%`)
+      )
     )
   }
 

apps/sim/lib/copilot/generated/tool-catalog-v1.ts

@@ -3675,7 +3675,7 @@ export const SearchDocs: ToolCatalogEntry = {
           'Optional docs/ VFS path (a page such as docs/workflows/blocks/agent.mdx, or a section such as docs/workflows) that limits the search scope',
       },
       query: { type: 'string', description: 'The search query' },
-      topK: { type: 'number', description: 'Number of results (default 10, max 25)' },
+      topK: { type: 'number', description: 'Number of results (default 5, max 25)' },
     },
     required: ['query'],
   },
@@ -3690,7 +3690,7 @@ export const SearchDocumentation: ToolCatalogEntry = {
     type: 'object',
     properties: {
       query: { type: 'string', description: 'The search query' },
-      topK: { type: 'number', description: 'Number of results (default 10, max 25)' },
+      topK: { type: 'number', description: 'Number of results (default 5, max 25)' },
     },
     required: ['query'],
   },

apps/sim/lib/copilot/generated/tool-schemas-v1.ts

@@ -3459,7 +3459,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
         },
         topK: {
           type: 'number',
-          description: 'Number of results (default 10, max 25)',
+          description: 'Number of results (default 5, max 25)',
         },
       },
       required: ['query'],
@@ -3476,7 +3476,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
         },
         topK: {
           type: 'number',
-          description: 'Number of results (default 10, max 25)',
+          description: 'Number of results (default 5, max 25)',
         },
       },
       required: ['query'],

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -97,8 +97,12 @@ function truncateDocsPageToInlineCap(page: { content: string; totalLines: number
   returnedLines: number
 } {
   const lines = page.content.split('\n')
+  // Route to ONE more fetch, not two. Telling the model to grep and then read
+  // costs two more uncached fetches of a page it already partly has; grep and
+  // read cost the same single fetch, so grep is an alternative to a read here,
+  // never a step before one.
   const notice = (shown: number) =>
-    `\n\n[Page truncated: showing lines 1-${shown} of ${page.totalLines}. Grep this path for the section you need, then read with offset/limit.]`
+    `\n\n[Page truncated: returned lines 1-${shown} of ${page.totalLines}. To continue, read this path with offset: ${shown}. To jump straight to a section, grep this path INSTEAD of reading it — grep is the same single fetch and returns only matching lines with their numbers.]`
 
   let kept = lines.length
   let content = page.content

scripts/sync-docs-manifest.ts

@@ -15,9 +15,10 @@
  *                                     into their parent URL; `/workflows/index.mdx`
  *                                     is a 404 on the site)
  *
- * Excluded, and intentionally absent from the VFS: `academy/` and
- * `api-reference/` (fetch those with the scrape tool if ever needed), the root
- * `index.mdx` (its URL is `/`, which redirects), and every non-`en` locale.
+ * Excluded, and intentionally absent from the VFS: every section in
+ * `UNMOUNTED_DOCS_SECTIONS` (fetch those with the scrape tool if ever needed),
+ * the root `index.mdx` (its URL is `/`, which redirects), and every non-`en`
+ * locale.
  *
  * Usage:
  *   bun run docs-manifest:generate   # write the manifest
@@ -26,16 +27,20 @@
 import { readdir, readFile, writeFile } from 'node:fs/promises'
 import { dirname, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { foldDocsIndexPath } from '../apps/sim/lib/copilot/docs/docs-path'
+import { foldDocsIndexPath, UNMOUNTED_DOCS_SECTIONS } from '../apps/sim/lib/copilot/docs/docs-path'
 import { formatGeneratedSource } from './format-generated-source'
 
 const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url))
 const ROOT = resolve(SCRIPT_DIR, '..')
 const DOCS_CONTENT_DIR = resolve(ROOT, 'apps/docs/content/docs/en')
 const OUTPUT_PATH = resolve(ROOT, 'apps/sim/lib/copilot/generated/docs-manifest.ts')
 
-/** Top-level docs sections deliberately left out of the copilot's `docs/` tree. */
-const EXCLUDED_SECTIONS = new Set(['academy', 'api-reference'])
+/**
+ * Top-level docs sections deliberately left out of the copilot's `docs/` tree.
+ * Shared with the vector search's unscoped filter so readability and
+ * findability cannot drift apart — see `UNMOUNTED_DOCS_SECTIONS`.
+ */
+const EXCLUDED_SECTIONS = new Set<string>(UNMOUNTED_DOCS_SECTIONS)
 
 /** Collect every `.mdx` file under `dir`, as paths relative to {@link DOCS_CONTENT_DIR}. */
 async function collectMdxPaths(dir: string, prefix = ''): Promise<string[]> {