feat(mcp): default to progressive tool discovery

.changeset/progressive-mcp-discovery.md

@@ -0,0 +1,5 @@
+---
+'incur': patch
+---
+
+Changed MCP servers to expose progressive tool discovery by default.

README.md

@@ -379,7 +379,7 @@ $ my-cli docs search --query tempo
 # → results: ...
 ```
 
-Each MCP tool becomes a plain incur subcommand, so it is also available through `cli.fetch` and through incur's own MCP server as `<group>_<tool>`.
+Each MCP tool becomes a plain incur subcommand, so it is also available through `cli.fetch` and through incur's own MCP server as `<group>_<tool>`. Progressive remote catalogs are resolved automatically.
 
 ### Serve CLIs as APIs
 
@@ -424,12 +424,20 @@ Async generator commands stream as NDJSON (`application/x-ndjson`). Middleware r
 
 #### MCP over HTTP
 
-The `fetch` handler automatically exposes an MCP endpoint at `/mcp`. Agents can discover and call your CLI's commands as MCP tools over HTTP — no stdio required:
+The `fetch` handler automatically exposes an MCP endpoint at `/mcp`. Agents can discover and call your CLI's commands over HTTP, with no stdio required:
 
 ```
 POST /mcp  { "jsonrpc": "2.0", "method": "initialize", ... }
 POST /mcp  { "jsonrpc": "2.0", "method": "tools/list", ... }
-POST /mcp  { "jsonrpc": "2.0", "method": "tools/call", "params": { "name": "users", ... } }
+POST /mcp  { "jsonrpc": "2.0", "method": "tools/call", "params": { "name": "search_tools", ... } }
+POST /mcp  { "jsonrpc": "2.0", "method": "tools/call", "params": { "name": "get_tool_details", ... } }
+POST /mcp  { "jsonrpc": "2.0", "method": "tools/call", "params": { "name": "call_read_tool", ... } }
+```
+
+MCP servers use progressive discovery by default: clients search a compact catalog, inspect one full schema, then execute through a read or write gate. This keeps command schemas out of `tools/list`. Set `mcp.tools.discovery` to `'direct'` for clients that require every command as a top-level tool:
+
+```ts
+Cli.create('my-cli', { mcp: { tools: { discovery: 'direct' } } })
 ```
 
 Non-`/mcp` paths continue routing to the command API as usual.
@@ -457,7 +465,7 @@ Most CLIs expose tools via MCP or a single monolithic skill file. incur combines
 
 The table below models a session with a 20-command CLI producing full output envelopes.
 
-- **Session start** – tokens consumed just by having the tool available. _MCP injects all tool schemas into every turn; skills only load frontmatter (name + description)._
+- **Session start** – tokens consumed just by having the tool available. _Traditional MCP servers inject all tool schemas into every turn; skills only load frontmatter (name + description)._
 - **Discovery** – tokens to learn what commands exist and how to call them. _MCP gets this at session start; skills load the full skill file on demand; incur splits by command group so only relevant commands are loaded._
 - **Invocation (×5)** – tokens per tool call.
 - **Response (×5)** – tokens in CLI output. _MCP and skills return JSON; incur defaults to TOON which strips braces, quotes, and keys._

src/Cli.test-d.ts

@@ -400,11 +400,22 @@ test('command mcp metadata accepts instructions and annotations', () => {
     mcp: {
       instructions: 'Use this server for test commands.',
       stateless: false,
-      tools: { include: ['read_*'], exclude: ['*_secret'] },
+      tools: {
+        discovery: 'progressive',
+        include: ['read_*'],
+        exclude: ['*_secret'],
+      },
       // @ts-expect-error -- annotations belong on command definitions
       annotations: { readOnlyHint: true },
     },
   })
+
+  Cli.create('test', {
+    mcp: {
+      // @ts-expect-error -- discovery only accepts supported strategies
+      tools: { discovery: 'lazy' },
+    },
+  })
 })
 
 test('command metadata accepts destructive flag', () => {

src/Cli.test.ts

@@ -2967,6 +2967,31 @@ describe('built-in commands', () => {
     })
   })
 
+  test('mcp doctor waits for delayed tool listings', async () => {
+    const spy = vi
+      .spyOn(Mcp, 'serve')
+      .mockImplementation(async (_name, _version, _commands, options) => {
+        await new Promise((resolve) => setTimeout(resolve, 50))
+        options!.output?.write(
+          `${JSON.stringify({ jsonrpc: '2.0', id: 1, result: { serverInfo: {} } })}\n`,
+        )
+        options!.output?.write(
+          `${JSON.stringify({ jsonrpc: '2.0', id: 2, result: { tools: [{ name: 'ping' }] } })}\n`,
+        )
+      })
+    try {
+      const cli = Cli.create('test')
+      cli.command('ping', { run: () => ({ pong: true }) })
+
+      const { output, exitCode } = await serve(cli, ['mcp', 'doctor', '--json'])
+
+      expect(exitCode).toBeUndefined()
+      expect(JSON.parse(output)).toMatchObject({ ok: true, toolCount: 1 })
+    } finally {
+      spy.mockRestore()
+    }
+  })
+
   test('mcp doctor applies root MCP tool filters', async () => {
     const cli = Cli.create('test', {
       version: '1.0.0',
@@ -5647,7 +5672,10 @@ describe('fetch', () => {
 
   describe('mcp over http', () => {
     function mcpCli() {
-      const cli = Cli.create('test', { version: '1.0.0' })
+      const cli = Cli.create('test', {
+        version: '1.0.0',
+        mcp: { tools: { discovery: 'direct' } },
+      })
       cli.command('greet', {
         description: 'Greet someone',
         args: z.object({ name: z.string() }),
@@ -5792,7 +5820,10 @@ describe('fetch', () => {
     })
 
     test('POST /mcp omits commands with mcp false while command routes still work', async () => {
-      const cli = Cli.create('test', { version: '1.0.0' })
+      const cli = Cli.create('test', {
+        version: '1.0.0',
+        mcp: { tools: { discovery: 'direct' } },
+      })
       cli.command('public', { run: () => ({ public: true }) })
       cli.command('secret', { mcp: false, run: () => ({ secret: true }) })
 
@@ -5826,7 +5857,13 @@ describe('fetch', () => {
     test('POST /mcp filters tools with root include and exclude patterns', async () => {
       const cli = Cli.create('test', {
         version: '1.0.0',
-        mcp: { tools: { include: ['docs_*'], exclude: ['*_secret'] } },
+        mcp: {
+          tools: {
+            discovery: 'direct',
+            include: ['docs_*'],
+            exclude: ['*_secret'],
+          },
+        },
       })
       cli.command('docs_list', { run: () => null })
       cli.command('docs_secret', { run: () => null })
@@ -5907,7 +5944,10 @@ describe('fetch', () => {
     })
 
     test('POST /mcp exposes per-request headers to command context', async () => {
-      const cli = Cli.create('test', { version: '1.0.0' })
+      const cli = Cli.create('test', {
+        version: '1.0.0',
+        mcp: { tools: { discovery: 'direct' } },
+      })
       cli.command('auth', {
         description: 'Auth',
         run: (c) => ({ authorization: c.request?.headers.get('authorization') }),

src/Cli.ts

@@ -594,7 +594,7 @@ export declare namespace create {
           instructions?: string | undefined
           /** Disable HTTP MCP session management. Defaults to `true`. */
           stateless?: boolean | undefined
-          /** Filters which command tools are exposed to MCP clients. */
+          /** Controls how command tools are exposed to MCP clients. */
           tools?: Mcp.ToolFilter | undefined
         }
       | undefined
@@ -1842,37 +1842,17 @@ function createMcpHttpHandler(
         await import('@modelcontextprotocol/server')
 
       const server = new McpServer({ name, version })
-
-      for (const tool of Mcp.collectTools(commands, [], [], options.tools)) {
-        const mergedShape: Record<string, any> = {
-          ...tool.command.args?.shape,
-          ...tool.command.options?.shape,
-        }
-        const hasInput = Object.keys(mergedShape).length > 0
-
-        server.registerTool(
-          tool.name,
-          {
-            ...(tool.description ? { description: tool.description } : undefined),
-            ...(hasInput ? { inputSchema: z.object(mergedShape) } : undefined),
-            ...(tool.outputSchema
-              ? { outputSchema: fromJsonSchema(tool.outputSchema) }
-              : undefined),
-          },
-          async (...callArgs: any[]) => {
-            const params = hasInput ? (callArgs[0] as Record<string, unknown>) : {}
-            const extra = hasInput ? callArgs[1] : callArgs[0]
-            return Mcp.callTool(tool, params, {
-              name,
-              version,
-              middlewares: mcpOptions?.middlewares,
-              env: mcpOptions?.env,
-              vars: mcpOptions?.vars,
-              request: extra?.http?.req,
-            })
-          },
-        )
-      }
+      Mcp.registerTools(server, commands, {
+        env: mcpOptions?.env,
+        fromJsonSchema,
+        middlewares: mcpOptions?.middlewares,
+        name,
+        request: (extra) => extra?.http?.req,
+        sendNotification: (notification) => server.server.notification(notification),
+        tools: options.tools,
+        vars: mcpOptions?.vars,
+        version,
+      })
 
       const transportOptions = stateless
         ? { enableJsonResponse: true }
@@ -2794,10 +2774,14 @@ async function runMcpDoctor(
     vars: options.vars,
     version: options.version,
     ...(options.mcp?.instructions ? { instructions: options.mcp.instructions } : undefined),
-    ...(options.mcp?.tools ? { tools: options.mcp.tools } : undefined),
+    tools: { ...options.mcp?.tools, discovery: 'direct' },
   }).catch((error) => {
     serveError = error
   })
+  let serveFinished = false
+  void done.finally(() => {
+    serveFinished = true
+  })
 
   input.write(
     `${Json.stringify({
@@ -2812,7 +2796,7 @@ async function runMcpDoctor(
     })}\n`,
   )
   input.write(`${Json.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} })}\n`)
-  await new Promise((resolve) => setTimeout(resolve, 20))
+  await waitForMcpDoctorResponses(chunks, () => serveFinished)
   input.end()
   await done
 
@@ -2875,6 +2859,14 @@ async function runMcpDoctor(
   }
 }
 
+async function waitForMcpDoctorResponses(chunks: string[], finished: () => boolean) {
+  const started = Date.now()
+  while (!finished() && (chunks.join('').match(/\n/g)?.length ?? 0) < 2) {
+    if (Date.now() - started >= 1_000) return
+    await new Promise((resolve) => setTimeout(resolve, 5))
+  }
+}
+
 function parseMcpDoctorResponses(chunks: string[]): Record<string, unknown>[] {
   const responses: Record<string, unknown>[] = []
   for (const line of chunks.join('').split(/\r?\n/)) {