diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 09cdf7dbb48..d5aa22c1ac2 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -123,6 +123,9 @@ jobs: - name: Repo audits run: bun run check:audits + - name: Verify docs manifest is in sync + run: bun run docs-manifest:check + - name: Migration safety (zero-downtime) audit run: | if [ "${{ github.event_name }}" = "pull_request" ]; then diff --git a/apps/sim/app/api/mothership/execute/route.test.ts b/apps/sim/app/api/mothership/execute/route.test.ts index 007f9424f1b..43893921a35 100644 --- a/apps/sim/app/api/mothership/execute/route.test.ts +++ b/apps/sim/app/api/mothership/execute/route.test.ts @@ -237,8 +237,13 @@ describe('mothership private trace provenance transport', () => { 'user-1', 'secret-value __var_FOREIGN', 'workspace-1', - 'chat-1' + 'chat-1', + expect.any(Object) ) + + const contextRegistry = mockProcessContextsServer.mock.calls.at(-1)?.[5] + const lifecycleOptions = mockRunHeadlessCopilotLifecycle.mock.calls.at(-1)?.[1] + expect(contextRegistry).toBe(lifecycleOptions.environmentContext?.resolvedSecretTraceRegistry) }) it('keeps context routing and display inputs raw until the lifecycle boundary', async () => { @@ -290,7 +295,8 @@ describe('mothership private trace provenance transport', () => { 'user-1', 'hello', 'workspace-1', - 'chat-1' + 'chat-1', + expect.any(Object) ) }) diff --git a/apps/sim/app/api/mothership/execute/route.ts b/apps/sim/app/api/mothership/execute/route.ts index 39f061a92d3..e01f727f591 100644 --- a/apps/sim/app/api/mothership/execute/route.ts +++ b/apps/sim/app/api/mothership/execute/route.ts @@ -244,7 +244,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => { userId, lastUserMessage, workspaceId, - effectiveChatId + effectiveChatId, + activeResolvedSecretTraceRegistry ).catch((error) => { reqLogger.warn('Failed to resolve agent contexts for execution', { error: toError(error).message, diff --git a/apps/sim/lib/copilot/chat/post.test.ts b/apps/sim/lib/copilot/chat/post.test.ts index 4f8efd52029..f7f71a24733 100644 --- a/apps/sim/lib/copilot/chat/post.test.ts +++ b/apps/sim/lib/copilot/chat/post.test.ts @@ -384,7 +384,8 @@ describe('handleUnifiedChatPost', () => { 'user-1', 'Hello', 'ws-1', - expect.anything() + expect.anything(), + expect.any(ResolvedSecretTraceRegistry) ) }) @@ -448,7 +449,8 @@ describe('handleUnifiedChatPost', () => { 'user-1', 'Explain these selections', 'ws-1', - 'chat-1' + 'chat-1', + expect.any(ResolvedSecretTraceRegistry) ) }) diff --git a/apps/sim/lib/copilot/chat/post.ts b/apps/sim/lib/copilot/chat/post.ts index 0bde82a6c45..710ea9ad544 100644 --- a/apps/sim/lib/copilot/chat/post.ts +++ b/apps/sim/lib/copilot/chat/post.ts @@ -461,9 +461,19 @@ async function resolveAgentContexts(params: { message: string workspaceId?: string chatId?: string + resolvedSecretTraceRegistry?: ExecutionContext['resolvedSecretTraceRegistry'] requestId: string }): Promise> { - const { contexts, resourceAttachments, userId, message, workspaceId, chatId, requestId } = params + const { + contexts, + resourceAttachments, + userId, + message, + workspaceId, + chatId, + resolvedSecretTraceRegistry, + requestId, + } = params let agentContexts: Array<{ type: string; content: string; tag?: string; path?: string }> = [] @@ -474,7 +484,8 @@ async function resolveAgentContexts(params: { userId, message, workspaceId, - chatId + chatId, + resolvedSecretTraceRegistry ) } catch (error) { logger.error(`[${requestId}] Failed to process contexts`, error) @@ -1267,7 +1278,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { }), activeOtelRoot.context ) - const agentContextsPromise = executionContextPromise.then(() => { + const agentContextsPromise = executionContextPromise.then((executionContext) => { return withCopilotSpan( TraceSpan.CopilotChatResolveAgentContexts, { @@ -1282,6 +1293,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { message: body.message, workspaceId, chatId: actualChatId, + resolvedSecretTraceRegistry: executionContext.resolvedSecretTraceRegistry, requestId, }), activeOtelRoot.context diff --git a/apps/sim/lib/copilot/chat/process-contents.test.ts b/apps/sim/lib/copilot/chat/process-contents.test.ts index 6570628e3bb..3e29c8b4046 100644 --- a/apps/sim/lib/copilot/chat/process-contents.test.ts +++ b/apps/sim/lib/copilot/chat/process-contents.test.ts @@ -10,6 +10,7 @@ import { MAX_TABLE_SELECTION_ROWS, } from '@/lib/copilot/chat/selection-context' import { DelegatedWorkspaceAuthorizationError } from '@/lib/core/application' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { ChatContext } from '@/stores/panel' const { @@ -25,6 +26,7 @@ const { readKnowledgeBase, getBlockVisibilityForCopilot, isIntegrationDeploymentAvailable, + searchDocsExecute, } = vi.hoisted(() => ({ discoverServerTools: vi.fn(), getBlock: vi.fn(), @@ -38,6 +40,7 @@ const { readKnowledgeBase: vi.fn(), getBlockVisibilityForCopilot: vi.fn(async () => null), isIntegrationDeploymentAvailable: vi.fn(() => true), + searchDocsExecute: vi.fn(), })) vi.mock('@/blocks/registry', () => ({ getBlock, getBlockRegistry })) @@ -57,6 +60,9 @@ vi.mock('@/lib/table/rows/service', () => ({ getRowsByIds })) vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ readKnowledgeBase: { execute: readKnowledgeBase }, })) +vi.mock('@/lib/copilot/tools/server/docs/search-docs', () => ({ + searchDocsServerTool: { execute: searchDocsExecute }, +})) /** * Overrides the global `@sim/db` mock: the logs-context tests below need @@ -282,6 +288,121 @@ describe('processContextsServer - skill contexts', () => { }) }) +describe('processContextsServer - docs contexts', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('routes @Docs to an unscoped search_docs query', async () => { + const resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry() + const results = [ + { + path: 'docs/workflows/loops.mdx', + url: 'https://docs.sim.ai/workflows/loops', + title: 'Loops', + content: 'Use a loop block to iterate.', + similarity: 0.9, + }, + ] + searchDocsExecute.mockResolvedValue({ results, query: 'how do loops work?', totalResults: 1 }) + + const result = await processContextsServer( + [{ kind: 'docs', label: 'Docs' }], + 'user-1', + '@Docs how do loops work?', + 'ws-1', + undefined, + resolvedSecretTraceRegistry + ) + + expect(searchDocsExecute).toHaveBeenCalledWith( + { query: 'how do loops work?' }, + { + userId: 'user-1', + workspaceId: 'ws-1', + chatId: undefined, + resolvedSecretTraceRegistry, + } + ) + expect(result).toEqual([ + { + type: 'docs', + tag: '@Docs', + content: JSON.stringify({ results }), + }, + ]) + }) + + it('preserves the search note when @Docs has no relevant matches', async () => { + const note = + 'No relevant matches. This does NOT mean the docs lack this topic. Rephrase the query.' + searchDocsExecute.mockResolvedValue({ results: [], query: 'new topic', totalResults: 0, note }) + + const result = await processContextsServer( + [{ kind: 'docs', label: 'Docs' }], + 'user-1', + '@Docs new topic', + 'ws-1', + undefined, + new ResolvedSecretTraceRegistry() + ) + + expect(result).toEqual([ + { + type: 'docs', + tag: '@Docs', + content: JSON.stringify({ results: [], note }), + }, + ]) + }) + + it('uses the Docs label when the message only contains the mention', async () => { + searchDocsExecute.mockResolvedValue({ results: [], query: 'Docs', totalResults: 0 }) + + await processContextsServer( + [{ kind: 'docs', label: 'Docs' }], + 'user-1', + '@Docs', + 'ws-1', + 'chat-1', + new ResolvedSecretTraceRegistry() + ) + + expect(searchDocsExecute).toHaveBeenCalledWith( + { query: 'Docs' }, + expect.objectContaining({ workspaceId: 'ws-1', chatId: 'chat-1' }) + ) + }) + + it('preserves an explicit unavailable note when docs search fails', async () => { + searchDocsExecute.mockRejectedValue(new Error('embedding service unavailable')) + + const result = await processContextsServer( + [{ kind: 'docs', label: 'Docs' }], + 'user-1', + '@Docs explain schedules', + 'ws-1', + 'chat-1', + new ResolvedSecretTraceRegistry() + ) + + expect(result).toEqual([ + { + type: 'docs', + tag: '@Docs', + content: JSON.stringify({ + results: [], + note: 'Documentation search is temporarily unavailable. Do not infer that the docs lack this topic; retry search_docs or browse docs/** later.', + }), + }, + ]) + expect(mockProcessContentsLogger.error).toHaveBeenCalledWith( + 'Failed to process docs context', + expect.any(Error) + ) + }) +}) + describe('processContextsServer - MCP contexts', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/copilot/chat/process-contents.ts b/apps/sim/lib/copilot/chat/process-contents.ts index 689eaed8a02..3ee1367159b 100644 --- a/apps/sim/lib/copilot/chat/process-contents.ts +++ b/apps/sim/lib/copilot/chat/process-contents.ts @@ -49,6 +49,7 @@ import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/rea import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import { escapeRegExp } from '@/executor/constants' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { BrowserTextSelection, ChatContext, TerminalTextSelection } from '@/stores/panel' type AgentContextType = @@ -124,7 +125,8 @@ export async function processContextsServer( userId: string, userMessage?: string, currentWorkspaceId?: string, - chatId?: string + chatId?: string, + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry ): Promise { if (!Array.isArray(contexts) || contexts.length === 0) return [] const tasks = contexts.map(async (ctx) => { @@ -314,17 +316,36 @@ export async function processContextsServer( } if (ctx.kind === 'docs') { try { - const { searchDocumentationServerTool } = await import( - '@/lib/copilot/tools/server/docs/search-documentation' + const { searchDocsServerTool } = await import( + '@/lib/copilot/tools/server/docs/search-docs' ) const rawQuery = (userMessage || '').trim() || ctx.label || 'Sim documentation' - const query = sanitizeMessageForDocs(rawQuery, contexts) - const res = await searchDocumentationServerTool.execute({ query, topK: 10 }) - const content = JSON.stringify(res?.results || []) + const query = + sanitizeMessageForDocs(rawQuery, contexts) || ctx.label || 'Sim documentation' + const res = await searchDocsServerTool.execute( + { query }, + { + userId, + workspaceId: currentWorkspaceId, + chatId, + resolvedSecretTraceRegistry, + } + ) + const content = JSON.stringify({ + results: res?.results || [], + ...(res?.note ? { note: res.note } : {}), + }) return { type: 'docs', tag: ctx.label ? `@${ctx.label}` : '@', content } } catch (e) { logger.error('Failed to process docs context', e) - return null + return { + type: 'docs', + tag: ctx.label ? `@${ctx.label}` : '@', + content: JSON.stringify({ + results: [], + note: 'Documentation search is temporarily unavailable. Do not infer that the docs lack this topic; retry search_docs or browse docs/** later.', + }), + } } } return null diff --git a/apps/sim/lib/copilot/docs/docs-corpus.test.ts b/apps/sim/lib/copilot/docs/docs-corpus.test.ts new file mode 100644 index 00000000000..54764c35625 --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-corpus.test.ts @@ -0,0 +1,235 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSleep } = vi.hoisted(() => ({ + mockSleep: vi.fn(() => Promise.resolve()), +})) + +vi.mock('@sim/utils/helpers', () => ({ + sleep: mockSleep, +})) + +import { + couldMatchDocsScope, + DocsCorpusError, + globDocs, + grepDocs, + isDocsPath, + readDocsPage, +} from '@/lib/copilot/docs/docs-corpus' +import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest' + +const SAMPLE_PAGE = DOCS_MANIFEST.find((path) => path === 'workflows/blocks/agent.mdx') + +function fetchResponse(status: number, content = '', headers: HeadersInit = {}) { + return { + ok: status >= 200 && status < 300, + status, + headers: new Headers(headers), + text: async () => content, + } +} + +describe('docs corpus scoping', () => { + it('recognizes docs paths', () => { + expect(isDocsPath('docs/workflows.mdx')).toBe(true) + expect(isDocsPath('docs')).toBe(true) + expect(isDocsPath('/docs/workflows.mdx')).toBe(true) + expect(isDocsPath('workflows.mdx')).toBe(false) + expect(isDocsPath('files/report.pdf')).toBe(false) + expect(isDocsPath('docsomething/x')).toBe(false) + expect(isDocsPath(undefined)).toBe(false) + }) + + it('is opt-in: only an explicit docs/ pattern can match', () => { + expect(couldMatchDocsScope('docs/**')).toBe(true) + expect(couldMatchDocsScope('docs/workflows/**')).toBe(true) + expect(couldMatchDocsScope('**')).toBe(false) + expect(couldMatchDocsScope('**/*.mdx')).toBe(false) + expect(couldMatchDocsScope('*')).toBe(false) + expect(couldMatchDocsScope(undefined)).toBe(false) + }) +}) + +describe('globDocs', () => { + it('lists the whole corpus under docs/**', () => { + const files = globDocs('docs/**') + expect(files.length).toBeGreaterThan(DOCS_MANIFEST.length) + expect(files).toContain('docs/workflows/blocks/agent.mdx') + expect(files).toContain('docs/workflows/blocks') + }) + + it('scopes to a section', () => { + const files = globDocs('docs/integrations/*.mdx') + expect(files).toContain('docs/integrations/gmail.mdx') + expect(files.every((path) => path.startsWith('docs/integrations/'))).toBe(true) + }) + + it('excludes academy and api-reference', () => { + expect(globDocs('docs/academy/**')).toEqual([]) + expect(globDocs('docs/api-reference/**')).toEqual([]) + }) + + it('maps section index pages onto their parent URL path', () => { + expect(globDocs('docs/workflows.mdx')).toEqual(['docs/workflows.mdx']) + expect(globDocs('docs/workflows/index.mdx')).toEqual([]) + }) + + it('treats a trailing-slash pattern like the bare directory instead of matching nothing', () => { + expect(globDocs('docs/')).toEqual(['docs']) + expect(globDocs('docs/integrations/')).toEqual(['docs/integrations']) + }) +}) + +describe('readDocsPage', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + fetchMock.mockReset() + mockSleep.mockReset() + mockSleep.mockResolvedValue(undefined) + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('fetches the manifest path verbatim from the docs site', async () => { + expect(SAMPLE_PAGE).toBeDefined() + fetchMock.mockResolvedValue(fetchResponse(200, '# Agent\n\nbody')) + + const page = await readDocsPage(`docs/${SAMPLE_PAGE}`) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock.mock.calls[0][0]).toBe(`https://docs.sim.ai/${SAMPLE_PAGE}`) + expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 }) + }) + + it('rejects an unknown page without fetching', async () => { + await expect(readDocsPage('docs/not-a-real-page.mdx')).rejects.toThrow(DocsCorpusError) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('points a directory read at glob', async () => { + await expect(readDocsPage('docs/workflows/blocks')).rejects.toThrow(/is a directory/) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('surfaces a docs-site outage as a retryable error after exhausting retries', async () => { + fetchMock.mockResolvedValue(fetchResponse(502)) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/) + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it('treats a network failure as retryable', async () => { + fetchMock.mockRejectedValue(new Error('socket hang up')) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/) + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it('recovers when a transient failure clears on retry', async () => { + fetchMock + .mockRejectedValueOnce(new Error('socket hang up')) + .mockResolvedValue(fetchResponse(200, '# Agent\n\nbody')) + + const page = await readDocsPage(`docs/${SAMPLE_PAGE}`) + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 }) + }) + + it('reports a page the site no longer serves as permanent, without retrying', async () => { + fetchMock.mockResolvedValue(fetchResponse(404)) + const error = await readDocsPage(`docs/${SAMPLE_PAGE}`).catch((e) => e) + expect(error).toBeInstanceOf(DocsCorpusError) + expect(error.message).toMatch(/does not serve it/) + expect(error.message).toMatch(/retrying will not help/) + expect(error.message).not.toMatch(/could not be reached/) + expect(fetchMock).toHaveBeenCalledOnce() + }) + + it('honors Retry-After while retrying a 429 response', async () => { + fetchMock.mockResolvedValue(fetchResponse(429, '', { 'Retry-After': '7' })) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/) + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(mockSleep).toHaveBeenNthCalledWith(1, 7_000) + expect(mockSleep).toHaveBeenNthCalledWith(2, 7_000) + }) + + it('treats 408 as retryable rather than a missing page', async () => { + fetchMock.mockResolvedValue(fetchResponse(408)) + await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/) + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it('aborts an in-flight fetch without retrying', async () => { + const controller = new AbortController() + fetchMock.mockImplementation((_url: string, init: RequestInit) => { + const signal = init.signal as AbortSignal + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + }) + + const request = readDocsPage(`docs/${SAMPLE_PAGE}`, controller.signal) + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce()) + controller.abort(new Error('user stopped docs read')) + + await expect(request).rejects.toThrow('user stopped docs read') + expect(fetchMock).toHaveBeenCalledOnce() + expect(mockSleep).not.toHaveBeenCalled() + }) + + it('aborts retry backoff before starting another fetch', async () => { + const controller = new AbortController() + fetchMock.mockResolvedValue(fetchResponse(502)) + mockSleep.mockImplementationOnce(() => new Promise(() => {})) + + const request = readDocsPage(`docs/${SAMPLE_PAGE}`, controller.signal) + await vi.waitFor(() => expect(mockSleep).toHaveBeenCalledOnce()) + controller.abort(new Error('user stopped docs retry')) + + await expect(request).rejects.toThrow('user stopped docs retry') + expect(fetchMock).toHaveBeenCalledOnce() + }) +}) + +describe('grepDocs', () => { + const fetchMock = vi.fn() + const SECTION_DIR = 'docs/workflows/blocks' + + beforeEach(() => { + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('greps exactly one page for a page path', async () => { + fetchMock.mockResolvedValue(fetchResponse(200, 'intro line\nsystemPrompt matters\ntail')) + + const matches = await grepDocs(`docs/${SAMPLE_PAGE}`, 'systemPrompt') + + expect(fetchMock).toHaveBeenCalledOnce() + expect(matches).toEqual([ + { path: `docs/${SAMPLE_PAGE}`, line: 2, content: 'systemPrompt matters' }, + ]) + }) + + it('rejects a directory without fetching any pages', async () => { + await expect(grepDocs(SECTION_DIR, 'cron marker')).rejects.toThrow( + /grep must target one docs page/ + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects a path that is neither a page nor a directory without fetching', async () => { + await expect(grepDocs('docs/not-a-real-page.mdx', 'cron')).rejects.toThrow(/not a docs page/) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/docs/docs-corpus.ts b/apps/sim/lib/copilot/docs/docs-corpus.ts new file mode 100644 index 00000000000..e8eca75761d --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-corpus.ts @@ -0,0 +1,251 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import { foldDocsIndexPath } from '@/lib/copilot/docs/docs-path' +import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest' +import type { GrepCountEntry, GrepMatch, GrepOptions } from '@/lib/copilot/vfs/operations' +import { glob as globPaths, grepReadResult } from '@/lib/copilot/vfs/operations' + +const logger = createLogger('DocsCorpus') + +/** The public docs site the `docs/` tree is a lazy view of. */ +const DOCS_BASE_URL = 'https://docs.sim.ai' + +/** VFS prefix the docs corpus is mounted at. */ +const DOCS_PREFIX = 'docs/' + +/** Per-attempt budget — the site is CDN-cached and normally answers in well under a second. */ +const FETCH_ATTEMPT_TIMEOUT_MS = 3_000 +const FETCH_MAX_ATTEMPTS = 3 + +/** + * Thrown for expected, user-facing docs-corpus conditions (unknown page, + * directory path, site unreachable). The VFS handlers return the message as the + * tool error instead of logging an internal failure. + */ +export class DocsCorpusError extends Error { + constructor(message: string) { + super(message) + this.name = 'DocsCorpusError' + } +} + +/** + * Keys-only view of the corpus for glob: every manifest path under `docs/`, + * mapped to empty content. `ops.glob` matches keys and derives the virtual + * directories from them, so this never touches the network. + */ +const docsKeyView: Map = new Map( + DOCS_MANIFEST.map((path) => [`${DOCS_PREFIX}${path}`, '']) +) + +/** + * Normalize a docs path and make `docs/` equivalent to `docs`, avoiding empty + * glob results caused only by a trailing slash. + */ +export function normalizeDocsPath(path: string): string { + return path.trim().replace(/^\/+/, '').replace(/\/+$/, '') +} + +/** + * True when a read/grep `path` addresses the docs corpus. Deliberately not a + * `path is string` type predicate: the callers chain it ahead of the other + * namespace checks, and a predicate would narrow `path` to `never` in every + * later branch. + */ +export function isDocsPath(path: string | undefined): boolean { + if (!path) return false + const normalized = normalizeDocsPath(path) + return normalized === 'docs' || normalized.startsWith(DOCS_PREFIX) +} + +/** + * True when a glob `pattern` could match the docs corpus. Like `uploads/` and + * `recently-deleted/`, the corpus is opt-in: only a pattern that explicitly + * starts with `docs/` (or is exactly `docs`) sees it, so a broad `**` glob never + * drags 300+ doc pages into the result. Same rule as {@link isDocsPath}; the + * separate name reads correctly at the glob call site. + */ +export function couldMatchDocsScope(pattern: string | undefined): boolean { + return isDocsPath(pattern) +} + +/** Manifest paths (and their virtual directories) matching an explicit `docs/` pattern. */ +export function globDocs(pattern: string): string[] { + return globPaths(docsKeyView, normalizeDocsPath(pattern)) +} + +/** True when `path` is a page in the docs tree. */ +export function isDocsPage(path: string): boolean { + return docsKeyView.has(normalizeDocsPath(path)) +} + +/** + * Map a `docs_embeddings.source_document` (the en-relative mdx file path) back to + * its `docs/` VFS path, applying the same index-page fold as the manifest + * generator. Returns null when the source has no live VFS path — an unmounted + * section (academy, api-reference) or a page deleted since the index was built. + */ +export function docsPathForSourceDocument(sourceDocument: string | null): string | null { + if (!sourceDocument) return null + const path = `${DOCS_PREFIX}${foldDocsIndexPath(sourceDocument.replace(/^\/+/, ''))}` + return docsKeyView.has(path) ? path : null +} + +/** True when `path` is a directory in the docs tree rather than a page. */ +export function isDocsDir(path: string): boolean { + const dir = `${normalizeDocsPath(path).replace(/\/+$/, '')}/` + if (dir === DOCS_PREFIX) return true + for (const key of docsKeyView.keys()) { + if (key.startsWith(dir)) return true + } + return false +} + +export interface DocsPage { + content: string + totalLines: number +} + +/** + * Fetch one docs page's raw markdown from the live site. The manifest path IS + * the URL path (`docs/workflows/blocks/agent.mdx` → + * `https://docs.sim.ai/workflows/blocks/agent.mdx`, which the docs app rewrites + * to its raw-markdown route), so no mapping table is needed. Transient failures + * (5xx, 429, network error, timeout) are retried with jittered backoff before + * being reported as unavailable. + */ +type DocsFetchResult = + | { outcome: 'ok'; content: string } + /** The site will not serve this path however many times we ask. */ + | { outcome: 'missing' } + /** Transient: 5xx, 429, network error, or timeout. */ + | { outcome: 'unavailable'; retryAfterMs: number | null } + +function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) { + throw toError(signal.reason ?? 'Docs request aborted') + } +} + +async function sleepForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (!signal) { + await sleep(delayMs) + return + } + + throwIfAborted(signal) + let abortListener: (() => void) | undefined + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(toError(signal.reason ?? 'Docs request aborted')) + signal.addEventListener('abort', abortListener, { once: true }) + if (signal.aborted) abortListener() + }) + + try { + await Promise.race([sleep(delayMs), aborted]) + } finally { + if (abortListener) signal.removeEventListener('abort', abortListener) + } +} + +async function fetchDocsPageOnce(url: string, signal?: AbortSignal): Promise { + throwIfAborted(signal) + const timeoutSignal = AbortSignal.timeout(FETCH_ATTEMPT_TIMEOUT_MS) + const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal + + try { + const response = await fetch(url, { + signal: requestSignal, + headers: { Accept: 'text/markdown, text/plain' }, + }) + if (!response.ok) { + logger.warn('Docs page fetch returned a non-OK status', { url, status: response.status }) + const permanent = + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 + if (permanent) return { outcome: 'missing' } + return { + outcome: 'unavailable', + retryAfterMs: parseRetryAfter(response.headers.get('retry-after')), + } + } + return { outcome: 'ok', content: await response.text() } + } catch (err) { + throwIfAborted(signal) + logger.warn('Docs page fetch failed', { url, error: toError(err).message }) + return { outcome: 'unavailable', retryAfterMs: null } + } +} + +async function fetchDocsPage(path: string, signal?: AbortSignal): Promise { + const key = normalizeDocsPath(path) + if (!docsKeyView.has(key)) return { outcome: 'missing' } + const url = `${DOCS_BASE_URL}/${key.slice(DOCS_PREFIX.length)}` + for (let attempt = 1; ; attempt++) { + throwIfAborted(signal) + const result = await fetchDocsPageOnce(url, signal) + throwIfAborted(signal) + if (result.outcome !== 'unavailable' || attempt >= FETCH_MAX_ATTEMPTS) return result + await sleepForRetry(backoffWithJitter(attempt, result.retryAfterMs), signal) + } +} + +/** + * Read one docs page. Throws {@link DocsCorpusError} for the expected user-facing + * conditions (directory path, unknown page, site unreachable) so the handler can + * surface the message verbatim. + */ +export async function readDocsPage(path: string, signal?: AbortSignal): Promise { + const key = normalizeDocsPath(path) + if (!docsKeyView.has(key)) { + if (isDocsDir(key)) { + const dir = key.replace(/\/+$/, '') + throw new DocsCorpusError(`${dir} is a directory — glob "${dir}/**" to list its pages.`) + } + throw new DocsCorpusError( + `Docs page not found: ${path}. Use glob("docs/**") to list the docs corpus.` + ) + } + const result = await fetchDocsPage(key, signal) + if (result.outcome === 'missing') { + throw new DocsCorpusError( + `${key} is in the docs index but ${DOCS_BASE_URL} does not serve it — the page was likely moved or removed. Use glob("docs/**") to find the current path; retrying will not help.` + ) + } + if (result.outcome === 'unavailable') { + throw new DocsCorpusError( + `Could not load ${key} from ${DOCS_BASE_URL} — the docs site could not be reached. Retry shortly.` + ) + } + return { content: result.content, totalLines: result.content.split('\n').length } +} + +/** + * Grep one docs page. Directory-wide grep is deliberately unsupported because + * each page is a separate network fetch; use `search_docs` for corpus search or + * `glob("docs/**")` to find a page first. + */ +export async function grepDocs( + path: string, + pattern: string, + options?: GrepOptions, + signal?: AbortSignal +): Promise { + const key = normalizeDocsPath(path) + if (!docsKeyView.has(key)) { + if (isDocsDir(key)) { + throw new DocsCorpusError( + `"${path}" is a docs directory; grep must target one docs page. Use search_docs to search the corpus or glob("${key}/**") to list its pages.` + ) + } + throw new DocsCorpusError( + `"${path}" is not a docs page. Use glob("docs/**") to list the docs corpus.` + ) + } + const page = await readDocsPage(key, signal) + return grepReadResult(key, page, pattern, key, options) +} diff --git a/apps/sim/lib/copilot/docs/docs-path.test.ts b/apps/sim/lib/copilot/docs/docs-path.test.ts new file mode 100644 index 00000000000..c40ba0a7abb --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-path.test.ts @@ -0,0 +1,35 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { docsSourceCandidates, foldDocsIndexPath } from '@/lib/copilot/docs/docs-path' +import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest' + +describe('foldDocsIndexPath', () => { + it('folds a section overview onto the section path', () => { + expect(foldDocsIndexPath('workflows/index.mdx')).toBe('workflows.mdx') + expect(foldDocsIndexPath('platform/enterprise/index.mdx')).toBe('platform/enterprise.mdx') + }) + + it('leaves a plain page untouched', () => { + expect(foldDocsIndexPath('workflows/blocks/agent.mdx')).toBe('workflows/blocks/agent.mdx') + expect(foldDocsIndexPath('agents.mdx')).toBe('agents.mdx') + }) + + it('does not fold a page merely named index', () => { + expect(foldDocsIndexPath('index.mdx')).toBe('index.mdx') + }) +}) + +describe('docsSourceCandidates', () => { + it('is the inverse of the fold — one candidate always reproduces the input', () => { + for (const publicPath of DOCS_MANIFEST) { + const candidates = docsSourceCandidates(publicPath) + expect(candidates.map(foldDocsIndexPath)).toContain(publicPath) + } + }) + + it('offers both on-disk layouts for a section path', () => { + expect(docsSourceCandidates('workflows.mdx')).toEqual(['workflows.mdx', 'workflows/index.mdx']) + }) +}) diff --git a/apps/sim/lib/copilot/docs/docs-path.ts b/apps/sim/lib/copilot/docs/docs-path.ts new file mode 100644 index 00000000000..61785418035 --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-path.ts @@ -0,0 +1,44 @@ +/** + * The single definition of how a docs source file maps onto its public path. + * + * Fumadocs folds a section's `index.mdx` into the section URL itself, so + * `workflows/index.mdx` on disk is `/workflows` on the site (and + * `/workflows/index.mdx` is a 404). Three places need that rule — the manifest + * generator, the `source_document` -> VFS reverse mapping, and the vector + * search's scope filter — and hand-syncing it has bitten this repo before, so + * it lives here. + * + * Deliberately dependency-free: `scripts/sync-docs-manifest.ts` imports this by + * relative path, and it must not pull in the manifest it generates. + */ + +/** Suffix that marks a section overview page on disk. */ +export const DOCS_INDEX_SUFFIX = '/index.mdx' + +/** + * Top-level docs sections deliberately left out of the copilot's `docs/` tree. + * + * The manifest generator and vector search share this list so search cannot + * return pages that the VFS cannot read. + */ +export const UNMOUNTED_DOCS_SECTIONS = ['academy', 'api-reference'] as const + +/** + * Fold an `en`-relative mdx file path onto its public path — the value used as + * both the `docs/`-relative VFS path and the docs.sim.ai URL path. + */ +export function foldDocsIndexPath(mdxPath: string): string { + return mdxPath.endsWith(DOCS_INDEX_SUFFIX) + ? `${mdxPath.slice(0, -DOCS_INDEX_SUFFIX.length)}.mdx` + : mdxPath +} + +/** + * The inverse of {@link foldDocsIndexPath}: the on-disk file names a public + * path could have come from. A page is stored either as `.mdx` or, when + * it is a section overview, as `/index.mdx`. + */ +export function docsSourceCandidates(publicPath: string): [string, string] { + const stem = publicPath.replace(/\.mdx$/, '') + return [`${stem}.mdx`, `${stem}${DOCS_INDEX_SUFFIX}`] +} diff --git a/apps/sim/lib/copilot/docs/docs-search.test.ts b/apps/sim/lib/copilot/docs/docs-search.test.ts new file mode 100644 index 00000000000..16d19141f43 --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-search.test.ts @@ -0,0 +1,292 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGenerateSearchEmbedding, capturedWhere, capturedLimit, mockRows } = vi.hoisted(() => ({ + mockGenerateSearchEmbedding: vi.fn(), + capturedWhere: { value: undefined as unknown }, + capturedLimit: { value: undefined as number | undefined }, + mockRows: { value: [] as unknown[] }, +})) + +vi.mock('@/lib/knowledge/embeddings', () => ({ + generateSearchEmbedding: mockGenerateSearchEmbedding, +})) + +/** + * Override the global drizzle mock with operators that record their arguments, + * so a test can assert on the `source_document` filter the scope produced. + */ +vi.mock('drizzle-orm', () => { + const op = + (name: string) => + (...args: unknown[]) => ({ op: name, args }) + return { + and: op('and'), + or: op('or'), + eq: op('eq'), + ne: op('ne'), + like: op('like'), + notLike: op('notLike'), + sql: (strings: TemplateStringsArray) => ({ op: 'sql', text: strings.join('?') }), + } +}) + +vi.mock('@sim/db', () => ({ + db: { + select: () => ({ + from: () => ({ + where: (condition: unknown) => { + capturedWhere.value = condition + return { + orderBy: () => ({ + limit: async (n: number) => { + capturedLimit.value = n + return mockRows.value + }, + }), + } + }, + }), + }), + }, +})) + +import { DocsSearchScopeError, searchDocs } from '@/lib/copilot/docs/docs-search' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +/** Render a drizzle condition to comparable SQL-ish text for assertions. */ +function whereText(): string { + return JSON.stringify(capturedWhere.value) +} + +describe('searchDocs path scoping', () => { + beforeEach(() => { + capturedWhere.value = undefined + mockRows.value = [] + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('excludes unmounted sections when unscoped', async () => { + await searchDocs('cron') + expect(whereText()).toContain('academy/%') + expect(whereText()).toContain('api-reference/%') + }) + + it('treats a bare docs prefix as unscoped', async () => { + await searchDocs('cron', { path: 'docs/' }) + expect(whereText()).toContain('academy/%') + }) + + it('excludes the root homepage when unscoped — its chunks have no live docs/ path', async () => { + await searchDocs('cron') + expect(whereText()).toContain('"op":"ne"') + expect(whereText()).toContain('index.mdx') + }) + + it('scopes a page to both on-disk layouts', async () => { + await searchDocs('cron', { path: 'docs/workflows/blocks/agent.mdx' }) + const text = whereText() + expect(text).toContain('workflows/blocks/agent.mdx') + expect(text).toContain('workflows/blocks/agent/index.mdx') + }) + + it('maps a section overview page onto its index file', async () => { + await searchDocs('cron', { path: 'docs/workflows.mdx' }) + const text = whereText() + expect(text).toContain('workflows/index.mdx') + }) + + it('scopes a directory to its subtree', async () => { + await searchDocs('cron', { path: 'docs/workflows' }) + expect(whereText()).toContain('workflows/%') + }) + + it('includes a section overview stored in either on-disk layout', async () => { + await searchDocs('cron', { path: 'docs/workflows' }) + const text = whereText() + expect(text).toContain('workflows/%') + expect(text).toContain('workflows.mdx') + }) + + it('rejects a path outside the docs corpus', async () => { + const error = await searchDocs('cron', { path: 'files/report.pdf' }).catch((cause) => cause) + expect(error).toBeInstanceOf(DocsSearchScopeError) + expect(error).toBeInstanceOf(OrchestrationError) + expect(error).toMatchObject({ code: 'validation' }) + }) + + it('rejects a docs path that is neither a page nor a section', async () => { + await expect(searchDocs('cron', { path: 'docs/not-a-real-section' })).rejects.toThrow( + /not a page or section/ + ) + }) + + it('rejects unmounted sections that exist on the site but not in the VFS', async () => { + await expect(searchDocs('cron', { path: 'docs/academy' })).rejects.toThrow( + /not a page or section/ + ) + }) +}) + +describe('searchDocs results', () => { + beforeEach(() => { + capturedWhere.value = undefined + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('returns the docs/ path to read next, folding index pages', async () => { + mockRows.value = [ + { + chunkText: 'body', + sourceDocument: 'workflows/index.mdx', + sourceLink: 'https://docs.sim.ai/workflows', + headerText: 'Overview', + similarity: 0.8, + }, + ] + const { results } = await searchDocs('cron') + expect(results).toEqual([ + { + path: 'docs/workflows.mdx', + url: 'https://docs.sim.ai/workflows', + title: 'Overview', + content: 'body', + similarity: 0.8, + }, + ]) + }) + + it('drops chunks whose source has no live docs/ path', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'academy/lesson-1.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.9, + }, + { + chunkText: 'b', + sourceDocument: 'deleted-page.mdx', + sourceLink: 'y', + headerText: 'h', + similarity: 0.9, + }, + ] + expect((await searchDocs('cron')).results).toEqual([]) + }) + + it('returns the zero-candidate outcome without querying when the embedding is empty', async () => { + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [] }) + const outcome = await searchDocs('cron') + expect(outcome).toEqual({ + results: [], + candidatesConsidered: 0, + droppedBelowThreshold: 0, + droppedStale: 0, + }) + }) + + it('drops chunks below the similarity threshold', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'agents.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.1, + }, + ] + expect((await searchDocs('cron')).results).toEqual([]) + }) +}) + +describe('searchDocs shortfall reporting', () => { + beforeEach(() => { + capturedWhere.value = undefined + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('counts why candidates were dropped so an empty set is explainable', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'agents.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.1, + }, + { + chunkText: 'b', + sourceDocument: 'deleted-page.mdx', + sourceLink: 'y', + headerText: 'h', + similarity: 0.9, + }, + ] + const outcome = await searchDocs('cron') + expect(outcome).toEqual({ + results: [], + candidatesConsidered: 2, + droppedBelowThreshold: 1, + droppedStale: 1, + }) + }) + + it('reports no drops when every candidate survives', async () => { + mockRows.value = [ + { + chunkText: 'a', + sourceDocument: 'agents.mdx', + sourceLink: 'x', + headerText: 'h', + similarity: 0.9, + }, + ] + const outcome = await searchDocs('cron') + expect(outcome.droppedBelowThreshold).toBe(0) + expect(outcome.droppedStale).toBe(0) + expect(outcome.results).toHaveLength(1) + }) +}) + +describe('searchDocs topK clamping', () => { + beforeEach(() => { + capturedLimit.value = undefined + mockRows.value = [] + mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] }) + }) + + it('defaults to 5 when unspecified', async () => { + await searchDocs('cron') + expect(capturedLimit.value).toBe(5) + }) + + it('caps at 25 — the documented max, which the old tool never enforced', async () => { + await searchDocs('cron', { topK: 500 }) + expect(capturedLimit.value).toBe(25) + }) + + it('floors at 1', async () => { + await searchDocs('cron', { topK: 0 }) + expect(capturedLimit.value).toBe(1) + await searchDocs('cron', { topK: -8 }) + expect(capturedLimit.value).toBe(1) + }) + + it('truncates a fractional count', async () => { + await searchDocs('cron', { topK: 7.9 }) + expect(capturedLimit.value).toBe(7) + }) + + it('falls back to the default rather than passing NaN to the query', async () => { + await searchDocs('cron', { topK: Number.NaN }) + expect(capturedLimit.value).toBe(5) + await searchDocs('cron', { topK: 'twelve' as unknown as number }) + expect(capturedLimit.value).toBe(5) + await searchDocs('cron', { topK: Number.POSITIVE_INFINITY }) + expect(capturedLimit.value).toBe(5) + }) +}) diff --git a/apps/sim/lib/copilot/docs/docs-search.ts b/apps/sim/lib/copilot/docs/docs-search.ts new file mode 100644 index 00000000000..e506dff48ef --- /dev/null +++ b/apps/sim/lib/copilot/docs/docs-search.ts @@ -0,0 +1,207 @@ +import { db } from '@sim/db' +import { docsEmbeddings } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { and, eq, like, ne, notLike, or, sql } from 'drizzle-orm' +import { escapeLikePattern } from '@/lib/api/list-query' +import { + docsPathForSourceDocument, + isDocsDir, + isDocsPage, + normalizeDocsPath, +} from '@/lib/copilot/docs/docs-corpus' +import { docsSourceCandidates, UNMOUNTED_DOCS_SECTIONS } from '@/lib/copilot/docs/docs-path' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' + +const logger = createLogger('DocsSearch') + +const SIMILARITY_THRESHOLD = 0.3 +const DEFAULT_TOP_K = 5 +const MAX_TOP_K = 25 + +export interface DocsSearchResult { + /** The `docs/` VFS path this chunk came from — pass it to `read` for the full page. */ + path: string + /** Public docs.sim.ai URL for the section, for citation. */ + url: string + title: string + content: string + similarity: number +} + +/** + * A search result set plus why it may be shorter than `topK`. The SQL LIMIT is + * applied before the threshold and liveness filters, so these counts are what + * distinguishes "nothing matched" from "matches were filtered out". + */ +export interface DocsSearchOutcome { + results: DocsSearchResult[] + /** Rows the vector search returned before filtering. */ + candidatesConsidered: number + /** Candidates dropped for scoring below the similarity threshold. */ + droppedBelowThreshold: number + /** Candidates dropped because their page is no longer in the docs manifest. */ + droppedStale: number +} + +/** + * Thrown when the caller scopes a search to a `path` that is not a real page or + * section in the docs corpus. Surfaced verbatim so the model can correct itself + * rather than reading an empty result as "the docs say nothing about this". + */ +export class DocsSearchScopeError extends OrchestrationError { + constructor(message: string) { + super('validation', message) + this.name = 'DocsSearchScopeError' + } +} + +/** + * Translate an optional `docs/` VFS path into a `source_document` filter. + * + * `source_document` stores the en-relative mdx file path, while VFS paths mirror + * the public URL — so a section overview is `docs/workflows.mdx` in the VFS but + * `workflows/index.mdx` (or `workflows.mdx`) on disk. A directory scope covers + * the whole subtree plus the overview in either layout. + * + * An unscoped search excludes every {@link UNMOUNTED_DOCS_SECTIONS} section: + * they are indexed but not mounted in the VFS, so a hit there would be a chunk + * the agent cannot then read. The root homepage (`index.mdx`) is excluded for + * the same reason — the manifest generator drops it (its URL is `/`, which + * redirects), so its chunks would only ever be counted against topK and then + * discarded as stale. + */ +function scopeCondition(path?: string) { + const normalized = normalizeDocsPath(path ?? '') + if (normalized === '' || normalized === 'docs') { + return and( + ne(docsEmbeddings.sourceDocument, 'index.mdx'), + ...UNMOUNTED_DOCS_SECTIONS.map((section) => + notLike(docsEmbeddings.sourceDocument, `${section}/%`) + ) + ) + } + + if (!normalized.startsWith('docs/')) { + throw new DocsSearchScopeError( + `path must be a docs/ VFS path (got "${path}"). Use glob("docs/**") to find one, or omit path to search everything.` + ) + } + + const tail = normalized.slice('docs/'.length) + + if (isDocsPage(normalized)) { + const [pageFile, indexFile] = docsSourceCandidates(tail) + return or( + eq(docsEmbeddings.sourceDocument, pageFile), + eq(docsEmbeddings.sourceDocument, indexFile) + ) + } + + if (isDocsDir(normalized)) { + return or( + like(docsEmbeddings.sourceDocument, `${escapeLikePattern(tail)}/%`), + eq(docsEmbeddings.sourceDocument, `${tail}.mdx`) + ) + } + + throw new DocsSearchScopeError( + `"${path}" is not a page or section in the docs corpus. Use glob("docs/**") to find a valid path, or omit path to search everything.` + ) +} + +/** + * Clamp a caller-supplied result count into [1, {@link MAX_TOP_K}]. + * + * Guards magnitude AND type: `Math.min`/`Math.max` propagate NaN, so a + * non-numeric value would otherwise reach the query as `.limit(NaN)`. The + * generated tool schema rejects a non-number upstream today, but this function + * is also called directly, so it does not rely on that. + */ +function clampTopK(requested: number | undefined): number { + if (requested === undefined || !Number.isFinite(requested)) return DEFAULT_TOP_K + return Math.min(Math.max(Math.trunc(requested), 1), MAX_TOP_K) +} + +/** + * Semantic search over the indexed docs corpus (`docs_embeddings`, rebuilt by + * `scripts/process-docs.ts` on release). Every result carries the `docs/` path + * it came from so the caller can `read` the full page next. + * + * The index lags the VFS: a page added since the last index rebuild is readable + * but not searchable, and a deleted one can still return chunks. Results whose + * source no longer maps to a live `docs/` path are dropped. + * + * Because those drops happen after the SQL LIMIT, a caller can get fewer hits + * than it asked for — or none at all when every candidate was filtered. The + * returned {@link DocsSearchOutcome} reports that explicitly so an empty result + * is never mistaken for "the documentation does not cover this". + */ +export async function searchDocs( + query: string, + options?: { path?: string; topK?: number } +): Promise { + if (!query || typeof query !== 'string') throw new Error('query is required') + + const topK = clampTopK(options?.topK) + const where = scopeCondition(options?.path) + + logger.info('Executing docs search', { + queryLength: query.length, + topK, + path: options?.path ?? null, + }) + + const { embedding: queryEmbedding } = await generateSearchEmbedding(query) + if (!queryEmbedding || queryEmbedding.length === 0) { + return { results: [], candidatesConsidered: 0, droppedBelowThreshold: 0, droppedStale: 0 } + } + const queryVector = JSON.stringify(queryEmbedding) + + const rows = await db + .select({ + chunkText: docsEmbeddings.chunkText, + sourceDocument: docsEmbeddings.sourceDocument, + sourceLink: docsEmbeddings.sourceLink, + headerText: docsEmbeddings.headerText, + similarity: sql`1 - (${docsEmbeddings.embedding} <=> ${queryVector}::vector)`, + }) + .from(docsEmbeddings) + .where(where) + .orderBy(sql`${docsEmbeddings.embedding} <=> ${queryVector}::vector`) + .limit(topK) + + const results: DocsSearchResult[] = [] + let droppedBelowThreshold = 0 + let droppedStale = 0 + for (const row of rows) { + if (row.similarity < SIMILARITY_THRESHOLD) { + droppedBelowThreshold++ + continue + } + const path = docsPathForSourceDocument(row.sourceDocument) + if (!path) { + droppedStale++ + continue + } + results.push({ + path, + url: row.sourceLink, + title: row.headerText, + content: row.chunkText, + similarity: row.similarity, + }) + } + + logger.info('Docs search complete', { + count: results.length, + droppedBelowThreshold, + droppedStale, + }) + return { + results, + candidatesConsidered: rows.length, + droppedBelowThreshold, + droppedStale, + } +} diff --git a/apps/sim/lib/copilot/generated/docs-manifest.ts b/apps/sim/lib/copilot/generated/docs-manifest.ts new file mode 100644 index 00000000000..c068a936569 --- /dev/null +++ b/apps/sim/lib/copilot/generated/docs-manifest.ts @@ -0,0 +1,388 @@ +/** + * AUTO-GENERATED FILE. DO NOT EDIT. + * Generated from apps/docs/content/docs/en by scripts/sync-docs-manifest.ts. + * Run: bun run docs-manifest:generate. + * + * Every page in the copilot's read-only `docs/` VFS tree, as a path that is + * simultaneously the `docs/`-relative VFS path and the docs.sim.ai URL path + * (so `docs/workflows/blocks/agent.mdx` reads + * `https://docs.sim.ai/workflows/blocks/agent.mdx`). Sorted. + */ +export const DOCS_MANIFEST: readonly string[] = [ + 'agents.mdx', + 'agents/choosing.mdx', + 'agents/custom-tools.mdx', + 'agents/mcp.mdx', + 'agents/skills.mdx', + 'chat.mdx', + 'chat/files.mdx', + 'chat/knowledge.mdx', + 'chat/mailer.mdx', + 'chat/research.mdx', + 'chat/tables.mdx', + 'chat/tasks.mdx', + 'chat/workflows.mdx', + 'files.mdx', + 'files/editor.mdx', + 'files/generating.mdx', + 'files/passing-files.mdx', + 'files/using-in-workflows.mdx', + 'getting-started.mdx', + 'integrations.mdx', + 'integrations/a2a.mdx', + 'integrations/agentmail.mdx', + 'integrations/agentphone.mdx', + 'integrations/agiloft.mdx', + 'integrations/ahrefs.mdx', + 'integrations/airtable-service-account.mdx', + 'integrations/airtable.mdx', + 'integrations/airweave.mdx', + 'integrations/algolia.mdx', + 'integrations/amplitude.mdx', + 'integrations/apify.mdx', + 'integrations/apollo.mdx', + 'integrations/appconfig.mdx', + 'integrations/arxiv.mdx', + 'integrations/asana-service-account.mdx', + 'integrations/asana.mdx', + 'integrations/ashby.mdx', + 'integrations/athena.mdx', + 'integrations/atlassian-service-account.mdx', + 'integrations/attio-service-account.mdx', + 'integrations/attio.mdx', + 'integrations/azure_devops.mdx', + 'integrations/box-service-account.mdx', + 'integrations/box.mdx', + 'integrations/brandfetch.mdx', + 'integrations/brex.mdx', + 'integrations/brightdata.mdx', + 'integrations/browser_use.mdx', + 'integrations/buffer.mdx', + 'integrations/calcom-service-account.mdx', + 'integrations/calcom.mdx', + 'integrations/calendly.mdx', + 'integrations/circleback.mdx', + 'integrations/clay.mdx', + 'integrations/clerk.mdx', + 'integrations/clickhouse.mdx', + 'integrations/clickup-service-account.mdx', + 'integrations/clickup.mdx', + 'integrations/cloudflare.mdx', + 'integrations/cloudformation.mdx', + 'integrations/cloudwatch.mdx', + 'integrations/codepipeline.mdx', + 'integrations/confluence.mdx', + 'integrations/context_dev.mdx', + 'integrations/convex.mdx', + 'integrations/crowdstrike.mdx', + 'integrations/cursor.mdx', + 'integrations/dagster.mdx', + 'integrations/databricks.mdx', + 'integrations/datadog.mdx', + 'integrations/datagma.mdx', + 'integrations/daytona.mdx', + 'integrations/deployments.mdx', + 'integrations/devin.mdx', + 'integrations/discord.mdx', + 'integrations/docusign.mdx', + 'integrations/downdetector.mdx', + 'integrations/dropbox.mdx', + 'integrations/dropcontact.mdx', + 'integrations/dspy.mdx', + 'integrations/dub.mdx', + 'integrations/duckduckgo.mdx', + 'integrations/dynamodb.mdx', + 'integrations/dynatrace.mdx', + 'integrations/elasticsearch.mdx', + 'integrations/elevenlabs.mdx', + 'integrations/emailbison.mdx', + 'integrations/embeddings.mdx', + 'integrations/enrich.mdx', + 'integrations/enrichment.mdx', + 'integrations/enrow.mdx', + 'integrations/evernote.mdx', + 'integrations/exa.mdx', + 'integrations/extend.mdx', + 'integrations/fathom.mdx', + 'integrations/file.mdx', + 'integrations/findymail.mdx', + 'integrations/firecrawl.mdx', + 'integrations/fireflies.mdx', + 'integrations/flint.mdx', + 'integrations/gamma.mdx', + 'integrations/github.mdx', + 'integrations/gitlab.mdx', + 'integrations/gmail.mdx', + 'integrations/gong.mdx', + 'integrations/google-service-account.mdx', + 'integrations/google_ads.mdx', + 'integrations/google_appsheet.mdx', + 'integrations/google_bigquery.mdx', + 'integrations/google_books.mdx', + 'integrations/google_calendar.mdx', + 'integrations/google_contacts.mdx', + 'integrations/google_docs.mdx', + 'integrations/google_drive.mdx', + 'integrations/google_forms.mdx', + 'integrations/google_groups.mdx', + 'integrations/google_maps.mdx', + 'integrations/google_meet.mdx', + 'integrations/google_pagespeed.mdx', + 'integrations/google_search.mdx', + 'integrations/google_sheets.mdx', + 'integrations/google_slides.mdx', + 'integrations/google_tasks.mdx', + 'integrations/google_translate.mdx', + 'integrations/google_vault.mdx', + 'integrations/grafana.mdx', + 'integrations/grain.mdx', + 'integrations/granola.mdx', + 'integrations/greenhouse.mdx', + 'integrations/greptile.mdx', + 'integrations/hex.mdx', + 'integrations/hubspot-service-account.mdx', + 'integrations/hubspot-setup.mdx', + 'integrations/hubspot.mdx', + 'integrations/huggingface.mdx', + 'integrations/hunter.mdx', + 'integrations/iam.mdx', + 'integrations/icypeas.mdx', + 'integrations/identity_center.mdx', + 'integrations/imap.mdx', + 'integrations/incidentio.mdx', + 'integrations/infisical.mdx', + 'integrations/instantly.mdx', + 'integrations/intercom.mdx', + 'integrations/jina.mdx', + 'integrations/jira.mdx', + 'integrations/jira_service_management.mdx', + 'integrations/jupyter.mdx', + 'integrations/kalshi.mdx', + 'integrations/ketch.mdx', + 'integrations/knowledge.mdx', + 'integrations/langsmith.mdx', + 'integrations/latex.mdx', + 'integrations/launchdarkly.mdx', + 'integrations/leadmagic.mdx', + 'integrations/lemlist.mdx', + 'integrations/linear-service-account.mdx', + 'integrations/linear.mdx', + 'integrations/linkedin.mdx', + 'integrations/linkup.mdx', + 'integrations/linq.mdx', + 'integrations/logfire.mdx', + 'integrations/logs.mdx', + 'integrations/loops.mdx', + 'integrations/luma.mdx', + 'integrations/mailchimp.mdx', + 'integrations/mailgun.mdx', + 'integrations/managed_agent.mdx', + 'integrations/mem0.mdx', + 'integrations/memory.mdx', + 'integrations/microsoft_ad.mdx', + 'integrations/microsoft_dataverse.mdx', + 'integrations/microsoft_excel.mdx', + 'integrations/microsoft_planner.mdx', + 'integrations/microsoft_teams.mdx', + 'integrations/millionverifier.mdx', + 'integrations/mintlify.mdx', + 'integrations/mistral_parse.mdx', + 'integrations/monday-service-account.mdx', + 'integrations/monday.mdx', + 'integrations/mongodb.mdx', + 'integrations/mysql.mdx', + 'integrations/neo4j.mdx', + 'integrations/neverbounce.mdx', + 'integrations/new_relic.mdx', + 'integrations/notion-service-account.mdx', + 'integrations/notion.mdx', + 'integrations/obsidian.mdx', + 'integrations/okta.mdx', + 'integrations/onedrive.mdx', + 'integrations/onepassword.mdx', + 'integrations/openai.mdx', + 'integrations/outlook.mdx', + 'integrations/pagerduty.mdx', + 'integrations/parallel_ai.mdx', + 'integrations/peopledatalabs.mdx', + 'integrations/perplexity.mdx', + 'integrations/persona.mdx', + 'integrations/pinecone.mdx', + 'integrations/pipedrive-service-account.mdx', + 'integrations/pipedrive.mdx', + 'integrations/polymarket.mdx', + 'integrations/postgresql.mdx', + 'integrations/posthog.mdx', + 'integrations/profound.mdx', + 'integrations/prospeo.mdx', + 'integrations/pulse.mdx', + 'integrations/qdrant.mdx', + 'integrations/quartr.mdx', + 'integrations/quiver.mdx', + 'integrations/railway.mdx', + 'integrations/rb2b.mdx', + 'integrations/rds.mdx', + 'integrations/reddit.mdx', + 'integrations/redis.mdx', + 'integrations/reducto.mdx', + 'integrations/resend.mdx', + 'integrations/revenuecat.mdx', + 'integrations/rippling.mdx', + 'integrations/rocketlane.mdx', + 'integrations/rootly.mdx', + 'integrations/s3.mdx', + 'integrations/salesforce-service-account.mdx', + 'integrations/salesforce.mdx', + 'integrations/sap_concur.mdx', + 'integrations/sap_s4hana.mdx', + 'integrations/secrets_manager.mdx', + 'integrations/sendblue.mdx', + 'integrations/sendgrid.mdx', + 'integrations/sentry.mdx', + 'integrations/serper.mdx', + 'integrations/servicenow.mdx', + 'integrations/ses.mdx', + 'integrations/sftp.mdx', + 'integrations/sharepoint.mdx', + 'integrations/shopify-service-account.mdx', + 'integrations/shopify.mdx', + 'integrations/similarweb.mdx', + 'integrations/sixtyfour.mdx', + 'integrations/slack.mdx', + 'integrations/smartlead.mdx', + 'integrations/smtp.mdx', + 'integrations/snowflake-service-account.mdx', + 'integrations/snowflake.mdx', + 'integrations/sportmonks.mdx', + 'integrations/sqs.mdx', + 'integrations/square.mdx', + 'integrations/ssh.mdx', + 'integrations/stagehand.mdx', + 'integrations/stripe.mdx', + 'integrations/sts.mdx', + 'integrations/supabase.mdx', + 'integrations/table.mdx', + 'integrations/tailscale.mdx', + 'integrations/tavily.mdx', + 'integrations/telegram.mdx', + 'integrations/temporal.mdx', + 'integrations/textract.mdx', + 'integrations/thrive.mdx', + 'integrations/tiktok.mdx', + 'integrations/tinybird.mdx', + 'integrations/trello-service-account.mdx', + 'integrations/trello.mdx', + 'integrations/trigger_dev.mdx', + 'integrations/twilio.mdx', + 'integrations/twilio_sms.mdx', + 'integrations/twilio_voice.mdx', + 'integrations/typeform.mdx', + 'integrations/upstash.mdx', + 'integrations/uptimerobot.mdx', + 'integrations/vanta.mdx', + 'integrations/vercel.mdx', + 'integrations/wealthbox-service-account.mdx', + 'integrations/wealthbox.mdx', + 'integrations/webflow-service-account.mdx', + 'integrations/webflow.mdx', + 'integrations/whatsapp.mdx', + 'integrations/wikipedia.mdx', + 'integrations/wiza.mdx', + 'integrations/wordpress.mdx', + 'integrations/workday.mdx', + 'integrations/x.mdx', + 'integrations/youtube.mdx', + 'integrations/zendesk.mdx', + 'integrations/zep.mdx', + 'integrations/zerobounce.mdx', + 'integrations/zoho-desk-service-account.mdx', + 'integrations/zoho_desk.mdx', + 'integrations/zoom-service-account.mdx', + 'integrations/zoom.mdx', + 'integrations/zoominfo.mdx', + 'introduction.mdx', + 'keyboard-shortcuts.mdx', + 'knowledgebase.mdx', + 'knowledgebase/chunking-strategies.mdx', + 'knowledgebase/connectors.mdx', + 'knowledgebase/debugging-retrieval.mdx', + 'knowledgebase/tags.mdx', + 'knowledgebase/using-in-workflows.mdx', + 'logs-debugging.mdx', + 'logs-debugging/alerts.mdx', + 'logs-debugging/logging.mdx', + 'platform/costs.mdx', + 'platform/credentials.mdx', + 'platform/enterprise.mdx', + 'platform/enterprise/access-control.mdx', + 'platform/enterprise/audit-logs.mdx', + 'platform/enterprise/custom-blocks.mdx', + 'platform/enterprise/data-drains.mdx', + 'platform/enterprise/data-retention.mdx', + 'platform/enterprise/forks.mdx', + 'platform/enterprise/self-hosted.mdx', + 'platform/enterprise/session-policies.mdx', + 'platform/enterprise/sso.mdx', + 'platform/enterprise/verified-domains.mdx', + 'platform/enterprise/whitelabeling.mdx', + 'platform/organization.mdx', + 'platform/permissions.mdx', + 'platform/self-hosting.mdx', + 'platform/self-hosting/architecture.mdx', + 'platform/self-hosting/authentication.mdx', + 'platform/self-hosting/background-jobs.mdx', + 'platform/self-hosting/docker.mdx', + 'platform/self-hosting/email.mdx', + 'platform/self-hosting/environment-variables.mdx', + 'platform/self-hosting/integrations-oauth.mdx', + 'platform/self-hosting/kubernetes.mdx', + 'platform/self-hosting/networking.mdx', + 'platform/self-hosting/object-storage.mdx', + 'platform/self-hosting/observability.mdx', + 'platform/self-hosting/platforms.mdx', + 'platform/self-hosting/redis.mdx', + 'platform/self-hosting/scaling.mdx', + 'platform/self-hosting/security.mdx', + 'platform/self-hosting/troubleshooting.mdx', + 'platform/self-hosting/upgrades.mdx', + 'platform/self-hosting/verify.mdx', + 'platform/workspaces.mdx', + 'quick-reference.mdx', + 'tables.mdx', + 'tables/using-in-workflows.mdx', + 'tables/workflow-columns.mdx', + 'workflows.mdx', + 'workflows/blocks/agent.mdx', + 'workflows/blocks/api.mdx', + 'workflows/blocks/condition.mdx', + 'workflows/blocks/credential.mdx', + 'workflows/blocks/evaluator.mdx', + 'workflows/blocks/function.mdx', + 'workflows/blocks/guardrails.mdx', + 'workflows/blocks/human-in-the-loop.mdx', + 'workflows/blocks/logs.mdx', + 'workflows/blocks/loop.mdx', + 'workflows/blocks/parallel.mdx', + 'workflows/blocks/pi.mdx', + 'workflows/blocks/response.mdx', + 'workflows/blocks/router.mdx', + 'workflows/blocks/variables.mdx', + 'workflows/blocks/wait.mdx', + 'workflows/blocks/webhook.mdx', + 'workflows/blocks/workflow.mdx', + 'workflows/connections.mdx', + 'workflows/data-flow.mdx', + 'workflows/deployment.mdx', + 'workflows/deployment/agent-events.mdx', + 'workflows/deployment/api.mdx', + 'workflows/deployment/chat.mdx', + 'workflows/deployment/mcp.mdx', + 'workflows/how-it-runs.mdx', + 'workflows/triggers/rss.mdx', + 'workflows/triggers/schedule.mdx', + 'workflows/triggers/sim.mdx', + 'workflows/triggers/start.mdx', + 'workflows/triggers/table.mdx', + 'workflows/triggers/webhook.mdx', + 'workflows/variables.mdx', +] diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 3d522d70351..d4d2df7141d 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -61,7 +61,6 @@ export interface ToolCatalogEntry { | 'get_deployed_workflow_state' | 'get_deployment_log' | 'get_page_contents' - | 'get_platform_actions' | 'get_workflow_data' | 'get_workflow_run_options' | 'glob' @@ -102,7 +101,7 @@ export interface ToolCatalogEntry { | 'run_workflow_until_block' | 'scrape_page' | 'search' - | 'search_documentation' + | 'search_docs' | 'search_integration_tools' | 'search_knowledge_base' | 'search_library_docs' @@ -182,7 +181,6 @@ export interface ToolCatalogEntry { | 'get_deployed_workflow_state' | 'get_deployment_log' | 'get_page_contents' - | 'get_platform_actions' | 'get_workflow_data' | 'get_workflow_run_options' | 'glob' @@ -223,7 +221,7 @@ export interface ToolCatalogEntry { | 'run_workflow_until_block' | 'scrape_page' | 'search' - | 'search_documentation' + | 'search_docs' | 'search_integration_tools' | 'search_knowledge_base' | 'search_library_docs' @@ -3026,14 +3024,6 @@ export const GetPageContents: ToolCatalogEntry = { }, } -export const GetPlatformActions: ToolCatalogEntry = { - id: 'get_platform_actions', - name: 'get_platform_actions', - route: 'sim', - mode: 'async', - parameters: { type: 'object', properties: {} }, -} - export const GetWorkflowData: ToolCatalogEntry = { id: 'get_workflow_data', name: 'get_workflow_data', @@ -3127,12 +3117,12 @@ export const Grep: ToolCatalogEntry = { path: { type: 'string', description: - "Optional scope. A prefix (e.g. 'workflows/', 'environment/', 'internal/') searches the VFS map under it. An exact single-file path under files/ or uploads/ (optionally with /content) searches that file's content only; folders and multi-file trees are rejected for content search.", + "Optional scope. A prefix (e.g. 'workflows/', 'environment/', 'internal/') searches the VFS map under it. An exact supported single-file path searches that file's content; folders and multi-file trees are rejected for content search.", }, pattern: { type: 'string', description: - "Regex pattern to search for. Searches VFS map entries (workflow JSON, metadata, memories) by default; searches a single file's extracted text when path is one files/ or uploads/ file leaf.", + "Regex pattern to search for. Searches VFS map entries (workflow JSON, metadata, memories) by default, or an exact supported file leaf's content when path selects one.", }, toolTitle: { type: 'string', @@ -4559,21 +4549,21 @@ export const Search: ToolCatalogEntry = { internal: true, } -export const SearchDocumentation: ToolCatalogEntry = { - id: 'search_documentation', - name: 'search_documentation', +export const SearchDocs: ToolCatalogEntry = { + id: 'search_docs', + name: 'search_docs', route: 'sim', mode: 'async', parameters: { type: 'object', properties: { - query: { type: 'string', description: 'The search query' }, - topK: { - type: 'number', + path: { + type: 'string', description: - 'Number of results to return (default 10). Not clamped — keep it small, since each result is a full doc chunk.', - default: 10, + 'Optional docs/ VFS path (a page such as docs/workflows/blocks/agent.mdx, or a section such as docs/workflows) that limits the search scope', }, + query: { type: 'string', description: 'The search query' }, + topK: { type: 'number', description: 'Number of results (default 5, max 25)', default: 5 }, }, required: ['query'], }, @@ -6560,7 +6550,6 @@ export const TOOL_CATALOG: Record = { [GetDeployedWorkflowState.id]: GetDeployedWorkflowState, [GetDeploymentLog.id]: GetDeploymentLog, [GetPageContents.id]: GetPageContents, - [GetPlatformActions.id]: GetPlatformActions, [GetWorkflowData.id]: GetWorkflowData, [GetWorkflowRunOptions.id]: GetWorkflowRunOptions, [Glob.id]: Glob, @@ -6601,7 +6590,7 @@ export const TOOL_CATALOG: Record = { [RunWorkflowUntilBlock.id]: RunWorkflowUntilBlock, [ScrapePage.id]: ScrapePage, [Search.id]: Search, - [SearchDocumentation.id]: SearchDocumentation, + [SearchDocs.id]: SearchDocs, [SearchIntegrationTools.id]: SearchIntegrationTools, [SearchKnowledgeBase.id]: SearchKnowledgeBase, [SearchLibraryDocs.id]: SearchLibraryDocs, diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index a3fd31e1c85..6654dc277b5 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -2918,13 +2918,6 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_platform_actions: { - parameters: { - type: 'object', - properties: {}, - }, - resultSchema: undefined, - }, get_workflow_data: { parameters: { type: 'object', @@ -3007,12 +3000,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { path: { type: 'string', description: - "Optional scope. A prefix (e.g. 'workflows/', 'environment/', 'internal/') searches the VFS map under it. An exact single-file path under files/ or uploads/ (optionally with /content) searches that file's content only; folders and multi-file trees are rejected for content search.", + "Optional scope. A prefix (e.g. 'workflows/', 'environment/', 'internal/') searches the VFS map under it. An exact supported single-file path searches that file's content; folders and multi-file trees are rejected for content search.", }, pattern: { type: 'string', description: - "Regex pattern to search for. Searches VFS map entries (workflow JSON, metadata, memories) by default; searches a single file's extracted text when path is one files/ or uploads/ file leaf.", + "Regex pattern to search for. Searches VFS map entries (workflow JSON, metadata, memories) by default, or an exact supported file leaf's content when path selects one.", }, toolTitle: { type: 'string', @@ -4403,19 +4396,23 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_documentation: { + search_docs: { parameters: { type: 'object', properties: { + path: { + type: 'string', + description: + 'Optional docs/ VFS path (a page such as docs/workflows/blocks/agent.mdx, or a section such as docs/workflows) that limits the search scope', + }, query: { type: 'string', description: 'The search query', }, topK: { type: 'number', - description: - 'Number of results to return (default 10). Not clamped — keep it small, since each result is a full doc chunk.', - default: 10, + description: 'Number of results (default 5, max 25)', + default: 5, }, }, required: ['query'], diff --git a/apps/sim/lib/copilot/generated/vfs-snapshot-v1.ts b/apps/sim/lib/copilot/generated/vfs-snapshot-v1.ts index ee8a2dc4f62..6559a690ca7 100644 --- a/apps/sim/lib/copilot/generated/vfs-snapshot-v1.ts +++ b/apps/sim/lib/copilot/generated/vfs-snapshot-v1.ts @@ -10,7 +10,6 @@ export interface VfsSnapshotV1 { envVars?: string[] files?: VfsSnapshotV1File[] integrations?: VfsSnapshotV1Integration[] - jobs?: VfsSnapshotV1Job[] knowledgeBases?: VfsSnapshotV1KnowledgeBase[] mcpServers?: VfsSnapshotV1McpServer[] members?: VfsSnapshotV1Member[] @@ -59,19 +58,6 @@ export interface VfsSnapshotV1Integration { providerId: string role?: string } -/** - * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema - * via the `definition` "VfsSnapshotV1Job". - */ -export interface VfsSnapshotV1Job { - cronExpression?: string - id: string - lifecycle?: string - prompt?: string - sourceTaskName?: string - status?: string - title?: string -} /** * This interface was referenced by `VfsSnapshotV1`'s JSON-Schema * via the `definition` "VfsSnapshotV1KnowledgeBase". diff --git a/apps/sim/lib/copilot/tool-executor/register-handlers.ts b/apps/sim/lib/copilot/tool-executor/register-handlers.ts index f2f9eb6d304..b5b9768ac14 100644 --- a/apps/sim/lib/copilot/tool-executor/register-handlers.ts +++ b/apps/sim/lib/copilot/tool-executor/register-handlers.ts @@ -16,7 +16,6 @@ import { GetBlockUpstreamReferences, GetDeployedWorkflowState, GetDeploymentLog, - GetPlatformActions, GetWorkflowData, GetWorkflowRunOptions, Glob as GlobTool, @@ -81,7 +80,6 @@ import { executeManageSandbox } from '../tools/handlers/management/manage-sandbo import { executeManageSkill } from '../tools/handlers/management/manage-skill' import { executeMaterializeFile } from '../tools/handlers/materialize-file' import { executeOAuthGetAuthLink, executeOAuthRequestAccess } from '../tools/handlers/oauth' -import { executeGetPlatformActions } from '../tools/handlers/platform' import { executeOpenResource } from '../tools/handlers/resources' import { executeRestoreResource } from '../tools/handlers/restore-resource' import { executeRunCode } from '../tools/handlers/run-code' @@ -192,7 +190,6 @@ function buildHandlerMap(): Record { [OauthRequestAccess.id]: h(executeOAuthRequestAccess), [OpenResource.id]: h(executeOpenResource), [RestoreResource.id]: h(executeRestoreResource), - [GetPlatformActions.id]: h(executeGetPlatformActions), [ListIntegrationTools.id]: h(executeListIntegrationTools), [MaterializeFile.id]: h(executeMaterializeFile), [FunctionExecute.id]: h(executeFunctionExecute), diff --git a/apps/sim/lib/copilot/tools/client/store-utils.test.ts b/apps/sim/lib/copilot/tools/client/store-utils.test.ts index 7a849821895..de756b58182 100644 --- a/apps/sim/lib/copilot/tools/client/store-utils.test.ts +++ b/apps/sim/lib/copilot/tools/client/store-utils.test.ts @@ -49,6 +49,26 @@ describe('resolveToolDisplay', () => { ).toBe('Read RET XYZ') }) + it('formats docs corpus reads as section/page', () => { + expect( + resolveToolDisplay(ReadTool.id, ClientToolCallState.success, { + path: 'docs/workflows/blocks/agent.mdx', + })?.text + ).toBe('Read workflows/agent') + + expect( + resolveToolDisplay(ReadTool.id, ClientToolCallState.executing, { + path: 'docs/integrations/gmail.mdx', + })?.text + ).toBe('Reading integrations/gmail') + + expect( + resolveToolDisplay(ReadTool.id, ClientToolCallState.error, { + path: 'docs/getting-started.mdx', + })?.text + ).toBe('Attempted to read getting-started') + }) + it('decodes percent-encoded VFS path segments for display', () => { expect( resolveToolDisplay(ReadTool.id, ClientToolCallState.executing, { diff --git a/apps/sim/lib/copilot/tools/client/store-utils.ts b/apps/sim/lib/copilot/tools/client/store-utils.ts index 343c9e2712d..5240d0629e6 100644 --- a/apps/sim/lib/copilot/tools/client/store-utils.ts +++ b/apps/sim/lib/copilot/tools/client/store-utils.ts @@ -97,6 +97,10 @@ function describeReadTarget(path: string | undefined): string | undefined { if (segments.length === 0) return undefined + if (segments[0] === 'docs') { + return describeDocsReadTarget(segments) + } + const resourceType = VFS_DIR_TO_RESOURCE[segments[0]] if (!resourceType) { return humanizeDisplayIdentifier(stripExtension(segments[segments.length - 1]), 'sentence') @@ -140,6 +144,19 @@ function describeFileReadTarget(segments: string[]): string { return lastSegment } +/** + * Labels a docs/ corpus read as `
/` (e.g. `workflows/agent` for + * docs/workflows/blocks/agent.mdx). Top-level pages show just their name (e.g. + * `getting-started` for docs/getting-started.mdx). + */ +function describeDocsReadTarget(segments: string[]): string { + const rest = segments.slice(1) + if (rest.length === 0) return 'docs' + const leaf = stripExtension(rest[rest.length - 1]) + if (rest.length === 1) return leaf + return `${rest[0]}/${leaf}` +} + function getLeafResourceSegment(segments: string[]): string { const lastSegment = segments[segments.length - 1] || '' if (hasFileExtension(lastSegment) && segments.length > 1) { diff --git a/apps/sim/lib/copilot/tools/handlers/platform-actions.ts b/apps/sim/lib/copilot/tools/handlers/platform-actions.ts deleted file mode 100644 index c3c3ac14384..00000000000 --- a/apps/sim/lib/copilot/tools/handlers/platform-actions.ts +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Static content for the get_platform_actions tool. - * Contains the Sim platform quick reference and keyboard shortcuts. - */ -export const PLATFORM_ACTIONS_CONTENT = `# Sim Platform Quick Reference & Keyboard Shortcuts - -## Keyboard Shortcuts -**Mod** = Cmd (macOS) / Ctrl (Windows/Linux). Shortcuts work when canvas is focused. - -### Workflow Actions -| Shortcut | Action | -|----------|--------| -| Mod+Enter | Run workflow (or cancel if running) | -| Mod+Z | Undo | -| Mod+Shift+Z | Redo | -| Mod+C | Copy selected blocks | -| Mod+X | Cut selected blocks | -| Mod+V | Paste blocks | -| Delete/Backspace | Delete selected blocks or edges | -| Shift+L | Auto-layout canvas | -| Mod+Shift+F | Fit to view | -| Mod+Shift+Enter | Accept Copilot changes | - -### Panel Navigation -| Shortcut | Action | -|----------|--------| -| Mod+F | Open workflow search and replace | -| Mod+Alt+F | Focus Toolbar search | - -### Global Navigation -| Shortcut | Action | -|----------|--------| -| Mod+K | Open search | -| Mod+Shift+A | Add new agent workflow | -| Mod+Shift+P | Create workflow | -| Mod+B | Toggle sidebar | -| Mod+L | Go to logs | - -### Utility -| Shortcut | Action | -|----------|--------| -| Mod+D | Clear terminal console | - -### Mouse Controls -| Action | Control | -|--------|---------| -| Pan/move canvas | Left-drag on empty space (hand mode, the default), middle-drag, scroll, or trackpad | -| Select multiple blocks | Shift+drag to draw a selection box. In cursor mode, left-drag on empty space draws it instead | -| Drag block | Left-drag on block header | -| Add to selection | Mod+Click or Shift+Click on blocks | - -## Quick Reference — Workspaces -| Action | How | -|--------|-----| -| Create workspace | Click workspace dropdown → New Workspace | -| Switch workspaces | Click workspace dropdown → Select workspace | -| Invite teammates | Sidebar → Invite | -| Rename/Duplicate/Export/Delete workspace | Right-click workspace → action | - -## Quick Reference — Workflows -| Action | How | -|--------|-----| -| Create workflow | Click + button in sidebar | -| Reorder/move workflows | Drag workflow up/down or onto a folder | -| Import workflow | Click import button in sidebar → Select file | -| Multi-select workflows | Mod+Click or Shift+Click workflows in sidebar | -| Open in new tab | Right-click workflow → Open in New Tab | -| Rename/Duplicate/Export/Delete | Right-click workflow → action | - -## Quick Reference — Blocks -| Action | How | -|--------|-----| -| Add a block | Drag from Toolbar panel, or right-click canvas → Add Block | -| Multi-select blocks | Mod+Click or Shift+Click additional blocks, or Shift+drag a selection box | -| Copy/Paste blocks | Mod+C / Mod+V | -| Duplicate/Delete blocks | Right-click → action | -| Rename a block | Click block name in header | -| Enable/Disable block | Right-click → Enable/Disable | -| Lock/Unlock block | Hover block → Click lock icon (Admin only) | -| Toggle handle orientation | Right-click → Toggle Handles | -| Open a block in the Editor panel | Right-click → Open Editor | -| Move a block out of a loop/parallel | Right-click → Remove from Subflow | -| Configure a block | Select block → use Editor panel on right | - -## Quick Reference — Connections -| Action | How | -|--------|-----| -| Create connection | Drag from output handle to input handle | -| Delete connection | Click edge to select → Delete key | -| Use output in another block | Drag connection tag into input field | - -## Quick Reference — Running & Testing -| Action | How | -|--------|-----| -| Run workflow | Click Run Workflow button or Mod+Enter | -| Stop workflow | Click Stop button or Mod+Enter while running | -| Test with chat | Use Chat panel on the right side | -| Run from block | Hover block → Click play button, or right-click → Run from block | -| Run until block | Right-click block → Run until block | -| View execution logs | Open terminal panel at bottom, or Mod+L | -| Filter/Search/Copy/Clear logs | Terminal panel controls | - -## Quick Reference — Deployment -| Action | How | -|--------|-----| -| Deploy workflow | Click Deploy button in panel | -| Update deployment | Click Update when changes are detected | -| Revert deployment | Previous versions in Deploy tab → Promote to live | -| Copy API endpoint | Deploy tab → API → Copy API cURL | - -## Quick Reference — Variables -| Action | How | -|--------|-----| -| Add/Edit/Delete workflow variable | Panel → Variables → Add Variable | -| Add environment variable | Settings → Environment Variables → Add | -| Reference workflow variable | Use syntax | -| Reference environment variable | Use {{ENV_VAR}} syntax | -` diff --git a/apps/sim/lib/copilot/tools/handlers/platform.ts b/apps/sim/lib/copilot/tools/handlers/platform.ts deleted file mode 100644 index f5cc43f910b..00000000000 --- a/apps/sim/lib/copilot/tools/handlers/platform.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' -import { PLATFORM_ACTIONS_CONTENT } from './platform-actions' - -export async function executeGetPlatformActions( - _rawParams: Record, - _context: ExecutionContext -): Promise { - return { success: true, output: { content: PLATFORM_ACTIONS_CONTENT } } -} diff --git a/apps/sim/lib/copilot/tools/handlers/vfs.test.ts b/apps/sim/lib/copilot/tools/handlers/vfs.test.ts index 283f63c0709..b360fd5b2af 100644 --- a/apps/sim/lib/copilot/tools/handlers/vfs.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/vfs.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TOOL_RESULT_MAX_INLINE_CHARS } from '@/lib/copilot/constants' const { getOrMaterializeVFS } = vi.hoisted(() => ({ @@ -702,3 +702,141 @@ describe('vfs uploads are opt-in (like recently-deleted/)', () => { expect(grepChatUpload).toHaveBeenCalledWith('report.json', 'chat-1', 'x', expect.any(Object)) }) }) + +describe('vfs handlers docs corpus routing', () => { + const fetchMock = vi.fn() + const DOCS_PAGE = 'docs/workflows/blocks/agent.mdx' + + beforeEach(() => { + vi.clearAllMocks() + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('globs the docs corpus without materializing the workspace VFS', async () => { + const result = await executeVfsGlob({ pattern: 'docs/**' }, GREP_CTX) + + expect(result.success).toBe(true) + expect((result.output as { files: string[] }).files).toContain(DOCS_PAGE) + expect(getOrMaterializeVFS).not.toHaveBeenCalled() + }) + + it('reads a docs page via the live-site fetch, not the workspace VFS', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + headers: new Headers(), + text: async () => 'line one\nline two', + }) + + const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX) + + expect(result.success).toBe(true) + expect(result.output).toEqual({ content: 'line one\nline two', totalLines: 2 }) + expect(getOrMaterializeVFS).not.toHaveBeenCalled() + }) + + it('surfaces DocsCorpusError messages verbatim from read, without fetching', async () => { + const unknown = await executeVfsRead({ path: 'docs/not-a-real-page.mdx' }, GREP_CTX) + expect(unknown.success).toBe(false) + expect(unknown.error).toContain('Docs page not found') + + const dir = await executeVfsRead({ path: 'docs/workflows/blocks' }, GREP_CTX) + expect(dir.success).toBe(false) + expect(dir.error).toContain('is a directory') + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('greps one docs page and rejects directory scope without touching the workspace VFS', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + headers: new Headers(), + text: async () => 'alpha\ncron beta\ngamma', + }) + + const single = await executeVfsGrep({ pattern: 'cron', path: DOCS_PAGE }, GREP_CTX) + expect(single.success).toBe(true) + + const directory = await executeVfsGrep( + { pattern: 'cron', path: 'docs/workflows', maxResults: 10_000 }, + GREP_CTX + ) + expect(directory.success).toBe(false) + expect(directory.error).toContain('grep must target one docs page') + expect(fetchMock).toHaveBeenCalledOnce() + + const invalid = await executeVfsGrep({ pattern: 'cron', path: 'docs/not-a-page.mdx' }, GREP_CTX) + expect(invalid.success).toBe(false) + expect(invalid.error).toContain('not a docs page') + expect(getOrMaterializeVFS).not.toHaveBeenCalled() + }) + + it('truncates an oversized multi-line docs page to fit the inline cap', async () => { + const line = 'y'.repeat(200) + const totalLines = Math.ceil((TOOL_RESULT_MAX_INLINE_CHARS * 2) / (line.length + 1)) + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + headers: new Headers(), + text: async () => Array.from({ length: totalLines }, () => line).join('\n'), + }) + + const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX) + + expect(result.success).toBe(true) + const output = result.output as { content: string; totalLines: number } + expect(output.totalLines).toBe(totalLines) + expect(output.content).toContain('[Page truncated: returned lines 1-') + expect(output.content).toMatch(/offset: \d+ and limit: \d+/) + expect(output.content).toContain('reduce the limit if that window is still too large') + expect(JSON.stringify(output).length).toBeLessThanOrEqual(TOOL_RESULT_MAX_INLINE_CHARS) + }) + + it('fails a docs page whose single line cannot fit inline instead of returning it oversized', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + headers: new Headers(), + text: async () => 'z'.repeat(TOOL_RESULT_MAX_INLINE_CHARS + 1000), + }) + + const result = await executeVfsRead({ path: DOCS_PAGE }, GREP_CTX) + + expect(result.success).toBe(false) + expect(result.error).toContain('Grep this page') + }) + + it('rejects an explicit window that still overflows instead of truncating it', async () => { + const line = 'y'.repeat(200) + const totalLines = Math.ceil((TOOL_RESULT_MAX_INLINE_CHARS * 2) / (line.length + 1)) + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + headers: new Headers(), + text: async () => Array.from({ length: totalLines }, () => line).join('\n'), + }) + + const result = await executeVfsRead({ path: DOCS_PAGE, offset: 0, limit: totalLines }, GREP_CTX) + + expect(result.success).toBe(false) + expect(result.error).toContain('still too large over the requested window') + }) + + it('forwards caller cancellation to docs read and grep without fetching', async () => { + const controller = new AbortController() + controller.abort(new Error('user stopped docs tool')) + const context = { ...GREP_CTX, abortSignal: controller.signal } + + const read = await executeVfsRead({ path: DOCS_PAGE }, context) + const grep = await executeVfsGrep({ pattern: 'agent', path: DOCS_PAGE }, context) + + expect(read).toEqual({ success: false, error: 'user stopped docs tool' }) + expect(grep).toEqual({ success: false, error: 'user stopped docs tool' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/handlers/vfs.ts b/apps/sim/lib/copilot/tools/handlers/vfs.ts index ba8bd734bca..1d07c8aa751 100644 --- a/apps/sim/lib/copilot/tools/handlers/vfs.ts +++ b/apps/sim/lib/copilot/tools/handlers/vfs.ts @@ -4,6 +4,14 @@ import { resolveCopilotKnowledgePrincipal } from '@/lib/copilot/application/exec import { resolveCopilotFilePrincipal } from '@/lib/copilot/auth/file-delegation' import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { TOOL_RESULT_MAX_INLINE_CHARS } from '@/lib/copilot/constants' +import { + couldMatchDocsScope, + DocsCorpusError, + globDocs, + grepDocs, + isDocsPath, + readDocsPage, +} from '@/lib/copilot/docs/docs-corpus' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' import { getOrMaterializeVFS } from '@/lib/copilot/vfs' import type { GrepCountEntry, GrepMatch } from '@/lib/copilot/vfs/operations' @@ -126,6 +134,42 @@ async function canReturnWorkspaceFileValue( return true } +/** + * Trim an oversized docs page to a whole-line prefix that fits the + * inline budget, preserving the true `totalLines` so the model can page through + * the rest with offset/limit. Returns null when not even one line fits — a + * single line longer than the cap — so the caller can fail instead of returning + * an over-cap payload as success. The notice offers grep as an alternative to + * another read because either operation fetches the page once. + */ +function truncateDocsPageToInlineCap(page: { content: string; totalLines: number }): { + output: { content: string; totalLines: number } + returnedLines: number +} | null { + const lines = page.content.split('\n') + const notice = (shown: number) => + `\n\n[Page truncated: returned lines 1-${shown} of ${page.totalLines}. To continue, read this path with offset: ${shown} and limit: ${shown}; reduce the limit if that window is still too large. To jump straight to a section, grep this path INSTEAD of reading it — grep is the same single fetch and returns only matching lines with their numbers.]` + + let kept = lines.length + while (kept > 0) { + const content = `${lines.slice(0, kept).join('\n')}${notice(kept)}` + if ( + serializedResultSize({ content, totalLines: page.totalLines }) <= TOOL_RESULT_MAX_INLINE_CHARS + ) { + return { output: { content, totalLines: page.totalLines }, returnedLines: kept } + } + kept = Math.floor(kept / 2) + } + return null +} + +/** + * Routes grep by content source. `docs/` uses one network-backed docs + * page; `uploads/` uses one chat-scoped upload; workspace file paths use + * one authorized file; all remaining paths use the materialized in-memory VFS. + * External and dynamic file contents are therefore opt-in and single-target, + * while an unscoped grep searches only static VFS resources and metadata. + */ export async function executeVfsGrep( params: Record, context: ExecutionContext @@ -152,16 +196,11 @@ export async function executeVfsGrep( context: (params.context as number) ?? 0, } - // Routing mirrors read/glob: - // - uploads/ -> grep one chat upload's content (chat-scoped) - // - files/ -> grep one workspace file's content (one file only) - // - everything else -> grep the in-memory VFS map (workflow JSON, metadata) - // Chat uploads are opt-in like recently-deleted/: they are never in the VFS - // map, so an unscoped grep can't touch them — only an explicit uploads/ - // path does, and only one upload at a time. let result: GrepMatch[] | string[] | GrepCountEntry[] let provenanceFile: WorkspaceFileSecretProvenanceIdentity | undefined - if (isChatUploadGrepPath(rawPath)) { + if (rawPath !== undefined && isDocsPath(rawPath)) { + result = await grepDocs(rawPath, pattern, grepOptions, context.abortSignal) + } else if (isChatUploadGrepPath(rawPath)) { if (!context.chatId) { return { success: false, error: 'No chat context available for uploads/' } } @@ -223,8 +262,8 @@ export async function executeVfsGrep( } catch (err) { // Expected single-file scoping / no-text / too-large conditions: surface the // message verbatim instead of logging an internal failure. - if (err instanceof WorkspaceFileGrepError) { - logger.debug('vfs_grep workspace file rejected', { + if (err instanceof WorkspaceFileGrepError || err instanceof DocsCorpusError) { + logger.debug('vfs_grep single-file scope rejected', { pattern, path: rawPath, error: err.message, @@ -255,6 +294,12 @@ export async function executeVfsGlob( } try { + if (couldMatchDocsScope(pattern)) { + const files = globDocs(pattern) + logger.debug('vfs_glob docs result', { pattern, fileCount: files.length }) + return { success: true, output: { files } } + } + const vfs = await getGatedVFS(context) let files = vfs.glob(pattern) @@ -323,6 +368,34 @@ export async function executeVfsRead( } } + if (isDocsPath(path)) { + const page = await readDocsPage(path, context.abortSignal) + const windowed = applyWindow(page) + if (serializedResultSize(windowed) > TOOL_RESULT_MAX_INLINE_CHARS) { + if (offset !== undefined || limit !== undefined) { + return { + success: false, + error: `${path} is still too large over the requested window. Narrow offset/limit, or grep this page for the section you need.`, + } + } + const truncated = truncateDocsPageToInlineCap(page) + if (!truncated) { + return { + success: false, + error: `${path} is too large to return inline even truncated. Grep this page for the section you need.`, + } + } + logger.debug('vfs_read truncated oversized docs page', { + path, + totalLines: page.totalLines, + returnedLines: truncated.returnedLines, + }) + return { success: true, output: truncated.output } + } + logger.debug('vfs_read resolved docs page', { path, totalLines: page.totalLines }) + return { success: true, output: windowed } + } + // Handle chat-scoped uploads via the uploads/ virtual prefix. // Uploads are flat and have no metadata/content split like files/ — the upload // IS the first path segment after uploads/. Any trailing segment (e.g. a @@ -482,6 +555,10 @@ export async function executeVfsRead( output: result, } } catch (err) { + if (err instanceof DocsCorpusError) { + logger.debug('vfs_read docs page rejected', { path, error: err.message }) + return { success: false, error: err.message } + } logger.error('vfs_read failed', { path, error: toError(err).message, diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/queries.test.ts b/apps/sim/lib/copilot/tools/handlers/workflow/queries.test.ts index f8f33c8289e..801372da6f1 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/queries.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/queries.test.ts @@ -2,8 +2,9 @@ import { getErrorMessage } from '@sim/utils/errors' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutionContext } from '@/lib/copilot/request/types' -const { executeWorkflowUseCaseMock } = vi.hoisted(() => ({ +const { executeWorkflowUseCaseMock, listUserWorkspacesMock } = vi.hoisted(() => ({ executeWorkflowUseCaseMock: vi.fn(), + listUserWorkspacesMock: vi.fn(), })) vi.mock('@/lib/copilot/application/execute-workflow-use-case', () => ({ @@ -12,7 +13,54 @@ vi.mock('@/lib/copilot/application/execute-workflow-use-case', () => ({ getErrorMessage(error, 'Workflow operation failed'), })) -import { executeGetBlockOutputs } from './queries' +vi.mock('@/lib/workspaces/utils', () => ({ + listUserWorkspaces: listUserWorkspacesMock, +})) + +import { + executeGetBlockOutputs, + executeListUserWorkspaces, +} from '@/lib/copilot/tools/handlers/workflow/queries' + +describe('executeListUserWorkspaces', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('marks the current workspace in the accessible workspace list', async () => { + listUserWorkspacesMock.mockResolvedValue([ + { workspaceId: 'workspace-1', workspaceName: 'One', role: 'owner' }, + { workspaceId: 'workspace-2', workspaceName: 'Two', role: 'read' }, + ]) + + const result = await executeListUserWorkspaces({ + userId: 'user-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-2', + }) + + expect(listUserWorkspacesMock).toHaveBeenCalledWith('user-1') + expect(result).toEqual({ + success: true, + output: { + workspaces: [ + { + workspaceId: 'workspace-1', + workspaceName: 'One', + role: 'owner', + isCurrent: false, + }, + { + workspaceId: 'workspace-2', + workspaceName: 'Two', + role: 'read', + isCurrent: true, + }, + ], + }, + }) + }) +}) describe('executeGetBlockOutputs', () => { beforeEach(() => { diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/queries.ts b/apps/sim/lib/copilot/tools/handlers/workflow/queries.ts index 8861dbc98d6..0291fdee8fc 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/queries.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/queries.ts @@ -34,7 +34,10 @@ export async function executeListUserWorkspaces( context: ExecutionContext ): Promise { try { - const workspaces = await listUserWorkspaces(context.userId) + const workspaces = (await listUserWorkspaces(context.userId)).map((workspace) => ({ + ...workspace, + isCurrent: workspace.workspaceId === context.workspaceId, + })) return { success: true, output: { workspaces } } } catch (error) { diff --git a/apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts b/apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts new file mode 100644 index 00000000000..3634648b5b8 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/docs/search-docs-dispatch.test.ts @@ -0,0 +1,39 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { TOOL_CATALOG } from '@/lib/copilot/generated/tool-catalog-v1' +import { isKnownTool, isSimExecuted } from '@/lib/copilot/tool-executor/router' +import { getHiddenToolNames } from '@/lib/copilot/tools/client/hidden-tools' +import { getRegisteredServerToolNames } from '@/lib/copilot/tools/server/router' + +/** + * `executeTool` gates on `isKnownTool` (catalog membership) before it ever + * consults the handler registry, so a sim-routed tool needs every link of this + * chain or dispatch rejects it before the handler is reached. These assertions + * pin that chain for search_docs. + */ +describe('search_docs dispatch chain', () => { + it('is in the catalog, so dispatch does not reject it as unknown', () => { + expect(isKnownTool('search_docs')).toBe(true) + }) + + it('routes to sim, so dispatch reaches the server tool registry', () => { + expect(isSimExecuted('search_docs')).toBe(true) + }) + + it('has a registered server handler', () => { + expect(getRegisteredServerToolNames()).toContain('search_docs') + }) +}) + +describe('removed docs-tool ids', () => { + for (const removed of ['search_documentation', 'get_platform_actions']) { + it(`${removed} is absent from the catalog, registries, and hidden-tool set`, () => { + expect(TOOL_CATALOG[removed]).toBeUndefined() + expect(isKnownTool(removed)).toBe(false) + expect(getRegisteredServerToolNames()).not.toContain(removed) + expect(getHiddenToolNames().has(removed)).toBe(false) + }) + } +}) diff --git a/apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts b/apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts new file mode 100644 index 00000000000..1b01c4c8dd5 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/docs/search-docs.test.ts @@ -0,0 +1,152 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { DocsSearchOutcome } from '@/lib/copilot/docs/docs-search' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const { mockSearchDocs } = vi.hoisted(() => ({ + mockSearchDocs: vi.fn(), +})) + +vi.mock('@/lib/copilot/docs/docs-search', () => ({ + searchDocs: mockSearchDocs, +})) + +import { searchDocsServerTool } from '@/lib/copilot/tools/server/docs/search-docs' + +function outcome(overrides: Partial): DocsSearchOutcome { + return { + results: [], + candidatesConsidered: 0, + droppedBelowThreshold: 0, + droppedStale: 0, + ...overrides, + } +} + +const RESULT = { + path: 'docs/agents.mdx', + url: 'https://docs.sim.ai/agents', + title: 'Agents', + content: 'body', + similarity: 0.9, +} + +const CONTEXT = { + userId: 'user-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), +} + +describe('searchDocsServerTool', () => { + beforeEach(() => { + mockSearchDocs.mockReset() + }) + + it('forwards query, path, and topK to the search layer', async () => { + mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 })) + + const output = await searchDocsServerTool.execute( + { + query: 'how do agents work', + path: 'docs/agents.mdx', + topK: 7, + }, + CONTEXT + ) + + expect(mockSearchDocs).toHaveBeenCalledWith('how do agents work', { + path: 'docs/agents.mdx', + topK: 7, + }) + expect(output).toEqual({ + results: [RESULT], + query: 'how do agents work', + totalResults: 1, + }) + }) + + it('omits the note when nothing was dropped', async () => { + mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 })) + + const output = await searchDocsServerTool.execute({ query: 'q' }, CONTEXT) + + expect(output.note).toBeUndefined() + }) + + it('explains when the index returns no candidates', async () => { + mockSearchDocs.mockResolvedValue(outcome({})) + + const output = await searchDocsServerTool.execute({ query: 'brand new feature' }, CONTEXT) + + expect(output.note).toContain('search index may lag') + expect(output.note).toContain('read it directly') + expect(output.note).toContain('glob("docs/**")') + }) + + it('explains an empty result set caused by filtering, so it does not read as missing docs', async () => { + mockSearchDocs.mockResolvedValue( + outcome({ candidatesConsidered: 2, droppedBelowThreshold: 1, droppedStale: 1 }) + ) + + const output = await searchDocsServerTool.execute({ query: 'q' }, CONTEXT) + + expect(output.note).toContain('does NOT mean the docs lack this topic') + expect(output.note).toContain('1 scored too low') + expect(output.note).toContain('1 point at pages no longer in the docs') + }) + + it('notes threshold-only drops on a partial result set', async () => { + mockSearchDocs.mockResolvedValue( + outcome({ results: [RESULT], candidatesConsidered: 3, droppedBelowThreshold: 2 }) + ) + + const output = await searchDocsServerTool.execute({ query: 'q' }, CONTEXT) + + expect(output.note).toContain('Returned 1 of 3 candidate(s)') + expect(output.note).toContain('2 scored too low') + expect(output.note).not.toContain('no longer in the docs') + }) + + it('notes stale-only drops on a partial result set', async () => { + mockSearchDocs.mockResolvedValue( + outcome({ results: [RESULT], candidatesConsidered: 2, droppedStale: 1 }) + ) + + const output = await searchDocsServerTool.execute({ query: 'q' }, CONTEXT) + + expect(output.note).toContain('1 point at pages no longer in the docs') + expect(output.note).not.toContain('scored too low') + }) + + it('projects resolved secrets before embedding or returning the query', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'DOCS_QUERY', + plaintext: 'private docs query', + encryptedValue: 'ciphertext', + }, + ]) + registry.recordResolved('DOCS_QUERY', 'private docs query') + mockSearchDocs.mockResolvedValue(outcome({ results: [RESULT], candidatesConsidered: 1 })) + + const output = await searchDocsServerTool.execute( + { query: 'private docs query' }, + { userId: 'user-1', resolvedSecretTraceRegistry: registry } + ) + + expect(mockSearchDocs).toHaveBeenCalledWith('{{DOCS_QUERY}}', { + path: undefined, + topK: undefined, + }) + expect(output.query).toBe('{{DOCS_QUERY}}') + expect(JSON.stringify(output)).not.toContain('private docs query') + }) + + it('fails closed when secret provenance is unavailable', async () => { + await expect(searchDocsServerTool.execute({ query: 'query' })).rejects.toThrow( + 'Docs search query could not be processed safely' + ) + expect(mockSearchDocs).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/docs/search-docs.ts b/apps/sim/lib/copilot/tools/server/docs/search-docs.ts new file mode 100644 index 00000000000..7603bc9e6b2 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/docs/search-docs.ts @@ -0,0 +1,79 @@ +import type { DocsSearchResult } from '@/lib/copilot/docs/docs-search' +import { searchDocs } from '@/lib/copilot/docs/docs-search' +import { SearchDocs } from '@/lib/copilot/generated/tool-catalog-v1' +import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool' +import { ServerToolModelInputError } from '@/lib/copilot/tools/server/model-input' +import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' + +interface SearchDocsParams { + query: string + topK?: number + path?: string +} + +interface SearchDocsOutput { + results: DocsSearchResult[] + query: string + totalResults: number + /** + * Present only when the vector search matched chunks that were then filtered + * out. Without it an empty result set reads as "the docs do not cover this", + * which sends the caller off to guess instead of rephrasing or falling back + * to glob. + */ + note?: string +} + +/** + * Explain a short or empty result set in terms the caller can act on. Returns + * undefined when nothing was dropped — the common case needs no commentary. + */ +function shortfallNote(outcome: Awaited>): string | undefined { + const { results, candidatesConsidered, droppedBelowThreshold, droppedStale } = outcome + if (results.length === 0 && candidatesConsidered === 0) { + return 'No indexed candidates were returned. The search index may lag the live docs. If you know the page, read it directly; otherwise use glob("docs/**") to find the current path.' + } + if (droppedBelowThreshold === 0 && droppedStale === 0) return undefined + + const reasons: string[] = [] + if (droppedBelowThreshold > 0) + reasons.push(`${droppedBelowThreshold} scored too low to be relevant`) + if (droppedStale > 0) { + reasons.push( + `${droppedStale} point at pages no longer in the docs (the search index lags the site)` + ) + } + const dropped = reasons.join(' and ') + + return results.length === 0 + ? `No relevant matches. The search index returned ${candidatesConsidered} candidate(s), but ${dropped} — this does NOT mean the docs lack this topic. Rephrase the query, widen it by dropping the path scope, or browse with glob("docs/**").` + : `Returned ${results.length} of ${candidatesConsidered} candidate(s); ${dropped}. Rephrase or widen the query if these look off-topic.` +} + +/** + * Vector search over Sim's product documentation, scoped to the same pages the + * agent can `read` from the `docs/` VFS tree. Normal delegation exposes it to + * the platform agent; the `@Docs` compatibility path also invokes it directly. + * Corpus logic lives in `@/lib/copilot/docs/docs-search`. + */ +export const searchDocsServerTool: BaseServerTool = { + name: SearchDocs.id, + async execute(params: SearchDocsParams, context?: ServerToolContext): Promise { + const queryProjection = projectResolvedSecretModelContent( + params.query, + context?.resolvedSecretTraceRegistry + ) + if (!queryProjection.safe || typeof queryProjection.value !== 'string') { + throw new ServerToolModelInputError('Docs search query could not be processed safely') + } + const query = queryProjection.value + const outcome = await searchDocs(query, { path: params.path, topK: params.topK }) + const note = shortfallNote(outcome) + return { + results: outcome.results, + query, + totalResults: outcome.results.length, + ...(note ? { note } : {}), + } + }, +} diff --git a/apps/sim/lib/copilot/tools/server/docs/search-documentation.test.ts b/apps/sim/lib/copilot/tools/server/docs/search-documentation.test.ts deleted file mode 100644 index 14693f75913..00000000000 --- a/apps/sim/lib/copilot/tools/server/docs/search-documentation.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @vitest-environment node - */ -import { loggerMock } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockGenerateSearchEmbedding } = vi.hoisted(() => ({ - mockGenerateSearchEmbedding: vi.fn(), -})) - -vi.mock('@/lib/copilot/generated/tool-catalog-v1', () => ({ - SearchDocumentation: { id: 'search_documentation' }, -})) -vi.mock('@/lib/knowledge/embeddings', () => ({ - generateSearchEmbedding: mockGenerateSearchEmbedding, -})) - -import { searchDocumentationServerTool } from '@/lib/copilot/tools/server/docs/search-documentation' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -describe('documentation search model boundary', () => { - beforeEach(() => { - vi.clearAllMocks() - mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [], isBYOK: false }) - }) - - it('preserves a query that merely collides with ambient secret plaintext', async () => { - const registry = new ResolvedSecretTraceRegistry([ - { - name: 'DOCS_QUERY', - plaintext: 'private documentation query', - encryptedValue: 'encrypted-query', - }, - ]) - registry.recordResolved('DOCS_QUERY', 'private documentation query') - - const result = await searchDocumentationServerTool.execute( - { query: 'private documentation query' }, - { userId: 'user-1', resolvedSecretTraceRegistry: registry } - ) - - expect(mockGenerateSearchEmbedding).toHaveBeenCalledWith('private documentation query') - expect(result).toEqual({ - results: [], - query: 'private documentation query', - totalResults: 0, - }) - - const logger = loggerMock.createLogger.mock.results.at(-1)?.value - expect(logger?.info).toHaveBeenCalledWith('Executing docs search', { - queryLength: 'private documentation query'.length, - topK: 10, - }) - expect(JSON.stringify(logger?.info.mock.calls)).not.toContain('private documentation query') - }) -}) diff --git a/apps/sim/lib/copilot/tools/server/docs/search-documentation.ts b/apps/sim/lib/copilot/tools/server/docs/search-documentation.ts deleted file mode 100644 index ad14c3937a6..00000000000 --- a/apps/sim/lib/copilot/tools/server/docs/search-documentation.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { db } from '@sim/db' -import { docsEmbeddings } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { sql } from 'drizzle-orm' -import { SearchDocumentation } from '@/lib/copilot/generated/tool-catalog-v1' -import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool' -import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' - -interface DocsSearchParams { - query: string - topK?: number - threshold?: number -} - -const DEFAULT_DOCS_SIMILARITY_THRESHOLD = 0.3 - -export const searchDocumentationServerTool: BaseServerTool = { - name: SearchDocumentation.id, - async execute(params: DocsSearchParams): Promise { - const logger = createLogger('SearchDocumentationServerTool') - const { query, topK = 10, threshold } = params - if (!query || typeof query !== 'string') throw new Error('query is required') - - logger.info('Executing docs search', { queryLength: query.length, topK }) - - const similarityThreshold = threshold ?? DEFAULT_DOCS_SIMILARITY_THRESHOLD - - const modelQuery = query - const { embedding: queryEmbedding } = await generateSearchEmbedding(modelQuery) - if (!queryEmbedding || queryEmbedding.length === 0) { - return { results: [], query, totalResults: 0 } - } - - const results = await db - .select({ - chunkId: docsEmbeddings.chunkId, - chunkText: docsEmbeddings.chunkText, - sourceDocument: docsEmbeddings.sourceDocument, - sourceLink: docsEmbeddings.sourceLink, - headerText: docsEmbeddings.headerText, - headerLevel: docsEmbeddings.headerLevel, - similarity: sql`1 - (${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector)`, - }) - .from(docsEmbeddings) - .orderBy(sql`${docsEmbeddings.embedding} <=> ${JSON.stringify(queryEmbedding)}::vector`) - .limit(topK) - - const filteredResults = results.filter((r) => r.similarity >= similarityThreshold) - const documentationResults = filteredResults.map((r, idx) => ({ - id: idx + 1, - title: String(r.headerText || 'Untitled Section'), - url: String(r.sourceLink || '#'), - content: String(r.chunkText || ''), - similarity: r.similarity, - })) - - logger.info('Docs search complete', { count: documentationResults.length }) - return { results: documentationResults, query, totalResults: documentationResults.length } - }, -} diff --git a/apps/sim/lib/copilot/tools/server/router.ts b/apps/sim/lib/copilot/tools/server/router.ts index 7d87b432218..ab6a882b30e 100644 --- a/apps/sim/lib/copilot/tools/server/router.ts +++ b/apps/sim/lib/copilot/tools/server/router.ts @@ -24,7 +24,7 @@ import { } from '@/lib/copilot/tools/server/base-tool' import { getBlocksMetadataServerTool } from '@/lib/copilot/tools/server/blocks/get-blocks-metadata-tool' import { getTriggerBlocksServerTool } from '@/lib/copilot/tools/server/blocks/get-trigger-blocks' -import { searchDocumentationServerTool } from '@/lib/copilot/tools/server/docs/search-documentation' +import { searchDocsServerTool } from '@/lib/copilot/tools/server/docs/search-docs' import { enrichmentRunServerTool } from '@/lib/copilot/tools/server/enrichment/enrichment-run' import { createFileServerTool } from '@/lib/copilot/tools/server/files/create-file' import { downloadToWorkspaceFileServerTool } from '@/lib/copilot/tools/server/files/download-to-workspace-file' @@ -168,7 +168,7 @@ const baseServerToolRegistry: Record = { [getTriggerBlocksServerTool.name]: getTriggerBlocksServerTool, [editWorkflowServerTool.name]: editWorkflowServerTool, [queryLogsServerTool.name]: queryLogsServerTool, - [searchDocumentationServerTool.name]: searchDocumentationServerTool, + [searchDocsServerTool.name]: searchDocsServerTool, [searchOnlineServerTool.name]: searchOnlineServerTool, [setEnvironmentVariablesServerTool.name]: setEnvironmentVariablesServerTool, [getCredentialsServerTool.name]: getCredentialsServerTool, diff --git a/apps/sim/lib/copilot/tools/tool-display.test.ts b/apps/sim/lib/copilot/tools/tool-display.test.ts index 027ce68d915..94cfedd34af 100644 --- a/apps/sim/lib/copilot/tools/tool-display.test.ts +++ b/apps/sim/lib/copilot/tools/tool-display.test.ts @@ -78,6 +78,24 @@ describe('getToolDisplayTitle natural-language coverage', () => { expect(getToolDisplayTitle('diff_workflows')).toBe('Comparing workflows') }) + it('includes the query in search_docs titles', () => { + expect(getToolDisplayTitle('search_docs')).toBe('Searching Sim docs') + expect(getToolDisplayTitle('search_docs', { query: 'loop blocks iteration' })).toBe( + 'Searching Sim docs for "loop blocks iteration"' + ) + expect( + getToolCompletedTitle( + getToolDisplayTitle('search_docs', { query: 'how to read workflow logs' }) + ) + ).toBe('Searched Sim docs for "how to read workflow logs"') + expect( + getToolDisplayTitle('search_docs', { + query: + 'reference block outputs connection tags blockname.field pass data between blocks in a workflow', + })?.length + ).toBeLessThanOrEqual('Searching Sim docs for ""'.length + 60 + '...'.length) + }) + it('falls back to running code for function_execute without a title', () => { expect(getToolDisplayTitle('function_execute')).toBe('Running code') expect(getToolDisplayTitle('function_execute', { title: 'Crunching numbers' })).toBe( diff --git a/apps/sim/lib/copilot/tools/tool-display.ts b/apps/sim/lib/copilot/tools/tool-display.ts index f15c80c5224..8f77f4cd856 100644 --- a/apps/sim/lib/copilot/tools/tool-display.ts +++ b/apps/sim/lib/copilot/tools/tool-display.ts @@ -1,4 +1,4 @@ -import { stripVersionSuffix } from '@sim/utils/string' +import { stripVersionSuffix, truncate } from '@sim/utils/string' /** * Single source of truth for copilot tool-call display titles. @@ -503,7 +503,7 @@ const TOOL_TITLES: Record = { restore_resource: 'Restoring resource', run_block: 'Running block', scheduled_task: 'Managing scheduled task', - search_documentation: 'Searching documentation', + search_docs: 'Searching Sim docs', search_patterns: 'Searching patterns', set_block_enabled: 'Toggling block', set_environment_variables: 'Setting environment variables', @@ -803,6 +803,10 @@ export function getToolDisplayTitle(name: string, args?: Record const target = firstStringArg(args, 'toolTitle', 'title') return target ? `Searching online for ${target}` : 'Searching online' } + case 'search_docs': { + const target = firstStringArg(args, 'toolTitle', 'title', 'query') + return target ? `Searching Sim docs for "${truncate(target, 60)}"` : 'Searching Sim docs' + } case 'grep': { const target = firstStringArg(args, 'toolTitle', 'title') return target ? `Searching for ${target}` : 'Searching' diff --git a/package.json b/package.json index 81f5d23783b..2826be7b11c 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,8 @@ "metrics-contract:check": "bun run scripts/sync-metrics-contract.ts --check", "vfs-snapshot-contract:generate": "bun run scripts/sync-vfs-snapshot-contract.ts", "vfs-snapshot-contract:check": "bun run scripts/sync-vfs-snapshot-contract.ts --check", + "docs-manifest:generate": "bun run scripts/sync-docs-manifest.ts", + "docs-manifest:check": "bun run scripts/sync-docs-manifest.ts --check", "mship:generate": "bun run scripts/generate-mship-contracts.ts", "mship:check": "bun run scripts/generate-mship-contracts.ts --check", "library:covers": "bun run scripts/generate-library-covers.tsx", diff --git a/scripts/sync-docs-manifest.ts b/scripts/sync-docs-manifest.ts new file mode 100644 index 00000000000..35a240bd836 --- /dev/null +++ b/scripts/sync-docs-manifest.ts @@ -0,0 +1,113 @@ +/** + * Generate the static docs manifest the copilot's `docs/` VFS tree is built from. + * + * Source of truth: `apps/docs/content/docs/en/**\/*.mdx` — the English docs + * corpus, whose folder structure mirrors the public docs.sim.ai URL structure. + * The copilot never reads those files from disk (they are not deployed with + * `apps/sim`); it globs this manifest for structure and fetches page content + * from the live site on demand. That makes the manifest the one thing that can + * drift, hence `--check` in CI. + * + * Path derivation (each entry is BOTH the `docs/`-relative VFS path and the + * docs.sim.ai URL path, so a read is a plain fetch of `https://docs.sim.ai/`): + * - `workflows/blocks/agent.mdx` → `workflows/blocks/agent.mdx` + * - `workflows/index.mdx` → `workflows.mdx` (fumadocs folds index pages + * into their parent URL; `/workflows/index.mdx` + * is a 404 on the site) + * + * Excluded, and intentionally absent from the VFS: every section in + * `UNMOUNTED_DOCS_SECTIONS` (fetch those with the scrape tool if ever needed), + * the root `index.mdx` (its URL is `/`, which redirects), and every non-`en` + * locale. + * + * Usage: + * bun run docs-manifest:generate # write the manifest + * bun run docs-manifest:check # fail (exit 1) if the manifest is stale + */ +import { readdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { foldDocsIndexPath, UNMOUNTED_DOCS_SECTIONS } from '../apps/sim/lib/copilot/docs/docs-path' +import { formatGeneratedSource } from './format-generated-source' + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) +const ROOT = resolve(SCRIPT_DIR, '..') +const DOCS_CONTENT_DIR = resolve(ROOT, 'apps/docs/content/docs/en') +const OUTPUT_PATH = resolve(ROOT, 'apps/sim/lib/copilot/generated/docs-manifest.ts') + +/** + * Top-level docs sections deliberately left out of the copilot's `docs/` tree. + * Shared with the vector search's unscoped filter so readability and + * findability cannot drift apart — see `UNMOUNTED_DOCS_SECTIONS`. + */ +const EXCLUDED_SECTIONS = new Set(UNMOUNTED_DOCS_SECTIONS) + +/** Collect every `.mdx` file under `dir`, as paths relative to {@link DOCS_CONTENT_DIR}. */ +async function collectMdxPaths(dir: string, prefix = ''): Promise { + const entries = await readdir(dir, { withFileTypes: true }) + const paths: string[] = [] + for (const entry of entries) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name + if (entry.isDirectory()) { + if (prefix === '' && EXCLUDED_SECTIONS.has(entry.name)) continue + paths.push(...(await collectMdxPaths(resolve(dir, entry.name), relative))) + continue + } + if (entry.isFile() && entry.name.endsWith('.mdx')) paths.push(relative) + } + return paths +} + +/** Map an `en`-relative mdx file path to its docs.sim.ai URL path, or null to drop it. */ +function toDocsPath(mdxPath: string): string | null { + if (mdxPath === 'index.mdx') return null + return foldDocsIndexPath(mdxPath) +} + +function render(paths: string[]): string { + const entries = paths.map((path) => ` '${path}',`).join('\n') + return `/** + * AUTO-GENERATED FILE. DO NOT EDIT. + * Generated from apps/docs/content/docs/en by scripts/sync-docs-manifest.ts. + * Run: bun run docs-manifest:generate. + * + * Every page in the copilot's read-only \`docs/\` VFS tree, as a path that is + * simultaneously the \`docs/\`-relative VFS path and the docs.sim.ai URL path + * (so \`docs/workflows/blocks/agent.mdx\` reads + * \`https://docs.sim.ai/workflows/blocks/agent.mdx\`). Sorted. + */ +export const DOCS_MANIFEST: readonly string[] = [ +${entries} +] +` +} + +async function main() { + const checkOnly = process.argv.includes('--check') + + const mdxPaths = await collectMdxPaths(DOCS_CONTENT_DIR) + const docsPaths = mdxPaths + .map(toDocsPath) + .filter((path): path is string => path !== null) + .sort() + + if (docsPaths.length === 0) { + throw new Error(`No docs pages found under ${DOCS_CONTENT_DIR}`) + } + + const rendered = formatGeneratedSource(render(docsPaths), OUTPUT_PATH, ROOT) + + if (checkOnly) { + const existing = await readFile(OUTPUT_PATH, 'utf8').catch(() => null) + if (existing !== rendered) { + throw new Error( + 'Generated docs manifest is stale — the docs tree changed (page added, removed, or renamed). Run: bun run docs-manifest:generate' + ) + } + return + } + + await writeFile(OUTPUT_PATH, rendered, 'utf8') +} + +await main()