From afb2172201c0eb25effc1c0243a3a497d179bb8c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:53:37 +0000 Subject: [PATCH 1/2] Initial plan From 588d91fa5b0cf4f78666f226b288773f3a98e383 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:00:13 +0000 Subject: [PATCH 2/2] feat: add benchmark tests for TfidfIndex and ToolIndex local search Co-authored-by: glebedel <10488548+glebedel@users.noreply.github.com> --- package.json | 1 + src/local-search.bench.ts | 104 +++++++++++++++++++++++++++++++++ src/utils/tfidf-index.bench.ts | 80 +++++++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 src/local-search.bench.ts create mode 100644 src/utils/tfidf-index.bench.ts diff --git a/package.json b/package.json index d9ed45b..051dc8f 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "preinstall": "npx only-allow pnpm", "prepack": "npm pkg delete scripts.preinstall && pnpm run build", "test": "vitest", + "bench": "vitest bench", "coverage": "vitest run --coverage" }, "dependencies": { diff --git a/src/local-search.bench.ts b/src/local-search.bench.ts new file mode 100644 index 0000000..610caef --- /dev/null +++ b/src/local-search.bench.ts @@ -0,0 +1,104 @@ +/** + * Benchmarks for ToolIndex (hybrid BM25 + TF-IDF search) - index construction + * and search performance at various scales. + */ +import { bench, describe } from 'vitest'; +import { ToolIndex } from './local-search'; +import { BaseTool } from './tool'; + +function createMockTools(count: number): BaseTool[] { + const verbs = ['create', 'update', 'delete', 'get', 'list', 'search']; + const nouns = [ + 'employee', + 'candidate', + 'contact', + 'document', + 'task', + 'event', + 'ticket', + 'account', + 'project', + 'invoice', + ]; + const integrations = [ + 'bamboohr', + 'workday', + 'salesforce', + 'hubspot', + 'jira', + 'slack', + 'github', + 'zendesk', + 'greenhouse', + 'lever', + ]; + + return Array.from({ length: count }, (_, i) => { + const integration = integrations[i % integrations.length] ?? 'bamboohr'; + const verb = verbs[i % verbs.length] ?? 'create'; + const noun = nouns[Math.floor(i / verbs.length) % nouns.length] ?? 'employee'; + const name = `${integration}_${verb}_${noun}`; + + return new BaseTool( + name, + `${verb.charAt(0).toUpperCase() + verb.slice(1)} ${noun} records in ${integration}`, + { type: 'object', properties: {} }, + { + kind: 'http', + method: verb === 'list' || verb === 'get' || verb === 'search' ? 'GET' : 'POST', + url: `https://api.example.com/${noun}s`, + bodyType: 'json', + params: [], + }, + ); + }); +} + +describe('ToolIndex - construction', () => { + bench('construct and initialize index with 10 tools', async () => { + const tools = createMockTools(10); + const index = new ToolIndex(tools); + // Await internal Orama promise to measure total initialization time + // (synchronous TF-IDF build + async BM25 index construction) + await index.search('', 1); + }); + + bench('construct and initialize index with 50 tools', async () => { + const tools = createMockTools(50); + const index = new ToolIndex(tools); + await index.search('', 1); + }); + + bench('construct and initialize index with 200 tools', async () => { + const tools = createMockTools(200); + const index = new ToolIndex(tools); + await index.search('', 1); + }); +}); + +describe('ToolIndex - search', () => { + const smallIndex = new ToolIndex(createMockTools(10)); + const mediumIndex = new ToolIndex(createMockTools(50)); + const largeIndex = new ToolIndex(createMockTools(200)); + + beforeAll(async () => { + // Ensure all indexes are fully initialized before measuring search latency + await Promise.all([ + smallIndex.search('', 1), + mediumIndex.search('', 1), + largeIndex.search('', 1), + ]); + }); + + bench('search in 10-tool index', async () => { + await smallIndex.search('create employee', 5); + }); + + bench('search in 50-tool index', async () => { + await mediumIndex.search('create employee', 5); + }); + + bench('search in 200-tool index', async () => { + await largeIndex.search('create employee', 5); + }); +}); diff --git a/src/utils/tfidf-index.bench.ts b/src/utils/tfidf-index.bench.ts new file mode 100644 index 0000000..e25e722 --- /dev/null +++ b/src/utils/tfidf-index.bench.ts @@ -0,0 +1,80 @@ +/** + * Benchmarks for TfidfIndex - build and search performance at various scales. + */ +import { bench, describe } from 'vitest'; +import { TfidfIndex } from './tfidf-index'; + +function generateCorpus(size: number): { id: string; text: string }[] { + const verbs = ['create', 'update', 'delete', 'get', 'list', 'search', 'fetch', 'sync']; + const nouns = [ + 'employee', + 'candidate', + 'contact', + 'document', + 'task', + 'event', + 'ticket', + 'account', + ]; + const integrations = [ + 'bamboohr', + 'workday', + 'salesforce', + 'hubspot', + 'jira', + 'slack', + 'github', + 'zendesk', + ]; + + return Array.from({ length: size }, (_, i) => { + const integration = integrations[i % integrations.length] ?? 'bamboohr'; + const verb = verbs[i % verbs.length] ?? 'create'; + const noun = nouns[Math.floor(i / verbs.length) % nouns.length] ?? 'employee'; + const name = `${integration}_${verb}_${noun}`; + return { + id: name, + text: `${name} ${verb} ${noun} in the ${integration} integration for ${noun} management`, + }; + }); +} + +describe('TfidfIndex - build', () => { + bench('build index with 10 documents', () => { + const index = new TfidfIndex(); + index.build(generateCorpus(10)); + }); + + bench('build index with 100 documents', () => { + const index = new TfidfIndex(); + index.build(generateCorpus(100)); + }); + + bench('build index with 500 documents', () => { + const index = new TfidfIndex(); + index.build(generateCorpus(500)); + }); +}); + +describe('TfidfIndex - search', () => { + const smallIndex = new TfidfIndex(); + smallIndex.build(generateCorpus(10)); + + const mediumIndex = new TfidfIndex(); + mediumIndex.build(generateCorpus(100)); + + const largeIndex = new TfidfIndex(); + largeIndex.build(generateCorpus(500)); + + bench('search in 10-document index', () => { + smallIndex.search('create employee'); + }); + + bench('search in 100-document index', () => { + mediumIndex.search('create employee'); + }); + + bench('search in 500-document index', () => { + largeIndex.search('create employee'); + }); +});