From 49cca59b70654f2d3e932686ccdc34a2caf420c1 Mon Sep 17 00:00:00 2001 From: Stuart Robinson Date: Fri, 14 Aug 2026 05:26:27 +0700 Subject: [PATCH 1/3] perf(embeddings): speed up and harden index builds --- README.md | 1 + docs/CONFIG.md | 12 +- docs/USAGE.md | 10 +- src/cli/argv-preprocessor.test.ts | 11 + src/cli/cli.test.ts | 1 + src/cli/commands/config-cmd.test.ts | 1 + src/cli/commands/index-cmd.ts | 9 + src/cli/commands/index-embeddings.test.ts | 83 +++++++ src/cli/commands/index-embeddings.ts | 70 ++++-- src/cli/commands/index-run.ts | 5 +- src/cli/commands/init-toml.ts | 1 + src/cli/error-handler.ts | 2 +- src/cli/flag-schemas.ts | 5 + src/cli/help.ts | 8 +- src/config/loader.test.ts | 12 +- src/config/schema.ts | 3 + src/config/validation.ts | 6 + src/embeddings/embed-batched.test.ts | 152 ++++++++++++ src/embeddings/embed-batched.ts | 223 ++++++++++++++---- src/embeddings/embedding-inputs.test.ts | 69 ++++++ src/embeddings/embedding-inputs.ts | 185 +++++++++++++++ .../semantic-search-build-path-filter.test.ts | 70 ++++++ src/embeddings/semantic-search-build.ts | 96 ++++++-- src/embeddings/semantic-search.ts | 2 + src/index/manifest-build.test.ts | 49 +++- src/index/semantic-refresh.ts | 4 + src/mcp/index-generation.test.ts | 1 + src/utils/tokens.test.ts | 31 ++- src/utils/tokens.ts | 72 ++++++ 29 files changed, 1100 insertions(+), 94 deletions(-) create mode 100644 src/cli/commands/index-embeddings.test.ts create mode 100644 src/embeddings/embed-batched.test.ts create mode 100644 src/embeddings/embedding-inputs.test.ts create mode 100644 src/embeddings/embedding-inputs.ts diff --git a/README.md b/README.md index b6541d1..59b815e 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ mdm index # Later: refresh all existing manifest directori mdm index ./docs # Append path, then refresh all directories mdm index . --embed # First index and build semantic embeddings mdm index --no-embed # Leave semantic vectors unchanged +mdm index --force-embed # Rebuild every semantic embedding mdm index --watch # Fails; multi-root manifest watch is unavailable mdm index --force # Bypass cache, re-process all files mdm index --exclude "*.draft.md,research/**" # Exclude patterns (comma-separated) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index d169a83..7c3b88a 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -59,6 +59,7 @@ snippetLength = 200 provider = "openai" model = "text-embedding-3-small" batchSize = 100 +concurrency = 4 ``` --- @@ -130,6 +131,7 @@ provider = "openai" model = "text-embedding-3-small" dimensions = 512 batchSize = 100 +concurrency = 4 maxRetries = 3 # baseURL = "https://custom-endpoint.example.com" # apiKey = "sk-..." @@ -283,12 +285,18 @@ Controls semantic search embedding generation. | `provider` | `string` | `'openai'` | Embedding provider (openai, ollama, lm-studio, openrouter) | | `model` | `string` | `'text-embedding-3-small'` | Embedding model name | | `dimensions` | `number` | (auto) | Vector dimensions (auto-detected from model if not set) | -| `batchSize` | `number` | `100` | Batch size for API calls | +| `batchSize` | `number` | `100` | Inputs per embedding API call | +| `concurrency` | `number` | `4` | Maximum embedding API calls in flight | | `maxRetries` | `number` | `3` | Maximum retries for failed API calls | | `retryDelayMs` | `number` | `1000` | Delay between retries in milliseconds | | `timeoutMs` | `number` | `30000` | Request timeout in milliseconds | | `apiKey` | `string` | (from env) | API key (prefer environment variable) | +Embedding inputs are token counted before submission. Sections above the +provider safe limit are split into bounded inputs, embedded with their heading +and document context, then pooled into one normalized section vector. Requests +are also packed below the provider aggregate token limit. + **Model Dimensions:** Dimensions are now automatically configured based on the model. If not explicitly set: @@ -323,6 +331,7 @@ model = "text-embedding-3-large" # Smaller batches for rate limiting batchSize = 50 +concurrency = 4 # More aggressive retries maxRetries = 5 @@ -563,6 +572,7 @@ environment var: MDM_INDEX_MAXDEPTH | `MDM_EMBEDDINGS_MODEL` | `embeddings.model` | | `MDM_EMBEDDINGS_DIMENSIONS` | `embeddings.dimensions` | | `MDM_EMBEDDINGS_BATCHSIZE` | `embeddings.batchSize` | +| `MDM_EMBEDDINGS_CONCURRENCY` | `embeddings.concurrency` | | `MDM_EMBEDDINGS_MAXRETRIES` | `embeddings.maxRetries` | | `MDM_EMBEDDINGS_RETRYDELAYMS` | `embeddings.retryDelayMs` | | `MDM_EMBEDDINGS_TIMEOUTMS` | `embeddings.timeoutMs` | diff --git a/docs/USAGE.md b/docs/USAGE.md index f7ccc04..c79188e 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -100,7 +100,8 @@ mdm index [path] [options] | ------------- | ------------------------------------------ | | `-e, --embed` | Also build semantic embeddings | | `-w, --watch` | Show deferred manifest watch guidance | -| `--force` | Force full rebuild (ignore cache) | +| `--force` | Rebuild the structural index | +| `--force-embed` | Rebuild all semantic embeddings | | `--json` | Output as JSON | | `--pretty` | Pretty-print JSON | @@ -121,8 +122,15 @@ mdm index --watch # Force rebuild mdm index --force + +# Force a semantic rebuild +mdm index --force-embed ``` +Sections larger than an embedding provider accepts are split automatically and +stored as one normalized section vector. mdm reports the document and heading +when this occurs. + **Index location:** `.mdm/indexes/` --- diff --git a/src/cli/argv-preprocessor.test.ts b/src/cli/argv-preprocessor.test.ts index 06b904d..6855832 100644 --- a/src/cli/argv-preprocessor.test.ts +++ b/src/cli/argv-preprocessor.test.ts @@ -70,6 +70,17 @@ describe('preprocessArgvWithValidation', () => { expect(result.error).toBeUndefined() }) + it('accepts the semantic rebuild flag', () => { + const result = preprocessArgvWithValidation([ + node, + script, + 'index', + '--force-embed', + ]) + expect(result.argv).toEqual([node, script, 'index', '--force-embed']) + expect(result.error).toBeUndefined() + }) + it('passes through --help flag', () => { const result = preprocessArgvWithValidation([ node, diff --git a/src/cli/cli.test.ts b/src/cli/cli.test.ts index 9514d97..e3a4748 100644 --- a/src/cli/cli.test.ts +++ b/src/cli/cli.test.ts @@ -191,6 +191,7 @@ describe('mdm CLI e2e', () => { expect(output).toContain('--hnsw-m') expect(output).toContain('--hnsw-ef-construction') expect(output).toContain('--force') + expect(output).toContain('--force-embed') expect(output).not.toContain('--all') expect(output).not.toContain('--timeout') }) diff --git a/src/cli/commands/config-cmd.test.ts b/src/cli/commands/config-cmd.test.ts index 46f0af7..dfaaf01 100644 --- a/src/cli/commands/config-cmd.test.ts +++ b/src/cli/commands/config-cmd.test.ts @@ -176,6 +176,7 @@ color = "yes" expect(result.code).toBe(0) expect(parsed.valid).toBe(true) expect(parsed.config.embeddings.maxRetries.source).toBe('file') + expect(parsed.config.embeddings.concurrency.source).toBe('file') expect(parsed.config.embeddings.retryDelayMs.source).toBe('file') expect(parsed.config.embeddings.timeoutMs.source).toBe('file') expect(parsed.config.embeddings.hnswM.source).toBe('file') diff --git a/src/cli/commands/index-cmd.ts b/src/cli/commands/index-cmd.ts index f8a3937..7482166 100644 --- a/src/cli/commands/index-cmd.ts +++ b/src/cli/commands/index-cmd.ts @@ -21,6 +21,13 @@ const noEmbedOption = Options.boolean('no-embed').pipe( Options.withDefault(false), ) +const forceEmbedOption = Options.boolean('force-embed').pipe( + Options.withDescription( + 'Rebuild every semantic embedding instead of reusing unchanged vectors', + ), + Options.withDefault(false), +) + const excludeOption = Options.text('exclude').pipe( Options.withAlias('x'), Options.withDescription( @@ -83,6 +90,7 @@ export const indexCommand = Command.make( path: pathArg, embed: embedOption, noEmbed: noEmbedOption, + forceEmbed: forceEmbedOption, exclude: excludeOption, noGitignore: noGitignoreOption, provider: providerOption, @@ -100,6 +108,7 @@ export const indexCommand = Command.make( path: Option.getOrUndefined(input.path), embed: input.embed, noEmbed: input.noEmbed, + forceEmbed: input.forceEmbed, exclude: Option.getOrUndefined(input.exclude), noGitignore: input.noGitignore, provider: Option.getOrUndefined(input.provider), diff --git a/src/cli/commands/index-embeddings.test.ts b/src/cli/commands/index-embeddings.test.ts new file mode 100644 index 0000000..3d47aae --- /dev/null +++ b/src/cli/commands/index-embeddings.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' + +import { defaultConfig } from '../../config/schema.js' +import type { ProviderId } from '../../providers/index.js' +import { + type EmbeddingRefreshInput, + semanticRefreshOptions, +} from './index-embeddings.js' + +const input = ( + overrides: Partial = {}, +): EmbeddingRefreshInput => ({ + embed: true, + noEmbed: false, + forceEmbed: false, + force: false, + json: true, + provider: undefined, + providerBaseUrl: undefined, + providerModel: undefined, + hnswM: undefined, + hnswEfConstruction: undefined, + ...overrides, +}) + +describe('semanticRefreshOptions', () => { + it('keeps structural force separate from semantic rebuilds', () => { + const result = semanticRefreshOptions( + input({ force: true }), + false, + defaultConfig.embeddings, + ) + + expect(result).toMatchObject({ + mode: 'build', + options: { force: false }, + }) + }) + + it('makes a full semantic rebuild explicit', () => { + const result = semanticRefreshOptions( + input({ embed: false, forceEmbed: true }), + false, + defaultConfig.embeddings, + ) + + expect(result).toMatchObject({ + mode: 'build', + options: { force: true }, + }) + }) + + it('passes configured batching policy and provider defaults', () => { + const config = { + ...defaultConfig.embeddings, + provider: 'openrouter' as ProviderId, + model: 'configured-model', + batchSize: 40, + concurrency: 3, + maxRetries: 6, + retryDelayMs: 250, + timeoutMs: 45_000, + } + const result = semanticRefreshOptions(input(), false, config) + + expect(result).toMatchObject({ + mode: 'build', + options: { + providerConfig: { + provider: 'openrouter', + model: 'configured-model', + }, + execution: { + batchSize: 40, + concurrency: 3, + maxRetries: 6, + retryDelayMs: 250, + timeoutMs: 45_000, + }, + }, + }) + }) +}) diff --git a/src/cli/commands/index-embeddings.ts b/src/cli/commands/index-embeddings.ts index 80acb35..4cbaa35 100644 --- a/src/cli/commands/index-embeddings.ts +++ b/src/cli/commands/index-embeddings.ts @@ -1,8 +1,10 @@ -import { Console, Effect } from 'effect' +import { Console, Effect, Option } from 'effect' +import type { EmbeddingsConfig } from '../../config/schema.js' import type { BuildEmbeddingsOptions, BuildEmbeddingsResult, + EmbeddingExecutionOptions, EmbeddingProviderConfig, } from '../../embeddings/semantic-search.js' import type { SemanticRefreshOptions } from '../../index/semantic-refresh.js' @@ -11,6 +13,7 @@ import type { ProviderId } from '../../providers/index.js' export interface EmbeddingRefreshInput { readonly embed: boolean readonly noEmbed: boolean + readonly forceEmbed: boolean readonly force: boolean readonly json: boolean readonly provider: ProviderId | undefined @@ -22,26 +25,39 @@ export interface EmbeddingRefreshInput { const providerConfig = ( input: EmbeddingRefreshInput, -): EmbeddingProviderConfig | undefined => { - if (input.provider !== undefined) { - return { - provider: input.provider, - baseURL: input.providerBaseUrl, - model: input.providerModel, - } - } - return undefined -} + config: EmbeddingsConfig, +): EmbeddingProviderConfig => ({ + provider: input.provider ?? config.provider, + baseURL: input.providerBaseUrl ?? Option.getOrUndefined(config.baseURL), + model: input.providerModel ?? config.model, + dimensions: config.dimensions, +}) -const hnswOptions = (input: EmbeddingRefreshInput) => - input.hnswM !== undefined || input.hnswEfConstruction !== undefined - ? { m: input.hnswM, efConstruction: input.hnswEfConstruction } - : undefined +const hnswOptions = ( + input: EmbeddingRefreshInput, + config: EmbeddingsConfig, +) => ({ + m: input.hnswM ?? config.hnswM, + efConstruction: input.hnswEfConstruction ?? config.hnswEfConstruction, +}) + +const executionOptions = ( + config: EmbeddingsConfig, +): EmbeddingExecutionOptions => ({ + batchSize: config.batchSize, + concurrency: config.concurrency, + maxRetries: config.maxRetries, + retryDelayMs: config.retryDelayMs, + timeoutMs: config.timeoutMs, +}) const progressOptions = ( input: EmbeddingRefreshInput, showProgress: boolean, -): Pick => ({ +): Pick< + BuildEmbeddingsOptions, + 'onBatchProgress' | 'onFileProgress' | 'onSectionChunked' +> => ({ onFileProgress: (progress) => { if (!input.json && showProgress) { process.stdout.write( @@ -56,21 +72,33 @@ const progressOptions = ( ) } }, + onSectionChunked: (progress) => { + if (input.json) return + if (showProgress) process.stdout.write('\x1b[2K\r') + process.stderr.write( + ` Chunking oversized section: ${progress.documentPath} > ${progress.heading} (${progress.tokenCount} tokens into ${progress.chunkCount} inputs)\n`, + ) + }, }) export const semanticRefreshOptions = ( input: EmbeddingRefreshInput, showProgress: boolean, + config: EmbeddingsConfig, ): SemanticRefreshOptions => { if (input.noEmbed) return { mode: 'skip' } const progress = progressOptions(input, showProgress) - if (!input.embed) return { mode: 'active', ...progress } + const execution = executionOptions(config) + if (!input.embed && !input.forceEmbed) { + return { mode: 'active', execution, ...progress } + } return { mode: 'build', options: { - force: input.force, - providerConfig: providerConfig(input), - hnswOptions: hnswOptions(input), + force: input.forceEmbed, + providerConfig: providerConfig(input, config), + hnswOptions: hnswOptions(input, config), + execution, ...progress, }, } @@ -83,7 +111,7 @@ const renderEmbeddingResult = (result: BuildEmbeddingsResult) => yield* Console.log( `Embeddings already exist (${result.existingVectors} vectors)`, ) - yield* Console.log(' Use --force to rebuild') + yield* Console.log(' Use --force-embed to rebuild') return } diff --git a/src/cli/commands/index-run.ts b/src/cli/commands/index-run.ts index e02aa06..fe86711 100644 --- a/src/cli/commands/index-run.ts +++ b/src/cli/commands/index-run.ts @@ -1,6 +1,6 @@ import { Console, Effect } from 'effect' -import { getConfigValue } from '../../config/service.js' +import { getConfigSection, getConfigValue } from '../../config/service.js' import { resolveMdmHome } from '../../home.js' import { refreshManifestIndex } from '../../index/manifest-refresh.js' import { ManifestError, manifestPath } from '../../manifest.js' @@ -39,6 +39,7 @@ export const runIndexCommand = (input: IndexCommandInput) => if (input.watch) return yield* rejectManifestWatch(home) const colorEnabled = yield* getConfigValue('output', 'color') + const embeddingsConfig = yield* getConfigSection('embeddings') const showProgress = Boolean(process.stdout.isTTY && colorEnabled) const exclude = parseExcludePatterns(input.exclude) @@ -59,7 +60,7 @@ export const runIndexCommand = (input: IndexCommandInput) => ) } }, - semantic: semanticRefreshOptions(input, showProgress), + semantic: semanticRefreshOptions(input, showProgress, embeddingsConfig), }) const result = published.value diff --git a/src/cli/commands/init-toml.ts b/src/cli/commands/init-toml.ts index 257e4ea..d93cb07 100644 --- a/src/cli/commands/init-toml.ts +++ b/src/cli/commands/init-toml.ts @@ -31,6 +31,7 @@ provider = "${defaultConfig.embeddings.provider}" model = "${defaultConfig.embeddings.model}" dimensions = ${defaultConfig.embeddings.dimensions} batchSize = ${defaultConfig.embeddings.batchSize} +concurrency = ${defaultConfig.embeddings.concurrency} maxRetries = ${defaultConfig.embeddings.maxRetries} retryDelayMs = ${defaultConfig.embeddings.retryDelayMs} timeoutMs = ${defaultConfig.embeddings.timeoutMs} diff --git a/src/cli/error-handler.ts b/src/cli/error-handler.ts index e688828..0af3369 100644 --- a/src/cli/error-handler.ts +++ b/src/cli/error-handler.ts @@ -308,7 +308,7 @@ export const formatError = (error: MdmError): FormattedError => e.corpusProvider ? `Switch back to original provider: --provider ${e.corpusProvider.split(':')[0]} --provider-model ${e.corpusProvider.split(':')[1] ?? ''}` : 'Check your embedding provider configuration', - "Rebuild corpus with current provider: 'mdm index --embed --force'", + "Rebuild corpus with current provider: 'mdm index --force-embed'", 'The corpus was created with different embedding dimensions than your current provider', ] as const, exitCode: EXIT_CODE.USER_ERROR, diff --git a/src/cli/flag-schemas.ts b/src/cli/flag-schemas.ts index 42e5fd9..d32f90c 100644 --- a/src/cli/flag-schemas.ts +++ b/src/cli/flag-schemas.ts @@ -87,6 +87,11 @@ export const indexSchema: CommandSchema = { type: 'boolean', description: 'Skip semantic vector pruning and refresh', }, + { + name: 'force-embed', + type: 'boolean', + description: 'Rebuild every semantic embedding', + }, { name: 'exclude', type: 'string', diff --git a/src/cli/help.ts b/src/cli/help.ts index 06f19ed..cf77448 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -59,6 +59,7 @@ export const helpContent: Record = { 'mdm index # Refresh the existing manifest', 'mdm index docs/ # Append docs/ then refresh all roots', 'mdm index --embed # Refresh and build embeddings', + 'mdm index --force-embed # Rebuild every semantic embedding', 'mdm index --watch # Fails; manifest watch is unavailable', 'mdm index --force # Bypass cache, re-process all files', '', @@ -113,6 +114,11 @@ export const helpContent: Record = { description: 'Bypass mtime/hash cache and re-process every file (does not delete index)', }, + { + name: '--force-embed', + description: + 'Rebuild every semantic embedding instead of reusing vectors', + }, { name: '--json', description: 'Output results as JSON' }, { name: '--pretty', description: 'Pretty-print JSON output' }, ], @@ -120,7 +126,7 @@ export const helpContent: Record = { 'With no path, index refreshes every directory in manifest.toml.', 'With a path, index appends its absolute declared path before refreshing.', 'An empty manifest requires mdm index ; the current directory is never implicit.', - 'Existing semantic embeddings refresh atomically; use --embed to create them or --no-embed to leave them unchanged.', + 'Existing semantic embeddings refresh atomically. Use --embed to create them, --force-embed to rebuild them, or --no-embed to leave them unchanged.', 'Providers: openai (default), ollama (free/local), lm-studio, openrouter, voyage.', 'Set API keys: OPENAI_API_KEY, OPENROUTER_API_KEY, or use local providers.', 'Manifest and index state are stored in the active MDM_HOME.', diff --git a/src/config/loader.test.ts b/src/config/loader.test.ts index 103f63f..7f16cf6 100644 --- a/src/config/loader.test.ts +++ b/src/config/loader.test.ts @@ -533,7 +533,11 @@ describe('validateConfig', () => { maxLimit: 5, defaultLimit: 10, }, - embeddings: { ...defaultConfig.embeddings, batchSize: 0 }, + embeddings: { + ...defaultConfig.embeddings, + batchSize: 0, + concurrency: 0, + }, } const result = validateConfig(config) @@ -542,6 +546,9 @@ describe('validateConfig', () => { expect(result.search.maxLimit).toBe(5) expect(result.search.defaultLimit).toBe(5) expect(result.embeddings.batchSize).toBe(defaultConfig.embeddings.batchSize) + expect(result.embeddings.concurrency).toBe( + defaultConfig.embeddings.concurrency, + ) expect(warnSpy).toHaveBeenCalled() warnSpy.mockRestore() }) @@ -668,6 +675,9 @@ describe('generateDefaultToml round-trip', () => { defaultConfig.embeddings.dimensions, ) expect(result.embeddings.batchSize).toBe(defaultConfig.embeddings.batchSize) + expect(result.embeddings.concurrency).toBe( + defaultConfig.embeddings.concurrency, + ) expect(result.embeddings.maxRetries).toBe( defaultConfig.embeddings.maxRetries, ) diff --git a/src/config/schema.ts b/src/config/schema.ts index 3260230..c2c6435 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -76,6 +76,8 @@ export interface EmbeddingsConfig { dimensions: number /** Batch size for embedding API calls. */ batchSize: number + /** Maximum number of embedding API calls in flight. */ + concurrency: number /** Maximum retries for failed API calls. */ maxRetries: number /** Delay between retries in milliseconds. */ @@ -232,6 +234,7 @@ export const defaultConfig: MdmConfig = { model: 'text-embedding-3-small', dimensions: 512, batchSize: 100, + concurrency: 4, maxRetries: 3, retryDelayMs: 1000, timeoutMs: 30000, diff --git a/src/config/validation.ts b/src/config/validation.ts index 2e4a47c..b43848f 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -170,6 +170,12 @@ const numberRules: ConfigRule[] = [ defaultConfig.embeddings.batchSize, isPositiveInteger, ), + numberRule( + 'embeddings.concurrency', + 'an integer greater than or equal to 1', + defaultConfig.embeddings.concurrency, + isPositiveInteger, + ), numberRule( 'embeddings.maxRetries', 'an integer greater than or equal to 0', diff --git a/src/embeddings/embed-batched.test.ts b/src/embeddings/embed-batched.test.ts new file mode 100644 index 0000000..2a0c172 --- /dev/null +++ b/src/embeddings/embed-batched.test.ts @@ -0,0 +1,152 @@ +import { Effect } from 'effect' +import { describe, expect, it, vi } from 'vitest' + +import { + type EmbeddingClient, + EmbeddingError as RuntimeEmbeddingError, +} from '../providers/index.js' +import { embedInBatches } from './embed-batched.js' + +describe('embedInBatches', () => { + it('bounds concurrent requests and preserves input order', async () => { + let activeRequests = 0 + let peakRequests = 0 + const progress: number[] = [] + const client: EmbeddingClient = { + embed: (texts) => + Effect.promise(async () => { + activeRequests++ + peakRequests = Math.max(peakRequests, activeRequests) + const first = Number(texts[0]) + await new Promise((resolve) => + setTimeout(resolve, first === 0 ? 30 : 2), + ) + activeRequests-- + return { + embeddings: texts.map((text) => [Number(text)]), + model: 'test-model', + usage: { inputTokens: texts.length }, + } + }), + } + + const result = await Effect.runPromise( + embedInBatches(client, ['0', '1', '2', '3', '4'], { + model: 'test-model', + batchSize: 2, + concurrency: 2, + onBatchProgress: ({ processedTexts }) => progress.push(processedTexts), + }), + ) + + expect(peakRequests).toBe(2) + expect(result.embeddings).toEqual([[0], [1], [2], [3], [4]]) + expect(progress.at(-1)).toBe(5) + expect(progress).toEqual([...progress].sort((left, right) => left - right)) + }) + + it('honors the configured retry budget and delay', async () => { + const info = vi.spyOn(console, 'info').mockImplementation(() => {}) + let attempts = 0 + const client: EmbeddingClient = { + embed: () => { + attempts++ + return attempts < 3 + ? Effect.fail( + new RuntimeEmbeddingError({ + provider: 'openai', + message: 'network connection reset', + }), + ) + : Effect.succeed({ + embeddings: [[1]], + model: 'test-model', + usage: { inputTokens: 1 }, + }) + }, + } + + try { + await Effect.runPromise( + embedInBatches(client, ['text'], { + model: 'test-model', + maxRetries: 2, + retryDelayMs: 0, + }), + ) + expect(attempts).toBe(3) + } finally { + info.mockRestore() + } + }) + + it('keeps each request within the configured token budget', async () => { + const calls: string[][] = [] + const completedIndexes: number[][] = [] + const client: EmbeddingClient = { + embed: (texts) => { + calls.push([...texts]) + return Effect.succeed({ + embeddings: texts.map((text) => [Number(text)]), + model: 'test-model', + usage: { inputTokens: texts.length }, + }) + }, + } + + const result = await Effect.runPromise( + embedInBatches(client, ['0', '1', '2', '3'], { + model: 'test-model', + batchSize: 4, + tokenCounts: [6, 6, 4, 4], + maxBatchTokens: 10, + onBatchProgress: ({ completedTextIndexes }) => + completedIndexes.push([...completedTextIndexes]), + }), + ) + + expect(calls).toEqual([['0'], ['1', '2'], ['3']]) + expect(completedIndexes).toEqual([[0], [1, 2], [3]]) + expect(result.embeddings).toEqual([[0], [1], [2], [3]]) + }) + + it('aborts requests at the configured timeout', async () => { + const client: EmbeddingClient = { + embed: (_texts, options) => + Effect.tryPromise({ + try: () => + new Promise((_resolve, reject) => { + options?.signal?.addEventListener( + 'abort', + () => reject(options.signal?.reason), + { once: true }, + ) + }), + catch: (cause) => + new RuntimeEmbeddingError({ + provider: 'openai', + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + }), + } + + const outcome = await Effect.runPromise( + Effect.either( + embedInBatches(client, ['text'], { + model: 'test-model', + maxRetries: 0, + timeoutMs: 5, + }), + ), + ) + + expect(outcome).toMatchObject({ + _tag: 'Left', + left: { + reason: 'Network', + message: 'Embedding request timed out after 5ms', + }, + }) + }) +}) diff --git a/src/embeddings/embed-batched.ts b/src/embeddings/embed-batched.ts index b0ddbd4..13756ac 100644 --- a/src/embeddings/embed-batched.ts +++ b/src/embeddings/embed-batched.ts @@ -7,10 +7,11 @@ * retry on transient failures. That logic lives here, in the consumer * layer, so the runtime stays minimal. * - * Behavior preserved from the previous `OpenAIProvider.embed` path: + * Default behavior: * - Batch size defaults to 100 documents per request. - * - Retry up to 5 attempts on RateLimit/Network errors with exponential - * backoff (1s, 2s, 4s, 8s, 16s) plus 0-1s random jitter. + * - Requests run serially unless the caller configures concurrency. + * - Retry up to 3 times on RateLimit/Network errors with exponential + * backoff plus random jitter. * - Per-batch progress callback fires after each successful batch. * - Aggregated `inputTokens` across batches. * - Maps the runtime's generic `EmbeddingError` into the centralized @@ -46,9 +47,18 @@ export interface BatchProgress { readonly totalBatches: number readonly processedTexts: number readonly totalTexts: number + readonly completedTextIndexes: readonly number[] } -export interface EmbedInBatchesOptions { +export interface EmbeddingExecutionOptions { + readonly batchSize?: number | undefined + readonly concurrency?: number | undefined + readonly maxRetries?: number | undefined + readonly retryDelayMs?: number | undefined + readonly timeoutMs?: number | undefined +} + +export interface EmbedInBatchesOptions extends EmbeddingExecutionOptions { readonly model: string /** * Output dimensions, only set when the model supports Matryoshka @@ -56,7 +66,8 @@ export interface EmbedInBatchesOptions { * (OpenAI text-embedding-3-*) and ignores it elsewhere. */ readonly dimensions?: number | undefined - readonly batchSize?: number | undefined + readonly tokenCounts?: readonly number[] | undefined + readonly maxBatchTokens?: number | undefined readonly onBatchProgress?: ((progress: BatchProgress) => void) | undefined readonly signal?: AbortSignal | undefined } @@ -66,7 +77,10 @@ export interface EmbedInBatchesOptions { // ============================================================================ const DEFAULT_BATCH_SIZE = 100 -const MAX_RETRY_ATTEMPTS = 5 +const DEFAULT_CONCURRENCY = 1 +const DEFAULT_MAX_RETRIES = 3 +const DEFAULT_RETRY_DELAY_MS = 1000 +export const EMBEDDING_REQUEST_TOKEN_LIMIT = 300_000 // ============================================================================ // Error Classification @@ -156,15 +170,51 @@ const toConsumerError = ( }) } +interface RequestSignal { + readonly signal: AbortSignal | undefined + readonly timedOut: () => boolean + readonly cleanup: () => void +} + +const requestSignal = ( + parent: AbortSignal | undefined, + timeoutMs: number | undefined, +): RequestSignal => { + if (timeoutMs === undefined) { + return { signal: parent, timedOut: () => false, cleanup: () => {} } + } + + const controller = new AbortController() + let timeoutReached = false + const abortFromParent = () => controller.abort(parent?.reason) + if (parent?.aborted) abortFromParent() + else parent?.addEventListener('abort', abortFromParent, { once: true }) + + const timeoutId = setTimeout(() => { + timeoutReached = true + controller.abort( + new Error(`Embedding request timed out after ${timeoutMs}ms`), + ) + }, timeoutMs) + + return { + signal: controller.signal, + timedOut: () => timeoutReached, + cleanup: () => { + clearTimeout(timeoutId) + parent?.removeEventListener('abort', abortFromParent) + }, + } +} + // ============================================================================ // Retry Wrapper // ============================================================================ /** - * Call `client.embed` with up to 5 attempts, retrying on RateLimit and - * Network categories with exponential backoff + jitter. The OpenAI SDK - * already retries internally (maxRetries: 2); this loop covers cases - * that slip through during long batch runs. + * Call `client.embed`, retrying RateLimit and Network categories with + * exponential backoff and jitter. The caller controls the retry budget, + * base delay, and request timeout. */ const embedBatchWithRetry = ( client: EmbeddingClient, @@ -173,22 +223,30 @@ const embedBatchWithRetry = ( readonly model: string readonly dimensions?: number | undefined readonly signal?: AbortSignal | undefined + readonly maxRetries?: number | undefined + readonly retryDelayMs?: number | undefined + readonly timeoutMs?: number | undefined }, ): Effect.Effect< EmbeddingResult, ApiKeyInvalidError | ConsumerEmbeddingError > => Effect.gen(function* () { - for (let attempt = 0; attempt < MAX_RETRY_ATTEMPTS; attempt++) { + const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES + const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS + const maxAttempts = maxRetries + 1 + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const request = requestSignal(options.signal, options.timeoutMs) const result = yield* Effect.either( client.embed(texts, { model: options.model, ...(options.dimensions !== undefined ? { dimensions: options.dimensions } : {}), - ...(options.signal !== undefined ? { signal: options.signal } : {}), + ...(request.signal !== undefined ? { signal: request.signal } : {}), }), - ) + ).pipe(Effect.ensuring(Effect.sync(request.cleanup))) if (result._tag === 'Right') { return result.right @@ -200,19 +258,31 @@ const embedBatchWithRetry = ( return yield* Effect.fail(toConsumerError(error)) } - const reason = classifyEmbeddingError(error) - const isLastAttempt = attempt === MAX_RETRY_ATTEMPTS - 1 + const reason = request.timedOut() + ? 'Network' + : classifyEmbeddingError(error) + const isLastAttempt = attempt === maxAttempts - 1 if (!isRetryable(reason) || isLastAttempt) { + if (request.timedOut()) { + return yield* Effect.fail( + new ConsumerEmbeddingError({ + reason: 'Network', + message: `Embedding request timed out after ${options.timeoutMs}ms`, + provider: error.provider, + cause: error.cause, + }), + ) + } return yield* Effect.fail(toConsumerError(error)) } - const baseDelay = 2 ** attempt * 1000 - const jitter = Math.random() * 1000 + const baseDelay = 2 ** attempt * retryDelayMs + const jitter = Math.random() * retryDelayMs const delay = Math.round(baseDelay + jitter) console.info( - `[mdm] Embedding API ${reason} error, retry ${attempt + 1}/${MAX_RETRY_ATTEMPTS} after ${delay}ms`, + `[mdm] Embedding API ${reason} error, retry ${attempt + 1}/${maxRetries} after ${delay}ms`, ) yield* Effect.sleep(`${delay} millis`) @@ -247,6 +317,48 @@ export const createEmbeddingClient = ( ApiKeyMissingError | CapabilityNotSupported | ProviderNotFound > => resolveClient('embed', id, overrides) +interface EmbeddingBatch { + readonly index: number + readonly texts: readonly string[] + readonly textIndexes: readonly number[] +} + +const createEmbeddingBatches = ( + texts: readonly string[], + batchSize: number, + tokenCounts: readonly number[] | undefined, + maxBatchTokens: number, +): readonly EmbeddingBatch[] => { + const batches: EmbeddingBatch[] = [] + let batchTexts: string[] = [] + let textIndexes: number[] = [] + let batchTokens = 0 + + const flush = () => { + if (batchTexts.length === 0) return + batches.push({ index: batches.length, texts: batchTexts, textIndexes }) + batchTexts = [] + textIndexes = [] + batchTokens = 0 + } + + for (let index = 0; index < texts.length; index++) { + const tokenCount = tokenCounts?.[index] ?? 0 + if ( + batchTexts.length > 0 && + (batchTexts.length >= batchSize || + batchTokens + tokenCount > maxBatchTokens) + ) { + flush() + } + batchTexts.push(texts[index]!) + textIndexes.push(index) + batchTokens += tokenCount + } + flush() + return batches +} + // ============================================================================ // Public API // ============================================================================ @@ -278,36 +390,53 @@ export const embedInBatches = ( } const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE - const totalBatches = Math.ceil(texts.length / batchSize) - const allEmbeddings: (readonly number[])[] = [] - let totalTokens = 0 - let resolvedModel = options.model - - for (let i = 0; i < texts.length; i += batchSize) { - const batch = texts.slice(i, i + batchSize) - const batchIndex = Math.floor(i / batchSize) - - const result = yield* embedBatchWithRetry(client, batch, { - model: options.model, - dimensions: options.dimensions, - signal: options.signal, - }) + const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY + const batches = createEmbeddingBatches( + texts, + batchSize, + options.tokenCounts, + options.maxBatchTokens ?? EMBEDDING_REQUEST_TOKEN_LIMIT, + ) + const totalBatches = batches.length + let completedBatches = 0 + let processedTexts = 0 + + const results = yield* Effect.forEach( + batches, + (batch) => + embedBatchWithRetry(client, batch.texts, { + model: options.model, + dimensions: options.dimensions, + signal: options.signal, + maxRetries: options.maxRetries, + retryDelayMs: options.retryDelayMs, + timeoutMs: options.timeoutMs, + }).pipe( + Effect.tap(() => + Effect.sync(() => { + completedBatches++ + processedTexts += batch.texts.length + options.onBatchProgress?.({ + batchIndex: completedBatches, + totalBatches, + processedTexts, + totalTexts: texts.length, + completedTextIndexes: batch.textIndexes, + }) + }), + ), + Effect.map((result) => ({ index: batch.index, result })), + ), + { concurrency }, + ) + results.sort((left, right) => left.index - right.index) - for (const embedding of result.embeddings) { - allEmbeddings.push(embedding) - } - totalTokens += result.usage?.inputTokens ?? 0 - resolvedModel = result.model - - if (options.onBatchProgress) { - options.onBatchProgress({ - batchIndex: batchIndex + 1, - totalBatches, - processedTexts: Math.min(i + batchSize, texts.length), - totalTexts: texts.length, - }) - } - } + const allEmbeddings = results.flatMap(({ result }) => result.embeddings) + const totalTokens = results.reduce( + (total, { result }) => total + (result.usage?.inputTokens ?? 0), + 0, + ) + const resolvedModel = results.at(-1)?.result.model ?? options.model return { embeddings: allEmbeddings, diff --git a/src/embeddings/embedding-inputs.test.ts b/src/embeddings/embedding-inputs.test.ts new file mode 100644 index 0000000..74c39fe --- /dev/null +++ b/src/embeddings/embedding-inputs.test.ts @@ -0,0 +1,69 @@ +import { Effect } from 'effect' +import { describe, expect, it } from 'vitest' + +import { countTokens } from '../utils/tokens.js' +import { + EMBEDDING_INPUT_TOKEN_LIMIT, + poolEmbeddingsBySource, + prepareEmbeddingInputs, +} from './embedding-inputs.js' + +describe('embedding inputs', () => { + it('preserves the existing input format when a section fits', async () => { + const prepared = await Effect.runPromise( + prepareEmbeddingInputs([ + { context: '# Heading\nDocument: guide', content: 'Section body' }, + ]), + ) + + expect(prepared.inputs).toMatchObject([ + { text: '# Heading\nDocument: guide\n\nSection body', sourceIndex: 0 }, + ]) + expect(prepared.chunkedSources).toEqual([]) + }) + + it('chunks oversized content and repeats section context', async () => { + const context = '# Large section\nDocument: guide' + const prepared = await Effect.runPromise( + prepareEmbeddingInputs([ + { context, content: 'semantic content '.repeat(10_000) }, + ]), + ) + + expect(prepared.inputs.length).toBeGreaterThan(1) + expect(prepared.chunkCounts).toEqual([prepared.inputs.length]) + expect(prepared.chunkedSources).toEqual([ + { + sourceIndex: 0, + chunkCount: prepared.inputs.length, + tokenCount: expect.any(Number), + }, + ]) + for (const input of prepared.inputs) { + expect(input.text.startsWith(`${context}\n\n`)).toBe(true) + expect(input.tokenCount).toBeLessThanOrEqual(EMBEDDING_INPUT_TOKEN_LIMIT) + expect(await Effect.runPromise(countTokens(input.text))).toBe( + input.tokenCount, + ) + } + }) + + it('returns one normalized vector per source', async () => { + const prepared = await Effect.runPromise( + prepareEmbeddingInputs([ + { context: '# Large', content: 'content '.repeat(12_000) }, + { context: '# Small', content: 'short content' }, + ]), + ) + const embeddings = prepared.inputs.map((_input, index) => + index === prepared.inputs.length - 1 ? [0, 5] : [3, index + 1], + ) + const pooled = poolEmbeddingsBySource(prepared, embeddings) + + expect(pooled).toHaveLength(2) + expect(Math.hypot(...pooled[0]!)).toBeCloseTo(1) + expect(pooled[0]![0]).toBeGreaterThan(0) + expect(pooled[0]![1]).toBeGreaterThan(0) + expect(pooled[1]).toEqual([0, 5]) + }) +}) diff --git a/src/embeddings/embedding-inputs.ts b/src/embeddings/embedding-inputs.ts new file mode 100644 index 0000000..bf6b155 --- /dev/null +++ b/src/embeddings/embedding-inputs.ts @@ -0,0 +1,185 @@ +import { Effect } from 'effect' + +import { countTokens, splitTextByTokens } from '../utils/tokens.js' + +export const EMBEDDING_INPUT_TOKEN_LIMIT = 8_000 + +const CONTEXT_JOIN_TOKEN_MARGIN = 32 +const CONTEXT_SEPARATOR = '\n\n' + +export interface EmbeddingSourceInput { + readonly context: string + readonly content: string +} + +export interface PreparedEmbeddingInput { + readonly text: string + readonly tokenCount: number + readonly weight: number + readonly sourceIndex: number +} + +export interface ChunkedEmbeddingSource { + readonly sourceIndex: number + readonly chunkCount: number + readonly tokenCount: number +} + +export interface PreparedEmbeddingInputs { + readonly inputs: readonly PreparedEmbeddingInput[] + readonly chunkCounts: readonly number[] + readonly chunkedSources: readonly ChunkedEmbeddingSource[] +} + +const prepareOversizedSource = ( + source: EmbeddingSourceInput, + sourceIndex: number, + fullText: string, + fullTokenCount: number, +) => + Effect.gen(function* () { + const contextTokenCount = yield* countTokens(source.context) + const contentTokenLimit = + EMBEDDING_INPUT_TOKEN_LIMIT - + contextTokenCount - + CONTEXT_JOIN_TOKEN_MARGIN + + if (contentTokenLimit < 1) { + const chunks = yield* splitTextByTokens( + fullText, + EMBEDDING_INPUT_TOKEN_LIMIT, + ) + return chunks.map((chunk) => ({ + ...chunk, + sourceIndex, + weight: chunk.tokenCount, + })) + } + + const contentChunks = yield* splitTextByTokens( + source.content, + contentTokenLimit, + ) + const contextualChunks = yield* Effect.forEach(contentChunks, (chunk) => + Effect.map( + countTokens(`${source.context}${CONTEXT_SEPARATOR}${chunk.text}`), + (tokenCount) => ({ + text: `${source.context}${CONTEXT_SEPARATOR}${chunk.text}`, + tokenCount, + sourceIndex, + weight: Math.max(1, chunk.tokenCount), + }), + ), + ) + + if ( + contextualChunks.every( + (chunk) => chunk.tokenCount <= EMBEDDING_INPUT_TOKEN_LIMIT, + ) + ) { + return contextualChunks + } + + const fallback = yield* splitTextByTokens( + fullText, + EMBEDDING_INPUT_TOKEN_LIMIT, + ) + return fallback.map((chunk) => ({ + ...chunk, + sourceIndex, + weight: chunk.tokenCount, + })) + }).pipe(Effect.map((inputs) => ({ inputs, fullTokenCount }))) + +export const prepareEmbeddingInputs = ( + sources: readonly EmbeddingSourceInput[], +): Effect.Effect => + Effect.gen(function* () { + const inputs: PreparedEmbeddingInput[] = [] + const chunkCounts: number[] = [] + const chunkedSources: ChunkedEmbeddingSource[] = [] + + for (let sourceIndex = 0; sourceIndex < sources.length; sourceIndex++) { + const source = sources[sourceIndex]! + const fullText = `${source.context}${CONTEXT_SEPARATOR}${source.content}` + const fullTokenCount = yield* countTokens(fullText) + if (fullTokenCount <= EMBEDDING_INPUT_TOKEN_LIMIT) { + inputs.push({ + text: fullText, + tokenCount: fullTokenCount, + weight: Math.max(1, fullTokenCount), + sourceIndex, + }) + chunkCounts.push(1) + continue + } + + const prepared = yield* prepareOversizedSource( + source, + sourceIndex, + fullText, + fullTokenCount, + ) + inputs.push(...prepared.inputs) + chunkCounts.push(prepared.inputs.length) + chunkedSources.push({ + sourceIndex, + chunkCount: prepared.inputs.length, + tokenCount: prepared.fullTokenCount, + }) + } + + return { inputs, chunkCounts, chunkedSources } + }) + +const normalizedWeightedMean = ( + embeddings: readonly (readonly number[])[], + weights: readonly number[], +): readonly number[] => { + const dimensions = embeddings[0]?.length ?? 0 + const combined = Array.from({ length: dimensions }, () => 0) + let totalWeight = 0 + + for (let index = 0; index < embeddings.length; index++) { + const embedding = embeddings[index] + if (embedding === undefined || embedding.length !== dimensions) continue + const weight = weights[index] ?? 1 + totalWeight += weight + for (let dimension = 0; dimension < dimensions; dimension++) { + combined[dimension] = + (combined[dimension] ?? 0) + (embedding[dimension] ?? 0) * weight + } + } + + if (totalWeight === 0) return combined + const mean = combined.map((value) => value / totalWeight) + const magnitude = Math.hypot(...mean) + return magnitude === 0 ? mean : mean.map((value) => value / magnitude) +} + +export const poolEmbeddingsBySource = ( + prepared: PreparedEmbeddingInputs, + embeddings: readonly (readonly number[])[], +): readonly (readonly number[])[] => { + const groupedEmbeddings = prepared.chunkCounts.map( + () => [] as (readonly number[])[], + ) + const groupedWeights = prepared.chunkCounts.map(() => [] as number[]) + + for (let index = 0; index < prepared.inputs.length; index++) { + const input = prepared.inputs[index] + const embedding = embeddings[index] + if (input === undefined || embedding === undefined) continue + groupedEmbeddings[input.sourceIndex]?.push(embedding) + groupedWeights[input.sourceIndex]?.push(input.weight) + } + + return groupedEmbeddings.map((sourceEmbeddings, sourceIndex) => + sourceEmbeddings.length <= 1 + ? (sourceEmbeddings[0] ?? []) + : normalizedWeightedMean( + sourceEmbeddings, + groupedWeights[sourceIndex] ?? [], + ), + ) +} diff --git a/src/embeddings/semantic-search-build-path-filter.test.ts b/src/embeddings/semantic-search-build-path-filter.test.ts index 13f3765..ab31674 100644 --- a/src/embeddings/semantic-search-build-path-filter.test.ts +++ b/src/embeddings/semantic-search-build-path-filter.test.ts @@ -10,6 +10,8 @@ import { loadSectionIndex, } from '../index/storage.js' import type { EmbeddingClient } from '../providers/index.js' +import { countTokens } from '../utils/tokens.js' +import { EMBEDDING_INPUT_TOKEN_LIMIT } from './embedding-inputs.js' import { getActiveProviderPath, getMetaPath, @@ -17,6 +19,7 @@ import { } from './embedding-namespace-paths.js' import { buildEmbeddings } from './semantic-search-build.js' import { createNamespacedVectorStore } from './vector-store.js' +import { loadVectorIndex } from './vector-store-codec.js' const providerConfig = { provider: 'openai' as const, @@ -263,3 +266,70 @@ it('publishes no semantic artifacts after a forced build has zero eligible secti await fs.rm(fixture.parent, { recursive: true, force: true }) } }) + +it('embeds an oversized section as one pooled vector', async () => { + const fixture = await makeEmbeddingFixture( + `# Oversized\n\n${'semantic content '.repeat(10_000)}`, + ) + const embeddedInputs: string[] = [] + let embeddingIndex = 0 + const embed = vi.fn((texts, options) => { + embeddedInputs.push(...texts) + return Effect.succeed({ + embeddings: texts.map(() => + embeddingIndex++ % 2 === 0 ? [3, 0] : [0, 4], + ), + model: options?.model ?? 'test-model', + usage: { inputTokens: texts.length }, + }) + }) + const batchProgress: { processedSections: number; totalSections: number }[] = + [] + const chunkedSections: { heading: string; chunkCount: number }[] = [] + + try { + await Effect.runPromise( + buildIndex(fixture.sourceRoot, { indexRoot: fixture.indexRoot }), + ) + const result = await Effect.runPromise( + buildEmbeddings(fixture.sourceRoot, { + indexRoot: fixture.indexRoot, + client: { embed }, + providerConfig: { + provider: 'openai', + model: 'text-embedding-3-small', + dimensions: 2, + }, + onBatchProgress: (progress) => batchProgress.push(progress), + onSectionChunked: ({ heading, chunkCount }) => + chunkedSections.push({ heading, chunkCount }), + }), + ) + const tokenCounts = await Promise.all( + embeddedInputs.map((text) => Effect.runPromise(countTokens(text))), + ) + const vectorIndex = await Effect.runPromise( + loadVectorIndex( + getMetaPath(fixture.indexRoot, 'openai_text-embedding-3-small_2'), + ), + ) + const vector = Object.values(vectorIndex.entries)[0] + + expect(result.sectionsEmbedded).toBe(1) + expect(embeddedInputs.length).toBeGreaterThan(1) + expect( + tokenCounts.every((count) => count <= EMBEDDING_INPUT_TOKEN_LIMIT), + ).toBe(true) + expect(chunkedSections).toEqual([ + { heading: 'Oversized', chunkCount: embeddedInputs.length }, + ]) + expect(batchProgress.at(-1)).toMatchObject({ + processedSections: 1, + totalSections: 1, + }) + expect(vector).toBeDefined() + expect(Math.hypot(...(vector?.embedding ?? []))).toBeCloseTo(1) + } finally { + await fs.rm(fixture.parent, { recursive: true, force: true }) + } +}) diff --git a/src/embeddings/semantic-search-build.ts b/src/embeddings/semantic-search-build.ts index 1371967..051f34d 100644 --- a/src/embeddings/semantic-search-build.ts +++ b/src/embeddings/semantic-search-build.ts @@ -42,7 +42,17 @@ import { resolveCanonicalSourceRoot, } from '../search/path-matcher.js' import { getRecommendedDimensions, supportsMatryoshka } from './dimensions.js' -import { createEmbeddingClient, embedInBatches } from './embed-batched.js' +import { + createEmbeddingClient, + EMBEDDING_REQUEST_TOKEN_LIMIT, + type EmbeddingExecutionOptions, + embedInBatches, +} from './embed-batched.js' +import { + type EmbeddingSourceInput, + poolEmbeddingsBySource, + prepareEmbeddingInputs, +} from './embedding-inputs.js' import { type EmbeddingNamespaceError, generateNamespace, @@ -82,6 +92,13 @@ export interface EmbeddingBatchProgress { readonly totalSections: number } +export interface SectionChunkProgress { + readonly documentPath: DocumentKey + readonly heading: string + readonly tokenCount: number + readonly chunkCount: number +} + export interface BuildEmbeddingsOptions { readonly force?: boolean | undefined readonly indexRoot: string @@ -93,11 +110,15 @@ export interface BuildEmbeddingsOptions { readonly client?: EmbeddingClient | undefined readonly providerConfig?: EmbeddingProviderConfig | undefined readonly excludePatterns?: readonly string[] | undefined + readonly execution?: EmbeddingExecutionOptions | undefined readonly onFileProgress?: ((progress: FileProgress) => void) | undefined /** Callback for batch progress during embedding API calls */ readonly onBatchProgress?: | ((progress: EmbeddingBatchProgress) => void) | undefined + readonly onSectionChunked?: + | ((progress: SectionChunkProgress) => void) + | undefined /** HNSW build parameters for vector index construction */ readonly hnswOptions?: HnswBuildOptions | undefined } @@ -123,12 +144,12 @@ export interface BuildEmbeddingsResult { * section body. The format is stable so HNSW cache keys derived from it * survive across builds. */ -const generateEmbeddingText = ( +const generateEmbeddingSource = ( section: SectionEntry, content: string, documentTitle: string, parentHeading?: string | undefined, -): string => { +): EmbeddingSourceInput => { const parts: string[] = [] parts.push(`# ${section.heading}`) @@ -136,10 +157,7 @@ const generateEmbeddingText = ( parts.push(`Parent section: ${parentHeading}`) } parts.push(`Document: ${documentTitle}`) - parts.push('') - parts.push(content) - - return parts.join('\n') + return { context: parts.join('\n'), content } } type DocSections = { @@ -216,7 +234,10 @@ const groupEligibleSections = ( } interface SectionsToEmbed { - readonly sectionsToEmbed: { section: SectionEntry; text: string }[] + readonly sectionsToEmbed: { + section: SectionEntry + source: EmbeddingSourceInput + }[] readonly filesProcessed: number } @@ -233,7 +254,10 @@ const readSectionsToEmbed = ( onFileProgress: ((progress: FileProgress) => void) | undefined, ): Effect.Effect => Effect.gen(function* () { - const sectionsToEmbed: { section: SectionEntry; text: string }[] = [] + const sectionsToEmbed: { + section: SectionEntry + source: EmbeddingSourceInput + }[] = [] const docPaths = Array.from(sectionsByDoc.keys()) let filesProcessed = 0 @@ -283,13 +307,13 @@ const readSectionsToEmbed = ( .slice(section.startLine - 1, section.endLine) .join('\n') - const text = generateEmbeddingText( + const source = generateEmbeddingSource( section, content, document.title, parentHeading, ) - sectionsToEmbed.push({ section, text }) + sectionsToEmbed.push({ section, source }) } } @@ -399,25 +423,55 @@ const embedSections = ( sectionsToEmbed: SectionsToEmbed['sectionsToEmbed'], docIndex: DocumentIndex, runtime: EmbeddingRuntime, + execution: EmbeddingExecutionOptions | undefined, onBatchProgress: ((progress: EmbeddingBatchProgress) => void) | undefined, + onSectionChunked: ((progress: SectionChunkProgress) => void) | undefined, ) => Effect.gen(function* () { - const texts = sectionsToEmbed.map((section) => section.text) + const prepared = yield* prepareEmbeddingInputs( + sectionsToEmbed.map((section) => section.source), + ) + for (const chunked of prepared.chunkedSources) { + const section = sectionsToEmbed[chunked.sourceIndex]?.section + if (section === undefined) continue + onSectionChunked?.({ + documentPath: section.documentPath, + heading: section.heading, + tokenCount: chunked.tokenCount, + chunkCount: chunked.chunkCount, + }) + } + + const remainingChunks = [...prepared.chunkCounts] + let processedSections = 0 + const texts = prepared.inputs.map((input) => input.text) const result = yield* embedInBatches(runtime.client, texts, { model: runtime.providerModel, ...(supportsMatryoshka(runtime.providerModel) ? { dimensions: runtime.dimensions } : {}), + ...execution, + tokenCounts: prepared.inputs.map((input) => input.tokenCount), + maxBatchTokens: EMBEDDING_REQUEST_TOKEN_LIMIT, onBatchProgress: onBatchProgress - ? (progress) => + ? (progress) => { + for (const inputIndex of progress.completedTextIndexes) { + const sourceIndex = prepared.inputs[inputIndex]?.sourceIndex + if (sourceIndex === undefined) continue + remainingChunks[sourceIndex] = + (remainingChunks[sourceIndex] ?? 1) - 1 + if (remainingChunks[sourceIndex] === 0) processedSections++ + } onBatchProgress({ batchIndex: progress.batchIndex, totalBatches: progress.totalBatches, - processedSections: progress.processedTexts, - totalSections: progress.totalTexts, + processedSections, + totalSections: sectionsToEmbed.length, }) + } : undefined, }) + const embeddings = poolEmbeddingsBySource(prepared, result.embeddings) const tokensUsed = result.usage?.inputTokens ?? 0 const pricePerMillion = lookupPricing('embed', runtime.providerModel)?.input ?? 0 @@ -425,8 +479,14 @@ const embedSections = ( for (let index = 0; index < sectionsToEmbed.length; index++) { const section = sectionsToEmbed[index]?.section - const embedding = result.embeddings[index] - if (!section || !embedding) continue + const embedding = embeddings[index] + if ( + section === undefined || + embedding === undefined || + embedding.length !== runtime.dimensions + ) { + continue + } const documentHash = docIndex.documents[section.documentPath]?.hash if (documentHash === undefined) continue @@ -575,7 +635,9 @@ export const buildEmbeddings = ( sectionsToEmbed, docIndex, runtime, + options.execution, options.onBatchProgress, + options.onSectionChunked, ) yield* runtime.vectorStore.add(embedded.entries) runtime.vectorStore.addCost(embedded.cost, embedded.tokensUsed) diff --git a/src/embeddings/semantic-search.ts b/src/embeddings/semantic-search.ts index 4c704bd..945560d 100644 --- a/src/embeddings/semantic-search.ts +++ b/src/embeddings/semantic-search.ts @@ -65,6 +65,7 @@ export type SemanticSearchError = // ---------------------------------------------------------------------------- export { checkPricingFreshness, getPricingDate } from '../providers/pricing.js' +export type { EmbeddingExecutionOptions } from './embed-batched.js' export { clearHnswCache, invalidateHnswCache } from './hnsw-cache.js' export { type BuildEmbeddingsOptions, @@ -72,6 +73,7 @@ export { buildEmbeddings, type EmbeddingBatchProgress, type FileProgress, + type SectionChunkProgress, } from './semantic-search-build.js' export { type DirectoryEstimate, diff --git a/src/index/manifest-build.test.ts b/src/index/manifest-build.test.ts index 9dd61ae..6af0b23 100644 --- a/src/index/manifest-build.test.ts +++ b/src/index/manifest-build.test.ts @@ -3,7 +3,7 @@ import * as os from 'node:os' import * as path from 'node:path' import { Effect } from 'effect' -import { afterEach, expect, it } from 'vitest' +import { afterEach, expect, it, vi } from 'vitest' import { type DocumentKey, expandDeclaredPath } from '../db/canonical.js' import { readCurrentGeneration } from '../db/generation-paths.js' @@ -626,3 +626,50 @@ it('detects semantic no-ops and embedding-only changes', async () => { changed: true, }) }) + +it('reuses copied vectors during a forced structural rebuild', async () => { + const fixture = await makeManifestRoots() + await Effect.runPromise( + appendManifestDirectory(fixture.home, { path: fixture.first }), + ) + const embed = vi.fn((texts) => + Effect.succeed({ + embeddings: texts.map(() => [1, 0]), + model: 'test-model', + usage: { inputTokens: texts.length }, + }), + ) + const semantic = { + mode: 'build' as const, + options: { + client: { embed } satisfies EmbeddingClient, + providerConfig: { + provider: 'openai' as const, + model: 'test-model', + dimensions: 2, + }, + }, + } + + const first = await Effect.runPromise( + refreshManifestIndex(fixture.home, undefined, { semantic }), + ) + const second = await Effect.runPromise( + refreshManifestIndex(fixture.home, undefined, { + force: true, + semantic, + }), + ) + + expect(embed).toHaveBeenCalledTimes(1) + expect(second.generation).not.toBe(first.generation) + expect(second.semantic).toMatchObject({ + sectionsEmbedded: 0, + cacheHit: true, + }) + expect(second.mutation).toEqual({ + structural: false, + semantic: false, + changed: false, + }) +}) diff --git a/src/index/semantic-refresh.ts b/src/index/semantic-refresh.ts index 1ddc295..ac5b3df 100644 --- a/src/index/semantic-refresh.ts +++ b/src/index/semantic-refresh.ts @@ -18,8 +18,10 @@ type SemanticBuildOptions = Omit interface ActiveSemanticOptions { readonly mode: 'active' readonly client?: EmbeddingClient | undefined + readonly execution?: BuildEmbeddingsOptions['execution'] readonly onFileProgress?: BuildEmbeddingsOptions['onFileProgress'] readonly onBatchProgress?: BuildEmbeddingsOptions['onBatchProgress'] + readonly onSectionChunked?: BuildEmbeddingsOptions['onSectionChunked'] } export type SemanticRefreshOptions = @@ -57,8 +59,10 @@ const refreshActiveSemanticGeneration = ( baseURL: metadata.providerBaseURL, }, hnswOptions: metadata.hnswParams, + execution: options.execution, onFileProgress: options.onFileProgress, onBatchProgress: options.onBatchProgress, + onSectionChunked: options.onSectionChunked, }) }) diff --git a/src/mcp/index-generation.test.ts b/src/mcp/index-generation.test.ts index 36ac602..9d7b25d 100644 --- a/src/mcp/index-generation.test.ts +++ b/src/mcp/index-generation.test.ts @@ -148,6 +148,7 @@ it('publishes equivalent semantic generations through CLI and MCP', async () => pretty: false, embed: false, noEmbed: false, + forceEmbed: false, force: false, json: true, provider: undefined, diff --git a/src/utils/tokens.test.ts b/src/utils/tokens.test.ts index 42ed393..b7a65a1 100644 --- a/src/utils/tokens.test.ts +++ b/src/utils/tokens.test.ts @@ -1,6 +1,11 @@ import { Effect } from 'effect' import { describe, expect, it } from 'vitest' -import { countTokens, countTokensApprox, countWords } from './tokens.js' +import { + countTokens, + countTokensApprox, + countWords, + splitTextByTokens, +} from './tokens.js' describe('token utilities', () => { describe('countWords', () => { @@ -139,4 +144,28 @@ describe('token utilities', () => { expect(approx).toBeLessThanOrEqual(actual * 2) }) }) + + describe('splitTextByTokens', () => { + it('preserves text while enforcing the exact token limit', async () => { + const text = `${'alpha beta gamma\n'.repeat(80)}終わり🙂` + const chunks = await Effect.runPromise(splitTextByTokens(text, 40)) + + expect(chunks.length).toBeGreaterThan(1) + expect(chunks.map((chunk) => chunk.text).join('')).toBe(text) + expect(chunks.every((chunk) => chunk.tokenCount <= 40)).toBe(true) + await Promise.all( + chunks.map(async (chunk) => + expect(await Effect.runPromise(countTokens(chunk.text))).toBe( + chunk.tokenCount, + ), + ), + ) + }) + + it('returns one chunk when the text already fits', async () => { + const chunks = await Effect.runPromise(splitTextByTokens('short text', 8)) + + expect(chunks).toEqual([{ text: 'short text', tokenCount: 2 }]) + }) + }) }) diff --git a/src/utils/tokens.ts b/src/utils/tokens.ts index d7f5a5c..cb45b09 100644 --- a/src/utils/tokens.ts +++ b/src/utils/tokens.ts @@ -30,6 +30,78 @@ export const countTokens = ( return tokens.length }) +export interface TokenTextChunk { + readonly text: string + readonly tokenCount: number +} + +const preferredChunkEnd = ( + characters: readonly string[], + maximumEnd: number, +): number => { + const minimumEnd = Math.floor(maximumEnd * 0.8) + for (let index = maximumEnd - 1; index >= minimumEnd; index--) { + if (/\s/u.test(characters[index] ?? '')) return index + 1 + } + return maximumEnd +} + +const largestTokenSafePrefix = ( + text: string, + maxTokens: number, + enc: NonNullable, +): TokenTextChunk => { + const characters = Array.from(text) + let low = 1 + let high = characters.length + let maximumEnd = 1 + + while (low <= high) { + const middle = Math.floor((low + high) / 2) + const candidate = characters.slice(0, middle).join('') + if (enc.encode(candidate).length <= maxTokens) { + maximumEnd = middle + low = middle + 1 + } else { + high = middle - 1 + } + } + + const end = preferredChunkEnd(characters, maximumEnd) + const chunk = characters.slice(0, end).join('') + return { text: chunk, tokenCount: enc.encode(chunk).length } +} + +/** + * Split text without loss while keeping every chunk within an exact + * cl100k_base token ceiling. Natural whitespace near the boundary is + * preferred, and Unicode code points are never split. + */ +export const splitTextByTokens = ( + text: string, + maxTokens: number, +): Effect.Effect => + Effect.gen(function* () { + if (text.length === 0) return [] + const enc = yield* getEncoder + const tokenCount = enc.encode(text).length + if (tokenCount <= maxTokens) return [{ text, tokenCount }] + + const chunks: TokenTextChunk[] = [] + let remaining = text + while (remaining.length > 0) { + const remainingTokens = enc.encode(remaining).length + if (remainingTokens <= maxTokens) { + chunks.push({ text: remaining, tokenCount: remainingTokens }) + break + } + const chunk = largestTokenSafePrefix(remaining, maxTokens, enc) + chunks.push(chunk) + remaining = remaining.slice(chunk.text.length) + } + return chunks + }) + /** * Synchronous token counting with improved approximation * From 19a43388b6c5bfa1bf975667ab078614632ffbfb Mon Sep 17 00:00:00 2001 From: Stuart Robinson Date: Fri, 14 Aug 2026 05:34:49 +0700 Subject: [PATCH 2/3] test: update semantic search integration runtime --- tests/integration/search-hyde.test.ts | 22 ++++++- tests/integration/search-semantic.test.ts | 62 +++++++++++-------- .../semantic-search-test-runtime.ts | 35 +++++++++++ 3 files changed, 91 insertions(+), 28 deletions(-) create mode 100644 tests/integration/semantic-search-test-runtime.ts diff --git a/tests/integration/search-hyde.test.ts b/tests/integration/search-hyde.test.ts index 0d7eef5..a61249d 100644 --- a/tests/integration/search-hyde.test.ts +++ b/tests/integration/search-hyde.test.ts @@ -26,15 +26,30 @@ import { buildEmbeddings, semanticSearch, } from '../../src/embeddings/semantic-search.js' +import type { SemanticSearchOptions } from '../../src/embeddings/types.js' import { buildIndex } from '../../src/index/indexer.js' import { registerDefaultProviders } from '../../src/providers/index.js' +import { + loadSemanticSearchTestRuntime, + resolveSemanticSearchTestOptions, + type SemanticSearchTestRuntime, +} from './semantic-search-test-runtime.js' const TEST_DIR = path.join(process.cwd(), 'tests', 'fixtures', 'search-hyde') const originalMdmHome = process.env.MDM_HOME +let testRuntime: SemanticSearchTestRuntime const runEffect = (effect: Effect.Effect) => Effect.runPromise(effect) +const search = (query: string, options: SemanticSearchOptions) => + semanticSearch( + testRuntime.session, + TEST_DIR, + query, + resolveSemanticSearchTestOptions(testRuntime, options), + ) + const skipIfNoApiKey = () => { if (!process.env.OPENAI_API_KEY && !process.env.INCLUDE_EMBED_TESTS) { return true @@ -261,6 +276,7 @@ describe('HyDE behavior integration', () => { force: shouldRebuild, }), ) + testRuntime = await runEffect(loadSemanticSearchTestRuntime(TEST_DIR)) }, 300000) afterAll(async () => { @@ -283,7 +299,7 @@ describe('HyDE behavior integration', () => { 'how does the platform keep a long running api call authenticated when its access token reaches the end of its lifetime mid flight' const withoutHyde = await runEffect( - semanticSearch(TEST_DIR, query, { + search(query, { limit: 10, threshold: 0, hyde: false, @@ -291,7 +307,7 @@ describe('HyDE behavior integration', () => { ) const withHyde = await runEffect( - semanticSearch(TEST_DIR, query, { + search(query, { limit: 10, threshold: 0, hyde: true, @@ -329,7 +345,7 @@ describe('HyDE behavior integration', () => { // return ranked results. This guards against accidental property // dropping in the hydeOptions forwarding path. const results = await runEffect( - semanticSearch(TEST_DIR, 'how do refresh tokens rotate', { + search('how do refresh tokens rotate', { limit: 5, threshold: 0, hyde: true, diff --git a/tests/integration/search-semantic.test.ts b/tests/integration/search-semantic.test.ts index 507c06d..bee5a7c 100644 --- a/tests/integration/search-semantic.test.ts +++ b/tests/integration/search-semantic.test.ts @@ -17,8 +17,14 @@ import { buildEmbeddings, semanticSearchWithStats, } from '../../src/embeddings/semantic-search.js' +import type { SemanticSearchOptions } from '../../src/embeddings/types.js' import { buildIndex } from '../../src/index/indexer.js' import { registerDefaultProviders } from '../../src/providers/index.js' +import { + loadSemanticSearchTestRuntime, + resolveSemanticSearchTestOptions, + type SemanticSearchTestRuntime, +} from './semantic-search-test-runtime.js' const TEST_DIR = path.join( process.cwd(), @@ -27,10 +33,19 @@ const TEST_DIR = path.join( 'semantic-search', ) const originalMdmHome = process.env.MDM_HOME +let testRuntime: SemanticSearchTestRuntime const runEffect = (effect: Effect.Effect) => Effect.runPromise(effect) +const searchWithStats = (query: string, options: SemanticSearchOptions) => + semanticSearchWithStats( + testRuntime.session, + TEST_DIR, + query, + resolveSemanticSearchTestOptions(testRuntime, options), + ) + const skipIfNoApiKey = () => { if (!process.env.OPENAI_API_KEY && !process.env.INCLUDE_EMBED_TESTS) { return true @@ -209,6 +224,7 @@ Database migrations cannot be automatically rolled back. force: shouldRebuild, }), ) + testRuntime = await runEffect(loadSemanticSearchTestRuntime(TEST_DIR)) }, 60000) afterAll(async () => { @@ -223,7 +239,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'how does user login work', { + searchWithStats('how does user login work', { limit: 5, threshold: 0.3, }), @@ -242,7 +258,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'authentication', { + searchWithStats('authentication', { limit: 10, threshold: 0.2, }), @@ -263,14 +279,14 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const lowThreshold = await runEffect( - semanticSearchWithStats(TEST_DIR, 'deployment', { + searchWithStats('deployment', { limit: 10, threshold: 0.2, }), ) const highThreshold = await runEffect( - semanticSearchWithStats(TEST_DIR, 'deployment', { + searchWithStats('deployment', { limit: 10, threshold: 0.5, }), @@ -289,7 +305,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'authentication', { + searchWithStats('authentication', { limit: 5, threshold: 0.99, }), @@ -310,7 +326,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'api endpoints', { + searchWithStats('api endpoints', { limit: 3, threshold: 0.2, }), @@ -323,7 +339,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'user', { + searchWithStats('user', { limit: 2, threshold: 0.2, }), @@ -340,7 +356,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'authentication', { + searchWithStats('authentication', { limit: 100, threshold: 0.3, }), @@ -356,7 +372,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'authentication', { + searchWithStats('authentication', { limit: 5, threshold: 0.3, contextBefore: 2, @@ -381,7 +397,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'quantum physics blockchain AI', { + searchWithStats('quantum physics blockchain AI', { limit: 5, threshold: 0.7, }), @@ -394,7 +410,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'api', { + searchWithStats('api', { limit: 5, threshold: 0.3, }), @@ -410,7 +426,7 @@ Database migrations cannot be automatically rolled back. 'I need to understand how the authentication system works including user login with email and password, OAuth integration with third-party providers like Google and GitHub, session management with Redis, and security best practices for production deployment' const result = await runEffect( - semanticSearchWithStats(TEST_DIR, longQuery, { + searchWithStats(longQuery, { limit: 5, threshold: 0.3, }), @@ -423,7 +439,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'OAuth 2.0 & JWT tokens', { + searchWithStats('OAuth 2.0 & JWT tokens', { limit: 5, threshold: 0.3, }), @@ -438,7 +454,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'database schema', { + searchWithStats('database schema', { limit: 5, threshold: 0.3, }), @@ -470,14 +486,10 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats( - TEST_DIR, - 'configuration environment variables', - { - limit: 10, - threshold: 0.3, - }, - ), + searchWithStats('configuration environment variables', { + limit: 10, + threshold: 0.3, + }), ) expect(result.results.length).toBeGreaterThan(0) @@ -490,7 +502,7 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const result = await runEffect( - semanticSearchWithStats(TEST_DIR, 'security and passwords', { + searchWithStats('security and passwords', { limit: 10, threshold: 0.3, }), @@ -505,14 +517,14 @@ Database migrations cannot be automatically rolled back. if (skipIfNoApiKey()) return const authResult = await runEffect( - semanticSearchWithStats(TEST_DIR, 'OAuth authentication', { + searchWithStats('OAuth authentication', { limit: 5, threshold: 0.2, }), ) const unrelatedResult = await runEffect( - semanticSearchWithStats(TEST_DIR, 'database backup', { + searchWithStats('database backup', { limit: 5, threshold: 0.2, }), diff --git a/tests/integration/semantic-search-test-runtime.ts b/tests/integration/semantic-search-test-runtime.ts new file mode 100644 index 0000000..317806a --- /dev/null +++ b/tests/integration/semantic-search-test-runtime.ts @@ -0,0 +1,35 @@ +import { Effect } from 'effect' + +import { defaultConfig } from '../../src/config/schema.js' +import { testGenerationSession } from '../../src/db/generation-test-fixture.js' +import type { GenerationReadSession } from '../../src/db/generation-types.js' +import { + type ResolvedQueryProviderConfig, + resolveQueryProviderConfig, +} from '../../src/embeddings/query-provider-config.js' +import type { + ResolvedSemanticSearchOptions, + SemanticSearchOptions, +} from '../../src/embeddings/types.js' + +export interface SemanticSearchTestRuntime { + readonly session: GenerationReadSession + readonly queryProvider: ResolvedQueryProviderConfig +} + +export const loadSemanticSearchTestRuntime = (indexRoot: string) => { + const session = testGenerationSession(indexRoot) + return Effect.map( + resolveQueryProviderConfig(session, defaultConfig.embeddings), + (queryProvider) => ({ session, queryProvider }), + ) +} + +export const resolveSemanticSearchTestOptions = ( + runtime: SemanticSearchTestRuntime, + options: SemanticSearchOptions, +): ResolvedSemanticSearchOptions => ({ + ...options, + providerConfig: runtime.queryProvider.providerConfig, + activeProvider: runtime.queryProvider.activeProvider, +}) From f75e5e391cd566efb37b6c0ec82e22158aefcad1 Mon Sep 17 00:00:00 2001 From: Stuart Robinson Date: Fri, 14 Aug 2026 05:34:49 +0700 Subject: [PATCH 3/3] docs: add agent coordination lessons --- LESSONS.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/LESSONS.md b/LESSONS.md index ba15e99..b38409e 100644 --- a/LESSONS.md +++ b/LESSONS.md @@ -12,6 +12,15 @@ garbage). Reserve the Agent/subagent tool for single, self-contained delegated lookups. When Stuart says "all general agent, claude use opus," he means the warroom's general-agent preset with opus on the Claude panes. (2026-07-04) +## Launch the full warroom roster up front, not incrementally +Always spawn the complete standing roster at warroom start so every agent holds a +fixed bus address for the whole job, rather than growing pane by pane per phase. +The standing team spans all three model families, one each: **codex** scout/code, +**opus** reviewer (cross-family vs the codex scout), **grok** personal +assistant/ad-hoc queries. Do not defer the reviewer and assistant seats to a later +phase; a fixed address that never churns is worth more than saving idle-pane +overhead. (2026-07-23) + ## Normalize paths at portable text boundaries Keep declared paths in the platform native form used by filesystem logic. Normalize backslashes to forward slashes only when writing portable text such @@ -105,3 +114,10 @@ when existing behavior already exercises the path. Verify low risk wording edits through the existing command and relevant established gates. Add tests when they protect meaningful logic, branching, or a demonstrated recurring failure. (2026-07-22) + +## Let bus nudges drive inbox checks +Do not poll Helioy Bus mail at task, turn, or session boundaries. Read the inbox +after a "you have mail!" nudge or an explicit user request. The bus sends a nudge +when mail arrives. If tmux readdresses the current pane, compare `whoami` with +the current target and re-register stale Bus identity using the stable pane ID. +(2026-07-22)