perf(embeddings): speed up and harden index builds

LESSONS.md

@@ -12,6 +12,15 @@ garbage). Reserve the Agent/subagent tool for single, self-contained delegated
 lookups. When Stuart says "all general agent, claude use opus," he means the
 warroom's general-agent preset with opus on the Claude panes. (2026-07-04)
 
+## Launch the full warroom roster up front, not incrementally
+Always spawn the complete standing roster at warroom start so every agent holds a
+fixed bus address for the whole job, rather than growing pane by pane per phase.
+The standing team spans all three model families, one each: **codex** scout/code,
+**opus** reviewer (cross-family vs the codex scout), **grok** personal
+assistant/ad-hoc queries. Do not defer the reviewer and assistant seats to a later
+phase; a fixed address that never churns is worth more than saving idle-pane
+overhead. (2026-07-23)
+
 ## Normalize paths at portable text boundaries
 Keep declared paths in the platform native form used by filesystem logic.
 Normalize backslashes to forward slashes only when writing portable text such
@@ -105,3 +114,10 @@ when existing behavior already exercises the path. Verify low risk wording edits
 through the existing command and relevant established gates. Add tests when they
 protect meaningful logic, branching, or a demonstrated recurring failure.
 (2026-07-22)
+
+## Let bus nudges drive inbox checks
+Do not poll Helioy Bus mail at task, turn, or session boundaries. Read the inbox
+after a "you have mail!" nudge or an explicit user request. The bus sends a nudge
+when mail arrives. If tmux readdresses the current pane, compare `whoami` with
+the current target and re-register stale Bus identity using the stable pane ID.
+(2026-07-22)

README.md

@@ -75,6 +75,7 @@ mdm index                       # Later: refresh all existing manifest directori
 mdm index ./docs                # Append path, then refresh all directories
 mdm index . --embed             # First index and build semantic embeddings
 mdm index --no-embed            # Leave semantic vectors unchanged
+mdm index --force-embed         # Rebuild every semantic embedding
 mdm index --watch               # Fails; multi-root manifest watch is unavailable
 mdm index --force               # Bypass cache, re-process all files
 mdm index --exclude "*.draft.md,research/**"  # Exclude patterns (comma-separated)

docs/CONFIG.md

@@ -59,6 +59,7 @@ snippetLength = 200
 provider = "openai"
 model = "text-embedding-3-small"
 batchSize = 100
+concurrency = 4
 ```
 
 ---
@@ -130,6 +131,7 @@ provider = "openai"
 model = "text-embedding-3-small"
 dimensions = 512
 batchSize = 100
+concurrency = 4
 maxRetries = 3
 # baseURL = "https://custom-endpoint.example.com"
 # apiKey = "sk-..."
@@ -283,12 +285,18 @@ Controls semantic search embedding generation.
 | `provider`     | `string` | `'openai'`                 | Embedding provider (openai, ollama, lm-studio, openrouter) |
 | `model`        | `string` | `'text-embedding-3-small'` | Embedding model name                     |
 | `dimensions`   | `number` | (auto)                     | Vector dimensions (auto-detected from model if not set) |
-| `batchSize`    | `number` | `100`                      | Batch size for API calls                 |
+| `batchSize`    | `number` | `100`                      | Inputs per embedding API call            |
+| `concurrency`  | `number` | `4`                        | Maximum embedding API calls in flight    |
 | `maxRetries`   | `number` | `3`                        | Maximum retries for failed API calls     |
 | `retryDelayMs` | `number` | `1000`                     | Delay between retries in milliseconds    |
 | `timeoutMs`    | `number` | `30000`                    | Request timeout in milliseconds          |
 | `apiKey`       | `string` | (from env)                 | API key (prefer environment variable) |
 
+Embedding inputs are token counted before submission. Sections above the
+provider safe limit are split into bounded inputs, embedded with their heading
+and document context, then pooled into one normalized section vector. Requests
+are also packed below the provider aggregate token limit.
+
 **Model Dimensions:**
 
 Dimensions are now automatically configured based on the model. If not explicitly set:
@@ -323,6 +331,7 @@ model = "text-embedding-3-large"
 
 # Smaller batches for rate limiting
 batchSize = 50
+concurrency = 4
 
 # More aggressive retries
 maxRetries = 5
@@ -563,6 +572,7 @@ environment var:   MDM_INDEX_MAXDEPTH
 | `MDM_EMBEDDINGS_MODEL`      | `embeddings.model`       |
 | `MDM_EMBEDDINGS_DIMENSIONS` | `embeddings.dimensions`  |
 | `MDM_EMBEDDINGS_BATCHSIZE`  | `embeddings.batchSize`   |
+| `MDM_EMBEDDINGS_CONCURRENCY` | `embeddings.concurrency` |
 | `MDM_EMBEDDINGS_MAXRETRIES` | `embeddings.maxRetries`  |
 | `MDM_EMBEDDINGS_RETRYDELAYMS` | `embeddings.retryDelayMs` |
 | `MDM_EMBEDDINGS_TIMEOUTMS`  | `embeddings.timeoutMs`   |

docs/USAGE.md

@@ -100,7 +100,8 @@ mdm index [path] [options]
 | ------------- | ------------------------------------------ |
 | `-e, --embed` | Also build semantic embeddings             |
 | `-w, --watch` | Show deferred manifest watch guidance      |
-| `--force`     | Force full rebuild (ignore cache)          |
+| `--force`     | Rebuild the structural index               |
+| `--force-embed` | Rebuild all semantic embeddings          |
 | `--json`      | Output as JSON                             |
 | `--pretty`    | Pretty-print JSON                          |
 
@@ -121,8 +122,15 @@ mdm index --watch
 
 # Force rebuild
 mdm index --force
+
+# Force a semantic rebuild
+mdm index --force-embed
 ```
 
+Sections larger than an embedding provider accepts are split automatically and
+stored as one normalized section vector. mdm reports the document and heading
+when this occurs.
+
 **Index location:** `.mdm/indexes/`
 
 ---

src/cli/argv-preprocessor.test.ts

@@ -70,6 +70,17 @@ describe('preprocessArgvWithValidation', () => {
       expect(result.error).toBeUndefined()
     })
 
+    it('accepts the semantic rebuild flag', () => {
+      const result = preprocessArgvWithValidation([
+        node,
+        script,
+        'index',
+        '--force-embed',
+      ])
+      expect(result.argv).toEqual([node, script, 'index', '--force-embed'])
+      expect(result.error).toBeUndefined()
+    })
+
     it('passes through --help flag', () => {
       const result = preprocessArgvWithValidation([
         node,

src/cli/cli.test.ts

@@ -191,6 +191,7 @@ describe('mdm CLI e2e', () => {
       expect(output).toContain('--hnsw-m')
       expect(output).toContain('--hnsw-ef-construction')
       expect(output).toContain('--force')
+      expect(output).toContain('--force-embed')
       expect(output).not.toContain('--all')
       expect(output).not.toContain('--timeout')
     })

src/cli/commands/config-cmd.test.ts

@@ -176,6 +176,7 @@ color = "yes"
     expect(result.code).toBe(0)
     expect(parsed.valid).toBe(true)
     expect(parsed.config.embeddings.maxRetries.source).toBe('file')
+    expect(parsed.config.embeddings.concurrency.source).toBe('file')
     expect(parsed.config.embeddings.retryDelayMs.source).toBe('file')
     expect(parsed.config.embeddings.timeoutMs.source).toBe('file')
     expect(parsed.config.embeddings.hnswM.source).toBe('file')

src/cli/commands/index-cmd.ts

@@ -21,6 +21,13 @@ const noEmbedOption = Options.boolean('no-embed').pipe(
   Options.withDefault(false),
 )
 
+const forceEmbedOption = Options.boolean('force-embed').pipe(
+  Options.withDescription(
+    'Rebuild every semantic embedding instead of reusing unchanged vectors',
+  ),
+  Options.withDefault(false),
+)
+
 const excludeOption = Options.text('exclude').pipe(
   Options.withAlias('x'),
   Options.withDescription(
@@ -83,6 +90,7 @@ export const indexCommand = Command.make(
     path: pathArg,
     embed: embedOption,
     noEmbed: noEmbedOption,
+    forceEmbed: forceEmbedOption,
     exclude: excludeOption,
     noGitignore: noGitignoreOption,
     provider: providerOption,
@@ -100,6 +108,7 @@ export const indexCommand = Command.make(
       path: Option.getOrUndefined(input.path),
       embed: input.embed,
       noEmbed: input.noEmbed,
+      forceEmbed: input.forceEmbed,
       exclude: Option.getOrUndefined(input.exclude),
       noGitignore: input.noGitignore,
       provider: Option.getOrUndefined(input.provider),

src/cli/commands/index-embeddings.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from 'vitest'
+
+import { defaultConfig } from '../../config/schema.js'
+import type { ProviderId } from '../../providers/index.js'
+import {
+  type EmbeddingRefreshInput,
+  semanticRefreshOptions,
+} from './index-embeddings.js'
+
+const input = (
+  overrides: Partial<EmbeddingRefreshInput> = {},
+): EmbeddingRefreshInput => ({
+  embed: true,
+  noEmbed: false,
+  forceEmbed: false,
+  force: false,
+  json: true,
+  provider: undefined,
+  providerBaseUrl: undefined,
+  providerModel: undefined,
+  hnswM: undefined,
+  hnswEfConstruction: undefined,
+  ...overrides,
+})
+
+describe('semanticRefreshOptions', () => {
+  it('keeps structural force separate from semantic rebuilds', () => {
+    const result = semanticRefreshOptions(
+      input({ force: true }),
+      false,
+      defaultConfig.embeddings,
+    )
+
+    expect(result).toMatchObject({
+      mode: 'build',
+      options: { force: false },
+    })
+  })
+
+  it('makes a full semantic rebuild explicit', () => {
+    const result = semanticRefreshOptions(
+      input({ embed: false, forceEmbed: true }),
+      false,
+      defaultConfig.embeddings,
+    )
+
+    expect(result).toMatchObject({
+      mode: 'build',
+      options: { force: true },
+    })
+  })
+
+  it('passes configured batching policy and provider defaults', () => {
+    const config = {
+      ...defaultConfig.embeddings,
+      provider: 'openrouter' as ProviderId,
+      model: 'configured-model',
+      batchSize: 40,
+      concurrency: 3,
+      maxRetries: 6,
+      retryDelayMs: 250,
+      timeoutMs: 45_000,
+    }
+    const result = semanticRefreshOptions(input(), false, config)
+
+    expect(result).toMatchObject({
+      mode: 'build',
+      options: {
+        providerConfig: {
+          provider: 'openrouter',
+          model: 'configured-model',
+        },
+        execution: {
+          batchSize: 40,
+          concurrency: 3,
+          maxRetries: 6,
+          retryDelayMs: 250,
+          timeoutMs: 45_000,
+        },
+      },
+    })
+  })
+})

src/cli/commands/index-embeddings.ts

@@ -1,8 +1,10 @@
-import { Console, Effect } from 'effect'
+import { Console, Effect, Option } from 'effect'
 
+import type { EmbeddingsConfig } from '../../config/schema.js'
 import type {
   BuildEmbeddingsOptions,
   BuildEmbeddingsResult,
+  EmbeddingExecutionOptions,
   EmbeddingProviderConfig,
 } from '../../embeddings/semantic-search.js'
 import type { SemanticRefreshOptions } from '../../index/semantic-refresh.js'
@@ -11,6 +13,7 @@ import type { ProviderId } from '../../providers/index.js'
 export interface EmbeddingRefreshInput {
   readonly embed: boolean
   readonly noEmbed: boolean
+  readonly forceEmbed: boolean
   readonly force: boolean
   readonly json: boolean
   readonly provider: ProviderId | undefined
@@ -22,26 +25,39 @@ export interface EmbeddingRefreshInput {
 
 const providerConfig = (
   input: EmbeddingRefreshInput,
-): EmbeddingProviderConfig | undefined => {
-  if (input.provider !== undefined) {
-    return {
-      provider: input.provider,
-      baseURL: input.providerBaseUrl,
-      model: input.providerModel,
-    }
-  }
-  return undefined
-}
+  config: EmbeddingsConfig,
+): EmbeddingProviderConfig => ({
+  provider: input.provider ?? config.provider,
+  baseURL: input.providerBaseUrl ?? Option.getOrUndefined(config.baseURL),
+  model: input.providerModel ?? config.model,
+  dimensions: config.dimensions,
+})
 
-const hnswOptions = (input: EmbeddingRefreshInput) =>
-  input.hnswM !== undefined || input.hnswEfConstruction !== undefined
-    ? { m: input.hnswM, efConstruction: input.hnswEfConstruction }
-    : undefined
+const hnswOptions = (
+  input: EmbeddingRefreshInput,
+  config: EmbeddingsConfig,
+) => ({
+  m: input.hnswM ?? config.hnswM,
+  efConstruction: input.hnswEfConstruction ?? config.hnswEfConstruction,
+})
+
+const executionOptions = (
+  config: EmbeddingsConfig,
+): EmbeddingExecutionOptions => ({
+  batchSize: config.batchSize,
+  concurrency: config.concurrency,
+  maxRetries: config.maxRetries,
+  retryDelayMs: config.retryDelayMs,
+  timeoutMs: config.timeoutMs,
+})
 
 const progressOptions = (
   input: EmbeddingRefreshInput,
   showProgress: boolean,
-): Pick<BuildEmbeddingsOptions, 'onBatchProgress' | 'onFileProgress'> => ({
+): Pick<
+  BuildEmbeddingsOptions,
+  'onBatchProgress' | 'onFileProgress' | 'onSectionChunked'
+> => ({
   onFileProgress: (progress) => {
     if (!input.json && showProgress) {
       process.stdout.write(
@@ -56,21 +72,33 @@ const progressOptions = (
       )
     }
   },
+  onSectionChunked: (progress) => {
+    if (input.json) return
+    if (showProgress) process.stdout.write('\x1b[2K\r')
+    process.stderr.write(
+      `  Chunking oversized section: ${progress.documentPath} > ${progress.heading} (${progress.tokenCount} tokens into ${progress.chunkCount} inputs)\n`,
+    )
+  },
 })
 
 export const semanticRefreshOptions = (
   input: EmbeddingRefreshInput,
   showProgress: boolean,
+  config: EmbeddingsConfig,
 ): SemanticRefreshOptions => {
   if (input.noEmbed) return { mode: 'skip' }
   const progress = progressOptions(input, showProgress)
-  if (!input.embed) return { mode: 'active', ...progress }
+  const execution = executionOptions(config)
+  if (!input.embed && !input.forceEmbed) {
+    return { mode: 'active', execution, ...progress }
+  }
   return {
     mode: 'build',
     options: {
-      force: input.force,
-      providerConfig: providerConfig(input),
-      hnswOptions: hnswOptions(input),
+      force: input.forceEmbed,
+      providerConfig: providerConfig(input, config),
+      hnswOptions: hnswOptions(input, config),
+      execution,
       ...progress,
     },
   }
@@ -83,7 +111,7 @@ const renderEmbeddingResult = (result: BuildEmbeddingsResult) =>
       yield* Console.log(
         `Embeddings already exist (${result.existingVectors} vectors)`,
       )
-      yield* Console.log('  Use --force to rebuild')
+      yield* Console.log('  Use --force-embed to rebuild')
       return
     }