feat(bench): realistic per-language fixture generators + scenarios (#845)

.bench/baseline.json

@@ -1,5 +1,5 @@
 {
-  "capturedAt": "2026-05-05T15:06:12.102Z",
+  "capturedAt": "2026-05-06T00:29:28.140Z",
   "node": "v22.13.0",
   "platform": "darwin-arm64",
   "options": {
@@ -22,19 +22,64 @@
       "fixture": "medium",
       "fileCount": 25,
       "approxTokens": 36150,
-      "durationMs": 30213,
+      "durationMs": 31124,
       "llmCalls": 20,
-      "llmTotalMs": 102723,
-      "llmTotalPromptTokens": 91766
+      "llmTotalMs": 106333,
+      "llmTotalPromptTokens": 34237
     },
     {
       "fixture": "large",
       "fileCount": 50,
       "approxTokens": 83410,
-      "durationMs": 70048,
+      "durationMs": 72151,
       "llmCalls": 41,
-      "llmTotalMs": 236818,
-      "llmTotalPromptTokens": 220199
+      "llmTotalMs": 244112,
+      "llmTotalPromptTokens": 74197
+    },
+    {
+      "fixture": "feature-add",
+      "fileCount": 14,
+      "approxTokens": 17600,
+      "durationMs": 15967,
+      "llmCalls": 11,
+      "llmTotalMs": 54726,
+      "llmTotalPromptTokens": 18937
+    },
+    {
+      "fixture": "refactor",
+      "fileCount": 30,
+      "approxTokens": 32650,
+      "durationMs": 33994,
+      "llmCalls": 28,
+      "llmTotalMs": 153871,
+      "llmTotalPromptTokens": 52430
+    },
+    {
+      "fixture": "initial-commit",
+      "fileCount": 50,
+      "approxTokens": 83410,
+      "durationMs": 72291,
+      "llmCalls": 41,
+      "llmTotalMs": 245148,
+      "llmTotalPromptTokens": 74546
+    },
+    {
+      "fixture": "docs-update",
+      "fileCount": 9,
+      "approxTokens": 15050,
+      "durationMs": 18563,
+      "llmCalls": 8,
+      "llmTotalMs": 56293,
+      "llmTotalPromptTokens": 13908
+    },
+    {
+      "fixture": "dep-bump",
+      "fileCount": 3,
+      "approxTokens": 8450,
+      "durationMs": 27158,
+      "llmCalls": 1,
+      "llmTotalMs": 27141,
+      "llmTotalPromptTokens": 19597
     }
   ]
 }

src/lib/parsers/default/__fixtures__/diffs.ts

@@ -0,0 +1,107 @@
+/**
+ * Diff-shape wrappers for the realistic fixture generators (#845).
+ * Real `git diff` output has different headers + line prefixes
+ * depending on whether the change is a pure addition, pure
+ * deletion, modification, or rename. The condensing pipeline's
+ * tokenizer counts those characters, and an upcoming "skip-trivial"
+ * optimization (PR 2) detects shape from the prefixes — so the
+ * fixture contents need to match real git output closely enough that
+ * those detection passes behave the same.
+ */
+
+export type DiffShape = 'addition' | 'deletion' | 'modification' | 'rename' | 'binary'
+
+function gitHeader(file: string, shape: DiffShape, oldFile?: string): string {
+  switch (shape) {
+    case 'addition':
+      return [
+        `diff --git a/${file} b/${file}`,
+        'new file mode 100644',
+        'index 0000000..1234567',
+        '--- /dev/null',
+        `+++ b/${file}`,
+      ].join('\n')
+    case 'deletion':
+      return [
+        `diff --git a/${file} b/${file}`,
+        'deleted file mode 100644',
+        'index 1234567..0000000',
+        `--- a/${file}`,
+        '+++ /dev/null',
+      ].join('\n')
+    case 'rename':
+      return [
+        `diff --git a/${oldFile || file} b/${file}`,
+        'similarity index 100%',
+        `rename from ${oldFile || file}`,
+        `rename to ${file}`,
+      ].join('\n')
+    case 'binary':
+      return [
+        `diff --git a/${file} b/${file}`,
+        `Binary files a/${file} and b/${file} differ`,
+      ].join('\n')
+    case 'modification':
+    default:
+      return [
+        `diff --git a/${file} b/${file}`,
+        'index 1234567..89abcde 100644',
+        `--- a/${file}`,
+        `+++ b/${file}`,
+      ].join('\n')
+  }
+}
+
+/**
+ * Pure-addition diff: every content line gets a `+` prefix. Mirrors
+ * git's output for a brand-new file.
+ */
+export function asAdditionDiff(file: string, content: string): string {
+  const lines = content.split('\n')
+  const body = `@@ -0,0 +1,${lines.length} @@`
+  const plus = lines.map((line) => `+${line}`).join('\n')
+  return `${gitHeader(file, 'addition')}\n${body}\n${plus}\n`
+}
+
+/** Pure-deletion diff: every line gets a `-` prefix. */
+export function asDeletionDiff(file: string, content: string): string {
+  const lines = content.split('\n')
+  const body = `@@ -1,${lines.length} +0,0 @@`
+  const minus = lines.map((line) => `-${line}`).join('\n')
+  return `${gitHeader(file, 'deletion')}\n${body}\n${minus}\n`
+}
+
+/**
+ * Modification diff: realistic mix of context lines, removals, and
+ * additions. The "modify ratio" picks roughly what fraction of lines
+ * are touched (default 30%) — the rest render as context (` ` prefix).
+ */
+export function asModificationDiff(
+  file: string,
+  oldContent: string,
+  newContent: string
+): string {
+  const oldLines = oldContent.split('\n')
+  const newLines = newContent.split('\n')
+  // Naive "diff": for the bench we don't need a real LCS — just a
+  // plausible interleaving of context, removals, and additions.
+  // Take the first half of old as context, second half as removals,
+  // then add the new lines.
+  const half = Math.floor(oldLines.length / 2)
+  const contextLines = oldLines.slice(0, half).map((line) => ` ${line}`)
+  const removedLines = oldLines.slice(half).map((line) => `-${line}`)
+  const addedLines = newLines.slice(0, Math.max(removedLines.length, 4)).map((line) => `+${line}`)
+  const hunkHeader = `@@ -1,${oldLines.length} +1,${contextLines.length + addedLines.length} @@`
+  const body = [...contextLines, ...removedLines, ...addedLines].join('\n')
+  return `${gitHeader(file, 'modification')}\n${hunkHeader}\n${body}\n`
+}
+
+/** Rename diff with no content change (shape #2 from PR 2 plan). */
+export function asRenameDiff(oldFile: string, newFile: string): string {
+  return `${gitHeader(newFile, 'rename', oldFile)}\n`
+}
+
+/** Binary file change (shape #3 from PR 2 plan). */
+export function asBinaryDiff(file: string): string {
+  return `${gitHeader(file, 'binary')}\n`
+}

src/lib/parsers/default/__fixtures__/generators.test.ts

@@ -0,0 +1,69 @@
+import {
+  generateContentForFile,
+  generateJson,
+  generateMarkdown,
+  generatePython,
+  generateTypeScript,
+  generateYaml,
+  seededRng,
+} from './generators'
+
+describe('bench fixture generators (#845)', () => {
+  describe('seededRng', () => {
+    it('produces identical sequences for the same seed', () => {
+      const a = seededRng(12345)
+      const b = seededRng(12345)
+      const aValues = Array.from({ length: 10 }, () => a())
+      const bValues = Array.from({ length: 10 }, () => b())
+      expect(aValues).toEqual(bValues)
+    })
+
+    it('produces different sequences for different seeds', () => {
+      const a = seededRng(1)
+      const b = seededRng(2)
+      const aValues = Array.from({ length: 10 }, () => a())
+      const bValues = Array.from({ length: 10 }, () => b())
+      expect(aValues).not.toEqual(bValues)
+    })
+  })
+
+  describe('per-language generators', () => {
+    const cases = [
+      { name: 'TypeScript', generate: generateTypeScript, mustContain: ['import', 'export'] },
+      { name: 'Python', generate: generatePython, mustContain: ['def ', 'import'] },
+      { name: 'Markdown', generate: generateMarkdown, mustContain: ['#'] },
+      { name: 'JSON', generate: generateJson, mustContain: ['{', '}', ':'] },
+      { name: 'YAML', generate: generateYaml, mustContain: ['name:', 'jobs:'] },
+    ]
+
+    it.each(cases)('$name output is deterministic and contains expected markers', ({ generate, mustContain }) => {
+      const a = generate(500, 42)
+      const b = generate(500, 42)
+      expect(a).toBe(b)
+      mustContain.forEach((token) => expect(a).toContain(token))
+    })
+
+    it.each(cases)('$name output scales roughly with the requested token target', ({ generate }) => {
+      const small = generate(100, 7)
+      const large = generate(2000, 7)
+      // The generators target chars/4, so large should be ~20x small.
+      // Loose lower bound: at least 5x bigger to confirm scaling works.
+      expect(large.length).toBeGreaterThan(small.length * 5)
+    })
+  })
+
+  describe('generateContentForFile dispatcher', () => {
+    it('routes by extension', () => {
+      expect(generateContentForFile('foo.ts', 200, 1)).toContain('import')
+      expect(generateContentForFile('foo.py', 200, 1)).toContain('def ')
+      expect(generateContentForFile('foo.md', 200, 1)).toContain('#')
+      expect(generateContentForFile('foo.json', 200, 1)).toContain('{')
+      expect(generateContentForFile('foo.yml', 200, 1)).toContain('jobs:')
+    })
+
+    it('falls back to TypeScript for unknown extensions', () => {
+      const out = generateContentForFile('foo.unknown', 200, 1)
+      expect(out).toContain('import')
+    })
+  })
+})