From f4c65b56a72ec9e2b3645992604b347ecd51e067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 3 May 2026 15:09:02 +0200 Subject: [PATCH 1/2] feat: render Mockolate benchmarks on the docs site Adds a 6-library ranked-bar benchmark visualization for Mockolate that mirrors the existing aweXpect benchmark page but extends to all six mocking libraries Mockolate compares against (Mockolate, Moq, NSubstitute, FakeItEasy, TUnit.Mocks, Imposter). - MockolateBenchmarks Nuke target: fetches limited-data.js from the Testably/Mockolate benchmarks branch, splits each chart key into a scenario plus optional [Params] value, and reduces it to one snapshot entry per scenario. Wired into the Pages target's DependsOn chain. - MockolateBenchmarkResult React component: renders all libraries sorted fastest-first, with Mockolate highlighted as the baseline and every other row showing its ratio relative to Mockolate (e.g. 1.50x). Inline parameter tabs surface the [Params(1, 10)] variants. - Seeded src/data/mockolate/benchmarks.json from the live benchmarks branch (commit 4c61cdf5) so npm start / npm run build work without network access in clean checkouts. --- .nuke/build.schema.json | 1 + .../MockolateBenchmarkResult/index.tsx | 271 +++ .../styles.module.css | 197 ++ Docs/pages/src/data/mockolate/benchmarks.json | 2157 +++++++++++++++++ Pipeline/Build.MockolateBenchmarks.cs | 193 ++ Pipeline/Build.Pages.cs | 1 + 6 files changed, 2820 insertions(+) create mode 100644 Docs/pages/src/components/MockolateBenchmarkResult/index.tsx create mode 100644 Docs/pages/src/components/MockolateBenchmarkResult/styles.module.css create mode 100644 Docs/pages/src/data/mockolate/benchmarks.json create mode 100644 Pipeline/Build.MockolateBenchmarks.cs diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index 9729375e..9d1166a1 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -25,6 +25,7 @@ "type": "string", "enum": [ "Benchmarks", + "MockolateBenchmarks", "Pages" ] }, diff --git a/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx b/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx new file mode 100644 index 00000000..45a5912b --- /dev/null +++ b/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx @@ -0,0 +1,271 @@ +/** + * Renders the latest Mockolate-vs-other-mocking-libraries snapshot for a + * single benchmark scenario (e.g. "Method", "Property", "Callback"). + * Reads from `src/data/mockolate/benchmarks.json` — refreshed at deploy + * time by Testably.Site's docs pipeline (the `MockolateBenchmarks` Nuke + * target) from the `Testably/Mockolate` benchmarks branch. The committed + * file doubles as a fallback so `npm start`/`npm run build` work in a + * clean checkout. + * + * For benchmarks parameterised with `[Params(1, 10)]`, the component + * renders inline N=1 / N=10 buttons so readers can switch between + * fixed-cost and per-call regimes. + */ +import React, {type ReactElement, useState} from 'react'; +import benchmarks from '@site/src/data/mockolate/benchmarks.json'; +import styles from './styles.module.css'; + +const BASELINE_LIBRARY = 'Mockolate'; +const LIBRARY_ORDER = [ + 'Mockolate', + 'Moq', + 'NSubstitute', + 'FakeItEasy', + 'TUnitMocks', + 'Imposter', +] as const; +const LIBRARY_DISPLAY_NAMES: Record = { + TUnitMocks: 'TUnit.Mocks', +}; + +type Sample = { + timeNs: number; + memoryBytes?: number; + history?: { + timeNs: number[]; + memoryBytes: number[]; + }; +}; + +type Scenario = { + parameters: string[]; + samples: Record>; +}; + +type Snapshot = { + capturedAt: { + sha: string; + shortSha: string; + date: string; + message?: string; + }; + environment?: { + runner?: string; + toolchain?: string; + }; + benchmarks: Record; +}; + +const data = benchmarks as Snapshot; + +function formatNs(ns: number): string { + if (ns < 1_000) return `${ns.toFixed(1)} ns`; + if (ns < 1_000_000) return `${(ns / 1_000).toFixed(2)} µs`; + return `${(ns / 1_000_000).toFixed(2)} ms`; +} + +function formatBytes(b: number): string { + if (b < 1_024) return `${b} B`; + if (b < 1_024 * 1_024) return `${(b / 1_024).toFixed(2)} KiB`; + return `${(b / (1_024 * 1_024)).toFixed(2)} MiB`; +} + +function formatRatio(value: number, baseline: number): string { + if (baseline === 0) return value === 0 ? '1.00x' : '∞'; + return `${(value / baseline).toFixed(2)}x`; +} + +const SPARK_WIDTH = 90; +const SPARK_HEIGHT = 16; +const SPARK_PADDING = 2; + +type SparklineProps = { + values: number[] | undefined; + className: string; +}; + +function Sparkline({values, className}: SparklineProps): ReactElement | null { + if (!values || values.length < 2) return null; + + const min = Math.min(...values); + const max = Math.max(...values); + const range = max - min; + const innerHeight = SPARK_HEIGHT - SPARK_PADDING * 2; + + const points = values.map((v, i) => { + const x = (i / (values.length - 1)) * SPARK_WIDTH; + const y = range === 0 + ? SPARK_HEIGHT / 2 + : SPARK_PADDING + innerHeight - ((v - min) / range) * innerHeight; + return [x, y] as const; + }); + + const d = points + .map(([x, y], i) => `${i === 0 ? 'M' : 'L'}${x.toFixed(2)} ${y.toFixed(2)}`) + .join(' '); + const [lastX, lastY] = points[points.length - 1]; + + return ( + + ); +} + +type LibraryRow = { + library: string; + value: number; + history?: number[]; +}; + +type MetricBlockProps = { + label: string; + rows: LibraryRow[]; + baselineValue: number; + format: (n: number) => string; +}; + +function MetricBlock({label, rows, baselineValue, format}: MetricBlockProps): ReactElement { + const max = Math.max(...rows.map(r => r.value), 0); + + return ( +
+
{label}
+
+ {rows.map(row => { + const isBaseline = row.library === BASELINE_LIBRARY; + const width = max === 0 ? 0 : (row.value / max) * 100; + return ( +
+ + {LIBRARY_DISPLAY_NAMES[row.library] ?? row.library} + + + + + {format(row.value)} + + {isBaseline ? 'baseline' : formatRatio(row.value, baselineValue)} + + +
+ ); + })} +
+
+ ); +} + +function buildRows( + samples: Record, + picker: (s: Sample) => number | undefined, + history: (s: Sample) => number[] | undefined, +): LibraryRow[] { + // Build all available rows for libraries whose dataset includes the metric, then + // sort by metric value ascending — fastest-first / smallest-allocation-first. + // Libraries missing this scenario or this metric are skipped silently. + const rows: LibraryRow[] = []; + for (const library of LIBRARY_ORDER) { + const sample = samples[library]; + if (!sample) continue; + const value = picker(sample); + if (value === undefined) continue; + rows.push({library, value, history: history(sample)}); + } + rows.sort((a, b) => a.value - b.value); + return rows; +} + +type Props = { + name: string; +}; + +export default function MockolateBenchmarkResult({name}: Props): ReactElement { + const scenario = data.benchmarks[name]; + if (!scenario) { + return ( +
+ No benchmark data for {name}. +
+ ); + } + + const params = scenario.parameters; + const [activeParam, setActiveParam] = useState(params[0]); + const samples = scenario.samples[activeParam] ?? {}; + const baselineSample = samples[BASELINE_LIBRARY]; + + const timeRows = buildRows(samples, s => s.timeNs, s => s.history?.timeNs); + const memoryRows = buildRows(samples, + s => s.memoryBytes, + s => s.history?.memoryBytes); + + const showParamTabs = params.length > 1 || (params.length === 1 && params[0] !== ''); + + return ( +
+ {showParamTabs && ( +
+ {params.map(p => ( + + ))} +
+ )} + + {memoryRows.length > 0 && baselineSample?.memoryBytes !== undefined && ( + + )} +
+ Captured on commit{' '} + + {data.capturedAt.shortSha} + {' '} + ({data.capturedAt.date}) + {data.environment?.runner ? ` on ${data.environment.runner}` : ''}. +
+
+ ); +} diff --git a/Docs/pages/src/components/MockolateBenchmarkResult/styles.module.css b/Docs/pages/src/components/MockolateBenchmarkResult/styles.module.css new file mode 100644 index 00000000..876869f9 --- /dev/null +++ b/Docs/pages/src/components/MockolateBenchmarkResult/styles.module.css @@ -0,0 +1,197 @@ +.result { + margin: 1rem 0 1.5rem; + border: 1px solid var(--ifm-color-emphasis-200); + border-radius: 6px; + padding: 1rem 1.25rem 0.75rem; + background: var(--ifm-card-background-color, var(--ifm-background-surface-color)); +} + +.paramTabs { + display: flex; + gap: 0.25rem; + margin: 0 0 0.75rem; + padding: 0.2rem; + background: var(--ifm-color-emphasis-100); + border-radius: 6px; + width: fit-content; +} + +.paramTab { + padding: 0.25rem 0.75rem; + border: 0; + background: transparent; + border-radius: 4px; + color: var(--ifm-color-emphasis-700); + font-size: 0.85rem; + font-family: var(--ifm-font-family-monospace); + cursor: pointer; + transition: background 0.1s, color 0.1s; +} + +.paramTab:hover { + background: var(--ifm-color-emphasis-200); +} + +.paramTabActive, +.paramTabActive:hover { + background: var(--ifm-card-background-color, var(--ifm-background-surface-color)); + color: var(--ifm-color-emphasis-900); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.08); +} + +.metricBlock { + margin: 0.75rem 0; +} + +.metricLabel { + font-weight: 600; + color: var(--ifm-color-emphasis-700); + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 0.04em; + margin-bottom: 0.4rem; +} + +.bars { + display: flex; + flex-direction: column; + gap: 0.2rem; +} + +.barRow { + /* Columns: library label, bar, value, ratio, sparkline. The ratio sits between + the value and the sparkline so the visual flow is "what" → "how much" → + "vs Mockolate" → "trend." */ + display: grid; + grid-template-columns: 9rem 1fr 5.5rem 4rem 6rem; + align-items: center; + column-gap: 0.5rem; + font-variant-numeric: tabular-nums; + font-size: 0.9rem; + padding: 0.15rem 0.4rem; + border-radius: 4px; +} + +.barRowBaseline { + background: var(--ifm-color-primary-contrast-background, rgba(99, 162, 172, 0.12)); + font-weight: 600; +} + +.libraryLabel { + color: var(--ifm-color-emphasis-800); + font-size: 0.85rem; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.bar { + position: relative; + height: 0.6rem; + background: var(--ifm-color-emphasis-100); + border-radius: 3px; + overflow: hidden; +} + +.barFill { + display: block; + height: 100%; + background: var(--ifm-color-emphasis-500); + border-radius: 3px; +} + +.barFillBaseline { + display: block; + height: 100%; + background: var(--ifm-color-primary); + border-radius: 3px; +} + +.barValue { + text-align: right; + color: var(--ifm-color-emphasis-800); +} + +.ratio { + text-align: right; + font-size: 0.85rem; + color: var(--ifm-color-emphasis-600); +} + +.barRowBaseline .ratio { + color: var(--ifm-color-emphasis-700); + font-style: italic; + font-weight: 400; +} + +.sparkline { + display: block; + justify-self: end; + color: var(--ifm-color-emphasis-500); +} + +.sparklineBaseline { + color: var(--ifm-color-primary); +} + +.caption { + margin-top: 0.75rem; + padding-top: 0.5rem; + border-top: 1px dashed var(--ifm-color-emphasis-200); + color: var(--ifm-color-emphasis-600); + font-size: 0.8rem; +} + +.caption code { + font-size: 0.78rem; +} + +.error { + margin: 1rem 0; + padding: 0.75rem 1rem; + border: 1px solid var(--ifm-color-danger); + border-radius: 6px; + color: var(--ifm-color-danger); + background: var(--ifm-color-danger-contrast-background); + font-size: 0.9rem; +} + +/* Anything narrower than wide-desktop: drop the sparkline so the value + and ratio columns keep their room. Threshold accounts for Docusaurus' + left sidebar (~250 px) and right TOC (~200 px) eating into the content + area. */ +@media (max-width: 1200px) { + .barRow { + grid-template-columns: 9rem 1fr 5.5rem 4rem; + } + .sparkline { + display: none; + } +} + +/* Phone: stack label / bar / value+ratio so values stay readable. */ +@media (max-width: 600px) { + .barRow { + grid-template-columns: 1fr 4.5rem 3.5rem; + column-gap: 0.4rem; + row-gap: 0.2rem; + } + .bar { + grid-column: 1 / -1; + order: 2; + } + .libraryLabel { + grid-column: 1 / -1; + order: 1; + } + .barValue { + grid-column: 1; + order: 3; + text-align: left; + } + .ratio { + grid-column: 2 / -1; + order: 4; + text-align: right; + } +} diff --git a/Docs/pages/src/data/mockolate/benchmarks.json b/Docs/pages/src/data/mockolate/benchmarks.json new file mode 100644 index 00000000..090399c6 --- /dev/null +++ b/Docs/pages/src/data/mockolate/benchmarks.json @@ -0,0 +1,2157 @@ +{ + "capturedAt": { + "sha": "4c61cdf53af2939394a923cb4639800a691a1598", + "shortSha": "4c61cdf5", + "date": "2026-05-03", + "message": "feat: collect benchmark data into \u0027benchmarks\u0027 branch (#762)" + }, + "environment": { + "runner": "ubuntu-latest", + "toolchain": "BenchmarkDotNet MediumRun (in-process)" + }, + "benchmarks": { + "Callback": { + "parameters": [ + "" + ], + "samples": { + "": { + "Mockolate": { + "timeNs": 356.5, + "memoryBytes": 1720, + "history": { + "timeNs": [ + 392.6, + 397.4, + 364.9, + 319, + 242.9, + 319, + 377.7, + 324.8, + 308.9, + 351.4, + 311.1, + 334.7, + 390.9, + 335.1, + 356.5 + ], + "memoryBytes": [ + 1832, + 1832, + 1832, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720, + 1720 + ] + } + }, + "Moq": { + "timeNs": 100665.3, + "memoryBytes": 9096, + "history": { + "timeNs": [ + 95985.2, + 70481.1, + 72225.4, + 97235.5, + 56782.5, + 98309.2, + 72812.7, + 71666, + 70116.8, + 98867.7, + 69679.7, + 97096.4, + 71915.6, + 97602.8, + 100665.3 + ], + "memoryBytes": [ + 9096, + 9096, + 9096, + 9096, + 9095, + 9096, + 9090, + 9096, + 9096, + 9096, + 9096, + 9096, + 9090, + 9096, + 9096 + ] + } + }, + "NSubstitute": { + "timeNs": 4400.6, + "memoryBytes": 7928, + "history": { + "timeNs": [ + 4435.9, + 4396.5, + 4180.6, + 4412, + 3446.5, + 4459, + 4236, + 4288, + 4139.8, + 4611.7, + 4001.7, + 4373.7, + 4357.9, + 4406.5, + 4400.6 + ], + "memoryBytes": [ + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928, + 7928 + ] + } + }, + "FakeItEasy": { + "timeNs": 4808.8, + "memoryBytes": 6970, + "history": { + "timeNs": [ + 4754.9, + 4672.8, + 4383.4, + 4884.8, + 3764.1, + 4837.1, + 4524.1, + 4572.8, + 4153.6, + 4896.9, + 4142.1, + 4716.1, + 4429.6, + 4998.8, + 4808.8 + ], + "memoryBytes": [ + 6970, + 6970, + 6970, + 6970, + 6970, + 6970, + 6959, + 6970, + 6970, + 6970, + 6970, + 6970, + 6959, + 6970, + 6970 + ] + } + }, + "TUnitMocks": { + "timeNs": 632.8, + "memoryBytes": 2688, + "history": { + "timeNs": [ + 602.7, + 736.9, + 654.1, + 589.8, + 538.4, + 703, + 706, + 655.1, + 604.5, + 619.9, + 624.8, + 621.7, + 749.7, + 625.5, + 632.8 + ], + "memoryBytes": [ + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688, + 2688 + ] + } + }, + "Imposter": { + "timeNs": 439.1, + "memoryBytes": 2440, + "history": { + "timeNs": [ + 419.8, + 460, + 409.6, + 406.9, + 321.5, + 384.7, + 450.9, + 418.5, + 406.2, + 442.2, + 406.7, + 434.1, + 487.5, + 430.4, + 439.1 + ], + "memoryBytes": [ + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440, + 2440 + ] + } + } + } + } + }, + "CreateMock": { + "parameters": [ + "" + ], + "samples": { + "": { + "Mockolate": { + "timeNs": 63.9, + "memoryBytes": 440, + "history": { + "timeNs": [ + 206.4, + 206.9, + 197.5, + 185.6, + 214.2, + 66.7, + 61.7, + 66.9, + 66.2, + 69.5, + 65.9, + 65.8, + 55.6, + 64.6, + 63.9 + ], + "memoryBytes": [ + 1048, + 1048, + 1048, + 1048, + 1048, + 440, + 440, + 440, + 440, + 440, + 440, + 440, + 440, + 440, + 440 + ] + } + }, + "Moq": { + "timeNs": 1428.8, + "memoryBytes": 2096, + "history": { + "timeNs": [ + 1486.6, + 1458.4, + 1394.3, + 1385.4, + 1459.2, + 1512.1, + 1460.6, + 1412.3, + 1315.9, + 1346.9, + 1365.5, + 1353.1, + 1125, + 1375.9, + 1428.8 + ], + "memoryBytes": [ + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096, + 2096 + ] + } + }, + "NSubstitute": { + "timeNs": 2075.5, + "memoryBytes": 5048, + "history": { + "timeNs": [ + 2113.6, + 2107.6, + 2031.9, + 1957.9, + 2111.4, + 1973.7, + 1948.9, + 2016.3, + 1890.9, + 1945.1, + 1846.3, + 1816.9, + 1635.3, + 1963.2, + 2075.5 + ], + "memoryBytes": [ + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048, + 5048 + ] + } + }, + "FakeItEasy": { + "timeNs": 1871.3, + "memoryBytes": 2763, + "history": { + "timeNs": [ + 1961.2, + 1838, + 1780.5, + 1739.6, + 1850.7, + 1779.2, + 1926.7, + 1833.6, + 1759.6, + 1735.5, + 1786.1, + 1726.4, + 1712.5, + 1727.9, + 1871.3 + ], + "memoryBytes": [ + 2763, + 2763, + 2763, + 2763, + 2763, + 2763, + 2763, + 2763, + 2772, + 2772, + 2763, + 2763, + 2763, + 2763, + 2763 + ] + } + }, + "TUnitMocks": { + "timeNs": 46.7, + "memoryBytes": 224, + "history": { + "timeNs": [ + 46.3, + 45.1, + 41, + 38.9, + 42.6, + 39.9, + 38.8, + 42.6, + 45.1, + 44.1, + 43.1, + 42, + 35, + 41.5, + 46.7 + ], + "memoryBytes": [ + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224, + 224 + ] + } + }, + "Imposter": { + "timeNs": 286.1, + "memoryBytes": 2248, + "history": { + "timeNs": [ + 325, + 291.8, + 284.3, + 265.2, + 283.3, + 293.8, + 272.5, + 302.8, + 309.7, + 312, + 284, + 272.8, + 259.8, + 282.6, + 286.1 + ], + "memoryBytes": [ + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248, + 2248 + ] + } + } + } + } + }, + "Event": { + "parameters": [ + "" + ], + "samples": { + "": { + "Mockolate": { + "timeNs": 307.2, + "memoryBytes": 1824, + "history": { + "timeNs": [ + 311.5, + 324.8, + 344.7, + 309.8, + 312.1, + 332.5, + 299, + 290.8, + 329.3, + 310.2, + 245.3, + 309.2, + 302.4, + 293.4, + 307.2 + ], + "memoryBytes": [ + 1872, + 1872, + 1872, + 1952, + 1952, + 1824, + 1824, + 1824, + 1824, + 1824, + 1824, + 1824, + 1824, + 1824, + 1824 + ] + } + }, + "Moq": { + "timeNs": 13914.1, + "memoryBytes": 12809, + "history": { + "timeNs": [ + 16604.9, + 15888.8, + 16421.7, + 15713.6, + 15935.9, + 16129.1, + 16067.3, + 16094.2, + 16278.9, + 14069.9, + 11181.4, + 15998.8, + 13854.3, + 16389.6, + 13914.1 + ], + "memoryBytes": [ + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809, + 12809 + ] + } + }, + "NSubstitute": { + "timeNs": 5249.9, + "memoryBytes": 9264, + "history": { + "timeNs": [ + 5855.1, + 5768.6, + 5853, + 5511.9, + 5582.9, + 5741.2, + 5514.8, + 5692.6, + 5889.4, + 5494.3, + 4060.9, + 5762.5, + 4970.8, + 5524.3, + 5249.9 + ], + "memoryBytes": [ + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264, + 9264 + ] + } + }, + "FakeItEasy": { + "timeNs": 228773.7, + "memoryBytes": 15628, + "history": { + "timeNs": [ + 216644.6, + 213260, + 213631.5, + 212169.7, + 211018.7, + 214018.2, + 210787.4, + 212882.1, + 219609.8, + 231348.6, + 183999.3, + 214921.3, + 230975, + 213204.5, + 228773.7 + ], + "memoryBytes": [ + 15907, + 15755, + 15755, + 15755, + 15755, + 15755, + 15628, + 15628, + 15628, + 15628, + 15628, + 15747, + 15628, + 15628, + 15628 + ] + } + }, + "TUnitMocks": { + "timeNs": 174.3, + "memoryBytes": 1400, + "history": { + "timeNs": [ + 216.2, + 190.7, + 190.5, + 180.8, + 174.1, + 203.5, + 184.4, + 183.9, + 209.1, + 183.2, + 139.5, + 194.4, + 174.7, + 189.7, + 174.3 + ], + "memoryBytes": [ + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400, + 1400 + ] + } + }, + "Imposter": { + "timeNs": 1317.1, + "memoryBytes": 9016, + "history": { + "timeNs": [ + 1648.3, + 1457.5, + 1360.1, + 1422.7, + 1330.8, + 1452.7, + 1310.9, + 1336.6, + 1469.6, + 1389.7, + 1066.9, + 1373, + 1339.8, + 1311.4, + 1317.1 + ], + "memoryBytes": [ + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016, + 9016 + ] + } + } + } + } + }, + "Indexer": { + "parameters": [ + "N=1", + "N=10" + ], + "samples": { + "N=1": { + "Mockolate": { + "timeNs": 905.4, + "memoryBytes": 3904, + "history": { + "timeNs": [ + 842.5, + 1069.6, + 903.1, + 880.4, + 933, + 1027.6, + 1012, + 1034.5, + 904.5, + 935.5, + 825.6, + 916.3, + 840.2, + 1066.4, + 905.4 + ], + "memoryBytes": [ + 3992, + 3992, + 3992, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904, + 3904 + ] + } + }, + "Moq": { + "timeNs": 216630.3, + "memoryBytes": 20860, + "history": { + "timeNs": [ + 134996, + 167449.6, + 219668.9, + 213131.4, + 214441.3, + 171488.1, + 217577.6, + 170082.2, + 215500.6, + 165984.5, + 134234.4, + 173250.3, + 131275.6, + 173923.3, + 216630.3 + ], + "memoryBytes": [ + 20732, + 20849, + 20860, + 20980, + 20860, + 20860, + 20732, + 20972, + 20860, + 20860, + 20875, + 21084, + 20860, + 20732, + 20860 + ] + } + }, + "NSubstitute": { + "timeNs": 9404.2, + "memoryBytes": 13144, + "history": { + "timeNs": [ + 6882, + 9481, + 9708.4, + 9254.7, + 9537.9, + 8917.5, + 9284.9, + 8541.7, + 9127.1, + 8324.3, + 6885.6, + 9167.1, + 6818.3, + 9079.2, + 9404.2 + ], + "memoryBytes": [ + 13088, + 13088, + 13088, + 13144, + 13088, + 13144, + 13088, + 13088, + 13088, + 13088, + 13144, + 13144, + 13144, + 13144, + 13144 + ] + } + }, + "FakeItEasy": { + "timeNs": 11720.4, + "memoryBytes": 13954, + "history": { + "timeNs": [ + 7956.2, + 11003.9, + 12260.5, + 11199.2, + 12207.3, + 10249.9, + 11884.9, + 10420.5, + 11822.1, + 9394.6, + 8015.3, + 10395.5, + 7837.5, + 10637.5, + 11720.4 + ], + "memoryBytes": [ + 14211, + 14199, + 14211, + 14211, + 14211, + 14211, + 13954, + 14291, + 13954, + 13954, + 13954, + 13954, + 13954, + 14067, + 13954 + ] + } + }, + "Imposter": { + "timeNs": 870.4, + "memoryBytes": 5280, + "history": { + "timeNs": [ + 771.9, + 991.8, + 947.2, + 897.4, + 982.2, + 949.9, + 946.4, + 855.2, + 842.1, + 823.1, + 675.7, + 849.1, + 691.3, + 939.9, + 870.4 + ], + "memoryBytes": [ + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280, + 5280 + ] + } + } + }, + "N=10": { + "Mockolate": { + "timeNs": 2486.7, + "memoryBytes": 4984, + "history": { + "timeNs": [ + 1958.2, + 2968.4, + 2434.5, + 2403.6, + 2494.5, + 2587.4, + 2706.6, + 2542.3, + 2381.7, + 2381.2, + 2044.9, + 2561.9, + 2081.1, + 2653, + 2486.7 + ], + "memoryBytes": [ + 5072, + 5072, + 5072, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984, + 4984 + ] + } + }, + "Moq": { + "timeNs": 227405.9, + "memoryBytes": 30610, + "history": { + "timeNs": [ + 143025.9, + 178483, + 227934.9, + 227241.4, + 227593.4, + 185177.3, + 229589.1, + 184168, + 225253.3, + 177573.6, + 142187.1, + 185280.2, + 140860.2, + 187425.8, + 227405.9 + ], + "memoryBytes": [ + 29332, + 30603, + 30610, + 30730, + 30610, + 30610, + 29332, + 30722, + 30610, + 30610, + 30610, + 30834, + 30610, + 29332, + 30610 + ] + } + }, + "NSubstitute": { + "timeNs": 22656.7, + "memoryBytes": 26249, + "history": { + "timeNs": [ + 17053, + 22527.5, + 22849.1, + 22323, + 23604.3, + 22861, + 22771.3, + 22713, + 22022.7, + 20563.8, + 17123.5, + 22663.7, + 16926, + 22719.5, + 22656.7 + ], + "memoryBytes": [ + 26193, + 26193, + 26193, + 26249, + 26193, + 26249, + 26193, + 26193, + 26193, + 26193, + 26249, + 26249, + 26249, + 26249, + 26249 + ] + } + }, + "FakeItEasy": { + "timeNs": 24249.8, + "memoryBytes": 33764, + "history": { + "timeNs": [ + 17197.6, + 23890.9, + 25817.7, + 24160, + 25188.3, + 22661.5, + 24895.6, + 21297.3, + 24328.5, + 20522.4, + 17004.6, + 21709.1, + 16719, + 22547.9, + 24249.8 + ], + "memoryBytes": [ + 36325, + 36333, + 36325, + 36325, + 36325, + 36325, + 33764, + 34101, + 33764, + 33764, + 33764, + 33764, + 33764, + 33877, + 33764 + ] + } + }, + "Imposter": { + "timeNs": 2091.7, + "memoryBytes": 8160, + "history": { + "timeNs": [ + 1745.6, + 2349.3, + 2164.9, + 2236, + 2245.4, + 2285.2, + 2146.8, + 2117.4, + 2031.6, + 1981.3, + 1664, + 2199.3, + 1701.2, + 2252.9, + 2091.7 + ], + "memoryBytes": [ + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160, + 8160 + ] + } + } + } + } + }, + "Method": { + "parameters": [ + "N=1", + "N=10" + ], + "samples": { + "N=1": { + "Mockolate": { + "timeNs": 367.7, + "memoryBytes": 2088, + "history": { + "timeNs": [ + 427.8, + 419.6, + 456.4, + 360.8, + 382.8, + 352.9, + 403.5, + 395, + 412.3, + 407.2, + 398, + 358.9, + 345.1, + 398.3, + 367.7 + ], + "memoryBytes": [ + 2200, + 2200, + 2200, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088, + 2088 + ] + } + }, + "Moq": { + "timeNs": 138328.8, + "memoryBytes": 15098, + "history": { + "timeNs": [ + 131590.9, + 130710.2, + 134776.7, + 130068.4, + 185743.7, + 180607.3, + 134752.7, + 187164.5, + 184460.3, + 182793.4, + 140440.2, + 131557.4, + 178977, + 134101.3, + 138328.8 + ], + "memoryBytes": [ + 14926, + 14926, + 14940, + 14926, + 14926, + 14926, + 15086, + 15086, + 14926, + 14926, + 15162, + 15086, + 14926, + 15098, + 15098 + ] + } + }, + "NSubstitute": { + "timeNs": 5764.3, + "memoryBytes": 9336, + "history": { + "timeNs": [ + 5350.8, + 5290.4, + 5637.1, + 5358.2, + 5897.2, + 5594.8, + 5511.4, + 5991.8, + 5955.8, + 5784.7, + 5669.7, + 5209.7, + 5609.6, + 5447.3, + 5764.3 + ], + "memoryBytes": [ + 9336, + 9336, + 9280, + 9336, + 9280, + 9336, + 9336, + 9336, + 9336, + 9336, + 9336, + 9336, + 9336, + 9336, + 9336 + ] + } + }, + "FakeItEasy": { + "timeNs": 5898.3, + "memoryBytes": 8251, + "history": { + "timeNs": [ + 5467.7, + 5347.6, + 5924.9, + 5663.3, + 6249.6, + 5975.1, + 5538.6, + 5979.5, + 6506.6, + 6088.5, + 5815.6, + 5276.2, + 6016.2, + 5414.5, + 5898.3 + ], + "memoryBytes": [ + 8309, + 8309, + 8316, + 8309, + 8309, + 8309, + 8244, + 8244, + 8244, + 8244, + 8251, + 8245, + 8244, + 8244, + 8251 + ] + } + }, + "TUnitMocks": { + "timeNs": 705.6, + "memoryBytes": 2968, + "history": { + "timeNs": [ + 694, + 661.6, + 738.1, + 742.9, + 712.1, + 664.3, + 679.5, + 704.8, + 728.2, + 689.4, + 761, + 699.2, + 630.1, + 742, + 705.6 + ], + "memoryBytes": [ + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968, + 2968 + ] + } + }, + "Imposter": { + "timeNs": 551.4, + "memoryBytes": 4136, + "history": { + "timeNs": [ + 547.2, + 541.8, + 652.1, + 594, + 639.8, + 565.3, + 532.9, + 609.6, + 659.2, + 600.6, + 638.1, + 533.6, + 544.3, + 560.5, + 551.4 + ], + "memoryBytes": [ + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136, + 4136 + ] + } + } + }, + "N=10": { + "Mockolate": { + "timeNs": 781.6, + "memoryBytes": 2304, + "history": { + "timeNs": [ + 710.7, + 684.8, + 898.1, + 645.4, + 737.7, + 651.8, + 660.5, + 668.7, + 725.7, + 722, + 846.8, + 641.5, + 645.6, + 651.4, + 781.6 + ], + "memoryBytes": [ + 2416, + 2416, + 2416, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304, + 2304 + ] + } + }, + "Moq": { + "timeNs": 143889.6, + "memoryBytes": 19089, + "history": { + "timeNs": [ + 134497.4, + 136995.3, + 141282.9, + 136760.2, + 193886, + 185087.5, + 141501.8, + 188731.7, + 191732.8, + 187149, + 146625.4, + 136998.7, + 184408.2, + 139343.6, + 143889.6 + ], + "memoryBytes": [ + 18925, + 18925, + 18929, + 18924, + 18925, + 18925, + 19085, + 19085, + 18925, + 18925, + 19153, + 19085, + 18925, + 19085, + 19089 + ] + } + }, + "NSubstitute": { + "timeNs": 8732, + "memoryBytes": 12360, + "history": { + "timeNs": [ + 7932.6, + 7836.2, + 8055, + 8306.8, + 8850.3, + 8468.9, + 8033.1, + 8714.2, + 8965.3, + 8736.5, + 8446.9, + 7846.1, + 8392.5, + 7980.1, + 8732 + ], + "memoryBytes": [ + 12361, + 12361, + 11800, + 12361, + 11800, + 12361, + 12361, + 12361, + 12361, + 12361, + 12360, + 12361, + 12361, + 12361, + 12360 + ] + } + }, + "FakeItEasy": { + "timeNs": 9302.3, + "memoryBytes": 15786, + "history": { + "timeNs": [ + 8488.3, + 8835.9, + 9586.2, + 8599.6, + 10585.5, + 9259.1, + 8716.4, + 9273.9, + 9998, + 9480.1, + 9431.4, + 8293.7, + 9332.3, + 8468.1, + 9302.3 + ], + "memoryBytes": [ + 16433, + 16433, + 16428, + 16433, + 16433, + 16433, + 15786, + 15786, + 15786, + 15786, + 15786, + 15786, + 15786, + 15786, + 15786 + ] + } + }, + "TUnitMocks": { + "timeNs": 2034.6, + "memoryBytes": 4600, + "history": { + "timeNs": [ + 1829.3, + 1748.6, + 2131.4, + 1789.9, + 1817.5, + 1683.8, + 1857.5, + 1678.6, + 1777.5, + 1735.9, + 2164.8, + 1778.7, + 1633.7, + 1780.4, + 2034.6 + ], + "memoryBytes": [ + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600, + 4600 + ] + } + }, + "Imposter": { + "timeNs": 1122.3, + "memoryBytes": 5648, + "history": { + "timeNs": [ + 1128.1, + 1146.6, + 1276.4, + 1180, + 1428.2, + 1217.1, + 1120.5, + 1126.6, + 1339.9, + 1161.2, + 1238.2, + 1076.5, + 1093.1, + 1090.9, + 1122.3 + ], + "memoryBytes": [ + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648, + 5648 + ] + } + } + } + } + }, + "Property": { + "parameters": [ + "N=1", + "N=10" + ], + "samples": { + "N=1": { + "Mockolate": { + "timeNs": 411.9, + "memoryBytes": 2520, + "history": { + "timeNs": [ + 537.5, + 505, + 623.5, + 489.6, + 618, + 549.5, + 580.5, + 512.1, + 538.2, + 502.2, + 501.9, + 509, + 525.5, + 562.7, + 411.9 + ], + "memoryBytes": [ + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520, + 2520 + ] + } + }, + "Moq": { + "timeNs": 8111.8, + "memoryBytes": 10640, + "history": { + "timeNs": [ + 9891.6, + 12099.9, + 12556.8, + 11752.5, + 11152.1, + 10916.6, + 12253, + 11860.3, + 12136.2, + 12437.5, + 11738, + 12201.8, + 11858.6, + 11472.6, + 8111.8 + ], + "memoryBytes": [ + 10513, + 10514, + 10641, + 10641, + 10641, + 10641, + 10753, + 10641, + 10641, + 10641, + 10513, + 10721, + 10513, + 10721, + 10640 + ] + } + }, + "NSubstitute": { + "timeNs": 5485.5, + "memoryBytes": 11720, + "history": { + "timeNs": [ + 6959.6, + 7639, + 8137.8, + 7575.3, + 7582.2, + 7028.9, + 7461.1, + 7411.1, + 7769.3, + 7460.8, + 7209.8, + 7463.7, + 7208.5, + 7566.9, + 5485.5 + ], + "memoryBytes": [ + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720, + 11720 + ] + } + }, + "FakeItEasy": { + "timeNs": 5638.9, + "memoryBytes": 11508, + "history": { + "timeNs": [ + 7076, + 8308.3, + 8980.5, + 8435.1, + 7862.4, + 8186.4, + 8793.7, + 8179.5, + 8445.1, + 8430.1, + 7994.9, + 8502.4, + 8441.8, + 8479.3, + 5638.9 + ], + "memoryBytes": [ + 11508, + 11508, + 11508, + 11508, + 11518, + 11518, + 11508, + 11508, + 11508, + 11508, + 11508, + 11508, + 11508, + 11758, + 11508 + ] + } + }, + "TUnitMocks": { + "timeNs": 565.1, + "memoryBytes": 2568, + "history": { + "timeNs": [ + 726.2, + 713.4, + 884.3, + 772.1, + 847, + 826.8, + 748.1, + 724.9, + 751.8, + 709.5, + 697.6, + 723.1, + 694.6, + 749.8, + 565.1 + ], + "memoryBytes": [ + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568, + 2568 + ] + } + }, + "Imposter": { + "timeNs": 342.4, + "memoryBytes": 3200, + "history": { + "timeNs": [ + 442.4, + 481.7, + 558.3, + 461.5, + 539.1, + 463.3, + 477.9, + 438, + 470.9, + 437.4, + 485.5, + 469.8, + 474.9, + 457.5, + 342.4 + ], + "memoryBytes": [ + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200, + 3200 + ] + } + } + }, + "N=10": { + "Mockolate": { + "timeNs": 796, + "memoryBytes": 3024, + "history": { + "timeNs": [ + 1030.7, + 1012.7, + 1129, + 1063.3, + 1275.5, + 1243, + 1162.4, + 1016, + 1109.8, + 1031.3, + 1041.6, + 1054.5, + 1022.2, + 1266.8, + 796 + ], + "memoryBytes": [ + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024, + 3024 + ] + } + }, + "Moq": { + "timeNs": 13318.2, + "memoryBytes": 18720, + "history": { + "timeNs": [ + 15851, + 18691.2, + 19641, + 18716, + 17826.1, + 17536.8, + 19197.2, + 19004.9, + 19559.7, + 19420.8, + 18476.5, + 18905.9, + 18810.1, + 18073.8, + 13318.2 + ], + "memoryBytes": [ + 17441, + 17441, + 18721, + 18721, + 18721, + 18721, + 18833, + 18721, + 18721, + 18721, + 17441, + 18801, + 17441, + 18801, + 18720 + ] + } + }, + "NSubstitute": { + "timeNs": 12957.8, + "memoryBytes": 21585, + "history": { + "timeNs": [ + 16176.9, + 17353.1, + 18737, + 17340.2, + 16760.4, + 16452.4, + 17037.9, + 17109.9, + 17597.3, + 17078.6, + 17004.3, + 16667.7, + 16657.4, + 17377, + 12957.8 + ], + "memoryBytes": [ + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585, + 21585 + ] + } + }, + "FakeItEasy": { + "timeNs": 13823.1, + "memoryBytes": 31546, + "history": { + "timeNs": [ + 16887.9, + 19718.6, + 21012.2, + 20603.9, + 18810.3, + 18316.5, + 20139.4, + 19596.3, + 20680.9, + 20650.8, + 19685.5, + 20133, + 20151.9, + 19220.6, + 13823.1 + ], + "memoryBytes": [ + 31546, + 31546, + 31546, + 31546, + 31543, + 31543, + 31546, + 31546, + 31546, + 31546, + 31546, + 31546, + 31546, + 31783, + 31546 + ] + } + }, + "TUnitMocks": { + "timeNs": 1693.2, + "memoryBytes": 4776, + "history": { + "timeNs": [ + 2232.2, + 2089.5, + 2409.8, + 2167.2, + 2749, + 2653.1, + 2234.4, + 2130.5, + 2206.7, + 2196.3, + 2160.7, + 2148.6, + 2269.8, + 2681, + 1693.2 + ], + "memoryBytes": [ + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776, + 4776 + ] + } + }, + "Imposter": { + "timeNs": 830.6, + "memoryBytes": 4784, + "history": { + "timeNs": [ + 1093.7, + 1133.7, + 1298.9, + 1096.1, + 1410.6, + 1243.1, + 1204.4, + 1143.2, + 1133.3, + 1079, + 1123.3, + 1154.9, + 1155.2, + 1283.3, + 830.6 + ], + "memoryBytes": [ + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784, + 4784 + ] + } + } + } + } + } + } +} diff --git a/Pipeline/Build.MockolateBenchmarks.cs b/Pipeline/Build.MockolateBenchmarks.cs new file mode 100644 index 00000000..cc848d4a --- /dev/null +++ b/Pipeline/Build.MockolateBenchmarks.cs @@ -0,0 +1,193 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Serialization; +using Nuke.Common; +using Nuke.Common.IO; +using Serilog; + +// ReSharper disable AllUnderscoreLocalParameterName + +namespace Build; + +partial class Build +{ + /// + /// Fetches the latest Mockolate benchmark snapshot from the long-lived + /// benchmarks branch of Testably/Mockolate and reduces it + /// to the shape consumed by the MockolateBenchmarkResult React + /// component. Mirrors the target but for the + /// 6-library Mockolate dataset (Mockolate, Moq, NSubstitute, FakeItEasy, + /// TUnitMocks, Imposter) and groups [Params] variants under the + /// same scenario key. + /// + Target MockolateBenchmarks => _ => _ + .Executes(async () => + { + AbsolutePath outputDirectory = RootDirectory / "Docs" / "pages" / "src" / "data" / "mockolate"; + outputDirectory.CreateDirectory(); + AbsolutePath outputPath = outputDirectory / "benchmarks.json"; + + Log.Information( + "Fetching benchmark snapshot from {Repo} branch {Branch}", + MockolateBenchmarksRepo, BenchmarksBranch); + + using HttpClient client = new(); + client.DefaultRequestHeaders.UserAgent.ParseAdd("Testably.Site"); + if (!string.IsNullOrEmpty(GithubToken)) + { + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", GithubToken); + } + + string raw = await client.GetStringAsync( + $"https://raw.githubusercontent.com/{MockolateBenchmarksRepo}/refs/heads/{BenchmarksBranch}/Docs/pages/static/js/{BenchmarksFile}"); + if (!raw.StartsWith(BenchmarksJsPrefix, StringComparison.Ordinal)) + { + throw new NotSupportedException( + $"{BenchmarksFile} does not start with '{BenchmarksJsPrefix}' — upstream format changed?"); + } + string json = raw.Substring(BenchmarksJsPrefix.Length).TrimEnd(';', '\r', '\n', ' ', '\t'); + + Dictionary rawData = + JsonSerializer.Deserialize>(json, RawJsonOptions) + ?? throw new NotSupportedException($"Could not deserialize {BenchmarksFile}"); + + MockolateSnapshot snapshot = ReduceToMockolateSnapshot(rawData, BenchmarkHistoryLength); + + string pretty = JsonSerializer.Serialize(snapshot, OutputJsonOptions); + await File.WriteAllTextAsync(outputPath, pretty + Environment.NewLine); + + Log.Information( + "Wrote {ScenarioCount} scenarios to {Path} (captured commit {Sha} on {Date})", + snapshot.Benchmarks.Count, outputPath, snapshot.CapturedAt.ShortSha, snapshot.CapturedAt.Date); + }); + + const string MockolateBenchmarksRepo = "Testably/Mockolate"; + + static readonly string[] MockolateLibraries = + ["Mockolate", "Moq", "NSubstitute", "FakeItEasy", "TUnitMocks", "Imposter",]; + + static MockolateSnapshot ReduceToMockolateSnapshot(Dictionary rawData, int historyLength) + { + // All benchmark groups share the same commit list (the `BenchmarkReport` target in Mockolate + // appends one new entry per push to main, in lockstep across every chart). Pick the first + // non-empty list to source the "captured at" metadata. + RawCommit? latestCommit = rawData.Values + .Where(b => b.Commits is {Count: > 0,}) + .Select(b => b.Commits![^1]) + .FirstOrDefault(); + if (latestCommit is null) + { + throw new NotSupportedException("Benchmark data has no commits"); + } + + // Group chart keys (e.g. "Method (N=1)", "Method (N=10)", "Callback") by scenario name. + // Scenarios without parameters keep an empty parameter key. + Dictionary scenarios = new(); + foreach ((string chartKey, RawBenchmark benchmark) in rawData.OrderBy(kvp => kvp.Key, StringComparer.Ordinal)) + { + if (benchmark.Datasets is null) continue; + + (string scenario, string parameter) = SplitChartKey(chartKey); + Dictionary samples = new(); + foreach (string library in MockolateLibraries) + { + MockolateLibrarySample? sample = BuildMockolateSample(benchmark, library, historyLength); + if (sample is not null) + { + samples[library] = sample; + } + } + + if (samples.Count == 0) continue; + + if (!scenarios.TryGetValue(scenario, out MockolateScenario? entry)) + { + entry = new MockolateScenario(); + scenarios[scenario] = entry; + } + + entry.Parameters.Add(parameter); + entry.Samples[parameter] = samples; + } + + string sha = latestCommit.Sha ?? string.Empty; + string shortSha = sha.Length >= 8 ? sha[..8] : sha; + string date = ParseShortDate(latestCommit.Date); + + return new MockolateSnapshot + { + CapturedAt = new SnapshotCommit + { + Sha = sha, + ShortSha = shortSha, + Date = date, + Message = latestCommit.Message, + }, + Environment = new SnapshotEnvironment + { + Runner = "ubuntu-latest", + Toolchain = "BenchmarkDotNet MediumRun (in-process)", + }, + Benchmarks = scenarios, + }; + } + + static (string Scenario, string Parameter) SplitChartKey(string chartKey) + { + int open = chartKey.IndexOf('('); + if (open <= 0 || !chartKey.EndsWith(')')) return (chartKey, string.Empty); + string scenario = chartKey[..open].TrimEnd(); + string parameter = chartKey.Substring(open + 1, chartKey.Length - open - 2).Trim(); + return (scenario, parameter); + } + + static MockolateLibrarySample? BuildMockolateSample(RawBenchmark benchmark, string library, int historyLength) + { + double[]? time = FindData(benchmark.Datasets!, library, "y"); + double[]? memory = FindData(benchmark.Datasets!, library, "y1"); + if (time is not {Length: > 0,}) return null; + + return new MockolateLibrarySample + { + TimeNs = Round1(time[^1]), + MemoryBytes = memory is {Length: > 0,} ? (long)Math.Round(memory[^1]) : null, + History = new SampleHistory + { + TimeNs = TakeLast(time, historyLength).Select(Round1).ToArray(), + MemoryBytes = memory is null + ? Array.Empty() + : TakeLast(memory, historyLength).Select(v => (long)Math.Round(v)).ToArray(), + }, + }; + } + + // ─── Snapshot shape consumed by MockolateBenchmarkResult.tsx ─── + sealed class MockolateSnapshot + { + [JsonPropertyName("capturedAt")] public SnapshotCommit CapturedAt { get; init; } = new(); + [JsonPropertyName("environment")] public SnapshotEnvironment? Environment { get; init; } + [JsonPropertyName("benchmarks")] public Dictionary Benchmarks { get; init; } = new(); + } + + sealed class MockolateScenario + { + [JsonPropertyName("parameters")] public List Parameters { get; init; } = new(); + + [JsonPropertyName("samples")] + public Dictionary> Samples { get; init; } = new(); + } + + sealed class MockolateLibrarySample + { + [JsonPropertyName("timeNs")] public double TimeNs { get; init; } + [JsonPropertyName("memoryBytes")] public long? MemoryBytes { get; init; } + [JsonPropertyName("history")] public SampleHistory? History { get; init; } + } +} diff --git a/Pipeline/Build.Pages.cs b/Pipeline/Build.Pages.cs index ffd724a7..34386548 100644 --- a/Pipeline/Build.Pages.cs +++ b/Pipeline/Build.Pages.cs @@ -79,6 +79,7 @@ record ReadmeSubstitution( Target Pages => _ => _ .DependsOn(Benchmarks) + .DependsOn(MockolateBenchmarks) .Executes(async () => { AbsolutePath docsRoot = RootDirectory / "Docs" / "pages" / "docs"; From 5adf444fe21d76c61b763e889c3b8920d9411e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 3 May 2026 15:35:38 +0200 Subject: [PATCH 2/2] fix: review feedback on mockolate benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pages.yml: trigger the Pages workflow when Build.MockolateBenchmarks.cs changes too, so reducer-only fixes redeploy the docs. - MockolateBenchmarkResult: drop role="tablist"/role="tab" — the param toggle is a button group, not a real tab pattern (no panel linkage, no roving tabIndex, no arrow-key handling). Use aria-pressed instead so screen readers announce it as a toggle. - MockolateBenchmarkResult: render libraries with no sample for the scenario as italicised "not available" rows rather than dropping them silently. The seeded snapshot has no TUnitMocks data for the Indexer benchmark and that absence was previously hidden. - docusaurus.config.ts: migrate onBrokenMarkdownLinks under markdown.hooks (the top-level option is deprecated for Docusaurus v4). --- .github/workflows/pages.yml | 1 + Docs/pages/docusaurus.config.ts | 6 +- .../MockolateBenchmarkResult/index.tsx | 73 +++++++++++-------- .../styles.module.css | 11 +++ 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index cd888972..7f601da7 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -7,6 +7,7 @@ on: - 'Docs/pages/**' - 'Pipeline/Build.Pages.cs' - 'Pipeline/Build.Benchmarks.cs' + - 'Pipeline/Build.MockolateBenchmarks.cs' - '.github/workflows/pages.yml' repository_dispatch: types: [ extension-documentation-updated-event ] diff --git a/Docs/pages/docusaurus.config.ts b/Docs/pages/docusaurus.config.ts index 4dedc9b9..a43efe9d 100644 --- a/Docs/pages/docusaurus.config.ts +++ b/Docs/pages/docusaurus.config.ts @@ -23,7 +23,11 @@ const config: Config = { // original single-library sites have not yet been fixed up. onBrokenLinks: 'warn', onBrokenAnchors: 'warn', - onBrokenMarkdownLinks: 'warn', + markdown: { + hooks: { + onBrokenMarkdownLinks: 'warn', + }, + }, i18n: { defaultLocale: 'en', diff --git a/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx b/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx index 45a5912b..02cf7310 100644 --- a/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx +++ b/Docs/pages/src/components/MockolateBenchmarkResult/index.tsx @@ -127,7 +127,7 @@ function Sparkline({values, className}: SparklineProps): ReactElement | null { type LibraryRow = { library: string; - value: number; + value: number | null; history?: number[]; }; @@ -139,7 +139,7 @@ type MetricBlockProps = { }; function MetricBlock({label, rows, baselineValue, format}: MetricBlockProps): ReactElement { - const max = Math.max(...rows.map(r => r.value), 0); + const max = Math.max(...rows.map(r => r.value ?? 0), 0); return (
@@ -147,28 +147,38 @@ function MetricBlock({label, rows, baselineValue, format}: MetricBlockProps): Re
{rows.map(row => { const isBaseline = row.library === BASELINE_LIBRARY; - const width = max === 0 ? 0 : (row.value / max) * 100; + const isMissing = row.value === null; + const width = isMissing || max === 0 ? 0 : (row.value! / max) * 100; + const rowClass = [ + styles.barRow, + isBaseline && styles.barRowBaseline, + isMissing && styles.barRowMissing, + ].filter(Boolean).join(' '); return ( -
+
{LIBRARY_DISPLAY_NAMES[row.library] ?? row.library} - + {!isMissing && ( + + )} + + + {isMissing ? 'not available' : format(row.value!)} - {format(row.value)} - {isBaseline ? 'baseline' : formatRatio(row.value, baselineValue)} + {isMissing ? '—' : isBaseline ? 'baseline' : formatRatio(row.value!, baselineValue)} - + {!isMissing && ( + + )}
); })} @@ -182,19 +192,25 @@ function buildRows( picker: (s: Sample) => number | undefined, history: (s: Sample) => number[] | undefined, ): LibraryRow[] { - // Build all available rows for libraries whose dataset includes the metric, then - // sort by metric value ascending — fastest-first / smallest-allocation-first. - // Libraries missing this scenario or this metric are skipped silently. - const rows: LibraryRow[] = []; + // Render every library in LIBRARY_ORDER so absences are visible: a library + // with no sample for this scenario / metric appears as a "not available" + // placeholder rather than silently disappearing from the comparison. + // Present rows sort by value ascending (fastest first); missing rows fall + // through to the bottom in declared order so the chart's shape stays stable + // across scenarios. + const present: LibraryRow[] = []; + const missing: LibraryRow[] = []; for (const library of LIBRARY_ORDER) { const sample = samples[library]; - if (!sample) continue; - const value = picker(sample); - if (value === undefined) continue; - rows.push({library, value, history: history(sample)}); + const value = sample ? picker(sample) : undefined; + if (value === undefined) { + missing.push({library, value: null}); + } else { + present.push({library, value, history: sample ? history(sample) : undefined}); + } } - rows.sort((a, b) => a.value - b.value); - return rows; + present.sort((a, b) => (a.value as number) - (b.value as number)); + return [...present, ...missing]; } type Props = { @@ -226,13 +242,12 @@ export default function MockolateBenchmarkResult({name}: Props): ReactElement { return (
{showParamTabs && ( -
+
{params.map(p => (