From 8c45adeeb8794244095c0500c3823eb6da9d5f88 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Fri, 8 May 2026 16:52:08 -0400 Subject: [PATCH 1/9] Fix tooltip coordinates when scrolled down The tooltip uses `position: fixed` which interprets `left/top` as viewport-relative, but `ChartCanvas` was storing `event.pageX/pageY` (document-relative, including scroll offsets). In a non-scrolling page these match, but inside a scrollable container like our benchmark page they diverge by `window.scrollY`, dragging the tooltip down. Switched the state field from `pageX/pageY` to `clientX/clientY`. --- src/components/shared/chart/Canvas.tsx | 49 +++++++++++++----------- src/test/components/MarkerChart.test.tsx | 14 +++++++ 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/components/shared/chart/Canvas.tsx b/src/components/shared/chart/Canvas.tsx index 4f12710e52..978060258a 100644 --- a/src/components/shared/chart/Canvas.tsx +++ b/src/components/shared/chart/Canvas.tsx @@ -43,13 +43,16 @@ type Props = { } | null; }; -// The naming of the X and Y coordinates here correspond to the ones -// found on the MouseEvent interface. +// Mouse coordinates passed to the Tooltip component. The Tooltip uses +// `position: fixed`, so these must be VIEWPORT-relative (i.e. clientX/clientY, +// not pageX/pageY). Otherwise the tooltip is mispositioned whenever the page +// is scrolled (the chart's container can be in a scrollable region — e.g. the +// benchmark-comparison page — even though the chart itself doesn't scroll). type State = { hoveredItem: Item | null; selectedItem: Item | null; - pageX: CssPixels; - pageY: CssPixels; + clientX: CssPixels; + clientY: CssPixels; }; export type ChartCanvasScale = { @@ -103,8 +106,8 @@ export class ChartCanvas extends React.Component< override state: State = { hoveredItem: null, selectedItem: null, - pageX: 0, - pageY: 0, + clientX: 0, + clientY: 0, }; _scheduleDraw( @@ -238,8 +241,8 @@ export class ChartCanvas extends React.Component< if (this.props.stickyTooltips) { this.setState((state) => ({ selectedItem: state.hoveredItem, - pageX: e.pageX, - pageY: e.pageY, + clientX: e.clientX, + clientY: e.clientY, })); } @@ -292,20 +295,18 @@ export class ChartCanvas extends React.Component< const maybeHoveredItem = this.props.hitTest(offsetX, offsetY); if (maybeHoveredItem !== null) { if (this.state.selectedItem === null) { - // Update both the hovered item and the pageX and pageY values. The - // pageX and pageY values are used to change the position of the tooltip - // and if there is no selected item, it means that we can update this - // position freely. + // Update both the hovered item and the cursor position. The cursor + // position is used to position the tooltip; if there is no selected + // item we can update it freely. this.setState({ hoveredItem: maybeHoveredItem, - pageX: event.pageX, - pageY: event.pageY, + clientX: event.clientX, + clientY: event.clientY, }); } else { // If there is a selected item, only update the hoveredItem and not the - // pageX and pageY values which is used for the position of the tooltip. - // By keeping the x and y values the same, we make sure that the tooltip - // stays in its initial position where it's clicked. + // cursor position used to position the tooltip. By keeping the x and y + // values the same, the tooltip stays at the position where it was clicked. this.setState({ hoveredItem: maybeHoveredItem, }); @@ -385,9 +386,11 @@ export class ChartCanvas extends React.Component< const { offsetX, offsetY } = selectedItemTooltipOffset; const canvasRect = this._canvas.getBoundingClientRect(); - const pageX = canvasRect.left + window.scrollX + offsetX; - const pageY = canvasRect.top + window.scrollY + offsetY; - this.setState({ selectedItem, pageX, pageY }); + // Viewport-relative coordinates (no scroll offsets), since the Tooltip is + // positioned with `position: fixed`. See the State type comment above. + const clientX = canvasRect.left + offsetX; + const clientY = canvasRect.top + offsetY; + this.setState({ selectedItem, clientX, clientY }); }; override UNSAFE_componentWillReceiveProps() { @@ -495,7 +498,7 @@ export class ChartCanvas extends React.Component< override render() { const { isDragging } = this.props; - const { hoveredItem, pageX, pageY } = this.state; + const { hoveredItem, clientX, clientY } = this.state; const className = classNames({ chartCanvas: true, @@ -519,8 +522,8 @@ export class ChartCanvas extends React.Component< /> {!isDragging && tooltipContents ? ( Date: Mon, 1 Jun 2026 17:11:34 -0400 Subject: [PATCH 2/9] Add profiler-edit --canonicalize-js-location. This is a workaround for Firefox and Chrome using different syntax to indicate the location of a JS function. On Windows, the difference is samply's fault: The JIT ETW events already allow specifying the URL and line/col separately from the function name, but samply puts it back into the function name. And Firefox doesn't make use of those ETW events yet. --- src/node-tools/profiler-edit.ts | 80 +++++++++++++++++++++++++++++ src/test/unit/profiler-edit.test.ts | 23 +++++++++ 2 files changed, 103 insertions(+) diff --git a/src/node-tools/profiler-edit.ts b/src/node-tools/profiler-edit.ts index ed67f35e43..d2753957d1 100644 --- a/src/node-tools/profiler-edit.ts +++ b/src/node-tools/profiler-edit.ts @@ -46,6 +46,7 @@ import { mergeNonOverlappingThreadsByName, remapCountersAndProfilerOverhead, } from 'firefox-profiler/profile-logic/merge-compare'; +import { StringTable } from 'firefox-profiler/utils/string-table'; /** * A CLI tool for editing profiles. @@ -70,6 +71,9 @@ import { * node node-tools-dist/profiler-edit.js -i big.json.gz -o small.json.gz \ * --only-keep-threads-with-markers-matching '-async,-sync' \ * --merge-non-overlapping-threads-by-name + * + * node node-tools-dist/profiler-edit.js -i in.json -o out.json \ + * --canonicalize-js-location */ export type ProfileSource = @@ -97,6 +101,7 @@ export interface CliOptions { onlyKeepThreadsWithMarkersMatching?: string; mergeNonOverlappingThreadsByName?: boolean; setName?: string; + canonicalizeJsLocation?: boolean; } export function loadWasmSymbolicationSpecs( @@ -133,6 +138,72 @@ export function collectFuncNames(profile: Profile): string[] { return result; } +/** + * Strip ` (file:line:col)` or ` file:line:col` location suffixes from JS func + * names and move the location into the funcTable + sources columns instead. + * Idempotent: re-running on an already-canonicalized profile is a no-op + * because the trailing suffix is gone. + */ +function canonicalizeJsLocations(profile: Profile): Profile { + const { funcTable, sources, stringArray } = profile.shared; + const stringTable = StringTable.withBackingArray(stringArray); + + // Reuse existing source entries that already cover a whole file at the + // standard (1, 1) origin, keyed by the filename's string index. + const filenameToSourceIndex = new Map(); + for (let i = 0; i < sources.length; i++) { + if (sources.startLine[i] === 1 && sources.startColumn[i] === 1) { + const filenameIndex = sources.filename[i]; + if (!filenameToSourceIndex.has(filenameIndex)) { + filenameToSourceIndex.set(filenameIndex, i); + } + } + } + + // The filename may contain colons (URLs), so we rely on greedy matching + // to anchor `:line:col` at the very end of the string. + const parenRegex = /^(.+) \((.+):(\d+):(\d+)\)$/; + const plainRegex = /^(.+) (.+):(\d+):(\d+)$/; + + let canonicalized = 0; + for (let i = 0; i < funcTable.length; i++) { + if (!funcTable.isJS[i]) { + continue; + } + const name = stringArray[funcTable.name[i]]; + const match = parenRegex.exec(name) ?? plainRegex.exec(name); + if (match === null) { + continue; + } + const cleanName = match[1]; + const filename = match[2]; + const line = parseInt(match[3], 10); + const col = parseInt(match[4], 10); + + const filenameIndex = stringTable.indexForString(filename); + let sourceIndex = filenameToSourceIndex.get(filenameIndex); + if (sourceIndex === undefined) { + sourceIndex = sources.length; + sources.id[sourceIndex] = null; + sources.filename[sourceIndex] = filenameIndex; + sources.startLine[sourceIndex] = 1; + sources.startColumn[sourceIndex] = 1; + sources.sourceMapURL[sourceIndex] = null; + sources.length++; + filenameToSourceIndex.set(filenameIndex, sourceIndex); + } + + funcTable.name[i] = stringTable.indexForString(cleanName); + funcTable.source[i] = sourceIndex; + funcTable.lineNumber[i] = line; + funcTable.columnNumber[i] = col; + canonicalized++; + } + + console.log(`Canonicalized location of ${canonicalized} JS function(s).`); + return profile; +} + export type ParsedLabelToml = { labels: LabelDescription[]; autoLabels: AutoLabel[]; @@ -317,6 +388,10 @@ export async function run(options: CliOptions) { profile = mergeNonOverlappingThreadsByName(profile); } + if (options.canonicalizeJsLocation) { + profile = canonicalizeJsLocations(profile); + } + if (options.setName !== undefined) { profile.meta.product = options.setName; } @@ -402,6 +477,10 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { '--set-name ', 'Override the profile product name', requireNonEmpty('--set-name') + ) + .option( + '--canonicalize-js-location', + 'Move "name (file:line:col)" suffixes on JS functions into the funcTable + sources columns' ); program.parse(processArgv); @@ -462,6 +541,7 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { mergeNonOverlappingThreadsByName: opts.mergeNonOverlappingThreadsByName === true, setName: typeof opts.setName === 'string' ? opts.setName : undefined, + canonicalizeJsLocation: opts.canonicalizeJsLocation === true, }; } diff --git a/src/test/unit/profiler-edit.test.ts b/src/test/unit/profiler-edit.test.ts index 01ff263120..92bda583a9 100644 --- a/src/test/unit/profiler-edit.test.ts +++ b/src/test/unit/profiler-edit.test.ts @@ -127,6 +127,29 @@ describe('makeOptionsFromArgv', function () { expect(options.insertLabelFrames).toEqual('/path/to/labels.toml'); }); + it('recognizes optional --canonicalize-js-location', function () { + const options = makeOptionsFromArgv([ + ...commonArgs, + '-i', + '/path/to/profile.json', + '-o', + '/path/to/output.json', + '--canonicalize-js-location', + ]); + expect(options.canonicalizeJsLocation).toBe(true); + }); + + it('defaults --canonicalize-js-location to false', function () { + const options = makeOptionsFromArgv([ + ...commonArgs, + '-i', + '/path/to/profile.json', + '-o', + '/path/to/output.json', + ]); + expect(options.canonicalizeJsLocation).toBe(false); + }); + it('throws when no input is provided', function () { expect(() => makeOptionsFromArgv([...commonArgs, '-o', '/path/to/output.json']) From e679ada56a4322bc2e4904bfadf0c5c87bd7c6bc Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Tue, 6 May 2025 15:04:46 -0400 Subject: [PATCH 3/9] Add Speedometer benchmark analysis scripts and comparison view. --- scripts/build-node-tools.mjs | 24 + scripts/generate-known-functions-toml.mjs | 24 + src/actions/app.ts | 7 + src/actions/receive-profile.ts | 3 +- src/app-logic/url-handling.ts | 11 + src/components/app/AppViewRouter.tsx | 7 + src/components/app/BenchmarkCompareViewer.css | 292 +++++++++ src/components/app/BenchmarkCompareViewer.tsx | 616 ++++++++++++++++++ src/components/app/BucketFlameGraphPair.tsx | 160 +++++ src/components/app/CompareHome.css | 10 +- src/components/app/CompareHome.tsx | 45 +- src/components/app/MenuButtons/index.tsx | 1 + src/components/app/ProfileLoader.tsx | 1 + src/components/app/ServiceWorkerManager.tsx | 2 + src/components/app/WindowTitle.tsx | 3 + src/node-tools/analyze-benchmark.ts | 282 ++++++++ src/node-tools/compare-benchmark-stats.ts | 239 +++++++ src/node-tools/extract-benchmark-stats.ts | 39 ++ src/node-tools/profile-insert-labels.ts | 208 ++++++ src/node-tools/profiler-edit.ts | 11 +- src/node-tools/symbolicator-cli.ts | 151 +++++ .../benchmark/benchmark-stuff.ts | 591 +++++++++++++++++ .../benchmark/bucket-flame-graph-data.ts | 359 ++++++++++ .../benchmark/compare-benchmark-stats.ts | 262 ++++++++ .../benchmark/extract-benchmark-stats.ts | 304 +++++++++ .../benchmark/perf-compare-stats.ts | 502 ++++++++++++++ src/reducers/url-state.ts | 3 + .../__snapshots__/CompareHome.test.tsx.snap | 21 +- src/types/actions.ts | 7 + src/utils/stats.ts | 92 +++ 30 files changed, 4257 insertions(+), 20 deletions(-) create mode 100644 scripts/generate-known-functions-toml.mjs create mode 100644 src/components/app/BenchmarkCompareViewer.css create mode 100644 src/components/app/BenchmarkCompareViewer.tsx create mode 100644 src/components/app/BucketFlameGraphPair.tsx create mode 100644 src/node-tools/analyze-benchmark.ts create mode 100644 src/node-tools/compare-benchmark-stats.ts create mode 100644 src/node-tools/extract-benchmark-stats.ts create mode 100644 src/node-tools/profile-insert-labels.ts create mode 100644 src/node-tools/symbolicator-cli.ts create mode 100644 src/profile-logic/benchmark/benchmark-stuff.ts create mode 100644 src/profile-logic/benchmark/bucket-flame-graph-data.ts create mode 100644 src/profile-logic/benchmark/compare-benchmark-stats.ts create mode 100644 src/profile-logic/benchmark/extract-benchmark-stats.ts create mode 100644 src/profile-logic/benchmark/perf-compare-stats.ts create mode 100644 src/utils/stats.ts diff --git a/scripts/build-node-tools.mjs b/scripts/build-node-tools.mjs index 72aa260e96..b453f3e2ea 100644 --- a/scripts/build-node-tools.mjs +++ b/scripts/build-node-tools.mjs @@ -10,9 +10,33 @@ const profilerEditConfig = { outfile: 'node-tools-dist/profiler-edit.js', }; +const analyzeBenchmarkConfig = { + ...nodeBaseConfig, + entryPoints: ['src/node-tools/analyze-benchmark.ts'], + outfile: 'node-tools-dist/analyze-benchmark.js', +}; + +const extractBenchmarkStatsConfig = { + ...nodeBaseConfig, + entryPoints: ['src/node-tools/extract-benchmark-stats.ts'], + outfile: 'node-tools-dist/extract-benchmark-stats.js', +}; + +const compareBenchmarkStatsConfig = { + ...nodeBaseConfig, + entryPoints: ['src/node-tools/compare-benchmark-stats.ts'], + outfile: 'node-tools-dist/compare-benchmark-stats.js', +}; + async function build() { await esbuild.build(profilerEditConfig); console.log('✅ profiler-edit build completed'); + await esbuild.build(analyzeBenchmarkConfig); + console.log('✅ analyze-benchmark build completed'); + await esbuild.build(extractBenchmarkStatsConfig); + console.log('✅ extract-benchmark-stats build completed'); + await esbuild.build(compareBenchmarkStatsConfig); + console.log('✅ compare-benchmark-stats build completed'); } build().catch(console.error); diff --git a/scripts/generate-known-functions-toml.mjs b/scripts/generate-known-functions-toml.mjs new file mode 100644 index 0000000000..037394084b --- /dev/null +++ b/scripts/generate-known-functions-toml.mjs @@ -0,0 +1,24 @@ +import { execSync } from 'child_process'; +import { writeFileSync } from 'fs'; + +const jsCode = execSync( + 'node_modules/.bin/esbuild src/node-tools/profile-insert-labels/known-functions.ts --platform=node --format=esm' +).toString(); + +const dataUrl = 'data:text/javascript,' + encodeURIComponent(jsCode); +const { BREAK_OUT_BUCKETS } = await import(dataUrl); + +let toml = ''; +for (const bucket of BREAK_OUT_BUCKETS) { + toml += `[[buckets]]\n`; + toml += `name = ${JSON.stringify(bucket.name)}\n`; + toml += `funcPrefixes = [\n`; + for (const prefix of bucket.funcPrefixes) { + toml += ` ${JSON.stringify(prefix)},\n`; + } + toml += `]\n\n`; +} + +const outPath = 'src/node-tools/profile-insert-labels/known-functions.toml'; +writeFileSync(outPath, toml.trimEnd() + '\n'); +console.log(`Wrote ${outPath}`); diff --git a/src/actions/app.ts b/src/actions/app.ts index 5792312927..b391dc1858 100644 --- a/src/actions/app.ts +++ b/src/actions/app.ts @@ -78,6 +78,13 @@ export function changeProfilesToCompare(profiles: string[]): Action { }; } +export function changeProfilesToCompareBenchmark(profiles: string[]): Action { + return { + type: 'CHANGE_PROFILES_TO_COMPARE_BENCHMARK', + profiles, + }; +} + export function startFetchingProfiles(): Action { return { type: 'START_FETCHING_PROFILES' }; } diff --git a/src/actions/receive-profile.ts b/src/actions/receive-profile.ts index e197bee440..3c7da1219b 100644 --- a/src/actions/receive-profile.ts +++ b/src/actions/receive-profile.ts @@ -1254,7 +1254,7 @@ export function viewProfileFromPostMessage( // Given a profile view URL, extract the raw URL needed to fetch the profile // data. This mirrors the manual pathname splitting done in retrieveProfileForRawUrl, // so we can fetch the profile before calling stateFromLocation. -function getProfileFetchUrl(urlString: string): string { +export function getProfileFetchUrl(urlString: string): string { const pathParts = new URL(urlString).pathname.split('/').filter((d) => d); const dataSource = ensureIsValidDataSource(pathParts[0]); switch (dataSource) { @@ -1459,6 +1459,7 @@ export function retrieveProfileForRawUrl( case 'uploaded-recordings': case 'none': case 'local': + case 'compare-benchmark': // There is no profile to download for these datasources. break; default: diff --git a/src/app-logic/url-handling.ts b/src/app-logic/url-handling.ts index 70c6f026f2..d8d6eec751 100644 --- a/src/app-logic/url-handling.ts +++ b/src/app-logic/url-handling.ts @@ -132,6 +132,8 @@ function getPathParts(urlState: UrlState): string[] { return ['compare']; } return ['compare', urlState.selectedTab]; + case 'compare-benchmark': + return ['compare-benchmark']; case 'uploaded-recordings': return ['uploaded-recordings']; case 'from-browser': @@ -267,6 +269,14 @@ export function getQueryStringFromUrlState(urlState: UrlState): string { return ''; } break; + case 'compare-benchmark': + if (urlState.profilesToCompare === null) { + return ''; + } + return queryString.stringify( + { profiles: urlState.profilesToCompare }, + { arrayFormat: 'bracket' } + ); case 'public': case 'local': case 'from-browser': @@ -457,6 +467,7 @@ export function ensureIsValidDataSource( case 'public': case 'from-url': case 'compare': + case 'compare-benchmark': case 'uploaded-recordings': return coercedDataSource; default: diff --git a/src/components/app/AppViewRouter.tsx b/src/components/app/AppViewRouter.tsx index 8c025eab8b..e9ff5b092d 100644 --- a/src/components/app/AppViewRouter.tsx +++ b/src/components/app/AppViewRouter.tsx @@ -9,6 +9,7 @@ import { ProfileViewer } from './ProfileViewer'; import { ZipFileViewer } from './ZipFileViewer'; import { Home } from './Home'; import { CompareHome } from './CompareHome'; +import { BenchmarkCompareViewer } from './BenchmarkCompareViewer'; import { ProfileRootMessage } from './ProfileRootMessage'; import { getView } from 'firefox-profiler/selectors/app'; import { getHasZipFile } from 'firefox-profiler/selectors/zipped-profiles'; @@ -34,6 +35,7 @@ const ERROR_MESSAGES_L10N_ID: { [key: string]: string } = Object.freeze({ public: 'AppViewRouter--error-public', 'from-url': 'AppViewRouter--error-from-url', compare: 'AppViewRouter--error-compare', + 'compare-benchmark': 'AppViewRouter--error-compare', }); type AppViewRouterStateProps = { @@ -61,6 +63,11 @@ class AppViewRouterImpl extends PureComponent { return ; } break; + case 'compare-benchmark': + if (profilesToCompare === null) { + return ; + } + return ; case 'uploaded-recordings': return ; case 'from-browser': diff --git a/src/components/app/BenchmarkCompareViewer.css b/src/components/app/BenchmarkCompareViewer.css new file mode 100644 index 0000000000..26f2d27682 --- /dev/null +++ b/src/components/app/BenchmarkCompareViewer.css @@ -0,0 +1,292 @@ +.benchmarkCompareViewer { + width: 100%; + min-height: 100%; + box-sizing: border-box; + + /* this element is a child in a horizontal flex parent */ + align-self: flex-start; /* allow extending beyond the parent's height */ + padding: 2em 3em; + background: var(--base-background-color); +} + +.benchmarkResults { + font-size: 14px; +} + +.benchmarkTitle { + margin-bottom: 1em; +} + +/* Loading */ + +.benchmarkLoading { + display: flex; + flex-direction: column; + align-items: center; + padding: 4em; + color: var(--grey-60); + gap: 1em; +} + +.benchmarkSpinner { + width: 2em; + height: 2em; + border: 3px solid var(--grey-30); + border-radius: 50%; + border-top-color: var(--blue-50); + animation: benchmarkSpin 0.8s linear infinite; +} + +@keyframes benchmarkSpin { + to { + transform: rotate(360deg); + } +} + +/* Error */ + +.benchmarkError { + padding: 1em; + border: 1px solid var(--red-50, #ff0039); + border-radius: 4px; + background: var(--red-10, #ffe8e8); + color: var(--red-70, #a4000f); +} + +/* Profile URLs */ + +.benchmarkProfileUrls { + display: flex; + flex-direction: column; + margin-bottom: 1.5em; + color: var(--grey-70); + font-size: 0.9em; + gap: 0.25em; + word-break: break-all; +} + +/* Section headings */ + +.benchmarkSectionTitle { + margin: 1.5em 0 0.5em; + font-size: 1.1em; + font-weight: bold; +} + +/* Suite details */ + +.benchmarkSuiteDetails > .benchmarkSectionTitle { + cursor: pointer; + user-select: none; +} + +.benchmarkSuiteDetails > .benchmarkSectionTitle::marker { + font-size: 0.8em; +} + +.benchmarkNoChanges { + padding: 0.5em 0; + color: var(--grey-60); + font-style: italic; +} + +/* Tables */ + +.benchmarkTable { + width: 100%; + margin-bottom: 1em; + border-collapse: collapse; + font-size: 0.9em; + + /* Fixed layout so the numeric column widths are honored exactly. With + * the same numeric widths in both the outer score table and the inner + * subtest tables, and zero right-padding on the expansion row, the + * numeric columns line up vertically across both tables. The first + * column (label / bucket name) flexes to fill the remaining space. */ + table-layout: fixed; +} + +.benchmarkTable th, +.benchmarkTable td { + box-sizing: border-box; + padding: 4px 8px; + border-bottom: 1px solid var(--grey-20); + text-align: left; + white-space: nowrap; +} + +.benchmarkTable th { + position: sticky; + z-index: 1; + top: 0; + background: var(--grey-10); + font-weight: bold; +} + +.benchmarkCell--number { + font-variant-numeric: tabular-nums; + text-align: right; +} + +.benchmarkCell--colFixed { + width: 9rem; +} + +.benchmarkCell--bucketName, +.benchmarkCell--scoreLabel { + overflow: hidden; + text-overflow: ellipsis; +} + +.benchmarkRow--overall td { + border-bottom: 2px solid var(--grey-40); + font-weight: bold; +} + +.benchmarkCell--indented { + padding-left: 1.5em; +} + +/* Expandable subtest rows in the score table */ + +.benchmarkRow--suite-expandable { + cursor: pointer; +} + +.benchmarkRow--suite-expandable:hover { + background: var(--grey-10); +} + +.benchmarkCell--suiteLabel { + position: relative; +} + +.benchmarkDisclosure { + display: inline-block; + width: 1em; + margin-right: 0.25em; + color: var(--grey-60); + font-size: 0.75em; + text-align: center; + user-select: none; +} + +.benchmarkRow--expansion > td { + padding: 0.5em 0 0.5em 2.5em; + background: var(--grey-10); +} + +.benchmarkRow--expansion .benchmarkTable { + margin-bottom: 0; +} + +/* Color coding. + * + * Confidence drives shading (background): HIGH = full, MEDIUM = muted, + * LOW = no shading. Effect size drives boldness: Large = bold, + * Moderate = semibold, Small/Negligible = normal. + */ + +.benchmarkCell--regressed { + color: var(--red-70, #a4000f); +} + +.benchmarkCell--improved { + color: var(--green-80, #006504); +} + +.benchmarkCell--regressed.benchmarkCell--conf-high { + background: var(--red-10, #ffe8e8); +} + +.benchmarkCell--regressed.benchmarkCell--conf-medium { + background: color-mix(in srgb, var(--red-10, #ffe8e8) 40%, transparent); +} + +.benchmarkCell--improved.benchmarkCell--conf-high { + background: var(--green-10, #d3f3d8); +} + +.benchmarkCell--improved.benchmarkCell--conf-medium { + background: color-mix(in srgb, var(--green-10, #d3f3d8) 40%, transparent); +} + +.benchmarkCell--effect-large { + font-weight: bold; +} + +.benchmarkCell--effect-moderate { + font-weight: 600; +} + +/* Expandable bucket rows (inner table). Mirror the suite-row styling. */ + +.benchmarkRow--bucket-expandable { + cursor: pointer; +} + +.benchmarkRow--bucket-expandable:hover { + background: var(--grey-20); +} + +.benchmarkRow--bucket-expansion > td { + padding: 0.5em 0; + background: var(--grey-20); + white-space: normal; +} + +/* Stacked base / new flame graphs for one expanded bucket. The base flame + * graph is on top, the new one below, so they can be visually compared by + * scanning vertically. Each side's width is set inline (as a percentage) + * proportional to its total sample count, so 1 sample takes up the same + * pixel width on both sides. */ + +.bucketFlameGraphPair { + display: flex; + flex-direction: column; + padding: 0.5em 1em; + gap: 0.5em; +} + +.bucketFlameGraphSide { + display: flex; + overflow: hidden; + min-width: 0; + height: 280px; + flex-direction: column; + border: 1px solid var(--grey-30); + border-radius: 4px; + background: var(--base-background-color); +} + +.bucketFlameGraphSide__label { + padding: 4px 8px; + border-bottom: 1px solid var(--grey-30); + background: var(--grey-20); + color: var(--grey-70); + font-size: 0.85em; + font-weight: bold; + text-transform: uppercase; +} + +.bucketFlameGraphSide__sampleCount { + color: var(--grey-60); + font-weight: normal; + text-transform: none; +} + +.bucketFlameGraphSide__chart { + display: flex; + overflow: hidden; + flex: 1; + flex-direction: column; +} + +.bucketFlameGraphSide__empty { + display: flex; + flex: 1; + align-items: center; + justify-content: center; + color: var(--grey-60); + font-style: italic; +} diff --git a/src/components/app/BenchmarkCompareViewer.tsx b/src/components/app/BenchmarkCompareViewer.tsx new file mode 100644 index 0000000000..5ca656a094 --- /dev/null +++ b/src/components/app/BenchmarkCompareViewer.tsx @@ -0,0 +1,616 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Fragment, useState, useEffect, useMemo } from 'react'; +import { useSelector } from 'react-redux'; + +import { AppHeader } from './AppHeader'; +import { getProfilesToCompare } from 'firefox-profiler/selectors/url-state'; +import { fetchProfile } from 'firefox-profiler/utils/profile-fetch'; +import { unserializeProfileOfArbitraryFormat } from 'firefox-profiler/profile-logic/process-profile'; +import { expandUrl } from 'firefox-profiler/utils/shorten-url'; +import { getProfileFetchUrl } from 'firefox-profiler/actions/receive-profile'; +import { extractBenchmarkStatsFromProfile } from 'firefox-profiler/profile-logic/benchmark/extract-benchmark-stats'; +import { + compareBuckets, + compareIterationTotals, + suiteIterationTotals, +} from 'firefox-profiler/profile-logic/benchmark/compare-benchmark-stats'; +import type { + BucketComparison, + ScoreComparison, +} from 'firefox-profiler/profile-logic/benchmark/compare-benchmark-stats'; +import type { + ConfidenceRating, + EffectSize, +} from 'firefox-profiler/profile-logic/benchmark/perf-compare-stats'; +import type { Profile } from 'firefox-profiler/types'; +import { BucketFlameGraphPair } from './BucketFlameGraphPair'; +import { + makeBucketProfileBundle, + makeSuiteFilteredThread, +} from 'firefox-profiler/profile-logic/benchmark/bucket-flame-graph-data'; +import type { BucketProfileBundle } from 'firefox-profiler/profile-logic/benchmark/bucket-flame-graph-data'; +import './BenchmarkCompareViewer.css'; + +type ComparisonData = { + baseUrl: string; + newUrl: string; + /** The loaded source profiles, retained so we can render flame graphs of + * individual buckets on demand (focusSelf on a bucket's representative func). */ + baseProfile: Profile; + newProfile: Profile; + overallScore: ScoreComparison; + suiteScores: ScoreComparison[]; + suiteComparisons: Array<{ + suiteName: string; + comparisons: BucketComparison[]; + }>; +}; + +type State = + | { phase: 'loading' } + | { phase: 'error'; error: string } + | { phase: 'done'; data: ComparisonData }; + +const TOP_N = 100; + +async function loadOneProfile(viewerUrl: string) { + let url = viewerUrl; + if ( + url.startsWith('https://perfht.ml/') || + url.startsWith('https://share.firefox.dev/') || + url.startsWith('https://bit.ly/') + ) { + url = await expandUrl(url); + } + const dataUrl = getProfileFetchUrl(url); + const response = await fetchProfile({ + url: dataUrl, + onTemporaryError: () => {}, + }); + if (response.responseType !== 'BYTES') { + throw new Error('Expected a profile, not a zip file.'); + } + return unserializeProfileOfArbitraryFormat(response.bytes, dataUrl); +} + +async function computeComparison( + baseUrl: string, + newUrl: string +): Promise { + const [baseProfile, newProfile] = await Promise.all([ + loadOneProfile(baseUrl), + loadOneProfile(newUrl), + ]); + + const baseStats = extractBenchmarkStatsFromProfile(baseProfile); + const newStats = extractBenchmarkStatsFromProfile(newProfile); + + const iterationCount = baseStats.suites[0]?.iterationCount ?? 1; + + const baseGlobalIter = suiteIterationTotals( + baseStats.globalBuckets, + iterationCount + ); + const newGlobalIter = suiteIterationTotals( + newStats.globalBuckets, + iterationCount + ); + const overallScore = compareIterationTotals( + 'Overall (geomean-normalised)', + baseGlobalIter, + newGlobalIter + ); + + const suiteScores: ScoreComparison[] = []; + for (const baseSuite of baseStats.suites) { + const newSuite = newStats.suites.find( + (s) => s.suiteName === baseSuite.suiteName + ); + const baseIter = suiteIterationTotals( + baseSuite.buckets, + baseSuite.iterationCount + ); + const newIter = newSuite + ? suiteIterationTotals(newSuite.buckets, newSuite.iterationCount) + : new Array(baseSuite.iterationCount).fill(0); + suiteScores.push( + compareIterationTotals(baseSuite.suiteName, baseIter, newIter) + ); + } + suiteScores.sort((a, b) => a.label.localeCompare(b.label)); + + const suiteComparisons = baseStats.suites.flatMap((baseSuite) => { + const newSuite = newStats.suites.find( + (s) => s.suiteName === baseSuite.suiteName + ); + if (!newSuite) { + return []; + } + const comparisons = compareBuckets( + baseSuite.buckets, + newSuite.buckets, + baseStats.bucketNames, + newStats.bucketNames, + baseStats.bucketFuncs, + newStats.bucketFuncs, + baseSuite.iterationCount, + false, + baseStats.bucketKeys ?? baseStats.bucketNames, + newStats.bucketKeys ?? newStats.bucketNames + ); + return [{ suiteName: baseSuite.suiteName, comparisons }]; + }); + suiteComparisons.sort((a, b) => a.suiteName.localeCompare(b.suiteName)); + + return { + baseUrl, + newUrl, + baseProfile, + newProfile, + overallScore, + suiteScores, + suiteComparisons, + }; +} + +/** + * Given a relative change of a single subtest's mean, compute the resulting + * relative change in the overall geomean across `numSuites` subtests, assuming + * the other subtests are unchanged. Exact (not a linearization): + * newGeomean / baseGeomean = (newSuiteMean / baseSuiteMean)^(1/N) + */ +function impactOnGeomean(suiteRel: number, numSuites: number): number { + if (!isFinite(suiteRel)) { + return suiteRel; + } + return Math.pow(1 + suiteRel, 1 / numSuites) - 1; +} + +function formatChange(rel: number): string { + if (!isFinite(rel)) { + return rel > 0 ? 'appeared' : 'disappeared'; + } + const pct = (rel * 100).toFixed(2); + return rel >= 0 ? `+${pct}%` : `${pct}%`; +} + +function changeClass( + relChange: number, + confidence: ConfidenceRating, + effectSize: EffectSize +): string { + if (!isFinite(relChange) || relChange === 0) { + return ''; + } + const direction = relChange > 0 ? 'regressed' : 'improved'; + const classes = []; + // Only color the text (and add background shading) when we have at least + // medium confidence. Below that, leave the text in the default color. + if (confidence === 'HIGH') { + classes.push(`benchmarkCell--${direction}`, 'benchmarkCell--conf-high'); + } else if (confidence === 'MEDIUM') { + classes.push(`benchmarkCell--${direction}`, 'benchmarkCell--conf-medium'); + } + if (effectSize === 'Large') { + classes.push('benchmarkCell--effect-large'); + } else if (effectSize === 'Moderate') { + classes.push('benchmarkCell--effect-moderate'); + } + // Small / Negligible: normal weight. + return classes.join(' '); +} + +const SCORE_TABLE_COLUMN_COUNT = 6; + +function ScoreRow({ + row, + isOverall, + numSuites, +}: { + row: ScoreComparison; + isOverall: boolean; + numSuites: number; +}) { + const absDiff = row.newMean - row.baseMean; + const absDiffStr = (absDiff >= 0 ? '+' : '') + absDiff.toFixed(2); + // For the overall row, the score IS the geomean — there's no enclosing + // subtest, so leave the subtest column blank, and the overall column shows + // the actual measured geomean relChange. For a subtest row, the subtest's + // relChange is its own, and we compute its impact on the overall geomean + // assuming only this subtest changed. + const subtestRel = isOverall ? null : row.relChange; + const overallRel = isOverall + ? row.relChange + : impactOnGeomean(row.relChange, numSuites); + return ( + <> + {row.baseMean.toFixed(2)} + {row.newMean.toFixed(2)} + {absDiffStr} + + {subtestRel === null ? '—' : formatChange(subtestRel)} + + + {formatChange(overallRel)} + + + ); +} + +function ScoreTable({ + overallScore, + suiteScores, + suiteComparisonsByName, + baseBundle, + newBundle, +}: { + overallScore: ScoreComparison; + suiteScores: ScoreComparison[]; + suiteComparisonsByName: Map; + baseBundle: BucketProfileBundle; + newBundle: BucketProfileBundle; +}) { + const [expanded, setExpanded] = useState>(new Set()); + const numSuites = suiteScores.length; + + const toggle = (label: string) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(label)) { + next.delete(label); + } else { + next.add(label); + } + return next; + }); + }; + + return ( + + + + + + + + + + + + + + + + + {suiteScores.map((row) => { + const isExpanded = expanded.has(row.label); + const comparisons = suiteComparisonsByName.get(row.label); + const expandable = comparisons !== undefined; + return ( + + toggle(row.label) : undefined} + > + + + + {isExpanded && comparisons ? ( + + + + ) : null} + + ); + })} + +
Score + Base mean + + New mean + + Δ abs + + Δ% subtest + + Δ% overall +
+ {overallScore.label} +
+ {expandable ? ( + + ) : null} + {row.label} +
+ +
+ ); +} + +function BucketTable({ + comparisons, + label, + baseSubtestMean, + numSuites, + baseBundle, + newBundle, +}: { + comparisons: BucketComparison[]; + label: string; + /** When provided together with numSuites, two percent columns are shown + * (Δ% overall and Δ% subtest) instead of the bucket-relative Δ%. */ + baseSubtestMean?: number; + numSuites?: number; + baseBundle: BucketProfileBundle; + newBundle: BucketProfileBundle; +}) { + const showSubtestColumns = + baseSubtestMean !== undefined && numSuites !== undefined; + const columnCount = showSubtestColumns ? 6 : 5; + + // Build per-suite bundles whose `thread.samples.weight` is zeroed outside + // this suite's iteration markers, so flame graphs reflect only the samples + // that contribute to this suite's score. + const baseSuiteBundle = useMemo( + () => withSuiteFilteredThread(baseBundle, label), + [baseBundle, label] + ); + const newSuiteBundle = useMemo( + () => withSuiteFilteredThread(newBundle, label), + [newBundle, label] + ); + + // Keyed by row index in `significant`, not bucketName: multiple buckets in + // the same suite can share a display name (e.g. several `get` accessors on + // different classes — distinguished by their source-location key but + // collapsed to the same name for display). + const [expanded, setExpanded] = useState>(new Set()); + const toggle = (rowIndex: number) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(rowIndex)) { + next.delete(rowIndex); + } else { + next.add(rowIndex); + } + return next; + }); + }; + + const significant = comparisons + // .filter((c) => c.confidence !== 'LOW' && c.effectSize !== 'Negligible') + .sort( + (a, b) => + Math.abs(b.newMean - b.baseMean) - Math.abs(a.newMean - a.baseMean) + ) + .slice(0, TOP_N); + + if (significant.length === 0) { + return ( +

+ No bucket changes in {label} with at least medium confidence and a + non-negligible effect size. +

+ ); + } + + return ( + + {/* Column widths come from the colgroup so we don't need a thead. The + * headers in the outer score table double as labels for these aligned + * columns. */} + + + + + + + {showSubtestColumns ? ( + + ) : null} + + + {significant.map((c, i) => { + const absDiff = c.newMean - c.baseMean; + const absDiffStr = (absDiff >= 0 ? '+' : '') + absDiff.toFixed(2); + let pctCells; + if (showSubtestColumns) { + const subtestRel = + baseSubtestMean === 0 ? Infinity : absDiff / baseSubtestMean!; + const overallRel = impactOnGeomean(subtestRel, numSuites!); + pctCells = ( + <> + + + + ); + } else { + pctCells = ( + + ); + } + // A bucket can be expanded if at least one side has a func index. + // (If both are null it's a degenerate "appeared/disappeared with no + // attributable func" case.) + const expandable = c.baseFunc !== null || c.newFunc !== null; + const isExpanded = expanded.has(i); + return ( + + toggle(i) : undefined} + > + + + + + {pctCells} + + {expandable && isExpanded ? ( + + + + ) : null} + + ); + })} + +
+ {formatChange(subtestRel)} + + {formatChange(overallRel)} + + {formatChange(c.relChange)} +
+ {expandable ? ( + + ) : null} + {c.bucketName} + + {c.baseMean.toFixed(2)} + + {c.newMean.toFixed(2)} + {absDiffStr}
+ +
+ ); +} + +/** Return a copy of `bundle` whose `thread` has sample weights zeroed outside + * this suite's iteration markers (matching the filtering applied to the suite + * count). All other bundle fields are shared with the input. */ +function withSuiteFilteredThread( + bundle: BucketProfileBundle, + suiteName: string +): BucketProfileBundle { + return { ...bundle, thread: makeSuiteFilteredThread(bundle, suiteName) }; +} + +function ComparisonResults({ data }: { data: ComparisonData }) { + const suiteComparisonsByName = new Map( + data.suiteComparisons.map(({ suiteName, comparisons }) => [ + suiteName, + comparisons, + ]) + ); + + const baseBundle = useMemo( + () => makeBucketProfileBundle(data.baseProfile, 'speedometer'), + [data.baseProfile] + ); + const newBundle = useMemo( + () => makeBucketProfileBundle(data.newProfile, 'speedometer'), + [data.newProfile] + ); + + return ( +
+
+ + Base:{' '} + + {data.baseUrl} + + + + New:{' '} + + {data.newUrl} + + +
+ +

Score and subtest totals

+ +
+ ); +} + +export function BenchmarkCompareViewer() { + const profilesToCompare = useSelector(getProfilesToCompare); + const [state, setState] = useState({ phase: 'loading' }); + + useEffect(() => { + if (!profilesToCompare || profilesToCompare.length < 2) { + setState({ phase: 'error', error: 'Two profile URLs are required.' }); + return; + } + setState({ phase: 'loading' }); + const [baseUrl, newUrl] = profilesToCompare; + computeComparison(baseUrl, newUrl) + .then((data) => setState({ phase: 'done', data })) + .catch((err) => + setState({ phase: 'error', error: String(err?.message ?? err) }) + ); + }, [profilesToCompare]); + + return ( +
+ +

Benchmark Comparison

+ + {state.phase === 'loading' && ( +
+
+

Loading profiles and computing statistics…

+
+ )} + + {state.phase === 'error' && ( +
+

+ Error: {state.error} +

+
+ )} + + {state.phase === 'done' && } +
+ ); +} diff --git a/src/components/app/BucketFlameGraphPair.tsx b/src/components/app/BucketFlameGraphPair.tsx new file mode 100644 index 0000000000..112faab970 --- /dev/null +++ b/src/components/app/BucketFlameGraphPair.tsx @@ -0,0 +1,160 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { useMemo, useState } from 'react'; + +import { FlameGraph } from 'firefox-profiler/components/flame-graph/FlameGraph'; +import { computeBucketFlameGraphData } from 'firefox-profiler/profile-logic/benchmark/bucket-flame-graph-data'; + +import type { + BucketFlameGraphData, + BucketProfileBundle, +} from 'firefox-profiler/profile-logic/benchmark/bucket-flame-graph-data'; +import type { + IndexIntoFuncTable, + IndexIntoCallNodeTable, +} from 'firefox-profiler/types'; + +export type { BucketProfileBundle }; + +type SideProps = { + label: string; + data: BucketFlameGraphData | null; + /** Stable React key (e.g. "base"/"new"); also used as the FlameGraph + * `threadsKey` to scope its internal state per side. */ + sideKey: string; + /** Width as a fraction (0..1) of the row, so the side with fewer samples is + * narrower and 1 sample takes the same pixel width on both sides. */ + widthFraction: number; +}; + +function BucketFlameGraphSide({ + label, + data, + sideKey, + widthFraction, +}: SideProps) { + const [selectedCallNodeIndex, setSelectedCallNodeIndex] = + useState(null); + + const sampleCountText = + data === null ? '' : ` — ${data.rootTotalSummary.toFixed(0)} samples`; + + return ( +
+
+ {label} + + {sampleCountText} + +
+
+ {data === null ? ( +
+ No data for this bucket. +
+ ) : ( + + )} +
+
+ ); +} + +function noop() {} + +type Props = { + baseBundle: BucketProfileBundle; + newBundle: BucketProfileBundle; + baseFunc: IndexIntoFuncTable | null; + newFunc: IndexIntoFuncTable | null; +}; + +function computeForBundle( + bundle: BucketProfileBundle, + funcIndex: IndexIntoFuncTable | null +): BucketFlameGraphData | null { + if (funcIndex === null) { + return null; + } + return computeBucketFlameGraphData( + bundle.profile, + bundle.thread, + funcIndex, + bundle.categories, + bundle.defaultCategory + ); +} + +/** Two flame graphs stacked vertically (base on top, new below). Each side's + * width is proportional to its sample-count total so 1 sample takes up the + * same pixel width across both. */ +export function BucketFlameGraphPair({ + baseBundle, + newBundle, + baseFunc, + newFunc, +}: Props) { + const baseData = useMemo( + () => computeForBundle(baseBundle, baseFunc), + [baseBundle, baseFunc] + ); + const newData = useMemo( + () => computeForBundle(newBundle, newFunc), + [newBundle, newFunc] + ); + + const baseTotal = baseData?.rootTotalSummary ?? 0; + const newTotal = newData?.rootTotalSummary ?? 0; + const maxTotal = Math.max(baseTotal, newTotal, 1); + + return ( +
+ + +
+ ); +} diff --git a/src/components/app/CompareHome.css b/src/components/app/CompareHome.css index 2cf68ab9b2..f2c183ef3d 100644 --- a/src/components/app/CompareHome.css +++ b/src/components/app/CompareHome.css @@ -21,11 +21,17 @@ grid-template-columns: auto 1fr; } -.compareHomeSubmitButton { - font-size: inherit; /* override the photon style to make it nicer with the rest of the form. */ +.compareHomeButtons { + display: flex; + flex-wrap: wrap; + gap: 0.5em; grid-column-start: span 2; } +.compareHomeButtons .photon-button { + font-size: inherit; /* override the photon style to make it nicer with the rest of the form. */ +} + .compareHomeFormLabel { white-space: nowrap; } diff --git a/src/components/app/CompareHome.tsx b/src/components/app/CompareHome.tsx index 71a1923459..a87b659d0a 100644 --- a/src/components/app/CompareHome.tsx +++ b/src/components/app/CompareHome.tsx @@ -6,13 +6,17 @@ import { PureComponent } from 'react'; import { Localized } from '@fluent/react'; import { AppHeader } from './AppHeader'; -import { changeProfilesToCompare } from 'firefox-profiler/actions/app'; +import { + changeProfilesToCompare, + changeProfilesToCompareBenchmark, +} from 'firefox-profiler/actions/app'; import explicitConnect from 'firefox-profiler/utils/connect'; import type { ConnectedProps } from 'firefox-profiler/utils/connect'; import './CompareHome.css'; type DispatchProps = { readonly changeProfilesToCompare: typeof changeProfilesToCompare; + readonly changeProfilesToCompareBenchmark: typeof changeProfilesToCompareBenchmark; }; type Props = ConnectedProps<{}, {}, DispatchProps>; @@ -33,8 +37,17 @@ class CompareHomeImpl extends PureComponent { handleFormSubmit = (e: React.FormEvent) => { e.preventDefault(); const { profile1, profile2 } = this.state; - const { changeProfilesToCompare } = this.props; - changeProfilesToCompare([profile1, profile2]); + const { changeProfilesToCompare, changeProfilesToCompareBenchmark } = + this.props; + const submitter = (e.nativeEvent as SubmitEvent).submitter; + if ( + submitter instanceof HTMLButtonElement && + submitter.name === 'benchmark' + ) { + changeProfilesToCompareBenchmark([profile1, profile2]); + } else { + changeProfilesToCompare([profile1, profile2]); + } }; override render() { @@ -86,13 +99,22 @@ class CompareHomeImpl extends PureComponent { onChange={this.handleInputChange} value={profile2} /> - - + + + + + ); @@ -100,6 +122,9 @@ class CompareHomeImpl extends PureComponent { } export const CompareHome = explicitConnect<{}, {}, DispatchProps>({ - mapDispatchToProps: { changeProfilesToCompare }, + mapDispatchToProps: { + changeProfilesToCompare, + changeProfilesToCompareBenchmark, + }, component: CompareHomeImpl, }); diff --git a/src/components/app/MenuButtons/index.tsx b/src/components/app/MenuButtons/index.tsx index 92c75be4e6..78fb17519e 100644 --- a/src/components/app/MenuButtons/index.tsx +++ b/src/components/app/MenuButtons/index.tsx @@ -109,6 +109,7 @@ class MenuButtonsImpl extends React.PureComponent { return isLocalURL(profileUrl) ? 'local' : 'uploaded'; case 'none': case 'uploaded-recordings': + case 'compare-benchmark': throw new Error(`The datasource ${dataSource} shouldn't happen here.`); default: throw assertExhaustiveCheck(dataSource); diff --git a/src/components/app/ProfileLoader.tsx b/src/components/app/ProfileLoader.tsx index 3446b26c7d..f2d6cfeee5 100644 --- a/src/components/app/ProfileLoader.tsx +++ b/src/components/app/ProfileLoader.tsx @@ -78,6 +78,7 @@ class ProfileLoaderImpl extends PureComponent { case 'from-post-message': case 'uploaded-recordings': case 'unpublished': + case 'compare-benchmark': case 'none': // nothing to do /* istanbul ignore next */ diff --git a/src/components/app/ServiceWorkerManager.tsx b/src/components/app/ServiceWorkerManager.tsx index 6802df69be..cb9ed51ffc 100644 --- a/src/components/app/ServiceWorkerManager.tsx +++ b/src/components/app/ServiceWorkerManager.tsx @@ -166,6 +166,7 @@ class ServiceWorkerManagerImpl extends PureComponent { switch (dataSource) { case 'none': case 'uploaded-recordings': + case 'compare-benchmark': return false; case 'from-file': case 'from-browser': @@ -214,6 +215,7 @@ class ServiceWorkerManagerImpl extends PureComponent { switch (dataSource) { case 'none': case 'uploaded-recordings': + case 'compare-benchmark': // These datasources have no profile loaded, we can update it right away. return true; case 'from-file': diff --git a/src/components/app/WindowTitle.tsx b/src/components/app/WindowTitle.tsx index d30dc47c67..9e41f6d7d0 100644 --- a/src/components/app/WindowTitle.tsx +++ b/src/components/app/WindowTitle.tsx @@ -55,6 +55,9 @@ class WindowTitleImpl extends PureComponent { case 'compare': document.title = 'Compare Profiles' + SEPARATOR + PRODUCT; break; + case 'compare-benchmark': + document.title = 'Benchmark Comparison' + SEPARATOR + PRODUCT; + break; case 'public': case 'local': case 'unpublished': diff --git a/src/node-tools/analyze-benchmark.ts b/src/node-tools/analyze-benchmark.ts new file mode 100644 index 0000000000..fdce694a09 --- /dev/null +++ b/src/node-tools/analyze-benchmark.ts @@ -0,0 +1,282 @@ +/** + * Merge two existing profiles, taking the samples from the first profile and + * the markers from the second profile. + * + * This was useful during early 2025 when the Mozilla Performance team was + * doing a lot of Android startup profiling: + * + * - The "samples" profile would be collected using simpleperf and converted + * with samply import. + * - The "markers" profile would be collected using the Gecko profiler. + * + * To use this script, it first needs to be built: + * yarn build-node-tools + * + * Then it can be run from the `node-tools-dist` directory: + * node node-tools-dist/analyze-benchmark.js --input ~/Downloads/munged-profile.json + * + * For example: + * yarn build-node-tools && node node-tools-dist/analyze-benchmark.js --input ~/Downloads/munged-profile.json + * + */ + +import fs from 'fs'; +import minimist from 'minimist'; + +import { unserializeProfileOfArbitraryFormat } from '../profile-logic/process-profile'; +import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants'; + +import type { + IndexIntoFuncTable, + IndexIntoStackTable, + Profile, + RawProfileSharedData, + RawThread, +} from '../types/profile'; +import type { SamplesTableForThisStuff } from 'firefox-profiler/profile-logic/benchmark/benchmark-stuff'; +import { + computeBenchmarkScores, + computeIterationMarkersAndMeasuredSamples, + computeSampleWeightsWithSuiteFactorsApplied, + getBenchmarkInfo, +} from 'firefox-profiler/profile-logic/benchmark/benchmark-stuff'; +import { + correlateIPCMarkers, + deriveMarkersFromRawMarkerTable, +} from 'firefox-profiler/profile-logic/marker-data'; +import { + computeTimeColumnForRawSamplesTable, + getTimeRangeForThread, +} from 'firefox-profiler/profile-logic/profile-data'; +import { StringTable } from 'firefox-profiler/utils/string-table'; +import { compress } from 'firefox-profiler/utils/gz'; + +type ProfileSource = + | { + type: 'HASH'; + hash: string; + } + | { + type: 'FILE'; + file: string; + }; + +interface CliOptions { + profile: ProfileSource; + outputProfilePath: string | undefined; + outputJsonPath: string | undefined; +} + +export function getProfileUrlForHash(hash: string): string { + // See https://cloud.google.com/storage/docs/access-public-data + // The URL is https://storage.googleapis.com//. + // https://.storage.googleapis.com/ seems to also work but + // is not documented nowadays. + + // By convention, "profile-store" is the name of our bucket, and the file path + // is the hash we receive in the URL. + return `https://storage.googleapis.com/${GOOGLE_STORAGE_BUCKET}/${hash}`; +} + +async function fetchProfileWithHash(hash: string): Promise { + const response = await fetch(getProfileUrlForHash(hash)); + const serializedProfile = await response.json(); + return unserializeProfileOfArbitraryFormat(serializedProfile); +} + +async function loadProfileFromFile(path: string): Promise { + const uint8Array = fs.readFileSync(path, null); + return unserializeProfileOfArbitraryFormat(uint8Array.buffer); +} + +async function loadProfile(source: ProfileSource): Promise { + switch (source.type) { + case 'HASH': + return fetchProfileWithHash(source.hash); + case 'FILE': + return loadProfileFromFile(source.file); + default: + return source; + } +} + +function computeJsOnlySampleBuckets( + shared: RawProfileSharedData, + sampleStacks: Array +): { + bucketFuncs: Array; + sampleBuckets: Int32Array; +} { + const { funcTable, stackTable, frameTable } = shared; + const bucketFuncs = new Array(); + const funcIndexToBucketIndex = new Map(); + + const stackIndexToJsOnlyFuncIndex = new Int32Array(stackTable.length); + for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) { + const frameIndex = stackTable.frame[stackIndex]; + const funcIndex = frameTable.func[frameIndex]; + if (funcTable.isJS[funcIndex] || funcTable.relevantForJS[funcIndex]) { + stackIndexToJsOnlyFuncIndex[stackIndex] = funcIndex; + } else { + const prefixOffset = stackTable.prefixOffset[stackIndex]; + if (prefixOffset !== 0) { + const parentStackIndex = stackIndex - prefixOffset; + stackIndexToJsOnlyFuncIndex[stackIndex] = + stackIndexToJsOnlyFuncIndex[parentStackIndex]; + } else { + stackIndexToJsOnlyFuncIndex[stackIndex] = -1; + } + } + } + + const sampleBuckets = new Int32Array(sampleStacks.length); + for (let sampleIndex = 0; sampleIndex < sampleBuckets.length; sampleIndex++) { + const stackIndex = sampleStacks[sampleIndex]; + if (stackIndex !== null) { + const jsOnlyFuncIndex = stackIndexToJsOnlyFuncIndex[stackIndex]; + let bucketIndex = + jsOnlyFuncIndex !== -1 + ? funcIndexToBucketIndex.get(jsOnlyFuncIndex) + : -1; + if (bucketIndex === undefined) { + bucketIndex = bucketFuncs.length; + bucketFuncs[bucketIndex] = jsOnlyFuncIndex; + funcIndexToBucketIndex.set(jsOnlyFuncIndex, bucketIndex); + } + sampleBuckets[sampleIndex] = bucketIndex; + } else { + sampleBuckets[sampleIndex] = -1; + } + } + + return { bucketFuncs, sampleBuckets }; +} + +export async function run(options: CliOptions) { + const profile: Profile = await loadProfile(options.profile); + const benchmarkInfo = getBenchmarkInfo(profile, 'speedometer'); + const { shared } = profile; + const thread = profile.threads[benchmarkInfo.threadIndex]; + const { markers } = deriveMarkersFromRawMarkerTable( + thread.markers, + shared.stringArray, + thread.tid, + getTimeRangeForThread(thread, profile.meta.interval), + correlateIPCMarkers(profile.threads, shared) + ); + const stringTable = StringTable.withBackingArray(shared.stringArray); + const sampleCount = thread.samples.length; + const { sampleBuckets, bucketFuncs } = computeJsOnlySampleBuckets( + shared, + thread.samples.stack + ); + const profileOverheadBucket = bucketFuncs.findIndex( + (func) => + shared.stringArray[shared.funcTable.name[func]] === 'Profiling overhead' + ); + const bucketsToIgnore = + profileOverheadBucket !== -1 ? [profileOverheadBucket] : []; + const samples: SamplesTableForThisStuff = { + length: sampleCount, + time: new Float64Array(computeTimeColumnForRawSamplesTable(thread.samples)), + weight: thread.samples.weight + ? new Float64Array(thread.samples.weight) + : new Float64Array(sampleCount).fill(1), + bucketIndex: sampleBuckets, + bucketCount: bucketFuncs.length, + }; + const iterationMarkersAndMeasuredSamples = + computeIterationMarkersAndMeasuredSamples( + benchmarkInfo, + markers, + samples, + stringTable, + bucketsToIgnore + ); + const benchmarkScores = computeBenchmarkScores( + iterationMarkersAndMeasuredSamples + ); + const sampleWeightsWithSuiteFactorsApplied = + computeSampleWeightsWithSuiteFactorsApplied( + iterationMarkersAndMeasuredSamples, + benchmarkScores.factorPerSuite + ); + console.log(benchmarkScores); + + const bucketNames = bucketFuncs.map( + (funcIndex) => shared.stringArray[shared.funcTable.name[funcIndex]] + ); + + const profileBenchmarkInfo = { + bucketFuncs, + bucketNames, + // bucketKeys, (type: label | js, when js include path and start line/col) + benchmarkScores, + }; + if (options.outputJsonPath !== undefined) { + fs.writeFileSync( + options.outputJsonPath, + JSON.stringify(profileBenchmarkInfo) + ); + } + + const adjustedWeightThread: RawThread = { + ...thread, + samples: { + ...thread.samples, + weight: [...sampleWeightsWithSuiteFactorsApplied], + // weightType: 'tracing-ms', + }, + }; + const adjustedWeightThreads = profile.threads.slice(); + adjustedWeightThreads[benchmarkInfo.threadIndex] = adjustedWeightThread; + const adjustedWeightProfile: Profile = { + ...profile, + threads: adjustedWeightThreads, + }; + + if (options.outputProfilePath !== undefined) { + if (options.outputProfilePath.endsWith('.gz')) { + fs.writeFileSync( + options.outputProfilePath, + await compress(JSON.stringify(adjustedWeightProfile)) + ); + } + } +} + +export function makeOptionsFromArgv(processArgv: string[]): CliOptions { + const argv = minimist(processArgv.slice(2)); + + const hasSamplesHash = 'hash' in argv && typeof argv.hash === 'string'; + const hasSamplesFile = 'input' in argv && typeof argv.input === 'string'; + + if (!hasSamplesHash && !hasSamplesFile) { + throw new Error('Either --input or --hash must be supplied'); + } + if (hasSamplesHash && hasSamplesFile) { + throw new Error('Only one of --input or --hash can be supplied'); + } + + const profile: ProfileSource = hasSamplesHash + ? { type: 'HASH', hash: argv.hash } + : { type: 'FILE', file: argv.input }; + + return { + profile, + outputProfilePath: argv['output-profile'], + outputJsonPath: argv['output-json'], + }; +} + +if (!module.parent) { + try { + const options = makeOptionsFromArgv(process.argv); + run(options).catch((err) => { + throw err; + }); + } catch (e) { + console.error(e); + process.exit(1); + } +} diff --git a/src/node-tools/compare-benchmark-stats.ts b/src/node-tools/compare-benchmark-stats.ts new file mode 100644 index 0000000000..5f9416ec5e --- /dev/null +++ b/src/node-tools/compare-benchmark-stats.ts @@ -0,0 +1,239 @@ +/** + * CLI entry point for compare-benchmark-stats. + * See compare-benchmark-stats.ts for the browser-safe library logic. + */ + +import fs from 'fs'; +import minimist from 'minimist'; +import type { ProfileBenchmarkStats } from 'firefox-profiler/profile-logic/benchmark/extract-benchmark-stats'; +import { + compareBuckets, + compareIterationTotals, + suiteIterationTotals, +} from 'firefox-profiler/profile-logic/benchmark/compare-benchmark-stats'; +import type { + BucketComparison, + ScoreComparison, +} from 'firefox-profiler/profile-logic/benchmark/compare-benchmark-stats'; + +// --------------------------------------------------------------------------- +// Formatting +// --------------------------------------------------------------------------- + +function formatChange(rel: number): string { + if (!isFinite(rel)) { + return rel > 0 ? 'appeared' : 'disappeared'; + } + const pct = (rel * 100).toFixed(2); + return rel >= 0 ? `+${pct}%` : `${pct}%`; +} + +function printScoreAndSubtests( + overall: ScoreComparison, + suites: ScoreComparison[] +) { + const COL = 45; + const overallAbsDiff = overall.newMean - overall.baseMean; + const overallAbsStr = + (overallAbsDiff >= 0 ? '+' : '') + overallAbsDiff.toFixed(2); + console.log( + `${'Score'.padEnd(COL)} ${'base mean'.padStart(10)} ${'new mean'.padStart(10)} ${'Δ abs'.padStart(10)} ${'Δ%'.padStart(10)} ${'effect'.padStart(10)} ${'confidence'.padStart(12)}` + ); + console.log('-'.repeat(COL + 64)); + console.log( + `${'Overall (geomean-normalised)'.padEnd(COL)} ${overall.baseMean.toFixed(2).padStart(10)} ${overall.newMean.toFixed(2).padStart(10)} ${overallAbsStr.padStart(10)} ${formatChange(overall.relChange).padStart(10)} ${overall.effectSize.padStart(10)} ${overall.confidence.padStart(12)}` + ); + console.log(''); + for (const s of suites) { + const absDiff = s.newMean - s.baseMean; + const absDiffStr = (absDiff >= 0 ? '+' : '') + absDiff.toFixed(2); + const label = + s.label.length > COL - 2 ? s.label.slice(0, COL - 5) + '...' : s.label; + console.log( + `${' ' + label.padEnd(COL - 2)} ${s.baseMean.toFixed(2).padStart(10)} ${s.newMean.toFixed(2).padStart(10)} ${absDiffStr.padStart(10)} ${formatChange(s.relChange).padStart(10)} ${s.effectSize.padStart(10)} ${s.confidence.padStart(12)}` + ); + } +} + +function printBucketResults( + label: string, + comparisons: BucketComparison[], + topN: number | null +) { + const significant = comparisons + .filter((c) => c.confidence !== 'LOW') + .sort( + (a, b) => + Math.abs(b.newMean - b.baseMean) - Math.abs(a.newMean - a.baseMean) + ); + + if (significant.length === 0) { + console.log(`\n[${label}] No significant bucket changes.`); + return; + } + + const shown = topN !== null ? significant.slice(0, topN) : significant; + console.log( + `\n[${label}] ${significant.length} significant buckets` + + (topN !== null && significant.length > topN + ? `, showing top ${topN} by absolute impact:` + : ':') + ); + console.log( + `${'Bucket name'.padEnd(60)} ${'base mean'.padStart(10)} ${'new mean'.padStart(10)} ${'Δ abs'.padStart(10)} ${'Δ%'.padStart(10)} ${'effect'.padStart(10)} ${'confidence'.padStart(12)}` + ); + console.log('-'.repeat(125)); + for (const c of shown) { + const name = + c.bucketName.length > 59 + ? c.bucketName.slice(0, 56) + '...' + : c.bucketName; + const absDiff = c.newMean - c.baseMean; + const absDiffStr = (absDiff >= 0 ? '+' : '') + absDiff.toFixed(2); + console.log( + `${name.padEnd(60)} ${c.baseMean.toFixed(2).padStart(10)} ${c.newMean.toFixed(2).padStart(10)} ${absDiffStr.padStart(10)} ${formatChange(c.relChange).padStart(10)} ${c.effectSize.padStart(10)} ${c.confidence.padStart(12)}` + ); + } +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (!argv.base || !argv.new) { + console.error( + 'Usage: compare-benchmark-stats --base --new \n' + + ' [--suite ] [--global] [--top 100] [--all] [--no-appeared]' + ); + process.exit(1); + } + + const topN: number | null = argv.all ? null : (argv.top ?? 100); + const suiteFilter: string | undefined = argv.suite; + const showGlobal: boolean = !suiteFilter || argv.global; + // minimist turns --no-appeared into { appeared: false } + const excludeAppearedDisappeared: boolean = argv.appeared === false; + + const base: ProfileBenchmarkStats = JSON.parse( + fs.readFileSync(argv.base, 'utf8') + ); + const newStats: ProfileBenchmarkStats = JSON.parse( + fs.readFileSync(argv.new, 'utf8') + ); + + // bucketFuncs was added later; older stats files don't include it. The CLI + // doesn't need real func indices (no flame graph here), so fill with -1. + if (!base.bucketFuncs) { + base.bucketFuncs = new Array(base.bucketNames.length).fill(-1); + } + if (!newStats.bucketFuncs) { + newStats.bucketFuncs = new Array(newStats.bucketNames.length).fill(-1); + } + // bucketKeys was added later too; fall back to bucketNames so older stats + // files still match using the prior name-based behaviour. + if (!base.bucketKeys) { + base.bucketKeys = base.bucketNames; + } + if (!newStats.bucketKeys) { + newStats.bucketKeys = newStats.bucketNames; + } + + const iterationCount = base.suites[0]?.iterationCount ?? 1; + + if (showGlobal) { + const baseGlobalIter = suiteIterationTotals( + base.globalBuckets, + iterationCount + ); + const newGlobalIter = suiteIterationTotals( + newStats.globalBuckets, + iterationCount + ); + const overallScore = compareIterationTotals( + 'Overall', + baseGlobalIter, + newGlobalIter + ); + + const suiteScores: ScoreComparison[] = []; + for (const baseSuite of base.suites) { + const newSuite = newStats.suites.find( + (s) => s.suiteName === baseSuite.suiteName + ); + const baseIter = suiteIterationTotals( + baseSuite.buckets, + baseSuite.iterationCount + ); + const newIter = newSuite + ? suiteIterationTotals(newSuite.buckets, newSuite.iterationCount) + : new Array(baseSuite.iterationCount).fill(0); + suiteScores.push( + compareIterationTotals(baseSuite.suiteName, baseIter, newIter) + ); + } + + console.log('\n--- Score and subtest totals ---\n'); + printScoreAndSubtests(overallScore, suiteScores); + + const globalComparisons = compareBuckets( + base.globalBuckets, + newStats.globalBuckets, + base.bucketNames, + newStats.bucketNames, + base.bucketFuncs, + newStats.bucketFuncs, + iterationCount, + excludeAppearedDisappeared, + base.bucketKeys, + newStats.bucketKeys + ); + printBucketResults('Global (geomean-normalised)', globalComparisons, topN); + } + + if (suiteFilter !== undefined) { + const matchingSuites = base.suites.filter((s) => + s.suiteName.toLowerCase().includes(suiteFilter.toLowerCase()) + ); + + if (matchingSuites.length === 0) { + console.error(`No suites matching "${suiteFilter}". Available suites:`); + for (const s of base.suites) { + console.error(` ${s.suiteName}`); + } + process.exit(1); + } + + for (const baseSuite of matchingSuites) { + const newSuite = newStats.suites.find( + (s) => s.suiteName === baseSuite.suiteName + ); + if (newSuite === undefined) { + console.warn( + `Suite "${baseSuite.suiteName}" not found in new stats, skipping.` + ); + continue; + } + const comparisons = compareBuckets( + baseSuite.buckets, + newSuite.buckets, + base.bucketNames, + newStats.bucketNames, + base.bucketFuncs, + newStats.bucketFuncs, + baseSuite.iterationCount, + excludeAppearedDisappeared, + base.bucketKeys, + newStats.bucketKeys + ); + printBucketResults(baseSuite.suiteName, comparisons, topN); + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/node-tools/extract-benchmark-stats.ts b/src/node-tools/extract-benchmark-stats.ts new file mode 100644 index 0000000000..c680084775 --- /dev/null +++ b/src/node-tools/extract-benchmark-stats.ts @@ -0,0 +1,39 @@ +/** + * CLI entry point for extract-benchmark-stats. + * See extract-benchmark-stats.ts for the browser-safe library logic. + */ + +import fs from 'fs'; +import minimist from 'minimist'; +import { unserializeProfileOfArbitraryFormat } from '../profile-logic/process-profile'; +import { extractBenchmarkStatsFromProfile } from 'firefox-profiler/profile-logic/benchmark/extract-benchmark-stats'; +import type { BenchmarkHarness } from 'firefox-profiler/profile-logic/benchmark/benchmark-stuff'; + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (!argv.input || !argv.output) { + console.error( + 'Usage: extract-benchmark-stats --input --output [--harness speedometer|jetstream]' + ); + process.exit(1); + } + + const harness: BenchmarkHarness = argv.harness ?? 'speedometer'; + const uint8Array = fs.readFileSync(argv.input, null); + const profile = await unserializeProfileOfArbitraryFormat(uint8Array.buffer); + const stats = extractBenchmarkStatsFromProfile(profile, harness); + + fs.writeFileSync(argv.output, JSON.stringify(stats)); + console.log( + `Wrote ${stats.suites.length} suites, ` + + `${stats.globalBuckets.length} global buckets, ` + + `${stats.suites.reduce((s, su) => s + su.buckets.length, 0)} suite-bucket pairs ` + + `to ${argv.output}` + ); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/node-tools/profile-insert-labels.ts b/src/node-tools/profile-insert-labels.ts new file mode 100644 index 0000000000..2a64e5e8da --- /dev/null +++ b/src/node-tools/profile-insert-labels.ts @@ -0,0 +1,208 @@ +/** + * Merge two existing profiles, taking the samples from the first profile and + * the markers from the second profile. + * + * This was useful during early 2025 when the Mozilla Performance team was + * doing a lot of Android startup profiling: + * + * - The "samples" profile would be collected using simpleperf and converted + * with samply import. + * - The "markers" profile would be collected using the Gecko profiler. + * + * To use this script, it first needs to be built: + * yarn build-node-tools + * + * Then it can be run from the `node-tools-dist` directory: + * node node-tools-dist/profile-insert-labels.js --labels src/node-tools/profile-insert-labels/known-functions.toml --hash w1spyw917hgfw56x5jzfs27q89dkphhqqzw2nag --output-file ~/Downloads/labeled-profile.json.gz + * + * For example: + * yarn build-node-tools && node node-tools-dist/profile-insert-labels.js --labels src/node-tools/profile-insert-labels/known-functions.toml --hash w1spyw917hgfw56x5jzfs27q89dkphhqqzw2nag --output-file ~/Downloads/labeled-profile.json.gz + * + */ + +import fs from 'fs'; +import minimist from 'minimist'; +import { parse as parseToml } from 'smol-toml'; + +import { unserializeProfileOfArbitraryFormat } from 'firefox-profiler/profile-logic/process-profile'; +import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants'; + +import type { Profile } from 'firefox-profiler/types/profile'; +import { compress } from 'firefox-profiler/utils/gz'; +import { insertStackLabels } from 'firefox-profiler/profile-logic/insert-stack-labels'; + +type ProfileSource = + | { + type: 'HASH'; + hash: string; + } + | { + type: 'FILE'; + file: string; + }; + +interface CliOptions { + profile: ProfileSource; + labelsFile: string; + outputFile: string; +} + +export function getProfileUrlForHash(hash: string): string { + // See https://cloud.google.com/storage/docs/access-public-data + // The URL is https://storage.googleapis.com//. + // https://.storage.googleapis.com/ seems to also work but + // is not documented nowadays. + + // By convention, "profile-store" is the name of our bucket, and the file path + // is the hash we receive in the URL. + return `https://storage.googleapis.com/${GOOGLE_STORAGE_BUCKET}/${hash}`; +} + +async function fetchProfileWithHash(hash: string): Promise { + const response = await fetch(getProfileUrlForHash(hash)); + const serializedProfile = await response.json(); + return unserializeProfileOfArbitraryFormat(serializedProfile); +} + +async function loadProfileFromFile(path: string): Promise { + const uint8Array = fs.readFileSync(path, null); + return unserializeProfileOfArbitraryFormat(uint8Array.buffer); +} + +async function loadProfile(source: ProfileSource): Promise { + switch (source.type) { + case 'HASH': + return fetchProfileWithHash(source.hash); + case 'FILE': + return loadProfileFromFile(source.file); + default: + return source; + } +} + +interface Template { + name: string; + patterns: string[]; +} + +interface BucketConfig { + name: string; + funcPrefixes?: string[]; + apply?: Array<{ template: string; [key: string]: string }>; +} + +export function applyModifier( + value: string, + modifier: string | undefined +): string { + switch (modifier) { + case 'pascal': + return value.charAt(0).toUpperCase() + value.slice(1); + case 'snake': + return value + .replace(/([a-z])([A-Z])/g, '$1_$2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') + .toLowerCase(); + case undefined: + return value; + default: + throw new Error(`Unknown template modifier: ${modifier}`); + } +} + +export function expandPattern( + pattern: string, + vars: Record +): string { + return pattern.replace( + /\{(\w+)(?::(\w+))?\}/g, + (_match, name: string, modifier: string | undefined) => { + if (!(name in vars)) { + throw new Error(`Template variable "${name}" not provided`); + } + return applyModifier(vars[name], modifier); + } + ); +} + +export function resolveTemplates( + bucketConfigs: BucketConfig[], + templates: Template[] +): Array<{ name: string; funcPrefixes: string[] }> { + const templateMap = new Map(templates.map((t) => [t.name, t])); + return bucketConfigs.map((bucket) => { + const funcPrefixes = [...(bucket.funcPrefixes ?? [])]; + for (const { template: templateName, ...vars } of bucket.apply ?? []) { + const template = templateMap.get(templateName); + if (!template) { + throw new Error(`Unknown template: "${templateName}"`); + } + for (const pattern of template.patterns) { + funcPrefixes.push(expandPattern(pattern, vars)); + } + } + return { name: bucket.name, funcPrefixes }; + }); +} + +export async function run(options: CliOptions) { + const tomlText = fs.readFileSync(options.labelsFile, 'utf8'); + const { buckets: bucketConfigs, templates = [] } = parseToml( + tomlText + ) as unknown as { + buckets: BucketConfig[]; + templates?: Template[]; + }; + const buckets = resolveTemplates(bucketConfigs, templates); + const oldProfile: Profile = await loadProfile(options.profile); + const profile: Profile = insertStackLabels(oldProfile, buckets); + + if (options.outputFile.endsWith('.gz')) { + fs.writeFileSync( + options.outputFile, + await compress(JSON.stringify(profile)) + ); + } else { + fs.writeFileSync(options.outputFile, JSON.stringify(profile)); + } +} + +export function makeOptionsFromArgv(processArgv: string[]): CliOptions { + const argv = minimist(processArgv.slice(2)); + + const hasSamplesHash = 'hash' in argv && typeof argv.hash === 'string'; + const hasSamplesFile = 'input' in argv && typeof argv.input === 'string'; + + if (!hasSamplesHash && !hasSamplesFile) { + throw new Error('Either --input or --hash must be supplied'); + } + if (hasSamplesHash && hasSamplesFile) { + throw new Error('Only one of --input or --hash can be supplied'); + } + + const profile: ProfileSource = hasSamplesHash + ? { type: 'HASH', hash: argv.hash } + : { type: 'FILE', file: argv.input }; + + if (!('labels' in argv) || typeof argv.labels !== 'string') { + throw new Error('--labels must be supplied'); + } + + return { + profile, + labelsFile: argv.labels, + outputFile: argv['output-file'], + }; +} + +if (!module.parent) { + try { + const options = makeOptionsFromArgv(process.argv); + run(options).catch((err) => { + throw err; + }); + } catch (e) { + console.error(e); + process.exit(1); + } +} diff --git a/src/node-tools/profiler-edit.ts b/src/node-tools/profiler-edit.ts index d2753957d1..82210ced2f 100644 --- a/src/node-tools/profiler-edit.ts +++ b/src/node-tools/profiler-edit.ts @@ -162,8 +162,15 @@ function canonicalizeJsLocations(profile: Profile): Profile { // The filename may contain colons (URLs), so we rely on greedy matching // to anchor `:line:col` at the very end of the string. - const parenRegex = /^(.+) \((.+):(\d+):(\d+)\)$/; - const plainRegex = /^(.+) (.+):(\d+):(\d+)$/; + // + // The function name is matched with `.*` (not `.+`) because V8 reports an + // empty name for anonymous functions: samply then formats them as + // ` filename:line:col` (leading space, empty name) or ` :line:col` (empty + // name and empty filename). Requiring at least one character for the name + // would leave those funcs uncanonicalized, with junk names like ` :1:20`. + // The filename is matched with `.*` for the same reason. + const parenRegex = /^(.*) \((.*):(\d+):(\d+)\)$/; + const plainRegex = /^(.*) (.*):(\d+):(\d+)$/; let canonicalized = 0; for (let i = 0; i < funcTable.length; i++) { diff --git a/src/node-tools/symbolicator-cli.ts b/src/node-tools/symbolicator-cli.ts new file mode 100644 index 0000000000..9817cff688 --- /dev/null +++ b/src/node-tools/symbolicator-cli.ts @@ -0,0 +1,151 @@ +/* + * This implements a simple CLI to symbolicate profiles captured by the profiler + * or by samply. + * + * To use it it first needs to be built: + * yarn build-symbolicator-cli + * + * Then it can be run from the `dist` directory: + * node dist/symbolicator-cli.js --input --output --server + * + * For example: + * node dist/symbolicator-cli.js --input samply-profile.json --output profile-symbolicated.json --server http://localhost:3000 + * + */ + +import fs from 'fs'; +import minimist from 'minimist'; + +import { unserializeProfileOfArbitraryFormat } from 'firefox-profiler/profile-logic/process-profile'; +import { SymbolStore } from 'firefox-profiler/profile-logic/symbol-store'; +import { + symbolicateProfile, + applySymbolicationSteps, +} from 'firefox-profiler/profile-logic/symbolication'; +import type { SymbolicationStepInfo } from 'firefox-profiler/profile-logic/symbolication'; +import * as MozillaSymbolicationAPI from 'firefox-profiler/profile-logic/mozilla-symbolication-api'; + +export interface CliOptions { + input: string; + output: string; + server: string; +} + +export async function run(options: CliOptions) { + console.log(`Loading profile from ${options.input}`); + + // Read the raw bytes from the file. It might be a JSON file, but it could also + // be a binary file, e.g. a .json.gz file, or any of the binary formats supported + // by our importers. + const bytes = fs.readFileSync(options.input, null); + + // Load the profile. + const profile = await unserializeProfileOfArbitraryFormat(bytes); + if (profile === undefined) { + throw new Error('Unable to parse the profile.'); + } + + /** + * SymbolStore implementation which just forwards everything to the symbol server in + * MozillaSymbolicationAPI format. No support for getting symbols from 'the browser' as + * there is no browser in this context. + */ + const symbolStore = new SymbolStore({ + requestSymbolsFromServer: async (requests) => { + for (const { lib } of requests) { + console.log(` Loading symbols for ${lib.debugName}`); + } + try { + return await MozillaSymbolicationAPI.requestSymbols( + 'symbol server', + requests, + async (path, json) => { + const response = await fetch(options.server + path, { + body: json, + method: 'POST', + }); + return response.json(); + } + ); + } catch (e) { + throw new Error( + `There was a problem with the symbolication API request to the symbol server: ${e.message}` + ); + } + }, + + requestSymbolsFromBrowser: async () => { + return []; + }, + + requestSymbolsViaSymbolTableFromBrowser: async () => { + throw new Error('Not supported in this context'); + }, + }); + + console.log('Symbolicating...'); + + const symbolicationSteps: SymbolicationStepInfo[] = []; + await symbolicateProfile( + profile, + symbolStore, + (symbolicationStepInfo: SymbolicationStepInfo) => { + symbolicationSteps.push(symbolicationStepInfo); + } + ); + + console.log('Applying collected symbolication steps...'); + + const { shared, threads } = applySymbolicationSteps( + profile.threads, + profile.shared, + symbolicationSteps + ); + profile.shared = shared; + profile.threads = threads; + profile.meta.symbolicated = true; + + console.log(`Saving profile to ${options.output}`); + fs.writeFileSync(options.output, JSON.stringify(profile)); + console.log('Finished.'); +} + +export function makeOptionsFromArgv(processArgv: string[]): CliOptions { + const argv = minimist(processArgv.slice(2)); + + if (!('input' in argv && typeof argv.input === 'string')) { + throw new Error( + 'Argument --input must be supplied with the path to the input profile' + ); + } + + if (!('output' in argv && typeof argv.output === 'string')) { + throw new Error( + 'Argument --output must be supplied with the path to the output profile' + ); + } + + if (!('server' in argv && typeof argv.server === 'string')) { + throw new Error( + 'Argument --server must be supplied with the URI of the symbol server endpoint' + ); + } + + return { + input: argv.input, + output: argv.output, + server: argv.server, + }; +} + +if (!module.parent) { + try { + const options = makeOptionsFromArgv(process.argv); + run(options).catch((err) => { + throw err; + }); + } catch (e) { + console.error(e); + process.exit(1); + } +} diff --git a/src/profile-logic/benchmark/benchmark-stuff.ts b/src/profile-logic/benchmark/benchmark-stuff.ts new file mode 100644 index 0000000000..812ccd65f1 --- /dev/null +++ b/src/profile-logic/benchmark/benchmark-stuff.ts @@ -0,0 +1,591 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { + Marker, + Profile, + RawProfileSharedData, + RawThread, + StartEndRange, +} from 'firefox-profiler/types'; +import type { StringTable } from 'firefox-profiler/utils/string-table'; +import { ensureExists } from 'firefox-profiler/utils/types'; +// import { computeBucketStats } from 'firefox-profiler/utils/stats'; + +export type BenchmarkHarness = 'speedometer' | 'jetstream'; + +export type BenchmarkInfo = { + suiteNameIfSingleSuite: string | null; + threadIndex: number; + getMeasuredTimeRanges: ( + markers: any, + stringTable: any + ) => StartEndRange[] | null; + getMarkersPerSuite: (markers: any, stringTable: any) => Map; +}; + +export function getBenchmarkInfo( + profile: Profile, + benchmarkHarness: BenchmarkHarness +): BenchmarkInfo { + if (benchmarkHarness === 'speedometer') { + return getSpeedometerBenchmarkInfo(profile); + } + if (benchmarkHarness === 'jetstream') { + return getJetStreamBenchmarkInfo(profile); + } + throw new Error(`Unknown benchmarkHarness: ${benchmarkHarness}`); +} + +export function getSpeedometerBenchmarkInfo(profile: Profile): BenchmarkInfo { + const { threads, shared } = profile; + for (let threadIndex = 0; threadIndex < threads.length; threadIndex++) { + const thread = threads[threadIndex]; + const suiteNames = speedometerSuiteNamesOnThread(thread, shared); + if (suiteNames.length !== 0) { + const suiteNameIfSingleSuite = + suiteNames.length === 1 ? suiteNames[0] : null; + return { + suiteNameIfSingleSuite, + threadIndex, + getMarkersPerSuite: getSpeedometerMarkersPerSuite, + getMeasuredTimeRanges: getSpeedometerMeasuredTimeRanges, + }; + } + } + throw new Error( + "Could not find a thread with markers that start with 'suite-'" + ); +} + +export function getSpeedometerMarkersPerSuite( + markers: Marker[], + stringTable: StringTable +): Map { + const markersPerSuiteName: Map = new Map(); + for (const m of markers) { + if ( + (m.name === 'UserTiming' || m.name === 'SimpleMarker') && + m.end !== null && + m.data && + 'name' in m.data && + m.data.name + ) { + const nameOrNameIndex = m.data.name; + let markerName = ''; + if (typeof nameOrNameIndex === 'number') { + markerName = stringTable.getString(nameOrNameIndex); + } + if (markerName.startsWith('suite-') && !markerName.endsWith('-prepare')) { + const suiteName = markerName.slice('suite-'.length); + let markersForThisSuite = markersPerSuiteName.get(suiteName); + if (markersForThisSuite === undefined) { + markersForThisSuite = []; + markersPerSuiteName.set(suiteName, markersForThisSuite); + } + markersForThisSuite.push(m); + } + } + } + return markersPerSuiteName; +} + +export function getSpeedometerMeasuredTimeRanges( + markers: Marker[], + stringTable: StringTable +): StartEndRange[] | null { + const ranges = []; + for (const m of markers) { + if ( + (m.name === 'UserTiming' || m.name === 'SimpleMarker') && + m.end !== null && + m.data && + 'name' in m.data && + m.data.name + ) { + const nameOrNameIndex = m.data.name; + let markerName = ''; + if (typeof nameOrNameIndex === 'number') { + markerName = stringTable.getString(nameOrNameIndex); + } + if (markerName.includes('-sync') || markerName.includes('-async')) { + ranges.push({ start: m.start, end: m.end }); + } + } + } + return ranges; +} + +export function getJetStreamBenchmarkInfo(profile: Profile): BenchmarkInfo { + const { threads, shared } = profile; + for (let threadIndex = 0; threadIndex < threads.length; threadIndex++) { + const thread = threads[threadIndex]; + const suiteNames = jetstreamSuiteNamesOnThread(thread, shared); + if (suiteNames.length !== 0) { + const suiteNameIfSingleSuite = + suiteNames.length === 1 ? suiteNames[0] : null; + return { + suiteNameIfSingleSuite, + threadIndex, + getMarkersPerSuite: getJetstreamMarkersPerSuite, + getMeasuredTimeRanges: () => null, + }; + } + } + throw new Error( + "Could not find a thread with markers that include '-iteration-'" + ); +} + +export function jetstreamSuiteNamesOnThread( + rawThread: RawThread, + shared: RawProfileSharedData +): string[] { + const names: Set = new Set(); + const { markers } = rawThread; + const { stringArray } = shared; + let userTimingMarkerNameStringIndex = stringArray.indexOf('UserTiming'); + const simpleMarkerNameStringIndex = stringArray.indexOf('SimpleMarker'); + if ( + userTimingMarkerNameStringIndex === -1 || + (simpleMarkerNameStringIndex !== -1 && + simpleMarkerNameStringIndex < userTimingMarkerNameStringIndex) + ) { + userTimingMarkerNameStringIndex = simpleMarkerNameStringIndex; + } + for (let i = 0; i < markers.length; i++) { + if (markers.phase[i] === 0) { + continue; + } + if (markers.name[i] !== userTimingMarkerNameStringIndex) { + continue; + } + const data = markers.data[i]; + if (!data || !('name' in data) || !data.name) { + continue; + } + + const markerName = + typeof data.name === 'string' ? data.name : stringArray[data.name]; + const match = markerName.match(/^(.*?)-iteration-[0-9]+$/); + if (match !== null) { + names.add(match[1]); + } + } + return [...names]; +} +export function getJetstreamMarkersPerSuite( + markers: Marker[], + stringTable: StringTable +): Map { + const markersPerSuiteName: Map = new Map(); + for (const m of markers) { + if ( + (m.name === 'UserTiming' || m.name === 'SimpleMarker') && + m.end !== null && + m.data && + 'name' in m.data && + m.data.name + ) { + const data = m.data; + const markerName = + typeof data.name === 'string' + ? data.name + : stringTable.getString(data.name); + const match = markerName.match(/^(.*?)-iteration-[0-9]+$/); + if (match !== null) { + const suiteName = match[1]; + let markersForThisSuite = markersPerSuiteName.get(suiteName); + if (markersForThisSuite === undefined) { + markersForThisSuite = []; + markersPerSuiteName.set(suiteName, markersForThisSuite); + } + markersForThisSuite.push(m); + } + } + } + return markersPerSuiteName; +} + +export function speedometerSuiteNamesOnThread( + rawThread: RawThread, + shared: RawProfileSharedData +): string[] { + const names: Set = new Set(); + const { markers } = rawThread; + const { stringArray } = shared; + let userTimingMarkerNameStringIndex = stringArray.indexOf('UserTiming'); + const simpleMarkerNameStringIndex = stringArray.indexOf('SimpleMarker'); + if ( + userTimingMarkerNameStringIndex === -1 || + (simpleMarkerNameStringIndex !== -1 && + simpleMarkerNameStringIndex < userTimingMarkerNameStringIndex) + ) { + userTimingMarkerNameStringIndex = simpleMarkerNameStringIndex; + } + for (let i = 0; i < markers.length; i++) { + if (markers.phase[i] === 0) { + continue; + } + if (markers.name[i] !== userTimingMarkerNameStringIndex) { + continue; + } + const data = markers.data[i]; + if (!data || !('name' in data) || !data.name) { + continue; + } + + const markerName = + typeof data.name === 'string' ? data.name : stringArray[data.name]; + if (markerName.startsWith('suite-')) { + const suiteName = ensureExists( + markerName.match(/^suite-(.*?)(-prepare)?$/) + )[1]; + names.add(suiteName); + } + } + return [...names]; +} + +export function threadHasMatchingMarkers( + rawThread: RawThread, + shared: RawProfileSharedData, + markerFilter: string +) { + const { markers } = rawThread; + const { stringArray } = shared; + let userTimingMarkerNameStringIndex = stringArray.indexOf('UserTiming'); + const simpleMarkerNameStringIndex = stringArray.indexOf('SimpleMarker'); + if ( + userTimingMarkerNameStringIndex === -1 || + (simpleMarkerNameStringIndex !== -1 && + simpleMarkerNameStringIndex < userTimingMarkerNameStringIndex) + ) { + userTimingMarkerNameStringIndex = simpleMarkerNameStringIndex; + } + for (let i = 0; i < markers.length; i++) { + if (markers.phase[i] === 0) { + continue; + } + if (markers.name[i] !== userTimingMarkerNameStringIndex) { + continue; + } + const data = markers.data[i]; + if (!data || !('name' in data) || !data.name) { + continue; + } + + const markerName = + typeof data.name === 'string' ? data.name : stringArray[data.name]; + // Check if the `markerFilter` string is contained in the marker name. + // TODO: Let the front-end do the matching, so that all the various search + // syntaxes work correctly (comma separated multi search, matching by field, etc) + if (markerName.includes(markerFilter)) { + return true; + } + } + return false; +} + +export type SamplesTableForThisStuff = { + time: Float64Array; + weight: Float64Array; + bucketIndex: Int32Array; + bucketCount: number; + length: number; +}; + +export type BenchmarkScores = { + geomean: number; + allSuiteScores: SuiteScores[]; + factorPerSuite: number[]; +}; + +export type IterationMarkersAndMeasuredSamples = { + markersPerSuite: Array<[string, Marker[]]>; + measuredSamples: SamplesTableForThisStuff; +}; + +/** + * Compute per-suite sample weights, filtered to (already-applied measured time + * ranges) ∩ (this suite's iteration marker ranges). The input weights are + * `measuredSamples.weight` (i.e. weights with -async/-sync filtering and + * ignored-bucket zeroing already applied). The output zeroes out any weight + * outside this suite's iteration markers, so the flame graph for this suite + * reflects exactly the same samples that the suite's score counts. + * + * Iteration markers are assumed to be sorted by start time and non-overlapping + * (matching the assumption in `computeSuiteScores`). + */ +export function computeSuiteFilteredSampleWeights( + measuredSampleWeights: Float64Array, + sampleTimes: Float64Array, + iterationMarkers: Marker[] +): Float64Array { + const filtered = measuredSampleWeights.slice(); + const ranges: StartEndRange[] = []; + for (const m of iterationMarkers) { + if (m.end !== null) { + ranges.push({ start: m.start, end: m.end }); + } + } + zeroWeightsOutsideRanges(filtered, sampleTimes, ranges); + return filtered; +} + +export function computeIterationMarkersAndMeasuredSamples( + benchmarkInfo: BenchmarkInfo, + filteredMarkers: Marker[], + samples: SamplesTableForThisStuff, + stringTable: StringTable, + bucketsToIgnore: number[] +): IterationMarkersAndMeasuredSamples { + const measuredTimeRanges = benchmarkInfo.getMeasuredTimeRanges( + filteredMarkers, + stringTable + ); + const measuredWeights = samples.weight.slice(); + if (measuredTimeRanges !== null) { + zeroWeightsOutsideRanges(measuredWeights, samples.time, measuredTimeRanges); + } + zeroWeightsForBuckets(measuredWeights, samples.bucketIndex, bucketsToIgnore); + const measuredSamples = { + ...samples, + weight: measuredWeights, + }; + const markersPerSuite = [ + ...benchmarkInfo.getMarkersPerSuite(filteredMarkers, stringTable), + ]; + return { markersPerSuite, measuredSamples }; +} + +export function computeBenchmarkScores( + iterationMarkersAndMeasuredSamples: IterationMarkersAndMeasuredSamples +): BenchmarkScores { + const { markersPerSuite, measuredSamples } = + iterationMarkersAndMeasuredSamples; + const allSuiteScores = markersPerSuite.map(([suiteName, iterationMarkers]) => + computeSuiteScores(suiteName, iterationMarkers, measuredSamples) + ); + const geomean = computeGeomean(allSuiteScores.map((s) => s.total)); + const factorPerSuite = allSuiteScores.map( + (suiteScores) => geomean / suiteScores.total + ); + return { geomean, allSuiteScores, factorPerSuite }; +} + +function computeGeomean(values: number[]): number { + let product = 1; + for (const value of values) { + product *= value; + } + return Math.pow(product, 1 / values.length); +} + +export function zeroWeightsOutsideRanges( + sampleWeights: Float64Array, + sampleTimes: Float64Array, + nonZeroRanges: StartEndRange[] +) { + let sampleIndex = 0; + const sampleCount = sampleTimes.length; + for (let rangeIndex = 0; rangeIndex < nonZeroRanges.length; rangeIndex++) { + const range = nonZeroRanges[rangeIndex]; + const rangeStart = range.start; + const rangeEnd = range.end; + + // Zero out sample weights before the range. + for (; sampleIndex < sampleCount; sampleIndex++) { + if (sampleTimes[sampleIndex] >= rangeStart) { + break; + } + sampleWeights[sampleIndex] = 0; + } + + // Skip over samples inside the range. + for (; sampleIndex < sampleCount; sampleIndex++) { + if (sampleTimes[sampleIndex] >= rangeEnd) { + break; + } + } + } + + // Zero out sample weights at the end + for (; sampleIndex < sampleCount; sampleIndex++) { + sampleWeights[sampleIndex] = 0; + } +} + +function zeroWeightsForBuckets( + sampleWeights: Float64Array, + sampleBuckets: Int32Array, + bucketsToZeroOut: number[] +) { + for (let i = 0; i < sampleWeights.length; i++) { + if (bucketsToZeroOut.includes(sampleBuckets[i])) { + sampleWeights[i] = 0; + } + } +} + +export function computeSampleWeightsWithSuiteFactorsApplied( + iterationMarkersAndMeasuredSamples: IterationMarkersAndMeasuredSamples, + suiteFactors: Array +): Float64Array { + const { markersPerSuite, measuredSamples: samples } = + iterationMarkersAndMeasuredSamples; + const newWeights = samples.weight.slice(); + for (let i = 0; i < markersPerSuite.length; i++) { + const [_suiteName, iterationMarkers] = markersPerSuite[i]; + const factor = suiteFactors[i]; + applySuiteFactor(samples.time, newWeights, iterationMarkers, factor); + } + return newWeights; +} + +function applySuiteFactor( + sampleTimes: Float64Array, + sampleWeights: Float64Array, + iterationMarkers: Marker[], + factor: number +) { + let sampleIndex = 0; + const sampleCount = sampleWeights.length; + for ( + let iterationIndex = 0; + iterationIndex < iterationMarkers.length; + iterationIndex++ + ) { + const marker = iterationMarkers[iterationIndex]; + const rangeStart = marker.start; + const rangeEnd = ensureExists(marker.end); + + // Skip over samples before the range. + for (; sampleIndex < sampleCount; sampleIndex++) { + if (sampleTimes[sampleIndex] >= rangeStart) { + break; + } + } + + // Process samples inside the range. + for (; sampleIndex < sampleCount; sampleIndex++) { + if (sampleTimes[sampleIndex] >= rangeEnd) { + break; + } + sampleWeights[sampleIndex] *= factor; + } + } +} + +function computeBucketStats( + bucketIterationTotals: Float64Array, + bucketCount: number, + iterationCount: number +): AllBucketStats { + const bucketMeans = new Float64Array(bucketCount); + const bucketVariances = new Float64Array(bucketCount); + for (let bucketIndex = 0; bucketIndex < bucketCount; bucketIndex++) { + const startIndex = bucketIndex * iterationCount; + let totalSum = 0; + for ( + let iterationIndex = 0; + iterationIndex < iterationCount; + iterationIndex++ + ) { + totalSum += bucketIterationTotals[startIndex + iterationIndex]; + } + const mean = totalSum / iterationCount; + let squareDiffSum = 0; + for ( + let iterationIndex = 0; + iterationIndex < iterationCount; + iterationIndex++ + ) { + const diff = bucketIterationTotals[startIndex + iterationIndex] - mean; + const squareDiff = diff * diff; + squareDiffSum += squareDiff; + } + const variance = squareDiffSum / (iterationCount - 1); + bucketMeans[bucketIndex] = mean; + bucketVariances[bucketIndex] = variance; + } + return { iterationCount, bucketMeans, bucketVariances }; +} + +export type AllBucketStats = { + iterationCount: number; + bucketMeans: Float64Array; + bucketVariances: Float64Array; +}; + +export type SuiteScores = { + suiteName: string; + total: number; + bucketTotals: Float64Array; + bucketIterationTotals: Float64Array; + bucketStats: AllBucketStats | null; +}; + +function computeSuiteScores( + suiteName: string, + iterationMarkers: Marker[], + samples: SamplesTableForThisStuff +): SuiteScores { + const iterationCount = iterationMarkers.length; + const bucketCount = samples.bucketCount; + const bucketTotals = new Float64Array(bucketCount); + const bucketIterationTotals = new Float64Array(bucketCount * iterationCount); + let total = 0; + + let sampleIndex = 0; + const sampleCount = samples.length; + for ( + let iterationIndex = 0; + iterationIndex < iterationMarkers.length; + iterationIndex++ + ) { + const marker = iterationMarkers[iterationIndex]; + const rangeStart = marker.start; + const rangeEnd = ensureExists(marker.end); + + // Skip over samples before the range. + for (; sampleIndex < sampleCount; sampleIndex++) { + if (samples.time[sampleIndex] >= rangeStart) { + break; + } + } + + // Process samples inside the range. + for (; sampleIndex < sampleCount; sampleIndex++) { + if (samples.time[sampleIndex] >= rangeEnd) { + break; + } + const bucketIndex = samples.bucketIndex[sampleIndex]; + if (bucketIndex === -1) { + continue; + } + + // Map this sample to its bucket and accumulate the weight. + const sampleWeight = samples.weight[sampleIndex]; + total += sampleWeight; + bucketTotals[bucketIndex] += sampleWeight; + bucketIterationTotals[bucketIndex * iterationCount + iterationIndex] += + sampleWeight; + } + } + + const bucketStats = computeBucketStats( + bucketIterationTotals, + bucketCount, + iterationCount + ); + + return { + suiteName, + total, + bucketTotals, + bucketIterationTotals, + bucketStats, + }; +} diff --git a/src/profile-logic/benchmark/bucket-flame-graph-data.ts b/src/profile-logic/benchmark/bucket-flame-graph-data.ts new file mode 100644 index 0000000000..08b402a80b --- /dev/null +++ b/src/profile-logic/benchmark/bucket-flame-graph-data.ts @@ -0,0 +1,359 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Pure helpers that compute everything needed to render a SelfWing-style + * flame graph (focusSelf with 'js' implementation filter) for one + * (Profile, funcIndex) pair, without going through the Redux store. + * + * Used by the benchmark-comparison page to expand a bucket row and show two + * flame graphs (base vs new). Each call here mirrors the chain of selectors + * in selectors/per-thread/stack-sample.ts (the "self wing" cluster) and + * selectors/per-thread/thread.tsx (`getThread`), but operates on a profile + * that is not the one currently loaded in Redux state. + */ + +import { + computeStackTableFromRawStackTable, + computeSamplesTableFromRawSamplesTable, + reserveFunctionsForCollapsedResources, + createThreadFromDerivedTables, + getCallNodeInfo, + getSampleIndexToCallNodeIndex, + getTimeRangeForThread, +} from '../profile-data'; +import * as Transforms from '../transforms'; +import * as CallTree from '../call-tree'; +import * as FlameGraph from '../flame-graph'; +import { computeReferenceCPUDeltaPerMs } from '../cpu'; +import { getDefaultCategories } from '../data-structures'; +import { StringTable } from '../../utils/string-table'; +import { base64StringToBytes } from '../../utils/base64'; +import { + correlateIPCMarkers, + deriveMarkersFromRawMarkerTable, +} from '../marker-data'; +import { + computeSuiteFilteredSampleWeights, + getBenchmarkInfo, + zeroWeightsOutsideRanges, +} from './benchmark-stuff'; +import type { BenchmarkHarness, BenchmarkInfo } from './benchmark-stuff'; + +import type { + Marker, + Thread, + Profile, + IndexIntoFuncTable, + IndexIntoCategoryList, + CategoryList, + StartEndRange, + WeightType, + SamplesLikeTable, + SampleCategoriesAndSubcategories, +} from '../../types'; +import type { CallNodeInfo } from '../call-node-info'; +import type { FlameGraphTiming } from '../flame-graph'; +import type { CallTree as CallTreeT } from '../call-tree'; + +export type BucketFlameGraphData = { + thread: Thread; + callNodeInfo: CallNodeInfo; + callTree: CallTreeT; + flameGraphTiming: FlameGraphTiming; + maxStackDepthPlusOne: number; + ctssSamples: SamplesLikeTable; + ctssSampleCategoriesAndSubcategories: SampleCategoriesAndSubcategories; + weightType: WeightType; + categories: CategoryList; + defaultCategory: IndexIntoCategoryList; + timeRange: StartEndRange; + interval: number; + /** Total weight of all samples in the focused thread. Used by callers to + * scale the flame-graph viewport so that 1 sample takes up the same pixel + * width across multiple flame graphs. */ + rootTotalSummary: number; +}; + +/** Categories list with fallback to defaults (matches selectors/profile.ts). */ +export function getCategoriesForProfile(profile: Profile): CategoryList { + return profile.meta.categories ?? getDefaultCategories(); +} + +/** Default category index — the "Other" / grey category. */ +export function getDefaultCategoryIndex( + categories: CategoryList +): IndexIntoCategoryList { + return categories.findIndex((c) => c.color === 'grey'); +} + +/** + * Build a derived `Thread` from `profile.threads[threadIndex]` without going + * through Redux. Equivalent to the `getThread` selector in + * selectors/per-thread/thread.tsx, minus the per-thread stuff that doesn't + * apply when there's no thread merging. + */ +export function buildDerivedThread( + profile: Profile, + threadIndex: number, + categories: CategoryList, + defaultCategory: IndexIntoCategoryList +): Thread { + const rawThread = profile.threads[threadIndex]; + const { shared, meta } = profile; + const stringTable = StringTable.withBackingArray( + shared.stringArray as string[] + ); + const stackTable = computeStackTableFromRawStackTable( + shared.stackTable, + shared.frameTable, + categories, + defaultCategory + ); + const { funcTable } = reserveFunctionsForCollapsedResources( + shared.funcTable, + shared.resourceTable + ); + const referenceCPUDeltaPerMs = computeReferenceCPUDeltaPerMs(profile); + const samples = computeSamplesTableFromRawSamplesTable( + rawThread.samples, + stackTable, + meta.sampleUnits, + referenceCPUDeltaPerMs, + defaultCategory + ); + const tracedValuesBuffer = rawThread.tracedValuesBuffer + ? base64StringToBytes(rawThread.tracedValuesBuffer) + : undefined; + return createThreadFromDerivedTables( + rawThread, + samples, + stackTable, + shared.frameTable, + funcTable, + shared.nativeSymbols, + shared.resourceTable, + stringTable, + shared.sources, + tracedValuesBuffer, + shared.sourceLocationTable + ); +} + +/** + * Compute everything needed to render one SelfWing-style flame graph for the + * given function in the given thread. Mirrors the `_getSelfWing*` selectors. + */ +export function computeBucketFlameGraphData( + profile: Profile, + thread: Thread, + funcIndex: IndexIntoFuncTable, + categories: CategoryList, + defaultCategory: IndexIntoCategoryList +): BucketFlameGraphData { + // 1. focusSelf with 'js' implementation filter — this is what "self wing" + // does in the call tree / function list. The 'js' filter matches the + // benchmark's bucketing logic in computeJsOnlySampleBuckets, so the flame + // graph reflects the same notion of "this bucket's time". + const selfWingThread = Transforms.focusSelf(thread, funcIndex, 'js'); + + // 2. Call-node info for the focused thread. + const callNodeInfo = getCallNodeInfo( + selfWingThread.stackTable, + selfWingThread.frameTable, + defaultCategory + ); + + // 3. CTSS samples (timing strategy → just thread.samples). + const ctssSamples = CallTree.extractSamplesLikeTable( + selfWingThread, + 'timing' + ); + + // 4. Map samples → call nodes. + const sampleIndexToCallNodeIndex = getSampleIndexToCallNodeIndex( + ctssSamples.stack, + callNodeInfo.getStackIndexToNonInvertedCallNodeIndex() + ); + + // 5. Per-callnode self-time + scaling totals. + const callNodeSelfAndSummary = CallTree.computeCallNodeSelfAndSummary( + ctssSamples, + sampleIndexToCallNodeIndex, + callNodeInfo.getCallNodeTable().length + ); + + // 6. Full timings. + const callTreeTimingsNonInverted = CallTree.computeCallTreeTimingsNonInverted( + callNodeInfo, + callNodeSelfAndSummary + ); + const callTreeTimings: CallTree.CallTreeTimings = { + type: 'NON_INVERTED', + timings: callTreeTimingsNonInverted, + }; + + // 7. Flame graph layout. + const flameGraphRows = FlameGraph.computeFlameGraphRows( + callNodeInfo.getCallNodeTable(), + selfWingThread.funcTable, + selfWingThread.stringTable + ); + const flameGraphTiming = FlameGraph.getFlameGraphTiming( + flameGraphRows, + callNodeInfo.getCallNodeTable(), + callTreeTimingsNonInverted + ); + + // 8. CallTree object (used by FlameGraph for tooltips and double-click). + const weightType: WeightType = ctssSamples.weightType ?? 'samples'; + const callTree = CallTree.getCallTree( + selfWingThread, + callNodeInfo, + categories, + ctssSamples, + callTreeTimings, + weightType + ); + + // 9. Per-sample categories. + const ctssSampleCategoriesAndSubcategories = + CallTree.computeUnfilteredCtssSampleCategoriesAndSubcategories( + selfWingThread, + ctssSamples, + defaultCategory + ); + + // Time range from the original (un-focused) thread's samples. The flame + // graph doesn't actually scrub by time, but ChartViewport requires a range. + const interval = profile.meta.interval; + const timeColumn = thread.samples.time; + const sampleCount = thread.samples.length; + const timeRange: StartEndRange = + sampleCount > 0 + ? { start: timeColumn[0], end: timeColumn[sampleCount - 1] + interval } + : { start: 0, end: interval }; + + return { + thread: selfWingThread, + callNodeInfo, + callTree, + flameGraphTiming, + maxStackDepthPlusOne: callNodeInfo.getCallNodeTable().maxDepth + 1, + ctssSamples, + ctssSampleCategoriesAndSubcategories, + weightType, + categories, + defaultCategory, + timeRange, + interval, + rootTotalSummary: callNodeSelfAndSummary.rootTotalSummary, + }; +} + +/** Per-profile prep data passed in from the viewer. The derived `thread` is + * expensive to build, so it's computed once at the viewer level and reused + * across every bucket the user expands. Also carries the benchmark marker + * info needed to lazily build per-suite filtered threads. */ +export type BucketProfileBundle = { + profile: Profile; + thread: Thread; + categories: CategoryList; + defaultCategory: IndexIntoCategoryList; + benchmarkInfo: BenchmarkInfo; + /** `thread.samples.time` as a Float64Array, for fast range filtering. */ + sampleTimes: Float64Array; + /** Sample weights with the global -async/-sync measured-time filter applied, + * matching the `measuredSamples.weight` used by score computation. */ + measuredSampleWeights: Float64Array; + /** Iteration markers per suite name. Sorted by start time, non-overlapping. */ + markersPerSuite: Map; +}; + +export function makeBucketProfileBundle( + profile: Profile, + benchmarkHarness: BenchmarkHarness +): BucketProfileBundle { + const categories = getCategoriesForProfile(profile); + const defaultCategory = getDefaultCategoryIndex(categories); + const benchmarkInfo = getBenchmarkInfo(profile, benchmarkHarness); + const thread = buildDerivedThread( + profile, + benchmarkInfo.threadIndex, + categories, + defaultCategory + ); + + const { shared } = profile; + const rawThread = profile.threads[benchmarkInfo.threadIndex]; + const stringTable = StringTable.withBackingArray(shared.stringArray); + const { markers: derivedMarkers } = deriveMarkersFromRawMarkerTable( + rawThread.markers, + shared.stringArray, + rawThread.tid, + getTimeRangeForThread(rawThread, profile.meta.interval), + correlateIPCMarkers(profile.threads, shared) + ); + + const sampleCount = thread.samples.length; + const sampleTimes = new Float64Array(thread.samples.time); + const measuredSampleWeights = thread.samples.weight + ? new Float64Array(thread.samples.weight) + : new Float64Array(sampleCount).fill(1); + const measuredTimeRanges = benchmarkInfo.getMeasuredTimeRanges( + derivedMarkers, + stringTable + ); + if (measuredTimeRanges !== null) { + zeroWeightsOutsideRanges( + measuredSampleWeights, + sampleTimes, + measuredTimeRanges + ); + } + + const markersPerSuite = benchmarkInfo.getMarkersPerSuite( + derivedMarkers, + stringTable + ); + + return { + profile, + thread, + categories, + defaultCategory, + benchmarkInfo, + sampleTimes, + measuredSampleWeights, + markersPerSuite, + }; +} + +/** + * Return a Thread that shares all tables with `bundle.thread` but has sample + * weights zeroed outside this suite's iteration marker ranges. The flame graph + * built from this thread then reflects only the samples that contribute to + * this suite's score (matching `computeSuiteScores`). + */ +export function makeSuiteFilteredThread( + bundle: BucketProfileBundle, + suiteName: string +): Thread { + const { thread, sampleTimes, measuredSampleWeights, markersPerSuite } = + bundle; + const iterationMarkers = markersPerSuite.get(suiteName) ?? []; + const filteredWeights = computeSuiteFilteredSampleWeights( + measuredSampleWeights, + sampleTimes, + iterationMarkers + ); + return { + ...thread, + samples: { + ...thread.samples, + weight: Array.from(filteredWeights), + weightType: thread.samples.weightType ?? 'samples', + }, + }; +} diff --git a/src/profile-logic/benchmark/compare-benchmark-stats.ts b/src/profile-logic/benchmark/compare-benchmark-stats.ts new file mode 100644 index 0000000000..628faa316a --- /dev/null +++ b/src/profile-logic/benchmark/compare-benchmark-stats.ts @@ -0,0 +1,262 @@ +/** + * Compare two benchmark profile stats files (produced by extract-benchmark-stats) + * and report which buckets changed significantly between them. + * + * Uses Mann-Whitney U test with normal approximation. + * + * Usage: + * yarn build-node-tools + * node node-tools-dist/compare-benchmark-stats.js \ + * --base /tmp/base-stats.json \ + * --new /tmp/new-stats.json + * + * Options: + * --suite Show per-suite results for this suite (substring match) + * --global Show results from the geomean-normalised global view (default) + * --pvalue <0.05> Significance threshold (default 0.05) + * --top <20> Show top N changed buckets (default 20) + * --all Show all significant buckets, not just top N + */ + +import type { SparseBucketEntry } from './extract-benchmark-stats'; +import { + mannWhitneyU, + mannWhitneyPValue, + cliffsDelta, + interpretEffectSize, + pValueToConfidence, +} from './perf-compare-stats'; +import type { EffectSize, ConfidenceRating } from './perf-compare-stats'; +import type { IndexIntoFuncTable } from '../../types/profile'; + +// --------------------------------------------------------------------------- +// Comparison logic +// --------------------------------------------------------------------------- + +export type BucketComparison = { + bucketName: string; + /** Func index of the bucket in the base profile, or null if absent there. + * If multiple funcs share this name within the profile, the one with the + * largest sum of iterationTotals is chosen (representative func). */ + baseFunc: IndexIntoFuncTable | null; + /** Func index of the bucket in the new profile, or null if absent there. */ + newFunc: IndexIntoFuncTable | null; + baseMean: number; + newMean: number; + /** Relative change: (newMean - baseMean) / baseMean */ + relChange: number; + cliffdsDelta: number; + effectSize: EffectSize; + confidence: ConfidenceRating; +}; + +type KeyMapEntry = { + /** Human-readable display name for this key (taken from the first bucket + * seen with this key — usually a function name). */ + displayName: string; + iterationTotals: number[]; + /** Func index of the highest-weight bucket with this key (representative). */ + representativeFunc: IndexIntoFuncTable; + /** Sum of iterationTotals for that representative bucket alone. */ + representativeWeight: number; +}; + +/** Build a key → iterationTotals + representative-func map for a set of sparse bucket entries. */ +function buildKeyMap( + buckets: SparseBucketEntry[], + bucketKeys: string[], + bucketNames: string[], + bucketFuncs: IndexIntoFuncTable[] +): Map { + const map = new Map(); + for (const entry of buckets) { + const key = bucketKeys[entry.bucketIndex] ?? `bucket#${entry.bucketIndex}`; + const name = + bucketNames[entry.bucketIndex] ?? `bucket#${entry.bucketIndex}`; + const func = bucketFuncs[entry.bucketIndex]; + let weight = 0; + for (const v of entry.iterationTotals) { + weight += v; + } + const existing = map.get(key); + if (existing !== undefined) { + // If the same key appears twice (two different funcs that collapsed to + // the same matching key — e.g. an inlined and non-inlined copy of the + // same JS function), sum their iteration totals together. Pick the + // heaviest as the representative func, since the flame graph can only + // focusSelf on one func. + for (let i = 0; i < existing.iterationTotals.length; i++) { + existing.iterationTotals[i] += entry.iterationTotals[i]; + } + if (weight > existing.representativeWeight) { + existing.representativeFunc = func; + existing.representativeWeight = weight; + existing.displayName = name; + } + } else { + map.set(key, { + displayName: name, + iterationTotals: entry.iterationTotals.slice(), + representativeFunc: func, + representativeWeight: weight, + }); + } + } + return map; +} + +/** + * Compare two sparse bucket lists, matching by bucket key across profiles. + * For JS funcs, the key is the source location (filename:line:col) so that + * naming differences across engines don't prevent the same function from + * matching. For everything else, the key is the bucket name. + * + * Buckets that appear in only one profile are treated as + * "appeared"/"disappeared" unless excludeAppearedDisappeared is set. + * + * `baseBucketKeys` / `newBucketKeys` may be missing (older stats files + * predate the cross-engine matching key); in that case we fall back to + * matching by name, which preserves prior behaviour. + */ +export function compareBuckets( + baseBuckets: SparseBucketEntry[], + newBuckets: SparseBucketEntry[], + baseBucketNames: string[], + newBucketNames: string[], + baseBucketFuncs: IndexIntoFuncTable[], + newBucketFuncs: IndexIntoFuncTable[], + iterationCount: number, + excludeAppearedDisappeared: boolean = false, + baseBucketKeys: string[] = baseBucketNames, + newBucketKeys: string[] = newBucketNames +): BucketComparison[] { + const baseMap = buildKeyMap( + baseBuckets, + baseBucketKeys, + baseBucketNames, + baseBucketFuncs + ); + const newMap = buildKeyMap( + newBuckets, + newBucketKeys, + newBucketNames, + newBucketFuncs + ); + + const allKeys = excludeAppearedDisappeared + ? new Set([...baseMap.keys()].filter((k) => newMap.has(k))) + : new Set([...baseMap.keys(), ...newMap.keys()]); + + const zeros = new Array(iterationCount).fill(0); + + const results: BucketComparison[] = []; + for (const key of allKeys) { + const baseEntry = baseMap.get(key); + const newEntry = newMap.get(key); + const baseIter = baseEntry?.iterationTotals ?? zeros; + const newIter = newEntry?.iterationTotals ?? zeros; + + const baseMean = mean(baseIter); + const newMean = mean(newIter); + + if (baseMean === 0 && newMean === 0) { + continue; + } + + const allValues = [...baseIter, ...newIter]; + const u = mannWhitneyU(baseIter, newIter); + const pValue = mannWhitneyPValue( + u, + baseIter.length, + newIter.length, + allValues + ); + const relChange = + baseMean === 0 ? Infinity : (newMean - baseMean) / baseMean; + const delta = cliffsDelta(u, baseIter.length, newIter.length); + const effectSize = interpretEffectSize(delta); + const confidence = pValueToConfidence(pValue); + + // Prefer the base profile's display name; fall back to the new one. + const displayName = baseEntry?.displayName ?? newEntry?.displayName ?? key; + + results.push({ + bucketName: displayName, + baseFunc: baseEntry?.representativeFunc ?? null, + newFunc: newEntry?.representativeFunc ?? null, + baseMean, + newMean, + relChange, + cliffdsDelta: delta, + effectSize, + confidence, + }); + } + + return results; +} + +export function mean(arr: number[]): number { + if (arr.length === 0) { + return 0; + } + let sum = 0; + for (const v of arr) { + sum += v; + } + return sum / arr.length; +} + +/** Sum all bucket iterationTotals element-wise to get a per-iteration total for a suite. */ +export function suiteIterationTotals( + buckets: SparseBucketEntry[], + iterationCount: number +): number[] { + const totals = new Array(iterationCount).fill(0); + for (const entry of buckets) { + for (let i = 0; i < iterationCount; i++) { + totals[i] += entry.iterationTotals[i]; + } + } + return totals; +} + +export type ScoreComparison = { + label: string; + baseMean: number; + newMean: number; + relChange: number; + cliffdsDelta: number; + effectSize: EffectSize; + confidence: ConfidenceRating; +}; + +export function compareIterationTotals( + label: string, + baseIter: number[], + newIter: number[] +): ScoreComparison { + const baseMean = mean(baseIter); + const newMean = mean(newIter); + const allValues = [...baseIter, ...newIter]; + const u = mannWhitneyU(baseIter, newIter); + const pValue = mannWhitneyPValue( + u, + baseIter.length, + newIter.length, + allValues + ); + const relChange = baseMean === 0 ? Infinity : (newMean - baseMean) / baseMean; + const delta = cliffsDelta(u, baseIter.length, newIter.length); + const effectSize = interpretEffectSize(delta); + const confidence = pValueToConfidence(pValue); + return { + label, + baseMean, + newMean, + relChange, + cliffdsDelta: delta, + effectSize, + confidence, + }; +} diff --git a/src/profile-logic/benchmark/extract-benchmark-stats.ts b/src/profile-logic/benchmark/extract-benchmark-stats.ts new file mode 100644 index 0000000000..d5833aa2bc --- /dev/null +++ b/src/profile-logic/benchmark/extract-benchmark-stats.ts @@ -0,0 +1,304 @@ +/** + * Extract per-bucket, per-iteration statistics from a benchmark profile into a + * compact intermediate JSON file suitable for cross-profile comparison. + * + * The output is intentionally sparse: only (suite, bucket) pairs with nonzero + * weight are stored. At 200 iterations × 10578 buckets × 20 suites a dense + * representation would be ~323 MB; the sparse form is ~2 MB. + * + * Usage: + * yarn build-node-tools + * node node-tools-dist/extract-benchmark-stats.js \ + * --input ~/Downloads/profile.json \ + * --output /tmp/profile-stats.json + */ + +import { + computeBenchmarkScores, + computeIterationMarkersAndMeasuredSamples, + getBenchmarkInfo, +} from 'firefox-profiler/profile-logic/benchmark/benchmark-stuff'; +import type { + BenchmarkHarness, + SamplesTableForThisStuff, +} from 'firefox-profiler/profile-logic/benchmark/benchmark-stuff'; +import { + correlateIPCMarkers, + deriveMarkersFromRawMarkerTable, +} from 'firefox-profiler/profile-logic/marker-data'; +import { + computeTimeColumnForRawSamplesTable, + getTimeRangeForThread, +} from 'firefox-profiler/profile-logic/profile-data'; +import { StringTable } from 'firefox-profiler/utils/string-table'; + +import type { + IndexIntoFuncTable, + IndexIntoStackTable, + Profile, + RawProfileSharedData, +} from '../../types/profile'; + +// --------------------------------------------------------------------------- +// Types for the intermediate JSON +// --------------------------------------------------------------------------- + +/** One (suite, bucket) pair with nonzero weight. */ +export type SparseBucketEntry = { + /** Index into the profile's global bucket list. */ + bucketIndex: number; + /** Weight sum per iteration, length = iterationCount. */ + iterationTotals: number[]; +}; + +export type SuiteStats = { + suiteName: string; + iterationCount: number; + /** Only buckets that have nonzero total weight across all iterations. */ + buckets: SparseBucketEntry[]; +}; + +export type ProfileBenchmarkStats = { + /** Name of each bucket (JS function name or similar). Length = total bucket count. */ + bucketNames: string[]; + /** + * Cross-engine matching key for each bucket. Same length as bucketNames. + * + * For JS funcs, this is `::` (using the + * source/lineNumber/columnNumber columns of the funcTable). Function names + * differ across engines for the same logical function — V8 reports + * `Template.show` where SpiderMonkey reports `Template.prototype.show`, and + * anonymous-arrow naming heuristics diverge entirely — but the source + * location is stable, so matching on it lets `compareBuckets` align the + * two engines' samples for the same function. + * + * For non-JS funcs (DOM-binding `relevantForJS` shims, inserted `Label` + * funcs, etc.), the key is the name itself: those are already engine- + * neutral by design. + * + * Falls back to the name if a JS func has no source/line/col (e.g. the + * profile wasn't run through `--canonicalize-js-location`). + */ + bucketKeys: string[]; + /** + * Func index (in profile.shared.funcTable) for each bucket. Same length as + * bucketNames. -1 for the synthetic "no JS frame" bucket. Useful when callers + * want to reach back into the source profile for a given bucket, e.g. to feed + * a focusSelf() flame graph. + */ + bucketFuncs: Array; + /** + * Per-bucket weight summed across all suites, with suite geomean factors applied, + * per iteration. Sparse: only buckets with nonzero global total. + * This is the "geomean-normalised" global view. + */ + globalBuckets: SparseBucketEntry[]; + /** Per-suite sparse bucket data. */ + suites: SuiteStats[]; +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function computeJsOnlySampleBuckets( + shared: RawProfileSharedData, + sampleStacks: Array +): { + bucketFuncs: Array; + sampleBuckets: Int32Array; +} { + const { funcTable, stackTable, frameTable } = shared; + const bucketFuncs = new Array(); + const funcIndexToBucketIndex = new Map(); + + const stackIndexToJsOnlyFuncIndex = new Int32Array(stackTable.length); + for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) { + const frameIndex = stackTable.frame[stackIndex]; + const funcIndex = frameTable.func[frameIndex]; + if (funcTable.isJS[funcIndex] || funcTable.relevantForJS[funcIndex]) { + stackIndexToJsOnlyFuncIndex[stackIndex] = funcIndex; + } else { + const prefixOffset = stackTable.prefixOffset[stackIndex]; + if (prefixOffset !== 0) { + const parentStackIndex = stackIndex - prefixOffset; + stackIndexToJsOnlyFuncIndex[stackIndex] = + stackIndexToJsOnlyFuncIndex[parentStackIndex]; + } else { + stackIndexToJsOnlyFuncIndex[stackIndex] = -1; + } + } + } + + const sampleBuckets = new Int32Array(sampleStacks.length); + for (let sampleIndex = 0; sampleIndex < sampleBuckets.length; sampleIndex++) { + const stackIndex = sampleStacks[sampleIndex]; + if (stackIndex !== null) { + const jsOnlyFuncIndex = stackIndexToJsOnlyFuncIndex[stackIndex]; + let bucketIndex = + jsOnlyFuncIndex !== -1 + ? funcIndexToBucketIndex.get(jsOnlyFuncIndex) + : -1; + if (bucketIndex === undefined) { + bucketIndex = bucketFuncs.length; + bucketFuncs[bucketIndex] = jsOnlyFuncIndex; + funcIndexToBucketIndex.set(jsOnlyFuncIndex, bucketIndex); + } + sampleBuckets[sampleIndex] = bucketIndex; + } else { + sampleBuckets[sampleIndex] = -1; + } + } + + return { bucketFuncs, sampleBuckets }; +} + +// --------------------------------------------------------------------------- +// Main extraction logic +// --------------------------------------------------------------------------- + +/** + * Extract per-bucket, per-iteration statistics from an already-loaded Profile. + * This is the browser-safe core of the extraction logic; it has no I/O dependencies. + */ +export function extractBenchmarkStatsFromProfile( + profile: Profile, + benchmarkHarness: BenchmarkHarness = 'speedometer' +): ProfileBenchmarkStats { + const benchmarkInfo = getBenchmarkInfo(profile, benchmarkHarness); + const { shared } = profile; + const thread = profile.threads[benchmarkInfo.threadIndex]; + + const { markers } = deriveMarkersFromRawMarkerTable( + thread.markers, + shared.stringArray, + thread.tid, + getTimeRangeForThread(thread, profile.meta.interval), + correlateIPCMarkers(profile.threads, shared) + ); + const stringTable = StringTable.withBackingArray(shared.stringArray); + + const sampleCount = thread.samples.length; + const { sampleBuckets, bucketFuncs } = computeJsOnlySampleBuckets( + shared, + thread.samples.stack + ); + + const profileOverheadBucket = bucketFuncs.findIndex( + (func) => + shared.stringArray[shared.funcTable.name[func]] === 'Profiling overhead' + ); + const bucketsToIgnore = + profileOverheadBucket !== -1 ? [profileOverheadBucket] : []; + + const samples: SamplesTableForThisStuff = { + length: sampleCount, + time: new Float64Array(computeTimeColumnForRawSamplesTable(thread.samples)), + weight: thread.samples.weight + ? new Float64Array(thread.samples.weight) + : new Float64Array(sampleCount).fill(1), + bucketIndex: sampleBuckets, + bucketCount: bucketFuncs.length, + }; + + const iterationMarkersAndMeasuredSamples = + computeIterationMarkersAndMeasuredSamples( + benchmarkInfo, + markers, + samples, + stringTable, + bucketsToIgnore + ); + + const benchmarkScores = computeBenchmarkScores( + iterationMarkersAndMeasuredSamples + ); + + const { funcTable: sharedFuncTable, sources: sharedSources } = shared; + const bucketNames = bucketFuncs.map( + (funcIndex) => shared.stringArray[sharedFuncTable.name[funcIndex]] + ); + // Build cross-engine matching keys: filename:line:col for JS funcs with a + // source, name otherwise. See ProfileBenchmarkStats.bucketKeys for the + // motivation (function names diverge across engines for the same source + // location). + const bucketKeys = bucketFuncs.map((funcIndex, b) => { + if (sharedFuncTable.isJS[funcIndex]) { + const sourceIndex = sharedFuncTable.source[funcIndex]; + const line = sharedFuncTable.lineNumber[funcIndex]; + const col = sharedFuncTable.columnNumber[funcIndex]; + if (sourceIndex !== null && line !== null && col !== null) { + const filename = + shared.stringArray[sharedSources.filename[sourceIndex]]; + return `${filename}:${line}:${col}`; + } + } + return bucketNames[b]; + }); + + const bucketCount = bucketFuncs.length; + const { allSuiteScores, factorPerSuite } = benchmarkScores; + + // Build per-suite sparse entries + const suites: SuiteStats[] = allSuiteScores.map((suiteScores) => { + const iterationCount = suiteScores.bucketStats!.iterationCount; + const buckets: SparseBucketEntry[] = []; + + for (let b = 0; b < bucketCount; b++) { + if (suiteScores.bucketTotals[b] === 0) { + continue; + } + const iterationTotals: number[] = new Array(iterationCount); + const base = b * iterationCount; + for (let i = 0; i < iterationCount; i++) { + iterationTotals[i] = suiteScores.bucketIterationTotals[base + i]; + } + buckets.push({ bucketIndex: b, iterationTotals }); + } + + return { + suiteName: suiteScores.suiteName, + iterationCount, + buckets, + }; + }); + + // Build global sparse entries: sum factorPerSuite[s] * bucketIterationTotals[s][b][i] + // All suites share the same iterationCount, so we can use the first suite's value. + const iterationCount = allSuiteScores[0].bucketStats!.iterationCount; + const globalIterTotals = new Float64Array(bucketCount * iterationCount); + + for (let suiteIndex = 0; suiteIndex < allSuiteScores.length; suiteIndex++) { + const factor = factorPerSuite[suiteIndex]; + const suiteScores = allSuiteScores[suiteIndex]; + for (let b = 0; b < bucketCount; b++) { + if (suiteScores.bucketTotals[b] === 0) { + continue; + } + const base = b * iterationCount; + for (let i = 0; i < iterationCount; i++) { + globalIterTotals[base + i] += + factor * suiteScores.bucketIterationTotals[base + i]; + } + } + } + + const globalBuckets: SparseBucketEntry[] = []; + for (let b = 0; b < bucketCount; b++) { + const base = b * iterationCount; + let total = 0; + for (let i = 0; i < iterationCount; i++) { + total += globalIterTotals[base + i]; + } + if (total === 0) { + continue; + } + const iterationTotals: number[] = new Array(iterationCount); + for (let i = 0; i < iterationCount; i++) { + iterationTotals[i] = globalIterTotals[base + i]; + } + globalBuckets.push({ bucketIndex: b, iterationTotals }); + } + + return { bucketNames, bucketKeys, bucketFuncs, globalBuckets, suites }; +} diff --git a/src/profile-logic/benchmark/perf-compare-stats.ts b/src/profile-logic/benchmark/perf-compare-stats.ts new file mode 100644 index 0000000000..96f7f49144 --- /dev/null +++ b/src/profile-logic/benchmark/perf-compare-stats.ts @@ -0,0 +1,502 @@ +// Non-parametric statistics for comparing performance samples. +// +// Mann-Whitney U: Wilcoxon (1945), Biometrics Bulletin 1(6):80-83 +// Cliff's delta: Cliff (1993), Psychological Bulletin 114(3):494-509 +// Shapiro-Wilk: Shapiro & Wilk (1965), Biometrika 52(3-4):591-611 +// Coefficients: Royston (1992), Statistics and Computing 2(3):117-119 +// p-value: Royston (1995) + +// --------------------------------------------------------------------------- +// Normal distribution +// --------------------------------------------------------------------------- + +function normalQuantile(p: number): number { + const a = [ + -3.969683028665376e1, 2.209460984245205e2, -2.759285104469687e2, + 1.38357751867269e2, -3.066479806614716e1, 2.506628277459239, + ]; + const b = [ + -5.447609879822406e1, 1.615858368580409e2, -1.556989798598866e2, + 6.680131188771972e1, -1.328068155288572e1, + ]; + const c = [ + -7.784894002430293e-3, -3.223964580411365e-1, -2.400758277161838, + -2.549732539343734, 4.374664141464968, 2.938163982698783, + ]; + const d = [ + 7.784695709041462e-3, 3.224671290700398e-1, 2.445134137142996, + 3.754408661907416, + ]; + const pLow = 0.02425; + const pHigh = 1 - pLow; + + if (p < pLow) { + const q = Math.sqrt(-2 * Math.log(p)); + return ( + (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + ); + } + if (p <= pHigh) { + const q = p - 0.5; + const r = q * q; + return ( + ((((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * + q) / + (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1) + ); + } + const q = Math.sqrt(-2 * Math.log(1 - p)); + return -( + (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + ); +} + +// Abramowitz & Stegun 7.1.26 via the error function. +// The coefficients are for erf(z), not Φ(x) directly. +export function normalCDF(x: number): number { + const z = Math.abs(x) / Math.SQRT2; + const t = 1 / (1 + 0.3275911 * z); + const poly = + t * + (0.254829592 + + t * + (-0.284496736 + + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))); + const erfVal = 1 - poly * Math.exp(-z * z); + return x >= 0 ? 0.5 * (1 + erfVal) : 0.5 * (1 - erfVal); +} + +// --------------------------------------------------------------------------- +// Median +// --------------------------------------------------------------------------- + +export function median(arr: number[]): number { + if (!arr.length) { + return NaN; + } + const s = arr.slice().sort((a, b) => a - b); + const m = s.length >> 1; + return s.length & 1 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +// --------------------------------------------------------------------------- +// Mann-Whitney U +// --------------------------------------------------------------------------- + +export function mannWhitneyU(a: number[], b: number[]): number { + let u = 0; + for (const ai of a) { + for (const bj of b) { + if (ai < bj) { + u += 1; + } else if (ai === bj) { + u += 0.5; + } + } + } + return u; +} + +export function mannWhitneyPValue( + u: number, + n1: number, + n2: number, + allValues: number[] +): number { + const mu = (n1 * n2) / 2; + const counts = new Map(); + for (const v of allValues) { + counts.set(v, (counts.get(v) ?? 0) + 1); + } + let tieCorrection = 0; + for (const t of counts.values()) { + if (t > 1) { + tieCorrection += t * t * t - t; + } + } + const n = n1 + n2; + const variance = ((n1 * n2) / 12) * (n + 1 - tieCorrection / (n * (n - 1))); + if (variance <= 0) { + return 1; + } + const z = (u - mu) / Math.sqrt(variance); + return 2 * (1 - normalCDF(Math.abs(z))); +} + +// --------------------------------------------------------------------------- +// Cliff's delta / CLES / effect size +// --------------------------------------------------------------------------- + +export type EffectSize = 'Negligible' | 'Small' | 'Moderate' | 'Large'; + +export function cliffsDelta(u: number, n1: number, n2: number): number { + return (2 * u) / (n1 * n2) - 1; +} + +export function cles(u: number, n1: number, n2: number): number { + return u / (n1 * n2); +} + +export function interpretEffectSize(delta: number): EffectSize { + const magnitude = Math.abs(delta); + if (magnitude < 0.15) { + return 'Negligible'; + } + if (magnitude < 0.33) { + return 'Small'; + } + if (magnitude < 0.47) { + return 'Moderate'; + } + return 'Large'; +} + +const EFFECT_SIZE_ORDER: EffectSize[] = [ + 'Negligible', + 'Small', + 'Moderate', + 'Large', +]; + +export function effectSizeLessThan(e1: EffectSize, e2: EffectSize): boolean { + return EFFECT_SIZE_ORDER.indexOf(e1) < EFFECT_SIZE_ORDER.indexOf(e2); +} + +// --------------------------------------------------------------------------- +// Confidence rating from p-value +// --------------------------------------------------------------------------- + +export type ConfidenceRating = 'HIGH' | 'MEDIUM' | 'LOW' | 'UNKNOWN'; + +export function pValueToConfidence(pValue: number): ConfidenceRating { + if (pValue <= 0.05) { + return 'HIGH'; + } + if (pValue <= 0.15) { + return 'MEDIUM'; + } + return 'LOW'; +} + +export function confidenceLessThan( + conf1: ConfidenceRating, + conf2: ConfidenceRating +): boolean { + return ( + (conf2 === 'HIGH' && conf1 !== 'HIGH') || + (conf2 === 'MEDIUM' && conf1 === 'LOW') + ); +} + +// --------------------------------------------------------------------------- +// Shapiro-Wilk normality test +// --------------------------------------------------------------------------- + +function poly5(coeffs: number[], u: number): number { + return ( + ((((coeffs[0] * u + coeffs[1]) * u + coeffs[2]) * u + coeffs[3]) * u + + coeffs[4]) * + u + + coeffs[5] + ); +} + +function iqrFilter(data: number[]): number[] { + if (data.length < 4) { + return data; + } + const s = [...data].sort((a, b) => a - b); + const n = s.length; + const q1 = s[Math.floor(n * 0.25)]; + const q3 = s[Math.floor(n * 0.75)]; + const iqr = q3 - q1; + return s.filter((x) => x >= q1 - 1.5 * iqr && x <= q3 + 1.5 * iqr); +} + +export function shapiroWilkTest( + data: number[] +): { w: number; pvalue: number } | null { + const x = iqrFilter(data).sort((a, b) => a - b); + const n = x.length; + if (n < 3 || n > 5000) { + return null; + } + + const m = Array.from({ length: n }, (_, i) => + normalQuantile((i + 1 - 0.375) / (n + 0.25)) + ); + const md = m.reduce((s, v) => s + v * v, 0); + const sqrtMd = Math.sqrt(md); + + const c1 = [-2.706056, 4.434685, -2.07119, -0.147981, 0.221157, 0]; + const c2 = [-3.582633, 5.682633, -1.752461, -0.293762, 0.042981, 0]; + const u = 1 / Math.sqrt(n); + c1[5] = m[n - 1] / sqrtMd; + c2[5] = m[n - 2] / sqrtMd; + const an = poly5(c1, u); + const ann = poly5(c2, u); + + const half = Math.floor(n / 2); + let phi: number; + if (n > 5) { + phi = + (md - 2 * m[n - 1] ** 2 - 2 * m[n - 2] ** 2) / + (1 - 2 * an ** 2 - 2 * ann ** 2); + } else { + phi = (md - 2 * m[n - 1] ** 2) / (1 - 2 * an ** 2); + } + const sqrtPhi = Math.sqrt(phi); + + const a: number[] = Array.from({ length: half }); + a[0] = an; + if (n > 5 && half > 1) { + a[1] = ann; + } + const startJ = n > 5 ? 2 : 1; + for (let j = startJ; j < half; j++) { + a[j] = m[n - 1 - j] / sqrtPhi; + } + + const xbar = x.reduce((s, v) => s + v, 0) / n; + const ss = x.reduce((s, v) => s + (v - xbar) ** 2, 0); + if (ss === 0) { + return null; + } + + let num = 0; + for (let j = 0; j < half; j++) { + num += a[j] * (x[n - 1 - j] - x[j]); + } + const w = Math.min(num ** 2 / ss, 1); + + const logn = Math.log(n); + let g: number, mu2: number, sigma: number; + if (n < 12) { + const gamma = 0.459 * n - 2.273; + g = -Math.log(gamma - Math.log(1 - w)); + mu2 = -0.0006714 * n ** 3 + 0.025054 * n ** 2 - 0.39978 * n + 0.544; + sigma = Math.exp( + -0.0020322 * n ** 3 + 0.062767 * n ** 2 - 0.77857 * n + 1.3822 + ); + } else { + g = Math.log(1 - w); + mu2 = + 0.0038915 * logn ** 3 - 0.083751 * logn ** 2 - 0.31082 * logn - 1.5861; + sigma = Math.exp(0.0030302 * logn ** 2 - 0.082676 * logn - 0.4803); + } + + const pvalue = 1 - normalCDF((g - mu2) / sigma); + return { w, pvalue }; +} + +// --------------------------------------------------------------------------- +// Bootstrap CI for the median difference (comp − base) +// --------------------------------------------------------------------------- + +export type BootstrapCIResult = { + shift: number; + lo: number; + hi: number; +}; + +export function bootstrapMedianCI( + base: number[], + comp: number[], + nIter: number = 500 +): BootstrapCIResult | null { + if (base.length < 2 || comp.length < 2) { + return null; + } + const shifts = new Array(nIter); + for (let i = 0; i < nIter; i++) { + shifts[i] = median(bootSample(comp)) - median(bootSample(base)); + } + shifts.sort((a, b) => a - b); + return { + shift: median(comp) - median(base), + lo: shifts[Math.floor(0.025 * nIter)], + hi: shifts[Math.ceil(0.975 * nIter) - 1], + }; +} + +function bootSample(arr: number[]): number[] { + const out = new Array(arr.length); + for (let i = 0; i < arr.length; i++) { + out[i] = arr[Math.floor(Math.random() * arr.length)]; + } + return out; +} + +// --------------------------------------------------------------------------- +// Mode matching — min-cost bipartite assignment (bitmask DP, exact for ≤8 modes) +// +// Cost = 0.75 × normalised location distance + 0.25 × fraction difference +// --------------------------------------------------------------------------- + +export type MatchResult = { + pairs: [number, number][]; + unmatchedBase: number[]; + unmatchedNew: number[]; +}; + +export function matchModes( + baseLocs: number[], + baseFracs: number[], + newLocs: number[], + newFracs: number[] +): MatchResult { + const n = baseLocs.length; + const m = newLocs.length; + if (!n || !m) { + return { pairs: [], unmatchedBase: range(n), unmatchedNew: range(m) }; + } + + if (n > m) { + const sw = matchModes(newLocs, newFracs, baseLocs, baseFracs); + return { + pairs: sw.pairs.map(([a, b]) => [b, a]), + unmatchedBase: sw.unmatchedNew, + unmatchedNew: sw.unmatchedBase, + }; + } + + // n <= m: assign all n base modes to n of the m new modes + const all = baseLocs.concat(newLocs); + let lo = all[0], + hi = all[0]; + for (let i = 1; i < all.length; i++) { + if (all[i] < lo) { + lo = all[i]; + } + if (all[i] > hi) { + hi = all[i]; + } + } + const span = hi - lo || 1; + + const cost = baseLocs.map((bl, i) => + newLocs.map( + (nl, j) => + (0.75 * Math.abs(bl - nl)) / span + + 0.25 * Math.abs(baseFracs[i] - newFracs[j]) + ) + ); + + const INF = 1e9; + const states = 1 << m; + const dp = new Float64Array(states).fill(INF); + const prev = new Int16Array(states).fill(-1); + dp[0] = 0; + for (let mask = 0; mask < states; mask++) { + if (dp[mask] === INF) { + continue; + } + const i = popcount(mask); + if (i >= n) { + continue; + } + for (let j = 0; j < m; j++) { + if ((mask >> j) & 1) { + continue; + } + const nm = mask | (1 << j); + const c = dp[mask] + cost[i][j]; + if (c < dp[nm]) { + dp[nm] = c; + prev[nm] = j; + } + } + } + + let best = -1; + let bc = INF; + for (let mask = 0; mask < states; mask++) { + if (popcount(mask) === n && dp[mask] < bc) { + bc = dp[mask]; + best = mask; + } + } + + const pairs: [number, number][] = []; + let cur = best; + for (let i = n - 1; i >= 0; i--) { + const j = prev[cur]; + pairs.unshift([i, j]); + cur ^= 1 << j; + } + const matchedNew = new Set(pairs.map(([, b]) => b)); + return { + pairs, + unmatchedBase: [], + unmatchedNew: range(m).filter((j) => !matchedNew.has(j)), + }; +} + +function popcount(x: number): number { + let c = 0; + while (x) { + c += x & 1; + x >>= 1; + } + return c; +} + +function range(n: number): number[] { + return Array.from({ length: n }, (_, i) => i); +} + +// --------------------------------------------------------------------------- +// Mode helpers +// --------------------------------------------------------------------------- + +// Split raw samples into mode buckets using boundary x-values. +export function splitByMode(data: number[], boundaries: number[]): number[][] { + const buckets: number[][] = Array.from( + { length: boundaries.length + 1 }, + () => [] + ); + for (const v of data) { + let m = 0; + while (m < boundaries.length && v > boundaries[m]) { + m++; + } + buckets[m].push(v); + } + return buckets; +} + +// Fraction of KDE area in each mode bucket (trapezoid rule). +export function areaFractions( + x: number[], + y: number[], + boundaries: number[] +): number[] { + const buckets = new Array(boundaries.length + 1).fill(0); + let total = 0; + for (let i = 1; i < x.length; i++) { + const area = 0.5 * (y[i] + y[i - 1]) * (x[i] - x[i - 1]); + total += area; + let m = 0; + while (m < boundaries.length && x[i] > boundaries[m]) { + m++; + } + buckets[m] += area; + } + return total > 0 + ? buckets.map((b: number) => b / total) + : buckets.map(() => 1 / buckets.length); +} + +// Assign letter labels: A = lowest value (fastest), B = next, etc. +export function assignModeLetters(peakLocs: number[]): string[] { + const sorted = peakLocs + .map((_, i) => i) + .sort((a, b) => peakLocs[a] - peakLocs[b]); + const letters = new Array(peakLocs.length); + sorted.forEach((idx, rank) => { + letters[idx] = String.fromCharCode(65 + rank); + }); + return letters; +} diff --git a/src/reducers/url-state.ts b/src/reducers/url-state.ts index 97b41cbeae..d04d60ea2f 100644 --- a/src/reducers/url-state.ts +++ b/src/reducers/url-state.ts @@ -57,6 +57,8 @@ const dataSource: Reducer = (state = 'none', action) => { return 'unpublished'; case 'SET_DATA_SOURCE': return action.dataSource; + case 'CHANGE_PROFILES_TO_COMPARE_BENCHMARK': + return 'compare-benchmark'; default: return state; } @@ -88,6 +90,7 @@ const profileUrl: Reducer = (state = '', action) => { const profilesToCompare: Reducer = (state = null, action) => { switch (action.type) { case 'CHANGE_PROFILES_TO_COMPARE': + case 'CHANGE_PROFILES_TO_COMPARE_BENCHMARK': return action.profiles; default: return state; diff --git a/src/test/components/__snapshots__/CompareHome.test.tsx.snap b/src/test/components/__snapshots__/CompareHome.test.tsx.snap index 100a0aac05..7e6ad50e09 100644 --- a/src/test/components/__snapshots__/CompareHome.test.tsx.snap +++ b/src/test/components/__snapshots__/CompareHome.test.tsx.snap @@ -106,11 +106,22 @@ compare. type="url" value="" /> - +
+ + +
`; diff --git a/src/types/actions.ts b/src/types/actions.ts index fdde883ad1..51ff4d2c26 100644 --- a/src/types/actions.ts +++ b/src/types/actions.ts @@ -86,6 +86,9 @@ export type DataSource = // This is used when comparing two profiles. The displayed profile is a // comparison profile created from two input profiles. | 'compare' + // This is used for the benchmark comparison page at /compare-benchmark/. + // It loads two benchmark profiles and shows statistical comparison tables. + | 'compare-benchmark' // This is a page which displays a list of profiles that were uploaded from // this browser, and allows deleting / unpublishing those profiles. | 'uploaded-recordings'; @@ -568,6 +571,10 @@ type UrlStateAction = readonly searchString: string; } | { readonly type: 'CHANGE_PROFILES_TO_COMPARE'; readonly profiles: string[] } + | { + readonly type: 'CHANGE_PROFILES_TO_COMPARE_BENCHMARK'; + readonly profiles: string[]; + } | { readonly type: 'CHANGE_PROFILE_NAME'; readonly profileName: string | null; diff --git a/src/utils/stats.ts b/src/utils/stats.ts new file mode 100644 index 0000000000..0b2c0c3e7f --- /dev/null +++ b/src/utils/stats.ts @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export type BucketStats = { + mean: number; + variance: number; + iterationCount: number; +}; + +export type ConfidenceRating = 'HIGH' | 'MEDIUM' | 'LOW' | 'UNKNOWN'; + +export type ComparedStats = { + pooledStdDev: number; + tStat: number; + pValue: number; + confidence: ConfidenceRating; +}; + +export function confidenceLessThan( + conf1: ConfidenceRating, + conf2: ConfidenceRating +): boolean { + return ( + (conf2 === 'HIGH' && conf1 !== 'HIGH') || + (conf2 === 'MEDIUM' && conf1 === 'LOW') + ); +} + +function pValueToConfidence(pValue: number): ConfidenceRating { + if (pValue <= 0.05) { + return 'HIGH'; + } + if (pValue <= 0.15) { + return 'MEDIUM'; + } + return 'LOW'; +} + +// Function to calculate the cumulative distribution function of the t-distribution +function cumulativeDistributionT(t: number, df: number): number { + const x = df / (t * t + df); + return 0.5 * (1 + (t > 0 ? 1 : -1) * Math.sqrt(1 - x)); +} + +export function computeComparedStats( + statsComp: BucketStats | null, + statsRef: BucketStats | null +): ComparedStats | null { + if (statsComp === null && statsRef !== null) { + statsComp = { + mean: 0, + variance: 0, + iterationCount: statsRef.iterationCount, + }; + } else if (statsRef === null && statsComp !== null) { + statsRef = { + mean: 0, + variance: 0, + iterationCount: statsComp.iterationCount, + }; + } else if (statsRef === null || statsComp === null) { + return null; + } + if (statsComp === statsRef) { + return null; + } + + // Calculate pooled standard deviation + const pooledStdDev = Math.sqrt( + statsComp.variance / statsComp.iterationCount + + statsRef.variance / statsRef.iterationCount + ); + + // Calculate t-statistic + const tStat = + (statsComp.mean - statsRef.mean) / + (pooledStdDev * + Math.sqrt(1 / statsComp.iterationCount + 1 / statsRef.iterationCount)); + + // Calculate degrees of freedom + const degreesOfFreedom = + statsComp.iterationCount + statsRef.iterationCount - 2; + + // Calculate the P-value + const pValue = + 2 * (1 - cumulativeDistributionT(Math.abs(tStat), degreesOfFreedom)); + + const confidence = pValueToConfidence(pValue); + + return { pooledStdDev, tStat, pValue, confidence }; +} From 3dc154061000ef7a55853bf895b39e427571dc3b Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Thu, 4 Jun 2026 17:18:55 -0400 Subject: [PATCH 4/9] Only non-negligible changes --- src/components/app/BenchmarkCompareViewer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/app/BenchmarkCompareViewer.tsx b/src/components/app/BenchmarkCompareViewer.tsx index 5ca656a094..5171557584 100644 --- a/src/components/app/BenchmarkCompareViewer.tsx +++ b/src/components/app/BenchmarkCompareViewer.tsx @@ -403,7 +403,7 @@ function BucketTable({ }; const significant = comparisons - // .filter((c) => c.confidence !== 'LOW' && c.effectSize !== 'Negligible') + .filter((c) => c.confidence !== 'LOW' && c.effectSize !== 'Negligible') .sort( (a, b) => Math.abs(b.newMean - b.baseMean) - Math.abs(a.newMean - a.baseMean) From f1f0c2e8a63426fdc428ea0e90a77e4fc8027dcf Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Fri, 3 Jul 2026 15:57:21 -0400 Subject: [PATCH 5/9] Simplify mannWhitneyPValue signature: take input arrays directly. The tie-correction counts can be built directly from the two input arrays, so mannWhitneyPValue no longer needs the pre-concatenated allValues array. Every caller was passing baseIter/newIter and separately concatenating them into allValues just for this function. --- .../benchmark/compare-benchmark-stats.ts | 16 ++-------------- .../benchmark/perf-compare-stats.ts | 12 ++++++++---- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/profile-logic/benchmark/compare-benchmark-stats.ts b/src/profile-logic/benchmark/compare-benchmark-stats.ts index 628faa316a..90dfbf68cc 100644 --- a/src/profile-logic/benchmark/compare-benchmark-stats.ts +++ b/src/profile-logic/benchmark/compare-benchmark-stats.ts @@ -163,14 +163,8 @@ export function compareBuckets( continue; } - const allValues = [...baseIter, ...newIter]; const u = mannWhitneyU(baseIter, newIter); - const pValue = mannWhitneyPValue( - u, - baseIter.length, - newIter.length, - allValues - ); + const pValue = mannWhitneyPValue(u, baseIter, newIter); const relChange = baseMean === 0 ? Infinity : (newMean - baseMean) / baseMean; const delta = cliffsDelta(u, baseIter.length, newIter.length); @@ -238,14 +232,8 @@ export function compareIterationTotals( ): ScoreComparison { const baseMean = mean(baseIter); const newMean = mean(newIter); - const allValues = [...baseIter, ...newIter]; const u = mannWhitneyU(baseIter, newIter); - const pValue = mannWhitneyPValue( - u, - baseIter.length, - newIter.length, - allValues - ); + const pValue = mannWhitneyPValue(u, baseIter, newIter); const relChange = baseMean === 0 ? Infinity : (newMean - baseMean) / baseMean; const delta = cliffsDelta(u, baseIter.length, newIter.length); const effectSize = interpretEffectSize(delta); diff --git a/src/profile-logic/benchmark/perf-compare-stats.ts b/src/profile-logic/benchmark/perf-compare-stats.ts index 96f7f49144..e5f3fad5aa 100644 --- a/src/profile-logic/benchmark/perf-compare-stats.ts +++ b/src/profile-logic/benchmark/perf-compare-stats.ts @@ -101,13 +101,17 @@ export function mannWhitneyU(a: number[], b: number[]): number { export function mannWhitneyPValue( u: number, - n1: number, - n2: number, - allValues: number[] + a: number[], + b: number[] ): number { + const n1 = a.length; + const n2 = b.length; const mu = (n1 * n2) / 2; const counts = new Map(); - for (const v of allValues) { + for (const v of a) { + counts.set(v, (counts.get(v) ?? 0) + 1); + } + for (const v of b) { counts.set(v, (counts.get(v) ?? 0) + 1); } let tieCorrection = 0; From af9ea6875ae0a4e62e6719761783826b2fd6380f Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Fri, 3 Jul 2026 15:58:17 -0400 Subject: [PATCH 6/9] Store benchmark per-iteration totals as Float64Array subarrays. The extractor previously produced a per-bucket number[] copy of the dense bucketIterationTotals buffer, at ~200k small allocations per profile. Widen SparseBucketEntry.iterationTotals to Float64Array | number[] and hand out subarray() views instead. The downstream consumers (mean, mannWhitneyU, mannWhitneyPValue) are widened to ArrayLike and use indexed loops, and buildKeyMap now borrows entries on first insert, only materialising a fresh Float64Array on the rare key-collision case. The CLI serialiser converts Float64Array to plain arrays via a JSON.stringify replacer, since typed arrays stringify as {"0": ...} otherwise. --- src/node-tools/extract-benchmark-stats.ts | 10 +++- .../benchmark/compare-benchmark-stats.ts | 53 +++++++++++++------ .../benchmark/extract-benchmark-stats.ts | 23 ++++---- .../benchmark/perf-compare-stats.ts | 23 +++++--- 4 files changed, 74 insertions(+), 35 deletions(-) diff --git a/src/node-tools/extract-benchmark-stats.ts b/src/node-tools/extract-benchmark-stats.ts index c680084775..fe4ecfeb2b 100644 --- a/src/node-tools/extract-benchmark-stats.ts +++ b/src/node-tools/extract-benchmark-stats.ts @@ -24,7 +24,15 @@ async function main() { const profile = await unserializeProfileOfArbitraryFormat(uint8Array.buffer); const stats = extractBenchmarkStatsFromProfile(profile, harness); - fs.writeFileSync(argv.output, JSON.stringify(stats)); + // Float64Array serializes as an object with numeric keys under the default + // JSON.stringify behaviour; the extractor returns typed subarrays for the + // browser hot path, so convert them here for the CLI's JSON output. + fs.writeFileSync( + argv.output, + JSON.stringify(stats, (_key, value) => + value instanceof Float64Array ? Array.from(value) : value + ) + ); console.log( `Wrote ${stats.suites.length} suites, ` + `${stats.globalBuckets.length} global buckets, ` + diff --git a/src/profile-logic/benchmark/compare-benchmark-stats.ts b/src/profile-logic/benchmark/compare-benchmark-stats.ts index 90dfbf68cc..a3485b15f4 100644 --- a/src/profile-logic/benchmark/compare-benchmark-stats.ts +++ b/src/profile-logic/benchmark/compare-benchmark-stats.ts @@ -54,14 +54,22 @@ type KeyMapEntry = { /** Human-readable display name for this key (taken from the first bucket * seen with this key — usually a function name). */ displayName: string; - iterationTotals: number[]; + /** Borrowed from `entry.iterationTotals` on first insert, then replaced by a + * fresh Float64Array on collision (see buildKeyMap). Callers must not mutate + * this unless `owned` is true. */ + iterationTotals: ArrayLike; + /** True iff `iterationTotals` is a fresh Float64Array owned by this entry. */ + owned: boolean; /** Func index of the highest-weight bucket with this key (representative). */ representativeFunc: IndexIntoFuncTable; /** Sum of iterationTotals for that representative bucket alone. */ representativeWeight: number; }; -/** Build a key → iterationTotals + representative-func map for a set of sparse bucket entries. */ +/** Build a key → iterationTotals + representative-func map for a set of sparse + * bucket entries. Iteration-total arrays are borrowed by reference until a + * collision forces a copy — the extraction step returns subarrays of a shared + * Float64Array, so avoiding copies here saves ~200k small allocations per side. */ function buildKeyMap( buckets: SparseBucketEntry[], bucketKeys: string[], @@ -74,19 +82,28 @@ function buildKeyMap( const name = bucketNames[entry.bucketIndex] ?? `bucket#${entry.bucketIndex}`; const func = bucketFuncs[entry.bucketIndex]; + const iterTotals = entry.iterationTotals; + const iterLen = iterTotals.length; let weight = 0; - for (const v of entry.iterationTotals) { - weight += v; + for (let i = 0; i < iterLen; i++) { + weight += iterTotals[i]; } const existing = map.get(key); if (existing !== undefined) { - // If the same key appears twice (two different funcs that collapsed to - // the same matching key — e.g. an inlined and non-inlined copy of the - // same JS function), sum their iteration totals together. Pick the - // heaviest as the representative func, since the flame graph can only - // focusSelf on one func. - for (let i = 0; i < existing.iterationTotals.length; i++) { - existing.iterationTotals[i] += entry.iterationTotals[i]; + // Two funcs collapsed to the same matching key (e.g. an inlined and + // non-inlined copy of the same JS function). Sum their iteration totals + // together; on the first collision materialise into a fresh Float64Array + // since the borrowed entry may be a subarray of the source buffer. + let dest: Float64Array; + if (existing.owned) { + dest = existing.iterationTotals as Float64Array; + } else { + dest = new Float64Array(existing.iterationTotals); + existing.iterationTotals = dest; + existing.owned = true; + } + for (let i = 0; i < iterLen; i++) { + dest[i] += iterTotals[i]; } if (weight > existing.representativeWeight) { existing.representativeFunc = func; @@ -96,7 +113,8 @@ function buildKeyMap( } else { map.set(key, { displayName: name, - iterationTotals: entry.iterationTotals.slice(), + iterationTotals: iterTotals, + owned: false, representativeFunc: func, representativeWeight: weight, }); @@ -190,15 +208,16 @@ export function compareBuckets( return results; } -export function mean(arr: number[]): number { - if (arr.length === 0) { +export function mean(arr: ArrayLike): number { + const n = arr.length; + if (n === 0) { return 0; } let sum = 0; - for (const v of arr) { - sum += v; + for (let i = 0; i < n; i++) { + sum += arr[i]; } - return sum / arr.length; + return sum / n; } /** Sum all bucket iterationTotals element-wise to get a per-iteration total for a suite. */ diff --git a/src/profile-logic/benchmark/extract-benchmark-stats.ts b/src/profile-logic/benchmark/extract-benchmark-stats.ts index d5833aa2bc..4056ab525e 100644 --- a/src/profile-logic/benchmark/extract-benchmark-stats.ts +++ b/src/profile-logic/benchmark/extract-benchmark-stats.ts @@ -47,8 +47,11 @@ import type { export type SparseBucketEntry = { /** Index into the profile's global bucket list. */ bucketIndex: number; - /** Weight sum per iteration, length = iterationCount. */ - iterationTotals: number[]; + /** Weight sum per iteration, length = iterationCount. Held as a typed + * subarray of the underlying `bucketIterationTotals` buffer in the browser + * path (no copy). CLI-loaded stats parse back to `number[]` — both types + * support indexed access/assignment and `.slice()`. */ + iterationTotals: Float64Array | number[]; }; export type SuiteStats = { @@ -248,11 +251,11 @@ export function extractBenchmarkStatsFromProfile( if (suiteScores.bucketTotals[b] === 0) { continue; } - const iterationTotals: number[] = new Array(iterationCount); const base = b * iterationCount; - for (let i = 0; i < iterationCount; i++) { - iterationTotals[i] = suiteScores.bucketIterationTotals[base + i]; - } + const iterationTotals = suiteScores.bucketIterationTotals.subarray( + base, + base + iterationCount + ); buckets.push({ bucketIndex: b, iterationTotals }); } @@ -293,10 +296,10 @@ export function extractBenchmarkStatsFromProfile( if (total === 0) { continue; } - const iterationTotals: number[] = new Array(iterationCount); - for (let i = 0; i < iterationCount; i++) { - iterationTotals[i] = globalIterTotals[base + i]; - } + const iterationTotals = globalIterTotals.subarray( + base, + base + iterationCount + ); globalBuckets.push({ bucketIndex: b, iterationTotals }); } diff --git a/src/profile-logic/benchmark/perf-compare-stats.ts b/src/profile-logic/benchmark/perf-compare-stats.ts index e5f3fad5aa..45a3ff4824 100644 --- a/src/profile-logic/benchmark/perf-compare-stats.ts +++ b/src/profile-logic/benchmark/perf-compare-stats.ts @@ -85,10 +85,17 @@ export function median(arr: number[]): number { // Mann-Whitney U // --------------------------------------------------------------------------- -export function mannWhitneyU(a: number[], b: number[]): number { +export function mannWhitneyU( + a: ArrayLike, + b: ArrayLike +): number { let u = 0; - for (const ai of a) { - for (const bj of b) { + const aLen = a.length; + const bLen = b.length; + for (let i = 0; i < aLen; i++) { + const ai = a[i]; + for (let j = 0; j < bLen; j++) { + const bj = b[j]; if (ai < bj) { u += 1; } else if (ai === bj) { @@ -101,17 +108,19 @@ export function mannWhitneyU(a: number[], b: number[]): number { export function mannWhitneyPValue( u: number, - a: number[], - b: number[] + a: ArrayLike, + b: ArrayLike ): number { const n1 = a.length; const n2 = b.length; const mu = (n1 * n2) / 2; const counts = new Map(); - for (const v of a) { + for (let i = 0; i < n1; i++) { + const v = a[i]; counts.set(v, (counts.get(v) ?? 0) + 1); } - for (const v of b) { + for (let i = 0; i < n2; i++) { + const v = b[i]; counts.set(v, (counts.get(v) ?? 0) + 1); } let tieCorrection = 0; From 6a15fb090c04e483cca4a9c7200e9326cee08468 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Fri, 3 Jul 2026 15:59:32 -0400 Subject: [PATCH 7/9] Speed up mannWhitneyU with sort + two-pointer merge. Replace the naive O(n1 * n2) nested loop with an O((n1 + n2) log(n1 + n2)) algorithm: sort a and b once each, then walk them in a single pass with two monotonic pointers into the sorted b that count b[j] < ai and b[j] <= ai respectively. At n=200 (Speedometer iterations) and ~200k MWU calls per profile pair, this drops the compare-benchmark load-time contribution of mannWhitneyU from ~30% to ~3%. --- .../benchmark/perf-compare-stats.ts | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/profile-logic/benchmark/perf-compare-stats.ts b/src/profile-logic/benchmark/perf-compare-stats.ts index 45a3ff4824..f95066b6a2 100644 --- a/src/profile-logic/benchmark/perf-compare-stats.ts +++ b/src/profile-logic/benchmark/perf-compare-stats.ts @@ -85,23 +85,37 @@ export function median(arr: number[]): number { // Mann-Whitney U // --------------------------------------------------------------------------- +// Sort a and b once, then walk both in a single pass to count pairs where +// ai < bj and ai === bj. Both pointers move monotonically because a and b +// are sorted, giving O((n1 + n2) log(n1 + n2)) time overall — a large win +// over the naive O(n1 * n2) at n ≈ 200, which the compare-benchmark path +// runs ~200k times per profile pair. export function mannWhitneyU( a: ArrayLike, b: ArrayLike ): number { - let u = 0; const aLen = a.length; const bLen = b.length; + const aSorted = + a instanceof Float64Array ? new Float64Array(a) : Float64Array.from(a); + const bSorted = + b instanceof Float64Array ? new Float64Array(b) : Float64Array.from(b); + aSorted.sort(); + bSorted.sort(); + let u = 0; + // jLo = count of bSorted[j] < ai (first j with bSorted[j] >= ai). + // jHi = count of bSorted[j] <= ai (first j with bSorted[j] > ai). + let jLo = 0; + let jHi = 0; for (let i = 0; i < aLen; i++) { - const ai = a[i]; - for (let j = 0; j < bLen; j++) { - const bj = b[j]; - if (ai < bj) { - u += 1; - } else if (ai === bj) { - u += 0.5; - } + const ai = aSorted[i]; + while (jLo < bLen && bSorted[jLo] < ai) { + jLo++; + } + while (jHi < bLen && bSorted[jHi] <= ai) { + jHi++; } + u += bLen - jHi + 0.5 * (jHi - jLo); } return u; } From 6e2a073f48a290f418db3827325d1cd275536b31 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Fri, 3 Jul 2026 16:44:15 -0400 Subject: [PATCH 8/9] Concatenate stream output manually. Before: https://share.firefox.dev/4azGzbu (230ms concatenation) After: https://share.firefox.dev/4vNj739 (60ms concatenation, 3.8x faster) --- src/utils/gz.worker.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/utils/gz.worker.js b/src/utils/gz.worker.js index 30230bb640..49acd93488 100644 --- a/src/utils/gz.worker.js +++ b/src/utils/gz.worker.js @@ -3,7 +3,24 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ async function readableStreamToBuffer(stream) { - return new Uint8Array(await new Response(stream).arrayBuffer()); + const reader = stream.getReader(); + const chunks = []; + let totalLength = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + totalLength += value.byteLength; + } + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; } onmessage = async (e) => { From 8600e06a84abe9029a3e416aa2ff101e4d3c683d Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Tue, 7 Jul 2026 16:39:31 -0400 Subject: [PATCH 9/9] Fix dark mode --- src/components/app/BenchmarkCompareViewer.css | 110 +++++++++++++----- 1 file changed, 82 insertions(+), 28 deletions(-) diff --git a/src/components/app/BenchmarkCompareViewer.css b/src/components/app/BenchmarkCompareViewer.css index 26f2d27682..4a7cc69062 100644 --- a/src/components/app/BenchmarkCompareViewer.css +++ b/src/components/app/BenchmarkCompareViewer.css @@ -1,4 +1,25 @@ .benchmarkCompareViewer { + --internal-text-muted: var(--grey-70); + --internal-text-hint: var(--grey-60); + --internal-cell-border: var(--grey-20); + --internal-strong-border: var(--grey-40); + --internal-header-background: var(--grey-10); + --internal-row-hover-background: var(--grey-10); + --internal-expansion-background: var(--grey-10); + --internal-inner-hover-background: var(--grey-20); + --internal-inner-expansion-background: var(--grey-20); + --internal-panel-border: var(--grey-30); + --internal-panel-label-background: var(--grey-20); + --internal-spinner-track: var(--grey-30); + --internal-spinner-active: var(--blue-50); + --internal-error-border: var(--red-50); + --internal-error-background: #ffe8e8; + --internal-error-color: var(--red-70); + --internal-regressed-color: var(--red-70); + --internal-improved-color: var(--green-80); + --internal-regressed-background: #ffe8e8; + --internal-improved-background: #d3f3d8; + width: 100%; min-height: 100%; box-sizing: border-box; @@ -9,6 +30,31 @@ background: var(--base-background-color); } +:root.dark-mode { + .benchmarkCompareViewer { + --internal-text-muted: var(--grey-30); + --internal-text-hint: var(--grey-40); + --internal-cell-border: var(--grey-70); + --internal-strong-border: var(--grey-50); + --internal-header-background: var(--grey-80); + --internal-row-hover-background: rgb(255 255 255 / 0.06); + --internal-expansion-background: rgb(255 255 255 / 0.04); + --internal-inner-hover-background: rgb(255 255 255 / 0.09); + --internal-inner-expansion-background: rgb(255 255 255 / 0.07); + --internal-panel-border: var(--grey-60); + --internal-panel-label-background: var(--grey-80); + --internal-spinner-track: var(--grey-70); + --internal-spinner-active: var(--blue-40); + --internal-error-border: var(--red-60); + --internal-error-background: rgb(255 0 57 / 0.15); + --internal-error-color: #ff8a8a; + --internal-regressed-color: #ff8a8a; + --internal-improved-color: #7ed77e; + --internal-regressed-background: rgb(255 0 57 / 0.22); + --internal-improved-background: rgb(48 230 11 / 0.18); + } +} + .benchmarkResults { font-size: 14px; } @@ -24,16 +70,16 @@ flex-direction: column; align-items: center; padding: 4em; - color: var(--grey-60); + color: var(--internal-text-hint); gap: 1em; } .benchmarkSpinner { width: 2em; height: 2em; - border: 3px solid var(--grey-30); + border: 3px solid var(--internal-spinner-track); border-radius: 50%; - border-top-color: var(--blue-50); + border-top-color: var(--internal-spinner-active); animation: benchmarkSpin 0.8s linear infinite; } @@ -47,10 +93,10 @@ .benchmarkError { padding: 1em; - border: 1px solid var(--red-50, #ff0039); + border: 1px solid var(--internal-error-border); border-radius: 4px; - background: var(--red-10, #ffe8e8); - color: var(--red-70, #a4000f); + background: var(--internal-error-background); + color: var(--internal-error-color); } /* Profile URLs */ @@ -59,7 +105,7 @@ display: flex; flex-direction: column; margin-bottom: 1.5em; - color: var(--grey-70); + color: var(--internal-text-muted); font-size: 0.9em; gap: 0.25em; word-break: break-all; @@ -86,7 +132,7 @@ .benchmarkNoChanges { padding: 0.5em 0; - color: var(--grey-60); + color: var(--internal-text-hint); font-style: italic; } @@ -110,7 +156,7 @@ .benchmarkTable td { box-sizing: border-box; padding: 4px 8px; - border-bottom: 1px solid var(--grey-20); + border-bottom: 1px solid var(--internal-cell-border); text-align: left; white-space: nowrap; } @@ -119,7 +165,7 @@ position: sticky; z-index: 1; top: 0; - background: var(--grey-10); + background: var(--internal-header-background); font-weight: bold; } @@ -139,7 +185,7 @@ } .benchmarkRow--overall td { - border-bottom: 2px solid var(--grey-40); + border-bottom: 2px solid var(--internal-strong-border); font-weight: bold; } @@ -154,7 +200,7 @@ } .benchmarkRow--suite-expandable:hover { - background: var(--grey-10); + background: var(--internal-row-hover-background); } .benchmarkCell--suiteLabel { @@ -165,7 +211,7 @@ display: inline-block; width: 1em; margin-right: 0.25em; - color: var(--grey-60); + color: var(--internal-text-hint); font-size: 0.75em; text-align: center; user-select: none; @@ -173,7 +219,7 @@ .benchmarkRow--expansion > td { padding: 0.5em 0 0.5em 2.5em; - background: var(--grey-10); + background: var(--internal-expansion-background); } .benchmarkRow--expansion .benchmarkTable { @@ -188,27 +234,35 @@ */ .benchmarkCell--regressed { - color: var(--red-70, #a4000f); + color: var(--internal-regressed-color); } .benchmarkCell--improved { - color: var(--green-80, #006504); + color: var(--internal-improved-color); } .benchmarkCell--regressed.benchmarkCell--conf-high { - background: var(--red-10, #ffe8e8); + background: var(--internal-regressed-background); } .benchmarkCell--regressed.benchmarkCell--conf-medium { - background: color-mix(in srgb, var(--red-10, #ffe8e8) 40%, transparent); + background: color-mix( + in srgb, + var(--internal-regressed-background) 40%, + transparent + ); } .benchmarkCell--improved.benchmarkCell--conf-high { - background: var(--green-10, #d3f3d8); + background: var(--internal-improved-background); } .benchmarkCell--improved.benchmarkCell--conf-medium { - background: color-mix(in srgb, var(--green-10, #d3f3d8) 40%, transparent); + background: color-mix( + in srgb, + var(--internal-improved-background) 40%, + transparent + ); } .benchmarkCell--effect-large { @@ -226,12 +280,12 @@ } .benchmarkRow--bucket-expandable:hover { - background: var(--grey-20); + background: var(--internal-inner-hover-background); } .benchmarkRow--bucket-expansion > td { padding: 0.5em 0; - background: var(--grey-20); + background: var(--internal-inner-expansion-background); white-space: normal; } @@ -254,23 +308,23 @@ min-width: 0; height: 280px; flex-direction: column; - border: 1px solid var(--grey-30); + border: 1px solid var(--internal-panel-border); border-radius: 4px; background: var(--base-background-color); } .bucketFlameGraphSide__label { padding: 4px 8px; - border-bottom: 1px solid var(--grey-30); - background: var(--grey-20); - color: var(--grey-70); + border-bottom: 1px solid var(--internal-panel-border); + background: var(--internal-panel-label-background); + color: var(--internal-text-muted); font-size: 0.85em; font-weight: bold; text-transform: uppercase; } .bucketFlameGraphSide__sampleCount { - color: var(--grey-60); + color: var(--internal-text-hint); font-weight: normal; text-transform: none; } @@ -287,6 +341,6 @@ flex: 1; align-items: center; justify-content: center; - color: var(--grey-60); + color: var(--internal-text-hint); font-style: italic; }