aggregate.py

#!/usr/bin/env python3
"""Aggregate bench-results CSVs (tour.csv, build-times.csv) into a markdown summary."""
import csv, os, statistics

ROOT = os.path.dirname(os.path.abspath(__file__))
RES = os.path.join(ROOT, 'bench-results')


def load(name):
    p = os.path.join(RES, name)
    if not os.path.exists(p):
        return []
    with open(p) as f:
        return list(csv.DictReader(f))


def stats(vals):
    if not vals:
        return None
    return {'n': len(vals), 'mean': statistics.mean(vals), 'min': min(vals), 'max': max(vals)}


def fmt(s, nd=1):
    return '-' if not s else f"{s['mean']:.{nd}f} (n={s['n']}, {s['min']:.{nd}f}–{s['max']:.{nd}f})"


def delta(a, b):  # CD vs AB reduction
    if not a or not b or a['mean'] == 0:
        return ''
    return f"{(1 - b['mean'] / a['mean']) * 100:+.0f}%"


def main():
    tours = [r for r in load('tour.csv') if r['status'] == 'Succeeded' and r['run'] != 'smoke']
    builds = load('build-times.csv')

    print('## Scene tour (Terminal -> Garden/Oasis/Cockpit -> Terminal, previews retained)')
    print('| metric | AB | CD | CD vs AB |')
    print('|---|---|---|---|')
    rows = [
        ('Garden load (ms)', 'gardenMs'), ('Oasis load (ms)', 'oasisMs'), ('Cockpit load (ms)', 'cockpitMs'),
        ('baseline allocated (MB)', 'baseAllocMB'), ('peak allocated (MB)', 'peakAllocMB'),
        ('RESIDUAL allocated (MB)', 'residAllocMB'), ('after UnloadUnusedAssets (MB)', 'uuaAllocMB'),
        ('residual reserved (MB)', 'residReservedMB'), ('residual system used (MB)', 'residSystemMB'),
    ]
    for name, key in rows:
        d = {}
        for v in ('AB', 'CD'):
            d[v] = stats([float(r[key]) for r in tours if r['variant'] == v and float(r[key]) >= 0])
        print(f"| {name} | {fmt(d['AB'])} | {fmt(d['CD'])} | {delta(d['AB'], d['CD'])} |")

    # residual delta over baseline: the memory the tour failed to return
    print()
    print('| derived | AB | CD |')
    print('|---|---|---|')
    d = {}
    for v in ('AB', 'CD'):
        vals = [float(r['residAllocMB']) - float(r['baseAllocMB']) for r in tours if r['variant'] == v]
        d[v] = stats(vals)
    print(f"| residual − baseline (unreturned MB) | {fmt(d['AB'])} | {fmt(d['CD'])} |")

    print()
    print('## Content build times (seconds, warm shader cache)')
    print('| scenario | AB | CD | CD vs AB |')
    print('|---|---|---|---|')
    for label, name in [('nochange', 'no change (cache hit)'), ('incr', '1 material changed')]:
        d = {}
        for v in ('AB', 'CD'):
            vals = [float(r['seconds']) for r in builds
                    if r['kind'] == 'contentBuild' and r['variant'] == v
                    and r['label'].startswith(label) and 'revert' not in r['label']
                    and r['error'] == '']
            d[v] = stats(vals)
        print(f"| {name} | {fmt(d['AB'])} | {fmt(d['CD'])} | {delta(d['AB'], d['CD'])} |")

    bad = [r for r in load('tour.csv') if r['status'] != 'Succeeded']
    if bad:
        print(f"\nWARNING: {len(bad)} tour rows not Succeeded")


if __name__ == '__main__':
    main()