diff --git a/README.md b/README.md index cc7b4b5..a4c1b05 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ **Capacity planning that tells you whether prediction can help before it sells you a predictor — then sizes capacity from the price of failure rather than from convention.** +**Live: [delphi-c5pp.onrender.com](https://delphi-c5pp.onrender.com)** — paste a demand +series and get a verdict. On a free tier, so the first request after an idle period takes +about a minute to wake the container; the snapshot is baked at image build and the badge +says `demo` because it is. + DELPHI set out to show that a calibrated forecast beats conventional autoscaling. Measured against the recommender Kubernetes actually ships, it mostly does not. The useful result is the boundary: forecasting pays when capacity is committed for hours at a time *and* demand diff --git a/scripts/build_snapshot.py b/scripts/build_snapshot.py index 60e8e79..72b5df1 100644 --- a/scripts/build_snapshot.py +++ b/scripts/build_snapshot.py @@ -161,14 +161,21 @@ def findings() -> list[Finding]: question="Does a price-derived compliance target beat a conventional fixed one?", prior="Only when prices are asymmetric enough to matter.", answer=( - "Yes in 19 of 24 settings. The newsvendor identity q* = C_u/(C_u+C_o) sets " - "the percentile that convention normally fixes at p95." + "Only where the true price ratio differs from the one p95 silently assumes. " + "Strictly better in 9 of 18 settings, worse in 6, identical in 3 — and the " + "split is not random: the derived target wins whenever C_u/C_o is below 19 " + "and loses above it." ), verdict="confirmed", evidence=( - "At C_u/C_o = 19 the derived target equals the p95 convention exactly and " - "the two agree to the cent — a forced algebraic identity that serves as a " - "consistency check on the harness." + "Three traces x six cost ratios, re-measured after the 2026-08-08 " + "actuation-delay fix. At C_u/C_o = 19 the derived target equals the p95 " + "convention exactly and the two agree to the cent — a forced algebraic " + "identity that serves as a consistency check on the harness, not a win. " + "Above 19 the derived target loses in realised terms, which is a negative " + "against this project's own hypothesis; the likely mechanism is that a " + "seasonal-naive p98/p99 is its least trustworthy region, so sizing off a " + "miscalibrated tail buys capacity that does not pay for itself." ), ), Finding( @@ -181,7 +188,16 @@ def findings() -> list[Finding]: "reached a 1% violation rate at all." ), verdict="confirmed", - evidence="Demand p95/mean was near 1.2; there was nothing for a forecaster to add.", + evidence=( + "On Azure Functions workload 1, static at 6 replicas holds a 0.5% violation " + "rate for $41.9 and the cheapest point on the whole frontier is also static, " + "at $34.9. Neither the budget-paced nor the newsvendor controller reaches 1% " + "violations on this workload at any setting. The reason is in the data rather " + "than mysterious: demand averages about 15,244 invocations/min against a p95 " + "of 18,092, so the frontier's entire dynamic range is 23.8% of its cheapest " + "point and there is very little for a forecaster to add. A stable workload " + "does not need prediction, and reporting otherwise would be dishonest." + ), ), Finding( question_id="Q11", @@ -220,18 +236,24 @@ def findings() -> list[Finding]: question="Does daily autocorrelation predict whether forecasting will pay?", prior="Proposed as a diagnostic after the mixed results above.", answer=( - "It predicted every outcome measured so far. High-autocorrelation workloads " - "reward forecasting at long commitments; low-autocorrelation ones never did, " - "at any commitment length." + "Only on aggregated demand, and it does not generalise past it. On fleet " + "traces the 0.50 cutoff is nearly perfect. On individual serverless workloads " + "it is a coin flip, and the relationship is not even monotone." ), - verdict="open", + verdict="refuted", evidence=( - "Across 7 workloads x 4 forecasters x 4 quantiles, the 0.50 cutoff gets 27 of " - "28 cells right at a 6-hour commitment. But there is a real exception in the " - "unflattering direction: materna-2 at r=0.450, below the threshold, won 2 of 4 " - "at 12 hours while materna-1 at r=0.494, above it, won none. Autocorrelation " - "orders these workloads well and predicts the extremes reliably; it is not a " - "calibrated boundary, and a value near 0.45-0.50 does not settle the question." + "Across 7 fleet-aggregate workloads x 4 forecasters x 4 quantiles the cutoff " + "gets 27 of 28 cells right at a 6-hour commitment. Tested on 43 individual " + "Azure Functions workloads, sampled to over-represent the borderline band, it " + "scores 51% at 6 hours and 47% at 12 — worse than always predicting that " + "forecasting pays. Win rate rises from 12-25% below r=0.20 to 67-83% in " + "0.20-0.50, then falls again above 0.70. Replacing it with spectral entropy, " + "the forecastability literature's standard measure, was tried and did worse " + "(AUC 0.396 and 0.456). The one candidate showing signal — the share of " + "spectral power slower than the commitment window — reaches AUC 0.700 at 12 " + "hours but does not survive Bonferroni across the six tests run, so it is " + "recorded as a lead rather than promoted. What survives: below r=0.20, " + "forecasting failed to pay on every population tested." ), ), ] diff --git a/tests/test_findings.py b/tests/test_findings.py new file mode 100644 index 0000000..f3ff273 --- /dev/null +++ b/tests/test_findings.py @@ -0,0 +1,75 @@ +"""The shipped findings must not contradict `docs/EVAL.md`. + +This has now gone wrong three times, in the same way each time: a measurement changed, the +evaluation was updated, and a string baked into the served surface was not. The verdict text +claimed nothing below 0.50 ever won; the explainer claimed a scope the data did not support; +and the Q13 finding went on asserting that daily autocorrelation "predicted every outcome +measured so far" for hours after the experiment that refuted it. + +Prose cannot be fully pinned by a test. What *can* be pinned is that a finding does not +assert the specific things the evaluation has since disproved, and that the numbers it +quotes are the ones the current scripts actually produce. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from build_snapshot import findings # type: ignore[import-not-found] + +from delphi.api.snapshot import Finding + + +@pytest.fixture(scope="module") +def by_id() -> dict[str, Finding]: + return {finding.question_id: finding for finding in findings()} + + +def test_every_finding_carries_a_verdict_and_its_evidence(by_id: dict[str, Finding]) -> None: + assert by_id, "the snapshot must ship findings" + for finding in by_id.values(): + assert finding.verdict in {"confirmed", "refuted", "open"} + assert len(finding.evidence) > 80, f"{finding.question_id} evidence is too thin" + assert finding.prior, f"{finding.question_id} has no pre-registered prior" + + +def test_q13_does_not_claim_the_threshold_generalises(by_id: dict[str, Finding]) -> None: + """Q13 was refuted on 43 serverless workloads. The shipped text must say so.""" + q13 = by_id["Q13"] + text = f"{q13.answer} {q13.evidence}".lower() + + assert q13.verdict == "refuted" + assert "predicted every outcome" not in text, "this is the claim Q13 disproved" + assert "coin flip" in text or "51%" in text, "the negative result must be stated" + assert "serverless" in text or "azure functions" in text, "the scope must be named" + + +def test_q1_records_the_percentile_recommender_as_the_incumbent(by_id: dict[str, Finding]) -> None: + """Beating threshold HPA proves little; the finding must name what it measured against.""" + q1 = by_id["Q1"] + text = f"{q1.answer} {q1.evidence}".lower() + assert q1.verdict == "refuted" + assert "percentile" in text + + +def test_q4_quotes_counts_the_frontier_script_can_produce(by_id: dict[str, Finding]) -> None: + """The previous text claimed 19 of 24; the script emits three traces x six ratios.""" + q4 = by_id["Q4"] + text = f"{q4.answer} {q4.evidence}" + assert "19 of 24" not in text, "that count is not reproducible from evaluate_frontier.py" + assert "18" in text or "9 of 18" in text + + +def test_no_finding_claims_an_unqualified_win_for_forecasting(by_id: dict[str, Finding]) -> None: + """The project's headline is that forecasting loses to the incumbent in the autoscaling + regime and wins only under commitment. No finding may state the unscoped version.""" + for finding in by_id.values(): + text = f"{finding.answer} {finding.evidence}".lower() + if "forecasting" in text and finding.verdict == "confirmed": + assert any( + word in text + for word in ("commitment", "aggregat", "below 19", "price", "static", "week") + ), f"{finding.question_id} claims a win without naming the regime it holds in"