From 2905e71cce5767577c8a43ad14da056ec15830d2 Mon Sep 17 00:00:00 2001 From: "@jaydenOoOo" Date: Sat, 8 Aug 2026 21:21:42 +0800 Subject: [PATCH] Stop the served findings contradicting the evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verifying the live deploy caught the shipped Q13 finding still asserting that daily autocorrelation "predicted every outcome measured so far" — the exact claim the Q13 experiment refuted hours earlier. The verdict string, the explainer and the docs were all updated; the Finding record baked into the snapshot was not, so the deployed site was making a claim the project had already disproved. Q13 is now recorded as refuted, with the serverless numbers, the non-monotone shape, the failed spectral-entropy replacement and the low-frequency lead that does not survive Bonferroni. Q4 quoted "19 of 24 settings", which no configuration of evaluate_frontier.py can produce — it emits three traces by six ratios. Replaced with the counts the corrected run actually gives: strictly better in 9 of 18, worse in 6, identical in 3 at the ratio where p95 and the derived target coincide by construction. The regime matters more than the count and is now stated: derived wins below 19:1 and loses above. Q10's evidence was one sentence with no numbers. It now carries the frontier figures that support it. This is the third time a served string has drifted from docs/EVAL.md, so it is now tested rather than remembered. test_findings.py pins that Q13 does not reassert the disproved claim, that Q1 names the percentile recommender rather than threshold HPA as what it lost to, that Q4 does not quote the unreproducible count, and that no confirmed finding claims a win for forecasting without naming the regime it holds in. Thin evidence is also caught — which is what flagged Q10. README now links the live deployment. --- README.md | 5 +++ scripts/build_snapshot.py | 54 +++++++++++++++++++--------- tests/test_findings.py | 75 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 16 deletions(-) create mode 100644 tests/test_findings.py diff --git a/README.md b/README.md index cc7b4b5..a4c1b05 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ **Capacity planning that tells you whether prediction can help before it sells you a predictor — then sizes capacity from the price of failure rather than from convention.** +**Live: [delphi-c5pp.onrender.com](https://delphi-c5pp.onrender.com)** — paste a demand +series and get a verdict. On a free tier, so the first request after an idle period takes +about a minute to wake the container; the snapshot is baked at image build and the badge +says `demo` because it is. + DELPHI set out to show that a calibrated forecast beats conventional autoscaling. Measured against the recommender Kubernetes actually ships, it mostly does not. The useful result is the boundary: forecasting pays when capacity is committed for hours at a time *and* demand diff --git a/scripts/build_snapshot.py b/scripts/build_snapshot.py index 60e8e79..72b5df1 100644 --- a/scripts/build_snapshot.py +++ b/scripts/build_snapshot.py @@ -161,14 +161,21 @@ def findings() -> list[Finding]: question="Does a price-derived compliance target beat a conventional fixed one?", prior="Only when prices are asymmetric enough to matter.", answer=( - "Yes in 19 of 24 settings. The newsvendor identity q* = C_u/(C_u+C_o) sets " - "the percentile that convention normally fixes at p95." + "Only where the true price ratio differs from the one p95 silently assumes. " + "Strictly better in 9 of 18 settings, worse in 6, identical in 3 — and the " + "split is not random: the derived target wins whenever C_u/C_o is below 19 " + "and loses above it." ), verdict="confirmed", evidence=( - "At C_u/C_o = 19 the derived target equals the p95 convention exactly and " - "the two agree to the cent — a forced algebraic identity that serves as a " - "consistency check on the harness." + "Three traces x six cost ratios, re-measured after the 2026-08-08 " + "actuation-delay fix. At C_u/C_o = 19 the derived target equals the p95 " + "convention exactly and the two agree to the cent — a forced algebraic " + "identity that serves as a consistency check on the harness, not a win. " + "Above 19 the derived target loses in realised terms, which is a negative " + "against this project's own hypothesis; the likely mechanism is that a " + "seasonal-naive p98/p99 is its least trustworthy region, so sizing off a " + "miscalibrated tail buys capacity that does not pay for itself." ), ), Finding( @@ -181,7 +188,16 @@ def findings() -> list[Finding]: "reached a 1% violation rate at all." ), verdict="confirmed", - evidence="Demand p95/mean was near 1.2; there was nothing for a forecaster to add.", + evidence=( + "On Azure Functions workload 1, static at 6 replicas holds a 0.5% violation " + "rate for $41.9 and the cheapest point on the whole frontier is also static, " + "at $34.9. Neither the budget-paced nor the newsvendor controller reaches 1% " + "violations on this workload at any setting. The reason is in the data rather " + "than mysterious: demand averages about 15,244 invocations/min against a p95 " + "of 18,092, so the frontier's entire dynamic range is 23.8% of its cheapest " + "point and there is very little for a forecaster to add. A stable workload " + "does not need prediction, and reporting otherwise would be dishonest." + ), ), Finding( question_id="Q11", @@ -220,18 +236,24 @@ def findings() -> list[Finding]: question="Does daily autocorrelation predict whether forecasting will pay?", prior="Proposed as a diagnostic after the mixed results above.", answer=( - "It predicted every outcome measured so far. High-autocorrelation workloads " - "reward forecasting at long commitments; low-autocorrelation ones never did, " - "at any commitment length." + "Only on aggregated demand, and it does not generalise past it. On fleet " + "traces the 0.50 cutoff is nearly perfect. On individual serverless workloads " + "it is a coin flip, and the relationship is not even monotone." ), - verdict="open", + verdict="refuted", evidence=( - "Across 7 workloads x 4 forecasters x 4 quantiles, the 0.50 cutoff gets 27 of " - "28 cells right at a 6-hour commitment. But there is a real exception in the " - "unflattering direction: materna-2 at r=0.450, below the threshold, won 2 of 4 " - "at 12 hours while materna-1 at r=0.494, above it, won none. Autocorrelation " - "orders these workloads well and predicts the extremes reliably; it is not a " - "calibrated boundary, and a value near 0.45-0.50 does not settle the question." + "Across 7 fleet-aggregate workloads x 4 forecasters x 4 quantiles the cutoff " + "gets 27 of 28 cells right at a 6-hour commitment. Tested on 43 individual " + "Azure Functions workloads, sampled to over-represent the borderline band, it " + "scores 51% at 6 hours and 47% at 12 — worse than always predicting that " + "forecasting pays. Win rate rises from 12-25% below r=0.20 to 67-83% in " + "0.20-0.50, then falls again above 0.70. Replacing it with spectral entropy, " + "the forecastability literature's standard measure, was tried and did worse " + "(AUC 0.396 and 0.456). The one candidate showing signal — the share of " + "spectral power slower than the commitment window — reaches AUC 0.700 at 12 " + "hours but does not survive Bonferroni across the six tests run, so it is " + "recorded as a lead rather than promoted. What survives: below r=0.20, " + "forecasting failed to pay on every population tested." ), ), ] diff --git a/tests/test_findings.py b/tests/test_findings.py new file mode 100644 index 0000000..f3ff273 --- /dev/null +++ b/tests/test_findings.py @@ -0,0 +1,75 @@ +"""The shipped findings must not contradict `docs/EVAL.md`. + +This has now gone wrong three times, in the same way each time: a measurement changed, the +evaluation was updated, and a string baked into the served surface was not. The verdict text +claimed nothing below 0.50 ever won; the explainer claimed a scope the data did not support; +and the Q13 finding went on asserting that daily autocorrelation "predicted every outcome +measured so far" for hours after the experiment that refuted it. + +Prose cannot be fully pinned by a test. What *can* be pinned is that a finding does not +assert the specific things the evaluation has since disproved, and that the numbers it +quotes are the ones the current scripts actually produce. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from build_snapshot import findings # type: ignore[import-not-found] + +from delphi.api.snapshot import Finding + + +@pytest.fixture(scope="module") +def by_id() -> dict[str, Finding]: + return {finding.question_id: finding for finding in findings()} + + +def test_every_finding_carries_a_verdict_and_its_evidence(by_id: dict[str, Finding]) -> None: + assert by_id, "the snapshot must ship findings" + for finding in by_id.values(): + assert finding.verdict in {"confirmed", "refuted", "open"} + assert len(finding.evidence) > 80, f"{finding.question_id} evidence is too thin" + assert finding.prior, f"{finding.question_id} has no pre-registered prior" + + +def test_q13_does_not_claim_the_threshold_generalises(by_id: dict[str, Finding]) -> None: + """Q13 was refuted on 43 serverless workloads. The shipped text must say so.""" + q13 = by_id["Q13"] + text = f"{q13.answer} {q13.evidence}".lower() + + assert q13.verdict == "refuted" + assert "predicted every outcome" not in text, "this is the claim Q13 disproved" + assert "coin flip" in text or "51%" in text, "the negative result must be stated" + assert "serverless" in text or "azure functions" in text, "the scope must be named" + + +def test_q1_records_the_percentile_recommender_as_the_incumbent(by_id: dict[str, Finding]) -> None: + """Beating threshold HPA proves little; the finding must name what it measured against.""" + q1 = by_id["Q1"] + text = f"{q1.answer} {q1.evidence}".lower() + assert q1.verdict == "refuted" + assert "percentile" in text + + +def test_q4_quotes_counts_the_frontier_script_can_produce(by_id: dict[str, Finding]) -> None: + """The previous text claimed 19 of 24; the script emits three traces x six ratios.""" + q4 = by_id["Q4"] + text = f"{q4.answer} {q4.evidence}" + assert "19 of 24" not in text, "that count is not reproducible from evaluate_frontier.py" + assert "18" in text or "9 of 18" in text + + +def test_no_finding_claims_an_unqualified_win_for_forecasting(by_id: dict[str, Finding]) -> None: + """The project's headline is that forecasting loses to the incumbent in the autoscaling + regime and wins only under commitment. No finding may state the unscoped version.""" + for finding in by_id.values(): + text = f"{finding.answer} {finding.evidence}".lower() + if "forecasting" in text and finding.verdict == "confirmed": + assert any( + word in text + for word in ("commitment", "aggregat", "below 19", "price", "static", "week") + ), f"{finding.question_id} claims a win without naming the regime it holds in"