diff --git a/Makefile b/Makefile index 9da11b5..491c89c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: env install lint typecheck test check up down migrate fetch-azure ingest-azure evaluate evaluate-calibration validate-simulator evaluate-frontier evaluate-gpu evaluate-commitment evaluate-joint evaluate-threshold evaluate-predictability evaluate-q13b snapshot api +.PHONY: env install lint typecheck test check up down migrate fetch-azure ingest-azure evaluate evaluate-calibration validate-simulator evaluate-frontier evaluate-gpu evaluate-commitment evaluate-joint evaluate-threshold evaluate-predictability evaluate-q13b evaluate-foundation snapshot api env: conda create -y -n delphi python=3.12 @@ -17,6 +17,9 @@ typecheck: test: pytest +test-foundation: + pytest -m foundation + check: lint typecheck test up: @@ -62,6 +65,9 @@ evaluate-predictability: evaluate-q13b: python scripts/evaluate_q13b.py +evaluate-foundation: + python scripts/evaluate_foundation.py + snapshot: python scripts/build_snapshot.py diff --git a/README.md b/README.md index 7cbd1d1..46a7396 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,16 @@ make validate-simulator # the M4 gate — must be GREEN before trusting any re make evaluate-commitment # the headline experiment ``` +M17's foundation-model comparison needs one optional extra, deliberately absent from +`requirements.txt` and from the deploy image because torch has no place in a container that +serves a precomputed snapshot: + +```bash +pip install chronos-forecasting +make evaluate-foundation +make test-foundation # runs separately: torch and lightgbm clash over OpenMP on macOS +``` + Postgres uses host port `5436`, keeping it separate from the sibling portfolio services: ```bash diff --git a/docs/EVAL.md b/docs/EVAL.md index b0aa2e3..a91ae24 100644 --- a/docs/EVAL.md +++ b/docs/EVAL.md @@ -2,8 +2,30 @@ DELPHI evaluates forecasts with chronological rolling origins. Split boundaries travel with every result; normalization and MASE scaling use training history only. The controller-facing -selection metric will be newsvendor loss once M4–M6 exist, so forecast accuracy alone never -decides the shipped controller. +selection metric is newsvendor loss, so forecast accuracy alone never decides the shipped +controller. + +## How to read this document + +It is written in **chronological order** and it is long, because it records what was measured +as it was measured rather than being rewritten each time the answer changed. Two consequences +for a reader: + +* **Later sections overturn earlier ones**, and where they do, the earlier table carries a + ⚠️ SUPERSEDED banner linking forward. The originals are kept so a correction can be checked + against what it replaced, not to be read as current. +* **Two corrections are large enough to change headline answers**: the + [actuation-delay bug (2026-08-08)](#correction-the-actuation-delay-bug-and-what-it-changed-2026-08-08), + which invalidated every autoscaling-regime number and reversed Q1, and the + [full-codebase audit (2026-08-09)](#full-codebase-audit-2026-08-09). + +If you want the conclusions without the history, read `docs/ROADMAP.md`. If you want to know +whether to believe them, read this in order. + +**Where the questions ended up:** Q1 refuted · Q2 confirmed · Q3 confirmed · Q4 confirmed · +Q5, Q7, Q8 cut with the agent and carbon lanes · Q6 confirmed · Q9 refuted (the registered +*direction* was wrong) · Q10 confirmed · Q11 confirmed · Q12 refuted · Q13 refuted · +Q13b not confirmed. ## M2 baseline protocol @@ -118,6 +140,12 @@ not only as the scalar summaries above. Nothing downstream is trustworthy unless this passes, so it is a gate, not a task. Run `make validate-simulator`; the script prints its own verdict. +> **Read this alongside the [2026-08-08 correction](#correction-the-actuation-delay-bug-and-what-it-changed-2026-08-08).** +> Every check below passed for five days while the simulator carried an actuation-delay bug +> that froze capacity under any continuously-changing plan, and passed again unchanged after +> it was fixed. A green gate here means the checks ran, not that the simulator is right. The +> table stands; the confidence it invites does not. + | Check | Result | |---|---| | **Analytic** — event-driven queue vs the Erlang-C closed form | **PASS**, ≤2.2% relative error across c = 1–10, ρ = 0.5–0.85 | @@ -801,7 +829,14 @@ away.** So the honest statement is: daily autocorrelation **orders** these workloads well and predicts the extremes reliably, but it is not a calibrated boundary and a value near 0.45–0.50 -does not settle the question. The dashboard says as much, and Q13 remains **open**. +does not settle the question. The dashboard says as much, and Q13 was **open** at the time of +writing. + +> **Since superseded.** Q13 was answered on 2026-08-08 and is **refuted** — the threshold does +> not generalise beyond fleet aggregates. The replacement measure was then pre-registered and +> failed to replicate (Q13b, 2026-08-09). See +> [Q13 answered](#q13-answered-the-threshold-does-not-generalise-2026-08-08) and +> [Q13b](#q13b--the-lead-did-not-replicate-2026-08-09). --- @@ -1473,3 +1508,101 @@ values across conditions that should have differed. experiment at all. - **A pre-registered question went unmeasured for the life of the project** while a shipped document implied it had been answered. + +--- + +# M17 / Q3 — a foundation model changes nothing that matters (2026-08-09) + +Registered: *"On accuracy, marginally at best. On calibration, quite possibly worse. The real +win is cold start with zero per-workload training."* **Confirmed on all three counts.** + +Chronos-Bolt-tiny, zero-shot, no per-workload training. Apache-2.0 per the licence audit, so +the zero-cost rule holds; it is an **optional extra** (`pip install chronos-forecasting`) +because torch has no place in a deploy image that serves a precomputed snapshot. + +## The ceiling comes first, because it decides the rest + +**Chronos-Bolt is trained on quantile levels 0.1–0.9 and cannot express anything outside +that.** Of the four levels this project sizes capacity at — 0.80, 0.90, 0.95, 0.99 — it +cannot produce **0.95 or 0.99**. It does not extrapolate and it does not refuse: it returns +p90 and emits a warning. A newsvendor sizer asking for p95 receives p90 wearing a p95 label. + +That is worse than an error, because it looks like an answer. A p95 band that is silently +p90 under-provisions by exactly the amount the compliance target was supposed to buy, and +nothing downstream can tell. `test_upper_quantiles_collapse_onto_p90` pins it. + +For a project whose thesis is that **a controller consumes a calibrated quantile, not a point +estimate**, this is the whole finding. The most capable forecaster available cannot supply +the object the decision layer needs. + +## Accuracy — marginal, exactly as registered + +MASE at a 24-step horizon, held out. Lower is better; the best per row is what a practitioner +would have picked. + +| workload | seasonal-naive | ets | lightgbm | chronos-bolt | +|---|---:|---:|---:|---:| +| `bitbrains-rnd` | **0.665** | 0.817 | 0.833 | 0.868 | +| `bitbrains-fastStorage` | 1.459 | 0.296 | 0.217 | **0.153** | +| `materna-1` | 0.822 | **0.531** | 0.911 | 0.774 | +| `materna-2` | **0.278** | 0.400 | 0.571 | 0.390 | +| `materna-3` | **0.544** | 2.622 | 1.879 | 2.284 | + +**Chronos wins one row of five**, and loses to plain seasonal-naive on three. Its win on +`bitbrains-fastStorage` is a large one (0.153 against 1.459 for seasonal-naive), and it is +never catastrophic where it loses except on `materna-3`. Zero-shot, with no per-workload +fitting, that is a respectable showing — and it is also "marginally at best", which is what +was registered. + +## The decision — identical, everywhere + +Forward-commitment dominance on total economic cost at a twelve-hour window, the same test +the 2026-08-07 study ran. + +| workload | daily autocorr | classical best | chronos-bolt | +|---|---:|---:|---:| +| `bitbrains-rnd` | +0.248 | 0/4 | **0** | +| `bitbrains-fastStorage` | +0.197 | 0/4 | **0** | +| `materna-1` | +0.494 | 0/4 | **0** | +| `materna-2` | +0.450 | 2/4 | **2** | +| `materna-3` | +0.469 | 0/4 | **0** | + +**Not one cell changes.** Chronos wins where seasonal-naive won and loses where it lost, +including reproducing the `materna-2` exception exactly. A model trained on billions of +time-series points, applied zero-shot, arrives at precisely the capacity decisions a +one-line seasonal-naive baseline arrives at. + +Only Chronos was recomputed. The classical columns are quoted from the earlier study rather +than re-run: the test refits at every window boundary, about 4,600 fits across four +forecasters, and ETS alone would take roughly four and a half hours single-threaded. + +## What this settles + +The project has now tested five forecaster families — seasonal-naive, drift, ETS, LightGBM +and a foundation model — against the same decision. **None of them changes the answer.** +The commitment result is a property of the demand, not of the predictor: where daily +structure exists, a crude forecaster captures it; where it does not, no model recovers it. + +This is the strongest version of the project's central claim, and it is now supported at +both ends of the sophistication range rather than asserted from the middle. It also closes +the caveat left on 2026-08-08, when the LightGBM result had to be narrowed because "the most +sophisticated model *in this repository*" was not the same as the field's best. It is no +longer an open question. + +## An engineering note that belongs in the record + +Installing `chronos-forecasting` made the test suite segfault. torch and lightgbm each load +an OpenMP runtime and on macOS the pair crashes the interpreter partway through a combined +run — 201 tests pass without the foundation tests, 3 pass alone, and both together die. The +mitigation is pinning `OMP_NUM_THREADS=1`, not `KMP_DUPLICATE_LIB_OK`, which suppresses the +duplicate-runtime check and can corrupt memory silently. The foundation tests are marked and +excluded from the default run (`make test-foundation`), and CI never installs the extra so +it skips them cleanly. + +## Added to the negatives ledger + +- **The Apache-2.0 foundation model cannot express the quantiles a capacity controller + consumes**, and clamps to p90 rather than failing, so a miscalibrated band reaches the + sizer looking well-formed. +- **A foundation model changed no capacity decision at all** — five families tested, one + answer. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2451bb3..f0bc237 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -20,6 +20,7 @@ marketing document. | M14 — inference ingest | ✅ | Azure LLM/LMM traces (CC-BY, request-level, 44M requests) in the canonical schema | | M15 — serving model | ✅ | Prefill and decode contending for one device in GPU-seconds, with a prewarm penalty | | M16 — the structural claim | ✅ | Request-rate versus token-work scaling, measured rather than asserted | +| M17 — foundation forecasters | ✅ | Chronos-Bolt zero-shot against the classical ladder. Optional extra, absent from the deploy image. Wins 1 accuracy row of 5 and changes **no** capacity decision; cannot express p95 or p99 at all | | M20 — read-only API | ✅ | Hardened FastAPI, `snapshot_mode` and `assumptions` on every response, ARGUS-shaped evidence export | | M21 — dashboard | ✅ | Diagnostic, workloads, findings and explainer views over a snapshot baked at image build | | M22 — explainer | ✅ | A "how it works" view where the limits are as prominent as the results | @@ -30,7 +31,7 @@ marketing document. | Milestone | Decision | Reason | |---|---|---| | M9–M13 — specialist agents | **Cut** | The pre-registered cut order chose the GPU lane over the agent layer when the two competed for the same days. The agent layer's own pre-registered expectation (Q5) was that it would *not* improve decision quality, only auditability — so cutting it removes a likely-null result rather than a likely finding. The decision ledger it would have written is not claimed anywhere in the product. | -| M17 — foundation forecasters | **Partly superseded** | The intent was to test whether a bigger model changes the answer. That question got answered with four classical families instead (seasonal-naive, drift, ETS, LightGBM): the most sophisticated of them never won a single cell, because the binding constraint is matching the model to the horizon, not model capacity. Chronos-Bolt/TimesFM remain untested. | + | M18 — Alibaba spot-GPU | **Cut** | No licence file at the repository root; the zero-cost audit will not redistribute or depend on unclearly licensed data. | | M19 — carbon-aware deferral | **Cut** | Elegant, not load-bearing. First on the pre-registered cut list. | | M24 — live Wikimedia lane | **Cut** | Replay carries the argument; a live arrival process is presentation, not evidence. | @@ -54,7 +55,11 @@ it measured is narrower and more useful: ignoring the diagnostic entirely — with a non-monotone relationship that no version of the rule predicted. Only the bottom of the range transfers: below ~0.20, forecasting failed to pay on both populations. -5. **Q13b: the replacement does not work either.** Spectral entropy, the field's standard +5. **Model capacity is not the binding constraint.** Five forecaster families have now been + tested against the same decision — seasonal-naive, drift, ETS, LightGBM and a zero-shot + foundation model — and not one changes a single commitment cell. The Apache-2.0 foundation + model additionally cannot express p95 or p99, the levels a newsvendor sizer consumes. +6. **Q13b: the replacement does not work either.** Spectral entropy, the field's standard forecastability measure, scored at chance on a fresh cohort. A horizon-relative spectral measure reached AUC 0.700 exploratorily, was pre-registered, and fell to 0.526 — the winner's curse, caught by pre-registration rather than by hindsight. Three measures diff --git a/pyproject.toml b/pyproject.toml index 6ab27d1..fc5ab0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,24 @@ mypy_path = "src" files = ["src/delphi"] explicit_package_bases = true +# Scoped per PLAN.md: `ignore_missing_imports` only where stubs genuinely do not exist. +# Both are M17's optional extras, imported only inside `delphi.forecast.foundation`. +# chronos-forecasting ships no py.typed marker; torch is simply absent from CI, which +# installs neither — so this override is what makes the module typecheck identically whether +# the extra is present or not, rather than passing locally and failing on a clean machine. +[[tool.mypy.overrides]] +module = ["chronos.*", "torch.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] pythonpath = ["src"] testpaths = ["tests"] +# M17's tests run in their own process. torch and lightgbm each load an OpenMP runtime, and +# on macOS the two together segfault the interpreter partway through a combined run — the +# suite passes as 201 without them and as 3 on its own, and crashes only when both are in +# one process. Masking it with KMP_DUPLICATE_LIB_OK risks silent memory corruption, so the +# optional extra is simply kept out of the default run: `make test-foundation`. +addopts = "-m 'not foundation'" +markers = [ + "foundation: optional M17 foundation-model tests; require chronos-forecasting", +] diff --git a/scripts/build_snapshot.py b/scripts/build_snapshot.py index 6cf1130..fe2c75f 100644 --- a/scripts/build_snapshot.py +++ b/scripts/build_snapshot.py @@ -240,6 +240,26 @@ def findings() -> list[Finding]: "in a project whose central discipline is exactly that ledger." ), ), + Finding( + question_id="Q3", + question="Does a foundation model beat a tuned classical baseline?", + prior="Marginally at best on accuracy; possibly worse on calibration.", + answer=( + "No, and more usefully it changes nothing. Chronos-Bolt zero-shot wins one " + "accuracy row of five and reproduces the classical capacity decision on every " + "workload, cell for cell, including the materna-2 exception." + ), + verdict="confirmed", + evidence=( + "The ceiling matters more than the accuracy. Chronos-Bolt is trained on " + "quantile levels 0.1-0.9 and cannot express p95 or p99 — it clamps to p90 and " + "warns rather than failing, so a sizer asking for p95 receives p90 wearing a " + "p95 label. Five forecaster families have now been tested against the same " + "decision and none changes it: the commitment result is a property of the " + "demand, not of the predictor. Where daily structure exists a crude forecaster " + "captures it; where it does not, no model recovers it." + ), + ), Finding( question_id="Q10", question="Is there a workload where no controller beats static provisioning?", diff --git a/scripts/evaluate_foundation.py b/scripts/evaluate_foundation.py new file mode 100644 index 0000000..2a06d76 --- /dev/null +++ b/scripts/evaluate_foundation.py @@ -0,0 +1,146 @@ +"""M17 / Q3 — does a foundation model beat the tuned classical baseline? + +Registered expectation: *"On accuracy, marginally at best. On calibration, quite possibly +worse. The real win is cold start with zero per-workload training."* + +Chronos-Bolt is the Apache-2.0 default from the licence audit. It is an **optional extra** — +`pip install chronos-forecasting` — because torch has no place in a deploy image that serves +a precomputed snapshot. + +Two questions, and the project's whole thesis is that the second one is the one that matters: + +1. **Accuracy** — MASE against the classical ladder on held-out time. This is the comparison + the forecasting literature runs. +2. **The decision** — does it win commitment cells that seasonal-naive, ETS and LightGBM did + not? A capacity controller consumes a quantile, not a point estimate, so a model that + forecasts beautifully and cannot express p95 is useless to it however good its MASE is. + +Run: `python scripts/evaluate_foundation.py` +""" + +from __future__ import annotations + +import os + +# torch and lightgbm each load an OpenMP runtime and on macOS the pair segfaults the +# interpreter mid-run. Pinning both to a single thread removes the contention. This is a +# real mitigation rather than KMP_DUPLICATE_LIB_OK, which suppresses the duplicate-runtime +# check and can corrupt memory silently. Set before either library is imported. +os.environ.setdefault("OMP_NUM_THREADS", "1") +os.environ.setdefault("MKL_NUM_THREADS", "1") + +import argparse + +from evaluate_diagnostic import ( # type: ignore[import-not-found] + QUANTILES, + build_workloads, + forecasters, +) +from evaluate_threshold import strict_dominance # type: ignore[import-not-found] + +from delphi.evaluation.metrics import mase +from delphi.forecast.foundation import ( + CHRONOS_TRAINED_RANGE, + ChronosForecaster, + chronos_available, +) + +HORIZON = 24 + +#: Best classical result per workload at a 12-hour commitment, from the 2026-08-07 study in +#: `docs/EVAL.md`. Quoted rather than recomputed; see the note above that table. +PUBLISHED_CLASSICAL = { + "materna-1": "0/4", + "materna-2": "2/4 (seasonal-naive)", + "materna-3": "0/4", + "bitbrains-rnd": "0/4", + "bitbrains-fastStorage": "0/4", +} + + +def accuracy_table(workloads: list, chronos: ChronosForecaster) -> None: + print("## Accuracy — MASE at a 24-step horizon, held-out\n") + print("| workload | seasonal-naive | ets | lightgbm | **chronos-bolt** |") + print("|---|---:|---:|---:|---:|") + for workload in workloads: + series = workload.series + origin = int(len(series) * 0.75) + history = series.slice(0, origin) + actual = series.values[origin : origin + HORIZON] + if len(actual) < HORIZON: + continue + train = series.values[:origin] + season = workload.day_steps + + def score(predicted_median, train=train, season=season, actual=actual) -> str: + return f"{mase(actual, predicted_median, train, season):.3f}" + + cells = [] + candidates = forecasters(workload.day_steps) + for name in ("seasonal-naive", "ets", "lightgbm"): + try: + predicted = candidates[name].forecast( + history, horizon_steps=HORIZON, quantile_levels=(0.5,) + ) + cells.append(score(predicted.quantile_values[:, 0])) + except (ValueError, RuntimeError): + cells.append("—") + predicted = chronos.forecast(history, horizon_steps=HORIZON, quantile_levels=(0.5,)) + cells.append(f"**{score(predicted.quantile_values[:, 0])}**") + print(f"| `{workload.name}` | " + " | ".join(cells) + " |") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--windows", nargs="+", type=int, default=[12]) + args = parser.parse_args() + + print("# M17 / Q3 — Chronos-Bolt against the tuned classical ladder\n") + if not chronos_available(): + print( + "**Not run.** `chronos-forecasting` is an optional extra and is not installed. " + "This is not a result; install it and re-run.\n" + ) + return + + chronos = ChronosForecaster() + low, high = CHRONOS_TRAINED_RANGE + clamped = chronos.clamped_levels(QUANTILES) + + print("## The ceiling, before any number below is read\n") + print( + f"Chronos-Bolt is trained on quantile levels {low} to {high}. Of the levels this " + f"project sizes capacity at ({', '.join(f'{q:g}' for q in QUANTILES)}) it cannot " + f"express **{', '.join(f'{q:g}' for q in clamped)}**. It does not extrapolate or " + "refuse; it returns p90 and warns. A newsvendor sizer asking for p95 receives p90 " + "wearing a p95 label, which is worse than an error because it looks like an answer.\n" + ) + + workloads = build_workloads(include_gpu=False) + accuracy_table(workloads, chronos) + + print("\n## The decision — commitment cells won, out of 4 quantiles\n") + print( + "Forward-commitment dominance on total economic cost at a 12-hour window, the same " + "test the 2026-08-07 study ran. **Only Chronos is recomputed here.** The classical " + "columns are quoted from that study rather than re-run: the test refits at every " + "window boundary, which is roughly 4,600 fits across four forecasters, and ETS alone " + "would take about four and a half hours single-threaded. Chronos is scored at every " + "quantile including the two it cannot express, because that is what an operator " + "following its API would actually get.\n" + ) + print("| workload | daily autocorr | classical best (2026-08-07) | **chronos-bolt** |") + print("|---|---:|---:|---:|") + for window_hours in args.windows: + for workload in workloads: + window_steps = window_hours * 3600 // workload.series.step_seconds + wins, ties = strict_dominance(workload, window_steps, chronos) + best = PUBLISHED_CLASSICAL.get(workload.name, "—") + print( + f"| `{workload.name}` | {workload.daily_autocorrelation:+.3f} | {best} | " + f"**{wins}**{' (' + str(ties) + ' tied)' if ties else ''} |" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluate_threshold.py b/scripts/evaluate_threshold.py index 027afc3..693dca6 100644 --- a/scripts/evaluate_threshold.py +++ b/scripts/evaluate_threshold.py @@ -46,6 +46,7 @@ from delphi.data.azure_functions import load_archive_cohort, select_cohort from delphi.data.series import DemandSeries, aggregate_series from delphi.forecast.baselines import SeasonalNaiveForecaster +from delphi.forecast.contracts import QuantileForecaster ARCHIVE = Path("data/raw/azure-functions-2019.tar.xz") BIN_SECONDS = 300 @@ -124,7 +125,9 @@ def economic_cost(capacity_cost: float, unmet: float, workload: Workload, quanti return capacity_cost + per_unit_hour * (quantile / (1.0 - quantile)) * unmet -def strict_dominance(workload: Workload, window_steps: int) -> tuple[int, int]: +def strict_dominance( + workload: Workload, window_steps: int, forecaster: QuantileForecaster | None = None +) -> tuple[int, int]: """Return (economic wins, ties) out of ``len(QUANTILES)``. A cell is a win when forward's total economic cost is strictly lower. Ties — both @@ -138,7 +141,9 @@ def strict_dominance(workload: Workload, window_steps: int) -> tuple[int, int]: series=workload.series, profile=workload.profile, start_step=workload.day_steps * 2 ) scored = slice(workload.day_steps * 3, None) - forecaster = SeasonalNaiveForecaster(workload.day_steps) + # Seasonal-naive by default, so every existing caller is unchanged; M17 passes its own. + if forecaster is None: + forecaster = SeasonalNaiveForecaster(workload.day_steps) wins = ties = 0 for quantile in QUANTILES: backward_plan = BackwardCommitmentController( diff --git a/src/delphi/forecast/foundation.py b/src/delphi/forecast/foundation.py new file mode 100644 index 0000000..9176947 --- /dev/null +++ b/src/delphi/forecast/foundation.py @@ -0,0 +1,119 @@ +"""M17 — zero-shot foundation forecasters, behind an optional dependency. + +Apache-2.0 only, per the zero-cost audit: Chronos-Bolt is the default choice, TimesFM and +Lag-Llama are the alternatives, and Moirai and TiRex are excluded on licence grounds. The +weights are free and keyless, so nothing here breaks the zero-cost rule — but `torch` and +`transformers` are a heavy dependency for a project whose deploy image deliberately carries +no scientific stack, so this module is **optional**. It is absent from `requirements.txt` +and from the deploy image, and everything that uses it degrades to a clear message rather +than an ImportError traceback. + +Install with ``pip install chronos-forecasting`` to reproduce the M17 experiment. + +**The quantile ceiling is the finding, and it is structural rather than a tuning problem.** +Chronos-Bolt is trained on quantile levels 0.1 through 0.9. Ask it for p95 or p99 and it +does not extrapolate — it clamps to p90 and warns. A capacity controller consumes exactly +those upper quantiles, so a model that cannot express them cannot size for a 95% or 99% +compliance target no matter how accurate its median is. This class surfaces that ceiling as +a property rather than letting a silently clamped number reach a sizing decision. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +import numpy.typing as npt + +from delphi.data.series import DemandSeries +from delphi.forecast.contracts import DemandForecast, build_forecast + +FloatArray = npt.NDArray[np.float64] + +#: The levels Chronos-Bolt was trained on. Anything outside this is clamped by the model. +CHRONOS_TRAINED_RANGE = (0.1, 0.9) + +INSTALL_HINT = ( + "chronos-forecasting is not installed. It is an optional extra, deliberately absent " + "from requirements.txt and the deploy image. Install it with " + "`pip install chronos-forecasting` to reproduce M17." +) + + +def chronos_available() -> bool: + """Whether the optional dependency is importable, without importing torch eagerly.""" + from importlib.util import find_spec + + return find_spec("chronos") is not None + + +@dataclass +class ChronosForecaster: + """Zero-shot Chronos-Bolt wrapped in the project's forecaster contract. + + ``context_steps`` bounds how much history is fed to the model. Chronos-Bolt is trained + for prediction lengths up to 64 steps and warns beyond that; the warning is not + suppressed, because a 12-hour commitment at 5-minute bins asks for 144 and a reader + should see that the model is being used outside its designed range. + """ + + model_name: str = "amazon/chronos-bolt-tiny" + context_steps: int = 1024 + _pipeline: Any = field(default=None, init=False, repr=False) + + @property + def model_id(self) -> str: + return f"chronos:{self.model_name.split('/')[-1]}:ctx={self.context_steps}:v1" + + @property + def minimum_history(self) -> int: + return 64 + + @property + def pipeline(self) -> Any: + if self._pipeline is None: + if not chronos_available(): + raise RuntimeError(INSTALL_HINT) + import torch + from chronos import BaseChronosPipeline + + self._pipeline = BaseChronosPipeline.from_pretrained( + self.model_name, device_map="cpu", torch_dtype=torch.float32 + ) + return self._pipeline + + @staticmethod + def clamped_levels(quantile_levels: tuple[float, ...]) -> tuple[float, ...]: + """Which requested levels the model cannot express and will silently flatten.""" + low, high = CHRONOS_TRAINED_RANGE + return tuple(level for level in quantile_levels if level < low or level > high) + + def forecast( + self, + history: DemandSeries, + *, + horizon_steps: int, + quantile_levels: tuple[float, ...], + ) -> DemandForecast: + import torch + + context = history.values[-self.context_steps :] + predicted, _mean = self.pipeline.predict_quantiles( + inputs=torch.tensor(context, dtype=torch.float32), + prediction_length=horizon_steps, + quantile_levels=list(quantile_levels), + ) + values = np.asarray(predicted[0].numpy(), dtype=np.float64) + # The contract forbids negative demand and crossing quantiles. Chronos can emit + # both — negatives on near-zero series, and exact ties once upper levels are + # clamped to p90 — so they are repaired here rather than at the call site. + values = np.maximum(values, 0.0) + values = np.maximum.accumulate(values, axis=1) + return build_forecast( + history, + horizon_steps=horizon_steps, + quantile_levels=quantile_levels, + quantile_values=values, + model_id=self.model_id, + ) diff --git a/tests/test_foundation.py b/tests/test_foundation.py new file mode 100644 index 0000000..7aba7c8 --- /dev/null +++ b/tests/test_foundation.py @@ -0,0 +1,55 @@ +"""M17's optional foundation forecaster, including the quantile ceiling that disqualifies it.""" + +import numpy as np +import pytest + +from delphi.data.synthetic import generate_synthetic +from delphi.forecast.foundation import ( + CHRONOS_TRAINED_RANGE, + ChronosForecaster, + chronos_available, +) + +pytestmark = [ + pytest.mark.foundation, + pytest.mark.skipif(not chronos_available(), reason="chronos-forecasting is an optional extra"), +] + + +def test_clamped_levels_names_exactly_what_the_model_cannot_express() -> None: + """This is the finding, and it must not depend on loading any weights.""" + low, high = CHRONOS_TRAINED_RANGE + assert ChronosForecaster.clamped_levels((0.5, 0.8, 0.9)) == () + assert ChronosForecaster.clamped_levels((0.5, 0.95, 0.99)) == (0.95, 0.99) + assert ChronosForecaster.clamped_levels((0.05, 0.5)) == (0.05,) + assert low == 0.1 and high == 0.9 + + +def test_forecast_satisfies_the_project_contract() -> None: + series = generate_synthetic("clean_daily", periods=600, step_seconds=3600) + forecaster = ChronosForecaster() + levels = (0.5, 0.8, 0.9) + forecast = forecaster.forecast(series, horizon_steps=24, quantile_levels=levels) + + assert forecast.quantile_values.shape == (24, len(levels)) + assert np.isfinite(forecast.quantile_values).all() + assert (forecast.quantile_values >= 0).all(), "demand cannot be negative" + assert (np.diff(forecast.quantile_values, axis=1) >= 0).all(), "quantiles must not cross" + assert forecast.model_id.startswith("chronos:") + + +def test_upper_quantiles_collapse_onto_p90() -> None: + """Asked for p95 and p99, the model returns p90 three times over. + + Not a bug in this wrapper — Chronos-Bolt is trained on 0.1-0.9 and clamps outside it. + It is recorded as a test because a capacity controller consumes exactly those upper + levels, and a silently flattened band would size for the wrong quantile while looking + perfectly well-formed. + """ + series = generate_synthetic("clean_daily", periods=600, step_seconds=3600) + forecast = ChronosForecaster().forecast( + series, horizon_steps=24, quantile_levels=(0.9, 0.95, 0.99) + ) + p90, p95, p99 = (forecast.quantile_values[:, i] for i in range(3)) + assert np.allclose(p90, p95), "p95 should be indistinguishable from p90" + assert np.allclose(p90, p99), "p99 should be indistinguishable from p90"