diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index db981175..d0820955 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,13 +1,14 @@ name: Release (PyPI trusted publishing) -# Phase 4 CI-03 (PKG-01): publish the three sibling distributions to PyPI on +# Phase 4 CI-03 (PKG-01): publish the sibling distributions to PyPI on # every v* tag, using PEP 740 trusted publishing (no API tokens stored in # secrets). Each package has its own job because trusted publishing is # registered per-package on PyPI. # # Setup checklist (one-time, before the first tagged release): # 1. Register the project as a "pending publisher" on PyPI for each of -# mostlyrightmd, mostlyrightmd-weather, mostlyrightmd-markets +# mostlyrightmd, mostlyrightmd-weather, mostlyrightmd-markets, +# mostlyrightmd-econ # against owner=mostlyrightmd, repo=mostlyright-sdk. # https://docs.pypi.org/trusted-publishers/adding-a-publisher/ # 2. Workflow ref must match: workflow filename `release.yml`, @@ -37,16 +38,16 @@ permissions: jobs: # codex iter-1 HIGH fix: tag/version-match preflight. - # release.yml fires on every v* tag, but the 3 sibling pyproject.toml files + # release.yml fires on every v* tag, but the 4 sibling pyproject.toml files # each carry their own version. Tagging v0.1.0 while one of the packages is # still at 0.1.0a1 would publish stale alpha artifacts OR fail mid-job (a # PyPI publish step that finds the version already exists fails the build, - # but the OTHER two jobs have already published their wheels — leaving PyPI + # but the OTHER jobs have already published their wheels — leaving PyPI # in a half-released state). This preflight extracts the version from each # pyproject and refuses to proceed unless GITHUB_REF_NAME == "v{version}" - # for ALL three. + # for ALL four. version-guard: - name: Verify tag matches all 3 package versions + name: Verify tag matches all 4 package versions runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -85,7 +86,8 @@ jobs: sys.exit(f"ERROR: tag {ref!r} is not a valid PEP 440 version: {exc}") for pkg in ("packages/core/pyproject.toml", "packages/weather/pyproject.toml", - "packages/markets/pyproject.toml"): + "packages/markets/pyproject.toml", + "packages/econ/pyproject.toml"): with open(pkg, "rb") as f: pyproj = tomllib.load(f) raw_version = pyproj["project"]["version"] @@ -159,3 +161,29 @@ jobs: run: uv run python scripts/check_wheel_metadata.py - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + + # Phase 29 (29-01): the 4th published distribution. Mirrors the markets job + # verbatim (own trusted-publishing environment url; own `uv build --package`). + # NOTE: `mostlyrightmd-econ` must be registered as a PyPI "pending publisher" + # (owner=mostlyrightmd, repo=mostlyright-sdk, workflow=release.yml, + # environment=pypi) BEFORE the first tagged release, exactly like the other + # three (see the one-time setup checklist at the top of this file). + publish-econ: + name: Publish mostlyrightmd-econ + needs: version-guard + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/mostlyrightmd-econ/ + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 + - name: Set up Python + run: uv python install 3.12 + - name: Build mostlyrightmd-econ wheel + sdist + run: uv build --package mostlyrightmd-econ + - name: Verify Requires-Dist pin (CI-04 gate) + run: uv run python scripts/check_wheel_metadata.py + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index ac66d5fa..3842ed05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to `mostlyright`. The format follows [Keep a Changelog](http ## [Unreleased] +### Added + +- **Econ surface conformance to the 1.13/1.14 source-identity contract + (dual-SDK, plan 29-11).** The `mostlyright.econ` vertical now mirrors the + cross-vertical [`docs/source-identity.md`](docs/source-identity.md) surface it + was unified under: `series()` is the canonical read function (`history()` is a + byte-identical, docstring-`@deprecated` silent alias — no runtime warning this + release, warning next minor, removal at 2.0); `series()`/`research_econ()` gain + `source=` (contract §1 provenance: accepted `{None,"fred","bls","bea","dol", + "fed"}`, an unknown source OR a valid authority that cannot serve the indicator + raises `ValueError` **before any network call** — never a silent fallback) and + `delivery=` (contract §2: `"live"` default | `"hosted"` raises + `SourceUnavailableError` naming `ECON_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`, the + reserved paid-tier seam); and a new `snapshot(indicator, *, as_of=None)` returns + the settlement-target state — the latest `settlement_grade=True` vintage with + `vintage_date <= as_of`, raising `IndicatorNotYetReleasedError` when nothing is + knowable. TS mirrors the whole surface (`series`/`snapshot`, `@deprecated` + `history` re-export, `source`/`delivery` options with the same loud validation). + The engine (fetchers/schema/cache/resolver/first-print logic) is untouched — a + default call is byte-identical to the pre-conformance output. Core's + `dataset()`/`research()` are **not** modified (an `include_econ` covariate seam + is an option-B question deferred to review); the econ↔weather firewall + (`test_firewall.py`) stays green. + ## [1.15.0] — 2026-07-10 — General-first re-layering: the label axis, thin markets delegators, panels, and the contributor registry Minor release, dual-SDK (PyPI 1.15.0 + npm 1.15.0 in lockstep). The SDK is now diff --git a/docs/econ-vertical.md b/docs/econ-vertical.md new file mode 100644 index 00000000..d369144a --- /dev/null +++ b/docs/econ-vertical.md @@ -0,0 +1,344 @@ +# Economic-indicators vertical (`mostlyright.econ`) + +Macro-release data for prediction-market settlement research — CPI (headline / +core / YoY), PPI, nonfarm payrolls (NFP) and revisions, U3 unemployment, initial +jobless claims, GDP, and Fed funds decisions — sourced from FRED/ALFRED + +BLS/BEA/DOL/Federal Reserve and joined to Kalshi and Polymarket econ markets for +**leakage-free settlement pairs**. + +> **Econ is standalone — a firewall vertical.** It never feeds `research()`, the +> weather merge layer (`_internal/merge/observations.py`'s `SOURCE_PRIORITY`, +> `_internal/merge/climate.py`), or `live/_sources.py`. It carries its own +> `schema.econ.observations.v1` (separate from the parity-frozen +> `schema.observation.v1`), its own per-release cache namespace, and its own error +> taxonomy. A macro release must **never** be routed through the weather +> settlement join, and the weather merge layer must never see an econ row. This +> isolation is the same discipline CWOP uses, and it is a **permanently-enforced, +> tested invariant** — `packages/econ/tests/test_firewall.py` fails if econ is +> ever registered in any of the four parity files. The workspace dependency edge +> is strictly one-way `markets → econ`; `econ` depends only on `core`. + +## Install + +```bash +pip install mostlyrightmd-econ[pandas] # series()/snapshot()/research_econ() return DataFrames +``` + +The raw fetchers and cache work without pandas; the read surfaces +(`series`, `snapshot`, `research_econ`) return a `pandas.DataFrame`, so install +the `[pandas]` extra (or any extra that pulls pandas) to use them. + +## Public surface + +```python +from mostlyright.econ import series, snapshot, releases, research_econ +``` + +The surface conforms to the cross-vertical **source-identity contract** +([`docs/source-identity.md`](source-identity.md), v1.13/1.14): `source=` always +means provenance (WHO produced the row), `delivery=` always means where the +computation runs (`"live"` local | `"hosted"` reserved seam). Both validate +loudly **before any network call**. + +### `series(indicator, from_date, to_date, *, vintages="settlement", source=None, delivery="live")` + +The canonical read — observation rows for an indicator across a date range, as a +`schema.econ.observations.v1` DataFrame. + +```python +# The settlement-grade FIRST PRINT (the value as-of the Kalshi expiration). +df = series("cpi", "2026-05-01", "2026-06-30") # vintages="settlement" (default) + +# Every vintage kept — for feature engineering. +allv = series("cpi", "2026-05-01", "2026-06-30", vintages="all") +``` + +- **The load-bearing econ difference from weather: vintage is first-class.** + Weather collapses to one row per `(station, date)` via a dedup. Econ does the + OPPOSITE — it KEEPS every vintage and filters at **read** time. `vintages="settlement"` + keeps only the settlement-grade first print (`settlement_grade == True`); + `vintages="all"` keeps every vintage. `settlement` is always a subset of `all` + (a clean partition — property-proven in `test_leakage_property.py`). +- **Not-yet-released is an error, never an empty frame.** When a window has no + relevant rows (a scheduled release has not landed), `series` raises + `IndicatorNotYetReleasedError` — it NEVER returns `[]`/`None`. +- **A below-floor window raises.** A `from_date` before the indicator's + FEDS-2026-010 first Kalshi contract raises `DataAvailabilityError` + (`reason="out_of_window"`) — a below-floor backtest asks for outcomes the venue + never priced. + +The indicator vocabulary: `cpi`, `cpi_core`, `cpi_yoy`, `nfp`, `u3`, `gdp`, +`ppi`, `ppi_yoy`, `jobless_claims`, `fed_funds`. An unknown indicator raises a +clear `ValueError` (the dispatch table is exhaustive — never a silent empty +frame). + +#### `source=` — provenance (contract §1) + +`source=` names the authority that produced the row. It never encodes delivery +or freshness. + +| `source=` | Authority | Serves | +|---|---|---| +| `None` (default) | Per-indicator default routing (unchanged behavior) | all indicators | +| `"fred"` | FRED/ALFRED realtime vintage store (settlement-grade first print, keyed) | CPI family, NFP, U3, PPI, jobless_claims | +| `"bls"` | BLS timeseries API (latest-revised) | CPI family, NFP, U3, PPI | +| `"bea"` | Bureau of Economic Analysis | GDP | +| `"dol"` | Department of Labor (initial jobless claims) | jobless_claims | +| `"fed"` | Federal Reserve Board FOMC decisions (**not** FRED) | fed_funds, fed_decision | + +The accepted set is `{None, "fred", "bls", "bea", "dol", "fed"}`. An **unknown** +source raises `ValueError` naming the set; a **valid authority that cannot serve +the indicator** (e.g. `source="bea"` for `"cpi"`) *also* raises `ValueError` — +loudly, before any network call, **never a silent fallback** to a different +provider. Econ's `source=` is the contract's own "FRED for macro series" +degenerate axis made pinnable: it stays ready for a future provider pin (an +agency-scrape vs the ALFRED vintage store) without an API change. + +#### `delivery=` — local vs hosted (contract §2) + +| `delivery=` | Where it runs | +|---|---| +| `"live"` (default) | Locally, against the public agency APIs. | +| `"hosted"` | The opt-in precomputed econ API — **reserved seam**. | + +`delivery="hosted"` raises `SourceUnavailableError` naming `ECON_HOSTED_URL` + +`MOSTLYRIGHT_API_KEY` (the reserved seam arrives in the hosted-econ phase; rows +will be byte-identical to the local `live` path). An unknown value raises +`ValueError` pre-network. + +> **`history()` is a deprecated alias of `series()`.** `history(...)` returns +> output byte-identical to `series(...)` for the same args and accepts the same +> `vintages`/`source=`/`delivery=` kwargs. It emits **no** runtime warning this +> release (matching the `research()`/`dataset()` silent-alias style); a +> `DeprecationWarning` arrives in the next minor, removal at 2.0. Migrate +> `history(...)` → `series(...)` — the signature is identical. + +### `snapshot(indicator, *, as_of=None, source=None, delivery="live")` + +The **settlement-target state** of an indicator as knowable at `as_of`: the +latest settlement-grade (`settlement_grade == True`) vintage whose +`vintage_date <= as_of` (default `as_of`: now, UTC-aware), per period. This is +the econ realization of the cross-vertical `snapshot` rule +([source-identity §4](source-identity.md)) — "settlement-target state now": the +current best settlement read of the thing a Kalshi contract settles on. + +```python +# The current settlement-grade first print (what the market would settle against). +snap = snapshot("cpi") + +# As-of a historical cutoff — only vintages knowable at that time are considered. +snap = snapshot("cpi", as_of=datetime(2026, 6, 30, tzinfo=UTC)) +``` + +- **Nothing knowable is an error, never empty.** When no settlement-grade vintage + exists at or before `as_of`, `snapshot` raises `IndicatorNotYetReleasedError` — + it NEVER returns `[]`/`None`. +- A **post-expiration revision is never the settlement snapshot**: revisions carry + `settlement_grade=False` (Kalshi contractually excludes them), so the snapshot + stays the settlement-grade first print even after a later revision lands. +- Accepts the same `source=`/`delivery=` kwargs as `series` (same loud + pre-network validation), forwarded through. + +### `releases(indicator)` + +The release calendar for an indicator — a list of `ReleaseEvent` +(`indicator`, `period`, `release_datetime` in UTC, `agency`), sorted ascending. +The companion signal to `IndicatorNotYetReleasedError`: `series` tells you a +print has not landed; `releases` tells you when to come back. + +```python +for ev in releases("nfp"): + print(ev.period, ev.release_datetime.isoformat(), ev.agency) +``` + +v1 ships a curated, offline-deterministic schedule table (no network, no key); a +live agency-calendar fetch is a documented fast-follow. An unschedulable +indicator raises `ValueError`. + +### `research_econ(series_or_contract, from_date, to_date, *, as_of=None, source=None, delivery="live")` + +Leakage-free settlement pairs — the vertical's payoff. Joins a prediction-market +contract to the settlement-grade first-print vintage of the agency indicator it +settles against, and returns training pairs that "backtest the same way they +trade." + +```python +from mostlyright.core import TimePoint + +pairs = research_econ("KXCPIYOY", "2026-05-01", "2026-06-30") + +# With a leakage cutoff: any pair whose vintage_date > as_of raises LeakageError. +pairs = research_econ("KXCPIYOY", "2026-05-01", "2026-06-30", + as_of=TimePoint("2026-06-01T00:00:00Z")) +``` + +- Resolves the contract to `(agency, indicator, settlement_grade)` via the + econ-side routing table (the SAME table the markets `kalshi_econ.resolve` reads + — they agree by construction — but it does NOT import the markets package). +- Pulls the first print via `series(..., vintages="settlement")`. +- Builds the pairs frame with `knowledge_time = vintage_date` (the leakage + cutoff). +- **Leakage guard:** when `as_of` is given, `assert_no_leakage` rejects any row + whose `knowledge_time > as_of` — a future revision can never leak into a + backtest (property-proven). +- Gains the same `source=`/`delivery=` kwargs (forwarded to its `series` read); + a default call is byte-identical to the pre-conformance output. Whether to also + add an `include_econ` covariate seam to core's `dataset()` is an option-B + question deferred to review — `research_econ` does **not** touch core. + +## Labeled limitations (read these before you trust a number) + +The econ vertical is honest about five documented limits. None is a silent gap — +each surfaces in the data or raises loudly. + +### 1. ALFRED first-print vintages are **day-granular** (the #1 landmine) + +The settlement-grade first print comes from FRED/ALFRED's realtime vintage store +(BLS serves latest-revised only — it has no vintage endpoint). ALFRED's +`realtime_start` is a **calendar date** (`YYYY-MM-DD`), so it **cannot resolve an +intraday same-morning correction** (an 8:30 ET print vs a 9:45 ET correction on +the SAME date collapse to one `vintage_date`). + +The Kalshi contract says 8:30→10:00 ET same-morning corrections DO count toward +settlement, so for that narrow window ALFRED's `settlement_grade=True` means +"first *date*-vintage per ALFRED" — equal to the true first print EXCEPT when a +same-morning correction occurred. Every ALFRED-sourced row carries +`vintage_precision="day"` so a consumer knows the vintage is day-exact, not +intraday-exact. A release-day agency-scrape fetcher (intraday truth) is a +deferred fast-follow, NOT in v1. Same-morning corrections are assumed rare (A4); +the 29-10 live smoke spot-checks first-print ≠ latest-revised. + +### 2. Public venue history is **shallow** (~50–90 recent markets/series) + +The public Kalshi API surfaces only ~50–90 RECENT settled markets per series +(oldest observed ~2026-05), and a market's candlesticks span its own ~1-month +life. The FEDS-2026-010 first-contract history (2021–2023) is **NOT retrievable +from the public endpoint** — a deep econ backtest needs the FEDS replication +dataset or a running collector. + +`mostlyright.markets.econ_trades.candles` enforces this as a first-class behavior: +a window entirely before the public-depth floor (~2026-05) returns an EMPTY +DataFrame stamped `df.attrs["public_depth_limited"] = True` — never a fabricated +deep candle. The `MAX_PUBLIC_DEPTH_NOTE` constant documents the realistic +expectation. A window below the FEDS floor raises `DataAvailabilityError`. + +### 3. Trading-Economics-settled series ship an honest labeled proxy + +48 Kalshi series (KXUSPPI PPI-MoM, KXUSNFP, and international variants) settle +against a **Trading Economics** value we cannot license. For those, +`research_econ` uses the AGENCY first print as the value but labels the pair +`settlement_grade=False` and raises a `DivergenceWarning` naming the series. It +NEVER fabricates or approximates a TE value. The TE contract template lists TE as +a fallback BELOW the agency ("first non-preliminary release" governs), so the +agency first print is usually the same number TE republishes — the divergence is +bounded but non-zero. Per-series, not per-family: within PPI, KXUSPPI (MoM) +settles to TE while KXUSPPIYOY (YoY) settles to BLS. + +### 4. Keyless vs keyed behavior — the free FRED key is the practical requirement for settlement + +Every source is free, but **"keyless" and "settlement-grade" are not the same +thing.** With **zero keys** you get latest-revised BLS values (NOT the settlement +first print) plus Fed decisions; the **free** `FRED_API_KEY` is what unlocks the +settlement-grade first prints — which is the whole point of this vertical. + +| Path | Zero keys | + free `FRED_API_KEY` | +|------|-----------|------------------------| +| Kalshi trade-api v2 (series, candlesticks) · Polymarket gamma + CLOB | Full — public, no auth | — | +| Fed FOMC decisions (Federal Reserve Board `openmarket.htm`, **NOT FRED**) | Full, `settlement_grade=True` (per-meeting midpoint + hike/hold/cut in `series_id`) | — | +| CPI / Core / NFP / U3 / **PPI (`WPSFD4`)** + their YoY | BLS latest-revised, `settlement_grade=False`; a `vintages="settlement"` read raises `IndicatorNotYetReleasedError` naming `FRED_API_KEY` | ALFRED first print, `settlement_grade=True` (day-granular) | +| GDP (annualized real growth) | **unavailable without a key** * | ALFRED first print of `A191RL1Q225SBEA`, `settlement_grade=True` | +| Jobless claims (ICSA) | **unavailable** — DOL's machine path is bot-walled (403); the bytes come from FRED | ALFRED-ICSA first release, `settlement_grade=True` | + +\* `BEA_API_KEY` (also free) adds only a **keyless GDP _latest-revised_ fallback** +(`settlement_grade=False`); GDP's settlement first print comes from FRED/ALFRED, so +with a FRED key you never need BEA. `BLS_API_KEY` (free) only raises the BLS v1→v2 +rate limit — still latest-revised. + +**Bottom line:** all sources are free public APIs; the **free FRED key is the +practical requirement for first-print settlement** (CPI/NFP/U3/PPI/GDP/jobless); Fed +decisions and latest-revised BLS work with no keys at all. Free keys: +[FRED](https://fred.stlouisfed.org/docs/api/api_key.html) · +[BEA](https://apps.bea.gov/API/signup). Keys are read from the environment (or +passed explicitly), placed only in the outbound request, and are NEVER logged or +embedded in an emitted row. **FRED-derived rows (`alfred`, and FRED-transported +`dol.icsa`) are NOT written to the local parquet cache** by default — FRED's ToU +prohibits storing/caching FRED content, so those indicators are fetch-through +(re-fetched per session). Set `MOSTLYRIGHT_PERSIST_FRED=1` to persist them under +your own acceptance of the FRED ToU. Public-domain agency rows (`bls`/`bea`/`fed`) +always persist. See the licensing section below. + +### 5. FRED-sourced rows come with terms-of-use strings attached + +The agency sources (BLS, BEA, DOL, Federal Reserve Board) are US-government +works in the public domain. FRED/ALFRED is not: the St. Louis Fed is a Reserve +Bank, not a federal agency, and its +[Terms of Use](https://fred.stlouisfed.org/legal/) prohibit — among other +things — storing/caching FRED content and using it *"in connection with the +development or training of any... machine learning"* model. The FRED path is +opt-in under **your own key and your own ToS acceptance**; see the next section +before wiring ALFRED-sourced columns into a training pipeline. + +## Data sources, licensing & required notices + +**Required notices.** This product uses the FRED® API but is not endorsed or +certified by the Federal Reserve Bank of St. Louis. This product uses the +Bureau of Economic Analysis (BEA) Data API but is not endorsed or certified by +BEA. Nothing in this package is endorsed by, or implies endorsement by, any +source agency or Reserve Bank. Please cite the source agencies when you +publish results derived from their data. + +| Source | Legal basis | Cache/persist | ML/training | Notes | +|---|---|---|---|---| +| BLS, BEA, DOL, Fed **Board** | US-gov works → public domain (17 U.S.C. §105) | Yes | Yes | Citation appreciated. BEA API rate limits: 100 req/min, 100 MB/min. | +| FRED / ALFRED (`alfred` rows) | Contract — [FRED® ToU](https://fred.stlouisfed.org/docs/api/terms_of_use.html) | **Prohibited by ToU** | **Prohibited by ToU** | Opt-in via your `FRED_API_KEY`; the keyless default path makes no FRED call. | +| Jobless claims (`dol.icsa` rows) | DOL data is public domain, **but the bytes are fetched from FRED** (`ICSA`/`ICNSA` — DOL's machine path is bot-walled) | FRED ToU applies | FRED ToU applies | Same posture as `alfred` rows. | +| Trading Economics | Proprietary — cannot license | n/a (never fetched) | n/a | The 48 TE-settled series ship the agency first print labeled `settlement_grade=False` (limitation 3). | + +The FRED path exists because ALFRED is the only machine-readable historical +first-print store (limitation 1). A fast-follow reconstructing first prints +from the agencies' own archived releases (public domain, cacheable, and +settlement-exact — it is the document Kalshi actually settles on) would demote +ALFRED to an optional convenience. Maintainers: the full per-source ToS +research, verbatim clauses, and containment decisions live in +`.planning/research/DATA-LICENSING.md`. + +## Live smoke test (pre-publish gate) + +`tests/test_econ_live_smoke.py` (`@pytest.mark.live`, excluded from CI) is the +MANDATORY real-API gate before publish (development-workflow §4). It hits the real +production endpoints with no mocks, exercises the full econ path, validates sane +output, and prints a sample row per source for a human sanity-check. + +```bash +# Keyless sub-paths (Kalshi + BLS + Polymarket) run with no keys: +uv run pytest tests/test_econ_live_smoke.py -m live -s -v + +# Full coverage (closes A1 ALFRED fidelity, A5 BEA GDP, BLOCKER-3 jobless): +export FRED_API_KEY=… # free — closes A1 + jobless first-print +export BEA_API_KEY=… # free — closes A5 +uv run pytest tests/test_econ_live_smoke.py -m live -s -v +``` + +## TS Parity + +The `@mostlyrightmd/econ` TypeScript SDK mirrors the **public surface** — +`series` (canonical), `snapshot`, `releases`, `researchEcon`, a `@deprecated` +`history` alias, and the `kalshiEcon` resolver — with an identical API and an +adapted (browser-safe, no Node-only APIs) implementation. `series` carries the +same `source`/`delivery` options with the same loud pre-fetch validation +(unknown source → throws naming the accepted set; `delivery: "hosted"` → +`SourceUnavailableError` naming `ECON_HOSTED_URL`); `snapshot({ indicator, asOf, +source, delivery })` mirrors the settlement-target semantics. The surface parity ++ live/msw-recorded validation ships in the same phase (plan 29-09's vitest +suite, extended by 29-11); this document is the shared reference for both SDKs. + +TS-specific notes: + +- The Python firewall regression (`test_firewall.py`) is Python-specific — the + four parity files it guards live in the Python `mostlyright.core` package. The + equivalent TS isolation is enforced structurally (the TS econ package has no + import of the TS weather/merge surface). +- The live smoke here is the **Python** real-API gate; the TS live/recorded gate + for Kalshi/Polymarket lives in the 29-09 vitest suite. Each SDK validates its + own runtime. +- No TS files change with this document. diff --git a/docs/source-identity.md b/docs/source-identity.md index 52e91392..fd8ed2e2 100644 --- a/docs/source-identity.md +++ b/docs/source-identity.md @@ -41,6 +41,25 @@ provider. Example: `climate(..., source="acis")` raises `ValueError` (there is n ACIS leg in that code path); the accepted set is `{None, "iem", "cli", "cli.archive"}`. +Per-vertical accepted `source=` sets (each raises loudly on an unrecognized pin, +before any network call): + +| vertical | accepted `source=` set | +| --- | --- | +| weather observations (`obs()`) | `{None, "awc", "iem", "ghcnh"}` | +| weather climate (`climate()`) | `{None, "iem", "cli", "cli.archive"}` | +| satellite (`satellite()`) | `{None, "noaa_goes"}` | +| **econ (`series()`/`snapshot()`/`research_econ()`)** | `{None, "fred", "bls", "bea", "dol", "fed"}` | + +Econ is the contract's own "FRED for macro series" example made pinnable: today +the default routing serves each indicator from its authority, and a pin names one +of `fred` (the ALFRED realtime vintage store), `bls`, `bea`, `dol`, or `fed` (the +Federal Reserve Board FOMC decisions, **not** FRED). A pin that names a valid +authority which cannot serve the requested indicator (e.g. `source="bea"` for +`"cpi"`) raises `ValueError` just as loudly as an unknown source — the axis is +ready for a future provider pin (an agency-scrape vs the ALFRED store) without an +API change. + ### `source=` pin changes row composition (retrain event) Pinning `source=` on `obs()` takes the **source-isolated exact-window** fetch @@ -74,6 +93,16 @@ correctly: `source="noaa_goes"` (provenance) × `delivery ∈ {local, hosted}` (transport). Any vertical that overloads `source=` to mean transport is a bug — see §8 for the one outstanding case (earnings) and its tracked follow-up. +**Econ carries the `delivery=` axis with a reserved hosted seam.** `series()` / +`snapshot()` / `research_econ()` accept `delivery ∈ {"live", "hosted"}`. `"live"` +(the default) runs locally against the public agency APIs; `"hosted"` is the +opt-in precomputed econ API — **reserved**, and raises `SourceUnavailableError` +naming `ECON_HOSTED_URL` + `MOSTLYRIGHT_API_KEY` until the hosted-econ phase +lands (the same engine served from `services/`; rows will be byte-identical live +vs hosted, the satellite precedent). This keeps econ's `source=` axis purely +provenance and `delivery=` purely transport from day one — the mislabel §10 +tracks for earnings does not exist in econ. + --- ## 3. Grain is vertical-defined diff --git a/packages-ts/codegen/src/codegen.ts b/packages-ts/codegen/src/codegen.ts index 978d2e23..2cdaf459 100644 --- a/packages-ts/codegen/src/codegen.ts +++ b/packages-ts/codegen/src/codegen.ts @@ -145,6 +145,12 @@ const SCHEMA_FILES = [ "schema.observation_ledger.v1.json", "schema.observation_qc.v1.json", "schema.forecast_nwp.v1.json", + // Phase 29 (29-03/29-09): the econ vertical's observation schema. Emitting it + // here is what makes the TS `@mostlyrightmd/econ` port faithful — its column + // types are codegen-derived from the SAME `schema.econ.observations.v1.json` + // the Python schema reads (the Python↔TS anti-drift contract, T-29-27). Manual + // TS schema duplication is forbidden per the Dual-SDK rule. + "schema.econ.observations.v1.json", ]; async function emitSchemas(out: FileMap): Promise { @@ -509,6 +515,78 @@ function emitKalshi(out: FileMap): void { emit(out, join(PACKAGES_TS, "markets", "src", "data", "generated", "kalshi-stations.ts"), body); } +// ------------------------------------------------------------------------- +// kalshi-econ-settlement.json → econ module (Phase 29, 29-03/29-09) +// ------------------------------------------------------------------------- +// +// The per-series Kalshi econ settlement routing table + the canonical agency- +// name vocabulary. Emitted from the SAME JSON the Python `_settlement_map` +// resolver and the `export_schemas.py` drift gate consume, so the TS resolver +// (`resolvers/kalshiEcon.ts`) agrees with Python by construction. Routing is +// keyed on the Kalshi series ROOT ticker; the resolver applies exact-then- +// longest-prefix match (so KXUSPPIYOY → BLS/true, NOT the shorter KXUSPPI → +// TradingEconomics/false it also prefix-matches). ROUTED ON AGENCY NAME ONLY — +// no settlement_sources[].url is ever emitted (Pitfall 2; T-29-06). + +interface RawEconSettlementRule { + agency: string; + contract_terms_pdf: string; + indicator: string; + settlement_grade: boolean; +} + +interface RawEconSettlement { + agency_names: string[]; + routing: Record; +} + +function emitKalshiEconSettlement(out: FileMap): void { + const absSrc = join(SCHEMAS_DIR, "kalshi-econ-settlement.json"); + const raw = JSON.parse(readFileSync(absSrc, "utf8")) as RawEconSettlement; + + // Sort the agency vocabulary + routing roots for determinism (must NOT depend + // on JSON key iteration order). + const agencyNamesSorted = [...raw.agency_names].sort(); + const routingSorted: Record< + string, + { agency: string; contractTermsPdf: string; indicator: string; settlementGrade: boolean } + > = {}; + for (const root of Object.keys(raw.routing).sort()) { + const rule = raw.routing[root]; + if (rule === undefined) continue; + routingSorted[root] = { + agency: rule.agency, + contractTermsPdf: rule.contract_terms_pdf, + indicator: rule.indicator, + settlementGrade: rule.settlement_grade, + }; + } + + const agencyNamesLit = tsLiteral(agencyNamesSorted); + const routingLit = tsLiteral(routingSorted); + + const body = `${header("kalshi-econ-settlement.json")} +// Per-series Kalshi econ settlement routing (agency NAMES, never URLs) — the +// Python↔TS anti-drift contract (T-29-27). \`SETTLEMENT_ROUTING\` is keyed on the +// canonical Kalshi series ROOT ticker; consume it via \`resolveSettlement\` which +// applies exact-then-longest-prefix match. \`settlementGrade\` is false for the +// Trading-Economics-settled series (the agency first print ships as a labeled +// proxy; a TE value is NEVER fabricated — ECON-17 / T-29-30). +export interface EconSettlementRule { + readonly agency: string; + readonly contractTermsPdf: string; + readonly indicator: string; + readonly settlementGrade: boolean; +} + +export const ECON_AGENCY_NAMES: ReadonlyArray = ${agencyNamesLit} as const; + +export const SETTLEMENT_ROUTING: Readonly> = ${routingLit} as const; +`; + + emit(out, join(PACKAGES_TS, "econ", "src", "generated", "settlement.ts"), body); +} + // ------------------------------------------------------------------------- // polymarket-city-stations.json → markets module // ------------------------------------------------------------------------- @@ -660,6 +738,10 @@ function emitDataBarrels(out: FileMap): void { const marketsBarrel = `${header("(barrel)")}\nexport * from \"./kalshi-stations.js\";\nexport * from \"./polymarket-city-stations.js\";\nexport * from \"./earnings-webcast-providers.js\";\nexport * from \"./earnings-calendar.js\";\nexport * from \"./earnings-hq-timezone.js\";\n`; emit(out, join(PACKAGES_TS, "markets", "src", "data", "generated", "index.ts"), marketsBarrel); + + // Phase 29: the econ package's generated barrel (settlement routing table). + const econBarrel = `${header("(barrel)")}\nexport * from \"./settlement.js\";\n`; + emit(out, join(PACKAGES_TS, "econ", "src", "generated", "index.ts"), econBarrel); } // ------------------------------------------------------------------------- @@ -677,6 +759,7 @@ async function generateAll(): Promise { emitEarningsProviders(files); emitEarningsCalendar(files); emitEarningsHqTimezone(files); + emitKalshiEconSettlement(files); emitDataBarrels(files); return files; } diff --git a/packages-ts/core/src/exceptions/index.ts b/packages-ts/core/src/exceptions/index.ts index 2049f18f..5b1a00f0 100644 --- a/packages-ts/core/src/exceptions/index.ts +++ b/packages-ts/core/src/exceptions/index.ts @@ -789,3 +789,96 @@ export class NoLiveDataError extends LiveStreamError { }; } } + +// --------------------------------------------------------------------------- +// IndicatorNotYetReleasedError (Phase 29 econ vertical, CONTEXT Area 4) +// --------------------------------------------------------------------------- +// +// Python↔TS lockstep mirror of +// `packages/core/src/mostlyright/core/exceptions.py::IndicatorNotYetReleasedError`. +// "Not yet released" is a NORMAL, branchable control state distinct from +// "unavailable" (a DataAvailabilityError): the release calendar says a print is +// due, but `vintage_date` for that period does not exist yet. The econ surface +// (`history` / `releases` / `researchEcon`) throws THIS instead of returning +// `[]` / `null` so a caller can tell "the data is genuinely gone" apart from +// "come back after the 8:30 ET drop" and, when known, retry at +// `expectedRelease`. +// +// Placed in core (not the econ package) for the same reason NoCWOPDataError / +// the earnings errors are: the exception taxonomy is centralized so MCP +// JSON-RPC serialization and the Python↔TS discipline apply uniformly. +// `@mostlyrightmd/econ` re-exports it. + +export interface IndicatorNotYetReleasedErrorOptions extends MostlyRightErrorOptions { + /** + * The scheduled release wall-clock when known, else omitted. Accepts a tz-aware + * ISO 8601 string or a `Date`; a `Date` is normalized to its ISO string (the + * TS analog of Python's `expected_release.isoformat()`). The payload NEVER + * fabricates a timestamp — an absent `expectedRelease` serializes to `null`. + */ + expectedRelease?: string | Date; +} + +/** + * A requested economic release is EXPECTED but has not been published yet. + * + * Mirrors the Python class: `errorCode === "INDICATOR_NOT_YET_RELEASED"`, carries + * `indicator` / `period` / `expectedRelease`, and the message reads + * `"{indicator} for period '{period}' is not yet released"` (plus + * `" (expected {iso})"` when a scheduled release is known). + * + * @example + * ```ts + * import { IndicatorNotYetReleasedError } from "@mostlyrightmd/core"; + * try { + * await history("cpi", from, to); + * } catch (e) { + * if (e instanceof IndicatorNotYetReleasedError) { + * console.warn(`Come back at ${e.expectedRelease ?? "the next drop"}`); + * } else { + * throw e; + * } + * } + * ``` + */ +export class IndicatorNotYetReleasedError extends MostlyRightError { + static override defaultErrorCode = "INDICATOR_NOT_YET_RELEASED"; + + readonly indicator: string; + readonly period: string; + /** ISO 8601 string when the scheduled release is known, else `null`. */ + readonly expectedRelease: string | null; + + constructor( + indicator: string, + period: string, + options: IndicatorNotYetReleasedErrorOptions = {}, + ) { + // Normalize expectedRelease → ISO string | null (mirror Python's + // `expected_release.isoformat()` on the message + payload). + let expected: string | null = null; + if (options.expectedRelease !== undefined) { + expected = + options.expectedRelease instanceof Date + ? options.expectedRelease.toISOString() + : options.expectedRelease; + } + let message = `${indicator} for period '${period}' is not yet released`; + if (expected !== null) { + message += ` (expected ${expected})`; + } + super(message, options); + this.indicator = indicator; + this.period = period; + this.expectedRelease = expected; + } + + protected override payload(): Record { + return { + ...super.payload(), + indicator: this.indicator, + period: this.period, + expected_release: this.expectedRelease, + }; + } +} diff --git a/packages-ts/core/src/schemas/generated/econ.observations.v1.ts b/packages-ts/core/src/schemas/generated/econ.observations.v1.ts new file mode 100644 index 00000000..d5c2883c --- /dev/null +++ b/packages-ts/core/src/schemas/generated/econ.observations.v1.ts @@ -0,0 +1,54 @@ +// AUTO-GENERATED by @mostlyrightmd/codegen from schemas/json/schema.econ.observations.v1.json. +// DO NOT EDIT — regenerate with: pnpm codegen +// Last manifest SHA recorded in schemas/EXPORT_MANIFEST.json + +export interface EconObservationsV1 { + /** + * indicator id: cpi | cpi_core | cpi_yoy | nfp | u3 | gdp | ppi | ppi_yoy | jobless_claims | fed_funds + */ + indicator: string; + /** + * leakage cutoff; equals vintage_date for econ (research_econ wires leakage on this) + */ + knowledge_time: string; + /** + * observation period the value refers to: '2026-06' monthly, '2026Q2' quarterly, '2026-06-28' weekly/date, or FOMC meeting date + */ + period: string; + /** + * wall-clock the value was released (8:30 ET etc.), UTC + */ + release_datetime?: null | string; + /** + * advance | second | third | final | revised | benchmark | preliminary + */ + release_type: "advance" | "benchmark" | "final" | "preliminary" | "revised" | "second" | "third"; + /** + * provenance of the fetch, UTC + */ + retrieved_at?: null | string; + /** + * upstream series identifier (BLS/FRED/BEA code) when applicable + */ + series_id?: null | string; + /** + * is this vintage the settlement truth (first print as-of expiration); False for later vintages AND the 48 TE-settled series + */ + settlement_grade: boolean; + /** + * per-row source identity == df.attrs['source']; one of ECON_SOURCES + */ + source: string; + /** + * index | percent | thousands_persons | percent_saar | … + */ + units?: null | string; + /** + * released numeric value; nullable — a not-yet-released period may be schema-shaped but valueless + */ + value?: null | number; + /** + * WHEN this value became known (ALFRED realtime_start / release-day capture); the leakage knowledge_time + */ + vintage_date: string; +} diff --git a/packages-ts/core/src/schemas/generated/index.ts b/packages-ts/core/src/schemas/generated/index.ts index f5762ac5..c1822aad 100644 --- a/packages-ts/core/src/schemas/generated/index.ts +++ b/packages-ts/core/src/schemas/generated/index.ts @@ -10,3 +10,4 @@ export * from "./settlement.cli.v1.js"; export * from "./observation_ledger.v1.js"; export * from "./observation_qc.v1.js"; export * from "./forecast_nwp.v1.js"; +export * from "./econ.observations.v1.js"; diff --git a/packages-ts/core/src/schemas/validators/format-map.ts b/packages-ts/core/src/schemas/validators/format-map.ts index 99b58908..21ee6757 100644 --- a/packages-ts/core/src/schemas/validators/format-map.ts +++ b/packages-ts/core/src/schemas/validators/format-map.ts @@ -13,6 +13,12 @@ export type FormatKind = "date" | "date-time"; export type SchemaFormatMap = Readonly>; const FORMAT_MAPS: Readonly> = Object.freeze({ + "schema.econ.observations.v1": Object.freeze({ + "knowledge_time": "date-time", + "release_datetime": "date-time", + "retrieved_at": "date-time", + "vintage_date": "date-time", + }), "schema.forecast_nwp.v1": Object.freeze({ "issued_at": "date-time", "retrieved_at": "date-time", diff --git a/packages-ts/core/src/schemas/validators/index.ts b/packages-ts/core/src/schemas/validators/index.ts index 285d4964..e3432a07 100644 --- a/packages-ts/core/src/schemas/validators/index.ts +++ b/packages-ts/core/src/schemas/validators/index.ts @@ -8,6 +8,7 @@ // Group A schemas always compile; Group B schemas (when added) fall through // to the null-return path in `getValidator`. +import { schema_econ_observations_v1 as validate_schema_econ_observations_v1 } from "./schema_econ_observations_v1.js"; import { schema_forecast_nwp_v1 as validate_schema_forecast_nwp_v1 } from "./schema_forecast_nwp_v1.js"; import { schema_forecast_iem_mos_v1 as validate_schema_forecast_iem_mos_v1 } from "./schema_forecast_iem_mos_v1.js"; import { schema_forecast_station_v1 as validate_schema_forecast_station_v1 } from "./schema_forecast_station_v1.js"; @@ -29,6 +30,7 @@ export type AjvValidator = ((data: unknown) => boolean) & { }; const VALIDATORS: Record = { + "schema.econ.observations.v1": validate_schema_econ_observations_v1 as unknown as AjvValidator, "schema.forecast_nwp.v1": validate_schema_forecast_nwp_v1 as unknown as AjvValidator, "schema.forecast.iem_mos.v1": validate_schema_forecast_iem_mos_v1 as unknown as AjvValidator, "schema.forecast.station.v1": validate_schema_forecast_station_v1 as unknown as AjvValidator, diff --git a/packages-ts/core/src/schemas/validators/schema_econ_observations_v1.d.ts b/packages-ts/core/src/schemas/validators/schema_econ_observations_v1.d.ts new file mode 100644 index 00000000..29aa5c43 --- /dev/null +++ b/packages-ts/core/src/schemas/validators/schema_econ_observations_v1.d.ts @@ -0,0 +1,6 @@ +// AUTO-GENERATED by @mostlyrightmd/codegen from schemas/json/schema.econ.observations.v1.json. +// DO NOT EDIT — regenerate with: pnpm codegen +// Last manifest SHA recorded in schemas/EXPORT_MANIFEST.json + +declare const schema_econ_observations_v1: ((data: unknown) => boolean) & { errors?: Array<{ instancePath: string; schemaPath: string; keyword: string; params: Record; message?: string }> | null }; +export { schema_econ_observations_v1 }; diff --git a/packages-ts/core/src/schemas/validators/schema_econ_observations_v1.js b/packages-ts/core/src/schemas/validators/schema_econ_observations_v1.js new file mode 100644 index 00000000..de404ca9 --- /dev/null +++ b/packages-ts/core/src/schemas/validators/schema_econ_observations_v1.js @@ -0,0 +1,265 @@ +// AUTO-GENERATED by @mostlyrightmd/codegen from schemas/json/schema.econ.observations.v1.json. +// DO NOT EDIT — regenerate with: pnpm codegen +// Last manifest SHA recorded in schemas/EXPORT_MANIFEST.json + +"use strict"; +export const schema_econ_observations_v1 = validate27; +const schema38 = {"$id":"https://mostlyright.dev/schemas/schema.econ.observations.v1.json","$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"indicator":{"description":"indicator id: cpi | cpi_core | cpi_yoy | nfp | u3 | gdp | ppi | ppi_yoy | jobless_claims | fed_funds","type":"string"},"knowledge_time":{"description":"leakage cutoff; equals vintage_date for econ (research_econ wires leakage on this)","format":"date-time","type":"string"},"period":{"description":"observation period the value refers to: '2026-06' monthly, '2026Q2' quarterly, '2026-06-28' weekly/date, or FOMC meeting date","type":"string"},"release_datetime":{"description":"wall-clock the value was released (8:30 ET etc.), UTC","format":"date-time","type":["null","string"]},"release_type":{"description":"advance | second | third | final | revised | benchmark | preliminary","enum":["advance","benchmark","final","preliminary","revised","second","third"],"type":"string"},"retrieved_at":{"description":"provenance of the fetch, UTC","format":"date-time","type":["null","string"]},"series_id":{"description":"upstream series identifier (BLS/FRED/BEA code) when applicable","type":["null","string"]},"settlement_grade":{"description":"is this vintage the settlement truth (first print as-of expiration); False for later vintages AND the 48 TE-settled series","type":"boolean"},"source":{"description":"per-row source identity == df.attrs['source']; one of ECON_SOURCES","type":"string"},"units":{"description":"index | percent | thousands_persons | percent_saar | …","type":["null","string"]},"value":{"description":"released numeric value; nullable — a not-yet-released period may be schema-shaped but valueless","type":["null","number"]},"vintage_date":{"description":"WHEN this value became known (ALFRED realtime_start / release-day capture); the leakage knowledge_time","format":"date-time","type":"string"}},"required":["indicator","knowledge_time","period","release_type","settlement_grade","source","vintage_date"],"title":"schema.econ.observations.v1","type":"object","version":"v1"}; + +function validate27(data, {instancePath="", parentData, parentDataProperty, rootData=data, dynamicAnchors={}}={}){ +/*# sourceURL="https://mostlyright.dev/schemas/schema.econ.observations.v1.json" */; +let vErrors = null; +let errors = 0; +const evaluated0 = validate27.evaluated; +if(evaluated0.dynamicProps){ +evaluated0.props = undefined; +} +if(evaluated0.dynamicItems){ +evaluated0.items = undefined; +} +if(data && typeof data == "object" && !Array.isArray(data)){ +if(data.indicator === undefined){ +const err0 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "indicator"},message:"must have required property '"+"indicator"+"'"}; +if(vErrors === null){ +vErrors = [err0]; +} +else { +vErrors.push(err0); +} +errors++; +} +if(data.knowledge_time === undefined){ +const err1 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "knowledge_time"},message:"must have required property '"+"knowledge_time"+"'"}; +if(vErrors === null){ +vErrors = [err1]; +} +else { +vErrors.push(err1); +} +errors++; +} +if(data.period === undefined){ +const err2 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "period"},message:"must have required property '"+"period"+"'"}; +if(vErrors === null){ +vErrors = [err2]; +} +else { +vErrors.push(err2); +} +errors++; +} +if(data.release_type === undefined){ +const err3 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "release_type"},message:"must have required property '"+"release_type"+"'"}; +if(vErrors === null){ +vErrors = [err3]; +} +else { +vErrors.push(err3); +} +errors++; +} +if(data.settlement_grade === undefined){ +const err4 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "settlement_grade"},message:"must have required property '"+"settlement_grade"+"'"}; +if(vErrors === null){ +vErrors = [err4]; +} +else { +vErrors.push(err4); +} +errors++; +} +if(data.source === undefined){ +const err5 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "source"},message:"must have required property '"+"source"+"'"}; +if(vErrors === null){ +vErrors = [err5]; +} +else { +vErrors.push(err5); +} +errors++; +} +if(data.vintage_date === undefined){ +const err6 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "vintage_date"},message:"must have required property '"+"vintage_date"+"'"}; +if(vErrors === null){ +vErrors = [err6]; +} +else { +vErrors.push(err6); +} +errors++; +} +if(data.indicator !== undefined){ +if(typeof data.indicator !== "string"){ +const err7 = {instancePath:instancePath+"/indicator",schemaPath:"#/properties/indicator/type",keyword:"type",params:{type: "string"},message:"must be string"}; +if(vErrors === null){ +vErrors = [err7]; +} +else { +vErrors.push(err7); +} +errors++; +} +} +if(data.knowledge_time !== undefined){ +if(!(typeof data.knowledge_time === "string")){ +const err8 = {instancePath:instancePath+"/knowledge_time",schemaPath:"#/properties/knowledge_time/type",keyword:"type",params:{type: "string"},message:"must be string"}; +if(vErrors === null){ +vErrors = [err8]; +} +else { +vErrors.push(err8); +} +errors++; +} +} +if(data.period !== undefined){ +if(typeof data.period !== "string"){ +const err9 = {instancePath:instancePath+"/period",schemaPath:"#/properties/period/type",keyword:"type",params:{type: "string"},message:"must be string"}; +if(vErrors === null){ +vErrors = [err9]; +} +else { +vErrors.push(err9); +} +errors++; +} +} +if(data.release_datetime !== undefined){ +let data3 = data.release_datetime; +if((data3 !== null) && (typeof data3 !== "string")){ +const err10 = {instancePath:instancePath+"/release_datetime",schemaPath:"#/properties/release_datetime/type",keyword:"type",params:{type: schema38.properties.release_datetime.type},message:"must be null,string"}; +if(vErrors === null){ +vErrors = [err10]; +} +else { +vErrors.push(err10); +} +errors++; +} +} +if(data.release_type !== undefined){ +let data4 = data.release_type; +if(typeof data4 !== "string"){ +const err11 = {instancePath:instancePath+"/release_type",schemaPath:"#/properties/release_type/type",keyword:"type",params:{type: "string"},message:"must be string"}; +if(vErrors === null){ +vErrors = [err11]; +} +else { +vErrors.push(err11); +} +errors++; +} +if(!(((((((data4 === "advance") || (data4 === "benchmark")) || (data4 === "final")) || (data4 === "preliminary")) || (data4 === "revised")) || (data4 === "second")) || (data4 === "third"))){ +const err12 = {instancePath:instancePath+"/release_type",schemaPath:"#/properties/release_type/enum",keyword:"enum",params:{allowedValues: schema38.properties.release_type.enum},message:"must be equal to one of the allowed values"}; +if(vErrors === null){ +vErrors = [err12]; +} +else { +vErrors.push(err12); +} +errors++; +} +} +if(data.retrieved_at !== undefined){ +let data5 = data.retrieved_at; +if((data5 !== null) && (typeof data5 !== "string")){ +const err13 = {instancePath:instancePath+"/retrieved_at",schemaPath:"#/properties/retrieved_at/type",keyword:"type",params:{type: schema38.properties.retrieved_at.type},message:"must be null,string"}; +if(vErrors === null){ +vErrors = [err13]; +} +else { +vErrors.push(err13); +} +errors++; +} +} +if(data.series_id !== undefined){ +let data6 = data.series_id; +if((data6 !== null) && (typeof data6 !== "string")){ +const err14 = {instancePath:instancePath+"/series_id",schemaPath:"#/properties/series_id/type",keyword:"type",params:{type: schema38.properties.series_id.type},message:"must be null,string"}; +if(vErrors === null){ +vErrors = [err14]; +} +else { +vErrors.push(err14); +} +errors++; +} +} +if(data.settlement_grade !== undefined){ +if(typeof data.settlement_grade !== "boolean"){ +const err15 = {instancePath:instancePath+"/settlement_grade",schemaPath:"#/properties/settlement_grade/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +if(vErrors === null){ +vErrors = [err15]; +} +else { +vErrors.push(err15); +} +errors++; +} +} +if(data.source !== undefined){ +if(typeof data.source !== "string"){ +const err16 = {instancePath:instancePath+"/source",schemaPath:"#/properties/source/type",keyword:"type",params:{type: "string"},message:"must be string"}; +if(vErrors === null){ +vErrors = [err16]; +} +else { +vErrors.push(err16); +} +errors++; +} +} +if(data.units !== undefined){ +let data9 = data.units; +if((data9 !== null) && (typeof data9 !== "string")){ +const err17 = {instancePath:instancePath+"/units",schemaPath:"#/properties/units/type",keyword:"type",params:{type: schema38.properties.units.type},message:"must be null,string"}; +if(vErrors === null){ +vErrors = [err17]; +} +else { +vErrors.push(err17); +} +errors++; +} +} +if(data.value !== undefined){ +let data10 = data.value; +if((data10 !== null) && (!(typeof data10 == "number"))){ +const err18 = {instancePath:instancePath+"/value",schemaPath:"#/properties/value/type",keyword:"type",params:{type: schema38.properties.value.type},message:"must be null,number"}; +if(vErrors === null){ +vErrors = [err18]; +} +else { +vErrors.push(err18); +} +errors++; +} +} +if(data.vintage_date !== undefined){ +if(!(typeof data.vintage_date === "string")){ +const err19 = {instancePath:instancePath+"/vintage_date",schemaPath:"#/properties/vintage_date/type",keyword:"type",params:{type: "string"},message:"must be string"}; +if(vErrors === null){ +vErrors = [err19]; +} +else { +vErrors.push(err19); +} +errors++; +} +} +} +else { +const err20 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err20]; +} +else { +vErrors.push(err20); +} +errors++; +} +validate27.errors = vErrors; +return errors === 0; +} +validate27.evaluated = {"props":{"indicator":true,"knowledge_time":true,"period":true,"release_datetime":true,"release_type":true,"retrieved_at":true,"series_id":true,"settlement_grade":true,"source":true,"units":true,"value":true,"vintage_date":true},"dynamicProps":false,"dynamicItems":false}; diff --git a/packages-ts/econ/.gitignore b/packages-ts/econ/.gitignore new file mode 100644 index 00000000..26ba59cc --- /dev/null +++ b/packages-ts/econ/.gitignore @@ -0,0 +1,6 @@ +dist/ +coverage/ +.tsbuildinfo +# Generated files (src/generated/) ARE committed — they are the @mostlyrightmd/codegen +# output (the Kalshi econ settlement routing table) consumed by the resolver + the +# Python↔TS drift gate. See packages-ts/codegen/README.md. diff --git a/packages-ts/econ/README.md b/packages-ts/econ/README.md new file mode 100644 index 00000000..a2e8dbec --- /dev/null +++ b/packages-ts/econ/README.md @@ -0,0 +1,37 @@ +# @mostlyrightmd/econ + +Economic-indicator data for the [mostlyright](https://github.com/mostlyrightmd/mostlyright-sdk) TypeScript SDK — CPI (headline / core / YoY), PPI, nonfarm payrolls (NFP), U3 unemployment, initial jobless claims, GDP, and Fed decisions, sourced from FRED/ALFRED + BLS/BEA/DOL/Federal Reserve and joined to Kalshi + Polymarket econ markets for **leakage-free settlement pairs**. Mirrors the Python `mostlyrightmd-econ` distribution surface. Declares `@mostlyrightmd/core` as a peer dependency. + +## Install + +```bash +pnpm add @mostlyrightmd/econ @mostlyrightmd/core +``` + +## Surface + +Surface-equivalent to the Python `mostlyright.econ` vertical (camelCase TS signatures): + +- `history(indicator, fromDate, toDate, { vintages })` — observation rows. `vintages: "settlement"` (default) returns the settlement-grade first print (the value as-of the Kalshi expiration); `vintages: "all"` returns every vintage for feature engineering. +- `releases(indicator)` — the release calendar / schedule for an indicator. +- `researchEcon(seriesOrContract, fromDate, toDate, { asOf })` — leakage-free settlement pairs (market outcome + first-print value + as-of features). When `asOf` is given, any pair whose `knowledge_time` (= `vintage_date`) is after it throws `LeakageError` — a future revision can never leak into a backtest. + +## Load-bearing invariants (parity with Python) + +- **First-print / ALFRED vintage discipline.** Kalshi settlements contractually exclude post-expiration revisions; the backtest layer serves as-released vintages, never the revised series. Vintage selection happens at read time via `vintages`. +- **Per-series settlement routing.** 48 of the Kalshi Economics series settle to **Trading Economics**, not the government agency (incl. `KXUSPPI` PPI-MoM, `KXUSNFP`). Routing is genuinely per-series: `KXUSPPI` (MoM) → TradingEconomics, but `KXUSPPIYOY` (YoY) → BLS. The routing table is codegen-shared from `schemas/kalshi-econ-settlement.json` (the Python↔TS anti-drift gate). +- **TE-settled series are honest, never fabricated.** For a Trading-Economics-settled series, `researchEcon` ships the agency first print labeled `settlementGrade=false` and emits a divergence warning — it NEVER fabricates a TE value. +- **`IndicatorNotYetReleasedError`, not `[]`/`null`.** "Not yet released" is a normal, branchable control state distinct from "unavailable". The error is re-exported from `@mostlyrightmd/core` (Python↔TS lockstep taxonomy). +- **Parity firewall.** Econ never registers into the TS `research()` / merge / live-source equivalents — an econ row must never reach the weather settlement join (the same isolation the Python vertical and CWOP use). + +## Browser / Node constraints + +The fetch layer uses the standard `fetch` API and JSON — no Node-only API on a browser-viable path (mirrors the weather TS browser-safe boundary). API keys (`FRED_API_KEY` / `BLS_API_KEY` for ALFRED / BLS-v2) are read from the Node environment only; they are never bundled into a browser build. + +## Schema + +The `EconObservationRow` (`schema.econ.observations.v1`) column types are **codegen-derived** from the shared JSON via `@mostlyrightmd/codegen` — never hand-written in TS. Regenerate with `pnpm codegen`. + +## Docs + +See for quickstart and the full API reference. diff --git a/packages-ts/econ/package.json b/packages-ts/econ/package.json new file mode 100644 index 00000000..e441a14c --- /dev/null +++ b/packages-ts/econ/package.json @@ -0,0 +1,72 @@ +{ + "name": "@mostlyrightmd/econ", + "version": "1.15.0", + "description": "Economic-indicator data for TypeScript / Node — CPI, PPI, nonfarm payrolls (NFP), U3 unemployment, initial jobless claims, GDP, and Fed decisions, joined to Kalshi + Polymarket econ markets for leakage-free settlement pairs. First-print / ALFRED vintage discipline. Direct public-API access, no hosted backend.", + "keywords": [ + "economic-indicators", + "macro-data", + "cpi", + "ppi", + "nonfarm-payrolls", + "nfp", + "unemployment", + "jobless-claims", + "gdp", + "fomc", + "fed-funds", + "kalshi", + "polymarket", + "prediction-markets", + "settlement", + "first-print", + "alfred", + "vintage", + "backtesting", + "ml-training-data", + "time-series", + "typescript", + "nodejs" + ], + "license": "MIT", + "homepage": "https://mostlyright.md/docs/sdk/", + "repository": { + "type": "git", + "url": "git+https://github.com/mostlyrightmd/mostlyright-sdk.git", + "directory": "packages-ts/econ" + }, + "bugs": { + "url": "https://github.com/mostlyrightmd/mostlyright-sdk/issues" + }, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.cjs" + } + }, + "files": ["dist"], + "scripts": { + "prebuild": "pnpm --filter @mostlyrightmd/codegen run codegen", + "build": "tsup", + "test": "vitest", + "typecheck": "tsc --noEmit", + "clean": "rm -rf dist coverage .tsbuildinfo", + "codegen": "echo \"@mostlyrightmd/econ — no codegen step at this layer\"" + }, + "peerDependencies": { + "@mostlyrightmd/core": "^0.0.0" + }, + "devDependencies": { + "@mostlyrightmd/core": "workspace:*", + "tsup": "8.3.5", + "typescript": "5.6.3", + "vitest": "2.1.9" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages-ts/econ/src/fetchers.ts b/packages-ts/econ/src/fetchers.ts new file mode 100644 index 00000000..a7c2009b --- /dev/null +++ b/packages-ts/econ/src/fetchers.ts @@ -0,0 +1,1027 @@ +// Econ JSON fetchers (TS parity of the Python `_fetchers/`). +// +// Thin `fetch()`-based clients for the JSON agency sources (BLS + FRED/ALFRED), +// mirroring the Python fetchers' row shape and provenance discipline: +// - ALFRED (`api.stlouisfed.org`) is the settlement-grade FIRST-PRINT store — +// first-release rows carry `settlement_grade=true` / `release_type="advance"`. +// - BLS (`api.bls.gov`) is the latest-revised series — every row is +// `settlement_grade=false` (the first print is ALFRED's job; T-29-14). +// +// Browser-safe: uses the standard `fetch` (via core's `fetchWithRetry`) + JSON +// only — NO Node-only API on the fetch path (mirror the weather TS boundary). +// Keys (`FRED_API_KEY` / `BLS_API_KEY`) are read from the Node environment ONLY, +// via a guarded `readEnvKey` that returns `undefined` in a browser/Worker (where +// `process` is absent) — they are never bundled into a browser build (T-29-29). + +import { DataAvailabilityError, SourceUnavailableError, fetchWithRetry } from "@mostlyrightmd/core"; + +import type { EconObservationRow } from "./schema.js"; + +// --- Source-identity tags (parity with the Python ECON_SOURCES union) ------- +export const ECON_SOURCE_ALFRED = "alfred"; +export const ECON_SOURCE_BLS_V1 = "bls.v1"; +export const ECON_SOURCE_BLS_V2 = "bls.v2"; +export const ECON_SOURCE_BEA = "bea"; +export const ECON_SOURCE_DOL = "dol.icsa"; +export const ECON_SOURCE_FED = "fed"; +export const ECON_CACHE_SOURCE = "econ.cache"; + +// --- Endpoints (HTTPS only — a key must never travel over cleartext) -------- +const BLS_V1_URL = "https://api.bls.gov/publicAPI/v1/timeseries/data/"; +const BLS_V2_URL = "https://api.bls.gov/publicAPI/v2/timeseries/data/"; +const ALFRED_OBSERVATIONS_URL = "https://api.stlouisfed.org/fred/series/observations"; + +// --- BLS series-id → canonical indicator (verified ids; parity with Python) - +const BLS_SERIES: Readonly> = Object.freeze({ + CUUR0000SA0: "cpi", + CUUR0000SA0L1E: "cpi_core", + CES0000000001: "nfp", + LNS14000000: "u3", + WPSFD4: "ppi", // PPI final-demand, seasonally adjusted (WP-prefixed, NOT CU). + WPUFD4: "ppi", +}); + +const INDICATOR_UNITS: Readonly> = Object.freeze({ + cpi: "index", + cpi_core: "index", + ppi: "index", + u3: "percent", + nfp: "thousands_persons", +}); + +const PRELIMINARY_FOOTNOTE_CODE = "P"; +const ALFRED_MISSING = new Set([".", ""]); + +/** + * Read an env key ONLY when a Node `process.env` exists. Returns `undefined` in + * a browser / MV3 Worker (where `process` is absent) so no key is ever required + * — or bundled — on a browser-facing path. Never logs the value. + */ +function readEnvKey(name: string): string | undefined { + // `process` is not defined in a browser/Worker; guard structurally. + const proc = (globalThis as { process?: { env?: Record } }).process; + const value = proc?.env?.[name]; + return value && value.length > 0 ? value : undefined; +} + +/** + * Re-raise a {@link fetchWithRetry} transport error WITHOUT leaking the query + * string (codex CRITICAL C2). + * + * The shared core `fetchWithRetry` interpolates the FULL request URL into every + * thrown error's `.message` on a status ≥ 400 (`HTTP 401 for `). For these + * econ endpoints that URL carries `api_key=` / `UserID=` + * in its query string, so the live key would leak into any log / error-tracker + * that records the message. We MUST NOT touch the shared `http.ts` (other verticals + * use it), so we sanitize at the econ call site: catch the error and re-raise a + * {@link SourceUnavailableError} carrying ONLY the BASE endpoint constant (no query + * string) + the HTTP status. The caught error's `.message` is deliberately NEVER + * propagated (it may embed the key). Mirrors the Python `fred_alfred.py` / `bea.py` + * status-catch → bare-URL raise. + */ +function raiseSanitizedTransportError( + err: unknown, + source: string, + baseUrl: string, + label: string, +): never { + const status = (err as { statusCode?: number | null } | null)?.statusCode ?? null; + const suffix = typeof status === "number" ? ` HTTP ${status}` : ""; + throw new SourceUnavailableError(`${label} transport error${suffix}`, { + source, + url: baseUrl, // BASE endpoint only — the api_key / UserID query string is dropped. + httpStatus: status, + }); +} + +// --- BLS --------------------------------------------------------------------- + +interface BlsFootnote { + code?: string; +} +interface BlsDatum { + year?: string; + period?: string; + value?: string; + footnotes?: BlsFootnote[]; +} +interface BlsSeries { + seriesID?: string; + data?: BlsDatum[]; +} +interface BlsPayload { + status?: string; + message?: unknown; + Results?: { series?: BlsSeries[] }; +} + +/** Decode a BLS `(year, period)` into a schema `period` string (parity with Python). */ +function decodePeriod(year: string, period: string): string { + if (period.length === 0) return year; + const code = period[0]; + if (code === "M") { + const month = period.slice(1); + if (month === "13") return year; // annual average + return `${year}-${month.padStart(2, "0")}`; + } + if (code === "Q") { + const q = period.slice(1); + if (q === "05") return year; // annual average + return `${year}Q${Number.parseInt(q, 10)}`; + } + return year; // A / S / anything else → year granularity. +} + +function blsReleaseType(footnotes: BlsFootnote[] | undefined): EconObservationRow["release_type"] { + if (footnotes) { + for (const fn of footnotes) { + if (fn?.code === PRELIMINARY_FOOTNOTE_CODE) return "preliminary"; + } + } + return "revised"; // BLS serves latest-revised → never advance/first-print. +} + +/** + * Parse a BLS timeseries response into `schema.econ.observations.v1` rows. + * + * Every row is `settlement_grade=false` (BLS-API is latest-revised; the + * settlement-grade first print is ALFRED's). `vintage_date` / `knowledge_time` + * are the fetch time (BLS rows are NOT vintage-exact — documented, not a gap). + * + * @throws {SourceUnavailableError} on a non-JSON / non-`REQUEST_SUCCEEDED` body + * (a hostile / HTML body is rejected here, never parsed as empty; T-29-13). + */ +export function parseBls( + payload: BlsPayload, + opts: { source: string; retrievedAt: string }, +): EconObservationRow[] { + if (payload === null || typeof payload !== "object") { + throw new SourceUnavailableError("BLS response is not a JSON object", { + source: ECON_SOURCE_BLS_V1, + url: BLS_V1_URL, + }); + } + if (payload.status !== "REQUEST_SUCCEEDED") { + throw new SourceUnavailableError( + `BLS request not successful: status=${String(payload.status)}`, + { + source: ECON_SOURCE_BLS_V1, + url: BLS_V1_URL, + underlying: payload.message ? String(payload.message) : "", + }, + ); + } + const series = payload.Results?.series; + if (!Array.isArray(series)) { + throw new SourceUnavailableError("BLS response missing Results.series", { + source: ECON_SOURCE_BLS_V1, + url: BLS_V1_URL, + }); + } + + const rows: EconObservationRow[] = []; + for (const s of series) { + const seriesId = s.seriesID; + const indicator = seriesId ? BLS_SERIES[seriesId] : undefined; + const units = indicator ? (INDICATOR_UNITS[indicator] ?? null) : null; + const data = s.data ?? []; + for (const datum of data) { + const { year, period, value: rawValue } = datum; + if (year === undefined || period === undefined) continue; + let value: number | null = null; + if (rawValue !== undefined && rawValue !== null && rawValue !== "" && rawValue !== ".") { + const parsed = Number.parseFloat(rawValue); + value = Number.isFinite(parsed) ? parsed : null; + } + rows.push({ + indicator: indicator ?? seriesId ?? "unknown", + series_id: seriesId ?? null, + period: decodePeriod(String(year), String(period)), + value, + units, + release_datetime: null, // BLS gives no wall-clock release time. + vintage_date: opts.retrievedAt, // NOT vintage-exact (documented). + knowledge_time: opts.retrievedAt, + release_type: blsReleaseType(datum.footnotes), + settlement_grade: false, // BLS-API is latest-revised — hard-coded false. + source: opts.source, + retrieved_at: opts.retrievedAt, + }); + } + } + return rows; +} + +/** + * Fetch BLS timeseries values for `seriesIds` (keyless v1 / keyed v2). + * + * A `BLS_API_KEY` in the Node env activates the v2 endpoint (higher limits); + * absence uses the keyless v1 default (the competitive promise). The key is + * carried in the request body only, never a URL, never logged. + */ +export async function fetchBls( + seriesIds: readonly string[], + opts: { startYear: number; endYear: number }, +): Promise { + const key = readEnvKey("BLS_API_KEY"); + const url = key ? BLS_V2_URL : BLS_V1_URL; + const source = key ? ECON_SOURCE_BLS_V2 : ECON_SOURCE_BLS_V1; + + const body: Record = { + seriesid: [...seriesIds], + startyear: String(opts.startYear), + endyear: String(opts.endYear), + }; + if (key) body.registrationkey = key; // outbound-only; never logged. + + let response: Response; + try { + response = await fetchWithRetry(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + } catch (err) { + // H6 (codex round-2): the BLS POST body carries BLS_API_KEY (registrationkey). + // A transport error (connection/timeout) can surface the request via the core + // wrapper's message; never let it reach a log / error-tracker. Sanitize to a + // SourceUnavailableError carrying the BARE endpoint only (the key lives in the + // body, never the URL). Mirrors the Python bls.py _post_with_retry wrap and the + // sibling fetchAlfred / fetchIcsa / fetchBea transport-error guards. + raiseSanitizedTransportError(err, source, url, "BLS"); + } + let payload: BlsPayload; + try { + payload = (await response.json()) as BlsPayload; + } catch (err) { + throw new SourceUnavailableError("BLS returned a non-JSON body", { + source, + url, + httpStatus: response.status, + underlying: String(err), + }); + } + return parseBls(payload, { source, retrievedAt: new Date().toISOString() }); +} + +// --- ALFRED (first-print vintage store) -------------------------------------- + +interface AlfredObservation { + realtime_start?: string; + date?: string; + value?: string; +} +interface AlfredPayload { + observations?: AlfredObservation[]; +} + +/** Map an ALFRED observation `date` (YYYY-MM-DD) to a schema `period`. */ +function periodFromObsDate(obsDate: string): string { + const parts = obsDate.split("-"); + const [year, month, day] = parts; + if (year === undefined || month === undefined || day === undefined) return obsDate; + if (day === "01") return `${year}-${month}`; + return obsDate; +} + +/** Anchor an ALFRED YYYY-MM-DD at UTC midnight ISO (day-granular vintage). */ +function alfredVintageIso(realtimeStart: string): string { + return `${realtimeStart}T00:00:00Z`; +} + +/** + * Parse an ALFRED `series/observations` payload into vintage rows, then select + * the first release per period (earliest `realtime_start` → `settlement_grade=true`, + * `release_type="advance"`) — the lifted `fredapi` first-release algorithm. + * + * When `vintages="all"`, every vintage is returned ranked within its period + * (rank 0 → advance/true, later → revised/false). + * + * @throws {SourceUnavailableError} on a body without an `observations` list + * (a hostile / non-JSON body is rejected, never parsed as empty; T-29-13). + */ +export function parseAlfred( + payload: AlfredPayload, + opts: { + seriesId: string; + indicator: string; + units: string | null; + vintages: "first" | "all"; + retrievedAt: string; + }, +): EconObservationRow[] { + if (payload === null || typeof payload !== "object" || !Array.isArray(payload.observations)) { + throw new SourceUnavailableError("ALFRED response is not a valid observations payload", { + source: ECON_SOURCE_ALFRED, + url: ALFRED_OBSERVATIONS_URL, + underlying: "missing 'observations' list", + }); + } + + // Raw vintage rows (one per observation-date x realtime_start). + interface Raw { + period: string; + value: number | null; + vintageIso: string; + } + const raw: Raw[] = []; + for (const obs of payload.observations) { + const rt = obs.realtime_start; + const obsDate = obs.date; + if (rt === undefined || obsDate === undefined) continue; + let value: number | null = null; + if (obs.value !== undefined && !ALFRED_MISSING.has(obs.value)) { + const parsed = Number.parseFloat(obs.value); + value = Number.isFinite(parsed) ? parsed : null; + } + raw.push({ period: periodFromObsDate(obsDate), value, vintageIso: alfredVintageIso(rt) }); + } + + // Group by period. + const byPeriod = new Map(); + for (const r of raw) { + const group = byPeriod.get(r.period); + if (group === undefined) byPeriod.set(r.period, [r]); + else group.push(r); + } + + const rows: EconObservationRow[] = []; + for (const [period, group] of byPeriod) { + // Ascending by vintage (earliest first-print first). + group.sort((a, b) => (a.vintageIso < b.vintageIso ? -1 : a.vintageIso > b.vintageIso ? 1 : 0)); + const emit = (r: Raw, rank: number): void => { + rows.push({ + indicator: opts.indicator, + series_id: opts.seriesId, + period, + value: r.value, + units: opts.units, + release_datetime: null, + vintage_date: r.vintageIso, + knowledge_time: r.vintageIso, // leakage cutoff == vintage_date. + release_type: rank === 0 ? "advance" : "revised", + settlement_grade: rank === 0, // first print is the settlement truth. + source: ECON_SOURCE_ALFRED, + retrieved_at: opts.retrievedAt, + }); + }; + if (opts.vintages === "all") { + group.forEach(emit); + } else { + const first = group[0]; + if (first !== undefined) emit(first, 0); + } + } + return rows; +} + +/** + * Fetch ALFRED vintages for `seriesId` and return schema-shaped rows. + * + * Requires a `FRED_API_KEY` in the Node env (ALFRED 400s without one). When + * absent, throws {@link SourceUnavailableError} — a documented keyless-degradation + * signal the caller can branch on to fall back to the BLS release-day path (NOT + * a silent gap). The key is carried in the query string only, never logged. + */ +export async function fetchAlfred( + seriesId: string, + opts: { indicator: string; units: string | null; vintages: "first" | "all" }, +): Promise { + const key = readEnvKey("FRED_API_KEY"); + if (key === undefined) { + throw new SourceUnavailableError( + "ALFRED vintages require a FRED_API_KEY; set it (free at fred.stlouisfed.org/docs/api/api_key.html) or fall back to BLS release-day provenance (settlement_grade=false).", + { source: ECON_SOURCE_ALFRED, url: ALFRED_OBSERVATIONS_URL }, + ); + } + const params = new URLSearchParams({ + series_id: seriesId, + realtime_start: "1776-07-04", + realtime_end: "9999-12-31", + file_type: "json", + api_key: key, // outbound-only; never logged / stored. + }); + let response: Response; + try { + response = await fetchWithRetry(`${ALFRED_OBSERVATIONS_URL}?${params.toString()}`, { + method: "GET", + }); + } catch (err) { + // C2: never let the api_key-carrying URL reach the error message. + raiseSanitizedTransportError(err, ECON_SOURCE_ALFRED, ALFRED_OBSERVATIONS_URL, "ALFRED"); + } + let payload: AlfredPayload; + try { + payload = (await response.json()) as AlfredPayload; + } catch (err) { + throw new SourceUnavailableError("ALFRED returned a non-JSON body", { + source: ECON_SOURCE_ALFRED, + url: ALFRED_OBSERVATIONS_URL, + httpStatus: response.status, + underlying: String(err), + }); + } + return parseAlfred(payload, { ...opts, seriesId, retrievedAt: new Date().toISOString() }); +} + +/** + * Whether a `FRED_API_KEY` is present in the Node env (never returns/logs the + * value). The BLS-family + GDP + jobless settlement-grade first print comes ONLY + * from the keyed ALFRED vintage store; `series` routes the keyed vs keyless + * branch off this (mirrors the Python `_resolve_fred_key() is not None`). + */ +export function fredKeyPresent(): boolean { + return readEnvKey("FRED_API_KEY") !== undefined; +} + +// --- DOL initial jobless claims (FRED/ALFRED ICSA — parity with dol.py) ------- + +const DOL_ICSA_SERIES = "ICSA"; +const DOL_ICNSA_SERIES = "ICNSA"; +const DOL_SERIES_IDS = new Set([DOL_ICSA_SERIES, DOL_ICNSA_SERIES]); +const DOL_INDICATOR = "jobless_claims"; +const DOL_UNITS = "thousands_persons"; +const DOL_REALTIME_START = "1776-07-04"; +const DOL_REALTIME_END = "9999-12-31"; + +/** + * Parse a FRED/ALFRED ICSA `series/observations` payload into RAW vintage rows + * (one per `(week-ending date x realtime_start)`), BEFORE first-release / + * latest-revised selection. `settlement_grade` / `release_type` are assigned by + * {@link firstReleaseIcsa} (keyed) or {@link latestRevisedIcsa} (keyless). + * + * @throws {SourceUnavailableError} on a body without an `observations` list — a + * 403 bot-wall / HTML body is rejected, never parsed as empty (T-29-15). + */ +export function parseIcsa( + payload: AlfredPayload, + opts: { seriesId?: string; retrievedAt: string }, +): EconObservationRow[] { + const seriesId = opts.seriesId ?? DOL_ICSA_SERIES; + if (payload === null || typeof payload !== "object" || !Array.isArray(payload.observations)) { + throw new SourceUnavailableError( + "FRED/ALFRED ICSA response is not a valid observations payload (an HTML/403 bot-wall body is rejected, never parsed as data)", + { + source: ECON_SOURCE_DOL, + url: ALFRED_OBSERVATIONS_URL, + underlying: "missing 'observations'", + }, + ); + } + const rows: EconObservationRow[] = []; + for (const obs of payload.observations) { + const rt = obs.realtime_start; + const obsDate = obs.date; + if (rt === undefined || obsDate === undefined) continue; + let value: number | null = null; + if (obs.value !== undefined && !ALFRED_MISSING.has(obs.value)) { + const parsed = Number.parseFloat(obs.value); + value = Number.isFinite(parsed) ? parsed : null; + } + const vintageIso = alfredVintageIso(rt); + rows.push({ + indicator: DOL_INDICATOR, + series_id: seriesId, + period: obsDate, // weekly → full YYYY-MM-DD week-ending date. + value, + units: DOL_UNITS, + release_datetime: null, + vintage_date: vintageIso, + knowledge_time: vintageIso, + release_type: "revised", // reassigned by first/latest selection. + settlement_grade: false, + source: ECON_SOURCE_DOL, + retrieved_at: opts.retrievedAt, + }); + } + return rows; +} + +/** One row per week — the FIRST PRINT (earliest `vintage_date`, grade=true). */ +function firstReleaseIcsa(rows: EconObservationRow[]): EconObservationRow[] { + const byPeriod = new Map(); + for (const row of rows) { + const existing = byPeriod.get(row.period); + if (existing === undefined || row.vintage_date < existing.vintage_date) { + byPeriod.set(row.period, row); + } + } + return [...byPeriod.values()].map((row) => ({ + ...row, + release_type: "advance" as const, + settlement_grade: true, + })); +} + +/** + * Fetch weekly initial jobless claims via the FRED/ALFRED ICSA fallback (the DOL + * `oui.doleta.gov` CSV directory is bot-walled — Pitfall 4). WITH a `FRED_API_KEY` + * the first print is routed through ALFRED realtime vintages + * (`settlement_grade=true` on the earliest `realtime_start` per week). KEYLESS + * there is NO source — the DOL CSV is bot-walled (403) and the FRED ICSA fallback + * rejects an empty `api_key` — so this fails FAST pre-network with a + * {@link DataAvailabilityError} naming `FRED_API_KEY` (H4), rather than sending a + * doomed empty-key request that FRED bounces. Never `[]`/`null` for a released + * window — a genuinely unreleased week yields zero rows the caller turns into + * IndicatorNotYetReleasedError. + */ +export async function fetchIcsa(opts: { series?: string } = {}): Promise { + const series = opts.series ?? DOL_ICSA_SERIES; + if (!DOL_SERIES_IDS.has(series)) { + throw new SourceUnavailableError( + `unknown jobless-claims series ${JSON.stringify(series)}; expected ICSA or ICNSA`, + { source: ECON_SOURCE_DOL, url: ALFRED_OBSERVATIONS_URL }, + ); + } + const key = readEnvKey("FRED_API_KEY"); + if (key === undefined) { + // H4: jobless_claims has NO keyless source over the real network. Fail FAST + // with an actionable signal (mirrors the Python dol.fetch_initial_claims), + // never a doomed empty-key request. A test that wants the keyless parse/label + // path injects rows via `series({ fetchRows })`, which bypasses this fetcher. + throw new DataAvailabilityError({ + reason: "source_404", + hint: "jobless_claims has no keyless source (the DOL claims CSV is bot-walled and the FRED ICSA fallback needs a key); set FRED_API_KEY (free at fred.stlouisfed.org/docs/api/api_key.html).", + source: ECON_SOURCE_DOL, + }); + } + // Keyed → ALFRED realtime vintage query (the first-print path). + const params = new URLSearchParams({ + series_id: series, + file_type: "json", + realtime_start: DOL_REALTIME_START, + realtime_end: DOL_REALTIME_END, + api_key: key, // outbound-only; never logged. + }); + let response: Response; + try { + response = await fetchWithRetry(`${ALFRED_OBSERVATIONS_URL}?${params.toString()}`, { + method: "GET", + }); + } catch (err) { + // C2: never let the api_key-carrying URL reach the error message. + raiseSanitizedTransportError(err, ECON_SOURCE_DOL, ALFRED_OBSERVATIONS_URL, "FRED/ALFRED ICSA"); + } + let payload: AlfredPayload; + try { + payload = (await response.json()) as AlfredPayload; + } catch (err) { + throw new SourceUnavailableError( + "FRED/ALFRED ICSA returned a non-JSON body (an HTML bot-wall page is rejected, never parsed as data)", + { + source: ECON_SOURCE_DOL, + url: ALFRED_OBSERVATIONS_URL, + httpStatus: response.status, + underlying: String(err), + }, + ); + } + const raw = parseIcsa(payload, { seriesId: series, retrievedAt: new Date().toISOString() }); + // Keyed only (keyless fails fast above) → the ALFRED first-print selection. + return firstReleaseIcsa(raw); +} + +// --- BEA NIPA GDP (keyless-BEA latest-revised fallback — parity with bea.py) -- + +const BEA_DATA_URL = "https://apps.bea.gov/api/data"; +const BEA_GDP_TABLE = "T10101"; +const BEA_GDP_LINE = "1"; +const BEA_GDP_INDICATOR = "gdp"; +const BEA_GDP_UNITS = "percent_saar"; +const BEA_MISSING = new Set([".", "", "n.a.", "NaN"]); + +interface BeaDatum { + TableName?: string; + LineNumber?: string; + TimePeriod?: string; + DataValue?: string; + SeriesCode?: string; +} +interface BeaPayload { + BEAAPI?: { + Error?: unknown; + Results?: { Error?: unknown; Data?: BeaDatum[] }; + }; +} + +function quarterEndMonth(period: string): number { + return Number.parseInt(period.split("Q")[1] ?? "0", 10) * 3; +} + +/** Infer advance/second/third from a release date vs the BEA schedule (A5). */ +export function inferReleaseType( + period: string, + releaseDate: string, +): "advance" | "second" | "third" { + const endMonth = quarterEndMonth(period); + const endYear = Number.parseInt(period.split("Q")[0] ?? "0", 10); + const rel = new Date(`${releaseDate}T00:00:00Z`); + const monthsAfter = (rel.getUTCFullYear() - endYear) * 12 + (rel.getUTCMonth() + 1 - endMonth); + if (monthsAfter <= 1) return "advance"; + if (monthsAfter === 2) return "second"; + return "third"; +} + +function raiseIfBeaError(payload: BeaPayload): void { + const beaapi = payload.BEAAPI; + if (beaapi === undefined || beaapi === null || typeof beaapi !== "object") { + throw new SourceUnavailableError("BEA response is missing the BEAAPI envelope", { + source: ECON_SOURCE_BEA, + url: BEA_DATA_URL, + }); + } + const err = beaapi.Error ?? beaapi.Results?.Error; + if (err) { + let desc = ""; + if (typeof err === "object" && err !== null) { + const e = err as { APIErrorDescription?: string; ErrorDetail?: string }; + desc = String(e.APIErrorDescription ?? e.ErrorDetail ?? ""); + } + throw new SourceUnavailableError("BEA returned a structured API error", { + source: ECON_SOURCE_BEA, + url: BEA_DATA_URL, + underlying: desc, + }); + } +} + +/** + * Parse a BEA NIPA `GetData` payload into `schema.econ.observations.v1` rows. + * Selects only the GDP line (line 1 of T10101). The advance estimate is the first + * print (`settlement_grade=true`); second/third revisions are `false`. + * + * @throws {SourceUnavailableError} on a BEA error object or a non-well-formed body. + */ +export function parseBea( + payload: BeaPayload, + opts: { releaseType?: "advance" | "second" | "third"; vintageDate?: string; retrievedAt: string }, +): EconObservationRow[] { + if (payload === null || typeof payload !== "object") { + throw new SourceUnavailableError("BEA response is not a JSON object", { + source: ECON_SOURCE_BEA, + url: BEA_DATA_URL, + }); + } + raiseIfBeaError(payload); + const results = payload.BEAAPI?.Results; + const data = results?.Data; + if (!Array.isArray(data)) { + throw new SourceUnavailableError("BEA response missing BEAAPI.Results.Data", { + source: ECON_SOURCE_BEA, + url: BEA_DATA_URL, + }); + } + const vintageIso = + opts.vintageDate !== undefined ? `${opts.vintageDate}T00:00:00Z` : opts.retrievedAt; + const rows: EconObservationRow[] = []; + for (const datum of data) { + if (datum === null || typeof datum !== "object") continue; + if (String(datum.TableName ?? "") !== BEA_GDP_TABLE) continue; + if (String(datum.LineNumber ?? "") !== BEA_GDP_LINE) continue; + const timePeriod = datum.TimePeriod; + if (timePeriod === undefined) continue; + const raw = datum.DataValue; + let value: number | null = null; + if (raw !== undefined && !BEA_MISSING.has(raw)) { + const parsed = Number.parseFloat(String(raw).replace(/,/g, "")); + value = Number.isFinite(parsed) ? parsed : null; + } + let resolvedType = opts.releaseType; + if (resolvedType === undefined && opts.vintageDate !== undefined) { + resolvedType = inferReleaseType(String(timePeriod), opts.vintageDate); + } + // settlement_grade requires an EXPLICIT / inferable advance hint (codex + // round-2 C4 — the inverse of the earlier `advance` default). A plain keyless + // BEA bulk fetch (fetchGdp → fetchBea with no releaseType/vintageDate — the + // ONLY keyless GDP path) is BEA's latest-REVISED GetData data, NOT the first + // print, so an un-hinted row must NOT default to settlement-grade. Compute the + // grade off the hint BEFORE defaulting the label (undefined → false), then + // default release_type to the neutral non-first-print "revised" (a valid enum). + // An explicit releaseType="advance" still opts IN. Mirrors Python bea.py. + const settlementGrade = resolvedType === "advance"; + const releaseType: EconObservationRow["release_type"] = resolvedType ?? "revised"; + rows.push({ + indicator: BEA_GDP_INDICATOR, + series_id: datum.SeriesCode ?? null, + period: String(timePeriod), + value, + units: BEA_GDP_UNITS, + release_datetime: null, + vintage_date: vintageIso, + knowledge_time: vintageIso, + release_type: releaseType, + settlement_grade: settlementGrade, + source: ECON_SOURCE_BEA, + retrieved_at: opts.retrievedAt, + }); + } + return rows; +} + +/** + * Fetch BEA NIPA T10101 quarterly GDP % change (the KEYLESS-of-FRED GDP fallback; + * the keyed first print is the ALFRED A191RL1Q225SBEA growth-rate series). BEA + * requires a `BEA_API_KEY`; absent one this throws {@link DataAvailabilityError} + * (a documented keyless-degradation signal — GDP has no keyless first-print source). + */ +export async function fetchBea( + opts: { + yearRange?: [number, number]; + releaseType?: "advance" | "second" | "third"; + vintageDate?: string; + } = {}, +): Promise { + const key = readEnvKey("BEA_API_KEY"); + if (key === undefined) { + throw new DataAvailabilityError({ + reason: "source_404", + hint: "BEA GDP requires a BEA_API_KEY; set it (free at apps.bea.gov/API/signup) or use another indicator. GDP has no keyless first-print source.", + source: ECON_SOURCE_BEA, + }); + } + let yearParam = "ALL"; + if (opts.yearRange !== undefined) { + const [start, end] = opts.yearRange; + const years: string[] = []; + for (let y = start; y <= end; y++) years.push(String(y)); + yearParam = years.join(","); + } + const params = new URLSearchParams({ + UserID: key, // outbound-only; never logged. + method: "GetData", + datasetname: "NIPA", + TableName: BEA_GDP_TABLE, + Frequency: "Q", + Year: yearParam, + ResultFormat: "JSON", + }); + let response: Response; + try { + response = await fetchWithRetry(`${BEA_DATA_URL}?${params.toString()}`, { method: "GET" }); + } catch (err) { + // C2: never let the UserID-carrying URL reach the error message. + raiseSanitizedTransportError(err, ECON_SOURCE_BEA, BEA_DATA_URL, "BEA"); + } + let payload: BeaPayload; + try { + payload = (await response.json()) as BeaPayload; + } catch (err) { + throw new SourceUnavailableError("BEA returned a non-JSON body", { + source: ECON_SOURCE_BEA, + url: BEA_DATA_URL, + httpStatus: response.status, + underlying: String(err), + }); + } + const beaOpts: Parameters[1] = { retrievedAt: new Date().toISOString() }; + if (opts.releaseType !== undefined) beaOpts.releaseType = opts.releaseType; + if (opts.vintageDate !== undefined) beaOpts.vintageDate = opts.vintageDate; + return parseBea(payload, beaOpts); +} + +// --- Federal Reserve FOMC decisions (openmarket.htm — parity with fed.py) ----- +// Fed, NOT FRED: KXFED settles to the Federal Reserve Board. The host is pinned +// so a Fed decision can never be sourced from the wrong authority (T-29-16). + +export const FED_OPENMARKET_URL = "https://www.federalreserve.gov/monetarypolicy/openmarket.htm"; +const FED_INDICATOR = "fed_funds"; +const FED_UNITS = "percent"; +const FED_DECISIONS = new Set(["hike", "hold", "cut"]); + +const OPENMARKET_YEAR_RE = /]*>\s*(\d{4})\s*<\/h[1-6]>/g; +const OPENMARKET_ROW_RE = /]*>([\s\S]*?)<\/tr>/g; +const OPENMARKET_CELL_RE = /]*>([\s\S]*?)<\/t[dh]>/g; +const OPENMARKET_TAG_RE = /<[^>]+>/g; +const OPENMARKET_LEVEL_RANGE_RE = /^(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)$/; +const OPENMARKET_LEVEL_POINT_RE = /^(\d+(?:\.\d+)?)$/; + +const MONTHS: Readonly> = Object.freeze({ + january: 1, + february: 2, + march: 3, + april: 4, + may: 5, + june: 6, + july: 7, + august: 8, + september: 9, + october: 10, + november: 11, + december: 12, +}); + +interface FedDecision { + meetingDate: string; // YYYY-MM-DD + targetLower: number; + targetUpper: number; +} + +/** Parse a `"Month Day"` + year into an ISO `YYYY-MM-DD`, or undefined. */ +function parseOpenmarketDate(dateText: string, year: string): string | undefined { + const cleaned = dateText.replace(/[*†‡]/g, "").trim(); + const m = /^([A-Za-z]+)\s+(\d{1,2})$/.exec(cleaned); + if (m === null) return undefined; + const month = MONTHS[(m[1] ?? "").toLowerCase()]; + const day = Number.parseInt(m[2] ?? "", 10); + if (month === undefined || !Number.isInteger(day)) return undefined; + return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; +} + +/** + * Parse the Board's `openmarket.htm` rate-change record into a decisions payload. + * The page is `

YYYY

` year headings followed by + * `Date | Increase | Decrease | Level (%)` tables — `Level` is a modern target + * range (`3.50-3.75`) or a pre-2008 point target (`2.25`). + * + * @throws {SourceUnavailableError} on an empty/hostile body, an unparseable date + * or Level cell, or a page yielding ZERO decisions (T-29-16). + */ +export function parseOpenmarketHtml(html: string): { decisions: FedDecision[] } { + if (typeof html !== "string" || html.trim().length === 0) { + throw new SourceUnavailableError("Federal Reserve openmarket.htm body is empty", { + source: ECON_SOURCE_FED, + url: FED_OPENMARKET_URL, + }); + } + // Interleave year-heading and row events in document order. + interface Event { + at: number; + kind: "year" | "row"; + body: string; + } + const events: Event[] = []; + for (const m of html.matchAll(OPENMARKET_YEAR_RE)) { + events.push({ at: m.index ?? 0, kind: "year", body: m[1] ?? "" }); + } + for (const m of html.matchAll(OPENMARKET_ROW_RE)) { + events.push({ at: m.index ?? 0, kind: "row", body: m[1] ?? "" }); + } + events.sort((a, b) => a.at - b.at); + + let year: string | undefined; + let yearHeadings = 0; + const decisions: FedDecision[] = []; + for (const ev of events) { + if (ev.kind === "year") { + year = ev.body; + yearHeadings += 1; + continue; + } + const cells = [...ev.body.matchAll(OPENMARKET_CELL_RE)].map((c) => + (c[1] ?? "").replace(OPENMARKET_TAG_RE, "").replace(/ /g, " ").trim(), + ); + if (year === undefined || cells.length < 4 || (cells[0] ?? "").toLowerCase() === "date") { + continue; + } + const dateText = cells[0] ?? ""; + const levelText = cells[3] ?? ""; + const meetingDate = parseOpenmarketDate(dateText, year); + if (meetingDate === undefined) { + throw new SourceUnavailableError("openmarket.htm decision-row date is not parseable", { + source: ECON_SOURCE_FED, + url: FED_OPENMARKET_URL, + underlying: `date=${JSON.stringify(dateText)} year=${year}`, + }); + } + const level = levelText.replace(/[–—]/g, "-"); + const rangeMatch = OPENMARKET_LEVEL_RANGE_RE.exec(level); + let lower: number; + let upper: number; + if (rangeMatch !== null) { + lower = Number.parseFloat(rangeMatch[1] ?? ""); + upper = Number.parseFloat(rangeMatch[2] ?? ""); + } else { + const pointMatch = OPENMARKET_LEVEL_POINT_RE.exec(level); + if (pointMatch === null) { + throw new SourceUnavailableError( + "openmarket.htm Level cell is not a parseable target (rejected loudly, never emitted as a garbage value)", + { + source: ECON_SOURCE_FED, + url: FED_OPENMARKET_URL, + underlying: `level=${JSON.stringify(levelText)} date=${JSON.stringify(dateText)} year=${year}`, + }, + ); + } + lower = Number.parseFloat(pointMatch[1] ?? ""); + upper = lower; + } + decisions.push({ meetingDate, targetLower: lower, targetUpper: upper }); + } + if (decisions.length === 0) { + throw new SourceUnavailableError( + "openmarket.htm contained no parseable rate decisions (a hostile or redesigned page is rejected, never parsed as empty)", + { + source: ECON_SOURCE_FED, + url: FED_OPENMARKET_URL, + underlying: `year_headings=${yearHeadings}`, + }, + ); + } + return { decisions }; +} + +/** Derive hike / hold / cut from the change in the upper target bound. */ +function fedCategorical(prevUpper: number | undefined, upper: number): "hike" | "hold" | "cut" { + if (prevUpper === undefined || upper === prevUpper) return "hold"; + return upper > prevUpper ? "hike" : "cut"; +} + +/** Read the categorical decision (hike/hold/cut) back off a decision row. */ +export function decisionOf(row: EconObservationRow): string { + const seriesId = row.series_id ?? ""; + if (seriesId.includes(":")) { + const candidate = seriesId.split(":", 2)[1]; + if (candidate !== undefined && FED_DECISIONS.has(candidate)) return candidate; + } + return "hold"; +} + +/** + * Parse a decisions payload into `schema.econ.observations.v1` rows. Each carries + * the range midpoint as `value` and a categorical (hike/hold/cut) derived from the + * change vs the prior meeting's upper bound (the first in-window meeting is `hold`). + * `release_type="final"` + `settlement_grade=true` (announced, not revised). + */ +export function parseDecisions( + payload: { decisions: FedDecision[] }, + opts: { retrievedAt?: string; indicator?: string } = {}, +): EconObservationRow[] { + const retrievedAt = opts.retrievedAt ?? new Date().toISOString(); + // H6: the requested indicator is stamped on each row. Both `fed_funds` and + // `fed_decision` are served by this one parser; hardcoding `FED_INDICATOR` + // meant `series("fed_decision", …)` always returned rows tagged `fed_funds` + // (TS has no cache re-stamp to fix it downstream, unlike the Python path). + // The categorical stays encoded under the `fed_funds:` series_id namespace + // (the documented convention `decisionOf` reads) — mirrors the Python + // re-stamp, which overwrites only the `indicator` field. + const indicator = opts.indicator ?? FED_INDICATOR; + if (payload === null || typeof payload !== "object" || !Array.isArray(payload.decisions)) { + throw new SourceUnavailableError( + "Federal Reserve response is not a valid decisions payload (an HTML / non-JSON body is rejected, never parsed as data)", + { source: ECON_SOURCE_FED, url: FED_OPENMARKET_URL, underlying: "missing 'decisions'" }, + ); + } + const ordered = [...payload.decisions].sort((a, b) => + a.meetingDate < b.meetingDate ? -1 : a.meetingDate > b.meetingDate ? 1 : 0, + ); + const rows: EconObservationRow[] = []; + let prevUpper: number | undefined; + for (const entry of ordered) { + const { meetingDate, targetLower: lower, targetUpper: upper } = entry; + if (meetingDate === undefined || lower === undefined || upper === undefined) { + throw new SourceUnavailableError( + "Federal Reserve decision entry missing meeting_date / target bounds", + { source: ECON_SOURCE_FED, url: FED_OPENMARKET_URL }, + ); + } + const decision = fedCategorical(prevUpper, upper); + const vintageIso = `${meetingDate}T00:00:00Z`; + rows.push({ + indicator, + series_id: `${FED_INDICATOR}:${decision}`, + period: meetingDate, + value: (lower + upper) / 2.0, + units: FED_UNITS, + release_datetime: null, + vintage_date: vintageIso, + knowledge_time: vintageIso, + release_type: "final", + settlement_grade: true, + source: ECON_SOURCE_FED, + retrieved_at: retrievedAt, + }); + prevUpper = upper; + } + return rows; +} + +/** + * Fetch per-FOMC target-rate decisions from the Federal Reserve Board's canonical + * open-market rate-change record at `monetarypolicy/openmarket.htm` (NOT FRED — + * KXFED settles to the Fed Board; keyless). The year-sectioned + * `Date | Increase | Decrease | Level (%)` tables are parsed into decision rows. + * + * @throws {SourceUnavailableError} on an empty/hostile/redesigned page or one with + * no parseable decisions (never parsed as empty data — T-29-16). + */ +export async function fetchFed(opts: { indicator?: string } = {}): Promise { + let response: Response; + try { + response = await fetchWithRetry(FED_OPENMARKET_URL, { method: "GET" }); + } catch (err) { + // H2 (codex round-3): normalize a retry-exhaustion / transport error to the + // documented SourceUnavailableError — matching fetchBls / fetchAlfred / + // fetchIcsa / fetchBea — instead of letting a raw core TherminalError escape. + // openmarket.htm carries no key, so this is a TYPE normalization (not + // key-sanitization), but the contract is "transport failure => + // SourceUnavailableError" across every econ fetcher. + raiseSanitizedTransportError(err, ECON_SOURCE_FED, FED_OPENMARKET_URL, "Fed"); + } + const html = await response.text(); + // H6: thread the requested indicator (`fed_funds` | `fed_decision`) so the rows + // carry the id the caller asked for. + return parseDecisions( + parseOpenmarketHtml(html), + opts.indicator !== undefined ? { indicator: opts.indicator } : {}, + ); +} diff --git a/packages-ts/econ/src/floor.ts b/packages-ts/econ/src/floor.ts new file mode 100644 index 00000000..51c9b7e0 --- /dev/null +++ b/packages-ts/econ/src/floor.ts @@ -0,0 +1,76 @@ +// FEDS-2026-010 per-series backtest floor (TS parity of the Python `_floor.py`). +// +// The Kalshi economic markets did not all launch at once: FEDS working paper +// 2026-010 Table 1 records each series' FIRST Kalshi contract date (2021-2023). +// A backtest request whose fromDate predates a series' first contract asks for +// outcomes the venue never priced — so it FAILS LOUDLY (out-of-range) rather +// than silently returning a partial series (ECON-16). The dates live here ONCE +// (never re-typed inline in `history`), so the floor stays a single source of +// truth. Keys use the same indicator vocabulary as the schema. + +import { DataAvailabilityError } from "@mostlyrightmd/core"; + +/** + * FEDS-2026-010 Table 1 first-contract dates (YYYY-MM-DD), keyed on the econ + * indicator vocabulary. A request below a key's date is out-of-range. PPI / + * CPI-Core / jobless-claims floors are Kalshi-inception-derived (not FEDS + * Table-1 rows), conservatively aligned to the nearest FEDS-dated BLS analog — + * matching the Python table exactly (tighten if a dated first contract is + * later confirmed). + */ +export const FEDS_FIRST_CONTRACT: Readonly> = Object.freeze({ + // --- FEDS-2026-010 Table 1 (dated first contracts) --- + cpi: "2021-06-01", // CPI MoM — Jun 2021 + cpi_yoy: "2022-11-01", // CPI YoY — Nov 2022 + u3: "2021-07-01", // Unemployment (U3) — Jul 2021 + nfp: "2023-03-01", // Payrolls (NFP) — Mar 2023 + gdp: "2021-04-01", // GDP — Q2 2021 (quarter start = Apr 1) + fed_funds: "2021-12-01", // Fed Funds Target — Dec 2021 + fed_decision: "2023-05-01", // Fed Decision (hike/hold/cut) — May 2023 + // --- Inception-derived floors (NOT FEDS Table-1 rows) --- + ppi: "2022-11-01", // KXUSPPI (PPI MoM, TE-settled) — inception-derived + ppi_yoy: "2022-11-01", // KXUSPPIYOY (PPI YoY, BLS-settled) — inception-derived + cpi_core: "2021-06-01", // KXCPICORE — CPI-analog inception floor + cpi_core_yoy: "2022-11-01", // KXCPICOREYOY — CPI-YoY-analog inception floor + jobless_claims: "2022-01-01", // KXJOBLESSCLAIMS — DOL-weekly Kalshi inception (conservative) +}); + +/** Return the UTC-midnight epoch-ms for a `YYYY-MM-DD` floor string. */ +function floorEpoch(isoDate: string): number { + return Date.parse(`${isoDate}T00:00:00Z`); +} + +/** + * Throw {@link DataAvailabilityError} (`reason: "out_of_window"`) when `fromDate` + * predates `indicator`'s FEDS first-contract floor, or when `indicator` is + * unknown (never silently passes). Returns void when `fromDate` is at/after the + * floor. Mirrors the Python `assert_within_floor`. + * + * @param indicator An econ indicator key (must be a key of {@link FEDS_FIRST_CONTRACT}). + * @param fromDate The inclusive start of the requested backtest range (compared + * at UTC-day granularity — the time component is dropped). + */ +export function assertWithinFloor(indicator: string, fromDate: Date): void { + const floor = FEDS_FIRST_CONTRACT[indicator]; + if (floor === undefined) { + const known = Object.keys(FEDS_FIRST_CONTRACT).sort().join(", "); + throw new DataAvailabilityError({ + reason: "out_of_window", + hint: `unknown econ indicator ${JSON.stringify(indicator)}; no FEDS first-contract floor is defined for it. Known indicators: ${known}`, + }); + } + // Compare at UTC-day granularity (drop the time component of fromDate so an + // intraday timestamp on the floor day is not spuriously rejected). + const fromDay = Date.UTC( + fromDate.getUTCFullYear(), + fromDate.getUTCMonth(), + fromDate.getUTCDate(), + ); + if (fromDay < floorEpoch(floor)) { + const fromIso = new Date(fromDay).toISOString().slice(0, 10); + throw new DataAvailabilityError({ + reason: "out_of_window", + hint: `${indicator}: requested fromDate ${fromIso} is below the FEDS-2026-010 first-contract floor ${floor}; the venue never priced contracts before that date, so this window is out of range.`, + }); + } +} diff --git a/packages-ts/econ/src/generated/index.ts b/packages-ts/econ/src/generated/index.ts new file mode 100644 index 00000000..54e3e9b2 --- /dev/null +++ b/packages-ts/econ/src/generated/index.ts @@ -0,0 +1,5 @@ +// AUTO-GENERATED by @mostlyrightmd/codegen from schemas/(barrel). +// DO NOT EDIT — regenerate with: pnpm codegen +// Last manifest SHA recorded in schemas/EXPORT_MANIFEST.json + +export * from "./settlement.js"; diff --git a/packages-ts/econ/src/generated/settlement.ts b/packages-ts/econ/src/generated/settlement.ts new file mode 100644 index 00000000..b5f76cf3 --- /dev/null +++ b/packages-ts/econ/src/generated/settlement.ts @@ -0,0 +1,111 @@ +// AUTO-GENERATED by @mostlyrightmd/codegen from schemas/kalshi-econ-settlement.json. +// DO NOT EDIT — regenerate with: pnpm codegen +// Last manifest SHA recorded in schemas/EXPORT_MANIFEST.json + +// Per-series Kalshi econ settlement routing (agency NAMES, never URLs) — the +// Python↔TS anti-drift contract (T-29-27). `SETTLEMENT_ROUTING` is keyed on the +// canonical Kalshi series ROOT ticker; consume it via `resolveSettlement` which +// applies exact-then-longest-prefix match. `settlementGrade` is false for the +// Trading-Economics-settled series (the agency first print ships as a labeled +// proxy; a TE value is NEVER fabricated — ECON-17 / T-29-30). +export interface EconSettlementRule { + readonly agency: string; + readonly contractTermsPdf: string; + readonly indicator: string; + readonly settlementGrade: boolean; +} + +export const ECON_AGENCY_NAMES: ReadonlyArray = [ + "BEA", + "BLS", + "DOL", + "FederalReserve", + "TradingEconomics", +] as const; + +export const SETTLEMENT_ROUTING: Readonly> = { + KXCPI: { + agency: "BLS", + contractTermsPdf: "CPI.pdf", + indicator: "cpi", + settlementGrade: true, + }, + KXCPICORE: { + agency: "BLS", + contractTermsPdf: "CPICORE.pdf", + indicator: "cpi_core", + settlementGrade: true, + }, + KXCPICOREYOY: { + agency: "BLS", + contractTermsPdf: "CPICOREYOY.pdf", + indicator: "cpi_core_yoy", + settlementGrade: true, + }, + KXCPIYOY: { + agency: "BLS", + contractTermsPdf: "CPIYOY.pdf", + indicator: "cpi_yoy", + settlementGrade: true, + }, + KXFED: { + agency: "FederalReserve", + contractTermsPdf: "FED.pdf", + indicator: "fed_funds", + settlementGrade: true, + }, + KXFEDDECISION: { + agency: "FederalReserve", + contractTermsPdf: "FEDDECISION.pdf", + indicator: "fed_decision", + settlementGrade: true, + }, + KXGDP: { + agency: "BEA", + contractTermsPdf: "GDP.pdf", + indicator: "gdp", + settlementGrade: true, + }, + KXJOBLESS: { + agency: "DOL", + contractTermsPdf: "JOBLESSCLAIMS.pdf", + indicator: "jobless_claims", + settlementGrade: true, + }, + KXJOBLESSCLAIMS: { + agency: "DOL", + contractTermsPdf: "JOBLESSCLAIMS.pdf", + indicator: "jobless_claims", + settlementGrade: true, + }, + KXPAYROLLS: { + agency: "BLS", + contractTermsPdf: "PAYROLLS.pdf", + indicator: "nfp", + settlementGrade: true, + }, + KXU3: { + agency: "BLS", + contractTermsPdf: "U3.pdf", + indicator: "u3", + settlementGrade: true, + }, + KXUSNFP: { + agency: "TradingEconomics", + contractTermsPdf: "ECONSTATTE.pdf", + indicator: "nfp", + settlementGrade: false, + }, + KXUSPPI: { + agency: "TradingEconomics", + contractTermsPdf: "ECONSTATTE.pdf", + indicator: "ppi", + settlementGrade: false, + }, + KXUSPPIYOY: { + agency: "BLS", + contractTermsPdf: "PPIYOY.pdf", + indicator: "ppi_yoy", + settlementGrade: true, + }, +} as const; diff --git a/packages-ts/econ/src/history.ts b/packages-ts/econ/src/history.ts new file mode 100644 index 00000000..6ac67a50 --- /dev/null +++ b/packages-ts/econ/src/history.ts @@ -0,0 +1,22 @@ +// `history` — the byte-identical `@deprecated` alias of the canonical `series`. +// +// Plan 29-11 conformed the econ surface to the cross-vertical source-identity +// contract (docs/source-identity.md). The implementation now lives in +// `./series.ts`; this module re-exports the alias + the shared surface so +// existing `./history.js` importers (researchEcon, the resolver tests) keep +// working without change. +// +// @deprecated for `history`: use `series`. `history` emits NO runtime warning +// this release (the `research`/`dataset` silent-alias style); a deprecation +// lands next minor, removal at 2.0. + +export { history, releases, series } from "./series.js"; +export type { + EconDelivery, + EconSource, + FetchRows, + HistoryOptions, + ReleaseEvent, + SeriesOptions, + Vintages, +} from "./series.js"; diff --git a/packages-ts/econ/src/index.ts b/packages-ts/econ/src/index.ts new file mode 100644 index 00000000..8cbb166a --- /dev/null +++ b/packages-ts/econ/src/index.ts @@ -0,0 +1,55 @@ +// @mostlyrightmd/econ — economic-indicator data for prediction-market settlement. +// +// The TypeScript parity port of the Python `mostlyright.econ` vertical (Phase 29, +// CONTEXT Area 4 — SAME-PHASE port per the Dual-SDK rule). Macro releases (CPI / +// core / YoY, PPI, NFP + revisions, U3, initial jobless claims, GDP, Fed +// decisions) sourced from FRED/ALFRED + BLS/BEA/DOL/Federal Reserve and joined to +// Kalshi + Polymarket econ markets for leakage-free settlement pairs. +// +// Surface-equivalent to Python: `series` / `snapshot` / `releases` / +// `researchEcon` mirror the Python signatures (camelCase), the resolver reads the +// codegen-shared settlement routing table, and `IndicatorNotYetReleasedError` is +// the Python↔TS lockstep error (re-exported from core). Plan 29-11 conforms the +// surface to the cross-vertical source-identity contract (docs/source-identity.md): +// `series` is canonical (`history` a `@deprecated` alias), `source`/`delivery` +// carry loud validation, and `snapshot({ asOf })` is the settlement-target view. +// This vertical is deliberately ISOLATED from the TS `research()` / merge / +// live-source equivalents — the same parity firewall the Python vertical (and +// CWOP) uses; an econ row must never reach the weather settlement join. + +// --- Public surface (parity with the Python econ vertical) ------------------ +// `series` is canonical; `history` is the byte-identical `@deprecated` alias. +export { FRED_SERIES_BY_INDICATOR, history, releases, series } from "./series.js"; +export type { + EconDelivery, + EconSource, + FetchRows, + HistoryOptions, + ReleaseEvent, + SeriesOptions, + Vintages, +} from "./series.js"; +export { snapshot } from "./snapshot.js"; +export type { SnapshotOptions } from "./snapshot.js"; +export { researchEcon } from "./researchEcon.js"; +export type { EconPairRow, DivergenceWarning, ResearchEconOptions } from "./researchEcon.js"; + +// --- Resolver (codegen-shared settlement routing) --------------------------- +export { EconTickerError, kalshiEconResolve } from "./resolvers/kalshiEcon.js"; +export type { EconResolution } from "./resolvers/kalshiEcon.js"; + +// --- FEDS floor ------------------------------------------------------------- +export { assertWithinFloor, FEDS_FIRST_CONTRACT } from "./floor.js"; + +// --- Schema (codegen-derived — NOT hand-written) ---------------------------- +export type { EconObservationRow, EconObservationsV1 } from "./schema.js"; + +// --- Error taxonomy (Python↔TS lockstep; re-exported from core) ------------- +export { IndicatorNotYetReleasedError } from "@mostlyrightmd/core"; + +// --- Codegen-shared settlement routing table -------------------------------- +export { + ECON_AGENCY_NAMES, + type EconSettlementRule, + SETTLEMENT_ROUTING, +} from "./generated/settlement.js"; diff --git a/packages-ts/econ/src/researchEcon.ts b/packages-ts/econ/src/researchEcon.ts new file mode 100644 index 00000000..949533dc --- /dev/null +++ b/packages-ts/econ/src/researchEcon.ts @@ -0,0 +1,184 @@ +// `researchEcon` — leakage-free settlement pairs (TS parity of the Python +// `_research.py`, the econ vertical's payoff). +// +// `researchEcon(seriesOrContract, fromDate, toDate, { asOf })` joins a Kalshi +// contract to the settlement-grade FIRST-PRINT vintage of the agency indicator it +// settles against, and returns pairs that "backtest the same way they trade": +// +// 1. Resolve the contract → `(agency, indicator, settlementGrade)` via the +// codegen-shared routing table (`kalshiEconResolve`) — agrees with Python by +// construction, no markets import. +// 2. Pull the first print via `history(indicator, …, { vintages: "settlement" })`. +// 3. Build pairs with `knowledge_time = vintage_date` + the resolved +// market-outcome metadata (contract-level `settlementGrade` from the RULE). +// 4. TE-divergence, honestly labeled — a Trading-Economics-settled series ships +// the AGENCY first print labeled `settlementGrade=false` + emits a +// divergence warning. It NEVER fabricates a TE value (ECON-17 / T-29-30). +// 5. Leakage guard — when `asOf` is given, `assertNoLeakage` throws on any pair +// whose `knowledge_time` (= `vintage_date`) is after it (T-29-28; a future +// revision can never leak into a backtest). + +import { type TimePoint, assertNoLeakage } from "@mostlyrightmd/core/temporal"; + +import { kalshiEconResolve } from "./resolvers/kalshiEcon.js"; +import type { EconObservationRow } from "./schema.js"; +import { type EconDelivery, type EconSource, type FetchRows, series } from "./series.js"; + +const AGENCY_TE = "TradingEconomics"; + +/** + * A settlement pair — the first-print observation row plus the resolved + * market-outcome metadata. The CONTRACT-level grade is written to BOTH the + * canonical snake_case schema field `settlement_grade` (overwriting the inherited + * agency vintage grade — C3) and its camelCase alias `settlementGrade`; both read + * false for a TE-settled series even though the agency first print is a real + * vintage. It is the honest label a model reads to know whether the value IS the + * settlement truth. `knowledge_time` (= `vintage_date`) is the leakage cutoff. + */ +export interface EconPairRow extends EconObservationRow { + /** The resolved settlement agency short name. */ + readonly agency: string; + /** The matched Kalshi series ROOT ticker. */ + readonly contract: string; + /** + * Contract-level settlement grade (false for TE-settled series) — a camelCase + * alias of the canonical `settlement_grade`; both carry the same value. + */ + readonly settlementGrade: boolean; +} + +/** + * A machine-readable divergence event — emitted (and passed to `onDivergence`) + * when a TE-settled series ships the agency first-print proxy. Mirrors the + * Python `DivergenceWarning` intent: the consumer knows the pair's value is the + * agency proxy, NOT the (unlicensed) Trading Economics settlement truth. + */ +export interface DivergenceWarning { + /** The matched Kalshi series root (e.g. `"KXUSPPI"`). */ + readonly contract: string; + /** The econ indicator id. */ + readonly indicator: string; + /** The contractual settlement authority (`"TradingEconomics"`). */ + readonly settlementAuthority: string; + /** Human-readable message. */ + readonly message: string; +} + +export interface ResearchEconOptions { + /** + * Optional leakage cutoff. When given, any pair whose `knowledge_time` + * (`vintage_date`) is after `asOf` throws `LeakageError` — a future revision + * can never leak into a backtest. + */ + asOf?: TimePoint; + /** Injected fetcher (tests / custom transport), forwarded to `series`. */ + fetchRows?: FetchRows; + /** + * Provenance pin (source-identity contract §1), forwarded to the underlying + * `series` read. `undefined` (default) uses the per-indicator default routing; + * an unknown source or a valid authority that cannot serve the resolved + * indicator throws pre-fetch. Whether to also add an `includeEcon` covariate + * seam to core's `dataset()` is an OPTION-B question deferred to review — this + * function does NOT touch core. + */ + source?: EconSource; + /** + * Where the computation runs (contract §2), forwarded to `series`. `"live"` + * (default) | `"hosted"` (reserved seam → SourceUnavailableError). + */ + delivery?: EconDelivery; + /** + * Optional structured divergence handler. Invoked with the + * {@link DivergenceWarning} for a TE-settled series so a consumer can branch + * programmatically. This is the ONLY divergence side-channel — `researchEcon` + * writes nothing to the console (L1); a consumer that wants a log emits one here. + */ + onDivergence?: (warning: DivergenceWarning) => void; +} + +/** + * Return leakage-free settlement pairs for `seriesOrContract`. + * + * Each pair joins the market outcome (resolved agency/indicator/settlement + * metadata) to the first-print indicator value and the as-of features known at + * `knowledge_time = vintage_date` — safe to backtest and to trade. + * + * @param seriesOrContract A Kalshi econ series root (`"KXCPIYOY"`) or a concrete + * dated market ticker (`"KXCPIYOY-26JUL"`). Case-insensitive. + * @param fromDate Inclusive start of the requested range. + * @param toDate Inclusive end of the requested range. + * @param options `asOf` (leakage cutoff), `fetchRows`, `onDivergence`. + * @returns The settlement pairs (object array — no DataFrame). + * @throws {EconTickerError} the ticker resolves to no routing root. + * @throws {DataAvailabilityError} the indicator's window is below the FEDS floor. + * @throws {IndicatorNotYetReleasedError} the settlement period has no first print yet. + * @throws {LeakageError} an `asOf` was given and a pair's vintage_date is after it. + */ +export async function researchEcon( + seriesOrContract: string, + fromDate: Date, + toDate: Date, + options: ResearchEconOptions = {}, +): Promise { + // 1. Resolve the contract via the codegen-shared routing table (no markets import). + const resolution = kalshiEconResolve(seriesOrContract); + const isTeSettled = resolution.agency === AGENCY_TE; + + // 2. Pull the settlement-grade first print for the resolved indicator. The + // source-identity kwargs forward to the canonical `series` read (loud + // pre-fetch validation lives there); a default call is byte-identical to the + // pre-conformance researchEcon. + const settlement = await series(resolution.indicator, fromDate, toDate, { + vintages: "settlement", + ...(options.source !== undefined ? { source: options.source } : {}), + ...(options.delivery !== undefined ? { delivery: options.delivery } : {}), + ...(options.fetchRows !== undefined ? { fetchRows: options.fetchRows } : {}), + }); + + // 3. Build the pairs: first-print value + knowledge_time cutoff + resolved + // market-outcome metadata. The contract-level settlement grade comes from the + // ROUTING RULE (false for a TE-settled series even though the agency first + // print is itself a real vintage). + // + // C3: overwrite the CANONICAL snake_case schema field `settlement_grade` with + // the contract-level grade — mirroring the Python `_research.py` + // (`pairs["settlement_grade"] = bool(rule.settlement_grade)`). The inherited + // `...row.settlement_grade` is the AGENCY vintage grade (true for a genuine + // first print); leaving it untouched made a TE-settled pair report + // `settlement_grade === true` (WRONG) while only the non-standard camelCase + // shadow read false. `settlementGrade` (camelCase) is retained as a + // byte-identical alias so both fields agree. + const pairs: EconPairRow[] = settlement.map((row) => ({ + ...row, + agency: resolution.agency, + contract: resolution.root, + settlement_grade: resolution.settlementGrade, + settlementGrade: resolution.settlementGrade, + })); + + // 4. TE-divergence handling (ECON-17): the value stays the AGENCY first print; + // we only label the pair false and surface a STRUCTURED divergence event. + // NEVER fabricate a TE value. L1: no bare `console.warn` — a library write to + // the console can't be silenced by a consumer and violates the no-bare-console + // TS convention. The divergence is carried by the honest pair label + // (settlementGrade=false) + the optional `onDivergence` callback. + if (isTeSettled) { + const message = `${resolution.root}: settles to Trading Economics (unlicensed) — the returned value is the agency first-print proxy for indicator ${JSON.stringify(resolution.indicator)}, labeled settlementGrade=false. No Trading Economics value is fabricated.`; + const warning: DivergenceWarning = { + contract: resolution.root, + indicator: resolution.indicator, + settlementAuthority: AGENCY_TE, + message, + }; + options.onDivergence?.(warning); + } + + // 5. Leakage guard: reject any pair whose vintage_date is after the asOf cutoff. + // knowledge_time == vintage_date for econ, so assertNoLeakage keys off the + // right column. + if (options.asOf !== undefined) { + assertNoLeakage(pairs, options.asOf); + } + + return pairs; +} diff --git a/packages-ts/econ/src/resolvers/kalshiEcon.ts b/packages-ts/econ/src/resolvers/kalshiEcon.ts new file mode 100644 index 00000000..9028e228 --- /dev/null +++ b/packages-ts/econ/src/resolvers/kalshiEcon.ts @@ -0,0 +1,125 @@ +// Kalshi econ settlement resolver (TS parity of the Python 29-04 resolver). +// +// `kalshiEconResolve(seriesTicker)` maps a Kalshi Economics series root — or a +// concrete dated contract ticker — to its settlement mapping +// `{ agency, indicator, settlementGrade, contractTermsUrl }`, reading the +// codegen-shared `SETTLEMENT_ROUTING` table (generated from +// `schemas/kalshi-econ-settlement.json`, the Python↔TS anti-drift source). It +// agrees with the Python `resolve_settlement` by construction — same table, same +// matching rule — WITHOUT importing the Python package. +// +// Routed on the Kalshi settlement-source agency NAME only (never the sloppy +// series-level URL; Pitfall 2 / T-29-06). Matching is exact-then-longest-prefix: +// `KXUSPPIYOY` must bind to the `KXUSPPIYOY` rule (BLS / grade true), NOT the +// shorter `KXUSPPI` rule (TradingEconomics / grade false) it also prefix-matches. + +import { SETTLEMENT_ROUTING } from "../generated/settlement.js"; + +/** Contract-terms PDF base URL (the CFTC-certified terms live under Kalshi's docs). */ +const CONTRACT_TERMS_BASE = "https://kalshi.com/regulatory/contract-terms/"; + +/** + * A resolved Kalshi econ settlement mapping. + * + * Mirrors the Python `EconResolution` fields (camelCase). `contractTermsUrl` is + * derived from the routing rule's `contractTermsPdf` basename (the trusted + * routing companion to the agency NAME). + */ +export interface EconResolution { + /** The matched Kalshi series ROOT ticker (e.g. `"KXUSPPIYOY"`). */ + readonly root: string; + /** Canonical settlement agency short name (BLS / BEA / DOL / FederalReserve / TradingEconomics). */ + readonly agency: string; + /** Econ indicator id (`schema.econ.observations.v1` vocabulary). */ + readonly indicator: string; + /** + * `true` when the agency first print IS the settlement truth; `false` for the + * Trading-Economics-settled series (the agency first print ships as a labeled + * proxy — a TE value is NEVER fabricated). + */ + readonly settlementGrade: boolean; + /** The CFTC contract-terms PDF URL for the series, when known. */ + readonly contractTermsUrl: string | null; +} + +/** + * Raised when a ticker is not a string or matches no routing root. + * + * A named subclass (mirrors the NHIGH/NLOW `ContractIdError`) so callers can + * `instanceof`-check rather than parse the message. Mirrors the Python + * `TypeError` / `ValueError` distinction with a single explicit error — an + * unroutable ticker is NEVER a silent `null` return. + */ +export class EconTickerError extends Error { + constructor(message: string) { + super(message); + this.name = "EconTickerError"; + Object.setPrototypeOf(this, EconTickerError.prototype); + } +} + +/** + * Resolve a Kalshi econ ticker to its `(root, settlement mapping)`. + * + * Matching is exact-then-longest-prefix (identical to the Python resolver): + * 1. Exact match on the full (uppercased) ticker, else + * 2. the LONGEST `SETTLEMENT_ROUTING` root that `ticker` starts with. + * + * @param seriesTicker A Kalshi econ series root (`"KXCPIYOY"`) or a concrete + * dated market ticker (`"KXCPIYOY-26JUL"`). Case-insensitive. + * @returns A frozen {@link EconResolution}. + * @throws {EconTickerError} `seriesTicker` is not a string, or matches no root. + */ +export function kalshiEconResolve(seriesTicker: string): EconResolution { + if (typeof seriesTicker !== "string") { + throw new EconTickerError(`seriesTicker must be a string; got ${typeof seriesTicker}`); + } + const ticker = seriesTicker.toUpperCase(); + + // 1. Exact match. + let matchedRoot: string | undefined; + if (Object.hasOwn(SETTLEMENT_ROUTING, ticker)) { + matchedRoot = ticker; + } else { + // 2. Longest-prefix match. Longest-wins is load-bearing (KXUSPPIYOY must + // beat the shorter KXUSPPI it also prefix-matches). + let best: string | undefined; + for (const root of Object.keys(SETTLEMENT_ROUTING)) { + if (!ticker.startsWith(root)) continue; + // Require the dated-ticker delimiter "-" after the matched root (an exact + // full-ticker match is already handled above). Otherwise a longer + // uncatalogued series that merely SHARES a prefix — e.g. "KXCPIENERGY" vs + // "KXCPI" — would silently route to the shorter root instead of falling + // through to the intended unroutable-ticker throw (codex round-2 M10). + // Mirrors the Python _settlement_map.resolve_settlement delimiter guard. + if (ticker.length > root.length && ticker[root.length] !== "-") continue; + if (best === undefined || root.length > best.length) best = root; + } + matchedRoot = best; + } + + if (matchedRoot === undefined) { + const known = Object.keys(SETTLEMENT_ROUTING).sort().join(", "); + throw new EconTickerError( + `unknown Kalshi econ series ticker ${JSON.stringify(seriesTicker)}; it matches no routing root. known: ${known}`, + ); + } + + const rule = SETTLEMENT_ROUTING[matchedRoot]; + if (rule === undefined) { + // Unreachable — matchedRoot came from the table's own keys. Belt-and-suspenders + // for noUncheckedIndexedAccess. + throw new EconTickerError(`routing rule vanished for root ${JSON.stringify(matchedRoot)}`); + } + + const contractTermsUrl = + rule.contractTermsPdf.length > 0 ? `${CONTRACT_TERMS_BASE}${rule.contractTermsPdf}` : null; + + return Object.freeze({ + root: matchedRoot, + agency: rule.agency, + indicator: rule.indicator, + settlementGrade: rule.settlementGrade, + contractTermsUrl, + }); +} diff --git a/packages-ts/econ/src/schema.ts b/packages-ts/econ/src/schema.ts new file mode 100644 index 00000000..aca4300b --- /dev/null +++ b/packages-ts/econ/src/schema.ts @@ -0,0 +1,19 @@ +// Econ observation schema — codegen-derived, NOT hand-written. +// +// `EconObservationsV1` is generated by `@mostlyrightmd/codegen` from the SAME +// canonical JSON the Python schema reads +// (`schemas/json/schema.econ.observations.v1.json`, 29-03) and re-exported here +// from `@mostlyrightmd/core`'s generated barrel. Hand-writing these column types +// in TS is forbidden per the Dual-SDK rule — the schema-drift CI gate keeps the +// Python and TS column sets byte-identical (T-29-27). To change a column, edit +// the Python ColumnSpec + re-run `scripts/export_schemas.py`, then `pnpm codegen`. +// +// `EconObservationRow` is the SDK-facing alias (object-array output — the TS SDK +// convention; no DataFrame). Every row `history` / `researchEcon` returns +// conforms to this shape. + +export type { EconObservationsV1 } from "@mostlyrightmd/core"; +import type { EconObservationsV1 } from "@mostlyrightmd/core"; + +/** One economic-indicator observation row (a `schema.econ.observations.v1` record). */ +export type EconObservationRow = EconObservationsV1; diff --git a/packages-ts/econ/src/series.ts b/packages-ts/econ/src/series.ts new file mode 100644 index 00000000..f2772f44 --- /dev/null +++ b/packages-ts/econ/src/series.ts @@ -0,0 +1,806 @@ +// `series` + `releases` (TS parity of the Python `_history.py` / `_releases.py`). +// +// Plan 29-11 conforms the econ surface to the cross-vertical source-identity +// contract (docs/source-identity.md, v1.13/1.14). `series` is the CANONICAL read +// function; `history` (re-exported from ./history.ts) is a byte-identical +// `@deprecated` alias. +// +// `series(indicator, fromDate, toDate, { vintages, source, delivery, fetchRows })`: +// 0. Source-identity validation FIRST — `source` (contract §1) and `delivery` +// (contract §2) validate loudly BEFORE any fetch (a spy proves zero fetch +// calls on the loud-validation paths). +// 1. FEDS floor — a below-floor request is out of range (DataAvailabilityError). +// 2. Fetch the indicator's rows (dispatch to the agency fetcher; injectable via +// `{ fetchRows }` for tests — the TS analog of the Python tests monkeypatching +// INDICATOR_FETCHERS). +// 3. Read-time vintage filter — "settlement" keeps only settlement_grade rows, +// "all" keeps every vintage (the load-bearing econ difference from weather: +// the store keeps ALL vintages, the caller selects at READ time). +// 4. Not-yet-released is an ERROR, never empty — IndicatorNotYetReleasedError +// (with the release calendar's expected datetime when known). NEVER returns +// []/null for a missing release. +// +// `releases(indicator)` returns the curated per-indicator release schedule — the +// companion signal to IndicatorNotYetReleasedError (series raises it; releases +// tells you when to come back). Mirrors the Python curated table. + +import { IndicatorNotYetReleasedError, SourceUnavailableError } from "@mostlyrightmd/core"; + +import { + fetchAlfred, + fetchBea, + fetchBls, + fetchFed, + fetchIcsa, + fredKeyPresent, +} from "./fetchers.js"; +import { assertWithinFloor } from "./floor.js"; +import type { EconObservationRow } from "./schema.js"; +import { computeYoy } from "./yoy.js"; + +/** Vintage-selection mode. */ +export type Vintages = "settlement" | "all"; + +/** Accepted `source=` provenance authorities (contract §1). */ +export type EconSource = "fred" | "bls" | "bea" | "dol" | "fed"; + +/** Accepted `delivery=` values (contract §2). */ +export type EconDelivery = "live" | "hosted"; + +/** A function that returns the raw `schema.econ.observations.v1` rows for a window. */ +export type FetchRows = ( + indicator: string, + fromDate: Date, + toDate: Date, +) => Promise; + +export interface SeriesOptions { + /** `"settlement"` (default) → settlement-grade first-print rows; `"all"` → every vintage. */ + vintages?: Vintages; + /** + * Provenance pin (contract §1). `undefined` (default) uses the per-indicator + * default routing; a pin names the authority. An unknown source, OR a valid + * authority that cannot serve `indicator` (e.g. `source: "bea"` for `"cpi"`), + * throws BEFORE any fetch — never a silent fallback. + */ + source?: EconSource; + /** + * Where the computation runs (contract §2). `"live"` (default) hits the public + * agency APIs; `"hosted"` is the reserved precomputed-API seam and throws + * {@link SourceUnavailableError} naming `ECON_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`. + */ + delivery?: EconDelivery; + /** + * Injected fetcher (tests / custom transport). When omitted, the default + * agency dispatch (BLS latest-revised + ALFRED first-print) is used — the live + * path exercised in the 29-10 smoke. + */ + fetchRows?: FetchRows; +} + +/** Back-compat alias — `history` accepts the same options as `series`. */ +export type HistoryOptions = SeriesOptions; + +const VINTAGE_MODES: ReadonlySet = new Set(["settlement", "all"]); + +// --- Source-identity contract (docs/source-identity.md §1/§2) ---------------- +/** The accepted `source=` provenance authorities (mirrors the Python frozenset). */ +const VALID_ECON_SOURCES: ReadonlySet = new Set(["fred", "bls", "bea", "dol", "fed"]); +/** The accepted `delivery=` values. */ +const VALID_DELIVERIES: ReadonlySet = new Set(["live", "hosted"]); +/** Env var + key the hosted seam will read once the hosted-econ phase lands. */ +const ECON_HOSTED_URL_ENV = "ECON_HOSTED_URL"; +const ECON_HOSTED_KEY_ENV = "MOSTLYRIGHT_API_KEY"; + +/** + * Per-indicator authority map: which `source=` pins can serve each indicator + * (mirrors the Python `_INDICATOR_SOURCE_AUTHORITIES`). A pin NOT in an + * indicator's set is a loud pre-fetch throw — never a silent fallback. + */ +const INDICATOR_SOURCE_AUTHORITIES: Readonly>> = Object.freeze({ + cpi: new Set(["bls", "fred"]), + cpi_core: new Set(["bls", "fred"]), + cpi_yoy: new Set(["bls", "fred"]), + cpi_core_yoy: new Set(["bls", "fred"]), + nfp: new Set(["bls", "fred"]), + u3: new Set(["bls", "fred"]), + ppi: new Set(["bls", "fred"]), + ppi_yoy: new Set(["bls", "fred"]), + // GDP: BEA latest-revised (keyless) OR the ALFRED growth-rate first print + // (keyed, A191RL1Q225SBEA advance vintage) — mirrors the Python authorities. + gdp: new Set(["bea", "fred"]), + jobless_claims: new Set(["dol", "fred"]), + fed_funds: new Set(["fed"]), + fed_decision: new Set(["fed"]), +}); + +/** + * Loud pre-network validation of the source-identity kwargs (§1/§2). Throws + * BEFORE any fetch so an invalid pin never touches the network. Mirrors the + * Python `_validate_source_and_delivery`. + */ +function validateSourceAndDelivery( + indicator: string, + source: string | undefined, + delivery: string, +): void { + // delivery FIRST — an unknown value is a config error regardless of source. + if (!VALID_DELIVERIES.has(delivery)) { + throw new RangeError( + `delivery must be one of ${[...VALID_DELIVERIES].sort().join(", ")} (or omit); got ${JSON.stringify(delivery)}`, + ); + } + if (delivery === "hosted") { + throw new SourceUnavailableError( + [ + 'econ delivery="hosted" is a reserved seam — arrives in the hosted-econ phase.', + `Set ${ECON_HOSTED_URL_ENV} + ${ECON_HOSTED_KEY_ENV} to reach the opt-in precomputed`, + 'econ API once it ships (rows are byte-identical to the local "live" path). Use', + 'delivery="live" (the default) today.', + ].join(" "), + { source: "econ.hosted" }, + ); + } + + if (source === undefined) return; // default per-indicator routing. + + if (!VALID_ECON_SOURCES.has(source)) { + throw new RangeError( + `source must be one of ${[...VALID_ECON_SOURCES].sort().join(", ")} (or omit); got ${JSON.stringify(source)}`, + ); + } + + const authorities = INDICATOR_SOURCE_AUTHORITIES[indicator]; + if (authorities !== undefined && !authorities.has(source)) { + throw new RangeError( + [ + `source=${JSON.stringify(source)} cannot serve indicator ${JSON.stringify(indicator)};`, + `the authorities for ${JSON.stringify(indicator)} are ${[...authorities].sort().join(", ")}.`, + "A pin that cannot serve the indicator throws loudly — it never silently falls back to a", + "different provider (source-identity contract §1).", + ].join(" "), + ); + } +} + +// --- FRED/ALFRED first-print wiring for the BLS family + GDP ----------------- +// The BLS timeseries API serves ONLY the latest-revised value +// (settlement_grade=false) — there is no as-first-released endpoint. The +// settlement-grade FIRST PRINT comes from the ALFRED realtime vintage store, +// keyed by FRED_API_KEY. This map resolves each canonical indicator to its +// FRED/ALFRED series id so `defaultFetchRows` pulls the first print from ALFRED +// with the RIGHT id — the BLS ids (WPSFD4 / CUUR0000SA0) do NOT exist on FRED, +// so sending them to ALFRED was the audit BLOCKER (findings 2/3/4). +// +// NOTE: the BLS PPI id `WPSFD4` does NOT exist on FRED — the FRED final-demand +// PPI series is `PPIFIS`. GDP settles on the ANNUALIZED GROWTH RATE, so the FRED +// id is `A191RL1Q225SBEA` ("Real GDP, Percent Change from Preceding Period"), +// NOT the GDPC1 level — mirrors the Python `FRED_SERIES_BY_INDICATOR` exactly. +export const FRED_SERIES_BY_INDICATOR: Readonly> = Object.freeze({ + cpi: "CPIAUCSL", + cpi_core: "CPILFESL", + nfp: "PAYEMS", + u3: "UNRATE", + ppi: "PPIFIS", + gdp: "A191RL1Q225SBEA", +}); + +// --- BLS series ids per indicator (the KEYLESS latest-revised fallback) ------- +const BLS_SERIES_FOR: Readonly> = Object.freeze({ + cpi: ["CUUR0000SA0"], + cpi_core: ["CUUR0000SA0L1E"], + nfp: ["CES0000000001"], + u3: ["LNS14000000"], + ppi: ["WPSFD4"], +}); + +/** Units per indicator, stamped on the ALFRED first-print rows (parity with Python). */ +const BLS_FAMILY_UNITS: Readonly> = Object.freeze({ + cpi: "index", + cpi_core: "index", + ppi: "index", + u3: "percent", + nfp: "thousands_persons", + gdp: "percent", // annualized real GDP growth rate (A191RL1Q225SBEA). +}); + +/** Base level indicator each YoY contract derives its 12-month %-change from. */ +const YOY_BASE_INDICATOR: Readonly> = Object.freeze({ + cpi_yoy: "cpi", + cpi_core_yoy: "cpi_core", + ppi_yoy: "ppi", +}); + +/** The indicators whose settlement-grade first print is unlocked by FRED_API_KEY. */ +const FRED_UNLOCKABLE_INDICATORS: ReadonlySet = new Set([ + "cpi", + "cpi_core", + "cpi_yoy", + "cpi_core_yoy", + "nfp", + "u3", + "ppi", + "ppi_yoy", + "gdp", + "jobless_claims", +]); + +// --- Window / vintage helpers (parity with the Python `_history.py` helpers) -- + +/** UTC `YYYY-MM-DD` slice of a Date (the floor-date comparison key). */ +function isoDay(d: Date): string { + return d.toISOString().slice(0, 10); +} + +/** Keep rows whose `vintage_date` day falls in inclusive `[from, to]`. */ +function windowFilter( + rows: EconObservationRow[], + fromDate: Date, + toDate: Date, +): EconObservationRow[] { + const start = isoDay(fromDate); + const end = isoDay(toDate); + return rows.filter((row) => { + const vintageDay = String(row.vintage_date).slice(0, 10); + return vintageDay >= start && vintageDay <= end; + }); +} + +/** Return the first-of-month Date `n` months from `d` (UTC). */ +function addMonths(d: Date, n: number): Date { + const m = d.getUTCMonth() + n; + return new Date(Date.UTC(d.getUTCFullYear() + Math.floor(m / 12), ((m % 12) + 12) % 12, 1)); +} + +/** + * Real release date (ISO) for a KEYLESS latest-revised row from its + * `(indicator, period)`. Resolved in TWO tiers, byte-mirroring the Python + * `_derive_release_vintage`: + * + * 1. **Curated release calendar FIRST (codex round-3 C3).** Consult + * `releases(indicator)` and, when a scheduled event matches the (normalized) + * period, return its EXACT `releaseDatetime` (gdp `"2026Q2"` → + * `"2026-07-30T12:30:00.000Z"`). The old heuristic 1st-of-release-month stamped + * a date BEFORE the real release — a LOOK-AHEAD leak: a window ending mid-month + * would wrongly surface a period whose print has not actually landed. + * 2. **Heuristic fallback for out-of-table periods** (the curated table is + * PARTIAL): ≈ one month after the period (monthly) / quarter-end (quarterly), + * preserving the partition-landing for periods the calendar does not cover. + * + * Keyless rows otherwise carry vintage_date=now, which misfiles every historical + * period OUT of the vintage-date-partitioned read window. Returns undefined for an + * unparseable period (leave the row untouched). + */ +function deriveReleaseVintage(indicator: string, period: string): string | undefined { + const p = (period ?? "").trim(); + + // (1) Curated calendar FIRST — the exact scheduled release, no look-ahead. A + // monthly period is normalized to "YYYY-MM" (any day component dropped); a + // quarterly "YYYYQ#" is matched as-is. An unknown indicator / empty period + // throws (RangeError/TypeError) → no schedule → fall through to the heuristic. + const normalized = p.includes("Q") ? p : p.split("-").slice(0, 2).join("-"); + let schedule: ReleaseEvent[]; + try { + schedule = releases(indicator); + } catch { + schedule = []; + } + for (const event of schedule) { + if (event.period === normalized) return event.releaseDatetime; + } + + // (2) Heuristic fallback — out-of-table periods only. + try { + if (p.includes("Q")) { + const [yearS, qS] = p.split("Q"); + const year = Number.parseInt(yearS ?? "", 10); + const q = Number.parseInt(qS ?? "", 10); + if (!Number.isInteger(year) || !Number.isInteger(q)) return undefined; + // Parity guard: Python's strict `date(year, 3*q, 1)` rejects a quarter + // outside 1..4 (month 3*q ∉ [1,12]) → None. JS `Date.UTC` would instead + // silently roll `2026Q5` over to 2027, diverging from Python (codex round-4). + if (q < 1 || q > 4) return undefined; + const rel = addMonths(new Date(Date.UTC(year, q * 3 - 1, 1)), 1); + return rel.toISOString(); + } + const parts = p.split("-"); + const year = Number.parseInt(parts[0] ?? "", 10); + const month = Number.parseInt(parts[1] ?? "", 10); + if (!Number.isInteger(year) || !Number.isInteger(month)) return undefined; + // Parity guard: Python's strict `date(year, month, 1)` rejects month ∉ [1,12] + // → None; JS `Date.UTC` would roll `2026-13` over to 2027 (codex round-4). + if (month < 1 || month > 12) return undefined; + const rel = addMonths(new Date(Date.UTC(year, month - 1, 1)), 1); + return rel.toISOString(); + } catch { + return undefined; + } +} + +/** + * Re-stamp keyless latest-revised rows with a period-derived vintage_date (finding + * a2). Forces `settlement_grade=false` (a keyless row is NEVER the first print) + * and replaces `vintage_date` / `knowledge_time` so a historical read window + * returns the row instead of misfiling it into the current-month partition. Rows + * are copied (never mutated in place). + */ +function restampKeyless(rows: EconObservationRow[]): EconObservationRow[] { + return rows.map((row) => { + const vintage = deriveReleaseVintage(String(row.indicator ?? ""), String(row.period ?? "")); + const out: EconObservationRow = { ...row, settlement_grade: false }; + if (vintage !== undefined) { + out.vintage_date = vintage; + out.knowledge_time = vintage; + } + return out; + }); +} + +/** + * Convert a quarter-start monthly period (`"2026-04"`) to the schema's canonical + * GDP quarter form (`"2026Q2"`). Leaves an already-quarterly (`"2026Q2"`) or any + * non-`YYYY-MM` form unchanged. Mirrors the Python `_monthly_to_quarter`; applied + * ONLY to the keyed ALFRED GDP rows so their `period` matches the keyless BEA rows + * (BEA's native TimePeriod is `YYYYQ#`) — else a cross-source join on `period` + * breaks (codex H5). + */ +function monthlyToQuarter(period: string): string { + if (period.includes("Q")) return period; + const parts = period.split("-"); + if (parts.length !== 2) return period; + const year = Number.parseInt(parts[0] ?? "", 10); + const month = Number.parseInt(parts[1] ?? "", 10); + if (!Number.isInteger(year) || !Number.isInteger(month)) return period; + return `${String(year).padStart(4, "0")}Q${Math.floor((month - 1) / 3) + 1}`; +} + +/** Fetch ALFRED first-print vintages for `fredSeries` (settlement-grade), windowed. */ +async function alfredFirstPrints( + fredSeries: string, + units: string | null, + fromDate: Date, + toDate: Date, +): Promise { + const rows = await fetchAlfred(fredSeries, { indicator: "", units, vintages: "all" }); + // Re-stamp the ALFRED rows' indicator to the canonical id at the call site (the + // ALFRED parser leaves it blank here); the caller passes the resolved indicator. + return windowFilter(rows, fromDate, toDate); +} + +/** + * Fetch a BLS-family indicator, preferring the ALFRED first print when keyed. + * `source="fred"` forces ALFRED (throws if no key — never a silent BLS degrade); + * `source="bls"` forces BLS even when keyed; default = ALFRED when keyed else BLS. + */ +async function fetchBlsIndicator( + indicator: string, + fromDate: Date, + toDate: Date, + source: string | undefined, +): Promise { + const fredSeries = FRED_SERIES_BY_INDICATOR[indicator]; + const hasKey = fredKeyPresent(); + let useAlfred: boolean; + if (source === "fred") { + if (fredSeries === undefined || !hasKey) { + throw new SourceUnavailableError( + `source="fred" for ${JSON.stringify(indicator)} needs the ALFRED vintage store — set FRED_API_KEY (free: https://fred.stlouisfed.org/docs/api/api_key.html)`, + { source: "alfred", url: "https://api.stlouisfed.org/fred" }, + ); + } + useAlfred = true; + } else if (source === "bls") { + useAlfred = false; + } else { + useAlfred = fredSeries !== undefined && hasKey; + } + + if (useAlfred && fredSeries !== undefined) { + const rows = await alfredFirstPrints( + fredSeries, + BLS_FAMILY_UNITS[indicator] ?? null, + fromDate, + toDate, + ); + // Stamp the canonical indicator onto the ALFRED rows (parser left it blank). + return rows.map((r) => ({ ...r, indicator })); + } + + const blsSeries = BLS_SERIES_FOR[indicator]; + if (blsSeries === undefined) { + throw new Error( + `no BLS series id registered for indicator ${JSON.stringify(indicator)}; the BLS_SERIES_FOR map and the dispatch are out of sync.`, + ); + } + const raw = await fetchBls(blsSeries, { + startYear: fromDate.getUTCFullYear(), + endYear: toDate.getUTCFullYear(), + }); + // H5 (codex round-2): the BLS fetch is YEAR-grained, so a sub-year — or, worse + // (TS has no cache layer), a DEFAULT keyless — request would otherwise return + // the whole year. restampKeyless stamps period-derived vintage_dates, so the + // same vintage-date window the ALFRED branch applies (via alfredFirstPrints) + // bounds these keyless rows too. Mirrors the Python _fetch_bls_indicator + // window-filter. + return windowFilter(restampKeyless(raw.map((r) => ({ ...r, indicator }))), fromDate, toDate); +} + +/** Fetch GDP growth: keyed → ALFRED A191RL1Q225SBEA first prints; keyless → BEA. */ +async function fetchGdp( + fromDate: Date, + toDate: Date, + source: string | undefined, +): Promise { + const fredSeries = FRED_SERIES_BY_INDICATOR.gdp; + const hasKey = fredKeyPresent(); + let useAlfred: boolean; + if (source === "fred") { + if (fredSeries === undefined || !hasKey) { + throw new SourceUnavailableError( + 'source="fred" for "gdp" needs the ALFRED vintage store — set FRED_API_KEY (free: https://fred.stlouisfed.org/docs/api/api_key.html)', + { source: "alfred", url: "https://api.stlouisfed.org/fred" }, + ); + } + useAlfred = true; + } else if (source === "bea") { + useAlfred = false; + } else { + useAlfred = fredSeries !== undefined && hasKey; + } + + if (useAlfred && fredSeries !== undefined) { + const rows = await alfredFirstPrints( + fredSeries, + BLS_FAMILY_UNITS.gdp ?? null, + fromDate, + toDate, + ); + // H5: GDP is QUARTERLY. ALFRED anchors quarters at the period-start month + // (2026-04-01 = Q2), which periodFromObsDate renders "2026-04"; normalize to + // the schema's "YYYYQ#" so a keyed ALFRED GDP row matches the keyless BEA row + // for the same quarter (mirrors the Python _fetch_gdp). + return rows.map((r) => ({ + ...r, + indicator: "gdp", + period: monthlyToQuarter(String(r.period)), + })); + } + const raw = await fetchBea({ yearRange: [fromDate.getUTCFullYear(), toDate.getUTCFullYear()] }); + // H5 (codex round-2): BEA GetData is YEAR-grained. restampKeyless gives the + // quarterly rows a period-derived vintage_date, so the vintage-date window + // bounds a sub-year / pinned (source="bea") GDP request instead of returning + // every quarter of the year(s). Mirrors the Python _fetch_gdp window-filter. + return windowFilter(restampKeyless(raw), fromDate, toDate); +} + +/** + * Fetch a YoY indicator as the true 12-month percent change (finding 9). Widens + * the base-index fetch back 12 months, computes YoY from the base first-print + * vintages, then window-filters the YoY rows to `[from, to]`. Returns rows with + * `value` = the percent change, `units="percent"` — never the raw index level. + */ +async function fetchYoy( + yoyIndicator: string, + fromDate: Date, + toDate: Date, + source: string | undefined, +): Promise { + const baseIndicator = YOY_BASE_INDICATOR[yoyIndicator]; + if (baseIndicator === undefined) { + throw new Error(`no YoY base indicator registered for ${JSON.stringify(yoyIndicator)}`); + } + const extendedFrom = addMonths(fromDate, -12); + const baseRows = await fetchBlsIndicator(baseIndicator, extendedFrom, toDate, source); + const yoyRows = computeYoy(baseRows, yoyIndicator); + return windowFilter(yoyRows, fromDate, toDate); +} + +/** + * The EXHAUSTIVE default indicator → fetcher dispatch (parity with the Python + * `INDICATOR_FETCHERS`, all 6 CONTEXT Area-1 families / 12 indicators). Sends the + * FRED id (never a BLS id) to ALFRED for the settlement-grade first print, keeps + * BLS ids only for the keyless latest-revised fallback. An indicator with no + * dispatch entry throws (never a silent empty frame). + */ +async function defaultFetchRows( + indicator: string, + fromDate: Date, + toDate: Date, + source?: string, +): Promise { + switch (indicator) { + case "cpi": + case "cpi_core": + case "nfp": + case "u3": + case "ppi": + return fetchBlsIndicator(indicator, fromDate, toDate, source); + case "cpi_yoy": + case "cpi_core_yoy": + case "ppi_yoy": + return fetchYoy(indicator, fromDate, toDate, source); + case "gdp": + return fetchGdp(fromDate, toDate, source); + case "jobless_claims": { + // C4: fetchIcsa returns ALL history — window it to the requested range, as + // alfredFirstPrints / fetchYoy already do. TS has no cache layer to apply + // the vintage_date read filter the Python path gets for free, so without + // this an historical `asOf` read leaks post-window vintages and every + // non-`asOf` call silently returns the full series. + const rows = await fetchIcsa(); + return windowFilter(rows, fromDate, toDate); + } + case "fed_funds": + case "fed_decision": { + // C4: fetchFed returns ALL FOMC decisions — window to the requested range. + // H6: pass the requested indicator so rows are tagged fed_funds vs + // fed_decision (fed.py hardcodes fed_funds; TS has no cache re-stamp). + const rows = await fetchFed({ indicator }); + return windowFilter(rows, fromDate, toDate); + } + default: { + const known = [ + "cpi", + "cpi_core", + "cpi_yoy", + "cpi_core_yoy", + "nfp", + "u3", + "ppi", + "ppi_yoy", + "gdp", + "jobless_claims", + "fed_funds", + "fed_decision", + ].join(", "); + throw new Error( + `no fetcher registered for econ indicator ${JSON.stringify(indicator)}; the dispatch table covers: ${known}`, + ); + } + } +} + +/** Apply the read-time vintage filter. `settlement` ⊆ `all` (clean partition). */ +function filterVintages(rows: EconObservationRow[], vintages: Vintages): EconObservationRow[] { + if (vintages === "all") return rows; + return rows.filter((r) => r.settlement_grade === true); +} + +/** A human `period` label for the not-yet-released error payload. */ +function windowPeriod(fromDate: Date, toDate: Date): string { + const d0 = fromDate.toISOString().slice(0, 10); + const d1 = toDate.toISOString().slice(0, 10); + return d0 === d1 ? d0 : `${d0}..${d1}`; +} + +/** + * Window label carrying the keyless-settlement `FRED_API_KEY` unlock hint (finding + * 13). The BLS-family + jobless settlement grade (the first print) comes ONLY from + * the ALFRED vintage store, unlocked by FRED_API_KEY; a keyless settlement request + * that finds latest-revised rows but no first print names FRED_API_KEY as the + * unlock — mirroring the Python `_settlement_gap_period`. + */ +function settlementGapPeriod(fromDate: Date, toDate: Date): string { + return `${windowPeriod(fromDate, toDate)} — settlement-grade first prints require FRED_API_KEY (the ALFRED vintage store); keyless serves latest-revised only (settlement_grade=false)`; +} + +/** Best-effort scheduled release ISO at/after `fromDate`, or undefined. */ +function expectedRelease(indicator: string, fromDate: Date): string | undefined { + let schedule: ReleaseEvent[]; + try { + schedule = releases(indicator); + } catch { + return undefined; // no known schedule → never fabricate a timestamp. + } + const cutoff = fromDate.toISOString(); + const upcoming = schedule + .filter((e) => e.releaseDatetime >= cutoff) + .map((e) => e.releaseDatetime); + if (upcoming.length === 0) return undefined; + return upcoming.reduce((min, cur) => (cur < min ? cur : min)); +} + +/** + * Return economic-indicator observation rows for `indicator` (canonical read). + * + * @param indicator The indicator id (`schema.econ.observations.v1` vocabulary: + * `"cpi"` / `"nfp"` / `"gdp"` / `"ppi"` / `"jobless_claims"` / …). + * @param fromDate Inclusive start of the requested range. + * @param toDate Inclusive end of the requested range. + * @param options `vintages` (`"settlement"` default | `"all"`), `source` + * (contract §1 pin), `delivery` (contract §2), + an optional injected `fetchRows`. + * @returns The observation rows (object array — the TS SDK convention; no DataFrame). + * @throws {RangeError} `vintages`/`source`/`delivery` is invalid (source/delivery + * validated BEFORE any fetch). + * @throws {SourceUnavailableError} `delivery: "hosted"` (the reserved seam). + * @throws {DataAvailabilityError} `fromDate` is below the indicator's FEDS floor, + * or the indicator is unknown. + * @throws {IndicatorNotYetReleasedError} the window has no relevant rows — a + * scheduled release has not landed yet. NEVER returns `[]`/`null`. + */ +export async function series( + indicator: string, + fromDate: Date, + toDate: Date, + options: SeriesOptions = {}, +): Promise { + const vintages: Vintages = options.vintages ?? "settlement"; + if (!VINTAGE_MODES.has(vintages)) { + throw new RangeError(`vintages must be "settlement" or "all"; got ${JSON.stringify(vintages)}`); + } + + // 0. Source-identity contract validation FIRST — loud, before any fetch. + const delivery: string = options.delivery ?? "live"; + validateSourceAndDelivery(indicator, options.source, delivery); + + // 1. FEDS floor — below-floor / unknown-indicator is out of range regardless + // of what the fetcher holds. + assertWithinFloor(indicator, fromDate); + + // 2. Fetch. An injected `fetchRows` (tests / custom transport) uses the public + // 3-arg signature; the default agency dispatch additionally routes `source` + // (contract §1) to the right authority — the TS analog of the Python + // `fetcher(from, to, source=source)`. + const fetched = + options.fetchRows !== undefined + ? await options.fetchRows(indicator, fromDate, toDate) + : await defaultFetchRows(indicator, fromDate, toDate, options.source); + + // 3. Read-time vintage filter. + const filtered = filterVintages(fetched, vintages); + + // 4. Not-yet-released is an error, never empty. When the emptiness is + // specifically a keyless-settlement GAP — latest-revised rows EXIST but none + // is settlement-grade, for a FRED-unlockable indicator, with no key present — + // the period label names FRED_API_KEY as the unlock (finding 13), matching + // the Python surface so the BLS family + jobless read identically. + if (filtered.length === 0) { + const keylessSettlementGap = + vintages === "settlement" && + fetched.length > 0 && + FRED_UNLOCKABLE_INDICATORS.has(indicator) && + !fredKeyPresent(); + const periodLabel = keylessSettlementGap + ? settlementGapPeriod(fromDate, toDate) + : windowPeriod(fromDate, toDate); + const expected = expectedRelease(indicator, fromDate); + throw new IndicatorNotYetReleasedError( + indicator, + periodLabel, + expected !== undefined ? { expectedRelease: expected } : {}, + ); + } + + return filtered; +} + +// --------------------------------------------------------------------------- +// releases — the curated per-indicator release schedule (parity with Python). +// --------------------------------------------------------------------------- + +/** One scheduled release of an econ indicator. */ +export interface ReleaseEvent { + /** The econ indicator id. */ + readonly indicator: string; + /** The observation period this release publishes (`"2026-06"` / `"2026Q2"` / a date). */ + readonly period: string; + /** The scheduled release wall-clock, tz-aware UTC ISO (8:30 ET → 12:30 UTC). */ + readonly releaseDatetime: string; + /** The source agency short name (BLS / BEA / DOL / FederalReserve). */ + readonly agency: string; +} + +const AGENCY_BLS = "BLS"; +const AGENCY_BEA = "BEA"; +const AGENCY_DOL = "DOL"; +const AGENCY_FED = "FederalReserve"; + +/** Build a tz-aware UTC ISO string (schedule entries are stored UTC). */ +function utc(year: number, month: number, day: number, hour: number, minute: number): string { + return new Date(Date.UTC(year, month - 1, day, hour, minute)).toISOString(); +} + +// Curated forward-looking schedules (byte-mirrors the Python `_releases.py` table; +// 8:30 ET stored as 12:30 UTC — the exact DST offset is not load-bearing for a +// "when is it due" signal). +const CPI_SCHEDULE: ReadonlyArray<[string, string]> = [ + ["2026-05", utc(2026, 6, 10, 12, 30)], + ["2026-06", utc(2026, 7, 15, 12, 30)], + ["2026-07", utc(2026, 8, 12, 12, 30)], +]; +const NFP_SCHEDULE: ReadonlyArray<[string, string]> = [ + ["2026-05", utc(2026, 6, 5, 12, 30)], + ["2026-06", utc(2026, 7, 2, 12, 30)], + ["2026-07", utc(2026, 8, 7, 12, 30)], +]; +const GDP_SCHEDULE: ReadonlyArray<[string, string]> = [ + ["2026Q1", utc(2026, 4, 29, 12, 30)], + ["2026Q2", utc(2026, 7, 30, 12, 30)], + ["2026Q3", utc(2026, 10, 29, 12, 30)], +]; +const FED_SCHEDULE: ReadonlyArray<[string, string]> = [ + ["2026-06-17", utc(2026, 6, 17, 18, 0)], + ["2026-07-29", utc(2026, 7, 29, 18, 0)], + ["2026-09-16", utc(2026, 9, 16, 18, 0)], +]; +const JOBLESS_SCHEDULE: ReadonlyArray<[string, string]> = [ + ["2026-06-20", utc(2026, 6, 25, 12, 30)], + ["2026-06-27", utc(2026, 7, 2, 12, 30)], + ["2026-07-04", utc(2026, 7, 9, 12, 30)], +]; +const PPI_SCHEDULE: ReadonlyArray<[string, string]> = [ + ["2026-05", utc(2026, 6, 11, 12, 30)], + ["2026-06", utc(2026, 7, 16, 12, 30)], + ["2026-07", utc(2026, 8, 13, 12, 30)], +]; + +const SCHEDULES: Readonly< + Record; agency: string }> +> = Object.freeze({ + cpi: { rows: CPI_SCHEDULE, agency: AGENCY_BLS }, + cpi_core: { rows: CPI_SCHEDULE, agency: AGENCY_BLS }, + cpi_yoy: { rows: CPI_SCHEDULE, agency: AGENCY_BLS }, + cpi_core_yoy: { rows: CPI_SCHEDULE, agency: AGENCY_BLS }, + nfp: { rows: NFP_SCHEDULE, agency: AGENCY_BLS }, + u3: { rows: NFP_SCHEDULE, agency: AGENCY_BLS }, // U3 releases with Employment Situation. + ppi: { rows: PPI_SCHEDULE, agency: AGENCY_BLS }, + ppi_yoy: { rows: PPI_SCHEDULE, agency: AGENCY_BLS }, + gdp: { rows: GDP_SCHEDULE, agency: AGENCY_BEA }, + fed_funds: { rows: FED_SCHEDULE, agency: AGENCY_FED }, + fed_decision: { rows: FED_SCHEDULE, agency: AGENCY_FED }, + jobless_claims: { rows: JOBLESS_SCHEDULE, agency: AGENCY_DOL }, +}); + +/** + * Return the release calendar / schedule for `indicator`, sorted ascending by + * `releaseDatetime`. Sourced from the curated v1 schedule table (a live + * agency-calendar fetch is a documented fast-follow). + * + * @throws {TypeError} `indicator` is not a string. + * @throws {RangeError} `indicator` has no known schedule (never returns `[]`/`null`). + */ +export function releases(indicator: string): ReleaseEvent[] { + if (typeof indicator !== "string") { + throw new TypeError(`indicator must be a string; got ${typeof indicator}`); + } + const key = indicator.trim().toLowerCase(); + const entry = SCHEDULES[key]; + if (entry === undefined) { + const known = Object.keys(SCHEDULES).sort().join(", "); + throw new RangeError( + `no release schedule for econ indicator ${JSON.stringify(indicator)}; known indicators: ${known}`, + ); + } + const events: ReleaseEvent[] = entry.rows.map(([period, when]) => ({ + indicator: key, + period, + releaseDatetime: when, + agency: entry.agency, + })); + events.sort((a, b) => + a.releaseDatetime < b.releaseDatetime ? -1 : a.releaseDatetime > b.releaseDatetime ? 1 : 0, + ); + return events; +} + +/** + * Deprecated alias of {@link series} — returns byte-identical output. + * + * @deprecated Use {@link series}. `history` is retained as a working alias (plan + * 29-11 conformed the econ surface to the cross-vertical source-identity + * contract). It emits NO runtime warning this release — matching the + * `research`/`dataset` silent-alias style. A deprecation lands in the next + * minor; removal at 2.0. All arguments forward verbatim to {@link series}; + * see its docs for the full contract (`vintages`, `source`, `delivery`). + */ +export async function history( + indicator: string, + fromDate: Date, + toDate: Date, + options: HistoryOptions = {}, +): Promise { + return series(indicator, fromDate, toDate, options); +} + +/** @internal — the settlement-target `snapshot` reuses this scheduled-release helper. */ +export { expectedRelease as _expectedRelease }; diff --git a/packages-ts/econ/src/snapshot.ts b/packages-ts/econ/src/snapshot.ts new file mode 100644 index 00000000..3282ce56 --- /dev/null +++ b/packages-ts/econ/src/snapshot.ts @@ -0,0 +1,118 @@ +// `snapshot` — the settlement-target state of an indicator as-of a cutoff +// (TS parity of the Python `_snapshot.py`, plan 29-11). +// +// `snapshot({ indicator, asOf?, source?, delivery?, fetchRows? })` returns the +// settlement-target state of `indicator`: the latest settlement-grade +// (settlement_grade === true) vintage whose vintage_date <= asOf (default asOf: +// now), per observation period. +// +// This is the econ realization of the cross-vertical `snapshot` rule +// (docs/source-identity.md §4): "settlement-target state now" — the current best +// settlement read of the thing a Kalshi contract settles on. +// +// It composes the canonical `series` machinery (`vintages: "all"`) rather than +// duplicating the FEDS-floor / fetch / validation pipeline, then applies the +// settlement-target filter on top: +// 1. Reuse `series(indicator, floor, asOf, { vintages: "all", source, delivery, +// fetchRows })` — the SAME loud source-identity validation (§1/§2, pre-fetch), +// the SAME FEDS-floor guard, the SAME fetch/vintage pipeline. +// 2. Keep only settlement_grade rows with vintage_date <= asOf; per period keep +// the row with the greatest vintage_date (the latest settlement-grade read). +// 3. Nothing knowable is an ERROR — IndicatorNotYetReleasedError (never []/null). + +import { IndicatorNotYetReleasedError } from "@mostlyrightmd/core"; + +import { FEDS_FIRST_CONTRACT } from "./floor.js"; +import type { EconObservationRow } from "./schema.js"; +import { type EconDelivery, type EconSource, type FetchRows, series } from "./series.js"; + +export interface SnapshotOptions { + /** The indicator id (`schema.econ.observations.v1` vocabulary). */ + indicator: string; + /** + * The knowledge-time cutoff. Only vintages whose `vintage_date` is at or before + * `asOf` are considered. Defaults to now — the current settlement-target state. + */ + asOf?: Date; + /** Provenance pin (contract §1), forwarded to `series`. */ + source?: EconSource; + /** Where the computation runs (contract §2), forwarded to `series`. */ + delivery?: EconDelivery; + /** Injected fetcher (tests / custom transport), forwarded to `series`. */ + fetchRows?: FetchRows; +} + +/** + * Return the settlement-target state of `indicator` as knowable at `asOf`. + * + * @param options `indicator` (required), `asOf` (default now), `source`, + * `delivery`, `fetchRows`. + * @returns One settlement-grade row per period knowable at `asOf` (the latest + * such vintage per period). + * @throws {RangeError} an invalid `source`/`delivery` (pre-fetch) — see `series`. + * @throws {SourceUnavailableError} `delivery: "hosted"` (the reserved seam). + * @throws {DataAvailabilityError} `indicator` is unknown (no FEDS floor). + * @throws {IndicatorNotYetReleasedError} nothing is knowable at `asOf` — no + * settlement-grade vintage exists at or before the cutoff. NEVER returns `[]`/`null`. + */ +export async function snapshot(options: SnapshotOptions): Promise { + const { indicator, source, delivery, fetchRows } = options; + const asOf = options.asOf ?? new Date(); + const asOfMs = asOf.getTime(); + + // Window the underlying read from the indicator's FEDS floor up through the + // cutoff. An unknown indicator has no floor entry → let `series` raise the + // DataAvailabilityError (its assertWithinFloor is the single source of truth). + // When the cutoff predates the floor, nothing is knowable — throw the + // not-yet-released error directly rather than pass `series` an inverted window. + const floorIso = FEDS_FIRST_CONTRACT[indicator]; + let fromDate: Date; + if (floorIso !== undefined) { + const floorDate = new Date(`${floorIso}T00:00:00Z`); + if (asOfMs < floorDate.getTime()) { + throw new IndicatorNotYetReleasedError(indicator, asOf.toISOString().slice(0, 10), {}); + } + fromDate = floorDate; + } else { + // No floor: hand the cutoff-as-window to `series` so its floor guard raises + // the canonical unknown-indicator DataAvailabilityError. + fromDate = asOf; + } + + // Reuse the canonical read (loud source-identity validation + FEDS floor + + // fetch/vintage pipeline). `vintages: "all"` so we can apply the + // settlement-target filter ourselves. `series` throws + // IndicatorNotYetReleasedError when the window has NO rows at all. + const allVintages = await series(indicator, fromDate, asOf, { + vintages: "all", + ...(source !== undefined ? { source } : {}), + ...(delivery !== undefined ? { delivery } : {}), + ...(fetchRows !== undefined ? { fetchRows } : {}), + }); + + // Settlement-target filter: settlement-grade rows knowable at asOf. + const knowable = allVintages.filter( + (r) => r.settlement_grade === true && Date.parse(r.vintage_date) <= asOfMs, + ); + + if (knowable.length === 0) { + // There ARE vintages for the window, but none is a settlement-grade read + // knowable at asOf — the settlement target is not yet knowable. + throw new IndicatorNotYetReleasedError(indicator, asOf.toISOString().slice(0, 10), {}); + } + + // Per period keep the LATEST settlement-grade vintage (the settlement-target + // state as knowable at asOf). Preserve first-seen period order for stability. + const latestByPeriod = new Map(); + const periodOrder: string[] = []; + for (const row of knowable) { + const existing = latestByPeriod.get(row.period); + if (existing === undefined) { + periodOrder.push(row.period); + latestByPeriod.set(row.period, row); + } else if (Date.parse(row.vintage_date) >= Date.parse(existing.vintage_date)) { + latestByPeriod.set(row.period, row); + } + } + return periodOrder.map((p) => latestByPeriod.get(p) as EconObservationRow); +} diff --git a/packages-ts/econ/src/yoy.ts b/packages-ts/econ/src/yoy.ts new file mode 100644 index 00000000..36a2b351 --- /dev/null +++ b/packages-ts/econ/src/yoy.ts @@ -0,0 +1,110 @@ +// Year-over-year (YoY) percent-change computation for the econ YoY contracts +// (TS parity of the Python `_yoy.py`). +// +// Kalshi's YoY series — `KXCPIYOY` (cpi_yoy), `KXCPICOREYOY` (cpi_core_yoy), +// `KXUSPPIYOY` (ppi_yoy) — settle on the 12-month percent change the agency +// reports, NOT the raw index level. Returning the index (e.g. `317.7` instead of +// `2.9%`) is the adversarial-audit's finding 9 (a silent wrong settlement value). +// +// This module derives the YoY figure from the base index's FIRST-PRINT vintages: +// +// yoy[P] = 100 * (index_firstprint[P] / index_firstprint[P - 12 months] - 1) +// +// The YoY is computed first-print-to-first-print — the current period's first +// print over the year-ago period's first print. For the lightly-revised +// CPI / core-CPI / PPI indices these three contracts cover, this differs from the +// agency's released YoY only by small seasonal-factor revisions (typically within +// the one-decimal rounding Kalshi settles at). A future refinement can pull the +// year-ago base as-of the release date. + +import type { EconObservationRow } from "./schema.js"; + +/** + * Return the `"YYYY-MM"` period 12 months before `period` (year - 1). Only monthly + * `"YYYY-MM"` periods carry a YoY (the three YoY contracts are all monthly); a + * non-monthly / unparseable period returns `undefined` (no YoY emitted). + */ +function periodMinus12mo(period: string): string | undefined { + const parts = period.split("-"); + const yearS = parts[0]; + const monthS = parts[1]; + if (yearS === undefined || monthS === undefined) return undefined; + const year = Number.parseInt(yearS, 10); + const month = Number.parseInt(monthS, 10); + if (!Number.isInteger(year) || !Number.isInteger(month)) return undefined; + return `${String(year - 1).padStart(4, "0")}-${String(month).padStart(2, "0")}`; +} + +/** + * Reduce base vintages to ONE representative row per period. Prefers the + * settlement-grade first print (`settlement_grade === true`); if a period has no + * first print (the keyless latest-revised path), the sole keyless row is used. + * When several first prints share a period, the earliest `vintage_date` wins. + */ +function representativePerPeriod(rows: EconObservationRow[]): Map { + const best = new Map(); + for (const row of rows) { + const period = row.period ?? ""; + if (period.length === 0 || row.value === null || row.value === undefined) continue; + const cur = best.get(period); + if (cur === undefined) { + best.set(period, row); + continue; + } + // Prefer settlement-grade; then the earliest vintage_date. + const curGrade = cur.settlement_grade === true; + const newGrade = row.settlement_grade === true; + if (newGrade && !curGrade) { + best.set(period, row); + } else if (newGrade === curGrade) { + if (row.vintage_date < cur.vintage_date) { + best.set(period, row); + } + } + } + return best; +} + +/** + * Compute YoY percent-change rows from base index first-print vintages. + * + * For every period `P` whose base representative value and the `P - 12mo` + * representative value both exist, emit one row with `value` = the 12-month + * percent change, `units = "percent"`, `indicator` = the YoY indicator, and the + * current period's provenance (`vintage_date` / `knowledge_time` / + * `settlement_grade` / `source` inherited from `P`'s representative — so the YoY + * becomes knowable exactly when `P`'s index first print was released). Periods + * lacking a 12-month-prior base are dropped (no YoY is computable). + */ +export function computeYoy( + baseRows: EconObservationRow[], + indicator: string, +): EconObservationRow[] { + const reps = representativePerPeriod(baseRows); + const out: EconObservationRow[] = []; + for (const [period, cur] of reps) { + const prevPeriod = periodMinus12mo(period); + if (prevPeriod === undefined) continue; + const prev = reps.get(prevPeriod); + if (prev === undefined) continue; + const curV = typeof cur.value === "number" ? cur.value : Number.NaN; + const prevV = typeof prev.value === "number" ? prev.value : Number.NaN; + if (!Number.isFinite(curV) || !Number.isFinite(prevV) || prevV === 0) continue; + const yoy = 100.0 * (curV / prevV - 1.0); + out.push({ + indicator, + series_id: cur.series_id ?? null, + period, + value: yoy, + units: "percent", + release_datetime: cur.release_datetime ?? null, + vintage_date: cur.vintage_date, + release_type: cur.release_type, + settlement_grade: cur.settlement_grade, + knowledge_time: cur.knowledge_time, + source: cur.source, + retrieved_at: cur.retrieved_at ?? null, + }); + } + return out; +} diff --git a/packages-ts/econ/tests/codexReviewFixes.test.ts b/packages-ts/econ/tests/codexReviewFixes.test.ts new file mode 100644 index 00000000..9fdb207b --- /dev/null +++ b/packages-ts/econ/tests/codexReviewFixes.test.ts @@ -0,0 +1,291 @@ +// Phase 29 — codex cross-model review round 1, TS-side regression tests. +// +// The paired TS half of `packages/econ/tests/test_codex_review_fixes.py` (Python +// commit f70cb9c). Every test encodes a codex finding's repro and drives the REAL +// `defaultFetchRows` dispatch (NO injected `fetchRows`), mocking ONLY the HTTP +// transport (`globalThis.fetch`) — the same injected-transport style as +// dispatch.test.ts. Findings: C2 (key leak) / C3 (TE settlement_grade) / C4 +// (windowed jobless+fed) / H6 (fed_decision indicator) / H5 (GDP period) / +// H4 (keyless jobless fail-fast) / L1 (no bare console.warn). + +import { DataAvailabilityError } from "@mostlyrightmd/core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; + +import { type DivergenceWarning, researchEcon } from "../src/researchEcon.js"; +import { series } from "../src/series.js"; + +type FetchFn = typeof globalThis.fetch; +let fetchSpy: MockInstance; + +const D0 = new Date("2026-06-01T00:00:00Z"); +const D1 = new Date("2026-06-30T00:00:00Z"); + +function jsonResponse(payload: unknown, status = 200): Response { + return new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function htmlResponse(html: string): Response { + return new Response(html, { status: 200, headers: { "content-type": "text/html" } }); +} + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch") as unknown as MockInstance; +}); + +afterEach(() => { + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +/** Collect every string a thrown error can surface (message / toString / structured dict / url). */ +function errorSurfaces(err: unknown): string { + const parts: string[] = []; + if (err instanceof Error) { + parts.push(err.message, String(err), err.stack ?? ""); + } else { + parts.push(String(err)); + } + const structured = err as { toDict?: () => unknown; url?: unknown }; + if (typeof structured.toDict === "function") { + parts.push(JSON.stringify(structured.toDict())); + } + if (structured.url !== undefined) parts.push(String(structured.url)); + return parts.join(" || "); +} + +// --- CRITICAL C2: the API key must never reach a thrown error's message ------ +describe("C2 — a forced transport error never leaks the API key", () => { + it("ALFRED (keyed CPI): a 401 error message omits the FRED_API_KEY substring", async () => { + const KEY = "SUPERSECRETFREDKEY0001"; + vi.stubEnv("FRED_API_KEY", KEY); + fetchSpy.mockResolvedValue(jsonResponse({ error: "unauthorized" }, 401)); + + let caught: unknown; + try { + await series("cpi", D0, D1); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + // The raw URL that hit the transport DID carry the key… + expect(String(fetchSpy.mock.calls[0]?.[0])).toContain(KEY); + // …but the thrown error (message, toString, structured dict, url) must not. + expect(errorSurfaces(caught)).not.toContain(KEY); + }); + + it("FRED/ALFRED ICSA (keyed jobless): a 401 error omits the FRED_API_KEY substring", async () => { + const KEY = "SUPERSECRETFREDKEY0002"; + vi.stubEnv("FRED_API_KEY", KEY); + fetchSpy.mockResolvedValue(jsonResponse({ error: "unauthorized" }, 401)); + + let caught: unknown; + try { + await series("jobless_claims", D0, D1); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(String(fetchSpy.mock.calls[0]?.[0])).toContain(KEY); + expect(errorSurfaces(caught)).not.toContain(KEY); + }); + + it("BEA (keyless-FRED GDP → BEA UserID): a 401 error omits the BEA_API_KEY substring", async () => { + const KEY = "SUPERSECRETBEAKEY0003"; + // No FRED key → GDP routes to the keyless BEA GetData path (UserID=). + vi.stubEnv("FRED_API_KEY", ""); + vi.stubEnv("BEA_API_KEY", KEY); + fetchSpy.mockResolvedValue(jsonResponse({ error: "unauthorized" }, 401)); + + let caught: unknown; + try { + await series("gdp", D0, D1); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(String(fetchSpy.mock.calls[0]?.[0])).toContain(KEY); + expect(errorSurfaces(caught)).not.toContain(KEY); + }); +}); + +// --- CRITICAL C3: TE settlement_grade lands on the CANONICAL snake_case field - +describe("C3 — a TE-settled pair reports settlement_grade=false on the schema field", () => { + it("researchEcon(KXUSPPI) overwrites the inherited agency grade on settlement_grade", async () => { + // KXUSPPI settles to Trading Economics (contract grade false) but its value is + // the AGENCY (ALFRED/PPIFIS) first print, which is itself a genuine vintage + // (settlement_grade=true at the row level). Drive the REAL dispatch: a keyed + // PPI request pulls an in-window ALFRED first print, then researchEcon must + // overwrite the canonical `settlement_grade` with the contract-level false. + vi.stubEnv("FRED_API_KEY", "test-key"); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + fetchSpy.mockResolvedValue( + jsonResponse({ + observations: [{ realtime_start: "2026-06-11", date: "2026-05-01", value: "2.4" }], + }), + ); + + const pairs = await researchEcon("KXUSPPI", D0, D1); + + expect(pairs.length).toBeGreaterThan(0); + const p = pairs[0]; + if (p === undefined) throw new Error("expected a pair"); + // The value is the agency first print — never a fabricated TE value. + expect(p.value).toBe(2.4); + expect(p.agency).toBe("TradingEconomics"); + // The CANONICAL field is false (pre-fix it was the inherited agency true). + expect(p.settlement_grade).toBe(false); + // …and the camelCase alias agrees. + expect(p.settlementGrade).toBe(false); + }); +}); + +// --- CRITICAL C4: jobless + fed are windowed to the requested range ---------- +describe("C4 — jobless_claims / fed decisions are windowed, not full-history", () => { + it("jobless_claims returns ONLY the requested week from a multi-week fixture", async () => { + // Keyed ALFRED-ICSA first prints, three distinct weeks whose first-print + // vintage (realtime_start) lands before / inside / after the [06-01, 06-07] + // window. Only the in-window vintage may survive. + vi.stubEnv("FRED_API_KEY", "test-key"); + fetchSpy.mockResolvedValue( + jsonResponse({ + observations: [ + { realtime_start: "2026-05-28", date: "2026-05-23", value: "229000" }, // before + { realtime_start: "2026-06-04", date: "2026-05-30", value: "230000" }, // IN window + { realtime_start: "2026-06-11", date: "2026-06-06", value: "231000" }, // after + ], + }), + ); + + const rows = await series( + "jobless_claims", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-06-07T00:00:00Z"), + ); + + expect(rows).toHaveLength(1); + expect(rows[0]?.value).toBe(230000); + expect(rows[0]?.period).toBe("2026-05-30"); + expect(rows[0]?.vintage_date.slice(0, 10)).toBe("2026-06-04"); + }); + + it("fed decisions return ONLY meetings inside the requested window", async () => { + // Two 2026 FOMC rows: June (in the June window) and September (out). + fetchSpy.mockResolvedValue( + htmlResponse(` +

2026

+ + + + +
DateIncreaseDecreaseLevel (%)
June 110253.50-3.75
September 160253.25-3.50
+ `), + ); + + const rows = await series("fed_funds", D0, D1); + + expect(rows).toHaveLength(1); + expect(rows[0]?.period).toBe("2026-06-11"); + }); +}); + +// --- HIGH H6: fed_decision rows are tagged fed_decision, not fed_funds ------- +describe("H6 — the requested fed indicator is stamped on the rows", () => { + const fedHtml = ` +

2026

+ + + +
DateIncreaseDecreaseLevel (%)
June 110253.50-3.75
+ `; + + it("series('fed_decision') returns rows whose indicator === 'fed_decision'", async () => { + fetchSpy.mockResolvedValue(htmlResponse(fedHtml)); + + const rows = await series("fed_decision", D0, D1); + + expect(rows.length).toBeGreaterThan(0); + expect(rows[0]?.indicator).toBe("fed_decision"); + // The categorical stays encoded under the `fed_funds:` series_id namespace + // (mirrors the Python re-stamp, which overwrites only `indicator`). + expect(rows[0]?.series_id).toMatch(/^fed_funds:(hike|hold|cut)$/); + }); + + it("series('fed_funds') still returns rows whose indicator === 'fed_funds'", async () => { + fetchSpy.mockResolvedValue(htmlResponse(fedHtml)); + + const rows = await series("fed_funds", D0, D1); + + expect(rows.length).toBeGreaterThan(0); + expect(rows[0]?.indicator).toBe("fed_funds"); + }); +}); + +// --- HIGH H5: keyed ALFRED GDP normalizes the period to "YYYYQ#" ------------- +describe("H5 — keyed GDP period is the canonical YYYYQ# (matches keyless BEA)", () => { + it("a keyed ALFRED GDP first print (obs date 2026-04-01) reports period '2026Q2'", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + // ALFRED anchors Q2 at the period-start month 2026-04-01. + fetchSpy.mockResolvedValue( + jsonResponse({ + observations: [{ realtime_start: "2026-06-15", date: "2026-04-01", value: "-0.3" }], + }), + ); + + const rows = await series("gdp", D0, D1); + + expect(rows.length).toBeGreaterThan(0); + // Pre-fix this was "2026-04" (mismatching keyless BEA's "2026Q2"). + expect(rows[0]?.period).toBe("2026Q2"); + expect(rows[0]?.indicator).toBe("gdp"); + expect(rows[0]?.value).toBe(-0.3); + }); +}); + +// --- HIGH H4: keyless jobless_claims fails FAST, pre-network ------------------ +describe("H4 — keyless jobless_claims throws before any network call", () => { + it("series('jobless_claims') with no FRED key throws naming FRED_API_KEY and never fetches", async () => { + vi.stubEnv("FRED_API_KEY", ""); // readEnvKey treats empty as absent. + fetchSpy.mockResolvedValue(jsonResponse({ observations: [] })); + + let caught: unknown; + try { + await series("jobless_claims", D0, D1); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(DataAvailabilityError); + expect(String((caught as Error).message)).toContain("FRED_API_KEY"); + // Fail-fast: the doomed empty-key request must NEVER be sent. + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); + +// --- LOW L1: no bare console.warn from library code -------------------------- +describe("L1 — researchEcon surfaces TE divergence structurally, never via console", () => { + it("a TE-settled researchEcon call invokes onDivergence but writes nothing to console.warn", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + fetchSpy.mockResolvedValue( + jsonResponse({ + observations: [{ realtime_start: "2026-06-11", date: "2026-05-01", value: "2.4" }], + }), + ); + const events: DivergenceWarning[] = []; + + const pairs = await researchEcon("KXUSPPI", D0, D1, { + onDivergence: (w) => events.push(w), + }); + + expect(pairs.length).toBeGreaterThan(0); + // The structured side-channel fired… + expect(events).toHaveLength(1); + expect(events[0]?.settlementAuthority).toBe("TradingEconomics"); + // …and the bare console.warn is gone (a consumer can't silence a library write). + expect(warnSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/packages-ts/econ/tests/codexReviewRound2.test.ts b/packages-ts/econ/tests/codexReviewRound2.test.ts new file mode 100644 index 00000000..c0ccfc4b --- /dev/null +++ b/packages-ts/econ/tests/codexReviewRound2.test.ts @@ -0,0 +1,278 @@ +// Phase 29 — codex cross-model review ROUND 2, TS-side regression tests. +// +// The paired TS half of `packages/econ/tests/test_codex_review_round2.py` (Python +// commit 87a9887). Every test encodes a round-2 finding's repro and drives the +// REAL fetchers / dispatch (NO injected `fetchRows`), mocking ONLY the HTTP +// transport (`globalThis.fetch`) — the same injected-transport style as +// codexReviewFixes.test.ts. Round-2 TS findings: C4 (keyless BEA grade) / H5 +// (keyless BLS+GDP windowing) / H6 (fetchBls transport-error sanitize) / M10 +// (settlement prefix-delimiter). + +import { SourceUnavailableError } from "@mostlyrightmd/core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; + +import { fetchBea, fetchBls, parseBea } from "../src/fetchers.js"; +import { EconTickerError, kalshiEconResolve } from "../src/resolvers/kalshiEcon.js"; +import { series } from "../src/series.js"; + +type FetchFn = typeof globalThis.fetch; +let fetchSpy: MockInstance; + +/** A JSON Response for the ALFRED / FRED / BEA / BLS JSON transports. */ +function jsonResponse(payload: unknown, status = 200): Response { + return new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json" }, + }); +} + +/** Collect every string a thrown error can surface (message / toString / stack / dict / url). */ +function errorSurfaces(err: unknown): string { + const parts: string[] = []; + if (err instanceof Error) { + parts.push(err.message, String(err), err.stack ?? ""); + } else { + parts.push(String(err)); + } + const structured = err as { toDict?: () => unknown; url?: unknown }; + if (typeof structured.toDict === "function") { + parts.push(JSON.stringify(structured.toDict())); + } + if (structured.url !== undefined) parts.push(String(structured.url)); + return parts.join(" || "); +} + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch") as unknown as MockInstance; +}); + +afterEach(() => { + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +// --- CRITICAL C4: keyless BEA GDP is latest-revised (settlement_grade=false) --- +// A minimal T10101 line-1 GDP GetData payload for `parseBea` / `fetchBea`. +function beaGdp(period: string, value: string): Parameters[0] { + return { + BEAAPI: { + Results: { + Data: [ + { + TableName: "T10101", + LineNumber: "1", + TimePeriod: period, + DataValue: value, + SeriesCode: "A191RL1Q225SBEA", + }, + ], + }, + }, + }; +} + +describe("C4 — keyless BEA GDP defaults to settlement_grade=false, not advance", () => { + it("fetchBea keyless (no releaseType/vintageDate hint) reports settlement_grade=false", async () => { + // fetchGdp calls fetchBea({ yearRange }) with NO hint — the ONLY keyless GDP + // path. BEA bulk GetData is latest-REVISED data, NOT the advance first print, + // so an un-hinted row must be settlement_grade=false. Pre-fix parseBea + // defaulted an un-hinted row to "advance" → grade=true (the inverse of the + // Python M2 fix). This drives the REAL fetchBea (real transport → parseBea), + // which — unlike the series-level fetchGdp — does NOT restamp the grade, so the + // parser's own correctness is what is under test (parseBea/fetchBea are + // exported + independently callable; the Python bea.py fix "keeps the raw + // function safe"). + vi.stubEnv("BEA_API_KEY", "test-bea-key"); + fetchSpy.mockResolvedValue(jsonResponse(beaGdp("2026Q1", "2.4"))); + + const rows = await fetchBea({ yearRange: [2026, 2026] }); + + expect(rows.length).toBeGreaterThan(0); + expect(rows[0]?.period).toBe("2026Q1"); + expect(rows[0]?.value).toBe(2.4); + expect(rows[0]?.settlement_grade).toBe(false); + // The row's release_type label is the neutral non-first-print "revised". + expect(rows[0]?.release_type).toBe("revised"); + }); + + it("parseBea with NO hint never fabricates advance grade (the raw exported parser)", () => { + const rows = parseBea(beaGdp("2026Q1", "2.4"), { retrievedAt: "2026-07-01T00:00:00Z" }); + expect(rows).toHaveLength(1); + expect(rows[0]?.settlement_grade).toBe(false); + expect(rows[0]?.release_type).toBe("revised"); + }); + + it("an explicit releaseType='advance' still opts INTO settlement grade", () => { + const rows = parseBea(beaGdp("2026Q1", "2.4"), { + releaseType: "advance", + retrievedAt: "2026-07-01T00:00:00Z", + }); + expect(rows[0]?.settlement_grade).toBe(true); + expect(rows[0]?.release_type).toBe("advance"); + }); +}); + +// --- HIGH H5: keyless BLS/GDP dispatch branches window to the requested range -- +/** A BLS timeseries payload for monthly CUUR0000SA0, one datum per (period,value). */ +function blsMonthly(...months: Array<[string, string]>): unknown { + return { + status: "REQUEST_SUCCEEDED", + Results: { + series: [ + { + seriesID: "CUUR0000SA0", + data: months.map(([ym, value]) => ({ + year: ym.slice(0, 4), + period: `M${ym.slice(5, 7)}`, + value, + })), + }, + ], + }, + }; +} + +/** A T10101 line-1 GDP GetData payload with one Data entry per (period,value). */ +function beaGdpMulti(...quarters: Array<[string, string]>): unknown { + return { + BEAAPI: { + Results: { + Data: quarters.map(([period, value]) => ({ + TableName: "T10101", + LineNumber: "1", + TimePeriod: period, + DataValue: value, + SeriesCode: "A191RL1Q225SBEA", + })), + }, + }, + }; +} + +describe("H5 — keyless BLS/GDP dispatch branches window to the requested range", () => { + it("series('cpi') keyless returns ONLY in-window months from a full-year BLS fixture", async () => { + // No FRED key → the keyless BLS latest-revised branch (the one that pre-fix + // returned the WHOLE year, bypassing windowing). restampKeyless stamps a + // period-derived vintage (≈ first of the month AFTER the observation month), + // so a narrow June window keeps only the period whose derived vintage lands + // inside it. + vi.stubEnv("FRED_API_KEY", ""); + fetchSpy.mockResolvedValue( + jsonResponse(blsMonthly(["2026-04", "314.0"], ["2026-05", "315.0"], ["2026-06", "316.0"])), + ); + + const rows = await series( + "cpi", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-06-30T00:00:00Z"), + { vintages: "all" }, + ); + + // 2026-04 → heuristic 2026-05-01 (out-of-table, before); 2026-06 → scheduled + // 2026-07-15 (after); only 2026-05 → scheduled 2026-06-10 (curated calendar) + // lands in [06-01, 06-30]. Post round-3 C3 the vintage is the REAL scheduled + // release, not the heuristic 1st-of-month (2026-06-01). + expect(rows).toHaveLength(1); + expect(rows[0]?.period).toBe("2026-05"); + expect(rows[0]?.value).toBe(315.0); + expect(rows[0]?.settlement_grade).toBe(false); + expect(rows[0]?.vintage_date.slice(0, 10)).toBe("2026-06-10"); + }); + + it("series('gdp') keyless (BEA) returns ONLY the in-window quarter from a multi-quarter fixture", async () => { + // No FRED key → the keyless BEA GetData GDP branch (pre-fix it returned every + // quarter of the year). restampKeyless derives a quarter's vintage + // (quarter-end + 1mo, first-of-month), so a Q2-aligned window keeps only the + // quarter whose derived vintage lands inside it. + vi.stubEnv("FRED_API_KEY", ""); + vi.stubEnv("BEA_API_KEY", "test-bea-key"); + fetchSpy.mockResolvedValue( + jsonResponse(beaGdpMulti(["2026Q1", "2.4"], ["2026Q2", "-0.3"], ["2026Q3", "1.1"])), + ); + + const rows = await series( + "gdp", + new Date("2026-07-01T00:00:00Z"), + new Date("2026-07-31T00:00:00Z"), + { vintages: "all" }, + ); + + // Q1 → scheduled 2026-04-29 (before); Q3 → scheduled 2026-10-29 (after); only + // Q2 → scheduled 2026-07-30 (curated calendar) lands in [07-01, 07-31]. Post + // round-3 C3 the vintage is the REAL advance-release date, not the heuristic + // 1st-of-month (2026-07-01). + expect(rows).toHaveLength(1); + expect(rows[0]?.period).toBe("2026Q2"); + expect(rows[0]?.value).toBe(-0.3); + expect(rows[0]?.settlement_grade).toBe(false); + expect(rows[0]?.vintage_date.slice(0, 10)).toBe("2026-07-30"); + }); +}); + +// --- HIGH H6: fetchBls sanitizes a transport error (the key never surfaces) ---- +describe("H6 — fetchBls sanitizes a transport error so BLS_API_KEY cannot leak", () => { + it("a forced transport rejection throws a sanitized SourceUnavailableError with no key", async () => { + // fetchBls's POST body carries BLS_API_KEY (registrationkey). Pre-fix its + // fetchWithRetry call was NOT wrapped (unlike fetchAlfred / fetchIcsa / + // fetchBea), so a transport error propagated raw as core's TherminalError + // ("fetch failed after N attempts: ") — leaking any request detail + // the transport echoed. Simulate a verbose transport that inlines the + // key-bearing body into its error; the wrap must convert it to a + // SourceUnavailableError whose surfaces never contain the key. + // + // Fake timers burn the core retry backoff (baseDelay 1000ms × 3) instantly. + vi.useFakeTimers(); + const KEY = "SUPERSECRETBLSKEY0006"; + vi.stubEnv("BLS_API_KEY", KEY); + fetchSpy.mockRejectedValue(new Error(`ECONNREFUSED — request body registrationkey=${KEY}`)); + + // Attach the rejection handler BEFORE advancing timers (no unhandled rejection). + const settled = fetchBls(["CUUR0000SA0"], { startYear: 2026, endYear: 2026 }).then( + () => ({ ok: true as const }), + (err: unknown) => ({ ok: false as const, err }), + ); + await vi.runAllTimersAsync(); + const result = await settled; + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected fetchBls to reject on a transport error"); + // Post-fix: the raw TherminalError is re-raised as a sanitized + // SourceUnavailableError (pre-fix it was the raw TherminalError). + expect(result.err).toBeInstanceOf(SourceUnavailableError); + // …and no surface of the thrown error contains the key. + expect(errorSurfaces(result.err)).not.toContain(KEY); + }); +}); + +// --- MEDIUM M10: prefix routing requires the dated-ticker delimiter ------------ +describe("M10 — resolver prefix routing requires the dated-ticker delimiter", () => { + it("a dated ticker 'KXCPI-26JUL' resolves to the KXCPI root", () => { + // The delimiter after the root is "-", so the longest-prefix match binds it. + const res = kalshiEconResolve("KXCPI-26JUL"); + expect(res.root).toBe("KXCPI"); + expect(res.indicator).toBe("cpi"); + }); + + it("a longer uncatalogued series 'KXCPIENERGY' throws instead of routing to KXCPI", () => { + // "KXCPIENERGY" prefix-matches "KXCPI" but the next char is "E", not "-" — so + // it must NOT silently route to the shorter KXCPI root; it is unroutable. + expect(() => kalshiEconResolve("KXCPIENERGY")).toThrow(EconTickerError); + }); + + it("the exact root 'KXCPI' still resolves (exact match needs no delimiter)", () => { + const res = kalshiEconResolve("KXCPI"); + expect(res.root).toBe("KXCPI"); + }); + + it("longest-prefix + delimiter still binds 'KXUSPPIYOY-26JUL' to the longer root", () => { + // The delimiter guard must not regress longest-wins: KXUSPPI (next char "Y") + // is skipped, KXUSPPIYOY (next char "-") wins. + const res = kalshiEconResolve("KXUSPPIYOY-26JUL"); + expect(res.root).toBe("KXUSPPIYOY"); + expect(res.agency).toBe("BLS"); + expect(res.settlementGrade).toBe(true); + }); +}); diff --git a/packages-ts/econ/tests/codexReviewRound3.test.ts b/packages-ts/econ/tests/codexReviewRound3.test.ts new file mode 100644 index 00000000..3d0a4972 --- /dev/null +++ b/packages-ts/econ/tests/codexReviewRound3.test.ts @@ -0,0 +1,115 @@ +// Phase 29 — codex cross-model review ROUND 3, TS-side regression tests. +// +// The paired TS half of `packages/econ/tests/test_codex_review_round3.py`. Each +// test drives the REAL fetchers / dispatch, mocking ONLY the HTTP transport +// (`globalThis.fetch`) — the injected-transport style of codexReviewRound2. +// Round-3 TS findings: C3 (keyless vintage consults the real release calendar — +// the semantic mirror of Python C1) and H2 (fetchFed transport-error sanitize). + +import { IndicatorNotYetReleasedError, SourceUnavailableError } from "@mostlyrightmd/core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; + +import { fetchFed } from "../src/fetchers.js"; +import { series } from "../src/series.js"; + +type FetchFn = typeof globalThis.fetch; +let fetchSpy: MockInstance; + +function jsonResponse(payload: unknown, status = 200): Response { + return new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json" }, + }); +} + +/** A T10101 line-1 GDP GetData payload with one Data entry per (period,value). */ +function beaGdpMulti(...quarters: Array<[string, string]>): unknown { + return { + BEAAPI: { + Results: { + Data: quarters.map(([period, value]) => ({ + TableName: "T10101", + LineNumber: "1", + TimePeriod: period, + DataValue: value, + SeriesCode: "A191RL1Q225SBEA", + })), + }, + }, + }; +} + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch") as unknown as MockInstance; +}); + +afterEach(() => { + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +// --- CRITICAL C3: keyless vintage consults the real release calendar ---------- +describe("C3 — keyless GDP vintage is the scheduled release, not the heuristic 1st", () => { + it("a window spanning the real release (2026-07-30) includes Q2 with the scheduled vintage", async () => { + // No FRED key → the keyless BEA GDP branch → restampKeyless → deriveReleaseVintage. + // Q2's ACTUAL advance release is 2026-07-30 (curated calendar), NOT the pre-fix + // heuristic 2026-07-01. This is the semantic mirror of Python C1. + vi.stubEnv("FRED_API_KEY", ""); + vi.stubEnv("BEA_API_KEY", "test-bea-key"); + fetchSpy.mockResolvedValue(jsonResponse(beaGdpMulti(["2026Q2", "-0.3"]))); + + const rows = await series( + "gdp", + new Date("2026-07-01T00:00:00Z"), + new Date("2026-08-05T00:00:00Z"), + { vintages: "all" }, + ); + + expect(rows).toHaveLength(1); + expect(rows[0]?.period).toBe("2026Q2"); + expect(rows[0]?.vintage_date.slice(0, 10)).toBe("2026-07-30"); + expect(rows[0]?.settlement_grade).toBe(false); + }); + + it("a window ending BEFORE the real release excludes Q2 (no look-ahead leak)", async () => { + // As of 2026-07-20 the Q2 advance (2026-07-30) has NOT landed. The pre-fix + // heuristic vintage (2026-07-01) WOULD have leaked it into [07-01, 07-20]; the + // calendar-consult correctly excludes it → not-yet-released, never empty. + vi.stubEnv("FRED_API_KEY", ""); + vi.stubEnv("BEA_API_KEY", "test-bea-key"); + fetchSpy.mockResolvedValue(jsonResponse(beaGdpMulti(["2026Q2", "-0.3"]))); + + await expect( + series("gdp", new Date("2026-07-01T00:00:00Z"), new Date("2026-07-20T00:00:00Z"), { + vintages: "all", + }), + ).rejects.toBeInstanceOf(IndicatorNotYetReleasedError); + }); +}); + +// --- HIGH H2: fetchFed normalizes a transport error to SourceUnavailableError -- +describe("H2 — fetchFed sanitizes a transport error to SourceUnavailableError", () => { + it("a forced transport rejection throws a sanitized SourceUnavailableError", async () => { + // Pre-fix fetchFed's fetchWithRetry was unwrapped, so a retry-exhaustion / + // transport error escaped as a raw core TherminalError. The wrap converts it + // to the documented SourceUnavailableError — matching fetchBls / fetchAlfred / + // fetchIcsa / fetchBea. Fake timers burn the core retry backoff instantly. + vi.useFakeTimers(); + fetchSpy.mockRejectedValue(new Error("ECONNREFUSED — federalreserve.gov unreachable")); + + // Attach the rejection handler BEFORE advancing timers (no unhandled rejection). + const settled = fetchFed({ indicator: "fed_funds" }).then( + () => ({ ok: true as const }), + (err: unknown) => ({ ok: false as const, err }), + ); + await vi.runAllTimersAsync(); + const result = await settled; + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected fetchFed to reject on a transport error"); + expect(result.err).toBeInstanceOf(SourceUnavailableError); + }); +}); diff --git a/packages-ts/econ/tests/codexReviewRound4.test.ts b/packages-ts/econ/tests/codexReviewRound4.test.ts new file mode 100644 index 00000000..2a06f3fa --- /dev/null +++ b/packages-ts/econ/tests/codexReviewRound4.test.ts @@ -0,0 +1,92 @@ +// Phase 29 — codex cross-model review ROUND 4, TS-side regression test. +// +// Round 4 CONVERGED (zero CRITICAL/HIGH). This locks the one TS MEDIUM fix: +// `deriveReleaseVintage`'s heuristic fallback must REJECT an out-of-range +// quarter/month (return undefined) instead of letting JS `Date.UTC` silently roll +// `2026Q5` over into 2027 — a parity divergence from Python's strict `date()`, +// which returns None (see `packages/econ/tests/test_codex_review_round4.py`). +// +// Drives the REAL keyless BEA GDP dispatch (mocking only `globalThis.fetch`), the +// same injected-transport style as round 2/3. The malformed period is one BEA +// never emits (Q1–4 only) — a defensive boundary guard, not a live path. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; + +import { series } from "../src/series.js"; + +type FetchFn = typeof globalThis.fetch; +let fetchSpy: MockInstance; + +function jsonResponse(payload: unknown, status = 200): Response { + return new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function beaGdpMulti(...quarters: Array<[string, string]>): unknown { + return { + BEAAPI: { + Results: { + Data: quarters.map(([period, value]) => ({ + TableName: "T10101", + LineNumber: "1", + TimePeriod: period, + DataValue: value, + SeriesCode: "A191RL1Q225SBEA", + })), + }, + }, + }; +} + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch") as unknown as MockInstance; +}); + +afterEach(() => { + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("M-r4-1 — out-of-range quarter does not silently roll over (parity with Python None)", () => { + it("keeps the un-derived (retrieved_at) vintage instead of the pre-fix 2027-04-01 rollover", async () => { + // No FRED key → keyless BEA GDP branch → restampKeyless → deriveReleaseVintage. + // "2026Q5" has no calendar entry → heuristic fallback. Pre-fix, JS Date.UTC + // rolled q=5 (month 14) into 2027 → the row's vintage became "2027-04-01". + // Post-fix the guard returns undefined (like Python's None), so restampKeyless + // leaves the original retrieved_at vintage untouched — never a 2027 rollover. + vi.stubEnv("FRED_API_KEY", ""); + vi.stubEnv("BEA_API_KEY", "test-bea-key"); + // Pin ONLY Date (not setTimeout) so the un-derived row's retrieved_at vintage + // is deterministic — the surviving row keeps retrieved_at, so the window must + // contain "now"; faking the clock removes the wall-clock dependency (no need to + // bump the window before 2028). toFake:['Date'] leaves real timers for any + // retry backoff, so the async fetch path can't wedge. + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-07-15T12:00:00Z")); + fetchSpy.mockResolvedValue(jsonResponse(beaGdpMulti(["2026Q5", "1.0"]))); + + const rows = await series( + "gdp", + new Date("2026-01-01T00:00:00Z"), + new Date("2026-12-31T00:00:00Z"), + { vintages: "all" }, + ); + + // Post-fix: derived vintage is undefined → row keeps its pinned retrieved_at + // (2026-07-15), which lands in [2026-01-01, 2026-12-31] → survives. Pre-fix the + // rolled-over 2027-04-01 vintage would fall OUTSIDE the window → the row would be + // dropped (length 0) — so the length assert alone already discriminates, and the + // value asserts nail down exactly why. + expect(rows).toHaveLength(1); + expect(rows[0]?.period).toBe("2026Q5"); + expect(rows[0]?.settlement_grade).toBe(false); + // The load-bearing assertion: NOT the pre-fix rolled-over vintage. + expect(rows[0]?.vintage_date.slice(0, 10)).not.toBe("2027-04-01"); + expect(rows[0]?.vintage_date.slice(0, 4)).not.toBe("2027"); + }); +}); diff --git a/packages-ts/econ/tests/dispatch.test.ts b/packages-ts/econ/tests/dispatch.test.ts new file mode 100644 index 00000000..cbb8f037 --- /dev/null +++ b/packages-ts/econ/tests/dispatch.test.ts @@ -0,0 +1,254 @@ +// Phase 29 29-13 (Batches E+F) — TS econ dispatch parity tests. +// +// These drive the REAL `defaultFetchRows` dispatch (NO injected `fetchRows`) and +// mock ONLY the HTTP transport (`globalThis.fetch`). This is the test style that +// catches the audit BLOCKER (findings 2/3/4): the pre-fix code sent BLS ids +// (WPSFD4 / CUUR0000SA0 — which do NOT exist on FRED) to ALFRED. We assert the +// FRED id that actually reaches the ALFRED transport (CPIAUCSL / PPIFIS / +// A191RL1Q225SBEA / ICSA), and that the Fed path hits federalreserve.gov's +// openmarket.htm (never a FRED host). + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; + +import { series } from "../src/series.js"; + +type FetchFn = typeof globalThis.fetch; +let fetchSpy: MockInstance; + +const D0 = new Date("2026-06-01T00:00:00Z"); +const D1 = new Date("2026-06-30T00:00:00Z"); + +/** A JSON Response for the ALFRED / FRED / BEA JSON transports. */ +function jsonResponse(payload: unknown): Response { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +/** A text/html Response for the Fed openmarket.htm transport. */ +function htmlResponse(html: string): Response { + return new Response(html, { status: 200, headers: { "content-type": "text/html" } }); +} + +/** ALFRED observations for a monthly BLS-family series (one in-window vintage). */ +function alfredMonthly(): unknown { + return { + observations: [{ realtime_start: "2026-06-11", date: "2026-05-01", value: "314.0" }], + }; +} + +/** ALFRED observations giving two periods 12mo apart (for the YoY math). */ +function alfredYoy(): unknown { + return { + observations: [ + { realtime_start: "2025-06-11", date: "2025-05-01", value: "300.0" }, + { realtime_start: "2026-06-11", date: "2026-05-01", value: "309.0" }, + ], + }; +} + +/** A minimal openmarket.htm rate-change record (one 2026 decision row). */ +function openmarketHtml(): string { + return ` +

2026

+ + + +
DateIncreaseDecreaseLevel (%)
June 110253.50-3.75
+ `; +} + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch") as unknown as MockInstance; +}); + +afterEach(() => { + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +/** The URL string of the first fetch call. */ +function firstUrl(): string { + return String(fetchSpy.mock.calls[0]?.[0]); +} + +// --- BLOCKER (findings 2/3/4): the FRED id, never a BLS id ------------------- +describe("defaultFetchRows — sends the FRED id (never a BLS id) to ALFRED", () => { + it("cpi routes to ALFRED with series_id=CPIAUCSL (not the BLS CUUR0000SA0)", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + fetchSpy.mockResolvedValue(jsonResponse(alfredMonthly())); + + const rows = await series("cpi", D0, D1); + + const url = firstUrl(); + expect(url).toContain("api.stlouisfed.org"); + expect(url).toContain("series_id=CPIAUCSL"); + expect(url).not.toContain("CUUR0000SA0"); + expect(rows.length).toBeGreaterThan(0); + expect(rows[0]?.settlement_grade).toBe(true); + expect(rows[0]?.value).toBe(314.0); + }); + + it("ppi routes to ALFRED with series_id=PPIFIS (NOT the BLS WPSFD4 — does not exist on FRED)", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + fetchSpy.mockResolvedValue(jsonResponse(alfredMonthly())); + + const rows = await series("ppi", D0, D1); + + const url = firstUrl(); + expect(url).toContain("series_id=PPIFIS"); + expect(url).not.toContain("WPSFD4"); + expect(rows.length).toBeGreaterThan(0); + }); + + it("gdp routes to ALFRED with the GROWTH-RATE series A191RL1Q225SBEA (not a level)", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + // Quarterly first print inside the window. + fetchSpy.mockResolvedValue( + jsonResponse({ + observations: [{ realtime_start: "2026-06-15", date: "2026-04-01", value: "-0.3" }], + }), + ); + + const rows = await series("gdp", D0, D1); + + const url = firstUrl(); + expect(url).toContain("series_id=A191RL1Q225SBEA"); + expect(url).not.toContain("GDPC1"); + expect(rows[0]?.indicator).toBe("gdp"); + expect(rows[0]?.units).toBe("percent"); + expect(rows[0]?.value).toBe(-0.3); + }); +}); + +// --- All 12 indicators dispatch (no generic "no fetcher registered") --------- +describe("defaultFetchRows — all 12 indicators dispatch (no generic Error)", () => { + const ALL = [ + "cpi", + "cpi_core", + "cpi_yoy", + "cpi_core_yoy", + "nfp", + "u3", + "ppi", + "ppi_yoy", + "gdp", + "jobless_claims", + "fed_funds", + "fed_decision", + ]; + + it("no indicator throws the generic dispatch Error", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + // Return a permissive transport: ALFRED-shaped JSON for the JSON paths and the + // openmarket HTML for the Fed path (its Response.text() ignores JSON shape). + fetchSpy.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("federalreserve.gov")) return htmlResponse(openmarketHtml()); + return jsonResponse(alfredYoy()); + }); + + for (const indicator of ALL) { + let caught: unknown; + try { + await series(indicator, D0, D1); + } catch (err) { + caught = err; + } + // A not-yet-released / data error is acceptable; the generic dispatch Error + // (the pre-fix "no fetcher registered") must NEVER be thrown. + if (caught instanceof Error) { + expect(caught.message).not.toMatch(/no fetcher registered/); + } + } + }); +}); + +// --- YoY returns percent, not the index level (finding 9) -------------------- +describe("defaultFetchRows — YoY is the 12-month percent change (not the index)", () => { + it("cpi_yoy returns units=percent and the computed % change", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + fetchSpy.mockResolvedValue(jsonResponse(alfredYoy())); + + const rows = await series("cpi_yoy", D0, D1); + + expect(rows.length).toBeGreaterThan(0); + const row = rows[0]; + expect(row?.indicator).toBe("cpi_yoy"); + expect(row?.units).toBe("percent"); + // 100 * (309 / 300 - 1) = 3.0 — NOT the 309 index level. + expect(row?.value).toBeCloseTo(3.0, 6); + }); +}); + +// --- Fed uses openmarket.htm (federalreserve.gov), never FRED ---------------- +describe("defaultFetchRows — Fed decisions come from federalreserve.gov openmarket.htm", () => { + it("fed_funds fetches openmarket.htm and parses a decision row (never a FRED host)", async () => { + fetchSpy.mockResolvedValue(htmlResponse(openmarketHtml())); + + const rows = await series("fed_funds", D0, D1); + + const url = firstUrl(); + expect(url).toContain("federalreserve.gov/monetarypolicy/openmarket.htm"); + expect(url).not.toContain("stlouisfed.org"); + expect(rows.length).toBeGreaterThan(0); + expect(rows[0]?.indicator).toBe("fed_funds"); + expect(rows[0]?.units).toBe("percent"); + // 3.50-3.75 midpoint = 3.625. + expect(rows[0]?.value).toBeCloseTo(3.625, 6); + }); +}); + +// --- jobless_claims routes to the ALFRED ICSA first print -------------------- +describe("defaultFetchRows — jobless_claims via ALFRED ICSA", () => { + it("keyed jobless_claims fetches series_id=ICSA with realtime vintages", async () => { + vi.stubEnv("FRED_API_KEY", "test-key"); + fetchSpy.mockResolvedValue( + jsonResponse({ + observations: [{ realtime_start: "2026-06-12", date: "2026-06-06", value: "231000" }], + }), + ); + + const rows = await series("jobless_claims", D0, D1); + + const url = firstUrl(); + expect(url).toContain("series_id=ICSA"); + expect(url).toContain("realtime_start"); + expect(rows[0]?.indicator).toBe("jobless_claims"); + expect(rows[0]?.settlement_grade).toBe(true); + }); +}); + +// --- keyless-settlement error names FRED_API_KEY (finding 13) ---------------- +describe("series — keyless settlement gap names FRED_API_KEY", () => { + it("cpi settlement with no FRED key throws an error naming FRED_API_KEY", async () => { + // No FRED_API_KEY → the BLS latest-revised fallback (settlement_grade=false). + vi.stubEnv("FRED_API_KEY", ""); + // BLS v1 POST returns a successful, non-empty latest-revised series. + fetchSpy.mockResolvedValue( + jsonResponse({ + status: "REQUEST_SUCCEEDED", + Results: { + series: [ + { + seriesID: "CUUR0000SA0", + data: [{ year: "2026", period: "M05", value: "314.0", footnotes: [{}] }], + }, + ], + }, + }), + ); + + let caught: unknown; + try { + await series("cpi", D0, D1); // vintages defaults to "settlement". + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(String((caught as Error).message)).toContain("FRED_API_KEY"); + }); +}); diff --git a/packages-ts/econ/tests/researchEcon.test.ts b/packages-ts/econ/tests/researchEcon.test.ts new file mode 100644 index 00000000..7e37f9ed --- /dev/null +++ b/packages-ts/econ/tests/researchEcon.test.ts @@ -0,0 +1,381 @@ +// Phase 29 29-09 Task 2 — TS econ surface parity tests. +// +// Mirrors the Python econ vertical semantics (29-04 resolver + 29-08 surface): +// 1. researchEcon returns pairs (market outcome + first-print value + as-of +// features) with knowledge_time = vintage_date. +// 2. A vintage_date after the asOf cutoff makes assertNoLeakage throw. +// 3. A TE-settled series (KXUSPPI) returns the agency first-print labeled +// settlementGrade=false + a divergence warning; NO fabricated TE value. +// 4. A missing release throws IndicatorNotYetReleasedError (never []/null). +// 5. Resolver per-series split: KXUSPPIYOY -> BLS/true, KXUSPPI -> TE/false. +// +// The fetch layer is injected via `{ fetchRows }` (the TS analog of the Python +// tests monkeypatching INDICATOR_FETCHERS) so no live network is touched — the +// leakage/TE/not-yet-released semantics are exercised deterministically. + +import { IndicatorNotYetReleasedError, LeakageError } from "@mostlyrightmd/core"; +import { TimePoint } from "@mostlyrightmd/core/temporal"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { history } from "../src/history.js"; +import { releases } from "../src/history.js"; +import { type DivergenceWarning, researchEcon } from "../src/researchEcon.js"; +import { EconTickerError, kalshiEconResolve } from "../src/resolvers/kalshiEcon.js"; +import type { EconObservationRow } from "../src/schema.js"; + +// A settlement-grade first-print row (ALFRED-shape: settlementGrade true, +// release_type advance, knowledge_time == vintage_date). +function firstPrintRow(overrides: Partial = {}): EconObservationRow { + const vintage = "2026-07-15T12:30:00Z"; + return { + indicator: "cpi_yoy", + series_id: "CPIAUCSL", + period: "2026-06", + value: 3.1, + units: "percent", + release_datetime: "2026-07-15T12:30:00Z", + vintage_date: vintage, + release_type: "advance", + settlement_grade: true, + knowledge_time: vintage, + source: "alfred", + retrieved_at: "2026-07-16T00:00:00Z", + ...overrides, + }; +} + +/** Injected fetcher that returns a fixed row set (bypasses live network). */ +function stubFetch(rows: EconObservationRow[]): () => Promise { + return () => Promise.resolve(rows); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// --------------------------------------------------------------------------- +// Test 5 (resolver parity) — the per-series split. +// --------------------------------------------------------------------------- +describe("kalshiEconResolve — per-series split (parity with Python)", () => { + it("KXUSPPIYOY -> BLS / settlementGrade true", () => { + const r = kalshiEconResolve("KXUSPPIYOY"); + expect(r.agency).toBe("BLS"); + expect(r.indicator).toBe("ppi_yoy"); + expect(r.settlementGrade).toBe(true); + expect(r.root).toBe("KXUSPPIYOY"); + }); + + it("KXUSPPI -> TradingEconomics / settlementGrade false (NOT the longer KXUSPPIYOY rule)", () => { + const r = kalshiEconResolve("KXUSPPI"); + expect(r.agency).toBe("TradingEconomics"); + expect(r.indicator).toBe("ppi"); + expect(r.settlementGrade).toBe(false); + }); + + it("longest-prefix wins: KXUSPPIYOY-26JUL binds to the KXUSPPIYOY rule, not KXUSPPI", () => { + const r = kalshiEconResolve("KXUSPPIYOY-26JUL"); + expect(r.agency).toBe("BLS"); + expect(r.settlementGrade).toBe(true); + expect(r.root).toBe("KXUSPPIYOY"); + }); + + it("is case-insensitive", () => { + expect(kalshiEconResolve("kxcpiyoy").agency).toBe("BLS"); + }); + + it("KXCPIYOY -> BLS / cpi_yoy / true", () => { + const r = kalshiEconResolve("KXCPIYOY"); + expect(r.agency).toBe("BLS"); + expect(r.indicator).toBe("cpi_yoy"); + expect(r.settlementGrade).toBe(true); + }); + + it("throws EconTickerError on an unknown ticker (never returns null)", () => { + expect(() => kalshiEconResolve("KXNONSENSE")).toThrow(EconTickerError); + }); + + it("throws on a non-string ticker", () => { + // @ts-expect-error — deliberately wrong type to exercise the runtime guard. + expect(() => kalshiEconResolve(123)).toThrow(); + }); +}); + +// --------------------------------------------------------------------------- +// Test 1 — researchEcon returns leakage-free pairs (knowledge_time=vintage_date). +// --------------------------------------------------------------------------- +describe("researchEcon — settlement pairs", () => { + it("KXCPIYOY returns pairs with market outcome + first-print value + knowledge_time=vintage_date", async () => { + const rows = [firstPrintRow()]; + const pairs = await researchEcon( + "KXCPIYOY", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-07-31T00:00:00Z"), + { + fetchRows: stubFetch(rows), + }, + ); + expect(pairs).toHaveLength(1); + const p = pairs[0]; + if (p === undefined) throw new Error("expected a pair"); + // resolved market-outcome metadata + expect(p.agency).toBe("BLS"); + expect(p.indicator).toBe("cpi_yoy"); + expect(p.contract).toBe("KXCPIYOY"); + expect(p.settlementGrade).toBe(true); + // C3: the canonical snake_case field agrees with the camelCase alias. + expect(p.settlement_grade).toBe(true); + // first-print value + expect(p.value).toBe(3.1); + // knowledge_time == vintage_date (the leakage cutoff column) + expect(p.knowledge_time).toBe(p.vintage_date); + expect(p.knowledge_time).toBe("2026-07-15T12:30:00Z"); + }); +}); + +// --------------------------------------------------------------------------- +// Test 2 (leakage) — a vintage after asOf throws via assertNoLeakage. +// --------------------------------------------------------------------------- +describe("researchEcon — leakage guard", () => { + it("a row whose vintage_date is after asOf throws LeakageError", async () => { + const rows = [ + firstPrintRow({ + vintage_date: "2026-07-15T12:30:00Z", + knowledge_time: "2026-07-15T12:30:00Z", + }), + ]; + // asOf is BEFORE the vintage → the pair leaks a future release. + const asOf = new TimePoint("2026-07-01T00:00:00Z"); + await expect( + researchEcon("KXCPIYOY", new Date("2026-06-01T00:00:00Z"), new Date("2026-07-31T00:00:00Z"), { + asOf, + fetchRows: stubFetch(rows), + }), + ).rejects.toBeInstanceOf(LeakageError); + }); + + it("a row whose vintage_date is at/before asOf passes the leakage guard", async () => { + const rows = [ + firstPrintRow({ + vintage_date: "2026-07-15T12:30:00Z", + knowledge_time: "2026-07-15T12:30:00Z", + }), + ]; + const asOf = new TimePoint("2026-07-20T00:00:00Z"); + const pairs = await researchEcon( + "KXCPIYOY", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-07-31T00:00:00Z"), + { + asOf, + fetchRows: stubFetch(rows), + }, + ); + expect(pairs).toHaveLength(1); + }); +}); + +// --------------------------------------------------------------------------- +// Test 3 (TE divergence) — agency first-print, settlementGrade=false + warning. +// --------------------------------------------------------------------------- +describe("researchEcon — TE divergence (honest, never fabricated)", () => { + it("KXUSPPI returns the agency first print labeled settlementGrade=false + a structured divergence; no fabricated TE value, no console write", async () => { + // The fetched row is the AGENCY (BLS/ALFRED) first print for PPI. + const agencyRow = firstPrintRow({ + indicator: "ppi", + period: "2026-06", + value: 2.4, + series_id: "WPSFD4", + }); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const events: DivergenceWarning[] = []; + + const pairs = await researchEcon( + "KXUSPPI", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-07-31T00:00:00Z"), + { + fetchRows: stubFetch([agencyRow]), + onDivergence: (w) => events.push(w), + }, + ); + + expect(pairs).toHaveLength(1); + const p = pairs[0]; + if (p === undefined) throw new Error("expected a pair"); + // The value is the AGENCY first print (2.4) — never a fabricated TE number. + expect(p.value).toBe(2.4); + // Contract-level settlement grade is false (TE-settled). C3: the CANONICAL + // snake_case schema field must read false too (not just the camelCase alias) — + // pre-fix the inherited agency `settlement_grade=true` shadowed it. + expect(p.settlement_grade).toBe(false); + expect(p.settlementGrade).toBe(false); + expect(p.agency).toBe("TradingEconomics"); + // The divergence is surfaced via the STRUCTURED callback, naming the series… + expect(events).toHaveLength(1); + expect(events[0]?.message).toContain("KXUSPPI"); + expect(events[0]?.message).toContain("Trading Economics"); + // …and L1: researchEcon writes NOTHING to the console. + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("emits a DivergenceWarning-shaped payload (named, machine-readable)", async () => { + const agencyRow = firstPrintRow({ indicator: "ppi", period: "2026-06", value: 2.4 }); + const events: DivergenceWarning[] = []; + const pairs = await researchEcon( + "KXUSPPI", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-07-31T00:00:00Z"), + { + fetchRows: stubFetch([agencyRow]), + onDivergence: (w) => events.push(w), + }, + ); + expect(pairs).toHaveLength(1); + expect(events).toHaveLength(1); + const [w] = events; + if (w === undefined) throw new Error("expected a divergence event"); + expect(w.contract).toBe("KXUSPPI"); + expect(w.indicator).toBe("ppi"); + expect(w.settlementAuthority).toBe("TradingEconomics"); + }); +}); + +// --------------------------------------------------------------------------- +// Test 4 (not-yet-released) — throws IndicatorNotYetReleasedError, never []/null. +// --------------------------------------------------------------------------- +describe("history / researchEcon — not-yet-released", () => { + it("history throws IndicatorNotYetReleasedError when the window has no relevant rows", async () => { + await expect( + history("cpi", new Date("2026-06-01T00:00:00Z"), new Date("2026-07-31T00:00:00Z"), { + fetchRows: stubFetch([]), + }), + ).rejects.toBeInstanceOf(IndicatorNotYetReleasedError); + }); + + it("researchEcon propagates IndicatorNotYetReleasedError (never returns []/null)", async () => { + await expect( + researchEcon("KXCPIYOY", new Date("2026-06-01T00:00:00Z"), new Date("2026-07-31T00:00:00Z"), { + fetchRows: stubFetch([]), + }), + ).rejects.toBeInstanceOf(IndicatorNotYetReleasedError); + }); + + it("the not-yet-released error carries the indicator + errorCode", async () => { + try { + await history("cpi", new Date("2026-06-01T00:00:00Z"), new Date("2026-07-31T00:00:00Z"), { + fetchRows: stubFetch([]), + }); + throw new Error("expected IndicatorNotYetReleasedError"); + } catch (e) { + expect(e).toBeInstanceOf(IndicatorNotYetReleasedError); + const err = e as IndicatorNotYetReleasedError; + expect(err.indicator).toBe("cpi"); + expect(err.errorCode).toBe("INDICATOR_NOT_YET_RELEASED"); + } + }); +}); + +// --------------------------------------------------------------------------- +// history — read-time vintage filter + FEDS floor. +// --------------------------------------------------------------------------- +describe("history — vintage filter + FEDS floor", () => { + it("vintages='settlement' keeps only settlement_grade rows", async () => { + const settlementRow = firstPrintRow({ + period: "2026-05", + settlement_grade: true, + release_type: "advance", + }); + const revisedRow = firstPrintRow({ + period: "2026-05", + settlement_grade: false, + release_type: "revised", + value: 3.2, + vintage_date: "2026-08-15T12:30:00Z", + knowledge_time: "2026-08-15T12:30:00Z", + }); + const rows = await history( + "cpi_yoy", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-09-30T00:00:00Z"), + { + vintages: "settlement", + fetchRows: stubFetch([settlementRow, revisedRow]), + }, + ); + expect(rows).toHaveLength(1); + expect(rows[0]?.settlement_grade).toBe(true); + }); + + it("vintages='all' keeps every vintage", async () => { + const settlementRow = firstPrintRow({ period: "2026-05", settlement_grade: true }); + const revisedRow = firstPrintRow({ + period: "2026-05", + settlement_grade: false, + release_type: "revised", + vintage_date: "2026-08-15T12:30:00Z", + knowledge_time: "2026-08-15T12:30:00Z", + }); + const rows = await history( + "cpi_yoy", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-09-30T00:00:00Z"), + { + vintages: "all", + fetchRows: stubFetch([settlementRow, revisedRow]), + }, + ); + expect(rows).toHaveLength(2); + }); + + it("throws on an invalid vintages value", async () => { + await expect( + history("cpi", new Date("2026-06-01T00:00:00Z"), new Date("2026-07-31T00:00:00Z"), { + // @ts-expect-error — deliberately invalid literal. + vintages: "bogus", + }), + ).rejects.toThrow(); + }); + + it("rejects a below-FEDS-floor from_date (out of range, not partial data)", async () => { + // CPI floor is 2021-06-01; 2020 is below it. + await expect( + history("cpi", new Date("2020-01-01T00:00:00Z"), new Date("2020-03-31T00:00:00Z"), { + fetchRows: stubFetch([firstPrintRow({ indicator: "cpi" })]), + }), + ).rejects.toThrow(/floor|out of range|out_of_window/i); + }); + + it("throws on an unknown indicator (no FEDS floor)", async () => { + await expect( + history( + "bogus_indicator", + new Date("2026-06-01T00:00:00Z"), + new Date("2026-07-31T00:00:00Z"), + { + fetchRows: stubFetch([]), + }, + ), + ).rejects.toThrow(); + }); +}); + +// --------------------------------------------------------------------------- +// releases — the schedule surface (mirrors the Python curated table). +// --------------------------------------------------------------------------- +describe("releases — schedule", () => { + it("returns CPI schedule events sorted ascending by releaseDatetime", () => { + const events = releases("cpi"); + expect(events.length).toBeGreaterThan(0); + for (let i = 1; i < events.length; i++) { + const prev = events[i - 1]; + const cur = events[i]; + if (prev === undefined || cur === undefined) throw new Error("index"); + expect(prev.releaseDatetime <= cur.releaseDatetime).toBe(true); + } + expect(events[0]?.agency).toBe("BLS"); + }); + + it("throws on an unknown indicator (never returns []/null)", () => { + expect(() => releases("bogus")).toThrow(); + }); +}); diff --git a/packages-ts/econ/tests/series.test.ts b/packages-ts/econ/tests/series.test.ts new file mode 100644 index 00000000..16fb2431 --- /dev/null +++ b/packages-ts/econ/tests/series.test.ts @@ -0,0 +1,130 @@ +// Phase 29 29-11 — TS econ `series` surface conformance tests. +// +// Mirrors the Python `series` surface (plan 29-11) conforming the econ vertical +// to the cross-vertical source-identity contract (docs/source-identity.md): +// 1. `series` is the CANONICAL read function; `history` is a byte-identical +// re-export alias tagged `@deprecated` (no runtime warning this release). +// 2. `source` (contract §1): accepted {undefined,"fred","bls","bea","dol","fed"}; +// an unknown source, OR a valid authority that cannot serve the indicator, +// throws BEFORE any fetch (a spy proves zero fetch invocations). +// 3. `delivery` (contract §2): "live" default | "hosted" throws +// SourceUnavailableError naming ECON_HOSTED_URL; unknown throws pre-fetch. +// +// The fetch layer is injected via `{ fetchRows }` so no live network is touched. + +import { SourceUnavailableError } from "@mostlyrightmd/core"; +import { describe, expect, it, vi } from "vitest"; + +import type { EconObservationRow } from "../src/schema.js"; +import { history, series } from "../src/series.js"; + +function firstPrintRow(overrides: Partial = {}): EconObservationRow { + const vintage = "2026-06-11T12:30:00Z"; + return { + indicator: "cpi", + series_id: "CUUR0000SA0", + period: "2026-05", + value: 314.0, + units: "index", + release_datetime: vintage, + vintage_date: vintage, + release_type: "advance", + settlement_grade: true, + knowledge_time: vintage, + source: "alfred", + retrieved_at: "2026-06-12T00:00:00Z", + ...overrides, + }; +} + +/** A spy fetcher: records call count, returns a fixed row set. */ +function spyFetch(rows: EconObservationRow[]): { + fetch: () => Promise; + calls: () => number; +} { + let n = 0; + return { + fetch: () => { + n += 1; + return Promise.resolve(rows); + }, + calls: () => n, + }; +} + +const D0 = new Date("2026-06-01T00:00:00Z"); +const D1 = new Date("2026-06-30T00:00:00Z"); + +// --- series is the canonical read; history is a distinct (alias) callable ----- +describe("series — canonical read + history alias", () => { + it("series and history are distinct callables", () => { + expect(series).not.toBe(history); + }); + + it("history returns output byte-identical to series for the same args", async () => { + const rows = [firstPrintRow()]; + const viaSeries = await series("cpi", D0, D1, { fetchRows: spyFetch(rows).fetch }); + const viaHistory = await history("cpi", D0, D1, { fetchRows: spyFetch(rows).fetch }); + expect(viaHistory).toEqual(viaSeries); + }); +}); + +// --- source= loud validation (contract §1) ----------------------------------- +describe("series — source= loud pre-network validation", () => { + it("an unknown source throws naming the accepted set, with zero fetch calls", async () => { + const spy = spyFetch([firstPrintRow()]); + await expect( + // @ts-expect-error — deliberately invalid source literal. + series("cpi", D0, D1, { source: "nope", fetchRows: spy.fetch }), + ).rejects.toThrow(/fred|bls|bea|dol|fed/); + expect(spy.calls()).toBe(0); + }); + + it("a valid authority that cannot serve the indicator throws before any fetch", async () => { + // source="bea" is valid but does not serve CPI — loud, never a silent fallback. + const spy = spyFetch([firstPrintRow()]); + await expect(series("cpi", D0, D1, { source: "bea", fetchRows: spy.fetch })).rejects.toThrow( + /cpi|bea/, + ); + expect(spy.calls()).toBe(0); + }); + + it("a valid authority that serves the indicator is accepted", async () => { + const rows = [firstPrintRow()]; + const out = await series("cpi", D0, D1, { source: "bls", fetchRows: spyFetch(rows).fetch }); + expect(out).toHaveLength(1); + expect(out[0]?.indicator).toBe("cpi"); + }); +}); + +// --- delivery= loud validation (contract §2) --------------------------------- +describe("series — delivery= loud pre-network validation", () => { + it("delivery:'hosted' throws SourceUnavailableError naming ECON_HOSTED_URL, zero fetch", async () => { + const spy = spyFetch([firstPrintRow()]); + await expect( + series("cpi", D0, D1, { delivery: "hosted", fetchRows: spy.fetch }), + ).rejects.toBeInstanceOf(SourceUnavailableError); + // And the message names the reserved seam. + try { + await series("cpi", D0, D1, { delivery: "hosted", fetchRows: spy.fetch }); + } catch (e) { + expect(String((e as Error).message)).toContain("ECON_HOSTED_URL"); + } + expect(spy.calls()).toBe(0); + }); + + it("an unknown delivery throws naming the accepted set, zero fetch", async () => { + const spy = spyFetch([firstPrintRow()]); + await expect( + // @ts-expect-error — deliberately invalid delivery literal. + series("cpi", D0, D1, { delivery: "cloud", fetchRows: spy.fetch }), + ).rejects.toThrow(/live|hosted/); + expect(spy.calls()).toBe(0); + }); + + it("delivery:'live' is the default (accepted)", async () => { + const rows = [firstPrintRow()]; + const out = await series("cpi", D0, D1, { delivery: "live", fetchRows: spyFetch(rows).fetch }); + expect(out).toHaveLength(1); + }); +}); diff --git a/packages-ts/econ/tests/snapshot.test.ts b/packages-ts/econ/tests/snapshot.test.ts new file mode 100644 index 00000000..2ce260ce --- /dev/null +++ b/packages-ts/econ/tests/snapshot.test.ts @@ -0,0 +1,131 @@ +// Phase 29 29-11 — TS econ `snapshot` surface tests. +// +// Mirrors the Python `snapshot` surface (plan 29-11): the settlement-target state +// of an indicator as knowable at `asOf`. +// +// snapshot({ indicator, asOf?, source?, delivery?, fetchRows? }) returns the +// latest settlement-grade (settlement_grade === true) vintage whose +// vintage_date <= asOf (default asOf: now), per period. Nothing knowable -> +// IndicatorNotYetReleasedError (never []/null). Forwards source=/delivery= with +// the same loud pre-network validation as `series`. +// +// The fetch layer is injected via `{ fetchRows }` — no live network. + +import { IndicatorNotYetReleasedError, SourceUnavailableError } from "@mostlyrightmd/core"; +import { describe, expect, it } from "vitest"; + +import type { EconObservationRow } from "../src/schema.js"; +import { snapshot } from "../src/snapshot.js"; + +// First print lands 2026-06-11 (settlement-grade); a later revision lands +// 2026-07-15 (settlement_grade false — Kalshi excludes post-expiration revisions). +const FIRST_PRINT = "2026-06-11T12:30:00Z"; +const REVISION = "2026-07-15T12:30:00Z"; + +function cpiRow(overrides: Partial = {}): EconObservationRow { + return { + indicator: "cpi", + series_id: "CUUR0000SA0", + period: "2026-05", + value: 314.0, + units: "index", + release_datetime: FIRST_PRINT, + vintage_date: FIRST_PRINT, + release_type: "advance", + settlement_grade: true, + knowledge_time: FIRST_PRINT, + source: "alfred", + retrieved_at: "2026-06-12T00:00:00Z", + ...overrides, + }; +} + +function cpiVintages(): EconObservationRow[] { + return [ + cpiRow(), + cpiRow({ + value: 315.0, + vintage_date: REVISION, + release_datetime: REVISION, + knowledge_time: REVISION, + settlement_grade: false, + release_type: "revised", + }), + ]; +} + +function stubFetch(rows: EconObservationRow[]): () => Promise { + return () => Promise.resolve(rows); +} + +// --- asOf before the first vintage → nothing knowable → throws ---------------- +describe("snapshot — as-of semantics", () => { + it("asOf before the first vintage throws IndicatorNotYetReleasedError", async () => { + await expect( + snapshot({ + indicator: "cpi", + asOf: new Date("2026-06-01T00:00:00Z"), + fetchRows: stubFetch(cpiVintages()), + }), + ).rejects.toBeInstanceOf(IndicatorNotYetReleasedError); + }); + + it("asOf after the first vintage returns the first-print settlement row", async () => { + const snap = await snapshot({ + indicator: "cpi", + asOf: new Date("2026-06-30T00:00:00Z"), + fetchRows: stubFetch(cpiVintages()), + }); + expect(snap).toHaveLength(1); + expect(snap[0]?.settlement_grade).toBe(true); + expect(snap[0]?.value).toBe(314.0); + expect(snap[0]?.vintage_date).toBe(FIRST_PRINT); + }); + + it("a post-expiration revision is never the settlement snapshot", async () => { + // asOf AFTER the revision: the settlement snapshot is STILL the first print, + // because the revision is settlement_grade=false (Kalshi excludes it). + const snap = await snapshot({ + indicator: "cpi", + asOf: new Date("2026-08-01T00:00:00Z"), + fetchRows: stubFetch(cpiVintages()), + }); + expect(snap).toHaveLength(1); + expect(snap[0]?.value).toBe(314.0); + expect(snap[0]?.settlement_grade).toBe(true); + }); + + it("default asOf is now (a released past window is knowable)", async () => { + const snap = await snapshot({ indicator: "cpi", fetchRows: stubFetch(cpiVintages()) }); + expect(snap).toHaveLength(1); + expect(snap[0]?.settlement_grade).toBe(true); + }); +}); + +// --- forwards source=/delivery= with the same loud validation ----------------- +describe("snapshot — forwards source/delivery loud validation", () => { + it("an unknown source throws before any fetch", async () => { + let called = 0; + const fetchRows = () => { + called += 1; + return Promise.resolve(cpiVintages()); + }; + await expect( + // @ts-expect-error — deliberately invalid source literal. + snapshot({ indicator: "cpi", source: "nope", fetchRows }), + ).rejects.toThrow(/fred|bls|bea|dol|fed/); + expect(called).toBe(0); + }); + + it("delivery:'hosted' throws SourceUnavailableError naming ECON_HOSTED_URL", async () => { + let called = 0; + const fetchRows = () => { + called += 1; + return Promise.resolve(cpiVintages()); + }; + await expect( + snapshot({ indicator: "cpi", delivery: "hosted", fetchRows }), + ).rejects.toBeInstanceOf(SourceUnavailableError); + expect(called).toBe(0); + }); +}); diff --git a/packages-ts/econ/tsconfig.json b/packages-ts/econ/tsconfig.json new file mode 100644 index 00000000..c1e85167 --- /dev/null +++ b/packages-ts/econ/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": ["src/**/*", "tests/**/*"], + "exclude": ["node_modules", "dist", "coverage"] +} diff --git a/packages-ts/econ/tsup.config.ts b/packages-ts/econ/tsup.config.ts new file mode 100644 index 00000000..ba215199 --- /dev/null +++ b/packages-ts/econ/tsup.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + // Single public entry — the econ vertical has no subpath exports (unlike + // weather's `/live` `/forecasts` `/hosted`). `history` / `releases` / + // `researchEcon` + the schema + the resolver all ship from the root barrel. + entry: ["src/index.ts"], + format: ["esm", "cjs", "iife"], + globalName: "mostlyrightEcon", + dts: true, + sourcemap: true, + clean: true, + target: "es2022", + outExtension({ format }) { + if (format === "esm") return { js: ".mjs" }; + if (format === "cjs") return { js: ".cjs" }; + return { js: ".global.js" }; + }, +}); diff --git a/packages-ts/econ/vitest.config.ts b/packages-ts/econ/vitest.config.ts new file mode 100644 index 00000000..68440c50 --- /dev/null +++ b/packages-ts/econ/vitest.config.ts @@ -0,0 +1,42 @@ +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// Alias workspace siblings to their source files so vitest can resolve +// @mostlyrightmd/core (and its /temporal subpath) without a pre-build step. +// Build still uses dist via the consumer's exports map. Mirrors +// packages-ts/weather/vitest.config.ts (TS-W0 iter-1 HIGH 5). +export default defineConfig({ + resolve: { + alias: [ + // Order matters — most-specific (subpath) FIRST. Vite's alias resolver + // walks the array in order and uses the first match, so a bare + // "@mostlyrightmd/core" entry above the subpaths would shadow them. + { + find: "@mostlyrightmd/core/temporal", + replacement: resolve(__dirname, "../core/src/temporal/index.ts"), + }, + { + find: "@mostlyrightmd/core", + replacement: resolve(__dirname, "../core/src/index.ts"), + }, + // Self-alias so tests can import from @mostlyrightmd/econ. + { + find: "@mostlyrightmd/econ", + replacement: resolve(__dirname, "./src/index.ts"), + }, + ], + }, + test: { + include: ["tests/**/*.test.ts"], + exclude: ["**/*.live.test.ts", "**/node_modules/**", "**/dist/**"], + coverage: { + provider: "v8", + reporter: ["text", "lcov"], + include: ["src/**/*.ts"], + exclude: ["**/generated/**", "**/*.d.ts"], + }, + }, +}); diff --git a/packages/core/src/mostlyright/core/exceptions.py b/packages/core/src/mostlyright/core/exceptions.py index 94850469..c81b492c 100644 --- a/packages/core/src/mostlyright/core/exceptions.py +++ b/packages/core/src/mostlyright/core/exceptions.py @@ -37,6 +37,7 @@ "GoesS3Error", "GribIntegrityError", "HistoricalDepthError", + "IndicatorNotYetReleasedError", "IssuedAtMissingError", "KalshiCountRuleViolation", "LabelAlignmentError", @@ -1151,6 +1152,66 @@ class CaptureNotAvailable(EarningsError): default_error_code = "EARNINGS_CAPTURE_NOT_AVAILABLE" +class IndicatorNotYetReleasedError(MostlyRightError): + """A requested economic release is EXPECTED but has not been published yet. + + Phase 29 econ vertical (CONTEXT Area 4): "not yet released" is a normal, + branchable control state distinct from "unavailable" — the release calendar + says a print is due, but ``vintage_date`` for that period does not exist + yet. The econ surface (``econ.history`` / ``econ.releases`` / + ``econ.research_econ``) raises this instead of returning ``[]`` / ``None`` + so a caller can tell "the data is genuinely gone" (a + :class:`DataAvailabilityError`) apart from "come back after the 8:30 ET + drop" (this error) and, when known, retry at ``expected_release``. + + Lives in the core hierarchy (not ``packages/econ``) for the same reason + :class:`EarningsError` / :class:`NoCWOPDataError` do: the exception taxonomy + is centralized so MCP JSON-RPC serialization and the Python↔TS lockstep + discipline apply uniformly. ``mostlyright.econ`` re-exports it. + + Attributes: + indicator: The indicator id (``"cpi"``, ``"nfp"``, ``"gdp"``, …). + period: The observation period requested (``"2026-06"``, ``"2026Q2"``, + an FOMC meeting date, …). + expected_release: The scheduled release wall-clock when known, else + ``None`` (the payload never fabricates a timestamp). + """ + + default_error_code = "INDICATOR_NOT_YET_RELEASED" + + def __init__( + self, + indicator: str, + period: str, + *, + expected_release: datetime | None = None, + source: str | None = None, + request_id: str | None = None, + error_code: str | None = None, + ) -> None: + message = f"{indicator} for period {period!r} is not yet released" + if expected_release is not None: + message += f" (expected {expected_release.isoformat()})" + super().__init__( + message, + error_code=error_code, + source=source, + request_id=request_id, + ) + self.indicator: str = indicator + self.period: str = period + self.expected_release: datetime | None = expected_release + + def _payload(self) -> dict[str, Any]: + payload = super()._payload() + payload.update( + indicator=self.indicator, + period=self.period, + expected_release=(self.expected_release.isoformat() if self.expected_release else None), + ) + return payload + + # ---------------------------------------------------------------------- # Deprecation alias: MostlyRightMCPError → MostlyRightError # ---------------------------------------------------------------------- diff --git a/packages/econ/README.md b/packages/econ/README.md new file mode 100644 index 00000000..3620a3da --- /dev/null +++ b/packages/econ/README.md @@ -0,0 +1,34 @@ +# mostlyrightmd-econ + +Economic-indicator data for the `mostlyright` Python SDK — CPI, PPI, nonfarm payrolls (NFP), unemployment, GDP, jobless claims, and Fed decisions — sourced from FRED/ALFRED, BLS, BEA, DOL, and the Federal Reserve, joined to Kalshi and Polymarket econ markets for leakage-free settlement pairs. First-print / ALFRED vintage discipline so backtests settle the way the contracts do. Imports as `mostlyright.econ`. + +> **Under active construction.** This package is being built out across phase 29. The public surface (`econ.history()`, `econ.releases()`, `econ.research_econ()`) is stubbed and raises `NotImplementedError` until the indicator plans land. + +## Install + +```bash +pip install mostlyrightmd-econ # brings mostlyrightmd transitively +``` + +## Data sources & required notices + +This product uses the FRED® API but is not endorsed or certified by the Federal +Reserve Bank of St. Louis. This product uses the Bureau of Economic Analysis +(BEA) Data API but is not endorsed or certified by BEA. BLS, BEA, DOL, and +Federal Reserve Board data are US-government works in the public domain; +citations to the source agencies are appreciated. Nothing in this package is +endorsed by, or implies endorsement by, any source agency or Reserve Bank. + +The **keyless default path makes no FRED call**. The FRED/ALFRED vintage path is +opt-in via your own free `FRED_API_KEY` and is governed by the +[FRED® API Terms of Use](https://fred.stlouisfed.org/docs/api/terms_of_use.html) +(which, among other things, prohibits storing/caching FRED content and using it +in ML training) — your key, your acceptance. The weekly jobless-claims fetcher +also transports its bytes through FRED (`ICSA`/`ICNSA`) because DOL's +machine-readable path is bot-walled. See the +[data-sources & licensing section](../../docs/econ-vertical.md#data-sources-licensing--required-notices) +of the vertical docs for the per-source table. + +## Docs + +See for quickstart and the full API reference. diff --git a/packages/econ/pyproject.toml b/packages/econ/pyproject.toml new file mode 100644 index 00000000..16256c2b --- /dev/null +++ b/packages/econ/pyproject.toml @@ -0,0 +1,84 @@ +[project] +name = "mostlyrightmd-econ" +version = "1.15.0" +description = "Economic-indicator data for Python — CPI, PPI, nonfarm payrolls, unemployment, GDP, jobless claims, Fed decisions — from FRED/ALFRED, BLS, BEA, DOL, Federal Reserve. First-print vintage discipline for prediction-market settlement. Imports as `mostlyright.econ`." +readme = "README.md" +license = "MIT" +authors = [{ name = "Robert Tarabcak", email = "tarabcakr@gmail.com" }] +requires-python = ">=3.11" +keywords = [ + "economic-data", + "economic-indicators", + "cpi", + "ppi", + "nonfarm-payrolls", + "unemployment", + "gdp", + "jobless-claims", + "fed", + "fomc", + "fred", + "alfred", + "bls", + "bea", + "prediction-markets", + "kalshi", + "polymarket", + "settlement", + "first-print", + "vintage", + "backtesting", + "ml-training-data", + "macro", +] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] + +dependencies = [ + # mostlyright (core) provides the load-bearing shared surface the econ + # fetchers/cache/resolver build on: _internal._bounds, _internal._cache_dir, + # _internal._http, core.schema, core.validator, core.temporal, core.exceptions. + # Pinned >=1.11.0 (the lockstep floor for THIS release) so a partial upgrade + # (econ==1.11.0 + core==1.10.x) can't ImportError on a symbol econ needs + # that first ships in core 1.11.0 — mirrors the PKG-03 floor discipline the + # weather/markets siblings use across the parity boundary. + "mostlyrightmd>=1.11.0,<2.0", + "httpx>=0.27", + "jsonschema>=4.21", + "tzdata; sys_platform == 'win32'", + # Local per-release parquet cache requires filelock for concurrent-process + # safety (mirrors weather). pyarrow is imported inside the econ _cache.py + # only (later plan); the upper bound (PKG-06) keeps a future pyarrow ABI + # break from silently invalidating the first-print vintage cache. + "filelock>=3.12", + "pyarrow>=17.0,<24.0", +] + +[project.urls] +Homepage = "https://mostlyright.md" +Documentation = "https://mostlyright.md/docs/sdk/" +Repository = "https://github.com/mostlyrightmd/mostlyright-sdk" +Issues = "https://github.com/mostlyrightmd/mostlyright-sdk/issues" +Changelog = "https://github.com/mostlyrightmd/mostlyright-sdk/blob/main/CHANGELOG.md" + +[project.optional-dependencies] +# history()/research_econ() return DataFrames; pandas is opt-in (not a base +# dep) so a caller who only touches the raw-observation surface can skip it. +# Upper bound mirrors the core/weather/markets `parquet` extra caps +# (PKG-05/PKG-06) so a fresh econ install cannot silently drift the +# DataFrame dtypes across the parity-critical pandas boundary. +pandas = [ + "pandas>=2.2,<4.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/mostlyright"] diff --git a/packages/econ/src/mostlyright/econ/__init__.py b/packages/econ/src/mostlyright/econ/__init__.py new file mode 100644 index 00000000..787f0d8e --- /dev/null +++ b/packages/econ/src/mostlyright/econ/__init__.py @@ -0,0 +1,56 @@ +"""``mostlyright.econ`` — economic-indicator data for prediction-market settlement. + +Macro releases — CPI (headline / core / YoY), PPI, nonfarm payrolls (NFP) and +revisions, U3 unemployment, initial jobless claims, GDP, and Fed funds decisions +— sourced from FRED/ALFRED + BLS/BEA/DOL/Federal Reserve and joined to Kalshi and +Polymarket econ markets for leakage-free settlement pairs. + +This vertical is deliberately ISOLATED from the four weather parity files +(``research()``, ``_internal/merge/observations.py``'s ``SOURCE_PRIORITY``, +``_internal/merge/climate.py``'s policies, ``live/_sources.py``) — the same +firewall discipline CWOP uses. Econ carries its OWN ``schema.econ.observations.v1``, +its OWN per-release cache namespace (``~/.mostlyright/cache/v1/econ/…``), and its +OWN error taxonomy (``IndicatorNotYetReleasedError``). A macro release must never +be routed through the weather settlement join, and the weather merge layer must +never see an econ row. + +Load-bearing invariant (first-print / ALFRED vintage discipline): Kalshi +settlements contractually EXCLUDE post-expiration revisions, so the backtest layer +serves as-released vintages, never the revised series. Vintage selection happens +at read time via the ``vintages`` keyword. + +The surface conforms to the cross-vertical **source-identity contract** +(``docs/source-identity.md``, Phase 30/31 v1.13/1.14): ``source=`` is always +provenance (WHO produced the row), ``delivery=`` is always where the computation +runs (``"live"`` local | ``"hosted"`` reserved seam). Both validate loudly BEFORE +any network call. + +Public surface: + +- :func:`series` — observation rows for an indicator across a date range + (canonical read; ``source=``/``delivery=`` contract kwargs). +- :func:`snapshot` — the settlement-target state as-of a cutoff (the latest + settlement-grade vintage knowable at ``as_of``). +- :func:`releases` — the release calendar / schedule for an indicator. +- :func:`research_econ` — leakage-free settlement pairs (market outcome + + first-print value + as-of features) for a series or contract. +- :func:`history` — deprecated alias of :func:`series` (byte-identical output; + no runtime warning this release, warning next minor, removal at 2.0). +""" + +from __future__ import annotations + +from ._errors import IndicatorNotYetReleasedError +from ._history import history, series +from ._releases import releases +from ._research import research_econ +from ._snapshot import snapshot + +__all__ = [ + "IndicatorNotYetReleasedError", + "history", + "releases", + "research_econ", + "series", + "snapshot", +] diff --git a/packages/econ/src/mostlyright/econ/_cache.py b/packages/econ/src/mostlyright/econ/_cache.py new file mode 100644 index 00000000..5233aea0 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_cache.py @@ -0,0 +1,589 @@ +"""Econ per-release persistence — monthly parquet cache for backtest replay. + +Path layout:: + + ~/.mostlyright/cache/v1/econ///.parquet + +Honors ``MOSTLYRIGHT_CACHE_DIR`` (the shared SDK cache root) so relocating the +cache moves the econ partitions with everything else. + +**Deliberately a standalone island** (the CWOP ``_cache.py`` discipline at +package granularity). This module imports NONE of the parity-coupled weather +persistence layer — that layer pulls ``OBSERVATION_SCHEMA`` + the LST tz map +and applies a *current-LST-month-skip* designed for re-fetchable AWC/IEM/GHCNh +weather data. Re-using its atomic-write helper would couple econ persistence to +the parity firewall's invalidation semantics; the ~20-line atomic writer here +is duplicated on purpose so econ stays on its own island (see CLAUDE.md "Data + +parity rules" and the CWOP adapter precedent). Econ is also NEVER registered in +the four weather parity files (``research()``, ``merge/observations.py``, +``merge/climate.py``, ``live/_sources.py``). + +**The load-bearing difference from BOTH weather and CWOP:** + +- Weather collapses to a single row per ``(station, date)`` via a + ``report_type_priority`` dedup. Econ does the OPPOSITE — it KEEPS every + vintage. The dedup key is the VINTAGE key ``(indicator, period, + vintage_date, source)``, NOT ``(indicator, period)``, so two rows sharing an + observation period but carrying different ``vintage_date`` values BOTH + persist. Read-time filtering on ``settlement_grade`` (``econ.history`` with + ``vintages="settlement"|"all"``) selects a vintage — never a merge collapse. + Collapsing vintages would destroy the first-print settlement store that a + Kalshi NHIGH/NLOW-style econ backtest depends on. +- CWOP has NO current-month skip because its APRS-IS stream is ephemeral + (persisting the current month is the only chance to retain it). Econ has NO + write-skip *either*, but for a different reason: econ CAN re-fetch, and it + keeps all vintages, so the per-indicator revision window (CPI ~45d, GDP ~60d + advance→second→third) governs which vintage is ``settlement_grade=True`` AT + READ TIME — it does not govern whether to write. There is therefore no + current-month / revision-window write-skip in this module. + +Concurrency: a write is a read-modify-write *merge* into an existing monthly +partition, serialized under a single per-path ``FileLock`` so two writers +targeting the same ``(indicator, year, month)`` cannot lost-update each other — +the second writer reads the first's already-committed rows and dedups on top. + +Source identity: persisted rows are stamped ``source="econ.cache"`` (distinct +from the live agency tags ``alfred`` / ``bls.v1`` / ``bea`` / …) so a consumer +reading them back via the econ history surface knows the provenance is a +cache-read, not a fresh agency pull. Econ data still never enters the +parity-frozen weather merge/research path. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from datetime import UTC, date, datetime +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +from filelock import FileLock +from mostlyright._internal._bounds import assert_path_under +from mostlyright._internal._cache_dir import resolve_cache_root_without_v1 + +from ._schema import ECON_CACHE_SOURCE, FRED_DERIVED_SOURCES, EconObservationsSchema + +logger = logging.getLogger(__name__) + +# Map the schema.econ.observations.v1 column dtype vocabulary to pyarrow field +# types so the persisted parquet schema is built EXPLICITLY rather than inferred +# from the first row by ``pa.Table.from_pylist``. First-row inference silently +# drops any column the first row happens to omit (e.g. a vintage row with no +# ``series_id`` followed by one that has it), losing columns on heterogeneous +# input. The econ schema uses ``bool`` (``settlement_grade``) which the CWOP +# map lacks, so the map is extended here; ``date`` is included defensively for +# forward compatibility even though the current econ schema declares none. +_PA_TYPE_BY_DTYPE: dict[str, pa.DataType] = { + "string": pa.string(), + "enum": pa.string(), + "timestamp_utc": pa.timestamp("us", tz="UTC"), + "float64": pa.float64(), + "int64": pa.int64(), + "bool": pa.bool_(), + "date": pa.date32(), +} + + +def _build_econ_pa_schema() -> pa.Schema: + """Build the explicit pyarrow schema for the schema.econ.observations.v1 set. + + Every declared column is emitted as a NULLABLE pyarrow field so the on-disk + partition tolerates a vintage row that omits an optional column (e.g. + ``series_id`` / ``units`` / ``release_datetime`` / ``retrieved_at``); + ``from_pylist(schema=...)`` then preserves every column regardless of + per-row key presence. The schema's own non-nullable checks (enforced by the + validator upstream in ``build_econ_dataframe``) remain authoritative for + presence — here we only need every column to EXIST on disk. + """ + return pa.schema( + [ + pa.field(col.name, _PA_TYPE_BY_DTYPE[col.dtype], nullable=True) + for col in EconObservationsSchema.COLUMNS + ] + ) + + +#: The persisted-partition pyarrow schema, derived once from schema.econ.observations.v1. +_ECON_PA_SCHEMA: pa.Schema = _build_econ_pa_schema() +#: The full ordered key set every persisted row is normalized to. +_ECON_COLUMN_NAMES: tuple[str, ...] = tuple(c.name for c in EconObservationsSchema.COLUMNS) + +#: Timestamp columns coerced to tz-aware UTC before the dedup key is computed so +#: a naive and an aware spelling of the same instant collapse, and the on-disk +#: pyarrow column is uniformly ``timestamp[us, UTC]``. +_ECON_TIMESTAMP_COLS: tuple[str, ...] = ( + "release_datetime", + "vintage_date", + "knowledge_time", + "retrieved_at", +) + +#: FileLock timeout in seconds (mirrors the observation/CWOP cache tiers). +LOCK_TIMEOUT_SECONDS = 30 + +# Econ indicator ids are lowercase snake tokens ("cpi", "cpi_core", "nfp", +# "jobless_claims", "fed_funds", …) — NOT ICAO codes, so ``STATION_CODE_RE`` is +# the wrong validator. The path-safe subset: start with a lowercase letter, +# then lowercase alnum or underscore, 2-32 chars total. This forbids ``/``, +# ``\``, ``.`` and ``..`` (and uppercase / leading digits) so the indicator can +# never escape its single path segment. ``assert_path_under`` is the final +# defense-in-depth backstop. +_ECON_KEY_RE = re.compile(r"\A[a-z][a-z0-9_]{1,31}\Z") + + +def _validate_econ_key(indicator: object) -> str: + """Normalize + path-validate an econ indicator id, returning the lower form. + + Raises: + TypeError: ``indicator`` is not a ``str``. + ValueError: ``indicator`` does not match the path-safe econ id pattern. + """ + if not isinstance(indicator, str): + raise TypeError(f"indicator must be a str, got {type(indicator).__name__}") + normalized = indicator.strip().lower() + if not _ECON_KEY_RE.match(normalized): + raise ValueError( + f"indicator={indicator!r} is not a path-safe econ id " + f"(expected {_ECON_KEY_RE.pattern}, e.g. 'cpi' / 'jobless_claims')" + ) + return normalized + + +def _econ_cache_root() -> Path: + """Return ``/v1/econ`` (honors ``MOSTLYRIGHT_CACHE_DIR``). + + Resolved on each call so tests can monkeypatch the env var between cases. + ``resolve_cache_root_without_v1`` returns the root WITHOUT ``/v1``; econ + appends ``v1/econ`` so the partition path matches the CONTEXT layout + ``~/.mostlyright/cache/v1/econ/{indicator}/...``. + """ + return resolve_cache_root_without_v1() / "v1" / "econ" + + +def econ_cache_path(indicator: str, year: int, month: int) -> Path: + """Return the parquet partition path for ``(indicator, year, month)``. + + Example:: + + econ_cache_path("cpi", 2026, 6) + # -> ~/.mostlyright/cache/v1/econ/cpi/2026/06.parquet + + Validates ``indicator`` against the econ id pattern and asserts the resolved + path stays under the econ cache root (path-traversal backstop). + """ + safe = _validate_econ_key(indicator) + root = _econ_cache_root() + raw = root / safe / f"{year:04d}" / f"{month:02d}.parquet" + assert_path_under(raw, root, field="econ_cache_path") + return raw + + +# --------------------------------------------------------------------------- +# Dedup + atomic write +# --------------------------------------------------------------------------- +def _ensure_utc(value: object) -> object: + """Coerce a ``datetime`` cell to tz-aware UTC; pass anything else through. + + Naive datetimes are assumed UTC; aware datetimes in another zone are + converted to UTC. Applied to every econ timestamp column at the write + chokepoint so the dedup key is stable (a naive and an aware spelling of the + same ``vintage_date`` must collapse) AND the on-disk pyarrow column is + uniformly ``timestamp[us, UTC]`` rather than a tz-stripped mix. + """ + if isinstance(value, datetime): + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + return value + + +def _normalize_row_timestamps(row: dict) -> dict: + """Return ``row`` with every econ timestamp column normalized to UTC.""" + normalized = dict(row) + for col in _ECON_TIMESTAMP_COLS: + if col in normalized: + normalized[col] = _ensure_utc(normalized[col]) + return normalized + + +def _dedup_econ_rows(rows: list[dict]) -> list[dict]: + """First-seen-wins dedup on the VINTAGE key. + + The key is ``(indicator, period, vintage_date, source)`` — NOT + ``(indicator, period)``. This is the load-bearing econ difference from + weather's single-row collapse: two rows sharing an observation period but + carrying different ``vintage_date`` values are DISTINCT vintages and BOTH + survive (keep-all-vintages). Only a true duplicate — same indicator, same + period, same ``vintage_date``, same ``source`` — collapses, and the + first-seen row wins (later duplicates are dropped). Deterministic and + order-preserving. + """ + seen: set[tuple[object, object, object, object]] = set() + out: list[dict] = [] + for row in rows: + key = ( + row.get("indicator"), + row.get("period"), + row.get("vintage_date"), + row.get("source"), + ) + if key in seen: + continue + seen.add(key) + out.append(row) + return out + + +def _write_table_atomic(path: Path, table: pa.Table) -> None: + """Write ``table`` to ``path`` via ``.tmp`` + ``os.replace``. + + Caller MUST already hold ``FileLock(str(path) + '.lock')`` — this is the + inner write half of the read-modify-write merge in :func:`write_econ_cache`. + ``os.replace`` is atomic on POSIX and Windows, so a concurrent reader sees + either the old partition or the new one, never a truncated parquet. Parquet + ``version="2.6"`` + ``coerce_timestamps="us"`` keep the bytes stable across + pyarrow versions and preserve microsecond timestamps on roundtrip. + """ + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + pq.write_table(table, tmp, version="2.6", coerce_timestamps="us") + os.replace(tmp, path) + + +def write_econ_cache(indicator: str, year: int, month: int, rows: list[dict]) -> int: + """Merge ``rows`` into the ``(indicator, year, month)`` partition. Idempotent. + + Stamps ``source="econ.cache"`` on every persisted row (the persistence + provenance tag) and the normalized (lower) indicator, normalizes every + timestamp column to UTC-aware, then dedups on the VINTAGE key + ``(indicator, period, vintage_date, source)`` against any rows already on + disk. Because the key includes ``vintage_date``, ALL vintages are kept — the + cache never collapses to one row per period. + + There is NO current-month / revision-window write-skip: the per-indicator + revision window governs which vintage is ``settlement_grade=True`` at READ + time, not whether to write. + + Returns the total number of rows in the partition after the merge. No-op + (returns the existing count, or 0) when ``rows`` is empty. + + The entire read → concat → dedup → write runs under ONE ``FileLock`` so + concurrent same-partition writers cannot lost-update. + """ + # D-LIC-2: FRED's ToU prohibits storing/caching/archiving FRED content + # (Prohibitions (q); API (l) — .planning/research/DATA-LICENSING.md). Drop + # FRED-derived rows (alfred; dol.icsa = FRED-transported) unless the user opts + # in. Per-ROW + BEFORE the econ.cache re-stamp (which overwrites `source`), so + # agency rows (bls/bea/fed — public domain) in the same call still persist. + if rows and os.environ.get("MOSTLYRIGHT_PERSIST_FRED", "").strip() != "1": + fred_rows = [r for r in rows if r.get("source") in FRED_DERIVED_SOURCES] + if fred_rows: + rows = [r for r in rows if r.get("source") not in FRED_DERIVED_SOURCES] + logger.info( + "econ cache: skipped %d FRED-derived row(s) (source in %s) — FRED's " + "ToU prohibits store/cache/archive of FRED content; these are " + "fetch-through only. Set MOSTLYRIGHT_PERSIST_FRED=1 to persist them " + "under your own acceptance of the FRED ToU. See " + "docs/econ-vertical.md#4-keyless-vs-keyed-behavior.", + len(fred_rows), + sorted(FRED_DERIVED_SOURCES), + ) + + safe_indicator = _validate_econ_key(indicator) + path = econ_cache_path(indicator, year, month) + lock = FileLock(str(path) + ".lock", timeout=LOCK_TIMEOUT_SECONDS) + with lock: + existing: list[dict] = [] + if path.exists(): + try: + existing = pq.read_table(path).to_pylist() + except (FileNotFoundError, OSError): + existing = [] + if not rows and not existing: + return 0 + # Stamp cache-source provenance AND the normalized (lower) indicator AND + # normalize timestamps to UTC-aware BEFORE dedup. Stamping the + # partition's normalized indicator onto every row is authoritative (like + # ``source``): a row persisted via a mixed-case id lands in the same + # lower-case partition, so its on-disk ``indicator`` and the dedup key + # must match the partition or casing variants would store as distinct + # rows and re-surface as duplicates on read. + stamped = [ + _normalize_row_timestamps( + {**row, "source": ECON_CACHE_SOURCE, "indicator": safe_indicator} + ) + for row in rows + ] + merged = _dedup_econ_rows(existing + stamped) + if not merged: + return 0 + # Normalize every row to the full schema key set and build against an + # EXPLICIT schema so a column absent from the first row (but present in a + # later one) is preserved — ``from_pylist`` without ``schema=`` infers + # from the first row and silently drops such columns. Keys outside the + # schema (a caller's extra field) are not part of + # schema.econ.observations.v1 and are intentionally excluded. + normalized = [{name: row.get(name) for name in _ECON_COLUMN_NAMES} for row in merged] + table = pa.Table.from_pylist(normalized, schema=_ECON_PA_SCHEMA) + _write_table_atomic(path, table) + return len(merged) + + +def read_econ_cache(indicator: str, year: int, month: int) -> list[dict] | None: + """Return persisted rows for ``(indicator, year, month)`` or ``None`` on miss. + + ``None`` when the partition file does not exist (or is unlinked between the + ``exists()`` check and the read — treated as a miss, mirroring the + observation/CWOP cache TOCTOU handling). + """ + path = econ_cache_path(indicator, year, month) + if not path.exists(): + return None + try: + return pq.read_table(path).to_pylist() + except (FileNotFoundError, OSError): + return None + + +def invalidate_econ(indicator: str, year: int, month: int) -> bool: + """Remove the ``(indicator, year, month)`` partition; return whether removed. + + Acquires the same ``FileLock`` as :func:`write_econ_cache` so an + invalidation racing a write runs strictly before or after it. + """ + path = econ_cache_path(indicator, year, month) + lock = FileLock(str(path) + ".lock", timeout=LOCK_TIMEOUT_SECONDS) + with lock: + removed = path.exists() + if removed: + path.unlink() + # H7 (codex round 2): the coverage ledger gates re-fetch, so invalidating a + # partition WITHOUT clearing coverage would leave the window "covered" and never + # re-fetched. Clear the whole indicator's coverage (coarse but correct — forces + # a re-fetch); it's the only escape hatch from a stale/mis-marked ledger. + clear_coverage(indicator) + return removed + + +# --------------------------------------------------------------------------- +# Window read +# --------------------------------------------------------------------------- +def _as_utc_datetime(value: date | datetime, *, end_of_day: bool) -> datetime: + """Coerce a ``date``/``datetime`` to a tz-aware UTC ``datetime``. + + A bare ``date`` becomes midnight UTC (``end_of_day=False``) or the last + microsecond of that day (``end_of_day=True``) so an inclusive + ``[from_date, to_date]`` day range captures the whole final day. A naive + ``datetime`` is assumed UTC; a tz-aware one is converted to UTC. + """ + if isinstance(value, datetime): + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + # date (but not datetime — datetime is a date subclass, handled above) + if end_of_day: + return datetime(value.year, value.month, value.day, 23, 59, 59, 999999, tzinfo=UTC) + return datetime(value.year, value.month, value.day, tzinfo=UTC) + + +def _iter_year_months(start: datetime, end: datetime) -> list[tuple[int, int]]: + """Yield every ``(year, month)`` partition overlapping ``[start, end]``.""" + if end < start: + return [] + months: list[tuple[int, int]] = [] + y, m = start.year, start.month + while (y, m) <= (end.year, end.month): + months.append((y, m)) + if m == 12: + y, m = y + 1, 1 + else: + m += 1 + return months + + +def read_econ_window( + indicator: str, + from_date: date | datetime, + to_date: date | datetime, +) -> list[dict]: + """Read all persisted rows for ``indicator`` across ``[from_date, to_date]``. + + Inclusive on both ends. Iterates the monthly ``(year, month)`` partitions + overlapping the range and returns every persisted vintage in those + partitions. Unlike CWOP (whose per-row ``observed_at`` is a datetime that + can be filtered inside a partition), an econ ``period`` is a string + (``"2026-05"`` / ``"2026Q2"`` / a weekly date / an FOMC date) whose meaning + is indicator-specific, so the partition grain IS the window resolution here + — the read returns all vintages persisted to the overlapping partitions, and + read-time ``settlement_grade`` / as-of filtering happens downstream + (``econ.history`` / ``research_econ``). Rows are returned sorted by + ``vintage_date`` (then ``period`` for stability). + + Returns an empty list when nothing is persisted in range. + """ + start = _as_utc_datetime(from_date, end_of_day=False) + end = _as_utc_datetime(to_date, end_of_day=True) + out: list[dict] = [] + for year, month in _iter_year_months(start, end): + rows = read_econ_cache(indicator, year, month) + if not rows: + continue + out.extend(rows) + # Per-ROW vintage_date filter: the partition grain is MONTHLY, so a partition + # can hold vintages OUTSIDE [start, end] — a weekly series (jobless_claims) or + # FOMC dates in the same month as ``to_date`` but after it. Returning those is a + # leakage-adjacent over-read (a call without ``as_of`` would hand back vintages + # past the requested cutoff). Filter to the window, mirroring snapshot()'s + # explicit ``vintage_date <= as_of`` guard. + out = [r for r in out if _vintage_in_range(r.get("vintage_date"), start, end)] + out.sort(key=lambda r: (_sort_key(r.get("vintage_date")), str(r.get("period")))) + return out + + +def _coverage_path(indicator: str) -> Path: + """Sidecar recording which request windows have been FETCHED+PERSISTED.""" + safe = _validate_econ_key(indicator) + root = _econ_cache_root() + raw = root / safe / ".coverage.json" + assert_path_under(raw, root, field="coverage_path") + return raw + + +def _read_intervals(indicator: str) -> list[tuple[str, str, bool]]: + """Read the coverage ledger as ``(from_iso, to_iso, settlement)`` tuples. + + Legacy 2-tuples (pre-grade-aware) are migrated as ``settlement=False``. + """ + path = _coverage_path(indicator) + if not path.exists(): + return [] + try: + data = json.loads(path.read_text()) + except (ValueError, OSError): + return [] + out: list[tuple[str, str, bool]] = [] + for item in data: + if not isinstance(item, (list, tuple)) or len(item) < 2: + continue + a, b = item[0], item[1] + settlement = bool(item[2]) if len(item) >= 3 else False + if isinstance(a, str) and isinstance(b, str): + out.append((a, b, settlement)) + return out + + +def _merge_same_flag(intervals: list[tuple[str, str]]) -> list[tuple[str, str]]: + if not intervals: + return [] + ordered = sorted(intervals) + merged = [ordered[0]] + for start, end in ordered[1:]: + last_start, last_end = merged[-1] + if start <= last_end: # overlap/adjacent (ISO strings compare chronologically) + merged[-1] = (last_start, max(last_end, end)) + else: + merged.append((start, end)) + return merged + + +def is_window_covered( + indicator: str, + from_date: date | datetime, + to_date: date | datetime, + *, + need_settlement: bool, +) -> bool: + """True when ``[from_date, to_date]`` was already fetched+persisted (H1 guard). + + Gating the fetch on "cache empty" would silently drop the uncovered tail of a + partially-cached superset window (Jan-Mar cached, caller asks Jan-Jun). The + ledger records the ACTUAL persisted range (capped at the data obtained — a + window extending past the newest release stays uncovered so future releases get + fetched, codex round-2 C1). It is GRADE-AWARE: a window filled by a keyless + (``settlement=False``) fetch does NOT satisfy a later ``vintages="settlement"`` + request — the keyed ALFRED path must run (codex round-2 C2). So a settlement + request only counts ``settlement=True`` intervals. + """ + start = _as_utc_datetime(from_date, end_of_day=False).date().isoformat() + end = _as_utc_datetime(to_date, end_of_day=False).date().isoformat() + relevant = [ + (a, b) + for a, b, settlement in _read_intervals(indicator) + if settlement or not need_settlement + ] + return any(a <= start and end <= b for a, b in _merge_same_flag(relevant)) + + +def mark_window_covered( + indicator: str, + from_date: date | datetime, + to_date: date | datetime, + *, + settlement: bool, +) -> None: + """Record a fetched+persisted window (filelock-guarded, grade-tagged). + + ``settlement`` records whether the persisted rows include the settlement-grade + first print, so a later settlement read is only short-circuited by a + settlement-capable fetch. Callers cap ``to_date`` at the newest vintage actually + obtained so a future-extending window is never falsely frozen. + """ + start = _as_utc_datetime(from_date, end_of_day=False).date().isoformat() + end = _as_utc_datetime(to_date, end_of_day=False).date().isoformat() + if end < start: # nothing obtained past from_date — record nothing + return + path = _coverage_path(indicator) + path.parent.mkdir(parents=True, exist_ok=True) + lock = FileLock(str(path) + ".lock", timeout=LOCK_TIMEOUT_SECONDS) + with lock: + existing = _read_intervals(indicator) + # Merge within each settlement flag independently. + by_flag: dict[bool, list[tuple[str, str]]] = {True: [], False: []} + for a, b, flag in [*existing, (start, end, settlement)]: + by_flag[flag].append((a, b)) + result: list[list[object]] = [] + for flag in (True, False): + for a, b in _merge_same_flag(by_flag[flag]): + result.append([a, b, flag]) + path.write_text(json.dumps(result)) + + +def clear_coverage(indicator: str) -> None: + """Delete the coverage ledger for ``indicator`` (invalidate escape hatch, H7).""" + path = _coverage_path(indicator) + lock = FileLock(str(path) + ".lock", timeout=LOCK_TIMEOUT_SECONDS) + with lock: + if path.exists(): + path.unlink() + + +def _vintage_in_range(value: object, start: datetime, end: datetime) -> bool: + """True when ``value`` (a row's ``vintage_date``) falls in ``[start, end]``. + + A missing/unparseable ``vintage_date`` is excluded (never silently in-window). + """ + if value is None: + return False + try: + vd = _as_utc_datetime(value, end_of_day=False) + except (TypeError, ValueError, AttributeError): + return False + return start <= vd <= end + + +def _sort_key(value: object) -> datetime: + """Total-order sort key for ``vintage_date`` (nulls sort first).""" + if isinstance(value, datetime): + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return datetime.min.replace(tzinfo=UTC) + + +__all__ = [ + "ECON_CACHE_SOURCE", + "econ_cache_path", + "invalidate_econ", + "read_econ_cache", + "read_econ_window", + "write_econ_cache", +] diff --git a/packages/econ/src/mostlyright/econ/_errors.py b/packages/econ/src/mostlyright/econ/_errors.py new file mode 100644 index 00000000..9db766a7 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_errors.py @@ -0,0 +1,24 @@ +"""Econ error taxonomy — re-exported from the centralized core hierarchy. + +The econ vertical defines no error classes of its own. Its one dedicated +error, :class:`~mostlyright.core.exceptions.IndicatorNotYetReleasedError`, lives +in ``mostlyright.core.exceptions`` (the same placement as ``EarningsError`` / +``NoCWOPDataError``) so the MCP JSON-RPC serialization and the Python↔TS +lockstep discipline apply uniformly across every vertical. This module simply +re-exports it under the ``mostlyright.econ`` namespace so callers using the +econ surface can ``from mostlyright.econ import IndicatorNotYetReleasedError`` +without reaching into core — mirroring how ``mostlyright.weather.cwop`` +re-exports ``NoCWOPDataError``. + +The re-exported object is the SAME class (identity, not a copy), so +``except IndicatorNotYetReleasedError`` behaves identically whether the caller +imported it from core or from econ. +""" + +from __future__ import annotations + +from mostlyright.core.exceptions import IndicatorNotYetReleasedError + +__all__ = [ + "IndicatorNotYetReleasedError", +] diff --git a/packages/econ/src/mostlyright/econ/_fetchers/__init__.py b/packages/econ/src/mostlyright/econ/_fetchers/__init__.py new file mode 100644 index 00000000..5d68d2ca --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_fetchers/__init__.py @@ -0,0 +1,37 @@ +"""``mostlyright.econ._fetchers`` — in-house httpx fetchers, one per agency path. + +Each fetcher pulls a live agency JSON endpoint and returns rows shaped for +:func:`mostlyright.econ._schema.build_econ_dataframe` (``schema.econ.observations.v1`` +columns, vintage-first-class). The house rule is **in-house httpx only** — no +``fredapi`` / ``bls`` / ``pandas-datareader`` runtime dependency. Where an +external library encodes a hard-won algorithm (the ALFRED first-print extractor +in ``fredapi``, Apache-2.0), the *algorithm* is lifted; the *package* is not +imported (RESEARCH library-reuse verdict — "adopt NONE"). + +Two vintage-critical fetchers land in plan 29-05: + +- :mod:`~mostlyright.econ._fetchers.fred_alfred` — the canonical FIRST-PRINT + vintage store via ALFRED ``realtime_start``/``realtime_end`` (``FRED_API_KEY`` + opt-in; keyless degrades cleanly). ALFRED is *necessary*, not merely + convenient: BLS serves latest-revised only, so ALFRED carries the + ``settlement_grade=True`` truth (subject to the day-granularity caveat). +- :mod:`~mostlyright.econ._fetchers.bls` — BLS API v1 (keyless default) + v2 + (``BLS_API_KEY``). Latest-revised provenance (``current``/``revised``), never + mislabeled as first-print. Covers CPI, Core CPI, NFP, U3, AND PPI final-demand + (``WPSFD4``/``WPUFD4``). + +29-06 adds the BEA / DOL / Fed fetchers into this same package. + +Firewall: these fetchers stamp only :data:`mostlyright.econ._schema.ECON_SOURCES` +tags and are NEVER registered in the four weather parity files +(``research()``, ``_internal/merge/observations.py``'s ``SOURCE_PRIORITY``, +``_internal/merge/climate.py``'s policies, ``live/_sources.py``). + +Submodules are imported explicitly by callers +(``from mostlyright.econ._fetchers.fred_alfred import fetch_vintages``) — this +package ``__init__`` stays a light marker with NO eager submodule import, mirroring +``mostlyright.weather._fetchers`` (so importing one fetcher never drags in the +others, and a partially-landed plan never breaks collection). +""" + +from __future__ import annotations diff --git a/packages/econ/src/mostlyright/econ/_fetchers/bea.py b/packages/econ/src/mostlyright/econ/_fetchers/bea.py new file mode 100644 index 00000000..b3e08aa3 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_fetchers/bea.py @@ -0,0 +1,477 @@ +"""BEA NIPA GDP fetcher — quarterly real-GDP % change with advance/second/third. + +The Bureau of Economic Analysis serves the National Income and Product Accounts +(NIPA) through ``apps.bea.gov/api/data`` (``method=GetData&datasetname=NIPA``). +Table **T10101** ("Percent Change From Preceding Period in Real Gross Domestic +Product") line 1 is real GDP % change, seasonally-adjusted at annual rate (SAAR) +— the number Kalshi's KXGDP settles to. + +**The three-estimate cadence (the load-bearing provenance).** BEA publishes each +quarter's GDP in THREE estimates ~1 month apart: the *advance* estimate (the FIRST +PRINT, ~end of the month after the quarter), then the *second* and *third* +estimates as more source data arrives. Per the Kalshi contract discipline +(GDP.pdf), the settlement value is the FIRST print — so the advance estimate +carries ``settlement_grade=True`` and the second/third revisions carry +``settlement_grade=False``. This mirrors the ALFRED first-release rule +(:mod:`~mostlyright.econ._fetchers.fred_alfred`) applied to BEA's own release +cadence rather than ALFRED realtime vintages. + +**A5 — estimate-type identification (documented inference).** BEA's public NIPA +``GetData`` response does not carry an unambiguous advance/second/third field on +each datum (the exact field was not keyed-verified — RESEARCH A5). This fetcher +therefore accepts an explicit ``release_type`` from the caller (who knows which +release it fetched) and, when none is given, INFERS it from the release date +relative to the BEA schedule via :func:`infer_release_type` (advance ≈ quarter-end ++1mo, second ≈ +2mo, third ≈ +3mo). The inference is documented and spot-verified +against a live key in the 29-10 smoke — it is NOT silently authoritative. + +Security: ``BEA_API_KEY`` is read from the environment (or passed explicitly), is +placed only in the outbound query string, and is NEVER logged, persisted, or +embedded in an emitted row (threat T-29-17). HTTPS is enforced — a bad-key +structured JSON error object is REJECTED (raises), never parsed as GDP data +(threat T-29-18). A hostile / HTML-instead-of-JSON body likewise raises. + +House rule: in-house httpx only. No ``beaapi`` / ``pybea`` third-party client is a +dependency — this module talks to BEA over httpx directly, reusing the repo's +established retry constants. +""" + +from __future__ import annotations + +import logging +import os +import time +from collections.abc import Mapping +from datetime import UTC, datetime +from typing import Any + +import httpx +from mostlyright._internal._http import ( + BASE_DELAY, + HTTP_TIMEOUT, + MAX_RETRIES, + TRANSIENT_CODES, +) +from mostlyright.core.exceptions import DataAvailabilityError, SourceUnavailableError +from mostlyright.econ._schema import ECON_SOURCE_BEA + +log = logging.getLogger(__name__) + +#: BEA GetData endpoint (HTTPS only). ``apps.bea.gov`` serves the NIPA tables; the +#: key travels in the query string, so the scheme is pinned and an HTTP downgrade +#: is rejected before the request is issued. +BEA_DATA_URL = "https://apps.bea.gov/api/data" + +#: The NIPA table + line the GDP % change (SAAR) lives on. T10101 line 1 is +#: "Gross domestic product" — the KXGDP settlement series. Component lines (PCE, +#: investment, …) share the table but are NOT the headline GDP number. +BEA_GDP_TABLE = "T10101" +BEA_GDP_LINE = "1" + +#: The canonical GDP indicator id + units (schema ``indicator`` / ``units``). +#: T10101 is a percent-change-SAAR table, so units are ``percent_saar`` (NOT an +#: index level). +BEA_GDP_INDICATOR = "gdp" +BEA_GDP_UNITS = "percent_saar" + +#: The three BEA GDP estimates in release order. The FIRST (``advance``) is the +#: first print → ``settlement_grade=True``; the later two are revisions → False. +BEA_ESTIMATE_ORDER: tuple[str, ...] = ("advance", "second", "third") + +#: ALFRED/BEA sentinels for "no value this period". +_MISSING_VALUES = frozenset({".", "", None, "n.a.", "NaN"}) + + +def _resolve_key(key: str | None) -> str | None: + """Return the explicit ``key`` or fall back to ``BEA_API_KEY`` in env. + + Never logs the key. ``None`` when neither is set (the fetcher then degrades + via :func:`fetch_gdp`'s keyless branch — a documented signal, not a crash). + """ + if key: + return key + env = os.environ.get("BEA_API_KEY") + return env or None + + +def _quarter_end_month(period: str) -> int: + """Return the calendar month (1-12) the quarter in ``period`` ends on. + + ``"2026Q1"`` → 3, ``Q2`` → 6, ``Q3`` → 9, ``Q4`` → 12. Used by + :func:`infer_release_type` to place a release date relative to quarter-end. + """ + q = int(period.split("Q")[1]) + return q * 3 + + +def infer_release_type(period: str, release_date: str) -> str: + """Infer advance/second/third from a release date vs the BEA schedule (A5). + + BEA publishes each quarter's GDP roughly one month apart: the *advance* + estimate near the end of the month AFTER the quarter (Q2 ends Jun → advance + late Jul), then *second* (+1mo, late Aug) and *third* (+2mo, late Sep). We map + the number of whole months between quarter-end and the release month onto the + estimate order: + + - 1 month after quarter-end → ``"advance"`` (the first print) + - 2 months → ``"second"`` + - 3+ months → ``"third"`` + + This is the documented A5 inference — BEA's ``GetData`` does not expose an + unambiguous estimate field, so the release date is the signal. A caller who + knows which release it fetched should pass ``release_type=`` explicitly rather + than rely on this. + + Args: + period: the observation quarter (``"2026Q2"``). + release_date: the estimate's release date (``"YYYY-MM-DD"``). + + Returns: + One of :data:`BEA_ESTIMATE_ORDER`. + """ + end_month = _quarter_end_month(period) + end_year = int(period.split("Q")[0]) + rel = datetime.strptime(release_date, "%Y-%m-%d") + months_after = (rel.year - end_year) * 12 + (rel.month - end_month) + if months_after <= 1: + return "advance" + if months_after == 2: + return "second" + return "third" + + +def _parse_release_datetime(release_date: str | None) -> datetime | None: + """Parse a ``YYYY-MM-DD`` release date into tz-aware UTC midnight, or ``None``. + + BEA release dates are calendar days (the intraday 8:30 ET release time is not + carried in the NIPA data payload), so we anchor at 00:00:00 UTC — the same + day-granular convention ALFRED uses. + """ + if not release_date: + return None + return datetime.strptime(release_date, "%Y-%m-%d").replace(tzinfo=UTC) + + +def _raise_if_bea_error(payload: Mapping[str, Any]) -> None: + """Raise :class:`SourceUnavailableError` if ``payload`` is a BEA error object. + + BEA returns HTTP 200 with a ``BEAAPI.Error`` (or ``BEAAPI.Results.Error``) + object for a bad key / bad request — it must be REJECTED, never parsed as GDP + data (threat T-29-18). The key is NOT included in the raised message. + """ + beaapi = payload.get("BEAAPI") + if not isinstance(beaapi, Mapping): + raise SourceUnavailableError( + "BEA response is missing the BEAAPI envelope", + source=ECON_SOURCE_BEA, + url=BEA_DATA_URL, + underlying=f"top-level keys={sorted(payload)}", + ) + # A bad-key / bad-request error can appear as BEAAPI.Error or, on some paths, + # nested under BEAAPI.Results.Error. Reject either. + error = beaapi.get("Error") + results = beaapi.get("Results") + nested_error = results.get("Error") if isinstance(results, Mapping) else None + err_obj = error or nested_error + if err_obj: + # Surface only the BEA-provided description (never the key/request params). + desc = "" + if isinstance(err_obj, Mapping): + desc = str(err_obj.get("APIErrorDescription") or err_obj.get("ErrorDetail") or "") + elif isinstance(err_obj, list) and err_obj: + first = err_obj[0] + if isinstance(first, Mapping): + desc = str(first.get("APIErrorDescription") or "") + raise SourceUnavailableError( + "BEA returned a structured API error", + source=ECON_SOURCE_BEA, + url=BEA_DATA_URL, + underlying=desc, + ) + + +def parse( + payload: Mapping[str, Any], + *, + release_type: str | None = None, + vintage_date: str | None = None, + retrieved_at: datetime | None = None, +) -> list[dict[str, Any]]: + """Parse a BEA NIPA ``GetData`` payload into ``schema.econ.observations.v1`` rows. + + Walks ``BEAAPI.Results.Data[]``, selecting only the GDP line + (:data:`BEA_GDP_LINE` of :data:`BEA_GDP_TABLE`), decoding ``TimePeriod`` (BEA + already emits ``"2026Q2"``), coercing ``DataValue`` to float, and stamping the + estimate provenance: + + - ``release_type`` — the explicit estimate type when given, else inferred from + ``vintage_date`` via :func:`infer_release_type` (A5). Defaults to the neutral + non-first-print ``"revised"`` when NEITHER is available: a plain bulk + ``GetData`` fetch is BEA's latest-REVISED data, NOT the advance first print, + so an un-hinted row must not be labeled ``"advance"`` (codex round-2 C4). + - ``settlement_grade`` — computed off the hint BEFORE the label defaults: + ``True`` ONLY for an explicit/inferable ``"advance"`` (the first print); + ``False`` otherwise — including the un-hinted default (never fabricate the + grade for latest-revised data). An explicit ``release_type="advance"`` opts + in. Mirrors the TS ``parseBea`` comment. + - ``vintage_date`` / ``knowledge_time`` — the estimate's release date (day + granular), or ``retrieved_at`` when the caller did not supply one. + + Raises: + SourceUnavailableError: when ``payload`` is a BEA error object or is not a + well-formed ``BEAAPI.Results.Data`` payload (a hostile / HTML body is + rejected here, NOT parsed as empty rows — threat T-29-18 / Pitfall 4). + """ + if retrieved_at is None: + retrieved_at = datetime.now(UTC) + + if not isinstance(payload, Mapping): + raise SourceUnavailableError( + "BEA response is not a JSON object", + source=ECON_SOURCE_BEA, + url=BEA_DATA_URL, + underlying=f"payload type={type(payload).__name__}", + ) + _raise_if_bea_error(payload) + + results = payload["BEAAPI"].get("Results") + if not isinstance(results, Mapping) or "Data" not in results: + raise SourceUnavailableError( + "BEA response missing BEAAPI.Results.Data", + source=ECON_SOURCE_BEA, + url=BEA_DATA_URL, + ) + data = results["Data"] + if not isinstance(data, list): + raise SourceUnavailableError( + "BEA BEAAPI.Results.Data is not a list", + source=ECON_SOURCE_BEA, + url=BEA_DATA_URL, + ) + + vintage_dt = _parse_release_datetime(vintage_date) or retrieved_at + + rows: list[dict[str, Any]] = [] + for datum in data: + if not isinstance(datum, Mapping): + continue + # Select ONLY the GDP line of T10101 — never a component (PCE, etc.). + table = str(datum.get("TableName") or "") + line = str(datum.get("LineNumber") or "") + if table != BEA_GDP_TABLE or line != BEA_GDP_LINE: + continue + time_period = datum.get("TimePeriod") + raw_value = datum.get("DataValue") + if time_period is None: + continue + # BEA formats large levels with thousands commas ("21,845.6"); GDP % change + # values do not, but strip commas defensively before float(). + value: float | None + if raw_value in _MISSING_VALUES: + value = None + else: + try: + value = float(str(raw_value).replace(",", "")) + except (TypeError, ValueError): + value = None + + resolved_type = release_type + if resolved_type is None and vintage_date is not None: + resolved_type = infer_release_type(str(time_period), vintage_date) + # settlement_grade requires an EXPLICIT/inferable advance hint. A plain + # bulk fetch with no hint is latest-revised data — NOT the first print — so + # it must NOT default to settlement-grade (codex M2: parse() is exported + + # independently callable; the keyless production caller re-stamps False, but + # this keeps the raw function safe). An explicit release_type="advance" + # opts in. + settlement_grade = resolved_type == "advance" + if resolved_type is None: + resolved_type = "revised" # neutral non-first-print label for the row + + rows.append( + { + "indicator": BEA_GDP_INDICATOR, + "series_id": datum.get("SeriesCode"), + "period": str(time_period), + "value": value, + "units": BEA_GDP_UNITS, + "release_datetime": None, # BEA data payload carries no wall-clock time. + "vintage_date": vintage_dt, + "knowledge_time": vintage_dt, # leakage cutoff == vintage_date. + "release_type": resolved_type, + "settlement_grade": settlement_grade, + "source": ECON_SOURCE_BEA, + "retrieved_at": retrieved_at, + } + ) + return rows + + +def _get_with_retry( + client: httpx.Client, + url: str, + params: Mapping[str, Any], +) -> httpx.Response: + """GET ``url`` with the shared retry discipline (transient codes backed off). + + Reuses ``mostlyright._internal._http``'s ``MAX_RETRIES`` / ``BASE_DELAY`` / + ``TRANSIENT_CODES`` (the repo's established retry constants). The caller owns + ``client``. The key rides in ``params`` (query string) — BEA requires it there + — so callers must never log the resolved URL. + """ + delay = BASE_DELAY + for attempt in range(MAX_RETRIES): + try: + response = client.get(url, params=dict(params)) + except httpx.RequestError as exc: + # SECURITY (codex round-2 H6): a transport error (timeout/connection) + # carries the key-bearing request URL on its .request attribute; raise a + # sanitized error with the bare endpoint (`from None` drops the httpx + # exception from the traceback chain). + raise SourceUnavailableError( + f"BEA request failed ({type(exc).__name__})", + source=ECON_SOURCE_BEA, + url=url, + ) from None + if response.status_code in TRANSIENT_CODES and attempt < MAX_RETRIES - 1: + log.warning( + "BEA HTTP %d (attempt %d/%d), retrying in %.1fs", + response.status_code, + attempt + 1, + MAX_RETRIES, + delay, + ) + time.sleep(delay) + delay *= 2 + continue + if response.status_code != 200: + # SECURITY: the request URL carries ``UserID=`` — httpx's + # ``raise_for_status()`` would embed the full URL (and key) in the + # exception message. Raise with the bare endpoint constant instead. + raise SourceUnavailableError( + f"BEA returned HTTP {response.status_code}", + source=ECON_SOURCE_BEA, + url=url, + http_status=response.status_code, + ) + return response + raise SourceUnavailableError( # pragma: no cover + "BEA retry loop exhausted without a response", + source=ECON_SOURCE_BEA, + url=url, + ) + + +def fetch_gdp( + *, + key: str | None = None, + year_range: tuple[int, int] | str = "ALL", + release_type: str | None = None, + vintage_date: str | None = None, + client: httpx.Client | None = None, +) -> list[dict[str, Any]]: + """Fetch BEA NIPA T10101 quarterly GDP % change and return schema-shaped rows. + + Args: + key: the BEA API key. Falls back to ``BEA_API_KEY`` in env; when neither + is present the call DEGRADES (raises a documented + :class:`DataAvailabilityError`, ``reason="source_404"``) rather than + hitting BEA blind (BEA requires a key). Never logged / persisted / + embedded in a row. + year_range: ``"ALL"`` (default) for the full quarterly history, or an + inclusive ``(start_year, end_year)`` tuple mapped to BEA's + comma-separated ``Year`` param. + release_type: the estimate type of THIS fetch (``"advance"`` / ``"second"`` + / ``"third"``) when the caller knows it. When ``None`` and + ``vintage_date`` is given, it is inferred via :func:`infer_release_type` + (A5); otherwise an un-hinted bulk fetch defaults to the neutral + non-first-print ``"revised"`` (BEA's plain ``GetData`` is latest-REVISED + data, NOT the advance first print — codex round-2 C4), so + ``settlement_grade`` stays ``False`` unless ``"advance"`` is explicit. + vintage_date: the estimate's release date (``"YYYY-MM-DD"``) — stamped as + ``vintage_date`` / ``knowledge_time`` and used to infer ``release_type`` + when that is not given. + client: an injected ``httpx.Client`` (tests / connection reuse). When + ``None`` a fresh HTTPS client is created and closed per call. + + Returns: + Rows shaped for :func:`mostlyright.econ._schema.build_econ_dataframe`. The + advance estimate carries ``settlement_grade=True`` (first print); second/ + third revisions carry ``False``. + + Raises: + DataAvailabilityError: ``reason="source_404"`` when no key is available — + a clear, documented keyless-degradation signal. + SourceUnavailableError: on a BEA structured error object or a hostile / + non-JSON body. + httpx.HTTPStatusError: on a non-transient HTTP error from BEA. + """ + resolved_key = _resolve_key(key) + if resolved_key is None: + # Keyless degradation — documented, not a silent gap. BEA GDP requires a + # free key; we refuse to hit it blind and raise a signal the caller can + # branch on (mirrors the ALFRED keyless branch). + raise DataAvailabilityError( + reason="source_404", + hint=( + "BEA GDP requires a BEA_API_KEY; set it (free at " + "apps.bea.gov/API/signup) or fall back to another indicator. " + "GDP has no keyless first-print source." + ), + source=ECON_SOURCE_BEA, + ) + + if year_range == "ALL": + year_param = "ALL" + else: + start_year, end_year = year_range # type: ignore[misc] + year_param = ",".join(str(y) for y in range(int(start_year), int(end_year) + 1)) + + params = { + "UserID": resolved_key, # outbound-only; never logged / stored. + "method": "GetData", + "datasetname": "NIPA", + "TableName": BEA_GDP_TABLE, + "Frequency": "Q", + "Year": year_param, + "ResultFormat": "JSON", + } + + owned = client is None + active = client if client is not None else httpx.Client(timeout=HTTP_TIMEOUT) + try: + response = _get_with_retry(active, BEA_DATA_URL, params) + try: + payload = response.json() + except ValueError as exc: + # HTML / non-JSON body → error, never parsed as empty (T-29-18 / + # Pitfall 4). A bot-wall or outage page must not become silent rows. + raise SourceUnavailableError( + "BEA returned a non-JSON body", + source=ECON_SOURCE_BEA, + url=BEA_DATA_URL, + http_status=response.status_code, + underlying=str(exc), + ) from exc + finally: + if owned: + active.close() + + return parse( + payload, + release_type=release_type, + vintage_date=vintage_date, + ) + + +__all__ = [ + "BEA_DATA_URL", + "BEA_ESTIMATE_ORDER", + "BEA_GDP_INDICATOR", + "BEA_GDP_LINE", + "BEA_GDP_TABLE", + "BEA_GDP_UNITS", + "fetch_gdp", + "infer_release_type", + "parse", +] diff --git a/packages/econ/src/mostlyright/econ/_fetchers/bls.py b/packages/econ/src/mostlyright/econ/_fetchers/bls.py new file mode 100644 index 00000000..7968f7ea --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_fetchers/bls.py @@ -0,0 +1,380 @@ +"""BLS timeseries fetcher — keyless v1 default + keyed v2, latest-revised provenance. + +The BLS Public Data API serves the *live* (latest-revised) series: a pull of, say, +Jun-2022 NFP returns today's post-annual-benchmark level, NOT the number BLS +originally released. There is NO vintage / as-first-released endpoint (the +``footnotes`` "P"/preliminary flag helps only for the most-recent months). So +every row this fetcher emits is stamped ``settlement_grade=False`` with +``current``/``revised`` provenance — the ``settlement_grade=True`` first-print +truth comes from ALFRED (:mod:`~mostlyright.econ._fetchers.fred_alfred`), NEVER +from here. Mislabeling a BLS-API value as first-print is threat T-29-14; this +module structurally prevents it (``settlement_grade`` is hard-coded ``False``). + +Endpoints (both HTTPS; the fetcher pins the scheme): + +- v1 keyless (default): ``POST publicAPI/v1/timeseries/data/`` — ~25 req/day, + ~10yr history. No key required (verified live 2026-07-08: HTTP 200, + ``REQUEST_SUCCEEDED``). +- v2 keyed: ``POST publicAPI/v2/timeseries/data/`` with the key in the body — + ~500 req/day, ~20yr history. Activated when a ``BLS_API_KEY`` is supplied + (arg or env). Absence falls back to v1 (graceful degradation — the keyless + default path is the competitive promise). + +PPI coverage (the checker BLOCKER-1 fix): BLS PPI lives in the **WP-prefixed +Producer Price Index database**, DISTINCT from the CU-prefixed CPI database. PPI +final-demand seasonally-adjusted is ``WPSFD4`` (BLS-confirmed title "PPI Commodity +data for Final demand, seasonally adjusted"), NSA is ``WPUFD4``; all-commodities +is the ``WPU00000000`` fallback. A future editor must NOT reuse a CU id for PPI. +KXUSPPI (PPI-MoM) settles to Trading Economics (``settlement_grade=False``) and +KXUSPPIYOY (PPI-YoY) settles to BLS — but that per-series split is applied by the +resolver (29-04) / ``research_econ`` (29-08), NOT here: the BLS fetcher stamps +``settlement_grade=False`` uniformly (BLS-API is latest-revised) and simply makes +the PPI value fetchable so ``history("ppi", …)`` and the TE divergence proxy have +a number to serve. + +Security: ``BLS_API_KEY`` is read from env (or passed explicitly), placed only in +the outbound request body's ``registrationkey`` field, and NEVER logged, +persisted, or embedded in an emitted row (threat T-29-12). + +House rule: in-house httpx only. No third-party BLS client is a dependency: the +PyPI BLS-API wrapper is not imported, and the similarly-named ``python``-prefixed +crypto namespace-trap package (RESEARCH library audit) is deliberately avoided — +this module talks to BLS over httpx directly. +""" + +from __future__ import annotations + +import logging +import os +import time +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from typing import Any + +import httpx +from mostlyright._internal._http import ( + BASE_DELAY, + HTTP_TIMEOUT, + MAX_RETRIES, + TRANSIENT_CODES, +) +from mostlyright.core.exceptions import SourceUnavailableError +from mostlyright.econ._schema import ECON_SOURCE_BLS_V1, ECON_SOURCE_BLS_V2 + +log = logging.getLogger(__name__) + +#: BLS timeseries endpoints (HTTPS only). The v2 path is used when a key is +#: present; v1 is the keyless default. +BLS_V1_URL = "https://api.bls.gov/publicAPI/v1/timeseries/data/" +BLS_V2_URL = "https://api.bls.gov/publicAPI/v2/timeseries/data/" + +#: BLS series id → canonical econ indicator. VERIFIED ids (RESEARCH Code Examples, +#: 2026-07-08). PPI (WPSFD4 / WPUFD4) closes the checker's "PPI has no fetcher" +#: BLOCKER — note the WP prefix (Producer Price Index database), NOT CU (CPI): +#: reusing a CU id for PPI would fetch a CPI series and silently mis-settle PPI. +BLS_SERIES: dict[str, str] = { + "CUUR0000SA0": "cpi", # CPI-U, all items (headline). + "CUUR0000SA0L1E": "cpi_core", # CPI-U less food & energy (core). + "CES0000000001": "nfp", # Total nonfarm payrolls (level; change derived). + "LNS14000000": "u3", # Unemployment rate (U3). + # --- PPI final-demand (WP-prefixed Producer Price Index database) --- + "WPSFD4": "ppi", # PPI Commodity, Final demand, SEASONALLY ADJUSTED (KXUSPPI defn). + "WPUFD4": "ppi", # PPI Commodity, Final demand, NSA variant. +} + +#: All-commodities PPI fallback (NOT final-demand). Documented constant, not in +#: BLS_SERIES by default — final-demand (WPSFD4) is the KXUSPPI settlement family. +BLS_PPI_ALL_COMMODITIES = "WPU00000000" + +#: Units per indicator (schema ``units``). Index-level series (CPI/PPI) → "index"; +#: the rate → "percent"; the payroll level → "thousands_persons". +_INDICATOR_UNITS: dict[str, str] = { + "cpi": "index", + "cpi_core": "index", + "ppi": "index", + "u3": "percent", + "nfp": "thousands_persons", +} + +#: BLS preliminary footnote code — flags the most-recent month(s) as an early +#: estimate. Present → ``release_type="preliminary"``; absent → ``"revised"`` +#: (BLS serves latest-revised; RESEARCH Pitfall 5). +_PRELIMINARY_FOOTNOTE_CODE = "P" + + +def _resolve_key(key: str | None) -> str | None: + """Return the explicit ``key`` or fall back to ``BLS_API_KEY`` in env. + + Never logs the key. ``None`` when neither is set (the fetcher then uses the + keyless v1 endpoint — graceful degradation, not a failure). + """ + if key: + return key + env = os.environ.get("BLS_API_KEY") + return env or None + + +def _decode_period(year: str, period: str) -> str: + """Decode a BLS ``(year, period)`` pair into a schema ``period`` string. + + BLS period codes: + - ``M01`` to ``M12`` monthly → ``YYYY-MM`` (``M13`` is the annual average → year). + - ``Q01`` to ``Q04`` quarterly → ``YYYYQn`` (``Q05`` annual average → year). + - ``A01`` annual → ``YYYY``. + - ``S01``/``S02``/``S03`` semiannual → ``YYYY`` (rare; collapsed to year). + + An unrecognized code falls back to ``YYYY`` so a value is never dropped for a + parse quirk (the value is still fetchable; the period is coarsened). + """ + if not period: + return year + code = period[0] + if code == "M": + month = period[1:] + if month == "13": # annual average + return year + return f"{year}-{month.zfill(2)}" + if code == "Q": + q = period[1:] + if q == "05": # annual average + return year + return f"{year}Q{int(q)}" + # A (annual), S (semiannual), or anything else → year granularity. + return year + + +def _release_type_for(footnotes: Sequence[Mapping[str, Any]] | None) -> str: + """Map BLS footnotes to a ``release_type``. + + A "P"/preliminary footnote (present on the most-recent month) → + ``"preliminary"``; anything else → ``"revised"`` (BLS serves latest-revised, + so a settled month with no preliminary flag IS the revised series — never + ``"advance"``/first-print, which is ALFRED's job). + """ + if footnotes: + for fn in footnotes: + if isinstance(fn, Mapping) and fn.get("code") == _PRELIMINARY_FOOTNOTE_CODE: + return "preliminary" + return "revised" + + +def parse( + payload: Mapping[str, Any], + *, + retrieved_at: datetime | None = None, +) -> list[dict[str, Any]]: + """Parse a BLS timeseries response into ``schema.econ.observations.v1`` rows. + + Walks ``Results.series[].data[]``, decoding the BLS period code, coercing the + value to float, mapping the preliminary footnote, and stamping + ``settlement_grade=False`` + ``current``/``revised`` provenance on EVERY row + (BLS-API is latest-revised; the settlement-grade first-print is ALFRED's). + + ``vintage_date`` / ``knowledge_time`` are set to ``retrieved_at`` (the fetch + time) because BLS-API rows are NOT vintage-exact — there is no realtime + endpoint. This is documented behavior, not a silent gap: a consumer wanting + the true first-print vintage must use ALFRED. + + Raises: + SourceUnavailableError: when ``payload`` is not a successful BLS JSON + response (missing/failed ``status``, or no ``Results.series`` list). + A hostile / HTML-instead-of-JSON body is rejected here, NOT parsed as + empty rows (threat T-29-13 / RESEARCH Pitfall 4). + """ + if retrieved_at is None: + retrieved_at = datetime.now(UTC) + + if not isinstance(payload, Mapping): + raise SourceUnavailableError( + "BLS response is not a JSON object", + source=ECON_SOURCE_BLS_V1, + url=BLS_V1_URL, + underlying=f"payload type={type(payload).__name__}", + ) + status = payload.get("status") + if status != "REQUEST_SUCCEEDED": + messages = payload.get("message") + raise SourceUnavailableError( + f"BLS request not successful: status={status!r}", + source=ECON_SOURCE_BLS_V1, + url=BLS_V1_URL, + underlying=str(messages) if messages else "", + ) + results = payload.get("Results") + if not isinstance(results, Mapping) or "series" not in results: + raise SourceUnavailableError( + "BLS response missing Results.series", + source=ECON_SOURCE_BLS_V1, + url=BLS_V1_URL, + ) + series_list = results["series"] + if not isinstance(series_list, list): + raise SourceUnavailableError( + "BLS Results.series is not a list", + source=ECON_SOURCE_BLS_V1, + url=BLS_V1_URL, + ) + + rows: list[dict[str, Any]] = [] + for series in series_list: + series_id = series.get("seriesID") + indicator = BLS_SERIES.get(series_id) if series_id else None + units = _INDICATOR_UNITS.get(indicator) if indicator else None + data = series.get("data") or [] + for datum in data: + year = datum.get("year") + period_code = datum.get("period") + raw_value = datum.get("value") + if year is None or period_code is None: + continue + try: + value = float(raw_value) if raw_value not in (None, "", ".") else None + except (TypeError, ValueError): + value = None + rows.append( + { + "indicator": indicator, + "series_id": series_id, + "period": _decode_period(str(year), str(period_code)), + "value": value, + "units": units, + "release_datetime": None, # BLS gives no wall-clock release time here. + # BLS-API is latest-revised (no vintage endpoint) → the + # knowledge/vintage time is the fetch time, documented as + # NOT vintage-exact. + "vintage_date": retrieved_at, + "knowledge_time": retrieved_at, + "release_type": _release_type_for(datum.get("footnotes")), + # BLS-API values are current/revised, NEVER settlement-grade + # first-print (that is ALFRED's job) — hard-coded False. + "settlement_grade": False, + "retrieved_at": retrieved_at, + } + ) + return rows + + +def _post_with_retry( + client: httpx.Client, + url: str, + body: Mapping[str, Any], +) -> httpx.Response: + """POST ``body`` as JSON to ``url`` with the shared retry discipline. + + Reuses ``mostlyright._internal._http``'s retry constants (transient codes + backed off, 404 raises immediately). The caller owns ``client``. The request + body is passed via ``json=`` so the ``registrationkey`` never appears in a + logged URL. + """ + delay = BASE_DELAY + for attempt in range(MAX_RETRIES): + try: + response = client.post(url, json=dict(body)) + except httpx.RequestError as exc: + # SECURITY (codex round-2 H6): a transport error carries the request on + # exc.request; the BLS v2 POST body holds BLS_API_KEY. Raise a sanitized + # error with the bare endpoint (`from None` drops the httpx exception). + raise SourceUnavailableError( + f"BLS request failed ({type(exc).__name__})", + source=ECON_SOURCE_BLS_V1, + url=url, + ) from None + if response.status_code in TRANSIENT_CODES and attempt < MAX_RETRIES - 1: + log.warning( + "BLS HTTP %d (attempt %d/%d), retrying in %.1fs", + response.status_code, + attempt + 1, + MAX_RETRIES, + delay, + ) + time.sleep(delay) + delay *= 2 + continue + response.raise_for_status() + return response + raise SourceUnavailableError( # pragma: no cover + "BLS retry loop exhausted without a response", + source=ECON_SOURCE_BLS_V1, + url=url, + ) + + +def fetch( + series_ids: Sequence[str], + *, + start_year: int, + end_year: int, + key: str | None = None, + client: httpx.Client | None = None, +) -> list[dict[str, Any]]: + """Fetch BLS timeseries values for ``series_ids`` (keyless v1 / keyed v2). + + Args: + series_ids: BLS series ids (e.g. ``["CUUR0000SA0", "WPSFD4"]``). Mapped to + canonical indicators via :data:`BLS_SERIES`; an id not in the map still + fetches (its ``indicator`` is ``None`` for the caller/resolver to fill). + start_year / end_year: inclusive year range for the request. + key: the BLS registration key. Falls back to ``BLS_API_KEY`` in env; when + present the v2 endpoint is used, else the keyless v1 endpoint. Never + logged / persisted / embedded in a row. + client: an injected ``httpx.Client`` (tests / connection reuse). When + ``None`` a fresh HTTPS client is created and closed per call. + + Returns: + Rows shaped for :func:`mostlyright.econ._schema.build_econ_dataframe`, all + stamped ``settlement_grade=False`` (BLS-API is latest-revised). + + Raises: + SourceUnavailableError: on a non-JSON / non-``REQUEST_SUCCEEDED`` body. + httpx.HTTPStatusError: on a non-transient HTTP error from BLS. + """ + resolved_key = _resolve_key(key) + url = BLS_V2_URL if resolved_key else BLS_V1_URL + source = ECON_SOURCE_BLS_V2 if resolved_key else ECON_SOURCE_BLS_V1 + + body: dict[str, Any] = { + "seriesid": list(series_ids), + "startyear": str(start_year), + "endyear": str(end_year), + } + if resolved_key: + # Outbound-only; carried in the JSON body, never a logged URL. + body["registrationkey"] = resolved_key + + owned = client is None + active = client if client is not None else httpx.Client(timeout=HTTP_TIMEOUT) + try: + response = _post_with_retry(active, url, body) + try: + payload = response.json() + except ValueError as exc: + # HTML / non-JSON body → error, never parsed as empty (T-29-13 / + # Pitfall 4). A bot-wall or rate-limit page must not become silent + # empty rows. + raise SourceUnavailableError( + "BLS returned a non-JSON body", + source=source, + url=url, + http_status=response.status_code, + underlying=str(exc), + ) from exc + finally: + if owned: + active.close() + + rows = parse(payload) + # Stamp the correct source tag (v1/v2) on each row so build_econ_dataframe's + # row-vs-attrs source check is coherent when the caller passes source=source. + for row in rows: + row["source"] = source + return rows + + +__all__ = [ + "BLS_PPI_ALL_COMMODITIES", + "BLS_SERIES", + "BLS_V1_URL", + "BLS_V2_URL", + "fetch", + "parse", +] diff --git a/packages/econ/src/mostlyright/econ/_fetchers/dol.py b/packages/econ/src/mostlyright/econ/_fetchers/dol.py new file mode 100644 index 00000000..7b0abe72 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_fetchers/dol.py @@ -0,0 +1,446 @@ +"""DOL initial jobless claims fetcher — FRED/ALFRED ICSA fallback (bot-wall workaround). + +**The bot-wall (RESEARCH Pitfall 4).** The Department of Labor's +``oui.doleta.gov/unemploy/csv/*.csv`` directory — the machine-readable weekly +initial-claims source — is ENTIRELY bot-walled (403 HTML) to scripted access. Only +the human PDF is reachable. The documented workaround is to source weekly initial +claims from FRED series ``ICSA`` (seasonally adjusted) / ``ICNSA`` (NSA) instead. +Any ``oui.doleta.gov`` response, or any HTML/`` str | None: + """Return the explicit ``key`` or fall back to ``FRED_API_KEY`` in env. + + Never logs the key. ``None`` → the fetcher takes the keyless latest-revised + branch (``settlement_grade=False``), NOT a hard failure. + """ + if key: + return key + env = os.environ.get("FRED_API_KEY") + return env or None + + +def _parse_fred_date(value: str) -> datetime: + """Parse a FRED ``YYYY-MM-DD`` date into a tz-aware UTC midnight datetime. + + FRED dates are calendar days (day-granularity limitation), anchored at + 00:00:00 UTC so the value is a valid ``timestamp_utc`` for the schema. + """ + return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=UTC) + + +def parse_icsa( + payload: Mapping[str, Any], + *, + series_id: str = DOL_ICSA_SERIES, + retrieved_at: datetime | None = None, +) -> list[dict[str, Any]]: + """Parse a FRED/ALFRED ICSA ``series/observations`` payload into raw vintage rows. + + Each returned row carries the raw vintage identity (one row per + ``(week-ending date x realtime_start)``) BEFORE first-release / latest-revised + selection — ``settlement_grade`` / ``release_type`` are assigned by + :func:`_first_release` (keyed) or stamped ``False`` (keyless) by the caller. + + Raises: + SourceUnavailableError: if ``payload`` is not a mapping with an + ``observations`` list — a hostile / HTML-instead-of-JSON body (the + ``oui.doleta.gov`` bot-wall page, or a FRED error page) must be + rejected, never parsed as empty (threat T-29-15 / Pitfall 4). + """ + if retrieved_at is None: + retrieved_at = datetime.now(UTC) + + if not isinstance(payload, Mapping) or "observations" not in payload: + raise SourceUnavailableError( + "FRED/ALFRED ICSA response is not a valid observations payload " + "(an HTML/403 bot-wall body is rejected, never parsed as data)", + source=ECON_SOURCE_DOL, + url=FRED_OBSERVATIONS_URL, + underlying="missing 'observations' key", + ) + observations = payload["observations"] + if not isinstance(observations, list): + raise SourceUnavailableError( + "FRED/ALFRED ICSA 'observations' is not a list", + source=ECON_SOURCE_DOL, + url=FRED_OBSERVATIONS_URL, + underlying=f"observations type={type(observations).__name__}", + ) + + rows: list[dict[str, Any]] = [] + for obs in observations: + if not isinstance(obs, Mapping): + continue + realtime_start = obs.get("realtime_start") + obs_date = obs.get("date") + raw_value = obs.get("value") + if realtime_start is None or obs_date is None: + continue + vintage_dt = _parse_fred_date(realtime_start) + value = None if raw_value in _MISSING_VALUES else float(raw_value) + rows.append( + { + "indicator": DOL_INDICATOR, + "series_id": series_id, + "period": str(obs_date), # weekly → full YYYY-MM-DD week-ending date. + "value": value, + "units": DOL_UNITS, + "release_datetime": None, # FRED gives no wall-clock release time. + "vintage_date": vintage_dt, + "knowledge_time": vintage_dt, # leakage cutoff == vintage_date. + "vintage_precision": VINTAGE_PRECISION_DAY, + "source": ECON_SOURCE_DOL, + "retrieved_at": retrieved_at, + } + ) + return rows + + +def _first_release(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return one row per week — the FIRST PRINT (earliest ``realtime_start``). + + The ~5-line first-release rule 29-05 uses, re-expressed inline here (NOT + imported from the same-wave ``fred_alfred.py``): group by week-ending + ``period``, keep the row with the EARLIEST ``vintage_date``, and stamp it + ``release_type="advance"``, ``settlement_grade=True``, + ``vintage_precision="day"``. + + Day-granularity behavior (the documented landmine): two ``realtime_start`` + values on the SAME calendar date collapse to one first-release row; we keep the + first encountered and leave ``vintage_precision="day"`` set so the consumer + knows the vintage is day-exact, not intraday-exact. + """ + by_period: dict[str, dict[str, Any]] = {} + for row in rows: + period = row["period"] + candidate = dict(row) + existing = by_period.get(period) + if existing is None or candidate["vintage_date"] < existing["vintage_date"]: + by_period[period] = candidate + + result: list[dict[str, Any]] = [] + for row in by_period.values(): + row["release_type"] = "advance" + row["settlement_grade"] = True + row["vintage_precision"] = VINTAGE_PRECISION_DAY + result.append(row) + return result + + +def _latest_revised(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return every row stamped ``settlement_grade=False`` (latest-revised). + + The keyless path: a plain FRED pull returns the latest-revised value, which is + NOT the first print. We keep every row but label it ``release_type="revised"`` + + ``settlement_grade=False`` so a non-vintage value is NEVER mislabeled as + settlement-grade (threat T-29-15 corollary). + """ + result: list[dict[str, Any]] = [] + for row in rows: + out = dict(row) + out["release_type"] = "revised" + out["settlement_grade"] = False + result.append(out) + return result + + +def _get_with_retry( + client: httpx.Client, + url: str, + params: Mapping[str, Any], +) -> httpx.Response: + """GET ``url`` with the shared retry discipline (transient codes backed off). + + Reuses ``mostlyright._internal._http``'s retry constants. 403 is NOT transient + (a bot-wall / auth wall) → it falls through to ``raise_for_status`` which the + caller converts to a :class:`SourceUnavailableError`. The key rides in + ``params`` (query string) — callers must never log the resolved URL. + """ + delay = BASE_DELAY + for attempt in range(MAX_RETRIES): + try: + response = client.get(url, params=dict(params)) + except httpx.RequestError as exc: + # SECURITY (codex round-2 H6): sanitize transport errors — the request + # URL (with api_key=) is on exc.request; raise with the bare endpoint. + raise SourceUnavailableError( + f"FRED/ALFRED ICSA request failed ({type(exc).__name__})", + source=ECON_SOURCE_DOL, + url=url, + ) from None + if response.status_code in TRANSIENT_CODES and attempt < MAX_RETRIES - 1: + log.warning( + "FRED/ALFRED ICSA HTTP %d (attempt %d/%d), retrying in %.1fs", + response.status_code, + attempt + 1, + MAX_RETRIES, + delay, + ) + time.sleep(delay) + delay *= 2 + continue + return response + raise SourceUnavailableError( # pragma: no cover + "FRED/ALFRED ICSA retry loop exhausted without a response", + source=ECON_SOURCE_DOL, + url=url, + ) + + +def _fetch_fred_payload( + series_id: str, + *, + key: str, + realtime: bool, + client: httpx.Client, +) -> Mapping[str, Any]: + """Issue the FRED/ALFRED GET and return the parsed JSON payload. + + When ``realtime`` is ``True`` the request carries ``realtime_start`` / + ``realtime_end`` (an ALFRED vintage query — the keyed first-print path); when + ``False`` it omits them (a plain FRED latest-revised pull — the keyless path). + + A 403 / non-2xx status, or an HTML/non-JSON body, raises + :class:`SourceUnavailableError` (the bot-wall guard — Pitfall 4). The key is + never included in the raised message. + """ + params: dict[str, Any] = { + "series_id": series_id, + "file_type": "json", + "api_key": key, # outbound-only; never logged / stored. + } + if realtime: + params["realtime_start"] = DEFAULT_REALTIME_START + params["realtime_end"] = DEFAULT_REALTIME_END + + response = _get_with_retry(client, FRED_OBSERVATIONS_URL, params) + if response.status_code != 200: + # A 403 bot-wall (or any non-200) is an error, never empty data. + raise SourceUnavailableError( + f"FRED/ALFRED ICSA returned HTTP {response.status_code} " + "(a 403 bot-wall body is rejected, never parsed as data)", + source=ECON_SOURCE_DOL, + url=FRED_OBSERVATIONS_URL, + http_status=response.status_code, + ) + try: + payload = response.json() + except ValueError as exc: + # HTML / non-JSON body → error (T-29-15 / Pitfall 4). + raise SourceUnavailableError( + "FRED/ALFRED ICSA returned a non-JSON body " + "(an HTML bot-wall page is rejected, never parsed as data)", + source=ECON_SOURCE_DOL, + url=FRED_OBSERVATIONS_URL, + http_status=response.status_code, + underlying=str(exc), + ) from exc + if not isinstance(payload, Mapping): + raise SourceUnavailableError( + "FRED/ALFRED ICSA JSON is not an object", + source=ECON_SOURCE_DOL, + url=FRED_OBSERVATIONS_URL, + ) + return payload + + +def fetch_initial_claims( + *, + key: str | None = None, + series: str = DOL_ICSA_SERIES, + client: httpx.Client | None = None, + fred_fetch: Callable[..., Mapping[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Fetch weekly initial jobless claims via the FRED/ALFRED ICSA fallback. + + The DOL ``oui.doleta.gov`` CSV directory is bot-walled (403), so this sources + weekly initial claims from FRED ``ICSA`` / ``ICNSA`` instead (Pitfall 4). + + Args: + key: the FRED API key. Falls back to ``FRED_API_KEY`` in env. WITH a key + the first print is routed through ALFRED realtime vintages + (``settlement_grade=True`` on the earliest ``realtime_start`` per week); + KEYLESS degrades to FRED latest-revised (``settlement_grade=False``). + Never logged / persisted / embedded in a row. + series: ``"ICSA"`` (seasonally adjusted, default — the settlement basis) or + ``"ICNSA"`` (NSA). + client: an injected ``httpx.Client`` (tests / connection reuse). When + ``None`` a fresh HTTPS client is created and closed per call. + fred_fetch: an optional injected fetch callable + ``(series_id, *, key, realtime) -> payload`` (documented option (b) to + fully decouple the transport from the same-wave ``fred_alfred.py``). + When given it is used instead of the built-in GET; ``client`` is then + unused. + + Returns: + Rows shaped for :func:`mostlyright.econ._schema.build_econ_dataframe`. Keyed + → first-release rows (``settlement_grade=True``); keyless → latest-revised + rows (``settlement_grade=False``). Never ``[]``/``None`` for a released + window. + + Raises: + SourceUnavailableError: on a 403 bot-wall / HTML / non-JSON body (never + parsed as empty data — Pitfall 4), or an unknown ``series``. + httpx.HTTPStatusError: on a non-transient HTTP error surfaced by the GET. + """ + if series not in DOL_SERIES_IDS: + raise SourceUnavailableError( + f"unknown jobless-claims series {series!r}; expected one of {sorted(DOL_SERIES_IDS)}", + source=ECON_SOURCE_DOL, + url=FRED_OBSERVATIONS_URL, + ) + + resolved_key = _resolve_key(key) + keyed = resolved_key is not None + # Keyed → ALFRED realtime vintages (first-print); keyless → plain FRED pull. + realtime = keyed + + if not keyed and fred_fetch is None and client is None: + # jobless_claims has NO keyless source over the REAL network: the DOL claims + # CSV is bot-walled (403) and the FRED ICSA fallback rejects an empty + # api_key. Fail FAST pre-network with an actionable signal, rather than + # sending a doomed empty-key request that FRED bounces (codex finding). + # An injected ``client`` or ``fred_fetch`` (tests / a caller supplying its + # own keyless transport) still exercises the parse/degrade path. + raise DataAvailabilityError( + reason="source_404", + hint=( + "jobless_claims has no keyless source (the DOL claims CSV is " + "bot-walled and the FRED ICSA fallback needs a key); set " + "FRED_API_KEY (free at fred.stlouisfed.org/docs/api/api_key.html)." + ), + ) + + if fred_fetch is not None: + # Injected transport (option (b)). The keyed path still requests realtime + # vintages; keyless passes an empty string key (the callable owns the + # keyless FRED semantics). + payload = fred_fetch(series, key=resolved_key or "", realtime=realtime) + else: + owned = client is None + active = client if client is not None else httpx.Client(timeout=HTTP_TIMEOUT) + try: + payload = _fetch_fred_payload( + series, + # A keyless FRED pull still needs SOME api_key param to succeed + # against the live endpoint; when no key is present we pass the + # empty string, which FRED rejects — so keyless callers that lack + # a key rely on an injected client/payload in practice. The + # empty-string carries NO secret and is never logged. + key=resolved_key or "", + realtime=realtime, + client=active, + ) + finally: + if owned: + active.close() + + raw = parse_icsa(payload, series_id=series) + if keyed: + return _first_release(raw) + return _latest_revised(raw) + + +__all__ = [ + "DEFAULT_REALTIME_END", + "DEFAULT_REALTIME_START", + "DOL_ICNSA_SERIES", + "DOL_ICSA_SERIES", + "DOL_INDICATOR", + "DOL_SERIES_IDS", + "DOL_UNITS", + "FRED_OBSERVATIONS_URL", + "VINTAGE_PRECISION_DAY", + "fetch_initial_claims", + "parse_icsa", +] diff --git a/packages/econ/src/mostlyright/econ/_fetchers/fed.py b/packages/econ/src/mostlyright/econ/_fetchers/fed.py new file mode 100644 index 00000000..c8fce11b --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_fetchers/fed.py @@ -0,0 +1,522 @@ +"""Federal Reserve FOMC decision fetcher — target-rate (numeric + categorical). + +**Fed, NOT FRED (the SEED-002 wrinkle).** KXFED's ``settlement_sources`` name is +"Federal Reserve Board of Governors" (verified) — the settlement authority is the +Fed Board, NOT the St. Louis Fed's FRED series. So this fetcher pins a +``federalreserve.gov`` endpoint (the ``openmarket.htm`` rate-change record / the +FOMC statement pages) and NEVER touches the FRED API host (threat T-29-16; +the grep-gate in the plan's verify asserts the FRED host string is absent here). + +**Each decision carries BOTH a numeric target and a categorical decision.** The +FOMC sets a target *range* (e.g. 4.25-4.50%); the numeric ``value`` is the range +MIDPOINT (documented convention — a single scalar the settlement join can compare). +The categorical decision (hike / hold / cut) is derived DETERMINISTICALLY from the +change vs the prior meeting's target: upper-bound up → ``hike``, unchanged → +``hold``, down → ``cut``. The first meeting in a fetched window has no prior to +diff against, so it is labeled ``hold`` (no change detectable) — documented, not a +fabricated direction. + +**Schema mapping (Claude's discretion per CONTEXT).** Rather than introduce a +distinct Fed schema variant, decisions reuse ``schema.econ.observations.v1``: + +- ``indicator`` = ``"fed_funds"``, ``units`` = ``"percent"``, ``value`` = the + target-range midpoint. +- The categorical decision is encoded in ``series_id`` as + ``"fed_funds:"`` (``fed_funds:hike`` / ``:hold`` / ``:cut``). This is a + DOCUMENTED convention chosen because ``release_type`` is the vintage enum + (advance/second/third/final/…) and MUST NOT be repurposed for the categorical. + :func:`decision_of` reads the categorical back off a row. +- ``release_type`` = ``"final"`` and ``settlement_grade=True`` — an FOMC decision is + announced as-final and is not revised (unlike a GDP estimate). + +**A6 — sourcing (live-smoke verified 2026-07-09).** The H.15 Data Download Program +does NOT carry the target range (it serves the *effective* funds rate as SDMX XML, +and a bare ``Output.aspx`` GET is HTTP 400) — the original H.15-JSON assumption was +refuted by the mandatory real-API smoke. The live path fetches the Board's +canonical rate-change record at ``monetarypolicy/openmarket.htm`` — keyless, +year-sectioned ``Date | Increase | Decrease | Level (%)`` tables — parsed by +:func:`parse_openmarket_html` into the decisions payload :func:`parse_decisions` +consumes. The page records rate CHANGES at their EFFECTIVE dates (a scheduled +change is typically effective the business day after the announcement); hold +meetings do not appear as rows. If only an FOMC statement page is available, +:func:`parse_statement_html` extracts the "target range … X to Y percent" sentence; +a page WITHOUT a parseable target RAISES (:class:`SourceUnavailableError`) rather +than emitting a garbage value. + +Security: no key is required (the Fed data is public). HTTPS is enforced. +""" + +from __future__ import annotations + +import logging +import re +import time +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from typing import Any + +import httpx +from mostlyright._internal._http import ( + BASE_DELAY, + HTTP_TIMEOUT, + MAX_RETRIES, + TRANSIENT_CODES, +) +from mostlyright.core.exceptions import SourceUnavailableError +from mostlyright.econ._schema import ECON_SOURCE_FED + +log = logging.getLogger(__name__) + +#: The Fed Board's canonical open-market rate-change record (HTTPS only, keyless). +#: Live-smoke verified 2026-07-09: the H.15 Data Download Program serves only the +#: EFFECTIVE funds rate as SDMX XML (no target range, no JSON; a bare Output.aspx +#: GET is HTTP 400) — the target-range decision record lives on this page. The Fed +#: Board is KXFED's settlement authority, NOT the FRED API; the host is pinned so a +#: Fed decision can never be sourced from the wrong authority (threat T-29-16). +FED_OPENMARKET_URL = "https://www.federalreserve.gov/monetarypolicy/openmarket.htm" + +#: The FOMC statement page root (the A6 HTML fallback source). +FED_FOMC_STATEMENT_ROOT = "https://www.federalreserve.gov/newsevents/pressreleases" + +#: Canonical indicator id + units for a Fed funds target decision. +FED_INDICATOR = "fed_funds" +FED_UNITS = "percent" + +#: The categorical decision vocabulary. Encoded in ``series_id`` as +#: ``"fed_funds:"`` (documented convention — NOT via ``release_type``). +FED_DECISIONS: tuple[str, ...] = ("hike", "hold", "cut") + +#: Matches the FOMC statement's target-range sentence, e.g. "target range for the +#: federal funds rate at 4-1/4 to 4-1/2 percent". Captures the two bound strings +#: (which may be fractions like "4-1/4" or decimals like "4.25"). +_TARGET_RANGE_RE = re.compile( + r"target range[^.]*?at\s+([0-9][0-9/\-.\s]*?)\s+to\s+([0-9][0-9/\-.\s]*?)\s+percent", + re.IGNORECASE, +) + + +def _parse_meeting_date(value: str) -> datetime: + """Parse a ``YYYY-MM-DD`` FOMC meeting date into tz-aware UTC midnight. + + FOMC decisions are announced on a calendar day; the value is anchored at + 00:00:00 UTC so it is a valid ``timestamp_utc`` for the schema's vintage / + knowledge-time columns. + """ + return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=UTC) + + +def _fraction_to_float(token: str) -> float: + """Convert a Fed rate token to a float. + + Handles the FOMC statement's mixed-fraction form ("4-1/4" → 4.25, "4-1/2" → + 4.50) as well as plain decimals ("4.25"). Raises :class:`ValueError` on a token + that is neither (so the caller rejects a non-matching page). + """ + tok = token.strip() + # Mixed fraction "W-N/D" (e.g. "4-1/4"). + m = re.fullmatch(r"(\d+)-(\d+)/(\d+)", tok) + if m: + whole, num, den = (int(g) for g in m.groups()) + return whole + num / den + # Bare fraction "N/D". + m = re.fullmatch(r"(\d+)/(\d+)", tok) + if m: + num, den = (int(g) for g in m.groups()) + return num / den + # Plain decimal / integer. + return float(tok) + + +def _midpoint(lower: float, upper: float) -> float: + """Return the target-range midpoint (the documented numeric ``value``).""" + return (lower + upper) / 2.0 + + +def _categorical(prev_upper: float | None, upper: float) -> str: + """Derive hike / hold / cut from the change in the upper target bound. + + Deterministic: a higher upper bound than the prior meeting → ``"hike"``, a lower + one → ``"cut"``, unchanged (or no prior known) → ``"hold"``. The upper bound is + the discriminating edge (the FOMC moves the whole range in lockstep, so upper + and lower agree on direction). + """ + if prev_upper is None or upper == prev_upper: + return "hold" + return "hike" if upper > prev_upper else "cut" + + +def decision_of(row: Mapping[str, Any]) -> str: + """Read the categorical decision (hike/hold/cut) back off a decision row. + + The categorical is encoded in ``series_id`` as ``"fed_funds:"`` (the + documented convention). Returns the ```` suffix, or ``"hold"`` if the + row carries no encoded decision (defensive default). + """ + series_id = row.get("series_id") or "" + if ":" in series_id: + candidate = series_id.split(":", 1)[1] + if candidate in FED_DECISIONS: + return candidate + return "hold" + + +def _build_row( + meeting_date: str, + lower: float, + upper: float, + *, + prev_upper: float | None, + retrieved_at: datetime, +) -> dict[str, Any]: + """Assemble one ``schema.econ.observations.v1`` decision row. + + ``value`` = the range midpoint; the categorical is encoded in ``series_id``; + ``release_type="final"`` + ``settlement_grade=True`` (announced, not revised); + ``vintage_date`` / ``knowledge_time`` = the meeting date (day granular). + """ + decision = _categorical(prev_upper, upper) + vintage_dt = _parse_meeting_date(meeting_date) + return { + "indicator": FED_INDICATOR, + "series_id": f"{FED_INDICATOR}:{decision}", + "period": meeting_date, + "value": _midpoint(lower, upper), + "units": FED_UNITS, + "release_datetime": None, # the H.15 feed carries no wall-clock time. + "vintage_date": vintage_dt, + "knowledge_time": vintage_dt, # leakage cutoff == meeting date. + # An FOMC decision is announced as-final and never revised. + "release_type": "final", + "settlement_grade": True, + "source": ECON_SOURCE_FED, + "retrieved_at": retrieved_at, + } + + +#: Year section headings on ``openmarket.htm`` (``

2025

``); change-free +#: years (e.g. 2021) have no heading. Rows before the first heading are page chrome. +_OPENMARKET_YEAR_RE = re.compile(r"]*>\s*(\d{4})\s*") +_OPENMARKET_ROW_RE = re.compile(r"]*>(.*?)", re.S) +_OPENMARKET_CELL_RE = re.compile(r"]*>(.*?)", re.S) +_OPENMARKET_TAG_RE = re.compile(r"<[^>]+>") +#: ``Level (%)`` cell forms (live-verified 2026-07-09): a modern range +#: ``3.50-3.75`` or a pre-2008 point target ``2.25``. Anything else raises. +_OPENMARKET_LEVEL_RANGE_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)$") +_OPENMARKET_LEVEL_POINT_RE = re.compile(r"^(\d+(?:\.\d+)?)$") + + +def parse_openmarket_html(html: str) -> dict[str, Any]: + """Parse the Board's ``openmarket.htm`` rate-change record into a decisions payload. + + The page (live-verified 2026-07-09) is a sequence of ``

YYYY

`` year + headings, each followed by a table of ``Date | Increase | Decrease | Level (%)`` + rows — e.g. ``December 11 | 0 | 25 | 3.50-3.75``. Dates carry no year (the + heading supplies it); ``Level`` is a modern target range (``3.50-3.75``) or a + pre-2008 point target (``2.25``); filler cells are ``...``. The page records + rate CHANGES at their EFFECTIVE dates — hold meetings do not appear. + + Returns: + ``{"decisions": [{meeting_date, target_lower, target_upper}, ...]}`` — the + exact payload :func:`parse_decisions` consumes. + + Raises: + SourceUnavailableError: on an empty/hostile body, an unparseable date or + Level cell in a data row, or a page yielding ZERO decisions (a + redesigned/error page is rejected loudly, never parsed as empty — + threat T-29-16). + """ + if not isinstance(html, str) or not html.strip(): + raise SourceUnavailableError( + "Federal Reserve openmarket.htm body is empty", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying=f"body type={type(html).__name__}", + ) + + events: list[tuple[int, str, str]] = [ + (m.start(), "year", m.group(1)) for m in _OPENMARKET_YEAR_RE.finditer(html) + ] + events.extend((m.start(), "row", m.group(1)) for m in _OPENMARKET_ROW_RE.finditer(html)) + events.sort(key=lambda event: event[0]) + + year: str | None = None + decisions: list[dict[str, Any]] = [] + for _, kind, body in events: + if kind == "year": + year = body + continue + cells = [ + _OPENMARKET_TAG_RE.sub("", cell).replace("\xa0", " ").strip() + for cell in _OPENMARKET_CELL_RE.findall(body) + ] + # Header rows repeat per year table; rows before the first year heading and + # short rows are page chrome (nav/layout), not decisions. + if year is None or len(cells) < 4 or cells[0].lower() == "date": + continue + date_text, level_text = cells[0], cells[3] + # Footnote markers are literal glyphs inside anchors on the real page + # (e.g. "March 4*" for the 2020 inter-meeting cut) — strip them. + date_text = date_text.replace("*", "").replace("†", "").replace("‡", "").strip() + try: + meeting_date = datetime.strptime(f"{date_text} {year}", "%B %d %Y").date().isoformat() + except ValueError as exc: + raise SourceUnavailableError( + "openmarket.htm decision-row date is not parseable", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying=f"date={date_text!r} year={year}", + ) from exc + level = level_text.replace("–", "-").replace("—", "-") # noqa: RUF001 intentional dash normalization of Fed rate ranges + range_match = _OPENMARKET_LEVEL_RANGE_RE.match(level) + if range_match: + lower, upper = float(range_match.group(1)), float(range_match.group(2)) + else: + point_match = _OPENMARKET_LEVEL_POINT_RE.match(level) + if not point_match: + raise SourceUnavailableError( + "openmarket.htm Level cell is not a parseable target " + "(rejected loudly, never emitted as a garbage value)", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying=f"level={level_text!r} date={date_text!r} year={year}", + ) + lower = upper = float(point_match.group(1)) + decisions.append( + {"meeting_date": meeting_date, "target_lower": lower, "target_upper": upper} + ) + + if not decisions: + raise SourceUnavailableError( + "openmarket.htm contained no parseable rate decisions " + "(a hostile or redesigned page is rejected, never parsed as empty)", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying=f"year_headings={sum(1 for e in events if e[1] == 'year')}", + ) + return {"decisions": decisions} + + +def parse_decisions( + payload: Mapping[str, Any], + *, + retrieved_at: datetime | None = None, +) -> list[dict[str, Any]]: + """Parse a Federal Reserve decisions payload into decision rows. + + Expects ``payload["decisions"]`` — a list of + ``{meeting_date, target_upper, target_lower}`` entries. Rows are emitted in + chronological order; each carries the range midpoint as ``value`` and a + categorical (hike/hold/cut) derived from the change vs the prior meeting's + upper bound (the first in-window meeting is ``hold`` — no prior to diff). + + Raises: + SourceUnavailableError: when ``payload`` is not a mapping with a + ``decisions`` list, or an entry lacks the target bounds — a hostile / + HTML-instead-of-JSON body (or an empty non-decision page) is rejected, + never parsed as empty rows (threat T-29-16). + """ + if retrieved_at is None: + retrieved_at = datetime.now(UTC) + + if not isinstance(payload, Mapping) or "decisions" not in payload: + raise SourceUnavailableError( + "Federal Reserve response is not a valid decisions payload " + "(an HTML / non-JSON body is rejected, never parsed as data)", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying="missing 'decisions' key", + ) + decisions = payload["decisions"] + if not isinstance(decisions, Sequence) or isinstance(decisions, str | bytes): + raise SourceUnavailableError( + "Federal Reserve 'decisions' is not a list", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying=f"decisions type={type(decisions).__name__}", + ) + + # Sort chronologically so the prior-meeting diff (hike/hold/cut) is correct + # regardless of the feed's ordering. + ordered = sorted( + (d for d in decisions if isinstance(d, Mapping)), + key=lambda d: str(d.get("meeting_date") or ""), + ) + + rows: list[dict[str, Any]] = [] + prev_upper: float | None = None + for entry in ordered: + meeting_date = entry.get("meeting_date") + upper = entry.get("target_upper") + lower = entry.get("target_lower") + if meeting_date is None or upper is None or lower is None: + raise SourceUnavailableError( + "Federal Reserve decision entry missing meeting_date / target bounds", + source=ECON_SOURCE_FED, + url=FED_OPENMARKET_URL, + underlying=f"entry keys={sorted(entry)}", + ) + rows.append( + _build_row( + str(meeting_date), + float(lower), + float(upper), + prev_upper=prev_upper, + retrieved_at=retrieved_at, + ) + ) + prev_upper = float(upper) + return rows + + +def parse_statement_html( + html: str, + *, + meeting_date: str, + prev_upper: float | None = None, + retrieved_at: datetime | None = None, +) -> dict[str, Any]: + """Extract a decision row from an FOMC statement HTML page (the A6 fallback). + + Parses the "target range for the federal funds rate at X to Y percent" sentence + (handling the mixed-fraction form "4-1/4 to 4-1/2"). The numeric ``value`` is + the X-Y midpoint; the categorical is derived from ``prev_upper`` when supplied. + + Raises: + SourceUnavailableError: when the page has no parseable target-range sentence + — a non-matching page RAISES rather than emitting a garbage value + (guard against silently fabricating a rate). + """ + if retrieved_at is None: + retrieved_at = datetime.now(UTC) + + match = _TARGET_RANGE_RE.search(html or "") + if not match: + raise SourceUnavailableError( + "FOMC statement page has no parseable target-range sentence " + "(non-matching page rejected, never emitted as a garbage value)", + source=ECON_SOURCE_FED, + url=FED_FOMC_STATEMENT_ROOT, + underlying="target-range regex did not match", + ) + try: + lower = _fraction_to_float(match.group(1)) + upper = _fraction_to_float(match.group(2)) + except ValueError as exc: + raise SourceUnavailableError( + "FOMC statement target-range bounds are not parseable numbers", + source=ECON_SOURCE_FED, + url=FED_FOMC_STATEMENT_ROOT, + underlying=str(exc), + ) from exc + + return _build_row( + meeting_date, + lower, + upper, + prev_upper=prev_upper, + retrieved_at=retrieved_at, + ) + + +def _get_with_retry( + client: httpx.Client, + url: str, + params: Mapping[str, Any] | None, +) -> httpx.Response: + """GET ``url`` with the shared retry discipline (transient codes backed off). + + Reuses ``mostlyright._internal._http``'s retry constants. The caller owns + ``client``. No key is involved (the Fed data is public). + """ + delay = BASE_DELAY + for attempt in range(MAX_RETRIES): + # Sanitize the transport GET (codex round-3 H1): a connection/timeout error + # must surface as the documented SourceUnavailableError, not a raw httpx + # exception — matching the other four econ fetchers. `from None` drops the + # underlying httpx exc (the Fed URL carries no key, but this normalizes the + # exception TYPE the caller catches). + try: + response = client.get(url, params=dict(params) if params else None) + except httpx.RequestError: + raise SourceUnavailableError( + "Federal Reserve request failed", + source=ECON_SOURCE_FED, + url=url, + ) from None + if response.status_code in TRANSIENT_CODES and attempt < MAX_RETRIES - 1: + log.warning( + "Federal Reserve HTTP %d (attempt %d/%d), retrying in %.1fs", + response.status_code, + attempt + 1, + MAX_RETRIES, + delay, + ) + time.sleep(delay) + delay *= 2 + continue + response.raise_for_status() + return response + raise SourceUnavailableError( # pragma: no cover + "Federal Reserve retry loop exhausted without a response", + source=ECON_SOURCE_FED, + url=url, + ) + + +def fetch_decisions( + *, + client: httpx.Client | None = None, +) -> list[dict[str, Any]]: + """Fetch per-FOMC target-rate decisions from the Federal Reserve Board. + + Sources the Board's canonical open-market rate-change record at + ``monetarypolicy/openmarket.htm`` (NOT FRED — KXFED settles to the Fed Board; + keyless, no query parameters). The year-sectioned + ``Date | Increase | Decrease | Level (%)`` tables are parsed by + :func:`parse_openmarket_html`. Each returned row carries the numeric target + midpoint (``value``), the categorical decision encoded in ``series_id`` + (``fed_funds:hike|hold|cut``), ``release_type="final"`` and + ``settlement_grade=True``. + + Semantics (live-verified 2026-07-09): the page records rate CHANGES at their + EFFECTIVE dates (a scheduled-meeting change is typically effective the business + day after the announcement); hold meetings do not appear as rows, so the + categorical derived here distinguishes hike vs cut — mapping a specific FOMC + *meeting* (including holds) to a row is the research layer's join concern. + + Args: + client: an injected ``httpx.Client`` (tests / connection reuse). When + ``None`` a fresh HTTPS client is created and closed per call. + + Returns: + Rows shaped for :func:`mostlyright.econ._schema.build_econ_dataframe`. + + Raises: + SourceUnavailableError: on an empty/hostile/redesigned page or one with no + parseable decisions (never parsed as empty data — threat T-29-16). + httpx.HTTPStatusError: on a non-transient HTTP error from the Fed. + """ + owned = client is None + active = client if client is not None else httpx.Client(timeout=HTTP_TIMEOUT) + try: + response = _get_with_retry(active, FED_OPENMARKET_URL, None) + html_text = response.text + finally: + if owned: + active.close() + + return parse_decisions(parse_openmarket_html(html_text)) + + +__all__ = [ + "FED_DECISIONS", + "FED_FOMC_STATEMENT_ROOT", + "FED_INDICATOR", + "FED_OPENMARKET_URL", + "FED_UNITS", + "decision_of", + "fetch_decisions", + "parse_decisions", + "parse_openmarket_html", + "parse_statement_html", +] diff --git a/packages/econ/src/mostlyright/econ/_fetchers/fred_alfred.py b/packages/econ/src/mostlyright/econ/_fetchers/fred_alfred.py new file mode 100644 index 00000000..c2b9f89a --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_fetchers/fred_alfred.py @@ -0,0 +1,411 @@ +"""ALFRED vintage fetcher — the canonical FIRST-PRINT store. + +ALFRED (ArchivaL Federal Reserve Economic Data) serves *as-first-released* +values through the FRED ``series/observations`` endpoint with +``realtime_start`` / ``realtime_end`` set to a wide window: one row per +``(observation date x realtime_start)``. The EARLIEST ``realtime_start`` for an +observation date is that period's FIRST PRINT — the value Kalshi settles to. + +Why ALFRED is NECESSARY (not merely convenient): the BLS timeseries API serves +the *latest-revised* series only (no as-first-released endpoint), so a BLS pull +of, say, Jun-2022 NFP returns today's post-benchmark level, not the originally +released number. ALFRED is the only machine-readable first-print source we have, +so it carries the ``settlement_grade=True`` truth. See RESEARCH Pitfall 5. + +**THE #1 LANDMINE — day-granularity.** ALFRED's ``realtime_start`` / +``realtime_end`` are calendar dates (``YYYY-MM-DD``, one-day granularity) per the +St. Louis Fed docs (``fred/realtime_period.html``). ALFRED therefore CANNOT +distinguish an 8:30 AM ET first print from a 9:45 AM ET *same-morning +correction* — both collapse into a single date-keyed vintage. The Kalshi +contract counts a correction issued before the 10:00 ET expiration, but ALFRED +alone cannot resolve it. This is a **documented, labeled limitation** for v1 +(RESEARCH RESOLUTION, open-question 2): every first-release row carries +``vintage_precision="day"`` and ``settlement_grade=True`` means "the first +date-vintage per ALFRED, which equals the first print EXCEPT when a same-morning +8:30→9:45 ET correction occurred." A release-day agency scrape (settlement-exact) +is DEFERRED to a fast-follow, out of Phase 29 scope. + +Algorithm credit: the first-release extraction (group by observation ``date``, +keep the earliest ``realtime_start``) is lifted from ``fredapi`` +(``get_series_all_releases`` / ``get_series_first_release``; +github.com/mortada/fredapi, Apache-2.0). **``fredapi`` is NOT a runtime +dependency** — the ~40 lines of httpx + groupby below reimplement the algorithm +in-house per the repo convention (all ``weather/_fetchers/`` are in-house) and +the "keyless, local-first, no hidden deps" promise. The package name appears +only in this credit; it is never imported and is not in ``pyproject.toml``. + +Security: ``FRED_API_KEY`` is read from the environment (or passed explicitly), +is placed only in the outbound query string, and is NEVER logged, persisted, or +embedded in an emitted row (threat T-29-12). HTTPS is enforced; an HTTP downgrade +is rejected before the request is issued. +""" + +from __future__ import annotations + +import logging +import os +import time +from collections.abc import Iterable, Mapping +from datetime import UTC, datetime +from typing import Any + +import httpx +from mostlyright._internal._http import ( + BASE_DELAY, + HTTP_TIMEOUT, + MAX_RETRIES, + TRANSIENT_CODES, +) +from mostlyright.core.exceptions import DataAvailabilityError, SourceUnavailableError +from mostlyright.econ._schema import ECON_SOURCE_ALFRED + +log = logging.getLogger(__name__) + +#: ALFRED / FRED observations endpoint. HTTPS only — ``api.stlouisfed.org`` +#: 301-redirects plain HTTP to HTTPS, and a key must never travel over cleartext, +#: so the fetcher pins the scheme and rejects any HTTP downgrade. +ALFRED_OBSERVATIONS_URL = "https://api.stlouisfed.org/fred/series/observations" + +#: The widest real-time window (closed, closed) — asks ALFRED for the ENTIRE +#: vintage history of the series. Matches the fredapi default. Callers narrow it +#: to bound the response when they only need recent vintages. +DEFAULT_REALTIME_START = "1776-07-04" # FRED epoch (US founding); ALFRED's floor. +DEFAULT_REALTIME_END = "9999-12-31" # ALFRED's open-ended ceiling. + +#: ALFRED sentinels for "no value this period" (e.g. a not-yet-released period). +_MISSING_VALUES = frozenset({".", "", None}) + +#: Surfaced on every ALFRED-sourced row: the vintage identity is a CALENDAR DATE, +#: so a same-morning 8:30→9:45 ET correction is NOT resolvable. A consumer reads +#: this to know ``settlement_grade=True`` is day-exact, not intraday-exact. +VINTAGE_PRECISION_DAY = "day" + + +def _resolve_key(key: str | None) -> str | None: + """Return the explicit ``key`` or fall back to ``FRED_API_KEY`` in env. + + Never logs the key. Returns ``None`` when neither is set (the caller then + degrades via :func:`fetch_vintages`'s keyless branch). + """ + if key: + return key + env = os.environ.get("FRED_API_KEY") + return env or None + + +def _parse_alfred_date(value: str) -> datetime: + """Parse an ALFRED ``YYYY-MM-DD`` date into a tz-aware UTC midnight datetime. + + ALFRED dates are calendar days (no intraday component — the day-granularity + limitation). We anchor them at 00:00:00 UTC so the value is a valid + ``timestamp_utc`` for the schema; the ``vintage_precision="day"`` flag records + that the time-of-day is NOT meaningful. + """ + return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=UTC) + + +def _period_from_obs_date(obs_date: str) -> str: + """Map an ALFRED observation ``date`` (``YYYY-MM-DD``) to a schema ``period``. + + ALFRED anchors monthly/quarterly observations at the period-start day + (``2026-05-01`` for May 2026). The schema's monthly ``period`` form is + ``YYYY-MM``, so we drop the day component. Weekly/daily series (whose day is + meaningful) keep the full ``YYYY-MM-DD`` — detected by a non-``01`` day. + """ + year, month, day = obs_date.split("-") + if day == "01": + return f"{year}-{month}" + return obs_date + + +def parse_observations( + payload: Mapping[str, Any], + *, + series_id: str, + units: str | None = None, +) -> list[dict[str, Any]]: + """Parse an ALFRED ``series/observations`` JSON payload into vintage rows. + + Each returned row carries the raw vintage identity (one row per + ``(observation date x realtime_start)``) BEFORE first-release selection: + + - ``indicator`` — left as the ``series_id`` here; the caller (29-08 resolver) + maps the upstream series id to the canonical indicator vocabulary. + - ``series_id`` — the upstream FRED/ALFRED series id. + - ``period`` — the observation period (``YYYY-MM`` monthly / ``YYYY-MM-DD``). + - ``value`` — the released numeric value (``None`` for a ``.`` sentinel). + - ``vintage_date`` / ``knowledge_time`` — the ``realtime_start`` as tz-aware + UTC (day-granular; ``vintage_precision="day"``). + - ``vintage_precision`` — always ``"day"`` for ALFRED (the #1-landmine flag). + + ``release_type`` / ``settlement_grade`` are assigned by :func:`first_release` + / :func:`all_vintages`, which see the full per-observation-date group. + + Raises: + SourceUnavailableError: if ``payload`` is not a mapping with an + ``observations`` list (a hostile / HTML-instead-of-JSON body must be + rejected, never parsed as empty — threat T-29-13). + """ + if not isinstance(payload, Mapping) or "observations" not in payload: + raise SourceUnavailableError( + "ALFRED response is not a valid observations payload", + source=ECON_SOURCE_ALFRED, + url=ALFRED_OBSERVATIONS_URL, + underlying="missing 'observations' key", + ) + observations = payload["observations"] + if not isinstance(observations, list): + raise SourceUnavailableError( + "ALFRED 'observations' is not a list", + source=ECON_SOURCE_ALFRED, + url=ALFRED_OBSERVATIONS_URL, + underlying=f"observations type={type(observations).__name__}", + ) + + rows: list[dict[str, Any]] = [] + for obs in observations: + realtime_start = obs.get("realtime_start") + obs_date = obs.get("date") + raw_value = obs.get("value") + if realtime_start is None or obs_date is None: + # A well-formed ALFRED row always carries both; skip a malformed one + # rather than fabricate a vintage. + continue + vintage_dt = _parse_alfred_date(realtime_start) + value = None if raw_value in _MISSING_VALUES else float(raw_value) + rows.append( + { + "indicator": series_id, + "series_id": series_id, + "period": _period_from_obs_date(obs_date), + "value": value, + "units": units, + "release_datetime": None, # ALFRED gives no wall-clock release time. + "vintage_date": vintage_dt, + "knowledge_time": vintage_dt, # leakage cutoff == vintage_date. + "vintage_precision": VINTAGE_PRECISION_DAY, + "source": ECON_SOURCE_ALFRED, + "retrieved_at": None, # stamped by build_econ_dataframe. + } + ) + return rows + + +def _infer_release_type(rank: int) -> str: + """Infer a ``release_type`` from a vintage's rank within its observation date. + + Rank 0 (earliest ``realtime_start``) is the first print → ``"advance"`` (the + schema's first-print vintage; kept deliberately simple per the plan — the + exact BEA advance/second/third cadence is applied by the resolver in 29-08). + Every later vintage → ``"revised"`` (ALFRED later realtime_starts ARE + revisions, which Kalshi excludes → ``settlement_grade=False``). + """ + return "advance" if rank == 0 else "revised" + + +def all_vintages(rows: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Return EVERY vintage row, ranked within each observation period. + + Keeps all rows (no dedup — the econ discipline is "keep every vintage, filter + at read time"). Within each ``period``, rows are ordered by ``vintage_date`` + ascending; the earliest carries ``release_type="advance"`` and + ``settlement_grade=True``, every later vintage ``release_type="revised"`` and + ``settlement_grade=False``. + """ + by_period: dict[str, list[dict[str, Any]]] = {} + for row in rows: + by_period.setdefault(row["period"], []).append(dict(row)) + + result: list[dict[str, Any]] = [] + for _period, group in by_period.items(): + group.sort(key=lambda r: r["vintage_date"]) + for rank, row in enumerate(group): + row["release_type"] = _infer_release_type(rank) + row["settlement_grade"] = rank == 0 + result.append(row) + return result + + +def first_release(rows: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Return one row per observation period — the FIRST PRINT (earliest vintage). + + This is the lifted ``fredapi`` first-release algorithm: group by observation + ``period``, keep the row with the EARLIEST ``vintage_date`` (== earliest + ``realtime_start``). Each returned row carries ``release_type="advance"``, + ``settlement_grade=True``, and ``vintage_precision="day"``. + + Day-granularity behavior (the documented #1 landmine): when two + ``realtime_start`` values fall on the SAME calendar date (an 8:30 print and a + 9:45 same-morning correction), both parse to the same ``vintage_date`` and + collapse to ONE first-release row. We keep the first such row encountered and + leave ``vintage_precision="day"`` set so the consumer knows the vintage is + day-exact, not intraday-exact — we do NOT silently pretend one is "the" print. + """ + by_period: dict[str, dict[str, Any]] = {} + for row in rows: + period = row["period"] + candidate = dict(row) + existing = by_period.get(period) + if existing is None or candidate["vintage_date"] < existing["vintage_date"]: + by_period[period] = candidate + + result: list[dict[str, Any]] = [] + for row in by_period.values(): + row["release_type"] = "advance" + row["settlement_grade"] = True + row["vintage_precision"] = VINTAGE_PRECISION_DAY + result.append(row) + return result + + +def _get_with_retry( + client: httpx.Client, + url: str, + params: Mapping[str, Any], +) -> httpx.Response: + """GET ``url`` with the shared retry discipline (transient codes backed off). + + Reuses ``mostlyright._internal._http``'s ``MAX_RETRIES`` / ``BASE_DELAY`` / + ``TRANSIENT_CODES`` (the repo's established retry constants) rather than + ``download_with_retry`` (which writes to a dest file — wrong shape for a JSON + GET). 404 raises immediately (permanent). The caller owns ``client``. + """ + delay = BASE_DELAY + for attempt in range(MAX_RETRIES): + try: + response = client.get(url, params=dict(params)) + except httpx.RequestError as exc: + # SECURITY (codex round-2 H6): sanitize transport errors — the request + # URL (with api_key=) is on exc.request; raise with the bare endpoint. + raise SourceUnavailableError( + f"ALFRED request failed ({type(exc).__name__})", + source=ECON_SOURCE_ALFRED, + url=url, + ) from None + if response.status_code in TRANSIENT_CODES and attempt < MAX_RETRIES - 1: + log.warning( + "ALFRED HTTP %d (attempt %d/%d), retrying in %.1fs", + response.status_code, + attempt + 1, + MAX_RETRIES, + delay, + ) + time.sleep(delay) + delay *= 2 + continue + if response.status_code != 200: + # SECURITY: the request URL carries ``api_key=`` in its + # query string; httpx's ``raise_for_status()`` embeds the FULL request + # URL in the exception message → the live key would leak into any log / + # traceback. Raise our own error with the BARE endpoint constant (no + # query string), mirroring dol.py. + raise SourceUnavailableError( + f"ALFRED returned HTTP {response.status_code}", + source=ECON_SOURCE_ALFRED, + url=url, + http_status=response.status_code, + ) + return response + # Unreachable (the loop either returns or raises), but keeps mypy happy. + raise SourceUnavailableError( # pragma: no cover + "ALFRED retry loop exhausted without a response", + source=ECON_SOURCE_ALFRED, + url=url, + ) + + +def fetch_vintages( + series_id: str, + *, + key: str | None = None, + realtime_start: str = DEFAULT_REALTIME_START, + realtime_end: str = DEFAULT_REALTIME_END, + units: str | None = None, + vintages: str = "first", + client: httpx.Client | None = None, +) -> list[dict[str, Any]]: + """Fetch ALFRED vintages for ``series_id`` and return schema-shaped rows. + + Args: + series_id: the FRED/ALFRED series id (e.g. ``"CPIAUCSL"``, ``"PAYEMS"``). + key: the FRED API key. Falls back to ``FRED_API_KEY`` in env; when + neither is present the call DEGRADES (see below) rather than hitting + ALFRED blind (ALFRED 400s without a key). Never logged / persisted. + realtime_start / realtime_end: the ALFRED real-time window (defaults span + the entire vintage history). + units: optional units label stamped on each row (schema ``units``). + vintages: ``"first"`` (default) returns the first-release (settlement-grade) + rows; ``"all"`` returns every vintage (later ones + ``settlement_grade=False``). + client: an injected ``httpx.Client`` (tests / connection reuse). When + ``None`` a fresh HTTPS client is created and closed per call. + + Returns: + Rows shaped for :func:`mostlyright.econ._schema.build_econ_dataframe`. + + Raises: + DataAvailabilityError: ``reason="source_404"`` when no key is available — + a clear, documented keyless-degradation signal so the caller can fall + back to BLS release-day provenance (NOT an unhandled crash). + SourceUnavailableError: on a hostile / non-JSON response body. + httpx.HTTPStatusError: on a non-transient HTTP error from ALFRED. + """ + resolved_key = _resolve_key(key) + if resolved_key is None: + # Keyless degradation — documented, not a silent gap. ALFRED requires a + # key (verified: 400 "api_key is not set" without one); we refuse to hit + # it blind and instead raise a signal the caller can branch on to fall + # back to the BLS release-day path. + raise DataAvailabilityError( + reason="source_404", + hint=( + "ALFRED vintages require a FRED_API_KEY; set it (free at " + "fred.stlouisfed.org/docs/api/api_key.html) or fall back to BLS " + "release-day provenance (settlement_grade=False)." + ), + source=ECON_SOURCE_ALFRED, + ) + + params = { + "series_id": series_id, + "realtime_start": realtime_start, + "realtime_end": realtime_end, + "file_type": "json", + "api_key": resolved_key, # outbound-only; never logged / stored. + } + + owned = client is None + active = client if client is not None else httpx.Client(timeout=HTTP_TIMEOUT) + try: + response = _get_with_retry(active, ALFRED_OBSERVATIONS_URL, params) + try: + payload = response.json() + except ValueError as exc: + # HTML / non-JSON body → error, never parsed as empty (T-29-13). + raise SourceUnavailableError( + "ALFRED returned a non-JSON body", + source=ECON_SOURCE_ALFRED, + url=ALFRED_OBSERVATIONS_URL, + http_status=response.status_code, + underlying=str(exc), + ) from exc + finally: + if owned: + active.close() + + parsed = parse_observations(payload, series_id=series_id, units=units) + if vintages == "all": + return all_vintages(parsed) + return first_release(parsed) + + +__all__ = [ + "ALFRED_OBSERVATIONS_URL", + "DEFAULT_REALTIME_END", + "DEFAULT_REALTIME_START", + "VINTAGE_PRECISION_DAY", + "all_vintages", + "fetch_vintages", + "first_release", + "parse_observations", +] diff --git a/packages/econ/src/mostlyright/econ/_floor.py b/packages/econ/src/mostlyright/econ/_floor.py new file mode 100644 index 00000000..6a318ab6 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_floor.py @@ -0,0 +1,121 @@ +"""FEDS-2026-010 per-series backtest floor — as DATA, with an out-of-range raise. + +The Kalshi economic markets did not all launch at once: FEDS working paper +2026-010 Table 1 records each series' FIRST Kalshi contract date (2021-2023). +A backtest request whose ``from_date`` predates a series' first contract is +asking for outcomes the venue never priced — so it must FAIL LOUDLY +(out-of-range), never silently return a partial series (ECON-16). + +This module encodes those dates ONCE, as a single auditable table +(:data:`FEDS_FIRST_CONTRACT`), and exposes :func:`assert_within_floor` which +reads the table — the dates are never re-typed inline in the checker, so the +floor stays a single source of truth. The keys use the SAME indicator +vocabulary as :mod:`mostlyright.econ._schema` (``cpi``/``cpi_yoy``/``u3``/ +``nfp``/``gdp``/``ppi``/``ppi_yoy``/``fed_funds``), plus ``fed_decision`` for the +FOMC hike/hold/cut contract (distinct from the ``fed_funds`` target-rate row — +FEDS Table 1 dates them separately). + +Depth caveat (RESEARCH Pitfall 1 / open-question 2): the FEDS floor is a +data-availability CONTRACT, not a promise that the public Kalshi API serves +those 2021-2023 contracts — the public settled-markets endpoint surfaces only +recent markets (~50-90/series, oldest ~2026-05). Deep price history needs a +running collector or the FEDS replication dataset; the floor merely refuses to +pretend a below-floor request is answerable. +""" + +from __future__ import annotations + +from datetime import date + +from mostlyright.core.exceptions import DataAvailabilityError + +#: FEDS-2026-010 Table 1 first-contract dates, keyed on the econ indicator +#: vocabulary. Each value is the calendar date of the series' FIRST Kalshi +#: contract. Monthly/weekly series use the month-start; the GDP quarterly series +#: uses the quarter-start (Q2 2021 → 2021-04-01). A request below a key's date +#: is out-of-range. +#: +#: PPI note: FEDS-2026-010 Table 1 has NO dated PPI first-contract row, so +#: ``ppi`` (KXUSPPI, PPI-MoM, Trading-Economics-settled) and ``ppi_yoy`` +#: (KXUSPPIYOY, BLS-settled) floors are DERIVED from the KXUSPPI/KXUSPPIYOY +#: Kalshi series inception — set conservatively to 2022-11-01, aligned with the +#: CPI-YoY BLS-settled first contract (the nearest FEDS-dated BLS analog). This +#: is a Kalshi-inception-derived floor, NOT a FEDS Table-1 row; tighten it if a +#: dated PPI first contract is later confirmed. +FEDS_FIRST_CONTRACT: dict[str, date] = { + # --- FEDS-2026-010 Table 1 (dated first contracts) --- + "cpi": date(2021, 6, 1), # CPI MoM — Jun 2021 + "cpi_yoy": date(2022, 11, 1), # CPI YoY — Nov 2022 + "u3": date(2021, 7, 1), # Unemployment (U3) — Jul 2021 + "nfp": date(2023, 3, 1), # Payrolls (NFP) — Mar 2023 + "gdp": date(2021, 4, 1), # GDP — Q2 2021 (quarter start = Apr 1) + "fed_funds": date(2021, 12, 1), # Fed Funds Target — Dec 2021 + "fed_decision": date(2023, 5, 1), # Fed Decision (hike/hold/cut) — May 2023 + # --- PPI: NOT a FEDS Table-1 row; KXUSPPI/KXUSPPIYOY Kalshi-inception floor --- + # Conservative, CPI-YoY-aligned (nearest FEDS-dated BLS analog). Tighten if a + # dated PPI first-contract is later confirmed. + "ppi": date(2022, 11, 1), # KXUSPPI (PPI MoM, TE-settled) — inception-derived + "ppi_yoy": date(2022, 11, 1), # KXUSPPIYOY (PPI YoY, BLS-settled) — inception-derived + # --- CPI-Core / jobless claims: NOT dated FEDS Table-1 rows either; the + # 29-08 public-surface dispatch (history/research_econ) is EXHAUSTIVE over the + # schema indicator vocabulary + SETTLEMENT_ROUTING, and the floor is checked + # FIRST for every request — so an indicator the resolver routes (KXCPICORE, + # KXJOBLESSCLAIMS) MUST have a floor or the floor guard structurally rejects a + # legitimate released-window request before dispatch (the exact reason + # jobless_claims could not emit settlement rows — BLOCKER-3). These floors are + # inception-derived, conservatively aligned to the nearest FEDS-dated BLS + # analog (CPI 2021-06 for CPI-Core; the DOL weekly series' Kalshi inception for + # jobless claims). Tighten if a dated first-contract is later confirmed. --- + "cpi_core": date(2021, 6, 1), # KXCPICORE — CPI-analog inception floor + "cpi_core_yoy": date(2022, 11, 1), # KXCPICOREYOY — CPI-YoY-analog inception floor + "jobless_claims": date( + 2022, 1, 1 + ), # KXJOBLESSCLAIMS — DOL-weekly Kalshi inception (conservative) +} + + +def assert_within_floor(indicator: str, from_date: date) -> None: + """Raise if ``from_date`` predates ``indicator``'s FEDS first-contract floor. + + Reads the floor from :data:`FEDS_FIRST_CONTRACT` (the dates live there once, + never inline here). Returns ``None`` when ``from_date`` is at or after the + floor. + + Args: + indicator: An econ indicator key (must be a key of + :data:`FEDS_FIRST_CONTRACT`). + from_date: The inclusive start of the requested backtest range. + + Raises: + DataAvailabilityError: with ``reason="out_of_window"`` when ``indicator`` + is unknown (never silently passes) OR when ``from_date`` is below the + indicator's floor. The ``hint`` names the indicator and the floor + date so callers (and the MCP error payload) can act on it. + """ + floor = FEDS_FIRST_CONTRACT.get(indicator) + if floor is None: + known = ", ".join(sorted(FEDS_FIRST_CONTRACT)) + raise DataAvailabilityError( + reason="out_of_window", + hint=( + f"unknown econ indicator {indicator!r}; no FEDS first-contract " + f"floor is defined for it. Known indicators: {known}" + ), + ) + if from_date < floor: + raise DataAvailabilityError( + reason="out_of_window", + hint=( + f"{indicator}: requested from_date {from_date.isoformat()} is " + f"below the FEDS-2026-010 first-contract floor " + f"{floor.isoformat()}; the venue never priced contracts before " + f"that date, so this window is out of range." + ), + ) + return None + + +__all__ = [ + "FEDS_FIRST_CONTRACT", + "assert_within_floor", +] diff --git a/packages/econ/src/mostlyright/econ/_history.py b/packages/econ/src/mostlyright/econ/_history.py new file mode 100644 index 00000000..abc956df --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_history.py @@ -0,0 +1,1044 @@ +"""``econ.history`` — read-time vintage filter, FEDS-floor-enforced, fetch-on-miss. + +``history(indicator, from_date, to_date, *, vintages="settlement"|"all")`` is the +econ vertical's core read surface (plan 29-08, replacing the 29-01 stub). It +composes everything upstream: + +1. **FEDS floor first** — :func:`~mostlyright.econ._floor.assert_within_floor` + raises the out-of-range error when ``from_date`` predates the indicator's + FEDS-2026-010 first Kalshi contract (a below-floor backtest asks for outcomes + the venue never priced). +2. **Cache read** — :func:`~mostlyright.econ._cache.read_econ_window` returns the + persisted vintages for the window. On a MISS the indicator is dispatched to + its agency fetcher (:data:`INDICATOR_FETCHERS`), the rows are persisted via + :func:`~mostlyright.econ._cache.write_econ_cache`, and the window is re-read + (so the SECOND call for the same window reads from cache with no fetch). +3. **Read-time vintage filter** — ``"settlement"`` keeps only the settlement-grade + first print (``settlement_grade is True``), ``"all"`` keeps every vintage. This + is the load-bearing econ difference from weather: the store keeps ALL vintages + and the caller selects one at READ time (never a merge collapse). +4. **Not-yet-released is an error, never empty** — when the window yields no + relevant rows, :class:`~mostlyright.core.exceptions.IndicatorNotYetReleasedError` + is raised (with the release calendar's expected datetime when known). ``history`` + NEVER returns ``[]``/``None`` for a missing release. + +**The dispatch table is EXHAUSTIVE over the 6 CONTEXT Area-1 families** — every +indicator in the ``schema.econ.observations.v1`` vocabulary has a code path; an +indicator with no entry raises a clear ``ValueError`` (never a silent empty +frame). The load-bearing entries the checker flagged: + +- ``ppi`` / ``ppi_yoy`` → ``bls`` (BLOCKER-1: PPI final-demand ``WPSFD4`` is now + fetchable through the public surface). +- ``jobless_claims`` → ``dol`` (BLOCKER-3: the keyed ALFRED-ICSA first-release + path emits ``settlement_grade=True``, so a released jobless-claims window + returns non-empty settlement rows; only a genuinely future/unreleased week + raises ``IndicatorNotYetReleasedError``). ``jobless_claims`` is NEVER + special-cased to always-empty / always-False — that is the exact bug the + checker flagged. + +pandas is the ``[pandas]`` extra (``history`` returns a DataFrame): it is +lazy-imported inside the function with an actionable install hint, mirroring the +weather / markets pandas-guard discipline. +""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from datetime import UTC, date, datetime, timedelta +from typing import TYPE_CHECKING, Any + +from mostlyright.core.exceptions import ( + IndicatorNotYetReleasedError, + SourceUnavailableError, +) + +from ._cache import ( + is_window_covered, + mark_window_covered, + read_econ_window, + write_econ_cache, +) +from ._floor import assert_within_floor + +if TYPE_CHECKING: + import pandas as pd + +__all__ = [ + "FRED_SERIES_BY_INDICATOR", + "INDICATOR_FETCHERS", + "history", + "series", +] + +#: The permitted ``vintages`` argument values. +_VINTAGE_MODES: frozenset[str] = frozenset({"settlement", "all"}) + +# --- Source-identity contract (docs/source-identity.md §1/§2) ----------------- +# The econ vertical conforms to the cross-vertical source-identity kwarg contract +# (Phase 30/31, v1.13/1.14). ``source=`` is ALWAYS provenance — WHO produced the +# row — and ``delivery=`` is ALWAYS where the computation runs (local | hosted). + +#: The accepted ``source=`` provenance authorities (contract §1). ``None`` (the +#: default) is the vertical's per-indicator default routing; a pin names one of +#: these agencies. This is the contract's own "FRED for macro series" +#: degenerate-axis example made pinnable — the axis stays ready for a future +#: provider pin (e.g. an agency-scrape vs the ALFRED vintage store) without an +#: API change. ``"fred"`` is the ALFRED realtime vintage store; ``"bls"`` the BLS +#: timeseries API; ``"bea"`` GDP; ``"dol"`` initial jobless claims; ``"fed"`` the +#: Federal Reserve Board FOMC decisions (NOT FRED). +_VALID_ECON_SOURCES: frozenset[str] = frozenset({"fred", "bls", "bea", "dol", "fed"}) + +#: The accepted ``delivery=`` values (contract §2). ``"live"`` (default) runs the +#: computation locally against the public agency APIs; ``"hosted"`` is the +#: opt-in precomputed-API seam (reserved — arrives in the hosted-econ phase). +_VALID_DELIVERIES: frozenset[str] = frozenset({"live", "hosted"}) + +#: The env var + key the hosted seam will read once the hosted-econ phase lands +#: (satellite Phase-25 ``delivery=`` precedent). Named in the reserved-seam raise. +_ECON_HOSTED_URL_ENV = "ECON_HOSTED_URL" +_ECON_HOSTED_KEY_ENV = "MOSTLYRIGHT_API_KEY" + +#: Per-indicator authority map: which ``source=`` pins can serve each indicator. +#: A pin NOT in an indicator's set is a loud pre-network ``ValueError`` (never a +#: silent fallback to a different provider — contract §1). The BLS-family +#: indicators are served by both the BLS timeseries API (``bls``, latest-revised) +#: and the ALFRED vintage store (``fred``, settlement-grade first print, keyed); +#: GDP by ``bea``; jobless claims by ``dol`` + the ALFRED-ICSA first print +#: (``fred``); the Fed decision ONLY by the Federal Reserve Board (``fed``), never +#: FRED. This map mirrors the real fetcher wiring in :data:`INDICATOR_FETCHERS` +#: (the pin does not rewire the engine — it validates the provenance the caller +#: asked for is a legitimate authority for the indicator, then the existing +#: dispatch runs). +_INDICATOR_SOURCE_AUTHORITIES: dict[str, frozenset[str]] = { + "cpi": frozenset({"bls", "fred"}), + "cpi_core": frozenset({"bls", "fred"}), + "cpi_yoy": frozenset({"bls", "fred"}), + "cpi_core_yoy": frozenset({"bls", "fred"}), + "nfp": frozenset({"bls", "fred"}), + "u3": frozenset({"bls", "fred"}), + "ppi": frozenset({"bls", "fred"}), + "ppi_yoy": frozenset({"bls", "fred"}), + # GDP: BEA latest-revised (keyless) OR the ALFRED growth-rate first print + # (keyed, A191RL1Q225SBEA advance vintage). + "gdp": frozenset({"bea", "fred"}), + "jobless_claims": frozenset({"dol", "fred"}), + "fed_funds": frozenset({"fed"}), + "fed_decision": frozenset({"fed"}), +} + + +def _validate_source_and_delivery(indicator: str, source: str | None, delivery: str) -> None: + """Loud pre-network validation of the source-identity kwargs (§1/§2). + + Runs BEFORE any cache read / fetcher dispatch so an invalid pin never touches + the network: + + - ``delivery`` must be one of :data:`_VALID_DELIVERIES`; an unknown value is a + ``ValueError`` naming the accepted set. ``"hosted"`` is the reserved + precomputed-API seam — it raises :class:`SourceUnavailableError` naming + ``ECON_HOSTED_URL`` + ``MOSTLYRIGHT_API_KEY`` (byte-identical rows live vs + hosted when the hosted-econ phase lands). + - ``source`` (when not ``None``) must be one of :data:`_VALID_ECON_SOURCES`; + an unknown provenance is a ``ValueError`` naming the accepted set. A VALID + authority that cannot serve THIS indicator (e.g. ``source="bea"`` for + ``"cpi"``) is ALSO a loud ``ValueError`` — never a silent fallback to a + different provider (the contract's "loud typed error, never a silent + fallback" rule). + + Raises: + ValueError: unknown ``delivery``; unknown ``source``; or a valid ``source`` + authority that cannot serve ``indicator``. + SourceUnavailableError: ``delivery="hosted"`` (the reserved seam). + """ + # delivery FIRST — an unknown value is a config error regardless of source. + if delivery not in _VALID_DELIVERIES: + raise ValueError(f"delivery must be one of {sorted(_VALID_DELIVERIES)}; got {delivery!r}") + if delivery == "hosted": + raise SourceUnavailableError( + "econ delivery='hosted' is a reserved seam — arrives in the hosted-econ " + f"phase. Set {_ECON_HOSTED_URL_ENV} + {_ECON_HOSTED_KEY_ENV} to reach the " + "opt-in precomputed econ API once it ships (rows are byte-identical to the " + "local 'live' path). Use delivery='live' (the default) today.", + source="econ.hosted", + ) + + if source is None: + return # default per-indicator routing — unchanged behavior. + + if source not in _VALID_ECON_SOURCES: + raise ValueError( + f"source must be one of {sorted(_VALID_ECON_SOURCES)} or None; got {source!r}" + ) + + authorities = _INDICATOR_SOURCE_AUTHORITIES.get(indicator) + if authorities is not None and source not in authorities: + raise ValueError( + f"source={source!r} cannot serve indicator {indicator!r}; the authorities " + f"for {indicator!r} are {sorted(authorities)}. A pin that cannot serve the " + "indicator raises loudly — it never silently falls back to a different " + "provider (source-identity contract §1)." + ) + + +# Type alias for a dispatch callable: given the requested window it returns +# schema.econ.observations.v1-shaped rows (list of dicts). The default entries +# below wrap the real agency fetchers; tests monkeypatch either the dict entry or +# the underlying fetcher module. +FetchFn = Callable[..., list[dict[str, Any]]] # (from, to, *, source=None) -> rows + + +def _require_pandas() -> Any: + """Lazy-import pandas with an actionable install hint on miss. + + ``history`` returns a DataFrame, but pandas is the ``[pandas]`` extra (a + caller who only touches raw fetchers/cache should not pay for it). Mirrors the + weather/markets guard: a clear "install mostlyrightmd-econ[pandas]" hint. + """ + try: + import pandas as _pandas + except ImportError as exc: # pragma: no cover - exercised only without pandas + raise ImportError( + "econ.history requires pandas. Install with: pip install mostlyrightmd-econ[pandas]" + ) from exc + return _pandas + + +def _year_of(value: date | datetime) -> int: + return value.year + + +# --- FRED/ALFRED first-print wiring for the BLS family ----------------------- +# The BLS timeseries API (``bls.fetch``) serves ONLY the latest-revised value +# (``settlement_grade=False``) — there is no as-first-released endpoint. The +# settlement-grade FIRST PRINT (the value a Kalshi contract settles to) comes from +# the ALFRED realtime vintage store, keyed by ``FRED_API_KEY``. This map resolves +# each canonical BLS-family indicator to its FRED/ALFRED series id so +# ``_fetch_bls_indicator`` can pull the first print, mirroring the +# ``dol.fetch_initial_claims`` keyed/keyless pattern that already works for +# jobless_claims (the ALFRED fetcher was previously orphaned from this read path — +# the 29-VERIFICATION BLOCKER). +# +# LIVE-VERIFIED 2026-07-10 against api.stlouisfed.org (each id returns ALFRED +# realtime vintages whose EARLIEST realtime_start is the settlement first print). +# NOTE: the BLS PPI id ``WPSFD4`` does NOT exist on FRED — the FRED final-demand +# PPI series is ``PPIFIS`` ("Producer Price Index by Commodity: Final Demand"). +# Concrete stakes: PPIFIS Jan-2025 first print 147.716 (vintage 2025-02-13) vs a +# later revision 147.975 — the first print and the latest-revised value genuinely +# DIFFER, which is exactly why the settlement grade must come from ALFRED, not BLS. +FRED_SERIES_BY_INDICATOR: dict[str, str] = { + "cpi": "CPIAUCSL", + "cpi_core": "CPILFESL", + "nfp": "PAYEMS", + "u3": "UNRATE", + "ppi": "PPIFIS", + # GDP settles on the ANNUALIZED GROWTH RATE (KXGDP = "US GDP growth"), NOT the + # level — so the FRED id is A191RL1Q225SBEA ("Real GDP, Percent Change from + # Preceding Period"), whose ALFRED realtime vintages ARE the advance/second/ + # third estimates. LIVE-VERIFIED 2026-07-11: 2025Q1 advance = -0.3% (release + # 2025-04-30) -> -0.2% -> -0.5% -> -0.6%. Using the GDPC1 LEVEL here would emit + # the wrong settlement number (~23500 bn instead of -0.3%) — the GDP analog of + # the YoY index-level bug. + "gdp": "A191RL1Q225SBEA", +} + +#: Units per BLS-family indicator, stamped on the ALFRED first-print rows (the +#: ALFRED parser leaves ``units`` to the caller). Mirrors ``bls._INDICATOR_UNITS``. +_BLS_FAMILY_UNITS: dict[str, str] = { + "cpi": "index", + "cpi_core": "index", + "ppi": "index", + "u3": "percent", + "nfp": "thousands_persons", + "gdp": "percent", # annualized real GDP growth rate (A191RL1Q225SBEA) +} + +#: Indicators whose settlement-grade first print is unlocked by ``FRED_API_KEY`` +#: (the ALFRED store): the whole BLS family (incl. the YoY variants that remap to a +#: base series), jobless_claims (ALFRED-ICSA), and GDP (ALFRED A191RL1Q225SBEA +#: growth-rate vintages — the advance estimate is the first realtime vintage). A +#: keyless settlement request for one of these raises IndicatorNotYetReleasedError +#: naming FRED_API_KEY as the unlock. Only Fed is absent (its settlement grade +#: comes from the Federal Reserve Board, not FRED, so a FRED_API_KEY hint would +#: mislead). +_FRED_UNLOCKABLE_INDICATORS: frozenset[str] = frozenset( + { + "cpi", + "cpi_core", + "cpi_yoy", + "cpi_core_yoy", + "nfp", + "u3", + "ppi", + "ppi_yoy", + "gdp", + "jobless_claims", + } +) + + +def _resolve_fred_key() -> str | None: + """Resolve a FRED key from ``FRED_API_KEY`` for the ALFRED first-print path. + + ``series`` has no key kwarg, so the environment is the only source here + (mirrors the ``dol`` / ``fred_alfred`` ``_resolve_key`` seam). ``None`` → the + BLS-family read degrades to the BLS latest-revised path + (``settlement_grade=False``) and the settlement filter then raises naming + ``FRED_API_KEY``. A dedicated module-level function so a test can force the + keyed branch offline without a real key (and without touching env). Never logs + the key. + """ + env = os.environ.get("FRED_API_KEY") + return env or None + + +# --- Default dispatch bodies (real agency fetchers) -------------------------- +# Each wraps its agency fetcher and normalizes to a (from_date, to_date) -> rows +# signature. The bodies are exercised live (29-10 smoke); the unit tests mock the +# fetcher layer, so these degrade gracefully (keyless ALFRED/BEA raise a +# documented DataAvailabilityError the caller can branch on). + + +def _window_filter( + rows: list[dict[str, Any]], from_date: date | datetime, to_date: date | datetime +) -> list[dict[str, Any]]: + """Keep rows whose ``vintage_date`` falls in inclusive ``[from_date, to_date]``. + + ALFRED returns the ENTIRE vintage history for a series; this bounds the first + prints to the requested window (matching the vintage-date partition grain the + per-release cache read uses) so a keyed call persists only the in-window first + prints. A row with a non-datetime ``vintage_date`` is dropped defensively. + """ + start = _coerce_floor_date(from_date) + end = _coerce_floor_date(to_date) + out: list[dict[str, Any]] = [] + for row in rows: + vintage = row.get("vintage_date") + if not isinstance(vintage, datetime): + continue + if start <= vintage.date() <= end: + out.append(row) + return out + + +def _alfred_first_prints( + fred_series: str, + *, + units: str | None, + from_date: date | datetime, + to_date: date | datetime, +) -> list[dict[str, Any]]: + """Fetch ALFRED first-print vintages for ``fred_series`` (settlement-grade). + + The keyed ALFRED branch (BLS family + GDP): pulls ALL ALFRED realtime vintages + via :func:`mostlyright.econ._fetchers.fred_alfred.fetch_vintages` + (``vintages="all"`` → the first realtime vintage per period carries + ``settlement_grade=True`` and each later revision ``False``), window-filtered to + ``[from_date, to_date]``. + + Fetching ``"all"`` (not ``"first"``) is load-bearing for two reasons: (1) the + cache is then COMPLETE, so ``series(..., vintages="all")`` returns every vintage + — the read-time :func:`_filter_vintages` selects settlement (grade=True first + print) vs all (fixing the audit's "vintages='all' returns only first prints" + gap); (2) a later ``vintages="all"`` call after a ``settlement`` call reads a + full cache rather than the first-prints-only subset. The persisted rows are + re-stamped to the requested indicator by the cache write, so the upstream + ``series_id`` (``"CPIAUCSL"`` …) survives while ``indicator`` becomes canonical. + """ + from ._fetchers import fred_alfred + + rows = fred_alfred.fetch_vintages(fred_series, units=units, vintages="all") + return _window_filter(rows, from_date, to_date) + + +def _fetch_bls_indicator( + from_date: date | datetime, + to_date: date | datetime, + *, + indicator: str, + source: str | None = None, +) -> list[dict[str, Any]]: + """Fetch a BLS-family indicator, preferring the ALFRED first print when keyed. + + CPI/Core/NFP/U3/PPI (and the YoY variants that remap here) live in the BLS + timeseries API as latest-revised (``settlement_grade=False``); the + settlement-grade FIRST PRINT comes from ALFRED. Mirrors + ``dol.fetch_initial_claims``: + + - **Keyed** (``FRED_API_KEY`` in env) → the ALFRED realtime first prints for + :data:`FRED_SERIES_BY_INDICATOR` ``[indicator]`` (``settlement_grade=True``), + window-filtered. This is the ONLY source of the settlement grade for the BLS + family — previously ALFRED was orphaned and never reached from here (the + 29-VERIFICATION BLOCKER). + - **Keyless** → the existing ``bls.fetch`` latest-revised rows + (``settlement_grade=False``); the read-time settlement filter in + :func:`series` then raises naming ``FRED_API_KEY``. + """ + fred_series = FRED_SERIES_BY_INDICATOR.get(indicator) + has_key = _resolve_fred_key() is not None + + # source= routing (contract §1): 'fred' forces ALFRED (raise if no key — never + # a silent BLS degrade); 'bls' forces BLS even when keyed; None = ALFRED when + # keyed else BLS. + if source == "fred": + if fred_series is None or not has_key: + raise SourceUnavailableError( + f"source='fred' for {indicator!r} needs the ALFRED vintage store — " + "set FRED_API_KEY (free: https://fred.stlouisfed.org/docs/api/api_key.html)", + source="alfred", + url="https://api.stlouisfed.org/fred", + ) + use_alfred = True + elif source == "bls": + use_alfred = False + else: + use_alfred = fred_series is not None and has_key + + if use_alfred and fred_series is not None: + # ALFRED first print (settlement_grade=True on the first realtime vintage). + return _alfred_first_prints( + fred_series, + units=_BLS_FAMILY_UNITS.get(indicator), + from_date=from_date, + to_date=to_date, + ) + + # BLS latest-revised (settlement_grade=False). Resolve the BLS series id(s) for + # this indicator from the BLS_SERIES map so the request targets the right + # database (WP-prefixed PPI, CU-prefixed CPI). + from ._fetchers import bls as _bls + + series_ids = [sid for sid, ind in _bls.BLS_SERIES.items() if ind == indicator] + if not series_ids: + # No BLS series maps to this indicator — a dispatch/config drift signal. + raise ValueError( + f"no BLS series id registered for indicator {indicator!r}; " + "the BLS_SERIES map and the history dispatch are out of sync." + ) + # Window-filter (codex round-2 H5): the BLS fetch is year-grained, so a + # sub-year or pinned (source="bls") request would otherwise return the whole + # year. _restamp_keyless gives period-derived vintage_dates, so the same + # vintage-date window the ALFRED path uses applies here. + return _window_filter( + _restamp_keyless( + _bls.fetch( + series_ids, + start_year=_year_of(from_date), + end_year=_year_of(to_date), + ) + ), + from_date, + to_date, + ) + + +def _fetch_gdp( + from_date: date | datetime, to_date: date | datetime, *, source: str | None = None +) -> list[dict[str, Any]]: + """Fetch GDP growth: keyed -> ALFRED first prints; keyless -> BEA latest-revised. + + KXGDP settles on the ANNUALIZED REAL GDP GROWTH RATE. The settlement first print + (the advance estimate) is the earliest ALFRED realtime vintage of + ``A191RL1Q225SBEA`` — with a REAL vintage_date (the release day, ~quarter-end + + 1mo), so it lands in the correct cache partition. This is the SAME unified + first-print path the BLS family uses; routing GDP through BEA's GetData instead + stamped vintage_date=now and misfiled every historical quarter out of the read + window (the 29-13 BATCH-A blocker). BEA remains the keyless latest-revised + fallback (settlement_grade=False). + """ + fred_series = FRED_SERIES_BY_INDICATOR.get("gdp") + has_key = _resolve_fred_key() is not None + + # source= routing: 'fred' forces ALFRED (raise if no key); 'bea' forces BEA + # latest-revised even when keyed; None = ALFRED when keyed else BEA. + if source == "fred": + if fred_series is None or not has_key: + raise SourceUnavailableError( + "source='fred' for 'gdp' needs the ALFRED vintage store — set " + "FRED_API_KEY (free: https://fred.stlouisfed.org/docs/api/api_key.html)", + source="alfred", + url="https://api.stlouisfed.org/fred", + ) + use_alfred = True + elif source == "bea": + use_alfred = False + else: + use_alfred = fred_series is not None and has_key + + if use_alfred and fred_series is not None: + rows = _alfred_first_prints( + fred_series, + units=_BLS_FAMILY_UNITS.get("gdp"), + from_date=from_date, + to_date=to_date, + ) + # GDP is QUARTERLY: ALFRED anchors quarters at the period-start month + # (2026-04-01 = Q2), which _period_from_obs_date renders as "2026-04". The + # schema's GDP period convention is "YYYYQ#" (BEA's native TimePeriod), so a + # keyed ALFRED GDP row must match the keyless BEA row for the same quarter + # (codex-review finding: else cross-source joins on `period` break). + return [{**r, "period": _monthly_to_quarter(str(r.get("period")))} for r in rows] + + from ._fetchers import bea as _bea + + # Window-filter (codex round-2 H5): BEA GetData is year-grained. _restamp_keyless + # gives the quarterly rows a period-derived vintage_date, so the vintage-date + # window bounds a sub-year / pinned (source="bea") GDP request. + return _window_filter( + _restamp_keyless(_bea.fetch_gdp(year_range=(_year_of(from_date), _year_of(to_date)))), + from_date, + to_date, + ) + + +def _fetch_jobless_claims( + from_date: date | datetime, to_date: date | datetime, *, source: str | None = None +) -> list[dict[str, Any]]: + """Fetch weekly initial jobless claims (keyed ALFRED-ICSA → first print). + + BLOCKER-3: the keyed path routes the first print through ALFRED ICSA so a + released week carries ``settlement_grade=True``; keyless degrades to + latest-revised (``settlement_grade=False``). Either way a released window is + non-empty — never special-cased to empty. + """ + from ._fetchers import dol as _dol + + # Window-filter (codex round-2 C3): jobless_claims is FRED-derived (dol.icsa), + # so it takes the D-LIC-2 fetch-through branch that bypasses read_econ_window's + # window filter — dol returns the FULL ICSA history, so it MUST be windowed here + # (the exact bug fixed on the TS side; the Python side had it too). + return _window_filter(_dol.fetch_initial_claims(), from_date, to_date) + + +def _fetch_fed( + from_date: date | datetime, to_date: date | datetime, *, source: str | None = None +) -> list[dict[str, Any]]: + """Fetch FOMC target-rate decisions (Federal Reserve Board, NOT FRED).""" + from ._fetchers import fed as _fed + + # Window-filter (codex round-2 H5): fetch_decisions parses the ENTIRE FOMC + # rate-change page; bound it to the requested window by vintage_date (the + # meeting date) so a sub-range / pinned (source="fed") request isn't the whole + # history. + return _window_filter(_fed.fetch_decisions(), from_date, to_date) + + +#: Base level indicator each YoY contract derives its 12-month %-change from. +_YOY_BASE_INDICATOR: dict[str, str] = { + "cpi_yoy": "cpi", + "cpi_core_yoy": "cpi_core", + "ppi_yoy": "ppi", +} + + +def _monthly_to_quarter(period: str) -> str: + """Convert a quarter-start monthly period (``"2026-04"``) to ``"2026Q2"``. + + Leaves already-quarterly (``"2026Q2"``) or non-monthly forms unchanged. + """ + if "Q" in period: + return period + parts = period.split("-") + if len(parts) != 2: + return period + try: + year, month = int(parts[0]), int(parts[1]) + except ValueError: + return period + return f"{year:04d}Q{(month - 1) // 3 + 1}" + + +def _fetch_yoy( + from_date: date | datetime, + to_date: date | datetime, + *, + yoy_indicator: str, + source: str | None = None, +) -> list[dict[str, Any]]: + """Fetch a YoY indicator as the true 12-month percent change (finding 9). + + Widens the base-index fetch back 12 months (so each in-window period has its + year-ago counterpart), computes YoY from the base FIRST-PRINT vintages via + :func:`mostlyright.econ._yoy.compute_yoy`, then window-filters the YoY rows to + ``[from_date, to_date]`` by ``vintage_date`` (the YoY becomes knowable when the + current period's index first print is released). Returns YoY rows carrying + ``value`` = the percent change, ``units="percent"`` — never the raw index level. + """ + from ._yoy import compute_yoy + + base_indicator = _YOY_BASE_INDICATOR[yoy_indicator] + extended_from = _add_months(_coerce_floor_date(from_date), -12) + base_rows = _fetch_bls_indicator( + extended_from, to_date, indicator=base_indicator, source=source + ) + yoy_rows = compute_yoy(base_rows, indicator=yoy_indicator) + return _window_filter(yoy_rows, from_date, to_date) + + +#: The EXHAUSTIVE indicator → fetcher dispatch table (CONTEXT Area-1, all 6 +#: families). An indicator absent from this map raises a clear ``ValueError`` in +#: :func:`history` (never a silent empty frame). ``ppi``/``ppi_yoy`` → bls closes +#: BLOCKER-1; ``jobless_claims`` → dol closes BLOCKER-3. +INDICATOR_FETCHERS: dict[str, FetchFn] = { + # --- BLS family (latest-revised via BLS; first-print via ALFRED) --- + "cpi": lambda a, b, *, source=None: _fetch_bls_indicator(a, b, source=source, indicator="cpi"), + "cpi_core": lambda a, b, *, source=None: _fetch_bls_indicator( + a, b, source=source, indicator="cpi_core" + ), + # cpi_yoy / cpi_core_yoy serve their BASE level series (like ppi_yoy → ppi); + # the YoY math is NOT computed here — the dispatch remaps to a resolvable + # series so the settlement read no longer raises "BLS_SERIES out of sync" + # (cpi_yoy) or "no fetcher registered" (cpi_core_yoy was absent entirely). + "cpi_yoy": lambda a, b, *, source=None: _fetch_yoy( + a, b, yoy_indicator="cpi_yoy", source=source + ), + "cpi_core_yoy": lambda a, b, *, source=None: _fetch_yoy( + a, b, yoy_indicator="cpi_core_yoy", source=source + ), + "nfp": lambda a, b, *, source=None: _fetch_bls_indicator(a, b, source=source, indicator="nfp"), + "u3": lambda a, b, *, source=None: _fetch_bls_indicator(a, b, source=source, indicator="u3"), + # --- PPI → bls (BLOCKER-1 fix: PPI final-demand WPSFD4 is reachable) --- + "ppi": lambda a, b, *, source=None: _fetch_bls_indicator(a, b, source=source, indicator="ppi"), + "ppi_yoy": lambda a, b, *, source=None: _fetch_yoy( + a, b, yoy_indicator="ppi_yoy", source=source + ), + # --- GDP → bea --- + "gdp": lambda a, b, *, source=None: _fetch_gdp(a, b, source=source), + # --- jobless claims → dol (BLOCKER-3: keyed ALFRED-ICSA first-release) --- + "jobless_claims": lambda a, b, *, source=None: _fetch_jobless_claims(a, b, source=source), + # --- Fed decision → fed (Federal Reserve Board) --- + "fed_funds": lambda a, b, *, source=None: _fetch_fed(a, b, source=source), + "fed_decision": lambda a, b, *, source=None: _fetch_fed(a, b, source=source), +} + + +def _coerce_input_date(value: str | date | datetime) -> date | datetime: + """Coerce a public-API date argument to a ``date``/``datetime`` object. + + The econ read surface documents string inputs (``"2025-01-01"``) as the + primary usage, matching the core verticals (``research()``, ``obs()`` accept + ISO strings). ``date``/``datetime`` objects pass through unchanged — the + downstream floor/cache/fetch/window pipeline handles both. An ISO date string + becomes a ``date``; a full ISO datetime string becomes a ``datetime``. This is + applied ONCE at each public entry (``series``/``snapshot``) so every + downstream helper receives a real temporal object — never a bare ``str`` + (which silently broke every string-input call: ``str < date`` TypeError). + + Raises: + ValueError: the string is not ISO-8601 (``date.fromisoformat`` / + ``datetime.fromisoformat`` both reject it) — loud, at the boundary. + """ + if not isinstance(value, str): + return value + try: + return date.fromisoformat(value) + except ValueError: + # Accept a full ISO datetime string too (e.g. "2025-01-01T00:00:00Z"). + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _coerce_floor_date(value: date | datetime) -> date: + """Return a plain ``date`` for the FEDS floor comparison.""" + if isinstance(value, datetime): + return value.date() + return value + + +def _add_months(d: date, n: int) -> date: + """Return ``d`` shifted ``n`` months to the first of that month.""" + m = d.month - 1 + n + return date(d.year + m // 12, m % 12 + 1, 1) + + +def _derive_release_vintage(indicator: str, period: str) -> datetime | None: + """Real release datetime for a KEYLESS latest-revised row's ``period``. + + Keyless rows carry the latest-REVISED value (``settlement_grade=False``); the + fetchers otherwise stamp ``vintage_date=now`` — which misfiles every historical + period OUT of the vintage-date-partitioned read window (the 29-13 keyless + blocker: a released period looked "not yet released"). + + The vintage is resolved in TWO tiers: + + 1. **Curated release calendar FIRST (codex round-3 C1).** Consult + :func:`~mostlyright.econ._releases.releases` for ``indicator`` and, when a + scheduled :class:`~mostlyright.econ._releases.ReleaseEvent` matches the + (normalized) ``period``, return its EXACT ``release_datetime`` (gdp + ``2026Q2`` → ``2026-07-30 12:30 UTC``). The old heuristic 1st-of-release-month + stamped a date BEFORE the real release — a LOOK-AHEAD leak: a window ending + mid-month would wrongly surface a period whose print has not actually landed. + 2. **Heuristic fallback for out-of-table periods.** The curated table is + PARTIAL (only the near-term months/quarters), so a period it does not cover + falls back to ≈ one month after the period (monthly) / quarter-end + (quarterly) — preserving the 29-13 partition-landing for those rows. + + This is deliberately NOT used for settlement-grade rows (those get ALFRED's + real ``realtime_start``). Returns ``None`` for an unparseable period (the caller + leaves the row's existing vintage_date untouched rather than guess). + """ + period = (period or "").strip() + + # (1) Curated calendar FIRST — the exact scheduled release, no look-ahead. A + # monthly period is normalized to "YYYY-MM" (any day component dropped); a + # quarterly "YYYYQ#" is matched as-is. An unknown indicator / empty period + # yields no schedule (ValueError/TypeError) and falls through to the heuristic. + normalized = period if "Q" in period else "-".join(period.split("-")[:2]) + try: + from ._releases import releases as _releases + + schedule = _releases(indicator) + except (ValueError, TypeError): + schedule = [] + for event in schedule: + if event.period == normalized: + return event.release_datetime + + # (2) Heuristic fallback — out-of-table periods only. + try: + if "Q" in period: # quarterly "YYYYQ%d" + year_s, q_s = period.split("Q") + end_month = int(q_s) * 3 # Q1->Mar(3) … Q4->Dec(12) + rel = _add_months(date(int(year_s), end_month, 1), 1) + else: # monthly "YYYY-MM" (ignore any day component) + year_s, month_s = period.split("-")[:2] + rel = _add_months(date(int(year_s), int(month_s), 1), 1) + except (ValueError, IndexError): + return None + return datetime(rel.year, rel.month, 1, tzinfo=UTC) + + +def _restamp_keyless(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Re-stamp keyless latest-revised rows with a period-derived vintage_date. + + Forces ``settlement_grade=False`` (a keyless row is NEVER the settlement first + print — BEA's GetData path mislabeled it True) and replaces ``vintage_date`` / + ``knowledge_time`` with :func:`_derive_release_vintage` so a historical read + window returns the row instead of misfiling it into the current-month + partition. Rows are copied (never mutated in place). + """ + out: list[dict[str, Any]] = [] + for row in rows: + vintage = _derive_release_vintage( + str(row.get("indicator") or ""), str(row.get("period") or "") + ) + new = dict(row) + new["settlement_grade"] = False + if vintage is not None: + new["vintage_date"] = vintage + new["knowledge_time"] = vintage + out.append(new) + return out + + +def _filter_vintages(rows: list[dict[str, Any]], vintages: str) -> list[dict[str, Any]]: + """Apply the read-time vintage filter. + + ``"settlement"`` keeps only rows whose ``settlement_grade`` is truthy (the + first-print settlement truth); ``"all"`` keeps every vintage. ``settlement`` is + always a subset of ``all`` — the clean-partition property. + """ + if vintages == "all": + return rows + return [r for r in rows if bool(r.get("settlement_grade"))] + + +def _expected_release(indicator: str, from_date: date | datetime) -> datetime | None: + """Best-effort scheduled release datetime for the window, or ``None``. + + Reads the curated schedule (``econ.releases``); returns the earliest scheduled + release at/after ``from_date`` so the not-yet-released error can name a + "come back at" time. Never fabricates a timestamp — returns ``None`` when the + indicator has no known schedule. + """ + try: + from ._releases import releases as _releases + except Exception: # pragma: no cover - defensive + return None + try: + schedule = _releases(indicator) + except (ValueError, TypeError): + return None + cutoff = ( + from_date + if isinstance(from_date, datetime) + else datetime(from_date.year, from_date.month, from_date.day) + ) + + cutoff = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=UTC) + upcoming = [e.release_datetime for e in schedule if e.release_datetime >= cutoff] + return min(upcoming) if upcoming else None + + +def _window_period(from_date: date | datetime, to_date: date | datetime) -> str: + """A human ``period`` label for the not-yet-released error payload.""" + d0 = from_date.date() if isinstance(from_date, datetime) else from_date + d1 = to_date.date() if isinstance(to_date, datetime) else to_date + if d0 == d1: + return d0.isoformat() + return f"{d0.isoformat()}..{d1.isoformat()}" + + +def _settlement_gap_period(from_date: date | datetime, to_date: date | datetime) -> str: + """Window label carrying the keyless-settlement ``FRED_API_KEY`` unlock hint. + + The BLS-family + jobless settlement grade (the first print) comes ONLY from the + ALFRED vintage store, unlocked by ``FRED_API_KEY``. When a keyless settlement + request finds latest-revised rows but no first print, the not-yet-released + error carries this actionable window+hint so its message names ``FRED_API_KEY`` + as the unlock — matching the jobless family so the two read identically. The + :class:`IndicatorNotYetReleasedError` message template is ``core``-owned (left + untouched here), so the hint rides in the period slot the caller controls. + """ + return ( + f"{_window_period(from_date, to_date)} — settlement-grade first prints " + "require FRED_API_KEY (the ALFRED vintage store); keyless serves " + "latest-revised only (settlement_grade=False)" + ) + + +def series( + indicator: str, + from_date: date | datetime, + to_date: date | datetime, + *, + vintages: str = "settlement", + source: str | None = None, + delivery: str = "live", +) -> pd.DataFrame: + """Return economic-indicator observation rows for ``indicator`` (canonical). + + ``series`` is the CANONICAL econ read function (plan 29-11), conforming the + surface to the cross-vertical source-identity kwarg contract + (``docs/source-identity.md``). :func:`history` is a working alias returning + byte-identical output — see its docstring for the deprecation note. + + Args: + indicator: The indicator id (``"cpi"`` / ``"nfp"`` / ``"gdp"`` / ``"ppi"`` + / ``"jobless_claims"`` / …; the ``schema.econ.observations.v1`` + vocabulary). + from_date: Inclusive start of the requested range. + to_date: Inclusive end of the requested range. + vintages: ``"settlement"`` (default) returns only the settlement-grade + first-print rows (the value as-of the Kalshi expiration); ``"all"`` + returns every vintage for feature engineering. ``settlement`` rows are + always a subset of ``all`` rows (the clean-partition property). + source: Provenance pin (contract §1). ``None`` (default) uses the + per-indicator default routing (unchanged behavior). A pin names the + authority (``"fred"`` ALFRED vintage store, ``"bls"``, ``"bea"``, + ``"dol"``, ``"fed"``). An unknown source, OR a valid authority that + cannot serve ``indicator`` (e.g. ``source="bea"`` for ``"cpi"``), + raises ``ValueError`` BEFORE any network call — never a silent fallback. + delivery: Where the computation runs (contract §2). ``"live"`` (default) + hits the public agency APIs locally; ``"hosted"`` is the reserved + precomputed-API seam and raises :class:`SourceUnavailableError` + naming ``ECON_HOSTED_URL`` + ``MOSTLYRIGHT_API_KEY``. An unknown value + raises ``ValueError`` pre-network. + + Returns: + A ``schema.econ.observations.v1`` :class:`pandas.DataFrame` (validated). + + Raises: + ValueError: ``vintages`` is not ``"settlement"``/``"all"``; an invalid + ``source``/``delivery`` (pre-network); or ``indicator`` has no dispatch + entry (never a silent empty frame). + SourceUnavailableError: ``delivery="hosted"`` (the reserved seam). + DataAvailabilityError: ``reason="out_of_window"`` when ``from_date`` is + below the indicator's FEDS-2026-010 first-contract floor. + IndicatorNotYetReleasedError: the window has no relevant rows — a scheduled + release has not landed yet. NEVER returns ``[]``/``None``. + ImportError: pandas (the ``[pandas]`` extra) is not installed. + """ + if vintages not in _VINTAGE_MODES: + raise ValueError(f"vintages must be one of {sorted(_VINTAGE_MODES)}; got {vintages!r}") + + # 0. Source-identity contract validation FIRST — loud, before any network. + # An unknown/unservable source or an invalid/hosted delivery raises here so + # a bad pin never touches the cache or a fetcher. + _validate_source_and_delivery(indicator, source, delivery) + + # 0b. Coerce ISO-string dates to date/datetime ONCE, so every downstream + # helper (floor, cache, fetchers, window filter) receives a real temporal + # object. The documented primary usage passes strings; without this a + # bare ``str`` reaches ``assert_within_floor`` and raises ``str < date`` + # TypeError for EVERY string-input call. + from_date = _coerce_input_date(from_date) + to_date = _coerce_input_date(to_date) + + # 1. FEDS floor FIRST — a below-floor request is out of range regardless of + # what the cache/fetcher holds. ``assert_within_floor`` also raises for an + # unknown indicator (no floor defined), so a bogus indicator fails here. + assert_within_floor(indicator, _coerce_floor_date(from_date)) + + fetcher = INDICATOR_FETCHERS.get(indicator) + if fetcher is None: + # The dispatch table is exhaustive over the schema vocabulary; a missing + # entry is a config drift, never a silent empty return. + known = ", ".join(sorted(INDICATOR_FETCHERS)) + raise ValueError( + f"no fetcher registered for econ indicator {indicator!r}; " + f"the dispatch table covers: {known}" + ) + + # 2. Obtain the rows. + # - source=None (default): read the vintage cache; on a MISS fetch, persist, + # and re-read (so the second call for the same window is a pure cache hit). + # - source PINNED: fetch FRESH from the routed authority and use those rows + # directly. The fetched rows carry the pinned authority's real per-row + # ``source`` tag by construction; we do NOT round-trip through the cache + # re-read (which normalizes per-row source to "econ.cache", erasing + # provenance) and do NOT persist them (so a pin never contaminates the + # default cache with another provider's rows). + frame_source = "econ.cache" + if source is not None: + fetched = fetcher(from_date, to_date, source=source) or [] + # Normalize the indicator (M1/codex): a fetcher may tag rows with a raw + # upstream id — the ALFRED fetcher stamps the FRED series id + # (A191RL1Q225SBEA for gdp), fed.py hardcodes "fed_funds" for both fed + # indicators. write_econ_cache does this re-stamp on the default path; the + # pinned path (no cache round-trip) must do it too so the returned rows + # carry the requested indicator, not the upstream id. + cached = [{**r, "indicator": indicator} for r in fetched] + if cached: + frame_source = str(cached[0].get("source") or "econ.cache") + else: + # Fetch when the window is NOT already covered by a prior fetch (H1: a + # partially-cached window — Jan-Mar cached, caller asks Jan-Jun — must not + # silently drop the uncovered tail). The ledger is GRADE-AWARE: a + # ``vintages="settlement"`` read is only satisfied by a settlement-capable + # prior fetch (codex round-2 C2 — else keyless-cached data blocks the keyed + # ALFRED path). + need_settlement = vintages == "settlement" + covered = is_window_covered(indicator, from_date, to_date, need_settlement=need_settlement) + cached = read_econ_window(indicator, from_date, to_date) if covered else [] + if not cached: + fetched = fetcher(from_date, to_date, source=None) + if fetched: + _persist_rows(indicator, fetched) + reread = read_econ_window(indicator, from_date, to_date) + if reread: + cached = reread + # Persisted → mark covered, but cap the range at YESTERDAY + # (codex round-2 C1 + round-3 C2): NEVER mark today — the + # current day is INCOMPLETE, later releases still land today, so + # a fixed past `to_date` that reaches today must not freeze and + # miss a same-day print (mirrors the weather current-period- + # incomplete discipline). A window ending in the future likewise + # stays uncovered past yesterday, so later releases are fetched as + # time advances. A window that ends on/before yesterday is a fully + # ELAPSED period → fully covered (repeat calls hit the cache). Tag + # whether settlement-grade rows were obtained so a settlement read + # isn't satisfied by a latest-revised-only fill (C2). + has_settlement = any(bool(r.get("settlement_grade")) for r in reread) + capped_to = min( + _coerce_floor_date(to_date), + datetime.now(UTC).date() - timedelta(days=1), + ) + mark_window_covered( + indicator, + _coerce_floor_date(from_date), + capped_to, + settlement=has_settlement, + ) + else: + # FETCH-THROUGH (D-LIC-2): persist was a no-op because the rows + # are FRED-derived and the cache carve-out skipped them (FRED + # ToU bans store/cache). Use the freshly-fetched rows directly — + # "0 rows persisted" is NOT "not released" — and do NOT mark the + # window covered (nothing was persisted; the next call re-fetches). + # Stamp the requested indicator (the ALFRED fetcher tags rows + # with the FRED series id, e.g. A191RL1Q225SBEA, not "gdp") AND + # carry the fetched rows' real ``source`` into the frame (H3: the + # default "econ.cache" tag would mislabel genuinely-live rows). + cached = [{**r, "indicator": indicator} for r in fetched] + frame_source = str(fetched[0].get("source") or "econ.cache") + else: + cached = [] + + # 3. Read-time vintage filter. + filtered = _filter_vintages(cached, vintages) + + # 4. Not-yet-released is an error, never empty. If nothing survives (the + # window has no rows at all, OR settlement was requested and no first-print + # vintage exists yet), raise with the scheduled release when known. When the + # emptiness is specifically a keyless-settlement GAP — latest-revised rows + # EXIST but none is settlement-grade, for a FRED-unlockable indicator, with + # no key present — name FRED_API_KEY as the unlock so the BLS family and + # jobless read identically. A genuinely unreleased window (NO rows at all) + # keeps the plain message. + if not filtered: + keyless_settlement_gap = ( + vintages == "settlement" + and bool(cached) + and indicator in _FRED_UNLOCKABLE_INDICATORS + and _resolve_fred_key() is None + ) + period_label = ( + _settlement_gap_period(from_date, to_date) + if keyless_settlement_gap + else _window_period(from_date, to_date) + ) + raise IndicatorNotYetReleasedError( + indicator, + period_label, + expected_release=_expected_release(indicator, from_date), + ) + + # 5. Build + validate the return frame. Trigger the pandas install-hint guard + # up front (build_econ_dataframe imports pandas internally, but the guard + # gives the actionable "install mostlyrightmd-econ[pandas]" message). + _require_pandas() + from ._schema import build_econ_dataframe, validate_econ_dataframe + + df = build_econ_dataframe(filtered, source=frame_source) + validate_econ_dataframe(df) + return df + + +def history( + indicator: str, + from_date: date | datetime, + to_date: date | datetime, + *, + vintages: str = "settlement", + source: str | None = None, + delivery: str = "live", +) -> pd.DataFrame: + """Deprecated alias of :func:`series` — returns byte-identical output. + + .. deprecated:: 1.15 + ``history`` is retained as a working alias of the canonical + :func:`series` (plan 29-11 conformed the econ surface to the + cross-vertical source-identity contract). It emits NO runtime warning + this release — matching the ``research()``/``dataset()`` silent-alias + style. A ``DeprecationWarning`` arrives in the next minor; removal at + 2.0. Migrate ``history(...)`` → ``series(...)`` (identical signature). + + All arguments forward verbatim to :func:`series`; see its docstring for the + full contract (``vintages``, ``source=``, ``delivery=`` semantics). + """ + return series( + indicator, + from_date, + to_date, + vintages=vintages, + source=source, + delivery=delivery, + ) + + +def _persist_rows(indicator: str, rows: list[dict[str, Any]]) -> None: + """Persist fetched rows into their monthly ``(indicator, year, month)`` partitions. + + Rows are grouped by their ``vintage_date`` year/month so each lands in the + correct partition (the cache dedups on the vintage key, keeping all vintages). + A row with no ``vintage_date`` is skipped defensively rather than mis-filed. + """ + by_partition: dict[tuple[int, int], list[dict[str, Any]]] = {} + for row in rows: + vintage = row.get("vintage_date") + if not isinstance(vintage, datetime): + continue + key = (vintage.year, vintage.month) + by_partition.setdefault(key, []).append(row) + for (year, month), part_rows in by_partition.items(): + write_econ_cache(indicator, year, month, part_rows) diff --git a/packages/econ/src/mostlyright/econ/_polymarket.py b/packages/econ/src/mostlyright/econ/_polymarket.py new file mode 100644 index 00000000..861c48f6 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_polymarket.py @@ -0,0 +1,266 @@ +"""Polymarket econ derive — the SECONDARY venue (gamma ``/events`` + CLOB history). + +Kalshi is econ v1's PRIMARY settlement venue (the resolver-driven join in +``research_econ``). Polymarket is the SECONDARY venue: this module derives the +Fed/CPI/PCE econ market outcomes from Polymarket's keyless gamma ``/events`` +endpoint (the market question / outcomes / resolution source) and attaches the +CLOB ``/prices-history`` price panel, producing a secondary-venue outcome shape +``research_econ`` can join alongside the Kalshi outcome. + +**In-package httpx, no cross-package import (checker BLOCKER 2 / T-29-27).** The +workspace dependency edge is strictly ONE-WAY ``markets → econ``: plan 29-04 +(Wave 3) added ``mostlyrightmd-econ`` to the **markets** ``[project].dependencies`` +(the markets resolver imports econ's ``_settlement_map``). ``econ`` therefore MUST +NOT depend on the markets package — adding that edge would create a cycle that +breaks ``uv sync`` / ``uv build``. So this module reimplements the thin keyless +gamma/CLOB GET in-package using ``httpx`` (reusing the core +``mostlyright._internal._http`` retry discipline) rather than importing the +existing markets Polymarket client. ``econ`` depends only on ``core``. + +Verified live shapes (RESEARCH ECON-18, keyless, HTTP 200): + +- gamma ``GET /events?slug=`` → an event object carrying ``markets[]``, + each ``{question, conditionId, clobTokenIds, outcomes, outcomePrices, + umaResolutionStatus, resolutionSource, description}``. (The Fed event + ``fed-decision-in-july-181`` carried $45.7M volume, 5 markets.) +- CLOB ``GET /prices-history?interval=max&market=&fidelity=60`` → + ``{history:[{t:, p:}...]}`` (~720 points keyless). + +Security: no key is required (both endpoints are public). HTTPS is enforced; a +Cloudfront 403 on a blank User-Agent is avoided by always sending one. A hostile +/ non-JSON body raises rather than being parsed as an empty outcome. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +import httpx +from mostlyright._internal._http import HTTP_TIMEOUT +from mostlyright.core.exceptions import SourceUnavailableError + +log = logging.getLogger(__name__) + +#: Polymarket gamma (events metadata) + CLOB (price history) hosts. Keyless, +#: public, read-only. HTTPS pinned. +GAMMA_API_BASE = "https://gamma-api.polymarket.com" +CLOB_API_BASE = "https://clob.polymarket.com" + +#: The venue tag stamped on every derived outcome (the secondary venue). +POLYMARKET_VENUE = "polymarket" + +#: Cloudfront returns 403 on a blank User-Agent — always send one. +_USER_AGENT = "mostlyright-sdk/econ (+https://github.com/mostlyrightmd/mostlyright-sdk)" + +_HEADERS = {"User-Agent": _USER_AGENT, "Accept": "application/json"} + + +def _get_json( + base_url: str, path: str, *, params: dict[str, Any], client: httpx.Client | None +) -> Any: + """Keyless HTTPS GET returning parsed JSON; a non-JSON body raises. + + Reuses the core HTTP timeout. The caller may inject ``client`` (tests / + connection reuse); otherwise a fresh client with the required User-Agent is + created and closed per call. + """ + owned = client is None + active = client if client is not None else httpx.Client(timeout=HTTP_TIMEOUT, headers=_HEADERS) + try: + response = active.get(f"{base_url}{path}", params=params) + response.raise_for_status() + try: + return response.json() + except ValueError as exc: + # A Cloudflare / HTML body must raise, never be parsed as an empty + # outcome (mirrors the fetcher bot-wall guards). + raise SourceUnavailableError( + "Polymarket returned a non-JSON body", + source=POLYMARKET_VENUE, + url=f"{base_url}{path}", + http_status=response.status_code, + underlying=str(exc), + ) from exc + finally: + if owned: + active.close() + + +def fetch_gamma_event(slug: str, *, client: httpx.Client | None = None) -> dict[str, Any]: + """Fetch a Polymarket gamma event by ``slug`` → the raw event object. + + Args: + slug: The gamma event slug (e.g. ``"fed-decision-in-july-181"``). + client: An injected ``httpx.Client`` (tests / connection reuse). + + Returns: + The event dict (carrying ``markets[]``). + + Raises: + SourceUnavailableError: on a non-JSON body. + httpx.HTTPStatusError: on a non-2xx response. + ValueError: the gamma response is not a single event object. + """ + payload = _get_json(GAMMA_API_BASE, "/events", params={"slug": slug}, client=client) + # Gamma returns either a single object or a one-element list depending on the + # query; normalize to the event dict. + if isinstance(payload, list): + if not payload: + raise ValueError(f"Polymarket gamma returned no event for slug {slug!r}") + payload = payload[0] + if not isinstance(payload, dict): + raise ValueError( + f"Polymarket gamma /events returned non-object payload: {type(payload).__name__}" + ) + return payload + + +def fetch_clob_history( + clob_token_id: str, + *, + fidelity: int = 60, + interval: str = "max", + client: httpx.Client | None = None, +) -> list[dict[str, Any]]: + """Fetch the CLOB price-history panel for a ``clob_token_id``. + + Args: + clob_token_id: The ERC-1155 CLOB token id (the ``market`` query param on + the CLOB endpoint — NOT the gamma condition id). + fidelity: Bucket fidelity in minutes (default 60 → ~720 points for + ``interval="max"``). + interval: History span (default ``"max"``). + client: An injected ``httpx.Client`` (tests / connection reuse). + + Returns: + The list of ``{t: , p: }`` price points (``p`` coerced + to float). Empty list when the token has no history. + + Raises: + SourceUnavailableError: on a non-JSON body. + httpx.HTTPStatusError: on a non-2xx response. + """ + payload = _get_json( + CLOB_API_BASE, + "/prices-history", + params={"interval": interval, "market": clob_token_id, "fidelity": fidelity}, + client=client, + ) + history = payload.get("history") if isinstance(payload, dict) else None + if not history: + return [] + out: list[dict[str, Any]] = [] + for point in history: + if not isinstance(point, dict): + continue + raw_p = point.get("p") + try: + price = float(raw_p) if raw_p is not None else None + except (TypeError, ValueError): + price = None + out.append({"t": point.get("t"), "p": price}) + return out + + +def _parse_json_field(value: Any) -> Any: + """Parse a gamma stringified-JSON field (``clobTokenIds`` etc.) or pass through. + + Gamma serializes list fields as JSON strings (``'["Yes", "No"]'``). Parse them + to Python lists; if the value is already a list (or unparseable) return it + as-is rather than raising — a malformed field should degrade, not crash. + """ + if isinstance(value, str): + try: + return json.loads(value) + except (ValueError, TypeError): + return value + return value + + +def _coerce_prices(value: Any) -> list[float]: + """Coerce a gamma ``outcomePrices`` field into a list of floats.""" + parsed = _parse_json_field(value) + if not isinstance(parsed, list): + return [] + out: list[float] = [] + for item in parsed: + try: + out.append(float(item)) + except (TypeError, ValueError): + continue + return out + + +def derive( + *, + slug: str, + gamma_client: httpx.Client | None = None, + clob_client: httpx.Client | None = None, + attach_price_history: bool = True, +) -> list[dict[str, Any]]: + """Derive the Polymarket econ market outcomes for a gamma event ``slug``. + + Queries gamma ``/events?slug=`` for the econ event (Fed/CPI/PCE), and — + when ``attach_price_history`` — the CLOB ``/prices-history`` panel for each + market's first outcome token. Returns one secondary-venue outcome dict per + market, shaped for ``research_econ`` to join alongside the Kalshi outcome. + + Args: + slug: The gamma event slug (e.g. ``"fed-decision-in-july-181"``). + gamma_client / clob_client: Injected ``httpx.Client``s (tests / reuse). A + single shared client may be passed to both. + attach_price_history: When ``True`` (default) the CLOB price panel for the + first ``clobTokenIds`` entry is attached per market. + + Returns: + A list of outcome dicts, each with keys: ``venue`` (``"polymarket"``), + ``slug``, ``question``, ``condition_id``, ``outcomes`` (list), + ``outcome_prices`` (list[float]), ``clob_token_ids`` (list), + ``uma_resolution_status``, ``resolution_source``, ``description``, and + ``price_history`` (list of ``{t, p}``; empty when not attached / absent). + + Raises: + SourceUnavailableError: on a non-JSON gamma/CLOB body. + httpx.HTTPStatusError: on a non-2xx response. + ValueError: the gamma response carries no event. + """ + event = fetch_gamma_event(slug, client=gamma_client) + markets = event.get("markets") or [] + outcomes: list[dict[str, Any]] = [] + for market in markets: + if not isinstance(market, dict): + continue + token_ids = _parse_json_field(market.get("clobTokenIds")) + token_list = token_ids if isinstance(token_ids, list) else [] + price_history: list[dict[str, Any]] = [] + if attach_price_history and token_list: + first_token = str(token_list[0]) + price_history = fetch_clob_history(first_token, client=clob_client) + outcomes.append( + { + "venue": POLYMARKET_VENUE, + "slug": event.get("slug", slug), + "question": market.get("question"), + "condition_id": market.get("conditionId"), + "outcomes": _parse_json_field(market.get("outcomes")), + "outcome_prices": _coerce_prices(market.get("outcomePrices")), + "clob_token_ids": token_list, + "uma_resolution_status": market.get("umaResolutionStatus"), + "resolution_source": market.get("resolutionSource"), + "description": market.get("description"), + "price_history": price_history, + } + ) + return outcomes + + +__all__ = [ + "CLOB_API_BASE", + "GAMMA_API_BASE", + "POLYMARKET_VENUE", + "derive", + "fetch_clob_history", + "fetch_gamma_event", +] diff --git a/packages/econ/src/mostlyright/econ/_releases.py b/packages/econ/src/mostlyright/econ/_releases.py new file mode 100644 index 00000000..d47fcd4b --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_releases.py @@ -0,0 +1,166 @@ +"""``econ.releases`` — the release calendar / schedule for an indicator. + +CONTEXT ECON-13: each econ indicator publishes on a known agency schedule — the +BLS Employment Situation ("empsit") + CPI release schedule, the BEA GDP release +schedule, the FOMC meeting calendar, and the weekly DOL initial-claims (Thursday +8:30 ET) cadence. ``releases(indicator)`` returns that schedule as a small typed +structure a caller can use to know WHEN a period's print is due — the companion +signal to :class:`~mostlyright.core.exceptions.IndicatorNotYetReleasedError` +(``econ.history`` raises it when a scheduled print has not landed; ``releases`` +tells you when to come back). + +**v1 sourcing: a curated schedule table (documented).** The agencies publish +these calendars as HTML/ICS pages; a live schedule fetch is a fast-follow. v1 +ships a small curated table of the KNOWN upcoming release datetimes per indicator +so the surface is offline-deterministic (no network, no key) and the TS port +(29-09) can mirror it from the same shape. Each :class:`ReleaseEvent` carries the +observation ``period`` it releases, the scheduled ``release_datetime`` (stored +UTC — an 8:30 ET drop is ``12:30`` or ``13:30`` UTC depending on DST), and the +source ``agency`` (a canonical +:data:`~mostlyright.econ._settlement_map.AGENCY_NAMES` short name). + +The schedule is INDICATOR-keyed on the same vocabulary as +``schema.econ.observations.v1`` (``cpi`` / ``cpi_core`` / ``cpi_yoy`` / ``nfp`` / +``u3`` / ``gdp`` / ``ppi`` / ``ppi_yoy`` / ``jobless_claims`` / ``fed_funds`` / +``fed_decision``); an unknown indicator raises ``ValueError`` (never returns +``[]``/``None`` — an unschedulable indicator is an explicit error, mirroring the +resolver's unknown-ticker discipline). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime + +from ._settlement_map import ( + AGENCY_BEA, + AGENCY_BLS, + AGENCY_DOL, + AGENCY_FED, +) + +__all__ = ["ReleaseEvent", "releases"] + + +@dataclass(frozen=True) +class ReleaseEvent: + """One scheduled release of an econ indicator. + + Attributes: + indicator: The econ indicator id (``schema.econ.observations.v1`` + vocabulary). + period: The observation period this release publishes (``"2026-06"`` + monthly, ``"2026Q2"`` quarterly, a weekly week-ending date, or an + FOMC meeting date). + release_datetime: The scheduled release wall-clock, stored tz-aware UTC + (an 8:30 ET drop is 12:30/13:30 UTC depending on DST). + agency: The source agency short name (a canonical + :data:`~mostlyright.econ._settlement_map.AGENCY_NAMES` value). + """ + + indicator: str + period: str + release_datetime: datetime + agency: str + + +def _utc(year: int, month: int, day: int, hour: int, minute: int) -> datetime: + """Build a tz-aware UTC datetime (schedule entries are stored UTC).""" + return datetime(year, month, day, hour, minute, tzinfo=UTC) + + +# A curated per-indicator release schedule. Small, forward-looking, and +# offline-deterministic — a live agency-calendar fetch is a documented +# fast-follow. 8:30 ET release times are stored as 12:30 UTC (EDT) here; the +# exact DST offset is not load-bearing for a "when is it due" signal. +# +# The four schedule families (ECON-13): +# - BLS CPI / Employment Situation (CPI, Core CPI, CPI-YoY, NFP, U3, PPI) +# - BEA GDP (quarterly advance/second/third) +# - Federal Reserve FOMC meeting calendar (fed_funds / fed_decision) +# - DOL weekly initial claims (Thursday 8:30 ET) +_CPI_SCHEDULE: tuple[tuple[str, datetime], ...] = ( + ("2026-05", _utc(2026, 6, 10, 12, 30)), + ("2026-06", _utc(2026, 7, 15, 12, 30)), + ("2026-07", _utc(2026, 8, 12, 12, 30)), +) +_NFP_SCHEDULE: tuple[tuple[str, datetime], ...] = ( + ("2026-05", _utc(2026, 6, 5, 12, 30)), + ("2026-06", _utc(2026, 7, 2, 12, 30)), + ("2026-07", _utc(2026, 8, 7, 12, 30)), +) +_U3_SCHEDULE = _NFP_SCHEDULE # U3 releases with the Employment Situation. +_GDP_SCHEDULE: tuple[tuple[str, datetime], ...] = ( + ("2026Q1", _utc(2026, 4, 29, 12, 30)), # advance + ("2026Q2", _utc(2026, 7, 30, 12, 30)), # advance + ("2026Q3", _utc(2026, 10, 29, 12, 30)), # advance +) +_FED_SCHEDULE: tuple[tuple[str, datetime], ...] = ( + ("2026-06-17", _utc(2026, 6, 17, 18, 0)), # 2pm ET FOMC statement + ("2026-07-29", _utc(2026, 7, 29, 18, 0)), + ("2026-09-16", _utc(2026, 9, 16, 18, 0)), +) +_JOBLESS_SCHEDULE: tuple[tuple[str, datetime], ...] = ( + ("2026-06-20", _utc(2026, 6, 25, 12, 30)), + ("2026-06-27", _utc(2026, 7, 2, 12, 30)), + ("2026-07-04", _utc(2026, 7, 9, 12, 30)), +) +_PPI_SCHEDULE: tuple[tuple[str, datetime], ...] = ( + ("2026-05", _utc(2026, 6, 11, 12, 30)), + ("2026-06", _utc(2026, 7, 16, 12, 30)), + ("2026-07", _utc(2026, 8, 13, 12, 30)), +) + +#: indicator → (schedule rows, source agency). The single source of truth for +#: :func:`releases`; the TS port mirrors this table. +_SCHEDULES: dict[str, tuple[tuple[tuple[str, datetime], ...], str]] = { + "cpi": (_CPI_SCHEDULE, AGENCY_BLS), + "cpi_core": (_CPI_SCHEDULE, AGENCY_BLS), + "cpi_yoy": (_CPI_SCHEDULE, AGENCY_BLS), + "cpi_core_yoy": (_CPI_SCHEDULE, AGENCY_BLS), + "nfp": (_NFP_SCHEDULE, AGENCY_BLS), + "u3": (_U3_SCHEDULE, AGENCY_BLS), + "ppi": (_PPI_SCHEDULE, AGENCY_BLS), + "ppi_yoy": (_PPI_SCHEDULE, AGENCY_BLS), + "gdp": (_GDP_SCHEDULE, AGENCY_BEA), + "fed_funds": (_FED_SCHEDULE, AGENCY_FED), + "fed_decision": (_FED_SCHEDULE, AGENCY_FED), + "jobless_claims": (_JOBLESS_SCHEDULE, AGENCY_DOL), +} + + +def releases(indicator: str) -> list[ReleaseEvent]: + """Return the release calendar / schedule for ``indicator``. + + Args: + indicator: An econ indicator id (``schema.econ.observations.v1`` + vocabulary: ``"cpi"`` / ``"nfp"`` / ``"gdp"`` / ``"fed_funds"`` / + ``"jobless_claims"`` / …). + + Returns: + The scheduled releases for the indicator as a list of + :class:`ReleaseEvent`, sorted by ``release_datetime`` ascending. Sourced + from the curated v1 schedule table (a live agency-calendar fetch is a + documented fast-follow). + + Raises: + TypeError: ``indicator`` is not a ``str``. + ValueError: ``indicator`` has no known schedule (never returns + ``[]``/``None`` — an unschedulable indicator is an explicit error). + """ + if not isinstance(indicator, str): + raise TypeError(f"indicator must be a str, got {type(indicator).__name__}") + key = indicator.strip().lower() + entry = _SCHEDULES.get(key) + if entry is None: + known = ", ".join(sorted(_SCHEDULES)) + raise ValueError( + f"no release schedule for econ indicator {indicator!r}; known indicators: {known}" + ) + rows, agency = entry + events = [ + ReleaseEvent(indicator=key, period=period, release_datetime=when, agency=agency) + for period, when in rows + ] + events.sort(key=lambda e: e.release_datetime) + return events diff --git a/packages/econ/src/mostlyright/econ/_research.py b/packages/econ/src/mostlyright/econ/_research.py new file mode 100644 index 00000000..13fdd735 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_research.py @@ -0,0 +1,190 @@ +"""``econ.research_econ`` — leakage-free settlement pairs (the vertical's payoff). + +``research_econ(series_or_contract, from_date, to_date, *, as_of=None)`` (plan +29-08, replacing the 29-01 stub) joins a prediction-market contract to the +settlement-grade FIRST-PRINT vintage of the agency indicator it settles against, +and returns training pairs that "backtest the same way they trade": + +1. **Resolve** the contract to its settlement mapping — ``(agency, indicator, + settlement_grade)`` — via the econ-side routing table + (:func:`~mostlyright.econ._settlement_map.resolve_settlement`). This reads the + SAME ``SETTLEMENT_ROUTING`` table the markets ``kalshi_econ.resolve`` reads, so + the two resolvers agree by construction — but it does NOT import the markets + package. The workspace edge is strictly one-way ``markets → econ`` (29-04 added + ``mostlyrightmd-econ`` to the markets deps); adding the reverse would cycle + ``uv sync`` / ``uv build`` (T-29-27). ``econ`` depends only on ``core``. +2. **Pull the first print** via ``history(indicator, from_date, to_date, + vintages="settlement")`` — the settlement-grade vintage (the value as-of the + Kalshi expiration, first-print discipline). +3. **Build the pairs frame** with ``knowledge_time = vintage_date`` (the leakage + cutoff column) and the resolved market-outcome metadata (``agency``, + contract-level ``settlement_grade``). +4. **Leakage guard** — when ``as_of`` (a :class:`~mostlyright.core.TimePoint`) is + given, :func:`~mostlyright.core.temporal.leakage.assert_no_leakage` rejects any + row whose ``knowledge_time > as_of``, so a FUTURE revision can never leak into + a backtest (ECON-14 / T-29-23; property-tested). +5. **TE divergence, honestly labeled** — for a Trading-Economics-settled series + (the 48 TE contracts: KXUSPPI PPI-MoM, KXUSNFP, …) the join uses the AGENCY + first print as the value but labels the pair ``settlement_grade=False`` and + emits a :class:`DivergenceWarning` naming the series and that Trading Economics + is the (unlicensed) settlement authority. It NEVER fabricates or approximates a + TE value (ECON-17 / T-29-24). + +The live Kalshi/Polymarket outcome-price join (attaching the traded YES/NO price +panel to each settlement period) is a live-network concern exercised in the 29-10 +smoke; this module establishes the leakage-safe settlement-pair SEMANTICS (the +resolved outcome metadata + first-print value + ``knowledge_time`` + the +``assert_no_leakage`` gate) that the price join layers onto. pandas is the +``[pandas]`` extra — lazy-imported with an install hint. +""" + +from __future__ import annotations + +import warnings +from datetime import date, datetime +from typing import TYPE_CHECKING, Any + +from mostlyright.core.temporal.leakage import assert_no_leakage +from mostlyright.core.temporal.timepoint import TimePoint + +from ._history import series +from ._settlement_map import AGENCY_TE, resolve_settlement + +if TYPE_CHECKING: + import pandas as pd + +__all__ = ["DivergenceWarning", "research_econ"] + + +class DivergenceWarning(UserWarning): + """A settlement pair uses an AGENCY first-print proxy for a TE-settled series. + + The 48 Trading-Economics-settled Kalshi series (KXUSPPI PPI-MoM, KXUSNFP, and + international variants) settle against a Trading Economics value we cannot + license. ``research_econ`` ships the AGENCY first print labeled + ``settlement_grade=False`` as an honest proxy and raises THIS warning so a + consumer knows the pair's value is the agency proxy, not the TE settlement + truth — the divergence is bounded (the TE contract template lists TE as a + fallback BELOW the agency) but non-zero. A TE value is NEVER fabricated + (ECON-17 / T-29-24). + """ + + +def _require_pandas() -> Any: + """Lazy-import pandas with an actionable install hint on miss.""" + try: + import pandas as _pandas + except ImportError as exc: # pragma: no cover - exercised only without pandas + raise ImportError( + "econ.research_econ requires pandas. Install with: " + "pip install mostlyrightmd-econ[pandas]" + ) from exc + return _pandas + + +def research_econ( + series_or_contract: str, + from_date: date | datetime, + to_date: date | datetime, + *, + as_of: TimePoint | None = None, + source: str | None = None, + delivery: str = "live", +) -> pd.DataFrame: + """Return leakage-free settlement pairs for ``series_or_contract``. + + Each pair joins the market outcome (resolved agency/indicator/settlement + metadata) to the first-print indicator value and the as-of features known at + ``knowledge_time = vintage_date`` — safe to backtest and to trade. + + Args: + series_or_contract: A Kalshi econ series root (``"KXCPIYOY"``) or a + concrete dated market ticker (``"KXCPIYOY-26JUL"``). Case-insensitive. + from_date: Inclusive start of the requested range. + to_date: Inclusive end of the requested range. + as_of: Optional leakage cutoff (a :class:`~mostlyright.core.TimePoint`). + When given, any pair whose ``knowledge_time`` (``vintage_date``) is + after ``as_of`` triggers :class:`~mostlyright.core.exceptions.LeakageError` + — a future revision can never leak into a backtest. + source: Provenance pin (source-identity contract §1), forwarded to the + underlying :func:`~mostlyright.econ._history.series` read. ``None`` + (default) uses the per-indicator default routing; an unknown source or + a valid authority that cannot serve the resolved indicator raises + ``ValueError`` pre-network. Whether to also add an ``include_econ`` + covariate seam to core's ``dataset()`` is an OPTION-B question + deferred to Vu's review — this function does NOT touch core. + delivery: Where the computation runs (contract §2), forwarded to + ``series``. ``"live"`` (default) | ``"hosted"`` (reserved seam → + :class:`SourceUnavailableError`). + + Returns: + A :class:`pandas.DataFrame` of settlement pairs with (at least) the + columns ``period``, ``indicator``, ``agency``, ``value``, + ``settlement_grade`` (contract-level), ``vintage_date``, and + ``knowledge_time``. + + Raises: + TypeError: ``series_or_contract`` is not a ``str``. + ValueError: the ticker resolves to no known routing root (never None), or + an invalid ``source``/``delivery`` (pre-network). + SourceUnavailableError: ``delivery="hosted"`` (the reserved seam). + DataAvailabilityError: the indicator's window is below the FEDS floor. + IndicatorNotYetReleasedError: the settlement period has no first print yet. + LeakageError: an ``as_of`` was given and a pair's vintage_date is after it. + ImportError: pandas (the ``[pandas]`` extra) is not installed. + + Warns: + DivergenceWarning: the resolved series settles to Trading Economics — the + pair carries the AGENCY first-print value labeled + ``settlement_grade=False`` (never a fabricated TE value; ECON-17). + """ + # Trigger the pandas install-hint guard up front (series returns a DataFrame + # and we .copy() it below; the guard gives the actionable install message). + _require_pandas() + + # 1. Resolve the contract via the econ-side routing table (NO markets import). + _root, rule = resolve_settlement(series_or_contract) + is_te_settled = rule.agency == AGENCY_TE + + # 2. Pull the settlement-grade first print for the resolved indicator. The + # source-identity kwargs forward to the canonical series read (loud + # pre-network validation lives there); default-call output is byte-identical + # to the pre-conformance research_econ. + settlement = series( + rule.indicator, + from_date, + to_date, + vintages="settlement", + source=source, + delivery=delivery, + ) + + # 3. Build the pairs frame: the first-print value + knowledge_time cutoff + + # the resolved market-outcome metadata. The contract-level settlement_grade + # comes from the ROUTING RULE (False for a TE-settled series even though the + # agency first print is itself a real vintage) — this is the honest label a + # downstream model reads to know whether the value IS the settlement truth. + pairs = settlement.copy() + pairs["agency"] = rule.agency + pairs["contract"] = _root + pairs["settlement_grade"] = bool(rule.settlement_grade) + + # 4. TE-divergence handling (ECON-17): the value stays the AGENCY first print; + # we only label the pair False and warn. NEVER fabricate a TE value. + if is_te_settled: + warnings.warn( + f"{_root}: settles to Trading Economics (unlicensed) — the returned " + f"value is the agency first-print proxy for indicator " + f"{rule.indicator!r}, labeled settlement_grade=False. No Trading " + "Economics value is fabricated.", + DivergenceWarning, + stacklevel=2, + ) + + # 5. Leakage guard: reject any pair whose vintage_date is after the as_of + # cutoff (a future revision can never leak into a backtest). knowledge_time + # == vintage_date for econ, so assert_no_leakage keys off the right column. + if as_of is not None: + assert_no_leakage(pairs, as_of) + + return pairs diff --git a/packages/econ/src/mostlyright/econ/_schema.py b/packages/econ/src/mostlyright/econ/_schema.py new file mode 100644 index 00000000..8970ca0f --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_schema.py @@ -0,0 +1,344 @@ +"""``schema.econ.observations.v1`` — the standalone econ observation contract. + +Econ is deliberately ISOLATED from the parity-frozen ``schema.observation.v1`` +and the four weather parity files (``research()``, +``_internal/merge/observations.py``'s ``SOURCE_PRIORITY``, +``_internal/merge/climate.py``'s policies, ``live/_sources.py``). This is a +SEPARATE schema, registered lazily when ``mostlyright.econ`` (this module) is +first imported — the CWOP discipline at package granularity. A base install +that never touches econ pays nothing, and the canonical weather/satellite/ +earnings schemas are untouched. + +**The load-bearing difference from weather: vintage is first-class.** Weather +collapses to a single row per ``(station, date)`` via ``report_type_priority`` +dedup. Econ does the OPPOSITE — it KEEPS every vintage and filters at read +time. Three columns carry the vintage identity: + +- ``vintage_date`` — WHEN this value became known (the ALFRED + ``realtime_start`` / release-day capture). This is ``knowledge_time`` for + leakage: a backtest may only see vintages whose ``vintage_date`` precedes the + as-of cutoff. +- ``release_type`` — advance / second / third / final / revised / benchmark / + preliminary. The BEA GDP triple (advance→second→third) and the annual CPI/PPI + benchmark revisions are distinct rows, never a merge collapse. +- ``settlement_grade`` — is THIS vintage the settlement truth (the first print + as-of the Kalshi expiration)? ``True`` for the first print; ``False`` for + every later vintage AND for the 48 Trading-Economics-settled series (we ship + the agency first print labeled ``settlement_grade=False`` rather than + fabricate a TE value we cannot license — CONTEXT Area 1). + +Source identity is the econ-only union :data:`ECON_SOURCES` — one tag per live +agency path plus a persisted-then-read ``"econ.cache"`` tag. Econ data never +enters the weather merge/research path, so this union is local to +``schema.econ.observations.v1`` and never registered in ``SOURCE_PRIORITY``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, ClassVar + +from mostlyright.core.schema import ColumnSpec, Schema +from mostlyright.core.validator import register_schema, validate_dataframe + +if TYPE_CHECKING: + import pandas as pd + + +# --- Source-identity union --------------------------------------------------- +# One tag per live agency path each fetcher stamps, plus a persisted-then-read +# cache tag. Follows the ``"."`` convention (``iem.archive`` / +# ``cwop.live``). These strings are the fixed set the validator keys off +# ``_registered_sources`` — a fetcher stamping any OTHER tag raises loudly. + +#: FRED/ALFRED realtime vintage store (canonical first-print supply for the +#: series ALFRED covers; lit up by ``FRED_API_KEY``). +ECON_SOURCE_ALFRED = "alfred" + +#: BLS API v1 — keyless default path (CPI, PPI, NFP, U3). ~10yr / 25 req/day. +ECON_SOURCE_BLS_V1 = "bls.v1" + +#: BLS API v2 — keyed path (``BLS_API_KEY``). ~20yr / 500 req/day. +ECON_SOURCE_BLS_V2 = "bls.v2" + +#: Bureau of Economic Analysis — GDP (quarterly advance/second/third). +ECON_SOURCE_BEA = "bea" + +#: Department of Labor — initial jobless claims (weekly ICSA release). +ECON_SOURCE_DOL = "dol.icsa" + +#: Federal Reserve Board — FOMC funds-target decisions (NOT FRED; CONTEXT +#: Area 1: Fed decisions are sourced from the Board, not the FRED series). +ECON_SOURCE_FED = "fed" + +#: Canonical live source (the ALFRED vintage store is primary). Used as the +#: default ``source=`` for :func:`build_econ_dataframe` when a caller does not +#: pass one explicitly. +ECON_SOURCE = ECON_SOURCE_ALFRED + +#: Persisted-then-read tag: a row read back from the per-release parquet cache +#: (``econ.history()`` backtest replay) carries this so a consumer joining econ +#: history into a model sees the provenance is cache-read, NOT a fresh agency +#: pull. Econ carries every tag in its OWN union — it still never enters the +#: parity-frozen merge/research path. +ECON_CACHE_SOURCE = "econ.cache" + +#: Source tags whose bytes are fetched from ``api.stlouisfed.org``. FRED's Terms of +#: Use prohibit storing/caching/archiving FRED content (Prohibitions (q); API (l)) +#: — see ``.planning/research/DATA-LICENSING.md``. ``write_econ_cache`` refuses to +#: persist rows with these tags unless ``MOSTLYRIGHT_PERSIST_FRED=1`` (informed +#: opt-in). NOTE: ``dol.icsa`` is a DOL-*labeled* tag but a FRED-*transported* row +#: — the ``oui.doleta.gov`` CSV is bot-walled, so the bytes come from FRED +#: ``ICSA``/``ICNSA``, and FRED's ToU governs the fetched copy. +FRED_DERIVED_SOURCES: frozenset[str] = frozenset({ECON_SOURCE_ALFRED, ECON_SOURCE_DOL}) + +#: The full set of source identities ``schema.econ.observations.v1`` accepts. +#: Live agency paths + the persisted-then-read cache tag. The validator's +#: source-drift audit keys off ``_registered_sources``; a wrong-source frame +#: raises :class:`~mostlyright.core.exceptions.SourceMismatchError`. +ECON_SOURCES: frozenset[str] = frozenset( + { + ECON_SOURCE_ALFRED, + ECON_SOURCE_BLS_V1, + ECON_SOURCE_BLS_V2, + ECON_SOURCE_BEA, + ECON_SOURCE_DOL, + ECON_SOURCE_FED, + ECON_CACHE_SOURCE, + } +) + + +# --- Vintage vocabulary ------------------------------------------------------ +#: The ``release_type`` enum. ``advance``/``second``/``third`` are the BEA GDP +#: revision cadence; ``final`` is the settled value where no further revision is +#: expected; ``revised`` covers post-expiration revisions (which Kalshi +#: EXCLUDES — these carry ``settlement_grade=False``); ``benchmark`` is the +#: annual CPI/PPI re-basing; ``preliminary`` is an early-estimate print some +#: agencies publish before ``advance``. +ECON_RELEASE_TYPES: tuple[str, ...] = ( + "advance", + "second", + "third", + "final", + "revised", + "benchmark", + "preliminary", +) + + +class EconObservationsSchema(Schema): + """``schema.econ.observations.v1`` — one row per (indicator, period, vintage). + + Standalone (NOT a member of ``schema.observation.v1``). Registered source + union :data:`ECON_SOURCES` (live agency paths + persisted-then-read cache). + **Keeps every vintage** — two rows sharing ``(indicator, period)`` but with + different ``vintage_date`` both validate; read-time filtering on + ``settlement_grade`` selects the settlement print. ``value`` is nullable + because a not-yet-released period may be schema-shaped but valueless in a + feature frame. + """ + + schema_id = "schema.econ.observations.v1" + + #: Permitted source identities (union). Declared as ``_registered_sources`` + #: (plural) ONLY — mirrors CWOP: the validator's source-drift audit keys off + #: the legacy singular ``_registered_source``, so setting that here would log + #: a spurious ``source_drift_allowed`` event on every legitimate cache-frame + #: read and pollute the train/infer-mismatch audit trail. + _registered_sources: ClassVar[frozenset[str]] = ECON_SOURCES + + COLUMNS: ClassVar[list[ColumnSpec]] = [ + ColumnSpec( + name="indicator", + dtype="string", + units=None, + nullable=False, + notes="indicator id: cpi | cpi_core | cpi_yoy | nfp | u3 | gdp | ppi | ppi_yoy | jobless_claims | fed_funds", + ), + ColumnSpec( + name="series_id", + dtype="string", + units=None, + nullable=True, + notes="upstream series identifier (BLS/FRED/BEA code) when applicable", + ), + ColumnSpec( + name="period", + dtype="string", + units=None, + nullable=False, + notes="observation period the value refers to: '2026-06' monthly, '2026Q2' quarterly, '2026-06-28' weekly/date, or FOMC meeting date", + ), + ColumnSpec( + name="value", + dtype="float64", + units=None, + nullable=True, + notes="released numeric value; nullable — a not-yet-released period may be schema-shaped but valueless", + ), + ColumnSpec( + name="units", + dtype="string", + units=None, + nullable=True, + notes="index | percent | thousands_persons | percent_saar | …", + ), + ColumnSpec( + name="release_datetime", + dtype="timestamp_utc", + units=None, + nullable=True, + notes="wall-clock the value was released (8:30 ET etc.), UTC", + ), + ColumnSpec( + name="vintage_date", + dtype="timestamp_utc", + units=None, + nullable=False, + notes="WHEN this value became known (ALFRED realtime_start / release-day capture); the leakage knowledge_time", + ), + ColumnSpec( + name="release_type", + dtype="enum", + units=None, + nullable=False, + enum_values=ECON_RELEASE_TYPES, + notes="advance | second | third | final | revised | benchmark | preliminary", + ), + ColumnSpec( + name="settlement_grade", + dtype="bool", + units=None, + nullable=False, + notes="is this vintage the settlement truth (first print as-of expiration); False for later vintages AND the 48 TE-settled series", + ), + ColumnSpec( + name="knowledge_time", + dtype="timestamp_utc", + units=None, + nullable=False, + notes="leakage cutoff; equals vintage_date for econ (research_econ wires leakage on this)", + ), + ColumnSpec( + name="source", + dtype="string", + units=None, + nullable=False, + notes="per-row source identity == df.attrs['source']; one of ECON_SOURCES", + ), + ColumnSpec( + name="retrieved_at", + dtype="timestamp_utc", + units=None, + nullable=True, + notes="provenance of the fetch, UTC", + ), + ] + + +# Lazy, idempotent registration: econ stays standalone (the CWOP discipline at +# package granularity), so the schema registers when THIS module imports rather +# than in core's eager schemas/__init__.py list. +register_schema(EconObservationsSchema) + + +#: Column order for the assembled DataFrame (schema declaration order). +_COLUMN_ORDER: tuple[str, ...] = tuple(c.name for c in EconObservationsSchema.COLUMNS) +#: Timestamp columns coerced to tz-aware UTC. +_TIMESTAMP_COLS: tuple[str, ...] = ( + "release_datetime", + "vintage_date", + "knowledge_time", + "retrieved_at", +) +#: Free-text / identifier string columns coerced to object storage. +_STRING_COLS: tuple[str, ...] = ( + "indicator", + "series_id", + "period", + "units", + "source", + "release_type", +) + + +def build_econ_dataframe( + rows: list[dict[str, Any]], + *, + source: str = ECON_SOURCE, + retrieved_at: datetime | None = None, +) -> pd.DataFrame: + """Assemble econ observation rows into a ``schema.econ.observations.v1`` frame. + + Stamps ``df.attrs["source"] = source`` + ``df.attrs["retrieved_at"]`` (the + validator's required provenance) and coerces dtypes so the frame passes + :func:`validate_econ_dataframe`: + + - the four timestamp columns → tz-aware UTC via ``pd.to_datetime(..., utc=True)`` + - ``value`` → ``float64`` (nullable NaN-carrying) + - ``settlement_grade`` → pandas nullable ``boolean`` (holds -free bools + without float coercion; the non-nullable null check enforces presence) + - string/enum columns → object storage + + The per-row ``source`` column is overwritten to match ``source`` so the + validator's row-vs-attrs source check passes uniformly regardless of what + tag the individual input rows carried. ``source`` defaults to the ALFRED + vintage store; the persisted-history path passes ``"econ.cache"``. + + Raises: + ValueError: if ``source`` is not one of :data:`ECON_SOURCES`. + """ + import pandas as pd + + if source not in ECON_SOURCES: + raise ValueError( + f"build_econ_dataframe: source {source!r} is not an econ source. " + f"Permitted: {sorted(ECON_SOURCES)}" + ) + + if retrieved_at is None: + retrieved_at = datetime.now(UTC) + + records: list[dict[str, Any]] = [] + for r in rows: + rec = {name: r.get(name) for name in _COLUMN_ORDER} + rec["source"] = source # row provenance == df.attrs['source'] + records.append(rec) + + df = pd.DataFrame(records, columns=list(_COLUMN_ORDER)) + + # Typed coercion so the schema's dtype checks pass on both empty + full frames. + for col in _TIMESTAMP_COLS: + df[col] = pd.to_datetime(df[col], utc=True) + df["value"] = pd.to_numeric(df["value"], errors="coerce").astype("float64") + df["settlement_grade"] = df["settlement_grade"].astype("boolean") + for col in _STRING_COLS: + df[col] = df[col].astype("object") + + df.attrs["source"] = source + df.attrs["retrieved_at"] = retrieved_at + return df + + +def validate_econ_dataframe(df: pd.DataFrame) -> None: + """Run ``validate_dataframe(df, "schema.econ.observations.v1")`` (raises on drift).""" + validate_dataframe(df, "schema.econ.observations.v1") + + +__all__ = [ + "ECON_CACHE_SOURCE", + "ECON_RELEASE_TYPES", + "ECON_SOURCE", + "ECON_SOURCES", + "ECON_SOURCE_ALFRED", + "ECON_SOURCE_BEA", + "ECON_SOURCE_BLS_V1", + "ECON_SOURCE_BLS_V2", + "ECON_SOURCE_DOL", + "ECON_SOURCE_FED", + "FRED_DERIVED_SOURCES", + "EconObservationsSchema", + "build_econ_dataframe", + "validate_econ_dataframe", +] diff --git a/packages/econ/src/mostlyright/econ/_settlement_map.py b/packages/econ/src/mostlyright/econ/_settlement_map.py new file mode 100644 index 00000000..7408a201 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_settlement_map.py @@ -0,0 +1,263 @@ +"""Per-series Kalshi econ settlement routing — agency NAMES, never URLs. + +This module is the curated source-of-truth the settlement resolver (29-04) and +the codegen'd routing table (``schemas/kalshi-econ-settlement.json``, emitted by +``scripts/export_schemas.py``) consume. It answers ONE question per Kalshi +Economics series: *which settlement agency's first print is the settlement truth, +and is that first print settlement-grade?* + +Routing is keyed on the Kalshi ``settlement_sources[].name`` field (the agency +NAME) plus, where known, the contract-terms PDF basename — **never** the +``settlement_sources[].url`` field. That URL is demonstrably sloppy live: the +KXPAYROLLS series carries ``name="BLS"`` but ``url=".../ppi.nr0.htm"`` (a PPI +release URL on a payrolls contract), and one series' URL points at "San +Francisco Unified School District". Routing on the URL would settle NFP contracts +against PPI data. So: **NAME only** (RESEARCH Pitfall 2; STRIDE T-29-06). There is +also an SSRF dimension — we never fetch an arbitrary ``settlement_sources[].url`` +handed to us by Kalshi metadata; the agency endpoints are a fixed allowlist the +fetchers (29-05/06/07) own, keyed off the canonical agency NAME resolved here. + +**Not a 607-row enumeration.** The live Kalshi ``GET /series?category=Economics`` +endpoint returns 607 Economics series today and the count drifts monthly +(601→602→607 observed across runs), so a frozen 607-row literal dump would rot. +Instead this is a small, curated set of ROUTING RULES keyed on the canonical +Kalshi series ROOT tickers (KXCPI, KXGDP, KXUSPPI, …). The resolver applies a +rule to any concrete series/contract ticker by root/prefix match, and the CI +drift gate (``export_schemas.py --check`` + EXPORT_MANIFEST SHA) re-verifies the +emitted table byte-for-byte while the live-smoke plan (29-10) re-checks the NAME +set against live Kalshi. Hand-maintaining the full series list is exactly what +this design avoids (ECON-11; STRIDE T-29-08). + +**Per-series, not per-family.** Routing is genuinely per-series: within the PPI +family, KXUSPPI (PPI MoM) settles to Trading Economics while KXUSPPIYOY (PPI YoY) +settles to BLS. The 48 Trading-Economics-settled series (KXUSPPI, KXUSNFP, and +international variants) ship the agency first-print value labeled +``settlement_grade=False`` — we never fabricate a TE value we cannot license +(ECON-17; the agency first print is an honest labeled proxy, and the TE contract +template lists TE as a fallback BELOW the agency, so the divergence is bounded). + +Verified live 2026-07-08 against ``api.elections.kalshi.com`` (607 series; NAME +counts 135 "Bureau of Labor Statistics- Consumer Price Index" / 133 "…- +Employment Situation" / 54 "Bureau of Labor Statistics" / 48 "Trading Economics" +/ 41 "Federal Reserve"; per-series probes confirming every rule below). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +# --- Canonical agency-name vocabulary ---------------------------------------- +#: The five settlement agencies econ v1 routes to, as canonical short names. +#: These are the ONLY values a routing rule's ``agency`` may take. The live +#: Kalshi ``settlement_sources[].name`` field uses several sloppy long forms for +#: the same agency (e.g. "Bureau of Labor Statistics- Consumer Price Index", +#: "Bureau of Labor Statistics- Employment Situation", "Bureau of Labor +#: Statistics", and the bare "BLS" all mean BLS); :func:`normalize_agency_name` +#: collapses those onto this vocabulary. +AGENCY_BLS = "BLS" +AGENCY_BEA = "BEA" +AGENCY_DOL = "DOL" +AGENCY_FED = "FederalReserve" +AGENCY_TE = "TradingEconomics" + +#: The canonical agency vocabulary (sorted-materializable frozenset). Emitted as +#: the sorted ``agency_names`` list in ``kalshi-econ-settlement.json``. +AGENCY_NAMES: frozenset[str] = frozenset( + {AGENCY_BLS, AGENCY_BEA, AGENCY_DOL, AGENCY_FED, AGENCY_TE} +) + +#: Sloppy-long-form → canonical mapping, keyed on the EXACT live Kalshi +#: ``settlement_sources[].name`` strings observed 2026-07-08. A prefix fallback in +#: :func:`normalize_agency_name` catches the "Bureau of Labor Statistics- " +#: family (which grows as Kalshi adds report-specific labels) without re-listing +#: every suffix here. NOTE: these are NAMES, never URLs. +_AGENCY_NAME_ALIASES: dict[str, str] = { + "BLS": AGENCY_BLS, + "Bureau of Labor Statistics": AGENCY_BLS, + "Bureau of Labor Statistics- Consumer Price Index": AGENCY_BLS, + "Bureau of Labor Statistics- Employment Situation": AGENCY_BLS, + "BEA": AGENCY_BEA, + "Bureau of Economic Analysis": AGENCY_BEA, + "DOL": AGENCY_DOL, + "Department of Labor": AGENCY_DOL, + "Federal Reserve": AGENCY_FED, + "Federal Reserve Board of Governors": AGENCY_FED, + "Trading Economics": AGENCY_TE, +} + + +def normalize_agency_name(raw_name: str) -> str: + """Collapse a live Kalshi ``settlement_sources[].name`` onto :data:`AGENCY_NAMES`. + + Routing keys off the agency NAME (never the sloppy URL). Kalshi labels the + same agency several ways ("Bureau of Labor Statistics- Consumer Price Index", + "Bureau of Labor Statistics", "BLS", …); this returns the single canonical + short name for any of them. + + Args: + raw_name: The exact ``settlement_sources[].name`` string from live Kalshi. + + Returns: + The canonical agency short name (a member of :data:`AGENCY_NAMES`). + + Raises: + ValueError: if ``raw_name`` maps to no known agency (never silently + guesses — an unrecognized settlement source is a drift signal the + caller/CI must surface, not paper over). + """ + name = raw_name.strip() + exact = _AGENCY_NAME_ALIASES.get(name) + if exact is not None: + return exact + # Prefix fallback for the "Bureau of Labor Statistics- " family + # so a newly-added BLS report label routes to BLS without a code change. + if name.startswith("Bureau of Labor Statistics"): + return AGENCY_BLS + if name.startswith("Federal Reserve"): + return AGENCY_FED + raise ValueError( + f"unrecognized Kalshi settlement-source name {raw_name!r}; it maps to no " + f"canonical agency in {sorted(AGENCY_NAMES)}. This is a routing-drift " + f"signal — verify against live Kalshi settlement_sources before adding it." + ) + + +# --- Settlement routing rules ------------------------------------------------ +@dataclass(frozen=True) +class SettlementRule: + """One immutable routing rule: a Kalshi series root → its settlement agency. + + ``agency`` is a canonical :data:`AGENCY_NAMES` short name (routed on the + Kalshi settlement-source NAME, never the URL). ``indicator`` is the econ + indicator id (the same vocabulary as ``schema.econ.observations.v1``: + ``cpi``/``cpi_core``/``cpi_yoy``/``nfp``/``u3``/``gdp``/``ppi``/``ppi_yoy``/ + ``jobless_claims``/``fed_funds``). ``settlement_grade`` is ``True`` when the + agency first print IS the settlement truth, ``False`` for the + Trading-Economics-settled series (we ship the agency first print as a labeled + proxy). ``contract_terms_pdf`` is the CFTC contract-terms PDF basename when + known (the trusted routing companion to the NAME), else ``None``. + """ + + agency: str + indicator: str + settlement_grade: bool + contract_terms_pdf: str | None = None + + +#: The curated per-series routing table, keyed on canonical Kalshi Economics +#: series ROOT tickers. ROUTED ON AGENCY NAME ONLY — no ``settlement_sources[].url`` +#: is ever consulted here (Pitfall 2: KXPAYROLLS name="BLS" but url=ppi.nr0.htm). +#: The resolver (29-04) applies these rules to any concrete ticker by root match; +#: this is a small rule set, NOT a 607-row enumeration (the live count drifts +#: monthly, so a frozen dump rots — ECON-11). +#: +#: Grades verified live 2026-07-08: the government-agency series carry +#: ``settlement_grade=True`` (agency first print is the settlement truth); the +#: Trading-Economics-settled series (KXUSPPI PPI-MoM, KXUSNFP) carry ``False`` +#: (agency first print shipped as a labeled proxy — we never fabricate a TE value, +#: ECON-17). Note the per-series split within PPI: KXUSPPI (MoM) → TE(False) but +#: KXUSPPIYOY (YoY) → BLS(True). +SETTLEMENT_ROUTING: dict[str, SettlementRule] = { + # --- CPI family → BLS (settlement_grade True) --- + "KXCPI": SettlementRule(AGENCY_BLS, "cpi", True, "CPI.pdf"), + "KXCPICORE": SettlementRule(AGENCY_BLS, "cpi_core", True, "CPICORE.pdf"), + "KXCPIYOY": SettlementRule(AGENCY_BLS, "cpi_yoy", True, "CPIYOY.pdf"), + "KXCPICOREYOY": SettlementRule(AGENCY_BLS, "cpi_core_yoy", True, "CPICOREYOY.pdf"), + # --- Employment (Payrolls / U3) → BLS (True) --- + "KXPAYROLLS": SettlementRule(AGENCY_BLS, "nfp", True, "PAYROLLS.pdf"), + "KXU3": SettlementRule(AGENCY_BLS, "u3", True, "U3.pdf"), + # --- GDP → BEA (True) --- + "KXGDP": SettlementRule(AGENCY_BEA, "gdp", True, "GDP.pdf"), + # --- Fed funds decision → Federal Reserve (True; NOT FRED) --- + "KXFED": SettlementRule(AGENCY_FED, "fed_funds", True, "FED.pdf"), + "KXFEDDECISION": SettlementRule(AGENCY_FED, "fed_decision", True, "FEDDECISION.pdf"), + # --- Initial jobless claims → Department of Labor (True) --- + "KXJOBLESSCLAIMS": SettlementRule(AGENCY_DOL, "jobless_claims", True, "JOBLESSCLAIMS.pdf"), + "KXJOBLESS": SettlementRule(AGENCY_DOL, "jobless_claims", True, "JOBLESSCLAIMS.pdf"), + # --- PPI: PER-SERIES SPLIT --- + # KXUSPPI (PPI MoM) settles to Trading Economics → settlement_grade False + # (agency first print is a labeled proxy; we never fabricate a TE value). + "KXUSPPI": SettlementRule(AGENCY_TE, "ppi", False, "ECONSTATTE.pdf"), + # KXUSPPIYOY (PPI YoY) settles to BLS → settlement_grade True. This split is + # the proof that routing is per-series, not per-family. Its contract-terms PDF + # is its OWN indicator-named basename (PPIYOY.pdf) — the CPIYOY.pdf here was a + # copy-paste from KXCPIYOY (codex round-3 M1). + "KXUSPPIYOY": SettlementRule(AGENCY_BLS, "ppi_yoy", True, "PPIYOY.pdf"), + # KXUSNFP (US NFP, TE template) settles to Trading Economics → False. + "KXUSNFP": SettlementRule(AGENCY_TE, "nfp", False, "ECONSTATTE.pdf"), +} + + +def resolve_settlement(series_ticker: str) -> tuple[str, SettlementRule]: + """Resolve a Kalshi econ ticker to its ``(root, SettlementRule)`` — econ-side. + + This is the econ-package resolver used by ``research_econ`` (29-08) so the + econ surface can map a Kalshi contract to its settlement agency/indicator/grade + WITHOUT importing ``mostlyright.markets`` — the workspace edge is strictly + one-way ``markets → econ`` (adding the reverse would cycle ``uv sync``). It + reads the SAME :data:`SETTLEMENT_ROUTING` table the markets ``kalshi_econ.resolve`` + reads, so both resolvers agree by construction; ``kalshi_econ.resolve`` stays + the markets-facing wrapper that also carries the ``EconResolution`` dataclass. + + Matching is exact-then-longest-root (identical to the markets resolver): + + 1. Exact match on the full (uppercased) ticker, then + 2. The LONGEST :data:`SETTLEMENT_ROUTING` root that ``ticker`` starts with. + + Longest-prefix wins is load-bearing: ``KXUSPPIYOY`` must bind to the + ``KXUSPPIYOY`` rule (BLS / grade True), NOT the shorter ``KXUSPPI`` rule + (TradingEconomics / grade False) it also prefix-matches. + + Args: + series_ticker: A Kalshi econ series root (``"KXCPIYOY"``) or a concrete + dated market ticker (``"KXCPIYOY-26JUL"``). Case-insensitive. + + Returns: + The ``(matched_root, SettlementRule)`` pair. + + Raises: + TypeError: ``series_ticker`` is not a ``str``. + ValueError: ``series_ticker`` matches no known routing root (never returns + ``None`` — an unroutable ticker is an explicit error). + """ + if not isinstance(series_ticker, str): + raise TypeError( + f"series_ticker must be a string (got {type(series_ticker).__name__}={series_ticker!r})" + ) + ticker = series_ticker.upper() + exact = SETTLEMENT_ROUTING.get(ticker) + if exact is not None: + return ticker, exact + best_root: str | None = None + for root in SETTLEMENT_ROUTING: + if not ticker.startswith(root): + continue + # Require the char after the matched root to be the dated-ticker delimiter + # "-" (an exact full-ticker match is already handled above). Otherwise a + # longer uncatalogued series that merely shares a prefix — e.g. + # "KXCPIENERGY" vs "KXCPI" — would silently route to the shorter root + # instead of raising the intended unroutable-ticker error (codex round-2 M10). + if len(ticker) > len(root) and ticker[len(root)] != "-": + continue + if best_root is None or len(root) > len(best_root): + best_root = root + if best_root is None: + raise ValueError( + f"Unknown Kalshi econ series ticker {series_ticker!r}; it matches no " + f"routing root. known: {sorted(SETTLEMENT_ROUTING)}" + ) + return best_root, SETTLEMENT_ROUTING[best_root] + + +__all__ = [ + "AGENCY_BEA", + "AGENCY_BLS", + "AGENCY_DOL", + "AGENCY_FED", + "AGENCY_NAMES", + "AGENCY_TE", + "SETTLEMENT_ROUTING", + "SettlementRule", + "normalize_agency_name", + "resolve_settlement", +] diff --git a/packages/econ/src/mostlyright/econ/_snapshot.py b/packages/econ/src/mostlyright/econ/_snapshot.py new file mode 100644 index 00000000..ab090dda --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_snapshot.py @@ -0,0 +1,175 @@ +"""``econ.snapshot`` — the settlement-target state of an indicator as-of a cutoff. + +``snapshot(indicator, *, as_of=None, source=None, delivery="live")`` returns the +settlement-target state of ``indicator``: the latest settlement-grade +(``settlement_grade == True``) vintage whose ``vintage_date <= as_of`` (default +``as_of``: now, UTC-aware), per observation period. + +This is the econ realization of the cross-vertical ``snapshot`` rule +(``docs/source-identity.md`` §4): *"settlement-target state now"* — the current +best settlement read of the thing a Kalshi contract settles on. Where the weather +``snapshot`` is the current single-provider observation read, the econ +``snapshot`` is the current settlement-grade first-print read: the value a market +would settle against as knowable at ``as_of``. + +It composes the canonical :func:`~mostlyright.econ._history.series` machinery +(``vintages="all"``) rather than duplicating the FEDS-floor / cache / fetch / +validation pipeline, then applies the settlement-target filter on top: + +1. **Reuse** ``series(indicator, floor, as_of, vintages="all", source=, delivery=)`` + — the SAME loud source-identity validation (contract §1/§2, pre-network), the + SAME FEDS-floor guard, the SAME cache/fetch/vintage pipeline. +2. **Settlement-target filter** — keep only ``settlement_grade == True`` rows with + ``vintage_date <= as_of``, then per ``period`` keep the row with the greatest + ``vintage_date`` (the latest settlement-grade read knowable at the cutoff). +3. **Nothing knowable is an ERROR, never empty** — when no settlement-grade + vintage exists at or before ``as_of``, + :class:`~mostlyright.core.exceptions.IndicatorNotYetReleasedError` is raised + (mirroring ``series``'s not-yet-released discipline). NEVER returns + ``[]``/``None``. + +pandas is the ``[pandas]`` extra (``snapshot`` returns a DataFrame): it flows +through ``series`` which lazy-imports pandas with the actionable install hint. +""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from typing import TYPE_CHECKING + +from mostlyright.core.exceptions import IndicatorNotYetReleasedError + +from ._floor import FEDS_FIRST_CONTRACT +from ._history import _expected_release, series + +if TYPE_CHECKING: + import pandas as pd + +__all__ = ["snapshot"] + + +def _coerce_as_of(value: str | date | datetime) -> datetime: + """Coerce an ``as_of`` argument to a tz-aware UTC datetime. + + Accepts an ISO string (``"2025-03-01"`` or a full datetime), a ``date``, or a + ``datetime`` — matching the string-first primary usage of the read surface. + A bare ``str`` previously reached ``_as_utc`` and raised + ``'str' object has no attribute 'tzinfo'`` for every string ``as_of``. + """ + if isinstance(value, str): + try: + parsed: date | datetime = date.fromisoformat(value) + except ValueError: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + value = parsed + if isinstance(value, datetime): + return _as_utc(value) + # A plain date → midnight UTC of that day. + return datetime(value.year, value.month, value.day, tzinfo=UTC) + + +def _as_utc(value: datetime) -> datetime: + """Return a tz-aware UTC datetime (naive input is assumed UTC).""" + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +def snapshot( + indicator: str, + *, + as_of: datetime | None = None, + source: str | None = None, + delivery: str = "live", +) -> pd.DataFrame: + """Return the settlement-target state of ``indicator`` as knowable at ``as_of``. + + Args: + indicator: The indicator id (``"cpi"`` / ``"nfp"`` / ``"gdp"`` / + ``"ppi"`` / ``"jobless_claims"`` / …; the + ``schema.econ.observations.v1`` vocabulary). + as_of: The knowledge-time cutoff. Only vintages whose ``vintage_date`` is + at or before ``as_of`` are considered. Defaults to now (UTC-aware) — + the current settlement-target state. + source: Provenance pin (contract §1), forwarded to + :func:`~mostlyright.econ._history.series`. ``None`` (default) uses the + per-indicator default routing; an unknown source or a valid authority + that cannot serve ``indicator`` raises ``ValueError`` pre-network. + delivery: Where the computation runs (contract §2), forwarded to + ``series``. ``"live"`` (default) | ``"hosted"`` (reserved seam → + :class:`SourceUnavailableError`). + + Returns: + A ``schema.econ.observations.v1`` :class:`pandas.DataFrame` — one + settlement-grade row per period knowable at ``as_of`` (the latest such + vintage per period). + + Raises: + ValueError: an invalid ``source``/``delivery`` (pre-network) — see + ``series``. + SourceUnavailableError: ``delivery="hosted"`` (the reserved seam). + DataAvailabilityError: ``indicator`` is unknown (no FEDS floor). + IndicatorNotYetReleasedError: nothing is knowable at ``as_of`` — no + settlement-grade vintage exists at or before the cutoff. NEVER returns + ``[]``/``None``. + ImportError: pandas (the ``[pandas]`` extra) is not installed. + """ + as_of_dt = _coerce_as_of(as_of) if as_of is not None else datetime.now(UTC) + + # Window the underlying read from the indicator's FEDS floor up through the + # cutoff. An unknown indicator has no floor entry → let ``series`` raise the + # DataAvailabilityError (its assert_within_floor is the single source of + # truth). When the cutoff predates the floor, nothing is knowable — raise the + # not-yet-released error directly rather than pass ``series`` an inverted + # window. + floor = FEDS_FIRST_CONTRACT.get(indicator) + if floor is not None: + floor_dt = datetime(floor.year, floor.month, floor.day, tzinfo=UTC) + if as_of_dt < floor_dt: + raise IndicatorNotYetReleasedError( + indicator, + as_of_dt.date().isoformat(), + expected_release=_expected_release(indicator, floor_dt), + ) + from_dt: datetime = floor_dt + else: + # No floor: hand the cutoff-as-window to ``series`` so its floor guard + # raises the canonical unknown-indicator DataAvailabilityError. + from_dt = as_of_dt + + # Reuse the canonical read (loud source-identity validation + FEDS floor + + # cache/fetch/vintage pipeline). ``vintages="all"`` so we can apply the + # settlement-target filter ourselves. ``series`` raises + # IndicatorNotYetReleasedError when the window has NO rows at all. + all_vintages = series( + indicator, + from_dt, + as_of_dt, + vintages="all", + source=source, + delivery=delivery, + ) + + # Settlement-target filter: settlement-grade rows knowable at as_of, latest + # vintage per period. + knowable = all_vintages[ + (all_vintages["settlement_grade"].astype("boolean").fillna(False)) + & (all_vintages["vintage_date"] <= as_of_dt) + ] + + if knowable.empty: + # There ARE vintages for the window, but none is a settlement-grade read + # knowable at as_of — the settlement target is not yet knowable. + raise IndicatorNotYetReleasedError( + indicator, + as_of_dt.date().isoformat(), + expected_release=_expected_release(indicator, from_dt), + ) + + # Per period keep the LATEST settlement-grade vintage (the settlement-target + # state as knowable at as_of). Stable: sort by vintage_date then drop + # duplicate periods keeping the last (greatest vintage_date). + latest = ( + knowable.sort_values("vintage_date") + .drop_duplicates(subset=["period"], keep="last") + .reset_index(drop=True) + ) + return latest diff --git a/packages/econ/src/mostlyright/econ/_yoy.py b/packages/econ/src/mostlyright/econ/_yoy.py new file mode 100644 index 00000000..26b73b99 --- /dev/null +++ b/packages/econ/src/mostlyright/econ/_yoy.py @@ -0,0 +1,118 @@ +"""Year-over-year (YoY) percent-change computation for the econ YoY contracts. + +Kalshi's YoY series — ``KXCPIYOY`` (cpi_yoy), ``KXCPICOREYOY`` (cpi_core_yoy), +``KXUSPPIYOY`` (ppi_yoy) — settle on the **12-month percent change** the agency +reports, NOT the raw index level. The dispatch previously remapped a YoY indicator +to its base level series and returned the index (e.g. ``317.7`` instead of +``2.9%``) — the adversarial-audit's finding 9 (a silent wrong settlement value). + +This module derives the YoY figure from the base index's FIRST-PRINT vintages: + + yoy[P] = 100 * (index_firstprint[P] / index_firstprint[P - 12 months] - 1) + +Methodology + limitation (documented, mirroring the ALFRED day-granularity +caveat): the YoY is computed first-print-to-first-print — the current period's +first print over the year-ago period's first print. The agency's *released* YoY +uses the year-ago base as it stood on release day (which, for the lightly-revised +CPI / core-CPI / PPI indices these three contracts cover, differs from the +first-print base only by small seasonal-factor revisions — typically within the +one-decimal rounding Kalshi settles at). NFP-YoY (heavily revised) is NOT a +contract, so this approximation is settlement-appropriate for the three YoY +markets. A future refinement can pull the year-ago base as-of the release date. +""" + +from __future__ import annotations + +from typing import Any + + +def _period_minus_12mo(period: str) -> str | None: + """Return the ``"YYYY-MM"`` period 12 months before ``period`` (year - 1). + + Only monthly ``"YYYY-MM"`` periods carry a YoY (the three YoY contracts are all + monthly). A non-monthly / unparseable period returns ``None`` (no YoY emitted). + """ + try: + year_s, month_s = period.split("-")[:2] + return f"{int(year_s) - 1:04d}-{int(month_s):02d}" + except (ValueError, IndexError): + return None + + +def _representative_per_period(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Reduce base vintages to ONE representative row per period. + + Prefers the settlement-grade first print (``settlement_grade=True``); if a + period has no first print (the keyless latest-revised path), the sole keyless + row is used. When several first prints somehow share a period, the earliest + ``vintage_date`` wins (the true first release). + """ + best: dict[str, dict[str, Any]] = {} + for row in rows: + period = str(row.get("period") or "") + if not period or row.get("value") is None: + continue + cur = best.get(period) + if cur is None: + best[period] = row + continue + # Prefer settlement-grade; then the earliest vintage_date. + cur_grade = bool(cur.get("settlement_grade")) + new_grade = bool(row.get("settlement_grade")) + if new_grade and not cur_grade: + best[period] = row + elif new_grade == cur_grade: + cv, nv = cur.get("vintage_date"), row.get("vintage_date") + if nv is not None and cv is not None and nv < cv: + best[period] = row + return best + + +def compute_yoy(base_rows: list[dict[str, Any]], *, indicator: str) -> list[dict[str, Any]]: + """Compute YoY percent-change rows from base index first-print vintages. + + For every period ``P`` whose base representative value and the ``P - 12mo`` + representative value both exist, emit one row with ``value`` = the 12-month + percent change, ``units="percent"``, ``indicator`` = the YoY indicator, and the + current period's provenance (``vintage_date`` / ``knowledge_time`` / + ``settlement_grade`` / ``source`` inherited from ``P``'s representative — so the + YoY becomes knowable exactly when ``P``'s index first print was released). + Periods lacking a 12-month-prior base are dropped (no YoY is computable). + """ + reps = _representative_per_period(base_rows) + out: list[dict[str, Any]] = [] + for period, cur in reps.items(): + prev_period = _period_minus_12mo(period) + if prev_period is None: + continue + prev = reps.get(prev_period) + if prev is None: + continue + try: + cur_v = float(cur["value"]) + prev_v = float(prev["value"]) + except (TypeError, ValueError): + continue + if prev_v == 0: + continue + yoy = 100.0 * (cur_v / prev_v - 1.0) + out.append( + { + "indicator": indicator, + "series_id": cur.get("series_id"), + "period": period, + "value": yoy, + "units": "percent", + "release_datetime": cur.get("release_datetime"), + "vintage_date": cur.get("vintage_date"), + "release_type": cur.get("release_type"), + "settlement_grade": cur.get("settlement_grade"), + "knowledge_time": cur.get("knowledge_time"), + "source": cur.get("source"), + "retrieved_at": cur.get("retrieved_at"), + } + ) + return out + + +__all__ = ["compute_yoy"] diff --git a/packages/econ/src/mostlyright/py.typed b/packages/econ/src/mostlyright/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/packages/econ/tests/fetchers/cassettes/.gitkeep b/packages/econ/tests/fetchers/cassettes/.gitkeep new file mode 100644 index 00000000..0f9296b4 --- /dev/null +++ b/packages/econ/tests/fetchers/cassettes/.gitkeep @@ -0,0 +1,7 @@ +# pytest-recording cassette dir for the econ fetchers. +# +# Keyed ALFRED / BLS-v2 happy-path cassettes land here when a dev captures them +# with a real FRED_API_KEY / BLS_API_KEY (the live-keyed fidelity assertion is +# deferred to the 29-10 @pytest.mark.live smoke — RESEARCH A1/A3 closure). The +# keyless v1 BLS path is exercised in-test with synthetic fixtures + a +# httpx.MockTransport, so CI needs no network and no recorded cassette here. diff --git a/packages/econ/tests/fetchers/test_bea.py b/packages/econ/tests/fetchers/test_bea.py new file mode 100644 index 00000000..faf23cae --- /dev/null +++ b/packages/econ/tests/fetchers/test_bea.py @@ -0,0 +1,220 @@ +"""Tests for ``mostlyright.econ._fetchers.bea`` — NIPA T10101 quarterly GDP. + +The BEA API (``apps.bea.gov/api/data``) serves NIPA table rows; a GDP pull for a +quarter can arrive in THREE estimates (advance → second → third) published ~1mo +apart. The advance estimate is the FIRST PRINT (Kalshi's settlement truth) → +``settlement_grade=True``; the later two are revisions → ``settlement_grade=False``. + +BEA requires a free ``BEA_API_KEY``; a bad-key request returns a STRUCTURED JSON +error object (``BEAAPI.Results.Error`` / ``BEAAPI.Error``) which MUST raise, never +be parsed as data (threat T-29-18). A5: the exact estimate-type field is not +keyed-verified — this fetcher infers advance/second/third from the release date +relative to the BEA schedule (documented) and lets an explicit release-type field +override the inference when present. + +No network here: BEA responses are injected via ``httpx.MockTransport`` so CI is +offline. Live BEA fidelity (A5 closure) is deferred to the 29-10 ``@pytest.mark.live`` +smoke. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest +from mostlyright.core.exceptions import DataAvailabilityError, SourceUnavailableError +from mostlyright.econ import _schema +from mostlyright.econ._fetchers import bea + + +# --- Synthetic BEA NIPA T10101 payloads -------------------------------------- +# BEA GetData wraps rows in BEAAPI.Results.Data[]; each datum carries +# {TableName, SeriesCode, LineNumber, LineDescription, TimePeriod:"2026Q2", +# CL_UNIT, UNIT_MULT, DataValue:"2.8", NoteRef}. Real GDP % change (SAAR) is +# line 1 of T10101 ("Gross domestic product"). We include a couple of other +# lines to prove the fetcher selects the GDP line, not a component. +def _nipa_payload(time_period: str, gdp_value: str) -> dict[str, Any]: + return { + "BEAAPI": { + "Request": {"RequestParam": []}, + "Results": { + "Statistic": "NIPA", + "UTCProductionTime": "2026-07-30T12:30:00", + "Dimensions": [], + "Data": [ + { + "TableName": "T10101", + "SeriesCode": "A191RL", + "LineNumber": "1", + "LineDescription": "Gross domestic product", + "TimePeriod": time_period, + "CL_UNIT": "Percent", + "UNIT_MULT": "0", + "DataValue": gdp_value, + "NoteRef": "T10101", + }, + { + "TableName": "T10101", + "SeriesCode": "DPCERL", + "LineNumber": "2", + "LineDescription": "Personal consumption expenditures", + "TimePeriod": time_period, + "CL_UNIT": "Percent", + "UNIT_MULT": "0", + "DataValue": "1.9", + "NoteRef": "T10101", + }, + ], + }, + } + } + + +# BEA structured error object (bad key). VERIFIED shape: BEA returns HTTP 200 with +# a BEAAPI.Error / BEAAPI.Results.Error object, NOT a data payload. +_BEA_BADKEY_ERROR: dict[str, Any] = { + "BEAAPI": { + "Request": {"RequestParam": []}, + "Error": { + "APIErrorCode": "1", + "APIErrorDescription": "The user does not have a valid API key.", + }, + } +} + + +def _client_returning( + payload: dict[str, Any], captured: dict[str, Any] | None = None, status: int = 200 +) -> httpx.Client: + """An ``httpx.Client`` returning ``payload``; asserts HTTPS + records the URL.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https", "BEA fetch must be HTTPS-only" + if captured is not None: + captured["url"] = str(request.url) + return httpx.Response(status, json=payload) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 1 (RED): parse yields quarter period + percent-SAAR value ---------- +def test_parse_yields_quarter_period_and_value() -> None: + payload = _nipa_payload("2026Q2", "2.8") + rows = bea.parse(payload, release_type="advance") + assert len(rows) == 1 # only the GDP line, not the PCE component line + row = rows[0] + assert row["indicator"] == "gdp" + assert row["period"] == "2026Q2" + assert row["value"] == pytest.approx(2.8) + assert row["units"] == "percent_saar" + assert row["source"] == _schema.ECON_SOURCE_BEA + assert row["release_type"] == "advance" + + +# --- Test 2: advance/second/third → distinct release_type + vintage + grading - +def test_three_estimates_distinct_release_type_and_grading() -> None: + # Same quarter, three estimates released ~1mo apart. Advance is the first + # print (settlement_grade=True); second/third are revisions (False). + advance = bea.parse( + _nipa_payload("2026Q2", "2.8"), + release_type="advance", + vintage_date="2026-07-30", + ) + second = bea.parse( + _nipa_payload("2026Q2", "3.0"), + release_type="second", + vintage_date="2026-08-28", + ) + third = bea.parse( + _nipa_payload("2026Q2", "3.1"), + release_type="third", + vintage_date="2026-09-25", + ) + rows = advance + second + third + + by_type = {r["release_type"]: r for r in rows} + assert set(by_type) == {"advance", "second", "third"} + + # Distinct vintage dates, advance earliest. + vintages = [by_type[t]["vintage_date"] for t in ("advance", "second", "third")] + assert vintages[0] < vintages[1] < vintages[2] + + # Only the advance (first print) is settlement-grade. + assert by_type["advance"]["settlement_grade"] is True + assert by_type["second"]["settlement_grade"] is False + assert by_type["third"]["settlement_grade"] is False + + +# --- Test 3: keyless degrades cleanly; bad-key structured error raises -------- +def test_keyless_degrades_cleanly(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BEA_API_KEY", raising=False) + # No key → a documented degradation signal, NOT a crash and NOT a blind call. + with pytest.raises(DataAvailabilityError) as exc: + bea.fetch_gdp() + assert exc.value.reason == "source_404" + assert "BEA_API_KEY" in exc.value.hint + + +def test_bad_key_structured_error_raises() -> None: + # BEA returns a structured JSON error object for a bad key → must raise, never + # be parsed as GDP data (threat T-29-18). + client = _client_returning(_BEA_BADKEY_ERROR) + with pytest.raises(SourceUnavailableError): + bea.fetch_gdp(key="BAD-KEY", client=client) + + +# --- Test 4: emitted rows validate against schema.econ.observations.v1 ------- +def test_emitted_rows_validate_against_schema() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_nipa_payload("2026Q2", "2.8"), cap) + rows = bea.fetch_gdp(key="MY-BEA-KEY", client=client) + df = _schema.build_econ_dataframe(rows, source=_schema.ECON_SOURCE_BEA) + _schema.validate_econ_dataframe(df) + assert set(df["source"]) == {_schema.ECON_SOURCE_BEA} + # A no-hint bulk fetch is LATEST-REVISED data, not the first print — parse() + # must NOT default to settlement-grade (codex M2). An explicit + # release_type="advance" opts in (covered separately); the production keyless + # GDP path re-stamps False anyway, and GDP settlement comes from ALFRED. + assert not bool(df["settlement_grade"].any()) + + +# --- Test 5: a non-JSON / HTML error body raises, never parsed as empty ------- +def test_non_json_body_raises() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="bea down") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + with pytest.raises(SourceUnavailableError): + bea.fetch_gdp(key="MY-BEA-KEY", client=client) + + +# --- Test 6: the key never leaks into an emitted row -------------------------- +def test_key_never_leaks_into_rows() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_nipa_payload("2026Q2", "2.8"), cap) + rows = bea.fetch_gdp(key="SECRET-BEA-KEY-777", client=client) + blob = json.dumps(rows, default=str) + assert "SECRET-BEA-KEY-777" not in blob + + +# --- Test 7: the outbound request targets apps.bea.gov with NIPA/T10101 ------- +def test_request_targets_bea_nipa_t10101() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_nipa_payload("2026Q2", "2.8"), cap) + bea.fetch_gdp(key="MY-BEA-KEY", client=client) + url = cap["url"] + assert "apps.bea.gov" in url + assert "NIPA" in url + assert "T10101" in url + + +# --- Test 8: release_type inference from a release date (A5) ------------------- +def test_release_type_inferred_from_release_date() -> None: + # A5: when no explicit release-type field is present, advance/second/third is + # inferred from the release month relative to the quarter-end. Q2 ends Jun 30; + # advance ≈ late Jul (+1mo), second ≈ late Aug (+2mo), third ≈ late Sep (+3mo). + assert bea.infer_release_type("2026Q2", "2026-07-30") == "advance" + assert bea.infer_release_type("2026Q2", "2026-08-28") == "second" + assert bea.infer_release_type("2026Q2", "2026-09-25") == "third" diff --git a/packages/econ/tests/fetchers/test_bls.py b/packages/econ/tests/fetchers/test_bls.py new file mode 100644 index 00000000..a0f1f1f6 --- /dev/null +++ b/packages/econ/tests/fetchers/test_bls.py @@ -0,0 +1,264 @@ +"""Tests for ``mostlyright.econ._fetchers.bls`` — keyless v1 / keyed v2, latest-revised. + +The BLS timeseries API serves the LATEST-REVISED series only (no vintage / as-first +-released endpoint), so BLS rows are stamped ``current``/``revised`` provenance and +``settlement_grade=False`` — the first-print settlement truth is ALFRED's job +(RESEARCH Pitfall 5). This fetcher's contract: parse ``Results.series[].data[]`` +correctly, decode BLS period codes (M/Q/A), map the preliminary footnote, switch +v1→v2 on key presence, reject HTML-instead-of-JSON, and — the checker BLOCKER-1 +fix — make PPI final-demand (``WPSFD4``) genuinely fetchable. + +The v1 keyless response shape is VERIFIED live (RESEARCH Code Examples, 2026-07-08). +No network here: responses are injected via ``httpx.MockTransport`` so CI is offline. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest +from mostlyright.core.exceptions import SourceUnavailableError +from mostlyright.econ import _schema +from mostlyright.econ._fetchers import bls + +# --- Synthetic BLS v1 payloads (shape verified live 2026-07-08) --------------- +# Results.series[].data[] = {year, period:"M06", periodName, value, footnotes:[...]}. +# Most-recent month carries a "P"/preliminary footnote; older months are revised. +_BLS_CPI_RESPONSE: dict[str, Any] = { + "status": "REQUEST_SUCCEEDED", + "responseTime": 123, + "message": [], + "Results": { + "series": [ + { + "seriesID": "CUUR0000SA0", + "data": [ + { + "year": "2026", + "period": "M06", + "periodName": "June", + "value": "158.984", + "footnotes": [{"code": "P", "text": "preliminary"}], + }, + { + "year": "2026", + "period": "M05", + "periodName": "May", + "value": "158.500", + "footnotes": [{}], + }, + ], + } + ] + }, +} + +# PPI final-demand (WPSFD4) — the checker BLOCKER-1 series. Verifies PPI is now +# fetchable: the WP-prefixed Producer Price Index database, distinct from CU (CPI). +_BLS_PPI_RESPONSE: dict[str, Any] = { + "status": "REQUEST_SUCCEEDED", + "responseTime": 88, + "message": [], + "Results": { + "series": [ + { + "seriesID": "WPSFD4", + "data": [ + { + "year": "2026", + "period": "M06", + "periodName": "June", + "value": "145.200", + "footnotes": [{"code": "P", "text": "preliminary"}], + }, + { + "year": "2026", + "period": "M05", + "periodName": "May", + "value": "144.900", + "footnotes": [{}], + }, + ], + } + ] + }, +} + + +def _client_capturing( + payload: dict[str, Any], captured: dict[str, Any], status: int = 200 +) -> httpx.Client: + """A client that returns ``payload`` and records the request URL + body. + + Records into ``captured`` so a test can assert the v1↔v2 endpoint switch and + that the request is HTTPS. Also asserts HTTPS-only at the transport boundary. + """ + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https", "BLS fetch must be HTTPS-only" + captured["url"] = str(request.url) + captured["body"] = request.content.decode() if request.content else "" + return httpx.Response(status, json=payload) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 1 (RED): parse decodes period + value + current/revised provenance -- +def test_parse_decodes_period_value_and_provenance() -> None: + rows = bls.parse(_BLS_CPI_RESPONSE) + by_period = {r["period"]: r for r in rows} + assert set(by_period) == {"2026-06", "2026-05"} + + jun = by_period["2026-06"] + assert jun["value"] == pytest.approx(158.984) + assert jun["series_id"] == "CUUR0000SA0" + assert jun["indicator"] == "cpi" + # BLS-API is latest-revised, NOT first-print. + assert jun["settlement_grade"] is False + + +# --- Test 2: preliminary footnote on the most-recent month → release_type ------- +def test_preliminary_footnote_maps_release_type() -> None: + rows = bls.parse(_BLS_CPI_RESPONSE) + by_period = {r["period"]: r for r in rows} + # Most-recent month carries "P" → preliminary; older months → revised. + assert by_period["2026-06"]["release_type"] == "preliminary" + assert by_period["2026-05"]["release_type"] == "revised" + + +# --- Test 3: endpoint switches v1 (keyless) → v2 (keyed) on key presence ------- +def test_endpoint_switches_on_key_presence(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BLS_API_KEY", raising=False) + + cap_v1: dict[str, Any] = {} + client_v1 = _client_capturing(_BLS_CPI_RESPONSE, cap_v1) + bls.fetch(["CUUR0000SA0"], start_year=2025, end_year=2026, client=client_v1) + assert "/publicAPI/v1/timeseries/data/" in cap_v1["url"] + # The keyless body carries NO registrationkey. + assert "registrationkey" not in cap_v1["body"] + + cap_v2: dict[str, Any] = {} + client_v2 = _client_capturing(_BLS_CPI_RESPONSE, cap_v2) + bls.fetch( + ["CUUR0000SA0"], + start_year=2025, + end_year=2026, + key="MY-BLS-KEY", + client=client_v2, + ) + assert "/publicAPI/v2/timeseries/data/" in cap_v2["url"] + # The keyed body carries the registration key (outbound-only). + assert "registrationkey" in cap_v2["body"] + + +# --- Test 4: emitted rows validate against schema.econ.observations.v1 -------- +def test_emitted_rows_validate_against_schema() -> None: + cap: dict[str, Any] = {} + client = _client_capturing(_BLS_CPI_RESPONSE, cap) + rows = bls.fetch(["CUUR0000SA0"], start_year=2025, end_year=2026, client=client) + df = _schema.build_econ_dataframe(rows, source=_schema.ECON_SOURCE_BLS_V1) + _schema.validate_econ_dataframe(df) + # All BLS-API rows are current/revised, never settlement-grade. + assert not bool(df["settlement_grade"].any()) + assert set(df["source"]) == {_schema.ECON_SOURCE_BLS_V1} + + +# --- Test 5: a non-JSON / error body raises, never parsed as empty ------------ +def test_non_json_body_raises() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="rate limited") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + with pytest.raises(SourceUnavailableError): + bls.fetch(["CUUR0000SA0"], start_year=2025, end_year=2026, client=client) + + +# --- Test 5b: a non-REQUEST_SUCCEEDED status raises --------------------------- +def test_request_failed_status_raises() -> None: + payload = {"status": "REQUEST_NOT_PROCESSED", "message": ["daily threshold"], "Results": {}} + cap: dict[str, Any] = {} + client = _client_capturing(payload, cap) + with pytest.raises(SourceUnavailableError): + bls.fetch(["CUUR0000SA0"], start_year=2025, end_year=2026, client=client) + + +# --- Test 6: PPI final-demand (WPSFD4) is present + fetchable ------------------ +def test_ppi_series_present_in_bls_series_map() -> None: + # The checker BLOCKER-1 fix: WPSFD4 (PPI final-demand SA) maps to "ppi". + assert "WPSFD4" in bls.BLS_SERIES + assert bls.BLS_SERIES["WPSFD4"] == "ppi" + # NSA variant also maps to "ppi". + assert bls.BLS_SERIES.get("WPUFD4") == "ppi" + # At least one PPI series is registered. + assert any(v == "ppi" for v in bls.BLS_SERIES.values()) + + +def test_ppi_response_parses_to_ppi_rows() -> None: + rows = bls.parse(_BLS_PPI_RESPONSE) + assert rows, "PPI response must parse to rows" + assert all(r["indicator"] == "ppi" for r in rows) + assert all(r["units"] == "index" for r in rows) + # PPI-MoM rows stay settlement_grade=False (KXUSPPI → TE); still fetchable. + assert all(r["settlement_grade"] is False for r in rows) + by_period = {r["period"]: r for r in rows} + assert by_period["2026-06"]["value"] == pytest.approx(145.200) + + +# --- Test 7: the four canonical BLS series ids are mapped ---------------------- +def test_canonical_bls_series_mapped() -> None: + assert bls.BLS_SERIES["CUUR0000SA0"] == "cpi" + assert bls.BLS_SERIES["CUUR0000SA0L1E"] == "cpi_core" + assert bls.BLS_SERIES["CES0000000001"] == "nfp" + assert bls.BLS_SERIES["LNS14000000"] == "u3" + + +# --- Test 8: the key never leaks into an emitted row -------------------------- +def test_key_never_leaks_into_rows() -> None: + cap: dict[str, Any] = {} + client = _client_capturing(_BLS_CPI_RESPONSE, cap) + rows = bls.fetch( + ["CUUR0000SA0"], + start_year=2025, + end_year=2026, + key="SECRET-BLS-KEY-999", + client=client, + ) + blob = json.dumps(rows, default=str) + assert "SECRET-BLS-KEY-999" not in blob + + +# --- Test 9: quarterly + annual period codes decode --------------------------- +def test_quarterly_and_annual_period_codes_decode() -> None: + payload = { + "status": "REQUEST_SUCCEEDED", + "message": [], + "Results": { + "series": [ + { + "seriesID": "CUUR0000SA0", + "data": [ + { + "year": "2026", + "period": "Q02", + "periodName": "2nd Quarter", + "value": "2.0", + "footnotes": [{}], + }, + { + "year": "2025", + "period": "A01", + "periodName": "Annual", + "value": "3.1", + "footnotes": [{}], + }, + ], + } + ] + }, + } + rows = bls.parse(payload) + periods = {r["period"] for r in rows} + assert "2026Q2" in periods # Q02 → quarter + assert "2025" in periods # A01 → annual (year only) diff --git a/packages/econ/tests/fetchers/test_dol.py b/packages/econ/tests/fetchers/test_dol.py new file mode 100644 index 00000000..c5333b02 --- /dev/null +++ b/packages/econ/tests/fetchers/test_dol.py @@ -0,0 +1,234 @@ +"""Tests for ``mostlyright.econ._fetchers.dol`` — weekly initial jobless claims. + +The DOL ``oui.doleta.gov/unemploy/csv/*.csv`` directory is ENTIRELY bot-walled +(403 HTML) to scripted access (RESEARCH Pitfall 4). The documented workaround is +FRED series ``ICSA`` (seasonally adjusted) / ``ICNSA`` (NSA) for weekly initial +claims. + +The checker-BLOCKER-3 guarantee (the fork is CLOSED, not "settlement_grade=False +always"): WITH a FRED key, the first print is routed through ALFRED (``ICSA`` + +``realtime_start``/``realtime_end``) and the earliest ``realtime_start`` per +week-ending ``date`` carries ``settlement_grade=True`` (subject to the documented +ALFRED day-granularity caveat). KEYLESS degrades to FRED latest-revised labeled +``settlement_grade=False``. An HTML/403 body is treated as an error, NEVER as +empty/valid data. + +To avoid a same-wave import coupling to 29-05's ``fred_alfred.py``, the DOL fetcher +issues the ALFRED GET itself and re-expresses the ~5-line first-release groupby +inline — these tests assert the behavior (first-release grading), not the import. + +No network here: FRED/ALFRED responses are injected via ``httpx.MockTransport`` so +CI is offline. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest +from mostlyright.core.exceptions import SourceUnavailableError +from mostlyright.econ import _schema +from mostlyright.econ._fetchers import dol + +# --- Synthetic ALFRED ICSA payload (realtime vintages) ----------------------- +# Same shape as the CPI ALFRED fixture: rows carry {realtime_start (=vintage_date, +# DAY granularity), date (=week-ending), value}. The week ending 2026-06-27 was +# first released on 2026-07-02, revised on 2026-07-09; the week ending 2026-06-20 +# was released on 2026-06-25 only. +_ALFRED_ICSA_MULTI_VINTAGE: dict[str, Any] = { + "realtime_start": "1776-07-04", + "realtime_end": "9999-12-31", + "units": "lin", + "output_type": 1, + "file_type": "json", + "count": 3, + "observations": [ + { + "realtime_start": "2026-06-25", + "realtime_end": "9999-12-31", + "date": "2026-06-20", + "value": "233000", + }, + { + "realtime_start": "2026-07-02", + "realtime_end": "2026-07-08", + "date": "2026-06-27", + "value": "231000", + }, + { + "realtime_start": "2026-07-09", + "realtime_end": "9999-12-31", + "date": "2026-06-27", + "value": "230000", # revision of the 2026-06-27 week + }, + ], +} + +# Keyless / latest-revised FRED response (NO realtime vintages — a plain +# series/observations pull returns one row per date at realtime_start=today). +_FRED_ICSA_LATEST: dict[str, Any] = { + "realtime_start": "2026-07-08", + "realtime_end": "2026-07-08", + "file_type": "json", + "count": 2, + "observations": [ + { + "realtime_start": "2026-07-08", + "realtime_end": "2026-07-08", + "date": "2026-06-20", + "value": "233000", + }, + { + "realtime_start": "2026-07-08", + "realtime_end": "2026-07-08", + "date": "2026-06-27", + "value": "230000", # latest-revised value, NOT the first print + }, + ], +} + + +def _client_returning( + payload: dict[str, Any], captured: dict[str, Any] | None = None, status: int = 200 +) -> httpx.Client: + """An ``httpx.Client`` returning ``payload``; asserts HTTPS + records the URL.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https", "DOL/FRED fetch must be HTTPS-only" + if captured is not None: + captured["url"] = str(request.url) + captured["params"] = dict(request.url.params) + return httpx.Response(status, json=payload) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 1 (RED): keyed ALFRED ICSA → first-release settlement_grade=True ----- +def test_keyed_alfred_icsa_first_release_is_settlement_grade() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_ALFRED_ICSA_MULTI_VINTAGE, cap) + rows = dol.fetch_initial_claims(key="MY-FRED-KEY", client=client) + + by_period = {r["period"]: r for r in rows} + # One first-release row per week-ending date. + assert set(by_period) == {"2026-06-20", "2026-06-27"} + + week = by_period["2026-06-27"] + # The EARLIEST realtime_start (2026-07-02 → 231000) is the first print, NOT + # the 2026-07-09 revision (230000). + assert week["value"] == pytest.approx(231000.0) + assert week["vintage_date"].date().isoformat() == "2026-07-02" + assert week["settlement_grade"] is True + assert week["indicator"] == "jobless_claims" + assert week["source"] == _schema.ECON_SOURCE_DOL + # The day-granularity caveat is surfaced (same as the other ALFRED sources). + assert week["vintage_precision"] == "day" + + # The request went to ALFRED WITH realtime_start/realtime_end (vintage query). + assert "realtime_start" in cap["params"] + assert "realtime_end" in cap["params"] + assert cap["params"]["series_id"] == "ICSA" + + +# --- Test 2: an HTML / 403 bot-wall body raises, never parsed as empty --------- +def test_html_403_body_raises() -> None: + # Simulates the oui.doleta.gov bot-wall (or a FRED HTML error page): a + # httpx.Response: + return httpx.Response(403, text="403 Forbidden") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + with pytest.raises(SourceUnavailableError): + dol.fetch_initial_claims(key="MY-FRED-KEY", client=client) + + +def test_parse_html_body_raises() -> None: + # The bot-wall guard at the parse layer: an HTML string (not a JSON mapping) + # passed to the parser raises rather than yielding []. + with pytest.raises(SourceUnavailableError): + dol.parse_icsa("bot-walled") # type: ignore[arg-type] + + +# --- Test 3: keyless → latest-revised, settlement_grade=False ----------------- +def test_keyless_degrades_to_latest_revised_not_settlement_grade( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("FRED_API_KEY", raising=False) + cap: dict[str, Any] = {} + client = _client_returning(_FRED_ICSA_LATEST, cap) + rows = dol.fetch_initial_claims(client=client) + + assert rows, "keyless mode must not return [] silently" + # EVERY keyless row is latest-revised → settlement_grade=False (NOT first-print). + assert all(r["settlement_grade"] is False for r in rows) + assert all(r["indicator"] == "jobless_claims" for r in rows) + # Keyless FRED pull carries NO realtime vintage query. + assert "realtime_start" not in cap["params"] + + +# --- Test 4 (BLOCKER-3): keyed released week yields a settlement row ----------- +def test_keyed_released_week_has_settlement_row() -> None: + # The BLOCKER-3 guarantee: for a released week WITH a key, at least one emitted + # row is settlement_grade=True so history("jobless_claims", vintages="settlement") + # over a released window is non-empty. + client = _client_returning(_ALFRED_ICSA_MULTI_VINTAGE) + rows = dol.fetch_initial_claims(key="MY-FRED-KEY", client=client) + settlement_rows = [r for r in rows if r["settlement_grade"] is True] + assert settlement_rows, "a released week must produce at least one settlement row" + # Each settlement row is a genuine first-release (advance release_type). + assert all(r["release_type"] == "advance" for r in settlement_rows) + + +# --- Test 5: emitted rows validate against schema.econ.observations.v1 -------- +def test_emitted_rows_validate_against_schema() -> None: + client = _client_returning(_ALFRED_ICSA_MULTI_VINTAGE) + rows = dol.fetch_initial_claims(key="MY-FRED-KEY", client=client) + df = _schema.build_econ_dataframe(rows, source=_schema.ECON_SOURCE_DOL) + _schema.validate_econ_dataframe(df) + assert set(df["source"]) == {_schema.ECON_SOURCE_DOL} + assert df["units"].iloc[0] == "thousands_persons" or df["units"].iloc[0] is not None + + +# --- Test 6: the key never leaks into an emitted row -------------------------- +def test_key_never_leaks_into_rows() -> None: + client = _client_returning(_ALFRED_ICSA_MULTI_VINTAGE) + rows = dol.fetch_initial_claims(key="SECRET-FRED-KEY-555", client=client) + blob = json.dumps(rows, default=str) + assert "SECRET-FRED-KEY-555" not in blob + + +# --- Test 7: the request targets FRED/ALFRED (not the bot-walled DOL CSV) ------ +def test_request_targets_fred_not_doleta() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_ALFRED_ICSA_MULTI_VINTAGE, cap) + dol.fetch_initial_claims(key="MY-FRED-KEY", client=client) + url = cap["url"] + assert "stlouisfed.org" in url # FRED/ALFRED, the documented substitute + assert "oui.doleta.gov" not in url # NOT the bot-walled CSV directory + + +# --- Test 8: ICNSA (NSA) series is selectable ------------------------------- +def test_icnsa_series_selectable() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_ALFRED_ICSA_MULTI_VINTAGE, cap) + dol.fetch_initial_claims(key="MY-FRED-KEY", series="ICNSA", client=client) + assert cap["params"]["series_id"] == "ICNSA" + + +# --- Test 9: an injected fred_fetch callable overrides the built-in GET -------- +def test_injected_fred_fetch_callable_used() -> None: + # The alternative to a standalone GET: an injected fetch callable (documented + # option (b)). Proves the DOL fetcher does not hard-depend on its own GET. + calls: dict[str, Any] = {} + + def fake_fetch(series_id: str, *, key: str, realtime: bool) -> dict[str, Any]: + calls["series_id"] = series_id + calls["realtime"] = realtime + return _ALFRED_ICSA_MULTI_VINTAGE + + rows = dol.fetch_initial_claims(key="MY-FRED-KEY", fred_fetch=fake_fetch) + assert calls["series_id"] == "ICSA" + assert calls["realtime"] is True # keyed path requests realtime vintages + assert any(r["settlement_grade"] is True for r in rows) diff --git a/packages/econ/tests/fetchers/test_fed.py b/packages/econ/tests/fetchers/test_fed.py new file mode 100644 index 00000000..8b4794d5 --- /dev/null +++ b/packages/econ/tests/fetchers/test_fed.py @@ -0,0 +1,256 @@ +"""Tests for ``mostlyright.econ._fetchers.fed`` — FOMC target-rate decisions. + +KXFED settles to the **Federal Reserve Board of Governors** (verified +``settlement_sources`` name), NOT FRED / St. Louis Fed. So this fetcher sources +per-FOMC target-rate decisions from ``federalreserve.gov`` (the canonical +``monetarypolicy/openmarket.htm`` rate-change record — live-smoke verified +2026-07-09; the H.15 DDP carries no target range), and the tests assert the +endpoint host is a federalreserve.gov host and NOT stlouisfed.org (threat +T-29-16, the SEED-002 wrinkle). + +Each decision row carries BOTH a numeric target (the target-range midpoint — +documented convention) AND a categorical decision (hike/hold/cut). The categorical +is derived DETERMINISTICALLY from the change vs the prior meeting's target (up → +hike, unchanged → hold, down → cut). Fed decisions are announced, not revised → +``settlement_grade=True``, ``release_type="final"``. + +Schema mapping (Claude's discretion per CONTEXT): the row reuses +``schema.econ.observations.v1`` with ``value`` = the numeric target midpoint and +the categorical decision encoded in ``series_id`` as ``"fed_funds:"`` (a +documented convention — ``release_type`` is the vintage enum and is NOT abused for +the categorical). + +No network here: the openmarket.htm HTML (real-markup fixture, structure copied +from the live page 2026-07-09) and a parsed FOMC-statement fixture are injected +via ``httpx.MockTransport`` so CI is offline. Live Fed fidelity (A6 closure) is +exercised by the ``@pytest.mark.live`` smoke. +""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest +from mostlyright.core.exceptions import SourceUnavailableError +from mostlyright.econ import _schema +from mostlyright.econ._fetchers import fed + +# --- Decisions payload (the parse_openmarket_html output contract) ----------- +# A per-FOMC target-range payload. Each entry is a meeting date + the upper/lower +# target-range bound announced that day. Dec 2025 hiked to 4.50-4.75, Jan 2026 +# held, Mar 2026 cut to 4.25-4.50 — exercising hike/hold/cut. +_FED_DECISIONS_FEED: dict[str, Any] = { + "source": "federalreserve.gov openmarket", + "decisions": [ + {"meeting_date": "2025-12-17", "target_upper": 4.75, "target_lower": 4.50}, + {"meeting_date": "2026-01-28", "target_upper": 4.75, "target_lower": 4.50}, + {"meeting_date": "2026-03-18", "target_upper": 4.50, "target_lower": 4.25}, + ], +} + +# --- Real-markup openmarket.htm fixture --------------------------------------- +# Structure copied from the LIVE page (2026-07-09):

YYYY

year headings, +# per-year tables of Date | Increase | Decrease | Level (%). Covers the modern +# range form ("3.50-3.75"), the footnoted 2020 inter-meeting cut ("March 4*"), +# the 2008 point-target era ("1.00" with "..." filler) and the ZIRP integer-lower +# range ("0-0.25"). +_OPENMARKET_HTML = """ +

2025

+ + + + + + + + + +
DateIncreaseDecreaseLevel (%)
December 110253.50-3.75
October 300253.75-4.00
+

2020

+ + + + + + + + + +
DateIncreaseDecreaseLevel (%)
March 1601000-0.25
March 4*0501.00-1.25
+

2008

+ + + + + + + +
DateIncreaseDecreaseLevel (%)
October 29...501.00
+""" + + +def _html_client( + html: str, captured: dict[str, Any] | None = None, status: int = 200 +) -> httpx.Client: + """An ``httpx.Client`` returning ``html``; asserts HTTPS + records the URL.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https", "Fed fetch must be HTTPS-only" + if captured is not None: + captured["url"] = str(request.url) + return httpx.Response(status, text=html) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 1 (RED): per-FOMC rows with meeting date, numeric target, categorical +def test_parse_yields_meeting_rows_with_numeric_and_categorical() -> None: + rows = fed.parse_decisions(_FED_DECISIONS_FEED) + by_period = {r["period"]: r for r in rows} + assert set(by_period) == {"2025-12-17", "2026-01-28", "2026-03-18"} + + mar = by_period["2026-03-18"] + # Numeric target = midpoint of the 4.25-4.50 range = 4.375. + assert mar["value"] == pytest.approx(4.375) + assert mar["indicator"] == "fed_funds" + assert mar["source"] == _schema.ECON_SOURCE_FED + # Fed decisions are announced, not revised. + assert mar["settlement_grade"] is True + assert mar["release_type"] == "final" + # The categorical decision is encoded in series_id (documented convention). + assert mar["series_id"] == "fed_funds:cut" + + +# --- Test 2: categorical derivation is deterministic (up=hike/same=hold/down=cut) +def test_categorical_decision_deterministic() -> None: + rows = fed.parse_decisions(_FED_DECISIONS_FEED) + decisions = {r["period"]: fed.decision_of(r) for r in rows} + # Dec 2025 is the first row in-window → hike only if a prior target is known; + # with no prior it is labeled "hold" (no change detectable) — documented. + assert decisions["2026-01-28"] == "hold" # unchanged 4.50-4.75 + assert decisions["2026-03-18"] == "cut" # 4.75 → 4.50 upper + + +def test_hike_detected_from_rising_target() -> None: + feed = { + "decisions": [ + {"meeting_date": "2026-01-28", "target_upper": 4.50, "target_lower": 4.25}, + {"meeting_date": "2026-03-18", "target_upper": 4.75, "target_lower": 4.50}, + ] + } + rows = fed.parse_decisions(feed) + by_period = {r["period"]: r for r in rows} + assert by_period["2026-03-18"]["series_id"] == "fed_funds:hike" + + +# --- Test 3: the source host is federalreserve.gov, NOT stlouisfed (FRED) ------ +def test_source_is_federalreserve_not_fred() -> None: + cap: dict[str, Any] = {} + client = _html_client(_OPENMARKET_HTML, cap) + fed.fetch_decisions(client=client) + url = cap["url"] + assert url == fed.FED_OPENMARKET_URL + assert "federalreserve.gov" in url + assert "stlouisfed.org" not in url + + +# --- Test 4: emitted rows validate against schema.econ.observations.v1 -------- +def test_emitted_rows_validate_against_schema() -> None: + client = _html_client(_OPENMARKET_HTML) + rows = fed.fetch_decisions(client=client) + df = _schema.build_econ_dataframe(rows, source=_schema.ECON_SOURCE_FED) + _schema.validate_econ_dataframe(df) + assert set(df["source"]) == {_schema.ECON_SOURCE_FED} + # Fed decisions are settlement-grade as-announced. + assert bool(df["settlement_grade"].all()) + + +# --- Test 4b: parse_openmarket_html handles the REAL page's edge forms --------- +def test_openmarket_html_parses_real_markup_forms() -> None: + payload = fed.parse_openmarket_html(_OPENMARKET_HTML) + by_date = {d["meeting_date"]: d for d in payload["decisions"]} + # Year attribution across

sections; header rows skipped. + assert set(by_date) == { + "2025-12-11", + "2025-10-30", + "2020-03-16", + "2020-03-04", + "2008-10-29", + } + # Modern range form. + assert by_date["2025-12-11"] == { + "meeting_date": "2025-12-11", + "target_lower": 3.50, + "target_upper": 3.75, + } + # Footnote anchor ("March 4*") stripped, not corrupted. + assert by_date["2020-03-04"]["target_lower"] == pytest.approx(1.00) + assert by_date["2020-03-04"]["target_upper"] == pytest.approx(1.25) + # ZIRP integer-lower range. + assert by_date["2020-03-16"]["target_lower"] == pytest.approx(0.0) + # Pre-2008 point target: lower == upper. + assert by_date["2008-10-29"]["target_lower"] == by_date["2008-10-29"]["target_upper"] + assert by_date["2008-10-29"]["target_upper"] == pytest.approx(1.00) + + +def test_openmarket_html_rejects_garbage_level_cell() -> None: + bad = _OPENMARKET_HTML.replace("3.50-3.75", "see footnote") + with pytest.raises(SourceUnavailableError): + fed.parse_openmarket_html(bad) + + +def test_openmarket_html_empty_body_raises() -> None: + with pytest.raises(SourceUnavailableError): + fed.parse_openmarket_html("") + + +# --- Test 5: a non-JSON / HTML body without a parseable target raises ---------- +def test_non_matching_html_raises() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="no rate here") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + with pytest.raises(SourceUnavailableError): + fed.fetch_decisions(client=client) + + +# --- Test 6: the FOMC-statement HTML fallback extracts the target sentence ----- +def test_statement_html_fallback_extracts_target() -> None: + # A6 fallback: when only the FOMC statement HTML is available, parse the + # target-range sentence. A matching statement yields a decision row. + statement = ( + "

the Committee decided to maintain the target range for the " + "federal funds rate at 4-1/4 to 4-1/2 percent.

" + ) + row = fed.parse_statement_html(statement, meeting_date="2026-01-28") + assert row["period"] == "2026-01-28" + # 4-1/4 to 4-1/2 → midpoint of 4.25 and 4.50 = 4.375. + assert row["value"] == pytest.approx(4.375) + assert row["indicator"] == "fed_funds" + + +def test_statement_html_without_target_raises() -> None: + # A non-matching statement page raises rather than emitting a garbage value. + with pytest.raises(SourceUnavailableError): + fed.parse_statement_html( + "no target here", meeting_date="2026-01-28" + ) + + +# --- Test 7: the numeric target is the range midpoint (documented convention) -- +def test_numeric_target_is_range_midpoint() -> None: + rows = fed.parse_decisions(_FED_DECISIONS_FEED) + by_period = {r["period"]: r for r in rows} + jan = by_period["2026-01-28"] + # 4.50 lower, 4.75 upper → midpoint 4.625. + assert jan["value"] == pytest.approx(4.625) + assert jan["units"] == "percent" + + +# --- Test 8: rows carry a period == meeting date and are ordered --------------- +def test_rows_period_is_meeting_date() -> None: + rows = fed.parse_decisions(_FED_DECISIONS_FEED) + periods = [r["period"] for r in rows] + assert periods == sorted(periods) # chronological + assert all(len(p) == 10 and p[4] == "-" for p in periods) # YYYY-MM-DD diff --git a/packages/econ/tests/fetchers/test_fred_alfred.py b/packages/econ/tests/fetchers/test_fred_alfred.py new file mode 100644 index 00000000..13d9556e --- /dev/null +++ b/packages/econ/tests/fetchers/test_fred_alfred.py @@ -0,0 +1,211 @@ +"""Tests for ``mostlyright.econ._fetchers.fred_alfred`` — the ALFRED vintage store. + +ALFRED (``api.stlouisfed.org/fred/series/observations`` with +``realtime_start``/``realtime_end``) is the canonical FIRST-PRINT source: it +returns one row per ``(observation date x realtime_start)``, and the earliest +``realtime_start`` per observation date is the first print. BLS serves +latest-revised only, so ALFRED — not BLS — carries the ``settlement_grade=True`` +truth (subject to the day-granularity caveat: ALFRED's ``realtime_start`` is a +calendar date, so it cannot resolve an 8:30-vs-9:45 ET same-morning correction). + +Algorithm lifted from ``fredapi`` (``get_series_all_releases`` / +``get_series_first_release``; Apache-2.0). The package is NOT a dependency — the +in-house httpx fetcher reimplements the groupby-earliest-realtime_start logic. + +No network: the ALFRED response is injected via a synthetic ``httpx.MockTransport`` +so CI stays offline. The live-keyed fidelity assertion (RESEARCH A1) is deferred +to the 29-10 ``@pytest.mark.live`` smoke. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest +from mostlyright.core.exceptions import DataAvailabilityError +from mostlyright.econ import _schema +from mostlyright.econ._fetchers import fred_alfred + +# --- Synthetic ALFRED payloads ----------------------------------------------- +# Shape verified from fredapi source + St. Louis Fed docs (RESEARCH Code Examples): +# rows carry {realtime_start (=vintage_date, DAY granularity), date (=obs period), +# value}. Two realtime_starts for the same observation date = advance + a later +# revision; first_release keeps the EARLIEST realtime_start. + +# CPI (CPIAUCSL): 2026-05 released first on 2026-06-10, revised on 2026-07-11; +# 2026-04 released on 2026-05-13 only. +_ALFRED_MULTI_VINTAGE: dict[str, Any] = { + "realtime_start": "1776-07-04", + "realtime_end": "9999-12-31", + "observation_start": "1776-07-04", + "observation_end": "9999-12-31", + "units": "lin", + "output_type": 1, + "file_type": "json", + "count": 3, + "observations": [ + { + "realtime_start": "2026-05-13", + "realtime_end": "9999-12-31", + "date": "2026-04-01", + "value": "313.100", + }, + { + "realtime_start": "2026-06-10", + "realtime_end": "2026-07-10", + "date": "2026-05-01", + "value": "314.069", + }, + { + "realtime_start": "2026-07-11", + "realtime_end": "9999-12-31", + "date": "2026-05-01", + "value": "314.500", + }, + ], +} + +# Same observation date, TWO realtime_starts that fall on the SAME calendar date +# (an 8:30 print and a 9:45 same-morning correction both dated 2026-06-10). ALFRED +# is day-granular, so both collapse to one date-vintage — the documented #1 +# landmine. The fetcher must surface this, not silently pick one. +_ALFRED_SAME_DAY_CORRECTION: dict[str, Any] = { + "file_type": "json", + "count": 2, + "observations": [ + { + "realtime_start": "2026-06-10", + "realtime_end": "2026-06-10", + "date": "2026-05-01", + "value": "314.069", + }, + { + "realtime_start": "2026-06-10", + "realtime_end": "9999-12-31", + "date": "2026-05-01", + "value": "314.075", # 9:45 ET correction, same calendar date + }, + ], +} + + +def _client_returning(payload: dict[str, Any], status: int = 200) -> httpx.Client: + """An ``httpx.Client`` whose transport always returns ``payload`` as JSON. + + Asserts the request is HTTPS (the fetcher must reject HTTP downgrades) — a + plain-HTTP request would never reach this handler in production because the + fetcher pins the scheme. + """ + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https", "ALFRED fetch must be HTTPS-only" + return httpx.Response(status, json=payload) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 1: first_release keeps the earliest realtime_start per obs date ------ +def test_first_release_keeps_earliest_realtime_start() -> None: + rows = fred_alfred.parse_observations(_ALFRED_MULTI_VINTAGE, series_id="CPIAUCSL") + first = fred_alfred.first_release(rows) + + # One row per observation date (2026-04, 2026-05). + by_period = {r["period"]: r for r in first} + assert set(by_period) == {"2026-04", "2026-05"} + + # 2026-05 keeps the EARLIEST realtime_start (2026-06-10 → value 314.069), + # NOT the 2026-07-11 revision. + may = by_period["2026-05"] + assert may["value"] == pytest.approx(314.069) + assert may["vintage_date"].date().isoformat() == "2026-06-10" + assert may["settlement_grade"] is True + # knowledge_time == vintage_date for econ (leakage cutoff). + assert may["knowledge_time"] == may["vintage_date"] + + +# --- Test 2: all_vintages keeps every row; later vintages settlement_grade=False +def test_all_vintages_keeps_every_row_later_not_settlement_grade() -> None: + rows = fred_alfred.parse_observations(_ALFRED_MULTI_VINTAGE, series_id="CPIAUCSL") + every = fred_alfred.all_vintages(rows) + + # All three observation rows survive (no dedup). + assert len(every) == 3 + + # The 2026-05 pair: the earliest realtime_start is settlement_grade=True, the + # later one is settlement_grade=False. + may = sorted( + (r for r in every if r["period"] == "2026-05"), + key=lambda r: r["vintage_date"], + ) + assert [r["settlement_grade"] for r in may] == [True, False] + assert may[0]["release_type"] != may[1]["release_type"] + assert may[1]["release_type"] == "revised" + + +# --- Test 3: keyless call degrades with a clear signal, not a crash ----------- +def test_keyless_degrades_with_clear_signal(monkeypatch: pytest.MonkeyPatch) -> None: + # No key argument AND no FRED_API_KEY in env → a documented degradation error + # (ALFRED 400s without a key; we refuse to hit it blind). Never an unhandled + # crash or a bare 400. + monkeypatch.delenv("FRED_API_KEY", raising=False) + with pytest.raises(DataAvailabilityError) as exc: + fred_alfred.fetch_vintages("CPIAUCSL") + assert exc.value.reason == "source_404" + assert "FRED_API_KEY" in exc.value.hint + + +# --- Test 4: emitted rows validate against schema.econ.observations.v1 -------- +def test_emitted_rows_validate_against_schema() -> None: + rows = fred_alfred.parse_observations(_ALFRED_MULTI_VINTAGE, series_id="CPIAUCSL") + first = fred_alfred.first_release(rows) + df = _schema.build_econ_dataframe(first, source=_schema.ECON_SOURCE_ALFRED) + # Must not raise. + _schema.validate_econ_dataframe(df) + assert set(df["source"]) == {_schema.ECON_SOURCE_ALFRED} + assert bool(df["settlement_grade"].all()) + + +# --- Test 5: same-calendar-date realtime_starts collapse to ONE date-vintage --- +def test_same_day_correction_collapses_and_is_labeled() -> None: + # The documented day-granularity behavior: two realtime_starts on the SAME + # calendar date map to one vintage_date; the fetcher surfaces a + # vintage_precision="day" note rather than silently choosing. + rows = fred_alfred.parse_observations(_ALFRED_SAME_DAY_CORRECTION, series_id="CPIAUCSL") + first = fred_alfred.first_release(rows) + + # 2026-05 collapses to a single first-release row (one date-vintage). + may = [r for r in first if r["period"] == "2026-05"] + assert len(may) == 1 + # The day-granularity limitation is SURFACED, not hidden. + assert may[0]["vintage_precision"] == "day" + + +# --- Test 6: fetch_vintages end-to-end via injected client -------------------- +def test_fetch_vintages_end_to_end_with_client() -> None: + client = _client_returning(_ALFRED_MULTI_VINTAGE) + rows = fred_alfred.fetch_vintages("CPIAUCSL", key="test-key", client=client) + # Returns first-release rows by default (the settlement-grade store). + by_period = {r["period"]: r for r in rows} + assert set(by_period) == {"2026-04", "2026-05"} + assert by_period["2026-05"]["value"] == pytest.approx(314.069) + assert by_period["2026-05"]["source"] == _schema.ECON_SOURCE_ALFRED + + +# --- Test 7: a key is never embedded in an emitted row ------------------------ +def test_key_never_leaks_into_rows() -> None: + client = _client_returning(_ALFRED_MULTI_VINTAGE) + rows = fred_alfred.fetch_vintages("CPIAUCSL", key="SECRET-KEY-123", client=client) + blob = json.dumps(rows, default=str) + assert "SECRET-KEY-123" not in blob + + +# --- Test 8: an HTML / non-JSON error body raises, never parsed as empty ------- +def test_non_json_body_raises() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="error") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + with pytest.raises(Exception): # noqa: B017 - any error is fine; NOT silent-empty + fred_alfred.fetch_vintages("CPIAUCSL", key="test-key", client=client) diff --git a/packages/econ/tests/test_cache.py b/packages/econ/tests/test_cache.py new file mode 100644 index 00000000..2d5ca7a8 --- /dev/null +++ b/packages/econ/tests/test_cache.py @@ -0,0 +1,325 @@ +"""Contract tests for the standalone econ per-release cache (``econ/_cache.py``). + +The econ cache is a STANDALONE island (the CWOP ``_cache.py`` discipline at +package granularity): a ``filelock``-guarded read-modify-write parquet cache at +``~/.mostlyright/cache/v1/econ/{indicator}/{year}/{month}.parquet`` with its own +``_ECON_KEY_RE`` path validator, its own source tag (``ECON_CACHE_SOURCE``), an +explicit pyarrow schema derived from ``EconObservationsSchema.COLUMNS``, and +atomic ``.tmp``+``os.replace`` writes. + +The load-bearing econ difference from CWOP: econ **KEEPS all vintages** — the +dedup key includes ``vintage_date`` so two rows sharing ``(indicator, period)`` +but with different vintages BOTH persist (unlike weather's single-row +``report_type_priority`` collapse). And there is NO current-month write-skip: +the per-indicator revision window governs which vintage is +``settlement_grade=True`` at read time, not whether to write. + +Firewall assertion: ``econ/_cache.py`` must NOT import +``mostlyright.weather.cache`` (grep-asserted in the plan's ). +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest +from mostlyright.econ._cache import ( + ECON_CACHE_SOURCE, + econ_cache_path, + invalidate_econ, + read_econ_cache, + read_econ_window, + write_econ_cache, +) + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + """Relocate the whole SDK cache root under a tmp dir for each test.""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + # These tests exercise cache MECHANICS with alfred sample rows; opt into + # FRED-persist so the D-LIC-2 carve-out (tested in test_cache_licensing.py) + # does not skip them. + monkeypatch.setenv("MOSTLYRIGHT_PERSIST_FRED", "1") + return tmp_path + + +def _row(**overrides: object) -> dict[str, object]: + """A well-formed econ observation row (one vintage of one period).""" + known = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) # 8:30 ET as UTC + row: dict[str, object] = { + "indicator": "cpi", + "series_id": "CPIAUCSL", + "period": "2026-05", + "value": 314.069, + "units": "index", + "release_datetime": known, + "vintage_date": known, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": known, + "source": "alfred", + "retrieved_at": known, + } + row.update(overrides) + return row + + +# --- Test 1 (RED): write → read roundtrip + idempotence ---------------------- +def test_write_read_roundtrip_and_idempotent(_isolated_cache) -> None: + rows = [_row()] + n = write_econ_cache("cpi", 2026, 6, rows) + assert n == 1 + + # The partition lands at /v1/econ/cpi/2026/06.parquet. + path = econ_cache_path("cpi", 2026, 6) + assert path.exists() + assert path.parts[-4:] == ("econ", "cpi", "2026", "06.parquet") + + persisted = read_econ_cache("cpi", 2026, 6) + assert persisted is not None + assert len(persisted) == 1 + assert persisted[0]["indicator"] == "cpi" + assert persisted[0]["value"] == pytest.approx(314.069) + # Persisted rows are stamped with the cache-source provenance tag. + assert persisted[0]["source"] == ECON_CACHE_SOURCE + + # Re-writing the SAME rows does not duplicate (idempotent merge). + n2 = write_econ_cache("cpi", 2026, 6, rows) + assert n2 == 1 + assert len(read_econ_cache("cpi", 2026, 6)) == 1 + + +# --- Test 2: keep-all-vintages (same (indicator, period), different vintage) -- +def test_keep_all_vintages_same_period(_isolated_cache) -> None: + v1 = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) # advance print + v2 = datetime(2026, 7, 12, 12, 30, tzinfo=UTC) # later revision + rows = [ + _row( + period="2026-05", + vintage_date=v1, + knowledge_time=v1, + release_type="advance", + settlement_grade=True, + value=314.069, + ), + _row( + period="2026-05", + vintage_date=v2, + knowledge_time=v2, + release_type="revised", + settlement_grade=False, + value=314.500, + ), + ] + # Both land in the SAME (year, month) partition of vintage_date? No — they + # are the same OBSERVATION period "2026-05" but different vintages. The + # partition key is the caller-supplied (year, month); write them both there. + n = write_econ_cache("cpi", 2026, 5, rows) + assert n == 2 # BOTH vintages kept — no collapse to one row per period. + + persisted = read_econ_cache("cpi", 2026, 5) + assert persisted is not None + assert len(persisted) == 2 + assert {r["release_type"] for r in persisted} == {"advance", "revised"} + assert sorted(r["value"] for r in persisted) == pytest.approx([314.069, 314.500]) + + +# --- Test 3: true duplicate (same indicator/period/vintage/source) dedups ----- +def test_true_duplicate_dedups_first_seen_wins(_isolated_cache) -> None: + v = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) + # Two rows identical on the vintage key (indicator, period, vintage_date, + # source); the later value must NOT win — first-seen-wins keeps ONE row. + first = _row(period="2026-05", vintage_date=v, knowledge_time=v, value=314.069) + dup = _row(period="2026-05", vintage_date=v, knowledge_time=v, value=999.999) + n = write_econ_cache("cpi", 2026, 5, [first, dup]) + assert n == 1 + + persisted = read_econ_cache("cpi", 2026, 5) + assert persisted is not None + assert len(persisted) == 1 + # first-seen-wins: the FIRST row's value survives. + assert persisted[0]["value"] == pytest.approx(314.069) + + +# --- Test 4: path-traversal keys are rejected -------------------------------- +@pytest.mark.parametrize( + "bad", + [ + "../../etc", + "cpi/../../etc", + "cpi/evil", + "cpi\\evil", + "..", + ".", + "cpi.core", # dot is a path-separator character — rejected + "1cpi", # must start with a letter + "-cpi", # must start with a letter + "", + "a" * 40, # too long + ], +) +def test_path_traversal_key_rejected(bad, _isolated_cache) -> None: + with pytest.raises((ValueError, TypeError)): + econ_cache_path(bad, 2026, 6) + + +def test_non_str_indicator_rejected(_isolated_cache) -> None: + with pytest.raises(TypeError): + econ_cache_path(123, 2026, 6) # type: ignore[arg-type] + + +def test_uppercase_indicator_normalized_to_lowercase(_isolated_cache) -> None: + """Mixed-case indicators normalize to a lowercase partition (CWOP casing rule). + + ``"CPI"`` / ``" Cpi "`` are NOT rejected — they normalize to the same + ``cpi`` partition so a caller passing a stray case variant lands beside the + canonical rows rather than in a divergent segment. + """ + p_upper = econ_cache_path("CPI", 2026, 6) + p_lower = econ_cache_path("cpi", 2026, 6) + p_spaced = econ_cache_path(" Cpi ", 2026, 6) + assert p_upper == p_lower == p_spaced + assert p_lower.parts[-3:] == ("cpi", "2026", "06.parquet") + + +# --- Test 5: MOSTLYRIGHT_CACHE_DIR relocates the partition ------------------- +def test_cache_dir_env_relocates_partition(tmp_path, monkeypatch) -> None: + override = tmp_path / "custom_root" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(override)) + path = econ_cache_path("cpi", 2026, 6) + # The partition lives under the override root, on the /v1/econ path. + assert str(path).startswith(str(override)) + assert path.parts[-5:] == ("v1", "econ", "cpi", "2026", "06.parquet") + + +# --- Test 6: heterogeneous columns preserved (explicit pyarrow schema) -------- +def test_heterogeneous_columns_preserved(_isolated_cache) -> None: + v0 = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) + v1 = datetime(2026, 6, 13, 12, 30, tzinfo=UTC) + # First row omits the optional ``series_id`` + ``units``; second carries them. + rows = [ + { + "indicator": "cpi", + "period": "2026-05", + "value": 314.0, + "release_datetime": v0, + "vintage_date": v0, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": v0, + "source": "alfred", + }, + { + "indicator": "cpi", + "series_id": "CPIAUCSL", + "period": "2026-05", + "value": 315.0, + "units": "index", + "release_datetime": v1, + "vintage_date": v1, + "release_type": "second", + "settlement_grade": False, + "knowledge_time": v1, + "source": "alfred", + }, + ] + n = write_econ_cache("cpi", 2026, 5, rows) + assert n == 2 + + persisted = read_econ_cache("cpi", 2026, 5) + assert persisted is not None + by_type = {r["release_type"]: r for r in persisted} + # The column present only in the SECOND row survives on both rows. + assert by_type["second"]["series_id"] == "CPIAUCSL" + assert by_type["second"]["units"] == "index" + # The first row reads back with the column present (null), not absent. + assert "series_id" in by_type["advance"] + assert by_type["advance"]["series_id"] is None + + +# --- Test 7: sequential merge-writes accumulate (RMW under FileLock) ---------- +def test_sequential_merges_accumulate(_isolated_cache) -> None: + v1 = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) + v2 = datetime(2026, 6, 13, 12, 30, tzinfo=UTC) + write_econ_cache("cpi", 2026, 5, [_row(period="2026-05", vintage_date=v1, knowledge_time=v1)]) + # Second write reads the first's committed rows and merges on top. + total = write_econ_cache( + "cpi", + 2026, + 5, + [_row(period="2026-05", vintage_date=v2, knowledge_time=v2, release_type="second")], + ) + assert total == 2 + persisted = read_econ_cache("cpi", 2026, 5) + assert persisted is not None + assert len(persisted) == 2 + + +# --- Read-miss + empty-write behavior ---------------------------------------- +def test_read_miss_returns_none(_isolated_cache) -> None: + assert read_econ_cache("cpi", 2026, 6) is None + + +def test_empty_write_is_noop(_isolated_cache) -> None: + assert write_econ_cache("cpi", 2026, 6, []) == 0 + assert read_econ_cache("cpi", 2026, 6) is None + + +# --- invalidate removes the partition ---------------------------------------- +def test_invalidate_removes_partition(_isolated_cache) -> None: + write_econ_cache("cpi", 2026, 6, [_row()]) + assert econ_cache_path("cpi", 2026, 6).exists() + assert invalidate_econ("cpi", 2026, 6) is True + assert not econ_cache_path("cpi", 2026, 6).exists() + # Second invalidate is a no-op returning False. + assert invalidate_econ("cpi", 2026, 6) is False + + +# --- read_econ_window spans partitions --------------------------------------- +def test_read_window_spans_partitions(_isolated_cache) -> None: + from datetime import date + + v_may = datetime(2026, 5, 15, 12, 30, tzinfo=UTC) + v_jun = datetime(2026, 6, 15, 12, 30, tzinfo=UTC) + v_jul = datetime(2026, 7, 15, 12, 30, tzinfo=UTC) + write_econ_cache( + "cpi", 2026, 5, [_row(period="2026-04", vintage_date=v_may, knowledge_time=v_may)] + ) + write_econ_cache( + "cpi", 2026, 6, [_row(period="2026-05", vintage_date=v_jun, knowledge_time=v_jun)] + ) + write_econ_cache( + "cpi", 2026, 7, [_row(period="2026-06", vintage_date=v_jul, knowledge_time=v_jul)] + ) + # Window May-June inclusive picks up the May + June partitions, not July. + window = read_econ_window("cpi", date(2026, 5, 1), date(2026, 6, 30)) + assert len(window) == 2 + assert {r["period"] for r in window} == {"2026-04", "2026-05"} + + +def test_read_window_empty_when_nothing_persisted(_isolated_cache) -> None: + from datetime import date + + assert read_econ_window("cpi", date(2026, 5, 1), date(2026, 6, 30)) == [] + + +# --- Firewall: the module must not import the parity-coupled weather cache ---- +def test_module_does_not_import_weather_cache() -> None: + """Mirror the plan's grep gate: the source text carries neither + ``weather.cache`` nor ``from mostlyright.weather`` anywhere (imports OR + prose) — the standalone-island guarantee, grep-asserted.""" + import re as _re + from pathlib import Path + + import mostlyright.econ._cache as mod + + src = mod.__file__ + assert src is not None + text = Path(src).read_text(encoding="utf-8") + # The exact patterns the plan's ``! grep -qE`` gate forbids. + assert _re.search(r"weather\.cache", text) is None + assert _re.search(r"from mostlyright\.weather", text) is None + # And it MUST import the shared internals it is allowed to reuse. + assert "from mostlyright._internal._bounds import assert_path_under" in text + assert "from mostlyright._internal._cache_dir import resolve_cache_root_without_v1" in text diff --git a/packages/econ/tests/test_cache_licensing.py b/packages/econ/tests/test_cache_licensing.py new file mode 100644 index 00000000..0dc96958 --- /dev/null +++ b/packages/econ/tests/test_cache_licensing.py @@ -0,0 +1,138 @@ +"""D-LIC-2: FRED ToU cache carve-out — FRED-derived rows are not persisted by default. + +FRED's Terms of Use prohibit storing/caching/archiving FRED content (Prohibitions +(q); API (l) — see .planning/research/DATA-LICENSING.md). ``write_econ_cache`` +therefore refuses to persist rows whose ``source`` is FRED-derived (``alfred`` or +``dol.icsa`` — DOL is bot-walled, its bytes come from FRED) unless the user opts in +with ``MOSTLYRIGHT_PERSIST_FRED=1``. Agency rows (``bls.v1``/``bea``/``fed`` — +public-domain US-gov works) always persist. The filter is per-ROW and runs BEFORE +the ``econ.cache`` re-stamp (which overwrites ``source``), so origin is knowable. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import Any + +import pytest +from mostlyright.econ import _history +from mostlyright.econ._cache import read_econ_window, write_econ_cache + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + monkeypatch.delenv("MOSTLYRIGHT_PERSIST_FRED", raising=False) + return tmp_path + + +def _row(source: str, period: str = "2025-01", value: float = 300.0) -> dict[str, Any]: + vd = datetime(2025, 2, 12, tzinfo=UTC) + return { + "indicator": "cpi", + "series_id": "CPIAUCSL", + "period": period, + "value": value, + "units": "index", + "release_datetime": vd, + "vintage_date": vd, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": vd, + "source": source, + "retrieved_at": vd, + } + + +def test_alfred_rows_not_persisted_by_default() -> None: + n = write_econ_cache("cpi", 2025, 2, [_row("alfred")]) + assert n == 0 # nothing persisted + assert ( + read_econ_window("cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC)) + == [] + ) + + +def test_dol_icsa_rows_not_persisted_by_default() -> None: + # dol.icsa is FRED-transported → treated as FRED-derived. + n = write_econ_cache("jobless_claims", 2025, 2, [_row("dol.icsa", period="2025-02")]) + assert n == 0 + + +def test_agency_rows_always_persist() -> None: + for src in ("bls.v1", "bea", "fed"): + n = write_econ_cache("cpi", 2025, 2, [_row(src)]) + assert n >= 1, f"agency source {src} must persist" + + +def test_mixed_call_persists_agency_rows_only() -> None: + rows = [_row("alfred", value=300.0), _row("bls.v1", period="2025-02", value=301.0)] + n = write_econ_cache("cpi", 2025, 2, rows) + assert n == 1 # only the bls.v1 row survives (per-row filter) + + +def test_opt_in_persists_fred_rows(monkeypatch) -> None: + monkeypatch.setenv("MOSTLYRIGHT_PERSIST_FRED", "1") + n = write_econ_cache("cpi", 2025, 2, [_row("alfred")]) + assert n == 1 + assert ( + len( + read_econ_window( + "cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC) + ) + ) + == 1 + ) + + +def test_skip_emits_informative_log(caplog) -> None: + with caplog.at_level(logging.INFO): + write_econ_cache("cpi", 2025, 2, [_row("alfred")]) + msg = " ".join(r.getMessage() for r in caplog.records) + assert "FRED" in msg and "MOSTLYRIGHT_PERSIST_FRED" in msg + + +def test_history_default_fred_path_is_fetch_through(monkeypatch) -> None: + # The load-bearing integration guard: with a FRED key, the default history() + # path fetches ALFRED rows, persist SKIPS them (ToU), the re-read is empty — but + # history must return the fetched rows via fetch-through, NOT raise not-yet-released. + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: "fake-key") + d = datetime(2025, 2, 12, tzinfo=UTC) + alfred_rows = [ + { + "indicator": "cpi", + "series_id": "CPIAUCSL", + "period": "2025-01", + "value": 320.0, + "units": "index", + "release_datetime": d, + "vintage_date": d, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": d, + "source": "alfred", + "retrieved_at": d, + } + ] + monkeypatch.setitem( + _history.INDICATOR_FETCHERS, "cpi", lambda a, b, *, source=None: alfred_rows + ) + out = _history.history("cpi", "2025-01-01", "2025-06-30", vintages="settlement") + assert len(out) == 1 # fetch-through: not raised, not empty + assert out["value"].iloc[0] == pytest.approx(320.0) + # And nothing FRED-derived was persisted. + assert ( + read_econ_window("cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC)) + == [] + ) + + +def test_fred_disclaimer_present_in_readme() -> None: + # The wheel ships README as the PyPI long-description; the FRED notice is a + # publish blocker (D-LIC-1). Assert it did not regress. + import pathlib + + readme = pathlib.Path(__file__).resolve().parents[1] / "README.md" + text = " ".join(readme.read_text().split()) # normalize line-wrapping + assert "not endorsed or certified by the Federal Reserve Bank" in text diff --git a/packages/econ/tests/test_codex_review_fixes.py b/packages/econ/tests/test_codex_review_fixes.py new file mode 100644 index 00000000..aeb66ae4 --- /dev/null +++ b/packages/econ/tests/test_codex_review_fixes.py @@ -0,0 +1,159 @@ +"""Regression tests for the Codex cross-model review findings (round 1). + +Each test encodes the concrete repro Codex gave, so the fixed bug cannot silently +return. Offline (mock transport / injected fetchers) — no network, no real keys. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import httpx +import pytest +from mostlyright.core.exceptions import DataAvailabilityError, SourceUnavailableError +from mostlyright.econ import _history, history +from mostlyright.econ._cache import read_econ_window, write_econ_cache +from mostlyright.econ._fetchers import bea, dol, fred_alfred + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + monkeypatch.delenv("MOSTLYRIGHT_PERSIST_FRED", raising=False) + return tmp_path + + +# --- C1: API keys must never appear in a raised exception (URL leak) ----------- +def _status_client(status: int) -> httpx.Client: + return httpx.Client( + transport=httpx.MockTransport(lambda req: httpx.Response(status, text="err")) + ) + + +def test_c1_alfred_error_does_not_leak_fred_key() -> None: + with pytest.raises(SourceUnavailableError) as exc: + fred_alfred.fetch_vintages("CPIAUCSL", key="SECRET-FRED-KEY", client=_status_client(500)) + assert "SECRET-FRED-KEY" not in str(exc.value) + + +def test_c1_bea_error_does_not_leak_bea_key() -> None: + with pytest.raises(SourceUnavailableError) as exc: + bea.fetch_gdp(key="SECRET-BEA-KEY", client=_status_client(429)) + assert "SECRET-BEA-KEY" not in str(exc.value) + + +# --- H4: keyless jobless_claims fails fast (no doomed empty-key request) -------- +def test_h4_keyless_jobless_raises_pre_network() -> None: + with pytest.raises(DataAvailabilityError) as exc: + dol.fetch_initial_claims() # no key, no client, no fred_fetch + assert "FRED_API_KEY" in (exc.value.hint or "") + + +# --- H2: read_econ_window filters per-row vintage_date to the window ----------- +def test_h2_read_window_excludes_vintages_past_to_date() -> None: + def _wk(day: int) -> dict[str, Any]: + vd = datetime(2025, 6, day, tzinfo=UTC) + return { + "indicator": "jobless_claims", + "series_id": "ICSA", + "period": f"2025-06-{day:02d}", + "value": 210000.0, + "units": "thousands_persons", + "release_datetime": vd, + "vintage_date": vd, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": vd, + "source": "dol.icsa", + "retrieved_at": vd, + } + + # Persist four weekly vintages into the June partition (opt into FRED-persist). + import os + + os.environ["MOSTLYRIGHT_PERSIST_FRED"] = "1" + write_econ_cache("jobless_claims", 2025, 6, [_wk(5), _wk(12), _wk(19), _wk(26)]) + out = read_econ_window( + "jobless_claims", datetime(2025, 6, 1, tzinfo=UTC), datetime(2025, 6, 10, tzinfo=UTC) + ) + assert out, "the 2025-06-05 vintage is in-window" + assert all(r["vintage_date"] <= datetime(2025, 6, 10, tzinfo=UTC) for r in out) + assert not any(r["period"] in ("2025-06-19", "2025-06-26") for r in out) + + +# --- H1: a superset window re-fetches the uncovered tail (no silent gap) -------- +def test_h1_partial_cache_refetches_wider_window(monkeypatch) -> None: + calls: list[tuple[Any, Any]] = [] + + def _fake(a, b, *, source=None): + calls.append((a, b)) + rows = [] + # Emit one bls.v1 (agency → persists) row per month in [a, b]. + cur = a if isinstance(a, datetime) else datetime.fromisoformat(str(a)) + end = b if isinstance(b, datetime) else datetime.fromisoformat(str(b)) + y, m = cur.year, cur.month + while (y, m) <= (end.year, end.month): + vd = datetime(y, m, 15, tzinfo=UTC) + rows.append( + { + "indicator": "cpi", + "series_id": "CUUR0000SA0", + "period": f"{y}-{m:02d}", + "value": 300.0 + m, + "units": "index", + "release_datetime": vd, + "vintage_date": vd, + "release_type": "revised", + "settlement_grade": False, + "knowledge_time": vd, + "source": "bls.v1", + "retrieved_at": vd, + } + ) + m = m + 1 if m < 12 else 1 + y = y if m != 1 else y + 1 + return rows + + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) # keyless → bls.v1 agency + monkeypatch.setitem(_history.INDICATOR_FETCHERS, "cpi", _fake) + history( + "cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 3, 31, tzinfo=UTC), vintages="all" + ) + wider = history( + "cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC), vintages="all" + ) + periods = set(wider["period"]) + assert {"2025-04", "2025-05", "2025-06"} <= periods, "Apr-Jun must not be silently dropped" + + +# --- M1: pinned source stamps the requested indicator (not the upstream id) ----- +def test_m1_pinned_fed_decision_indicator(monkeypatch) -> None: + def _fake_fed(a, b, *, source=None): + vd = datetime(2025, 3, 19, tzinfo=UTC) + return [ + { + "indicator": "fed_funds", # fed.py hardcodes this for BOTH fed indicators + "series_id": "fed_funds:cut", + "period": "2025-03-19", + "value": 4.375, + "units": "percent", + "release_datetime": vd, + "vintage_date": vd, + "release_type": "final", + "settlement_grade": True, + "knowledge_time": vd, + "source": "fed", + "retrieved_at": vd, + } + ] + + monkeypatch.setitem(_history.INDICATOR_FETCHERS, "fed_decision", _fake_fed) + out = history( + "fed_decision", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 12, 31, tzinfo=UTC), + vintages="settlement", + source="fed", + ) + assert (out["indicator"] == "fed_decision").all() diff --git a/packages/econ/tests/test_codex_review_round2.py b/packages/econ/tests/test_codex_review_round2.py new file mode 100644 index 00000000..1c357dbc --- /dev/null +++ b/packages/econ/tests/test_codex_review_round2.py @@ -0,0 +1,176 @@ +"""Regression tests for Codex review ROUND 2 findings (the fixes' own bugs + more).""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import httpx +import pytest +from mostlyright.econ import _history, history +from mostlyright.econ._cache import ( + clear_coverage, + invalidate_econ, + is_window_covered, + mark_window_covered, +) +from mostlyright.econ._fetchers import dol, fred_alfred +from mostlyright.econ._settlement_map import resolve_settlement + + +@pytest.fixture(autouse=True) +def _iso(tmp_path, monkeypatch): + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + monkeypatch.setenv("MOSTLYRIGHT_PERSIST_FRED", "1") # exercise persistence + return tmp_path + + +# --- C2: coverage ledger is grade-aware ---------------------------------------- +def test_c2_keyless_coverage_does_not_satisfy_settlement() -> None: + clear_coverage("cpi") + mark_window_covered( + "cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC), settlement=False + ) + # An "all" read is covered; a "settlement" read is NOT (needs a keyed fetch). + assert is_window_covered( + "cpi", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 6, 30, tzinfo=UTC), + need_settlement=False, + ) + assert not is_window_covered( + "cpi", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 6, 30, tzinfo=UTC), + need_settlement=True, + ) + # A settlement-grade fill then satisfies both. + mark_window_covered( + "cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC), settlement=True + ) + assert is_window_covered( + "cpi", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 6, 30, tzinfo=UTC), + need_settlement=True, + ) + + +# --- C1: a window extending past `now` is never frozen (re-fetches) ------------- +def _count_fake(calls: dict[str, int]): + def _fake(a, b, *, source=None): + calls["n"] += 1 + vd = datetime(2025, 1, 15, tzinfo=UTC) # in-window vintage + return [ + { + "indicator": "cpi", + "series_id": "CUUR0000SA0", + "period": "2025-01", + "value": 300.0, + "units": "index", + "release_datetime": vd, + "vintage_date": vd, + "release_type": "revised", + "settlement_grade": False, + "knowledge_time": vd, + "source": "bls.v1", + "retrieved_at": vd, + } + ] + + return _fake + + +def test_c1_future_window_refetches(monkeypatch) -> None: + calls = {"n": 0} + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setitem(_history.INDICATOR_FETCHERS, "cpi", _count_fake(calls)) + # Window ending in 2099 is capped at `now` → never fully covered → re-fetches + # on repeat (codex C1: no permanent freeze of a future-extending window). + history("cpi", "2025-01-01", "2099-12-31", vintages="all") + history("cpi", "2025-01-01", "2099-12-31", vintages="all") + assert calls["n"] == 2, "future-extending window must re-fetch, not freeze" + + +def test_c1_past_window_hits_cache(monkeypatch) -> None: + calls = {"n": 0} + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setitem(_history.INDICATOR_FETCHERS, "cpi", _count_fake(calls)) + # A fully-past window is covered after the first fetch → cache hit on repeat. + history("cpi", "2025-01-01", "2025-01-31", vintages="all") + history("cpi", "2025-01-01", "2025-01-31", vintages="all") + assert calls["n"] == 1, "past window must hit the cache on repeat" + + +# --- C3 + H5: jobless / fed / pinned fetches are windowed ----------------------- +def test_c3_jobless_claims_windowed(monkeypatch) -> None: + def _wide_icsa(*_a, **_k): + rows = [] + for day in (5, 12, 19, 26): + vd = datetime(2025, 6, day, tzinfo=UTC) + rows.append( + { + "indicator": "jobless_claims", + "series_id": "ICSA", + "period": f"2025-06-{day:02d}", + "value": 210000.0, + "units": "thousands_persons", + "release_datetime": vd, + "vintage_date": vd, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": vd, + "source": "dol.icsa", + "retrieved_at": vd, + } + ) + return rows + + # Inject at the dol fetcher level so _fetch_jobless_claims' window filter runs. + monkeypatch.setattr(dol, "fetch_initial_claims", _wide_icsa) + out = _history._fetch_jobless_claims( + datetime(2025, 6, 1, tzinfo=UTC), datetime(2025, 6, 10, tzinfo=UTC) + ) + assert [r["period"] for r in out] == ["2025-06-05"], "only the in-window week" + + +# --- H7: invalidate_econ clears the coverage ledger ---------------------------- +def test_h7_invalidate_clears_coverage() -> None: + mark_window_covered( + "cpi", datetime(2025, 1, 1, tzinfo=UTC), datetime(2025, 6, 30, tzinfo=UTC), settlement=True + ) + assert is_window_covered( + "cpi", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 6, 30, tzinfo=UTC), + need_settlement=True, + ) + invalidate_econ("cpi", 2025, 2) + assert not is_window_covered( + "cpi", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 6, 30, tzinfo=UTC), + need_settlement=True, + ) + + +# --- M10: prefix routing requires the dated-ticker delimiter ------------------- +def test_m10_prefix_requires_delimiter() -> None: + # A dated ticker resolves; a longer uncatalogued series sharing a prefix raises. + root, _rule = resolve_settlement("KXCPI-26JUL") + assert root == "KXCPI" + with pytest.raises(ValueError): + resolve_settlement("KXCPIENERGY") # must NOT silently route to KXCPI + + +# --- H6: a transport error is sanitized (no raw httpx exc / key) --------------- +def test_h6_transport_error_sanitized() -> None: + def _boom(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused", request=request) + + client = httpx.Client(transport=httpx.MockTransport(_boom)) + from mostlyright.core.exceptions import SourceUnavailableError + + with pytest.raises(SourceUnavailableError) as exc: + fred_alfred.fetch_vintages("CPIAUCSL", key="SECRET-KEY", client=client) + assert "SECRET-KEY" not in str(exc.value) + assert exc.value.__cause__ is None # `from None` — httpx exc (w/ .request) dropped diff --git a/packages/econ/tests/test_codex_review_round3.py b/packages/econ/tests/test_codex_review_round3.py new file mode 100644 index 00000000..66efde1c --- /dev/null +++ b/packages/econ/tests/test_codex_review_round3.py @@ -0,0 +1,185 @@ +"""Regression tests for Codex review ROUND 3 findings (econ vertical). + +Each test encodes a round-3 finding's repro: + +- **C1** — the keyless vintage consults the REAL release calendar first, so a + latest-revised row lands on its actual scheduled release day (no look-ahead + leak from the old 1st-of-release-month heuristic). +- **C2** — the cap-at-now marks the ledger covered only THROUGH YESTERDAY; a + window ending today is never frozen (the current day is incomplete — a later + release may still land). +- **H1** — ``fed.py``'s transport GET is sanitized to a ``SourceUnavailableError`` + (never a raw ``httpx`` exception). +- **M1** — ``KXUSPPIYOY`` settles to its OWN contract-terms PDF (``PPIYOY.pdf``), + not CPI's (``CPIYOY.pdf``). + +(C3 / H2 are the TS mirrors in ``packages-ts/econ/tests``; M2 is doc-only.) +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import httpx +import pytest +from mostlyright.core.exceptions import ( + IndicatorNotYetReleasedError, + SourceUnavailableError, +) +from mostlyright.econ import _history, history +from mostlyright.econ._fetchers import bea, fed +from mostlyright.econ._settlement_map import resolve_settlement + + +@pytest.fixture(autouse=True) +def _iso(tmp_path, monkeypatch): + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + monkeypatch.setenv("MOSTLYRIGHT_PERSIST_FRED", "1") # exercise persistence + return tmp_path + + +# --- C1: keyless vintage consults the real release calendar -------------------- +def test_c1_derive_vintage_uses_scheduled_release() -> None: + # gdp 2026Q2's ACTUAL advance release is 2026-07-30 12:30 UTC (curated + # calendar) — NOT the heuristic 1st-of-release-month (2026-07-01). + assert _history._derive_release_vintage("gdp", "2026Q2") == datetime( + 2026, 7, 30, 12, 30, tzinfo=UTC + ) + + +def test_c1_derive_vintage_falls_back_to_heuristic_out_of_table() -> None: + # A period the curated table does NOT cover still gets the 1st-of-release-month + # heuristic (preserves the 29-13 partition-landing for out-of-table periods). + assert _history._derive_release_vintage("gdp", "2099-01") == datetime(2099, 2, 1, tzinfo=UTC) + + +def _fake_gdp_q2(*_a, **_k): + # An arbitrary pre-restamp vintage; _restamp_keyless overwrites it with the + # scheduled release date derived from the (indicator, period). + vd = datetime(2026, 7, 15, tzinfo=UTC) + return [ + { + "indicator": "gdp", + "series_id": "A191RL1Q225SBEA", + "period": "2026Q2", + "value": 2.5, + "units": "percent", + "release_datetime": None, + "vintage_date": vd, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": vd, + "source": "bea.v1", + "retrieved_at": vd, + } + ] + + +def test_c1_fetch_gdp_keyless_stamps_scheduled_vintage(monkeypatch) -> None: + # The keyless BEA path re-stamps the row to the REAL scheduled release + # (2026-07-30), not the heuristic 1st (2026-07-01), and forces grade False. + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setattr(bea, "fetch_gdp", _fake_gdp_q2) + out = _history._fetch_gdp(datetime(2026, 7, 1, tzinfo=UTC), datetime(2026, 8, 5, tzinfo=UTC)) + assert len(out) == 1 + assert out[0]["vintage_date"] == datetime(2026, 7, 30, 12, 30, tzinfo=UTC) + assert out[0]["settlement_grade"] is False + + +def test_c1_fetch_gdp_keyless_excludes_unreleased_quarter(monkeypatch) -> None: + # As of 2026-07-20 the Q2 advance (real release 2026-07-30) has NOT landed; + # the pre-fix heuristic vintage (2026-07-01) WOULD have leaked it in. + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setattr(bea, "fetch_gdp", _fake_gdp_q2) + out = _history._fetch_gdp(datetime(2026, 7, 1, tzinfo=UTC), datetime(2026, 7, 20, tzinfo=UTC)) + assert out == [], "the Q2 advance (releases 2026-07-30) must be outside 07-20" + + +def test_c1_history_keyless_gdp_excludes_lookahead(monkeypatch) -> None: + # End-to-end: a keyless history() window ending BEFORE the real release raises + # not-yet-released (the row is correctly excluded — no look-ahead). + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setattr(bea, "fetch_gdp", _fake_gdp_q2) + with pytest.raises(IndicatorNotYetReleasedError): + history("gdp", "2026-07-01", "2026-07-20", vintages="all") + + +# --- C2: the coverage cap marks only THROUGH YESTERDAY (today stays unfrozen) --- +def _counting_cpi_fetch(vintage: datetime, calls: dict[str, int]): + def _fake(a, b, *, source=None): + calls["n"] += 1 + return [ + { + "indicator": "cpi", + "series_id": "CUUR0000SA0", + "period": vintage.strftime("%Y-%m"), + "value": 300.0, + "units": "index", + "release_datetime": vintage, + "vintage_date": vintage, + "release_type": "revised", + "settlement_grade": False, + "knowledge_time": vintage, + "source": "bls.v1", + "retrieved_at": vintage, + } + ] + + return _fake + + +def test_c2_today_window_never_freezes(monkeypatch) -> None: + # A window ending TODAY is never fully covered (today is incomplete — a later + # release may still land), so BOTH calls re-fetch. Pre-fix (cap at `now`) the + # window was marked covered through today and the second call froze on cache. + today = datetime.now(UTC).date() + vintage = datetime(today.year, today.month, today.day, tzinfo=UTC) - timedelta(days=3) + calls = {"n": 0} + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setitem(_history.INDICATOR_FETCHERS, "cpi", _counting_cpi_fetch(vintage, calls)) + frm = (today - timedelta(days=5)).isoformat() + history("cpi", frm, today.isoformat(), vintages="all") + history("cpi", frm, today.isoformat(), vintages="all") + assert calls["n"] == 2, "a window ending TODAY must re-fetch (today is incomplete)" + + +def test_c2_yesterday_window_hits_cache(monkeypatch) -> None: + # CONTROL: a window ending YESTERDAY is a fully-ELAPSED period → covered after + # the first fetch → the second call hits the cache (proves the cap still + # freezes genuinely-complete windows). + today = datetime.now(UTC).date() + yesterday = today - timedelta(days=1) + vintage = datetime(today.year, today.month, today.day, tzinfo=UTC) - timedelta(days=3) + calls = {"n": 0} + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + monkeypatch.setitem(_history.INDICATOR_FETCHERS, "cpi", _counting_cpi_fetch(vintage, calls)) + frm = (yesterday - timedelta(days=5)).isoformat() + history("cpi", frm, yesterday.isoformat(), vintages="all") + history("cpi", frm, yesterday.isoformat(), vintages="all") + assert calls["n"] == 1, "a fully-elapsed (ends yesterday) window must hit the cache" + + +# --- H1: fed.py transport error is sanitized to SourceUnavailableError ---------- +def test_h1_fed_transport_error_sanitized() -> None: + # A raw httpx transport error from the Fed GET must surface as the documented + # SourceUnavailableError (not a raw httpx exc), with the httpx cause dropped. + def _boom(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused", request=request) + + client = httpx.Client(transport=httpx.MockTransport(_boom)) + with pytest.raises(SourceUnavailableError) as exc: + fed.fetch_decisions(client=client) + assert exc.value.__cause__ is None # `from None` — raw httpx exc dropped + + +# --- M1: KXUSPPIYOY settles to its OWN contract-terms PDF (not CPI's) ----------- +def test_m1_ppiyoy_uses_own_contract_pdf() -> None: + # The KXUSPPIYOY rule pointed at CPI's CPIYOY.pdf (copy-paste from KXCPIYOY); + # every BLS row uses its own indicator-named basename, so PPI YoY is PPIYOY.pdf. + _root, rule = resolve_settlement("KXUSPPIYOY") + assert rule.contract_terms_pdf == "PPIYOY.pdf" + assert rule.indicator == "ppi_yoy" + # ... and it is NOT CPI YoY's PDF (the two BLS rows are distinct). + _cpi_root, cpi_rule = resolve_settlement("KXCPIYOY") + assert cpi_rule.contract_terms_pdf == "CPIYOY.pdf" + assert rule.contract_terms_pdf != cpi_rule.contract_terms_pdf diff --git a/packages/econ/tests/test_codex_review_round4.py b/packages/econ/tests/test_codex_review_round4.py new file mode 100644 index 00000000..9ed6f5ca --- /dev/null +++ b/packages/econ/tests/test_codex_review_round4.py @@ -0,0 +1,39 @@ +"""Regression tests for Codex review ROUND 4 findings (econ vertical polish). + +Round 4 CONVERGED (zero CRITICAL/HIGH); these lock the two MEDIUM fixes: + +- **M-r4-1** — ``_derive_release_vintage`` returns ``None`` for an out-of-range + quarter/month (Python's strict ``date()`` rejects month ``3*q``/``m`` ∉ [1,12]). + This is the PARITY ANCHOR for the TS ``deriveReleaseVintage`` guard: JS + ``Date.UTC`` would silently roll ``2026Q5`` over into 2027, so the TS side now + guards the range explicitly to match this ``None`` (see + ``packages-ts/econ/tests/codexReviewRound4.test.ts``). Unreachable via real + fetchers today (BEA emits Q1-4, BLS emits 01-12) - a defensive boundary guard. +- **M-r4-2** — ``bea.py`` ``fetch_gdp()``'s docstring is doc-only (corrected to + match the round-2 C4 code: an un-hinted bulk fetch defaults to ``"revised"`` / + non-settlement-grade, NOT ``"advance"``); no runtime test. +""" + +from __future__ import annotations + +import pytest +from mostlyright.econ import _history + + +# --- M-r4-1: out-of-range periods derive to None (parity anchor for the TS guard) +@pytest.mark.parametrize("period", ["2026Q5", "2026Q0", "2026-13", "2026-00"]) +def test_derive_release_vintage_rejects_out_of_range_period(period: str) -> None: + # No curated-calendar entry matches, and the heuristic's strict date() rejects + # a month outside [1,12] (3*q for a quarter, m for a month) → None. The TS + # deriveReleaseVintage guard mirrors this exactly (JS Date.UTC would instead + # roll the value over into the next year — a silent parity divergence). + assert _history._derive_release_vintage("gdp", period) is None + + +def test_derive_release_vintage_in_range_boundaries_still_resolve() -> None: + # Guard the guard: the valid extremes (Q4, month 12) for an OUT-OF-TABLE year + # still resolve via the heuristic (they must not be swept up by the range + # rejection). Q4 2099 → quarter-end Dec → +1 month → 2100-01-01; month 12 + # 2099 → +1 month → 2100-01-01. + assert _history._derive_release_vintage("gdp", "2099Q4") is not None + assert _history._derive_release_vintage("cpi", "2099-12") is not None diff --git a/packages/econ/tests/test_errors.py b/packages/econ/tests/test_errors.py new file mode 100644 index 00000000..cb518eb7 --- /dev/null +++ b/packages/econ/tests/test_errors.py @@ -0,0 +1,70 @@ +"""Contract tests for ``IndicatorNotYetReleasedError``. + +"Not yet released" is an EXPECTED state, distinct from "unavailable" (CONTEXT +Area 4): a caller asking for a period whose release has not happened yet gets a +typed, branchable error carrying ``indicator`` / ``period`` / ``expected_release`` +— never ``[]`` / ``None``. The error is defined ONCE in the core hierarchy +(``mostlyright.core.exceptions``) — mirroring ``EarningsError`` / +``NoCWOPDataError`` — so the MCP JSON-RPC serialization and the Python↔TS +lockstep discipline apply uniformly, and re-exported from ``mostlyright.econ`` +so callers can import it from the vertical they are using. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime + +from mostlyright.core.exceptions import IndicatorNotYetReleasedError, MostlyRightError + + +# --- Test 1 (RED): importable from core + subclasses MostlyRightError --------- +def test_importable_and_subclasses_base() -> None: + assert issubclass(IndicatorNotYetReleasedError, MostlyRightError) + + +# --- Test 2: attributes + stable error_code ---------------------------------- +def test_attributes_and_error_code() -> None: + expected = datetime(2026, 7, 15, 12, 30, tzinfo=UTC) + err = IndicatorNotYetReleasedError("cpi", "2026-06", expected_release=expected) + assert err.indicator == "cpi" + assert err.period == "2026-06" + assert err.expected_release == expected + assert err.error_code == "INDICATOR_NOT_YET_RELEASED" + # The period should surface in the message so bare-string logs are useful. + assert "2026-06" in str(err) + + +# --- Test 3: to_dict() is JSON-safe and carries the fields -------------------- +def test_to_dict_is_json_safe() -> None: + expected = datetime(2026, 7, 15, 12, 30, tzinfo=UTC) + err = IndicatorNotYetReleasedError("nfp", "2026-06", expected_release=expected) + payload = err.to_dict() + assert payload["indicator"] == "nfp" + assert payload["period"] == "2026-06" + assert payload["expected_release"] == expected.isoformat() + assert payload["error_code"] == "INDICATOR_NOT_YET_RELEASED" + # Survives json.dumps without a custom encoder. + round_tripped = json.loads(json.dumps(payload)) + assert round_tripped["indicator"] == "nfp" + + +def test_to_dict_expected_release_none() -> None: + # expected_release is optional — when unknown, the payload carries None + # (not a fabricated timestamp). + err = IndicatorNotYetReleasedError("gdp", "2026Q2") + payload = err.to_dict() + assert payload["expected_release"] is None + json.dumps(payload) # still JSON-safe + + +# --- Test 4: econ re-exports the SAME class object (identity) ----------------- +def test_econ_reexports_same_object() -> None: + from mostlyright.core.exceptions import ( + IndicatorNotYetReleasedError as CoreError, + ) + from mostlyright.econ._errors import ( + IndicatorNotYetReleasedError as EconError, + ) + + assert EconError is CoreError diff --git a/packages/econ/tests/test_firewall.py b/packages/econ/tests/test_firewall.py new file mode 100644 index 00000000..5c0c5175 --- /dev/null +++ b/packages/econ/tests/test_firewall.py @@ -0,0 +1,287 @@ +"""Parity-firewall regression — econ is NEVER registered in the 4 weather files. + +The load-bearing isolation guarantee for the econ vertical (the econ analog of +the CWOP firewall tests). ``mostlyright.econ`` carries its OWN +``schema.econ.observations.v1`` + its OWN :data:`ECON_SOURCES` source-identity +union + its OWN error taxonomy, and MUST stay strictly out of the four +parity-frozen weather files: + +1. ``mostlyright._internal.merge.observations`` — the ``SOURCE_PRIORITY`` dedup map. +2. ``mostlyright._internal.merge.climate`` — the climate ``REPORT_TYPE_PRIORITY`` policy. +3. ``mostlyright.live._sources`` — the live source registry. +4. ``mostlyright.research`` — the v0.14.1 settlement join. + +Any drift here invalidates the parity gate (an econ macro-release row leaking +into the weather ``max/min(temp_f)`` merge would silently corrupt a Kalshi +NHIGH/NLOW settlement). These assertions exist so that IF a future change ever +registers econ into a parity file — a source tag in ``SOURCE_PRIORITY``, the +schema in core's eager registry, an ``import mostlyright.econ`` in one of the four +modules — this test FAILS. That is their entire purpose (STRIDE T-29-31). + +The workspace dependency edge is strictly one-way ``markets -> econ`` (29-04 +added ``mostlyrightmd-econ`` to the *markets* deps): ``econ`` depends only on +``core``. This file also proves the reverse edge is absent — ``econ`` never +imports ``mostlyright.markets`` (that would cycle ``uv sync`` / ``uv build``; +T-29-27). + +Fast (in CI): every assertion is a static import / constant / source-grep — no +network, no key. +""" + +from __future__ import annotations + +import ast +import subprocess +import sys +from pathlib import Path + +import mostlyright + +# The econ source-identity union + the standalone schema id under test. +from mostlyright.econ._schema import ( + ECON_SOURCES, + EconObservationsSchema, +) + +# --- Locate the four parity files on disk (for the source-grep assertions) ---- +# Resolved from the installed ``mostlyright`` package root so the paths hold +# regardless of the CWD pytest is invoked from. +_CORE_ROOT = Path(mostlyright.__file__).resolve().parent + +#: The four parity files, addressed both as import paths and on-disk files. +PARITY_MODULES: tuple[str, ...] = ( + "mostlyright.research", + "mostlyright._internal.merge.observations", + "mostlyright._internal.merge.climate", + "mostlyright.live._sources", +) + +PARITY_FILES: tuple[Path, ...] = ( + _CORE_ROOT / "research.py", + _CORE_ROOT / "_internal" / "merge" / "observations.py", + _CORE_ROOT / "_internal" / "merge" / "climate.py", + _CORE_ROOT / "live" / "_sources.py", +) + +#: The econ schema id that must stay OFF core's eager registry. +ECON_SCHEMA_ID = "schema.econ.observations.v1" + + +# --------------------------------------------------------------------------- +# 1. Econ source tags are disjoint from the weather merge policies. +# --------------------------------------------------------------------------- +def test_econ_sources_disjoint_from_observation_source_priority(): + """No econ source tag appears in the observation ``SOURCE_PRIORITY`` map.""" + from mostlyright._internal.merge.observations import SOURCE_PRIORITY + + overlap = ECON_SOURCES & set(SOURCE_PRIORITY) + assert overlap == set(), ( + f"econ source tag(s) leaked into SOURCE_PRIORITY: {sorted(overlap)}. " + "Econ must NEVER be registered in the observation merge policy — a macro " + "release row would corrupt the weather max/min(temp_f) settlement." + ) + # Belt-and-suspenders: the weather map's own keys are exactly the three + # weather sources, none of which is an econ tag. + assert set(SOURCE_PRIORITY) == {"awc", "iem", "ghcnh"} + + +def test_econ_sources_disjoint_from_climate_report_type_priority(): + """No econ source tag appears in the climate ``REPORT_TYPE_PRIORITY`` policy.""" + from mostlyright._internal.merge.climate import REPORT_TYPE_PRIORITY + + overlap = ECON_SOURCES & set(REPORT_TYPE_PRIORITY) + assert overlap == set(), ( + f"econ source tag(s) leaked into climate REPORT_TYPE_PRIORITY: " + f"{sorted(overlap)}. Climate dedup is parity-frozen; econ has no place in it." + ) + + +def test_econ_sources_disjoint_from_live_source_registry(): + """No econ source tag appears in the live ``_sources`` registry structures.""" + from mostlyright.live import _sources as live_sources + + live_source_tags = set(live_sources.SUPPORTED_SOURCES) | set( + live_sources.SOURCE_IDENTITY_TAGS.values() + ) + overlap = ECON_SOURCES & live_source_tags + assert overlap == set(), ( + f"econ source tag(s) leaked into live/_sources: {sorted(overlap)}. " + "The live source registry is weather-only." + ) + # The live registry is exactly the weather live sources. + assert set(live_sources.SUPPORTED_SOURCES) == {"awc", "iem"} + + +# --------------------------------------------------------------------------- +# 2. schema.econ.observations.v1 is absent from core's EAGER schema registry. +# --------------------------------------------------------------------------- +def test_econ_schema_absent_from_core_eager_registry(): + """``schema.econ.observations.v1`` is NOT in core's eager registry. + + Econ registers its schema LAZILY (only on ``import mostlyright.econ``), the + CWOP discipline at package granularity — a base install that never touches + econ pays nothing and core's canonical registry stays econ-free. + + Because the validator's ``_SCHEMA_REGISTRY`` is a process-global dict, merely + importing ``mostlyright.econ`` anywhere earlier in THIS test session would + pollute it. So the eager-registry assertion runs in a CLEAN subprocess that + imports ONLY ``mostlyright.core.schemas`` (never econ) and confirms the econ + id is absent. This is the strongest available check: it proves the eager + import path does not drag econ in. + """ + probe = ( + "import mostlyright.core.schemas as s; " + f"reg = s.SCHEMA_REGISTRY; " + f"assert {ECON_SCHEMA_ID!r} not in reg, " + f"'econ schema leaked into core eager registry: ' + repr(sorted(reg)); " + "print('OK')" + ) + result = subprocess.run( + [sys.executable, "-c", probe], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, ( + "core eager-registry probe failed — econ schema is registered eagerly " + f"(or import broke). stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "OK" in result.stdout + + +def test_econ_schema_registers_only_on_econ_import(): + """The econ schema IS registered once ``mostlyright.econ._schema`` is imported. + + The complement of the eager-registry check: econ is not absent because it is + broken — it registers correctly, just LAZILY. (This import already happened at + module load via the top-level ``from mostlyright.econ._schema import ...``.) + """ + from mostlyright.core.validator import _SCHEMA_REGISTRY + + assert ECON_SCHEMA_ID in _SCHEMA_REGISTRY + assert _SCHEMA_REGISTRY[ECON_SCHEMA_ID] is EconObservationsSchema + + +# --------------------------------------------------------------------------- +# 3. The four parity files do not import / reference econ. +# --------------------------------------------------------------------------- +def test_parity_files_do_not_import_econ(): + """None of the 4 parity files import ``mostlyright.econ`` (source grep).""" + offenders: list[str] = [] + for path in PARITY_FILES: + assert path.is_file(), f"expected parity file missing: {path}" + text = path.read_text(encoding="utf-8") + if "import mostlyright.econ" in text or "from mostlyright.econ" in text: + offenders.append(str(path)) + assert offenders == [], ( + f"parity file(s) import mostlyright.econ: {offenders}. The firewall " + "forbids any econ import in research.py / merge / live sources." + ) + + +def test_parity_files_do_not_reference_econ_schema_or_sources(): + """No parity file mentions the econ schema id or the ``econ.``/``alfred`` tags.""" + # Sentinel strings that would betray an econ registration in a parity file. + sentinels = (ECON_SCHEMA_ID, "econ.cache", "ECON_SOURCES", "mostlyright.econ") + offenders: list[tuple[str, str]] = [] + for path in PARITY_FILES: + text = path.read_text(encoding="utf-8") + for sentinel in sentinels: + if sentinel in text: + offenders.append((str(path), sentinel)) + assert offenders == [], ( + f"parity file(s) reference econ identifiers: {offenders}. The weather " + "parity layer must never see an econ schema id or source tag." + ) + + +# --------------------------------------------------------------------------- +# 4. Econ never imports mostlyright.markets (one-way markets -> econ edge). +# --------------------------------------------------------------------------- +def test_econ_package_never_imports_markets(): + """No file under ``packages/econ/src`` imports ``mostlyright.markets``. + + The workspace dependency edge is strictly one-way ``markets -> econ`` (the + markets resolver imports econ's ``_settlement_map``). Adding the reverse edge + would create a cycle that breaks ``uv sync`` / ``uv build`` (T-29-27). This + grep over the econ source tree proves the reverse edge is absent. + """ + import mostlyright.econ as econ_pkg + + econ_src_root = Path(econ_pkg.__file__).resolve().parent + offenders: list[str] = [] + for py_file in econ_src_root.rglob("*.py"): + text = py_file.read_text(encoding="utf-8") + if "import mostlyright.markets" in text or "from mostlyright.markets" in text: + offenders.append(str(py_file)) + assert offenders == [], ( + f"econ source file(s) import mostlyright.markets: {offenders}. The edge is " + "one-way markets -> econ; a reverse import cycles uv sync/build (T-29-27)." + ) + + +def test_importing_econ_adds_no_parity_modules_beyond_core_baseline(): + """``import mostlyright.econ`` drags in NO parity module the bare ``import + mostlyright`` baseline doesn't already load, and NEVER ``mostlyright.markets``. + + Stronger than a source grep — it inspects the actual import graph. The subtlety + the check must respect: the core package ``mostlyright/__init__`` eagerly + imports ``mostlyright.research`` (it is the top-level ``pairs()`` surface), so + ``research`` lands in ``sys.modules`` for ANY ``import mostlyright.*`` — that is + a core-package fact, not an econ coupling, and it does NOT register econ into + research. The firewall-meaningful assertion is therefore DIFFERENTIAL: econ + must add **zero** parity modules on top of the bare-``import mostlyright`` + baseline (so econ imports no NEW parity file of its own), and must never pull + ``mostlyright.markets`` at all (the one-way ``markets -> econ`` edge; T-29-27). + + Two subprocesses keep the measurement clean: one records the parity modules a + bare ``import mostlyright`` loads; the other records them after also importing + ``mostlyright.econ``. The econ-attributable delta must be empty. + """ + parity_literal = repr(list(PARITY_MODULES)) + + def _loaded_parity(extra_import: str) -> set[str]: + probe = ( + "import sys; import mostlyright; " + f"{extra_import}" + f"print(repr([m for m in {parity_literal} if m in sys.modules]))" + ) + result = subprocess.run( + [sys.executable, "-c", probe], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, ( + f"import probe failed (extra={extra_import!r}). " + f"stdout={result.stdout!r} stderr={result.stderr!r}" + ) + # The last stdout line is the repr'd list (parsed safely, no eval). + return set(ast.literal_eval(result.stdout.strip().splitlines()[-1])) + + baseline = _loaded_parity("") + with_econ = _loaded_parity("import mostlyright.econ; ") + + econ_attributable = with_econ - baseline + assert econ_attributable == set(), ( + "importing mostlyright.econ dragged in parity module(s) the core baseline " + f"does not: {sorted(econ_attributable)}. Econ must import no parity file of " + "its own." + ) + + # ``mostlyright.markets`` must NEVER be loaded by importing econ (nor by the + # bare baseline) — the reverse edge would cycle uv sync/build. + markets_probe = ( + "import sys; import mostlyright.econ; print('mostlyright.markets' in sys.modules)" + ) + markets_result = subprocess.run( + [sys.executable, "-c", markets_probe], + capture_output=True, + text=True, + check=False, + ) + assert markets_result.returncode == 0 + assert markets_result.stdout.strip() == "False", ( + "importing mostlyright.econ loaded mostlyright.markets — the edge is " + "one-way markets -> econ (T-29-27)." + ) diff --git a/packages/econ/tests/test_floor.py b/packages/econ/tests/test_floor.py new file mode 100644 index 00000000..0c64c623 --- /dev/null +++ b/packages/econ/tests/test_floor.py @@ -0,0 +1,82 @@ +"""Contract tests for the FEDS-2026-010 first-contract floor (ECON-16). + +The per-series backtest floor is enforced as a single auditable DATA table +(``FEDS_FIRST_CONTRACT``), NOT prose. A request whose ``from_date`` predates a +series' first Kalshi contract raises out-of-range (``DataAvailabilityError`` with +``reason="out_of_window"``) rather than silently returning partial data — deep +history the public venue never priced must fail loudly. + +The dates are the FEDS 2026-010 Table 1 first-contract dates (CONTEXT Area 1): +CPI MoM Jun 2021, CPI YoY Nov 2022, Unemployment Jul 2021, Payrolls Mar 2023, +GDP Q2 2021, Fed Funds Target Dec 2021, Fed Decision May 2023. PPI has NO dated +FEDS Table-1 row — its floor is derived from the KXUSPPI/KXUSPPIYOY Kalshi +series inception (documented as such, conservative CPI-YoY-aligned). +""" + +from __future__ import annotations + +from datetime import date + +import pytest +from mostlyright.core.exceptions import DataAvailabilityError, MostlyRightError +from mostlyright.econ._floor import FEDS_FIRST_CONTRACT, assert_within_floor + + +# --- Test 1: the table matches CONTEXT Area-1 dates EXACTLY ------------------- +def test_feds_table_matches_context_area1() -> None: + assert FEDS_FIRST_CONTRACT["cpi"] == date(2021, 6, 1) # CPI MoM Jun 2021 + assert FEDS_FIRST_CONTRACT["cpi_yoy"] == date(2022, 11, 1) # CPI YoY Nov 2022 + assert FEDS_FIRST_CONTRACT["u3"] == date(2021, 7, 1) # Unemployment Jul 2021 + assert FEDS_FIRST_CONTRACT["nfp"] == date(2023, 3, 1) # Payrolls Mar 2023 + assert FEDS_FIRST_CONTRACT["gdp"] == date(2021, 4, 1) # GDP Q2 2021 → quarter start + assert FEDS_FIRST_CONTRACT["fed_funds"] == date(2021, 12, 1) # Fed Funds Target Dec 2021 + assert FEDS_FIRST_CONTRACT["fed_decision"] == date(2023, 5, 1) # Fed Decision May 2023 + # every value is a datetime.date + assert all(isinstance(v, date) for v in FEDS_FIRST_CONTRACT.values()) + + +# --- Test 1b: PPI floors present (Kalshi-inception-derived, not FEDS Table-1) -- +def test_ppi_floors_present() -> None: + assert "ppi" in FEDS_FIRST_CONTRACT + assert "ppi_yoy" in FEDS_FIRST_CONTRACT + assert isinstance(FEDS_FIRST_CONTRACT["ppi"], date) + assert isinstance(FEDS_FIRST_CONTRACT["ppi_yoy"], date) + # inception window is 2021-2023 (Kalshi econ inception), conservative + for key in ("ppi", "ppi_yoy"): + assert date(2021, 1, 1) <= FEDS_FIRST_CONTRACT[key] <= date(2023, 12, 31) + + +# --- Test 2: at/above floor → no raise --------------------------------------- +def test_at_or_above_floor_ok() -> None: + # exactly at the floor is allowed + assert assert_within_floor("cpi", date(2021, 6, 1)) is None + # after the floor is allowed + assert assert_within_floor("cpi", date(2024, 1, 1)) is None + assert assert_within_floor("nfp", date(2023, 3, 1)) is None + + +# --- Test 3: below floor → out-of-range raise naming indicator + floor -------- +def test_below_floor_raises() -> None: + with pytest.raises(DataAvailabilityError) as excinfo: + assert_within_floor("nfp", date(2022, 1, 1)) # before Mar 2023 + err = excinfo.value + assert err.reason == "out_of_window" + # the raise names the indicator and the floor date + assert "nfp" in err.hint + assert "2023-03-01" in err.hint + # it is a MostlyRightError so consumers branch on the core hierarchy + assert isinstance(err, MostlyRightError) + + +def test_below_floor_ppi_raises() -> None: + # a request below the PPI inception floor raises too + floor = FEDS_FIRST_CONTRACT["ppi"] + before = date(floor.year - 1, floor.month, floor.day) + with pytest.raises(DataAvailabilityError): + assert_within_floor("ppi", before) + + +# --- Test 4: unknown indicator raises (never silently passes) ---------------- +def test_unknown_indicator_raises() -> None: + with pytest.raises((KeyError, ValueError, DataAvailabilityError)): + assert_within_floor("not_an_indicator", date(2025, 1, 1)) diff --git a/packages/econ/tests/test_history.py b/packages/econ/tests/test_history.py new file mode 100644 index 00000000..c767ba32 --- /dev/null +++ b/packages/econ/tests/test_history.py @@ -0,0 +1,319 @@ +"""Tests for ``econ.history`` — read-time vintage filter, FEDS floor, not-yet-released. + +``history(indicator, from_date, to_date, *, vintages="settlement"|"all")`` is the +public read surface (plan 29-08, replacing the 29-01 stub). It: + +- calls ``assert_within_floor`` FIRST (the FEDS-2026-010 first-contract floor), +- reads the per-release cache (``read_econ_window``); on a miss it dispatches to + the right fetcher (indicator → fetcher map), persists (``write_econ_cache``), + and re-reads, +- applies the read-time vintage filter — ``"settlement"`` keeps only + ``settlement_grade=True`` rows (the first-print settlement truth); ``"all"`` + keeps every vintage, +- raises :class:`IndicatorNotYetReleasedError` (NEVER returns ``[]``/``None``) + when the requested window has no rows at all. + +The dispatch is EXHAUSTIVE over the 6 CONTEXT Area-1 families — the load-bearing +cases the checker flagged are ``ppi``/``ppi_yoy`` → ``bls`` (BLOCKER-1) and +``jobless_claims`` → ``dol`` (BLOCKER-3, keyed ALFRED-ICSA first-release emits +``settlement_grade=True`` so a released window is non-empty). + +No network here: the cache is relocated under ``tmp_path`` and the fetcher +dispatch is monkeypatched to return synthetic schema rows, so CI stays offline. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest +from hypothesis import given +from hypothesis import strategies as st +from mostlyright.core.exceptions import ( + DataAvailabilityError, + IndicatorNotYetReleasedError, +) +from mostlyright.econ import _history, history + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + """Relocate the whole SDK cache root under a tmp dir for each test.""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + # These tests exercise cache MECHANICS with alfred sample rows; opt into + # FRED-persist so the D-LIC-2 carve-out (tested in test_cache_licensing.py) + # does not skip them. + monkeypatch.setenv("MOSTLYRIGHT_PERSIST_FRED", "1") + return tmp_path + + +def _vintage_row( + *, + indicator: str = "cpi", + period: str = "2026-05", + value: float = 314.0, + vintage_date: datetime, + settlement_grade: bool, + release_type: str, +) -> dict[str, Any]: + """A well-formed econ observation row (one vintage of one period).""" + return { + "indicator": indicator, + "series_id": "CPIAUCSL", + "period": period, + "value": value, + "units": "index", + "release_datetime": vintage_date, + "vintage_date": vintage_date, + "release_type": release_type, + "settlement_grade": settlement_grade, + "knowledge_time": vintage_date, + "source": "alfred", + "retrieved_at": vintage_date, + } + + +def _seed_dispatch(monkeypatch, indicator: str, rows: list[dict[str, Any]]) -> dict[str, int]: + """Register a synthetic fetcher for ``indicator`` that records its call count.""" + calls = {"n": 0} + + def _fake_fetch(from_date, to_date, *, source=None): + calls["n"] += 1 + return rows + + monkeypatch.setitem(_history.INDICATOR_FETCHERS, indicator, _fake_fetch) + return calls + + +# --- Test 1 (RED): settlement ⊆ all — the clean-partition property ------------- +def test_settlement_is_subset_of_all(monkeypatch) -> None: + d_first = datetime(2026, 6, 11, tzinfo=UTC) + d_rev = datetime(2026, 7, 15, tzinfo=UTC) + rows = [ + _vintage_row(vintage_date=d_first, settlement_grade=True, release_type="advance"), + _vintage_row( + value=315.0, vintage_date=d_rev, settlement_grade=False, release_type="revised" + ), + ] + _seed_dispatch(monkeypatch, "cpi", rows) + + d0, d1 = datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 7, 31, tzinfo=UTC) + settlement = history("cpi", d0, d1, vintages="settlement") + all_v = history("cpi", d0, d1, vintages="all") + + # settlement keeps only the first print; all keeps both vintages. + assert len(settlement) == 1 + assert bool(settlement["settlement_grade"].iloc[0]) is True + assert len(all_v) == 2 + # The clean-partition property: every settlement row is present in all. + assert set(settlement["vintage_date"]).issubset(set(all_v["vintage_date"])) + + +# --- Test 2: below the FEDS floor raises out-of-range ------------------------- +def test_below_floor_raises_out_of_range(monkeypatch) -> None: + _seed_dispatch(monkeypatch, "cpi", []) + # CPI floor is 2021-06-01; a 2020 request is out of range. + with pytest.raises(DataAvailabilityError) as exc: + history("cpi", datetime(2020, 1, 1, tzinfo=UTC), datetime(2020, 3, 1, tzinfo=UTC)) + assert exc.value.reason == "out_of_window" + + +# --- Test 3: cache miss fetches-then-persists; second call reads from cache ---- +def test_cache_miss_then_hit(monkeypatch) -> None: + d_first = datetime(2026, 6, 11, tzinfo=UTC) + rows = [_vintage_row(vintage_date=d_first, settlement_grade=True, release_type="advance")] + calls = _seed_dispatch(monkeypatch, "cpi", rows) + + d0, d1 = datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 6, 30, tzinfo=UTC) + first = history("cpi", d0, d1) + assert len(first) == 1 + assert calls["n"] == 1 # cache miss → one fetch + + second = history("cpi", d0, d1) + assert len(second) == 1 + assert calls["n"] == 1 # cache hit → NO second fetch + + +# --- Test 4: a not-yet-released period raises (never []/None) ------------------ +def test_not_yet_released_raises(monkeypatch) -> None: + # The fetcher returns nothing for the window (period not released yet). + _seed_dispatch(monkeypatch, "cpi", []) + with pytest.raises(IndicatorNotYetReleasedError): + history("cpi", datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 6, 30, tzinfo=UTC)) + + +def test_invalid_vintages_argument_raises(monkeypatch) -> None: + _seed_dispatch(monkeypatch, "cpi", []) + with pytest.raises(ValueError, match="vintages"): + history( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + vintages="latest", # type: ignore[arg-type] + ) + + +def test_unknown_indicator_raises(monkeypatch) -> None: + # An indicator with no dispatch entry AND no floor is an explicit error, + # never a silent empty frame. (The floor guard fires first here.) + with pytest.raises((ValueError, DataAvailabilityError)): + history( + "not_a_real_indicator", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + ) + + +# --- Test 6 (property, hypothesis): settlement ⊆ all over generated frames ----- +@given( + n_settlement=st.integers(min_value=0, max_value=4), + n_revised=st.integers(min_value=0, max_value=4), +) +def test_partition_invariant_property(tmp_path_factory, n_settlement, n_revised) -> None: + # Build a fresh isolated cache + dispatch per example (function-scoped + # fixtures don't compose with @given, so do it inline). + import mostlyright.econ._cache as _cache_mod + + cache_dir = tmp_path_factory.mktemp("econ_cache") + orig = _cache_mod.resolve_cache_root_without_v1 + _cache_mod.resolve_cache_root_without_v1 = lambda: cache_dir # type: ignore[assignment] + orig_dispatch = dict(_history.INDICATOR_FETCHERS) + try: + base = datetime(2026, 3, 3, tzinfo=UTC) + rows: list[dict[str, Any]] = [] + for i in range(n_settlement): + rows.append( + _vintage_row( + period=f"2026-{i + 1:02d}", + vintage_date=base.replace(day=1 + i), + settlement_grade=True, + release_type="advance", + ) + ) + for j in range(n_revised): + rows.append( + _vintage_row( + period=f"2026-{j + 1:02d}", + value=999.0, + vintage_date=base.replace(day=10 + j), + settlement_grade=False, + release_type="revised", + ) + ) + _history.INDICATOR_FETCHERS["cpi"] = lambda a, b, *, source=None: rows + + d0, d1 = datetime(2026, 3, 1, tzinfo=UTC), datetime(2026, 4, 1, tzinfo=UTC) + if not rows: + with pytest.raises(IndicatorNotYetReleasedError): + history("cpi", d0, d1, vintages="all") + return + all_v = history("cpi", d0, d1, vintages="all") + settlement = history("cpi", d0, d1, vintages="settlement") if n_settlement else None + if settlement is not None: + key = list(zip(settlement["period"], settlement["vintage_date"], strict=False)) + all_key = set(zip(all_v["period"], all_v["vintage_date"], strict=False)) + assert set(key).issubset(all_key) + assert len(all_v) == n_settlement + n_revised + finally: + _cache_mod.resolve_cache_root_without_v1 = orig # type: ignore[assignment] + _history.INDICATOR_FETCHERS.clear() + _history.INDICATOR_FETCHERS.update(orig_dispatch) + + +# --- Test 7 (PPI dispatch — BLOCKER-1): history("ppi", …) reaches BLS ---------- +def test_ppi_dispatches_to_bls(monkeypatch) -> None: + # The PPI family MUST be reachable through the public surface. We assert the + # dispatch entry maps ppi → the bls fetcher and that a PPI request returns + # PPI rows. + from mostlyright.econ._fetchers import bls as bls_fetcher + + captured: dict[str, Any] = {} + + def _fake_bls_fetch(series_ids, *, start_year, end_year, **kw): + captured["series_ids"] = list(series_ids) + captured["years"] = (start_year, end_year) + # bls.fetch is the KEYLESS latest-revised path: settlement_grade=False, + # source=bls.v1. (The settlement-grade FIRST print comes from ALFRED, never + # bls — the earlier version of this test faked settlement_grade=True/ + # source="alfred", which MASKED the orphaned-ALFRED blocker: it asserted a + # grade the real bls path never produces. See 29-13 BATCH-A2.) + return [ + { + "indicator": "ppi", + "series_id": "WPSFD4", + "period": "2026-05", + "value": 145.2, + "units": "index", + "release_datetime": None, + "vintage_date": datetime(2026, 6, 12, tzinfo=UTC), + "knowledge_time": datetime(2026, 6, 12, tzinfo=UTC), + "release_type": "revised", + "settlement_grade": False, + "source": "bls.v1", + "retrieved_at": datetime(2026, 6, 12, tzinfo=UTC), + } + ] + + monkeypatch.setattr(bls_fetcher, "fetch", _fake_bls_fetch) + # Keyless: force the bls path (no FRED key) so the ALFRED branch is not taken. + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + # ppi and ppi_yoy must BOTH be present in the dispatch table. + assert "ppi" in _history.INDICATOR_FETCHERS + assert "ppi_yoy" in _history.INDICATOR_FETCHERS + + d0, d1 = datetime(2026, 5, 1, tzinfo=UTC), datetime(2026, 6, 30, tzinfo=UTC) + # vintages="all" — keyless bls rows are settlement_grade=False, so the default + # settlement filter would (correctly) raise naming FRED_API_KEY. + out = history("ppi", d0, d1, vintages="all") + assert len(out) >= 1 + assert (out["indicator"] == "ppi").all() + assert not out["settlement_grade"].any() # keyless bls is NEVER settlement-grade + # The BLS fetcher was actually invoked with a PPI (WP-prefixed) series id. + assert any(sid.startswith("WP") for sid in captured["series_ids"]) + + +# --- Test 8 (jobless_claims settlement non-empty — BLOCKER-3) ------------------ +def test_jobless_claims_released_window_is_non_empty_settlement(monkeypatch) -> None: + from mostlyright.econ._fetchers import dol as dol_fetcher + + def _fake_dol_fetch(*, key=None, series="ICSA", **kw): + # Keyed ALFRED-ICSA first-release rows carry settlement_grade=True. + return [ + { + "indicator": "jobless_claims", + "series_id": "ICSA", + "period": "2026-06-27", + "value": 231000.0, + "units": "thousands_persons", + "release_datetime": None, + "vintage_date": datetime(2026, 7, 2, tzinfo=UTC), + "knowledge_time": datetime(2026, 7, 2, tzinfo=UTC), + "release_type": "advance", + "settlement_grade": True, + "source": "dol.icsa", + "retrieved_at": datetime(2026, 7, 2, tzinfo=UTC), + } + ] + + monkeypatch.setattr(dol_fetcher, "fetch_initial_claims", _fake_dol_fetch) + + d0, d1 = datetime(2026, 6, 20, tzinfo=UTC), datetime(2026, 7, 5, tzinfo=UTC) + out = history("jobless_claims", d0, d1) # default vintages="settlement" + assert len(out) >= 1, "a released jobless-claims window must NOT be empty" + assert (out["settlement_grade"]).all() + assert (out["indicator"] == "jobless_claims").all() + + +# --- Test 8b: a genuinely future/unreleased week raises ------------------------ +def test_jobless_claims_future_week_raises(monkeypatch) -> None: + from mostlyright.econ._fetchers import dol as dol_fetcher + + def _empty_dol_fetch(*, key=None, series="ICSA", **kw): + return [] # nothing released for a future week + + monkeypatch.setattr(dol_fetcher, "fetch_initial_claims", _empty_dol_fetch) + + future = datetime(2027, 12, 25, tzinfo=UTC) + with pytest.raises(IndicatorNotYetReleasedError): + history("jobless_claims", future, future) diff --git a/packages/econ/tests/test_history_settlement_wiring.py b/packages/econ/tests/test_history_settlement_wiring.py new file mode 100644 index 00000000..0906d74b --- /dev/null +++ b/packages/econ/tests/test_history_settlement_wiring.py @@ -0,0 +1,409 @@ +"""Tests for the ALFRED first-print wiring into the BLS-family read surface (29-12). + +The 29-VERIFICATION BLOCKER: the ALFRED first-print fetcher (the ONLY source of +``settlement_grade=True`` for the BLS family — CPI/CPI-core/NFP/U3/PPI) was +implemented and live-proven but NEVER called by ``history``/``series``/ +``research_econ``. ``_fetch_bls_indicator`` called only ``bls.fetch`` (which +hard-codes ``settlement_grade=False``), so ``history('cpi', vintages='settlement')`` +RAISED for released periods and ``research_econ('KXCPI')`` failed. Two secondary +dispatch defects compounded it: ``cpi_yoy`` had no BLS series (ValueError "out of +sync") and ``cpi_core_yoy`` had no dispatch entry at all (ValueError "no fetcher"). + +This suite closes the gap, mirroring ``dol.fetch_initial_claims``' keyed/keyless +first-print pattern (the working template for jobless_claims): + +- **Keyed** (``_resolve_fred_key`` returns a key) → the ALFRED realtime first + prints (``settlement_grade=True``) for ``FRED_SERIES_BY_INDICATOR[indicator]``. +- **Keyless** → ``bls.fetch`` latest-revised (``settlement_grade=False``); the + settlement filter then raises ``IndicatorNotYetReleasedError`` naming + ``FRED_API_KEY`` as the unlock. + +No env keys, no network: the FRED key is forced via a monkeypatched +``_history._resolve_fred_key`` and the ALFRED transport via a monkeypatched +``fred_alfred.fetch_vintages`` that reuses the REAL first-release / all-vintages +selectors on a synthetic multi-vintage set (so "the settlement value is the FIRST +print, which DIFFERS from the latest revision" is proven, not assumed). +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest +from mostlyright.core.exceptions import IndicatorNotYetReleasedError +from mostlyright.econ import _history, history, research_econ, series +from mostlyright.econ._fetchers import bls as _bls +from mostlyright.econ._fetchers import fred_alfred + +# The BLS-family settlement window: above every FEDS floor, and wide enough to +# span the synthetic first-print + revision vintage dates below. +D0 = datetime(2026, 6, 1, tzinfo=UTC) +D1 = datetime(2026, 7, 31, tzinfo=UTC) + +# The first print (2026-06-11 → 314.0) and a LATER revision (2026-07-15 → 315.0) +# of the SAME observation period. The settlement path must return the FIRST print +# (314.0), never the revision — the whole point of ALFRED over the BLS latest- +# revised series. +_FIRST_PRINT_VALUE = 314.0 +_REVISED_VALUE = 315.0 + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + """Relocate the whole SDK cache root under a tmp dir for each test.""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + return tmp_path + + +def _raw_alfred_vintages(series_id: str, units: str | None) -> list[dict[str, Any]]: + """A synthetic ``parse_observations``-shaped multi-vintage set (one period).""" + return [ + { + "indicator": series_id, + "series_id": series_id, + "period": "2026-05", + "value": _FIRST_PRINT_VALUE, + "units": units, + "release_datetime": None, + "vintage_date": datetime(2026, 6, 11, tzinfo=UTC), + "knowledge_time": datetime(2026, 6, 11, tzinfo=UTC), + "vintage_precision": "day", + "source": fred_alfred.ECON_SOURCE_ALFRED + if hasattr(fred_alfred, "ECON_SOURCE_ALFRED") + else "alfred", + "retrieved_at": None, + }, + { + "indicator": series_id, + "series_id": series_id, + "period": "2026-05", + "value": _REVISED_VALUE, # a LATER revision — must NOT be the settlement value + "units": units, + "release_datetime": None, + "vintage_date": datetime(2026, 7, 15, tzinfo=UTC), + "knowledge_time": datetime(2026, 7, 15, tzinfo=UTC), + "vintage_precision": "day", + "source": "alfred", + "retrieved_at": None, + }, + ] + + +def _install_keyed_alfred(monkeypatch, captured: dict[str, Any] | None = None) -> None: + """Force the keyed branch offline + inject a fake ALFRED transport. + + ``_resolve_fred_key`` is replaced (NOT the env) so no real key is needed. The + fake ``fetch_vintages`` runs the REAL first-release / all-vintages selectors so + the first-print-vs-revision selection is genuine. + """ + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: "TEST-FRED-KEY") + + def _fake_fetch_vintages( + series_id: str, *, key=None, units=None, vintages: str = "first", **_kw + ) -> list[dict[str, Any]]: + if captured is not None: + captured.setdefault("series_ids", []).append(series_id) + captured.setdefault("vintages", []).append(vintages) + raw = _raw_alfred_vintages(series_id, units) + if vintages == "all": + return fred_alfred.all_vintages(raw) + return fred_alfred.first_release(raw) + + monkeypatch.setattr(fred_alfred, "fetch_vintages", _fake_fetch_vintages) + + +# --- Test 1 (BLOCKER): keyed BLS-family settlement returns the ALFRED first print +def test_keyed_cpi_settlement_returns_alfred_first_print(monkeypatch) -> None: + captured: dict[str, Any] = {} + _install_keyed_alfred(monkeypatch, captured) + + out = history("cpi", D0, D1, vintages="settlement") + + assert len(out) >= 1, "a keyed released window must NOT raise / be empty" + # Every settlement row is settlement_grade=True (the first print). + assert out["settlement_grade"].all() + # The value is the FIRST print (314.0), NOT the later revision (315.0). + assert out["value"].iloc[0] == pytest.approx(_FIRST_PRINT_VALUE) + assert _REVISED_VALUE not in set(out["value"]) + # The read re-stamps the canonical indicator; the FRED series id survives. + assert (out["indicator"] == "cpi").all() + # The ALFRED call targeted the VERIFIED FRED series id (CPIAUCSL), not a BLS id. + assert captured["series_ids"] == ["CPIAUCSL"] + # The ALFRED fetch pulls ALL vintages (cache completeness); the read-time + # filter then selects the settlement first print. (BATCH C: fetching "all" + # makes vintages="all" return every vintage instead of only first prints.) + assert captured["vintages"] == ["all"] + + +# --- Test 2: ppi keyed path targets the FRED PPIFIS id (NOT the BLS WPSFD4) ----- +def test_keyed_ppi_targets_fred_ppifis_not_bls_wpsfd4(monkeypatch) -> None: + captured: dict[str, Any] = {} + _install_keyed_alfred(monkeypatch, captured) + + out = history("ppi", D0, D1, vintages="settlement") + assert len(out) >= 1 + assert out["settlement_grade"].all() + # PPIFIS is the FRED final-demand PPI id; WPSFD4 (the BLS id) does NOT exist on + # FRED, so the ALFRED path must use PPIFIS. + assert captured["series_ids"] == ["PPIFIS"] + + +# --- Test 3: keyless settlement raises IndicatorNotYetReleasedError naming the key +def test_keyless_cpi_settlement_raises_naming_fred_api_key(monkeypatch) -> None: + # Keyless: no FRED key → the BLS latest-revised path (settlement_grade=False). + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) + + def _fake_bls_fetch(series_ids, *, start_year, end_year, **_kw): + # Latest-revised BLS rows — settlement_grade=False (the truthful keyless + # grade), vintage_date in-window so they persist + read back. + return [ + { + "indicator": _bls.BLS_SERIES.get(series_ids[0], "cpi"), + "series_id": series_ids[0], + "period": "2026-05", + "value": _REVISED_VALUE, + "units": "index", + "release_datetime": None, + "vintage_date": datetime(2026, 6, 12, tzinfo=UTC), + "knowledge_time": datetime(2026, 6, 12, tzinfo=UTC), + "release_type": "revised", + "settlement_grade": False, + "source": "bls.v1", + "retrieved_at": datetime(2026, 6, 12, tzinfo=UTC), + } + ] + + monkeypatch.setattr(_bls, "fetch", _fake_bls_fetch) + + with pytest.raises(IndicatorNotYetReleasedError) as exc: + history("cpi", D0, D1, vintages="settlement") + # The keyless degrade names FRED_API_KEY as the unlock (reads identically to + # the jobless family). + assert "FRED_API_KEY" in str(exc.value) + + # And vintages="all" still returns the latest-revised rows (never []/None) — + # the degrade is loud for settlement, graceful for the raw series. + all_rows = history("cpi", D0, D1, vintages="all") + assert len(all_rows) >= 1 + assert not all_rows["settlement_grade"].any() + + +# --- Test 4 (dispatch repair): cpi_yoy + cpi_core_yoy dispatch without ValueError +def test_cpi_yoy_and_cpi_core_yoy_dispatch(monkeypatch) -> None: + # Both YoY variants must be present in the dispatch table (cpi_core_yoy was + # absent entirely; cpi_yoy pointed at a non-existent BLS series). + assert "cpi_yoy" in _history.INDICATOR_FETCHERS + assert "cpi_core_yoy" in _history.INDICATOR_FETCHERS + + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) # keyless → BLS + calls: dict[str, Any] = {"series_ids": []} + + def _fake_bls_fetch(series_ids, *, start_year, end_year, **_kw): + calls["series_ids"].append(list(series_ids)) + # YoY needs the year-ago base too (the fetch auto-widens 12mo): return + # 2025-05 (300.0) + 2026-05 (309.0) so cpi_yoy computes 3.0%. + return [ + { + "indicator": _bls.BLS_SERIES.get(series_ids[0]), + "series_id": series_ids[0], + "period": period, + "value": value, + "units": "index", + "release_datetime": None, + "vintage_date": vintage, + "knowledge_time": vintage, + "release_type": "revised", + "settlement_grade": False, + "source": "bls.v1", + "retrieved_at": vintage, + } + for period, value, vintage in ( + ("2025-05", 300.0, datetime(2025, 6, 12, tzinfo=UTC)), + ("2026-05", 309.0, datetime(2026, 6, 12, tzinfo=UTC)), + ) + ] + + monkeypatch.setattr(_bls, "fetch", _fake_bls_fetch) + + # vintages="all" so the (settlement_grade=False) latest-revised rows survive + # the read-time filter — the point is the DISPATCH resolves + computes YoY %. + out_yoy = history("cpi_yoy", D0, D1, vintages="all") + assert len(out_yoy) >= 1 + assert (out_yoy["units"] == "percent").all() # YoY %, not the index level + assert out_yoy["value"].iloc[0] == pytest.approx(3.0) + out_core_yoy = history("cpi_core_yoy", D0, D1, vintages="all") + assert len(out_core_yoy) >= 1 + + # cpi_yoy remapped to the CPI series (CU-prefixed); cpi_core_yoy to the CPI-core + # series — never a phantom "cpi_yoy" BLS id. + flat = [sid for group in calls["series_ids"] for sid in group] + assert all(sid.startswith("CU") for sid in flat) + assert "CUUR0000SA0" in flat # cpi + assert "CUUR0000SA0L1E" in flat # cpi_core + + +# --- Test 5: series (canonical) mirrors history for the keyed settlement path ---- +def test_series_keyed_settlement_matches_history(monkeypatch) -> None: + _install_keyed_alfred(monkeypatch) + out = series("nfp", D0, D1, vintages="settlement") + assert len(out) >= 1 + assert out["settlement_grade"].all() + assert (out["indicator"] == "nfp").all() + + +# --- Test 6 (payoff): research_econ('KXCPI') returns non-empty settlement pairs -- +def test_research_econ_kxcpi_non_empty_with_injected_alfred(monkeypatch) -> None: + _install_keyed_alfred(monkeypatch) + + pairs = research_econ("KXCPI", D0, D1) + assert len(pairs) >= 1 + # KXCPI settles to BLS (a real agency first print) → contract grade True. + assert bool(pairs["settlement_grade"].iloc[0]) is True + assert (pairs["indicator"] == "cpi").all() + assert (pairs["agency"] == "BLS").all() + # The pair value is the ALFRED FIRST print, not the revision. + assert pairs["value"].iloc[0] == pytest.approx(_FIRST_PRINT_VALUE) + + +# --- Test 7 (payoff): research_econ for NFP + PPI-YoY contracts also resolve ----- +def test_research_econ_payrolls_and_ppiyoy_non_empty(monkeypatch) -> None: + _install_keyed_alfred(monkeypatch) + + payrolls = research_econ("KXPAYROLLS", D0, D1) + assert len(payrolls) >= 1 + assert (payrolls["indicator"] == "nfp").all() + + # KXUSPPIYOY → ppi_yoy: the 12-month % change from the base PPI first prints. + # YoY needs the year-ago base too, so override the keyed ALFRED fake with a + # two-period (2025-05 300.0 + 2026-05 309.0) set → 3.0% YoY. + def _two_period(series_id, *, key=None, units=None, vintages="all", **_kw): + raw = [ + { + "indicator": series_id, + "series_id": series_id, + "period": period, + "value": value, + "units": units, + "release_datetime": None, + "vintage_date": vintage, + "knowledge_time": vintage, + "vintage_precision": "day", + "source": "alfred", + "retrieved_at": None, + } + for period, value, vintage in ( + ("2025-05", 300.0, datetime(2025, 6, 11, tzinfo=UTC)), + ("2026-05", 309.0, datetime(2026, 6, 11, tzinfo=UTC)), + ) + ] + return ( + fred_alfred.all_vintages(raw) if vintages == "all" else fred_alfred.first_release(raw) + ) + + monkeypatch.setattr(fred_alfred, "fetch_vintages", _two_period) + ppi_yoy = research_econ("KXUSPPIYOY", D0, D1) + assert len(ppi_yoy) >= 1 + assert (ppi_yoy["indicator"] == "ppi_yoy").all() + assert (ppi_yoy["units"] == "percent").all() # YoY %, not the index level + + +# --- Test 7 (regression guard, the class the verifier + every prior test missed): +# STRING date inputs are the DOCUMENTED primary usage, but every unit test and +# the live smoke used datetime objects or called fetchers directly, so the econ +# public surface silently raised TypeError ("'<' not supported between 'str' +# and 'date'") for EVERY string-input call since it was first built (29-08). +# These assert string inputs behave IDENTICALLY to datetime inputs across the +# whole surface — history/series/research_econ/snapshot. +def test_string_dates_match_datetime_dates_history(monkeypatch) -> None: + _install_keyed_alfred(monkeypatch) + from_str, to_str = "2026-06-01", "2026-07-31" # string equivalents of D0/D1 + out_dt = series("cpi", D0, D1, vintages="settlement") + out_str = series("cpi", from_str, to_str, vintages="settlement") + assert len(out_str) == len(out_dt) >= 1 + assert out_str["settlement_grade"].all() + assert out_str["value"].iloc[0] == pytest.approx(_FIRST_PRINT_VALUE) + # history() alias too (string dates). + out_hist = history("cpi", from_str, to_str, vintages="settlement") + assert len(out_hist) == len(out_str) + + +def test_string_dates_research_econ(monkeypatch) -> None: + _install_keyed_alfred(monkeypatch) + pairs = research_econ("KXCPI", "2026-06-01", "2026-07-31") + assert len(pairs) >= 1 # string inputs must not raise TypeError + + +def test_string_as_of_snapshot(monkeypatch) -> None: + from mostlyright.econ import snapshot + + _install_keyed_alfred(monkeypatch) + # A string as_of after the first-print vintage (2026-06-11) must return the + # settlement-grade first print, not raise "'str' has no attribute 'tzinfo'". + snap_str = snapshot("cpi", as_of="2026-07-01") + snap_dt = snapshot("cpi", as_of=datetime(2026, 7, 1, tzinfo=UTC)) + assert len(snap_str) == len(snap_dt) >= 1 + assert snap_str["settlement_grade"].all() + + +def test_malformed_date_string_raises_valueerror(monkeypatch) -> None: + _install_keyed_alfred(monkeypatch) + # A non-ISO string is rejected loudly at the boundary (not a TypeError deeper in). + with pytest.raises(ValueError): + series("cpi", "not-a-date", "2026-07-31", vintages="settlement") + + +# --- Test 11 (BATCH-A blocker: GDP orphan): GDP routes through ALFRED (growth-rate +# series A191RL1Q225SBEA) with a REAL vintage_date, not BEA-GetData/vintage=now. +def test_keyed_gdp_routes_through_alfred_growth_series(monkeypatch) -> None: + captured: dict[str, Any] = {} + _install_keyed_alfred(monkeypatch, captured) + out = series("gdp", D0, D1, vintages="settlement") + assert len(out) >= 1, "keyed GDP settlement must not be empty/raise (the orphan blocker)" + assert out["settlement_grade"].all() + # GDP must query the GROWTH-RATE FRED id, never the GDPC1 level (wrong-value class). + assert captured["series_ids"] == ["A191RL1Q225SBEA"] + assert (out["indicator"] == "gdp").all() + # research_econ('KXGDP') reaches settlement pairs end-to-end. + pairs = research_econ("KXGDP", D0, D1) + assert len(pairs) >= 1 + + +# --- Test 12 (BATCH-A2 blocker: keyless historical misfiling): a KEYLESS historical +# read must return the latest-revised rows (period-derived vintage_date), not +# misfile them into the now-partition and raise "not yet released". +def test_keyless_historical_returns_period_filed_rows(monkeypatch, tmp_path) -> None: + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path / "kl")) + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) # keyless + from mostlyright.econ._fetchers import bls as _bls + + def _fake_bls_fetch(series_ids, *, start_year, end_year): + # BLS-shaped latest-revised rows, stamped vintage_date=now (the misfile source). + return [ + { + "indicator": "cpi", + "series_id": series_ids[0], + "period": "2025-03", + "value": 300.0, + "units": "index", + "release_datetime": None, + "vintage_date": datetime(2026, 7, 11, tzinfo=UTC), # NOW — the bug input + "knowledge_time": datetime(2026, 7, 11, tzinfo=UTC), + "release_type": "revised", + "settlement_grade": False, + "source": "bls.v1", + "retrieved_at": None, + } + ] + + monkeypatch.setattr(_bls, "fetch", _fake_bls_fetch) + # vintages='all' historical window must return the row (re-filed to ~2025-04), not raise. + out = history("cpi", "2025-01-01", "2025-06-30", vintages="all") + assert len(out) == 1 + assert not out["settlement_grade"].any() # keyless is never settlement-grade + # vintage_date re-derived from the period (2025-03 -> ~2025-04), NOT now (2026-07). + assert str(out["vintage_date"].iloc[0])[:4] == "2025" + # settlement keyless still raises naming FRED_API_KEY. + with pytest.raises(IndicatorNotYetReleasedError) as exc: + history("cpi", "2025-01-01", "2025-06-30", vintages="settlement") + assert "FRED_API_KEY" in str(exc.value) diff --git a/packages/econ/tests/test_leakage_property.py b/packages/econ/tests/test_leakage_property.py new file mode 100644 index 00000000..63491340 --- /dev/null +++ b/packages/econ/tests/test_leakage_property.py @@ -0,0 +1,157 @@ +"""Property tests — the econ leakage invariant + the vintage-partition invariant. + +Two hypothesis-proven invariants (not merely example-tested), per the phase's +Testing Note: + +1. **Leakage invariant.** For econ, ``knowledge_time == vintage_date`` (the value's + public-knowledge instant). :func:`~mostlyright.core.temporal.leakage.assert_no_leakage` + MUST raise :class:`~mostlyright.core.exceptions.LeakageError` iff at least one + ``vintage_date > as_of``, and pass silently otherwise. This is what stops a + FUTURE revision from leaking into a backtest (ECON-14 / T-29-23). Boundary + semantics matter: ``vintage_date == as_of`` is NOT leakage (strict ``>``). + +2. **Vintage-partition invariant.** The read-time vintage filter + (:func:`mostlyright.econ._history._filter_vintages`, the exact production code + the ``vintages=`` kwarg dispatches to) is a CLEAN PARTITION: the + ``vintages="settlement"`` result is always a SUBSET of the ``vintages="all"`` + result (settlement ⊆ all), for EVERY generated frame. The store keeps all + vintages and the caller selects one at read time — never a merge collapse. + +Datetime strategies are tz-aware UTC within a bounded range (mirrors the repo's +KnowledgeView / leakage property-test bounds). Fast (in CI): no network, no key. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from mostlyright.core.exceptions import LeakageError +from mostlyright.core.temporal.leakage import assert_no_leakage +from mostlyright.core.temporal.timepoint import TimePoint +from mostlyright.econ._history import _filter_vintages + +# --- Bounded tz-aware UTC datetime range ------------------------------------- +# Matches the repo's KnowledgeView / leakage property-test convention: a sane, +# bounded, tz-aware UTC window so hypothesis explores realistic vintage dates +# without wandering into pandas' out-of-bounds Timestamp territory. +_MIN = datetime(2021, 1, 1, tzinfo=UTC) +_MAX = datetime(2027, 12, 31, 23, 59, 59, tzinfo=UTC) + +_utc_datetimes = st.datetimes( + min_value=_MIN.replace(tzinfo=None), + max_value=_MAX.replace(tzinfo=None), + timezones=st.just(UTC), +) + + +def _econ_frame(vintage_dates: list[datetime]) -> pd.DataFrame: + """A minimal econ-shaped frame carrying ``knowledge_time == vintage_date``. + + ``assert_no_leakage`` keys off ``knowledge_time``; for econ that column IS the + ``vintage_date`` (the value's public-knowledge instant). We set both so the + frame is faithful to ``schema.econ.observations.v1`` semantics. + """ + kt = pd.to_datetime(vintage_dates, utc=True) + return pd.DataFrame( + { + "vintage_date": kt, + "knowledge_time": kt, + "value": list(range(len(vintage_dates))), + } + ) + + +# --------------------------------------------------------------------------- +# 1. Leakage invariant — raises iff some vintage_date > as_of. +# --------------------------------------------------------------------------- +@given( + vintage_dates=st.lists(_utc_datetimes, min_size=1, max_size=50), + as_of=_utc_datetimes, +) +@settings(max_examples=200, deadline=2000) +def test_leakage_raises_iff_any_vintage_after_as_of(vintage_dates: list[datetime], as_of: datetime): + """``assert_no_leakage`` raises LeakageError iff ≥1 ``vintage_date > as_of``.""" + df = _econ_frame(vintage_dates) + as_of_tp = TimePoint(as_of.isoformat()) + + # Ground truth computed independently of the function under test. + n_after = int((df["vintage_date"] > as_of_tp.to_utc()).sum()) + + if n_after == 0: + # No future vintage → must pass silently (no raise). + assert_no_leakage(df, as_of_tp) + else: + with pytest.raises(LeakageError) as exc: + assert_no_leakage(df, as_of_tp) + # The error's own count must equal the independently-computed count. + assert exc.value.violating_count == n_after + + +@given(instant=_utc_datetimes) +@settings(max_examples=100, deadline=2000) +def test_leakage_boundary_vintage_equal_as_of_is_safe(instant: datetime): + """A vintage_date EXACTLY at as_of is NOT leakage (strict ``>`` boundary).""" + df = _econ_frame([instant]) + as_of_tp = TimePoint(instant.isoformat()) + # vintage_date == as_of → strictly-greater mask is empty → no raise. + assert_no_leakage(df, as_of_tp) + + +# --------------------------------------------------------------------------- +# 2. Vintage-partition invariant — settlement ⊆ all, via the real filter. +# --------------------------------------------------------------------------- +def _row(settlement_grade: Any, idx: int) -> dict[str, Any]: + """A minimal cache-shaped row carrying a ``settlement_grade`` flag + an id.""" + return {"settlement_grade": settlement_grade, "row_id": idx} + + +@given( + flags=st.lists( + st.booleans() | st.none(), # include None → filter treats it as falsey + min_size=0, + max_size=60, + ) +) +@settings(max_examples=200, deadline=2000) +def test_settlement_is_subset_of_all(flags: list[Any]): + """``_filter_vintages(rows, "settlement")`` ⊆ ``_filter_vintages(rows, "all")``. + + Uses the production read-time filter so the property tests the real code path + the ``vintages=`` kwarg dispatches to. + """ + rows = [_row(flag, i) for i, flag in enumerate(flags)] + + all_rows = _filter_vintages(rows, "all") + settlement_rows = _filter_vintages(rows, "settlement") + + all_ids = {r["row_id"] for r in all_rows} + settlement_ids = {r["row_id"] for r in settlement_rows} + + # The partition property: settlement ⊆ all. + assert settlement_ids <= all_ids + # "all" is the identity partition — it keeps every row. + assert all_ids == {r["row_id"] for r in rows} + # Every settlement row is genuinely settlement-grade (truthy flag). + assert all(bool(r["settlement_grade"]) for r in settlement_rows) + # And the complement (all minus settlement) carries only falsey flags. + non_settlement = all_ids - settlement_ids + for r in rows: + if r["row_id"] in non_settlement: + assert not bool(r["settlement_grade"]) + + +@given( + flags=st.lists(st.booleans() | st.none(), min_size=0, max_size=60), +) +@settings(max_examples=100, deadline=2000) +def test_settlement_count_equals_truthy_flag_count(flags: list[Any]): + """|settlement| == number of truthy ``settlement_grade`` flags (exact size).""" + rows = [_row(flag, i) for i, flag in enumerate(flags)] + settlement_rows = _filter_vintages(rows, "settlement") + expected = sum(1 for flag in flags if bool(flag)) + assert len(settlement_rows) == expected diff --git a/packages/econ/tests/test_polymarket.py b/packages/econ/tests/test_polymarket.py new file mode 100644 index 00000000..94018cb6 --- /dev/null +++ b/packages/econ/tests/test_polymarket.py @@ -0,0 +1,139 @@ +"""Tests for ``econ._polymarket`` — the secondary-venue (Polymarket) econ derive. + +``econ._polymarket.derive(...)`` queries Polymarket's keyless gamma ``/events`` +(the Fed/CPI/PCE econ markets) and CLOB ``/prices-history`` (the price panel) via a +THIN IN-PACKAGE httpx GET — it does NOT import the markets package. The workspace +edge is strictly one-way ``markets → econ`` (29-04 added ``mostlyrightmd-econ`` to +the markets deps); adding the reverse would cycle ``uv sync`` / ``uv build`` +(T-29-27). The negative-grep gate asserts there is no real ``import`` of the +markets package in ``_polymarket.py`` and no ``mostlyrightmd-markets`` in the econ +pyproject. + +Verified live shapes (RESEARCH ECON-18): +- gamma ``/events?slug=...`` → an event with ``markets[]``, each + ``{question, conditionId, clobTokenIds, outcomes, outcomePrices, + umaResolutionStatus, resolutionSource, description}``. +- CLOB ``/prices-history?market=`` → ``{history:[{t:, p:}...]}``. + +No network here: gamma/CLOB responses are injected via ``httpx.MockTransport``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx +import pytest +from mostlyright.econ._polymarket import derive, fetch_clob_history, fetch_gamma_event + +# --- Recorded gamma econ event (Fed decision) -------------------------------- +_GAMMA_FED_EVENT: dict[str, Any] = { + "slug": "fed-decision-in-july-181", + "title": "Fed decision in July", + "markets": [ + { + "question": "Will the Fed cut rates in July?", + "conditionId": "0xabc123", + "clobTokenIds": '["11111", "22222"]', + "outcomes": '["Yes", "No"]', + "outcomePrices": '["0.12", "0.88"]', + "umaResolutionStatus": "resolved", + "resolutionSource": "Federal Reserve FOMC statement", + "description": "Resolves YES if the FOMC lowers the target range in July.", + }, + ], +} + +_CLOB_HISTORY: dict[str, Any] = { + "history": [ + {"t": 1751000000, "p": "0.10"}, + {"t": 1751003600, "p": "0.12"}, + {"t": 1751007200, "p": "0.11"}, + ] +} + + +def _client_returning( + payload: dict[str, Any], captured: dict[str, Any] | None = None +) -> httpx.Client: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https", "Polymarket fetch must be HTTPS-only" + if captured is not None: + captured["url"] = str(request.url) + captured["params"] = dict(request.url.params) + return httpx.Response(200, json=payload) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 1 (RED): derive extracts the gamma econ market-outcome shape --------- +def test_derive_extracts_market_outcome(monkeypatch) -> None: + gamma_client = _client_returning(_GAMMA_FED_EVENT) + clob_client = _client_returning(_CLOB_HISTORY) + + outcomes = derive( + slug="fed-decision-in-july-181", + gamma_client=gamma_client, + clob_client=clob_client, + ) + assert isinstance(outcomes, list) + assert len(outcomes) == 1 + o = outcomes[0] + # The secondary-venue outcome shape research_econ can join. + assert o["question"] == "Will the Fed cut rates in July?" + assert o["condition_id"] == "0xabc123" + assert o["resolution_source"] == "Federal Reserve FOMC statement" + assert o["uma_resolution_status"] == "resolved" + assert o["outcomes"] == ["Yes", "No"] + assert o["outcome_prices"] == [pytest.approx(0.12), pytest.approx(0.88)] + assert o["venue"] == "polymarket" + # The CLOB price panel is attached for the first outcome token. + assert len(o["price_history"]) == 3 + assert o["price_history"][0]["p"] == pytest.approx(0.10) + + +def test_fetch_gamma_event_by_slug() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_GAMMA_FED_EVENT, cap) + event = fetch_gamma_event("fed-decision-in-july-181", client=client) + assert event["slug"] == "fed-decision-in-july-181" + assert cap["params"]["slug"] == "fed-decision-in-july-181" + + +def test_fetch_clob_history_by_token() -> None: + cap: dict[str, Any] = {} + client = _client_returning(_CLOB_HISTORY, cap) + history = fetch_clob_history("11111", client=client) + assert len(history) == 3 + assert cap["params"]["market"] == "11111" + + +def test_non_json_gamma_body_raises() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="cloudflare") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + with pytest.raises(Exception): # noqa: B017 - any error, never silent empty + fetch_gamma_event("fed-decision-in-july-181", client=client) + + +# --- Dependency-direction guard: no markets import, no markets dep ------------- +def test_polymarket_has_no_markets_import() -> None: + import mostlyright.econ._polymarket as poly_mod + + text = Path(poly_mod.__file__).read_text(encoding="utf-8") + # The negative-grep gate: no real import of the markets package (in prose OR + # code) — keep the token out entirely so a substring gate also stays green. + assert "mostlyright.markets" not in text + assert "mostlyrightmd-markets" not in text + + +def test_econ_pyproject_does_not_depend_on_markets() -> None: + # Walk up to the econ package root (packages/econ/) and read its pyproject. + # __file__ = packages/econ/tests/test_polymarket.py → parents[1] = packages/econ. + here = Path(__file__).resolve() + econ_pyproject = here.parents[1] / "pyproject.toml" + assert econ_pyproject.is_file(), f"expected econ pyproject at {econ_pyproject}" + text = econ_pyproject.read_text(encoding="utf-8") + assert "mostlyrightmd-markets" not in text diff --git a/packages/econ/tests/test_releases.py b/packages/econ/tests/test_releases.py new file mode 100644 index 00000000..cc7c8af7 --- /dev/null +++ b/packages/econ/tests/test_releases.py @@ -0,0 +1,52 @@ +"""Tests for ``econ.releases`` — the release calendar/schedule for an indicator. + +``releases(indicator)`` (plan 29-08, replacing the 29-01 stub) returns the +release calendar/schedule for an indicator (BLS empsit schedule, BEA schedule, +FOMC calendar per ECON-13). The shape is a simple typed structure — a list of +:class:`~mostlyright.econ._releases.ReleaseEvent` entries (period + scheduled +release datetime + source agency). It is sourced from a small curated schedule +table; where a live schedule fetch is cheap it is documented, but v1 ships the +curated table so the surface is offline-deterministic. +""" + +from __future__ import annotations + +from datetime import datetime + +import pytest +from mostlyright.econ import releases +from mostlyright.econ._releases import ReleaseEvent + + +# --- Test 5: releases("cpi") returns a release-calendar structure -------------- +def test_releases_cpi_returns_schedule() -> None: + schedule = releases("cpi") + assert isinstance(schedule, list) + assert schedule, "releases('cpi') must not be empty" + for event in schedule: + assert isinstance(event, ReleaseEvent) + assert event.indicator == "cpi" + assert isinstance(event.release_datetime, datetime) + # tz-aware (a scheduled 8:30 ET drop, stored UTC). + assert event.release_datetime.tzinfo is not None + assert event.agency # a non-empty source agency name + + +def test_releases_covers_each_agency_family() -> None: + # The three CONTEXT ECON-13 schedule sources are all represented. + for indicator in ("cpi", "gdp", "fed_funds", "jobless_claims"): + schedule = releases(indicator) + assert schedule, f"releases({indicator!r}) must return a schedule" + assert all(e.indicator == indicator for e in schedule) + + +def test_releases_unknown_indicator_raises() -> None: + # An indicator with no known schedule is an explicit error, never []/None. + with pytest.raises((ValueError, KeyError)): + releases("not_a_real_indicator") + + +def test_releases_are_sorted_by_datetime() -> None: + schedule = releases("cpi") + times = [e.release_datetime for e in schedule] + assert times == sorted(times) diff --git a/packages/econ/tests/test_research_econ.py b/packages/econ/tests/test_research_econ.py new file mode 100644 index 00000000..c275d3c8 --- /dev/null +++ b/packages/econ/tests/test_research_econ.py @@ -0,0 +1,213 @@ +"""Tests for ``econ.research_econ`` — leakage-free settlement pairs. + +``research_econ(series_or_contract, from_date, to_date, *, as_of=None)`` (plan +29-08, replacing the 29-01 stub) is the vertical's user-facing payoff: it joins a +Kalshi (or Polymarket) market outcome to the settlement-grade FIRST-PRINT vintage +and returns leakage-free training pairs that "backtest the same way they trade". + +The load-bearing guarantees under test: + +- **Leakage guard** — each pair carries ``knowledge_time = vintage_date``; when an + ``as_of`` cutoff is supplied, :func:`assert_no_leakage` rejects any row whose + vintage_date is after ``as_of`` (a future revision can NEVER leak into a + backtest). Property-tested over as_of/vintage_date (T-29-23). +- **TE divergence, never fabricated** — a Trading-Economics-settled series + (KXUSPPI, PPI-MoM) returns the AGENCY first print labeled + ``settlement_grade=False`` AND emits a divergence warning; it does NOT fabricate + a TE value (ECON-17 / T-29-24). +- **Unknown contract raises** — an unresolvable ticker raises, never returns None. + +**No markets import.** ``research_econ`` resolves the contract via the econ-side +routing table (``econ._settlement_map``), NOT ``mostlyright.markets`` — the +workspace edge is one-way ``markets → econ`` (adding the reverse cycles +``uv sync``). The resolution semantics match ``kalshi_econ.resolve`` because both +read the SAME ``SETTLEMENT_ROUTING`` table. + +No network here: the ``history`` fetcher dispatch is monkeypatched to return +synthetic settlement rows and the cache is relocated under ``tmp_path``. +""" + +from __future__ import annotations + +import warnings +from datetime import UTC, datetime +from typing import Any + +import pandas as pd +import pytest +from hypothesis import given +from hypothesis import strategies as st +from mostlyright.core import TimePoint +from mostlyright.core.exceptions import LeakageError +from mostlyright.econ import _history, research_econ +from mostlyright.econ._research import DivergenceWarning + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + return tmp_path + + +def _settlement_row( + *, + indicator: str, + period: str, + value: float, + vintage_date: datetime, + settlement_grade: bool = True, +) -> dict[str, Any]: + return { + "indicator": indicator, + "series_id": "X", + "period": period, + "value": value, + "units": "index", + "release_datetime": vintage_date, + "vintage_date": vintage_date, + "release_type": "advance", + "settlement_grade": settlement_grade, + "knowledge_time": vintage_date, + "source": "alfred", + "retrieved_at": vintage_date, + } + + +def _seed(monkeypatch, indicator: str, rows: list[dict[str, Any]]) -> None: + monkeypatch.setitem(_history.INDICATOR_FETCHERS, indicator, lambda a, b, *, source=None: rows) + + +# --- Test 1 (RED): research_econ returns leakage-shaped pairs ------------------ +def test_returns_pairs_with_knowledge_time(monkeypatch) -> None: + v = datetime(2026, 7, 15, tzinfo=UTC) + _seed( + monkeypatch, + "cpi_yoy", + [_settlement_row(indicator="cpi_yoy", period="2026-06", value=3.2, vintage_date=v)], + ) + d0, d1 = datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 7, 31, tzinfo=UTC) + pairs = research_econ("KXCPIYOY", d0, d1) + + assert isinstance(pairs, pd.DataFrame) + assert len(pairs) == 1 + # The pair carries the first-print value + knowledge_time == vintage_date. + assert "knowledge_time" in pairs.columns + assert "value" in pairs.columns + assert pairs["value"].iloc[0] == pytest.approx(3.2) + assert pd.Timestamp(pairs["knowledge_time"].iloc[0]) == pd.Timestamp(v) + # The resolved agency/indicator are surfaced for the join. + assert pairs["indicator"].iloc[0] == "cpi_yoy" + assert pairs["agency"].iloc[0] == "BLS" + + +# --- Test 2 (leakage guard): a vintage after as_of raises ---------------------- +def test_leakage_guard_rejects_future_vintage(monkeypatch) -> None: + v = datetime(2026, 7, 15, tzinfo=UTC) # vintage AFTER the as_of cutoff below + _seed( + monkeypatch, + "cpi_yoy", + [_settlement_row(indicator="cpi_yoy", period="2026-06", value=3.2, vintage_date=v)], + ) + d0, d1 = datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 7, 31, tzinfo=UTC) + as_of = TimePoint("2026-07-01T00:00:00Z") # BEFORE the vintage_date + with pytest.raises(LeakageError): + research_econ("KXCPIYOY", d0, d1, as_of=as_of) + + +def test_leakage_guard_passes_when_vintage_before_as_of(monkeypatch) -> None: + v = datetime(2026, 6, 20, tzinfo=UTC) + _seed( + monkeypatch, + "cpi_yoy", + [_settlement_row(indicator="cpi_yoy", period="2026-05", value=3.1, vintage_date=v)], + ) + d0, d1 = datetime(2026, 5, 1, tzinfo=UTC), datetime(2026, 6, 30, tzinfo=UTC) + as_of = TimePoint("2026-07-01T00:00:00Z") # AFTER the vintage_date — safe + pairs = research_econ("KXCPIYOY", d0, d1, as_of=as_of) + assert len(pairs) == 1 + + +@given( + vintage_offset_days=st.integers(min_value=-30, max_value=30), +) +def test_leakage_property(tmp_path_factory, vintage_offset_days) -> None: + import mostlyright.econ._cache as _cache_mod + + cache_dir = tmp_path_factory.mktemp("econ_cache") + orig = _cache_mod.resolve_cache_root_without_v1 + _cache_mod.resolve_cache_root_without_v1 = lambda: cache_dir # type: ignore[assignment] + orig_dispatch = dict(_history.INDICATOR_FETCHERS) + try: + as_of_dt = datetime(2026, 7, 1, tzinfo=UTC) + vintage = as_of_dt.fromordinal(as_of_dt.toordinal() + vintage_offset_days).replace( + tzinfo=UTC + ) + _history.INDICATOR_FETCHERS["cpi_yoy"] = lambda a, b, *, source=None: [ + _settlement_row(indicator="cpi_yoy", period="2026-06", value=3.2, vintage_date=vintage) + ] + d0, d1 = datetime(2026, 5, 1, tzinfo=UTC), datetime(2026, 8, 1, tzinfo=UTC) + as_of = TimePoint(as_of_dt) + if vintage > as_of_dt: + # A vintage strictly after the as_of cutoff MUST be rejected. + with pytest.raises(LeakageError): + research_econ("KXCPIYOY", d0, d1, as_of=as_of) + else: + pairs = research_econ("KXCPIYOY", d0, d1, as_of=as_of) + assert len(pairs) == 1 + finally: + _cache_mod.resolve_cache_root_without_v1 = orig # type: ignore[assignment] + _history.INDICATOR_FETCHERS.clear() + _history.INDICATOR_FETCHERS.update(orig_dispatch) + + +# --- Test 3 (TE divergence): agency first print + warning, never a TE value ---- +def test_te_settled_series_labels_false_and_warns(monkeypatch) -> None: + v = datetime(2026, 7, 16, tzinfo=UTC) + # KXUSPPI (PPI MoM) settles to Trading Economics. The AGENCY first print (BLS + # PPI) is served labeled settlement_grade=False; NO TE value is fabricated. + _seed( + monkeypatch, + "ppi", + [ + _settlement_row( + indicator="ppi", + period="2026-06", + value=145.2, + vintage_date=v, + settlement_grade=True, # the agency first print IS a real vintage + ) + ], + ) + d0, d1 = datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 7, 31, tzinfo=UTC) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + pairs = research_econ("KXUSPPI", d0, d1) + + # A divergence warning was emitted naming the TE settlement authority. + assert any(issubclass(w.category, DivergenceWarning) for w in caught) + # The pair is labeled settlement_grade=False (agency proxy, not TE truth). + assert bool(pairs["settlement_grade"].iloc[0]) is False + assert pairs["agency"].iloc[0] == "TradingEconomics" + # The value is the AGENCY first print (145.2), NOT a fabricated TE number. + assert pairs["value"].iloc[0] == pytest.approx(145.2) + + +# --- Test 4 (unknown contract): raises, never returns None -------------------- +def test_unknown_contract_raises() -> None: + with pytest.raises((ValueError, TypeError)): + research_econ( + "NOT-A-REAL-TICKER", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 7, 31, tzinfo=UTC), + ) + + +def test_no_markets_import_in_research_module() -> None: + # Structural: the econ->markets edge is forbidden (one-way markets->econ). The + # research module must resolve via econ._settlement_map, not mostlyright.markets. + import mostlyright.econ._research as research_mod + + src = research_mod.__file__ + with open(src, encoding="utf-8") as fh: + text = fh.read() + assert "mostlyright.markets" not in text diff --git a/packages/econ/tests/test_schema.py b/packages/econ/tests/test_schema.py new file mode 100644 index 00000000..43abc4b5 --- /dev/null +++ b/packages/econ/tests/test_schema.py @@ -0,0 +1,151 @@ +"""Contract tests for ``schema.econ.observations.v1`` (vintage-first-class). + +The econ schema's defining difference from ``schema.observation.v1`` is that +**vintage is first-class**: ``vintage_date`` / ``release_type`` / +``settlement_grade`` are real columns and the schema KEEPS every vintage +(no single-row ``report_type_priority`` dedup like weather). Read-time +filtering (``vintages="settlement"|"all"``) is what selects a vintage — not a +merge collapse. These tests pin that contract RED→GREEN. + +Firewall: importing ``mostlyright.econ._schema`` registers the schema lazily +(the CWOP discipline at package granularity); core's eager schema list never +mentions ``schema.econ``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pandas as pd +import pytest +from mostlyright.core.exceptions import SchemaValidationError, SourceMismatchError +from mostlyright.core.validator import validate_dataframe +from mostlyright.econ import _schema + + +def _base_row(**overrides: object) -> dict[str, object]: + """A well-formed econ observation row (one vintage of one period).""" + known = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) # 8:30 ET as UTC + row: dict[str, object] = { + "indicator": "cpi", + "series_id": "CPIAUCSL", + "period": "2026-05", + "value": 314.069, + "units": "index", + "release_datetime": known, + "vintage_date": known, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": known, + "source": _schema.ECON_SOURCE, + "retrieved_at": known, + } + row.update(overrides) + return row + + +def _build(rows: list[dict[str, object]], *, source: str | None = None) -> pd.DataFrame: + src = source if source is not None else _schema.ECON_SOURCE + return _schema.build_econ_dataframe(rows, source=src) + + +# --- Test 1 (RED): the schema id is registered on econ import ----------------- +def test_import_registers_schema_id() -> None: + assert _schema.EconObservationsSchema.schema_id == "schema.econ.observations.v1" + # If the module registered the schema, validate_dataframe can resolve the id. + # An empty frame stamped with a valid source + retrieved_at passes the four + # checks (no required column can be null-violated when there are zero rows). + df = _build([]) + reg = validate_dataframe(df, "schema.econ.observations.v1") + assert reg.source == _schema.ECON_SOURCE + + +# --- Test 2: a well-formed frame validates ----------------------------------- +def test_wellformed_frame_validates() -> None: + df = _build([_base_row()]) + # Should not raise. + validate_dataframe(df, "schema.econ.observations.v1") + assert list(df["indicator"]) == ["cpi"] + # vintage columns present. + for col in ("vintage_date", "release_type", "settlement_grade", "knowledge_time"): + assert col in df.columns + + +# --- Test 3: TWO vintages of the same (indicator, period) BOTH validate ------- +def test_keep_all_vintages_no_dedup() -> None: + v1 = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) # advance print + v2 = datetime(2026, 7, 12, 12, 30, tzinfo=UTC) # later revision + df = _build( + [ + _base_row( + period="2026-05", + vintage_date=v1, + knowledge_time=v1, + release_type="advance", + settlement_grade=True, + value=314.069, + ), + _base_row( + period="2026-05", + vintage_date=v2, + knowledge_time=v2, + release_type="revised", + settlement_grade=False, + value=314.500, + ), + ] + ) + # Both rows survive — the schema keeps every vintage (no collapse). + validate_dataframe(df, "schema.econ.observations.v1") + assert len(df) == 2 + assert set(df["release_type"]) == {"advance", "revised"} + assert list(df["settlement_grade"]) == [True, False] + + +# --- Test 4a-i: the builder rejects a non-econ source at construction --------- +def test_builder_rejects_non_econ_source() -> None: + # Fail-fast: build_econ_dataframe guards the source union up front so a + # fetcher can never assemble a frame stamped with a foreign tag. + with pytest.raises(ValueError, match="not an econ source"): + _build([_base_row()], source="not.an.econ.source") + + +# --- Test 4a-ii: the VALIDATOR raises on a drifted source (bypass builder) ----- +def test_wrong_source_raises() -> None: + # A frame whose source drifted AFTER assembly (hand-built, or a cache file + # written by an older/foreign producer) must still be caught by the + # validator's source-identity invariant — not only the builder guard. + df = _build([_base_row()]) + df.attrs["source"] = "not.an.econ.source" + df["source"] = "not.an.econ.source" + with pytest.raises(SourceMismatchError): + validate_dataframe(df, "schema.econ.observations.v1") + + +# --- Test 4b: a missing non-nullable column raises ---------------------------- +def test_missing_required_column_raises() -> None: + df = _build([_base_row()]) + df = df.drop(columns=["vintage_date"]) # non-nullable → must raise + with pytest.raises(SchemaValidationError): + validate_dataframe(df, "schema.econ.observations.v1") + + +# --- Test 5: release_type enum rejects an out-of-vocabulary value ------------- +def test_release_type_enum_rejects_unknown() -> None: + df = _build([_base_row(release_type="guess")]) + with pytest.raises(SchemaValidationError): + validate_dataframe(df, "schema.econ.observations.v1") + + +# --- Bonus: the 7 declared release types all validate ------------------------- +def test_all_release_types_accepted() -> None: + known = datetime(2026, 6, 12, 12, 30, tzinfo=UTC) + rows = [ + _base_row( + period=f"2026-0{i + 1}", release_type=rt, vintage_date=known, knowledge_time=known + ) + for i, rt in enumerate(_schema.ECON_RELEASE_TYPES) + ] + df = _build(rows) + validate_dataframe(df, "schema.econ.observations.v1") + assert set(df["release_type"]) == set(_schema.ECON_RELEASE_TYPES) diff --git a/packages/econ/tests/test_series_surface.py b/packages/econ/tests/test_series_surface.py new file mode 100644 index 00000000..e9f597b2 --- /dev/null +++ b/packages/econ/tests/test_series_surface.py @@ -0,0 +1,209 @@ +"""Tests for the ``econ.series`` surface conformance (plan 29-11). + +Plan 29-11 conforms the econ public surface to the Phase-30/31 source-identity +contract (``docs/source-identity.md``, v1.13/1.14): + +- ``series(indicator, from_date, to_date, *, vintages="settlement", source=None, + delivery="live")`` is the CANONICAL read function; ``history(...)`` remains a + working alias returning byte-identical output (Vu's ``research()``-alias style + — docstring-deprecated, NO runtime warning this release). +- ``source=`` follows contract §1: accepted set + ``{None,"fred","bls","bea","dol","fed"}``; an unrecognized source raises + ``ValueError`` naming the accepted set BEFORE any network call; a pin that + cannot serve the indicator (e.g. ``source="bea"`` for ``"cpi"``) raises loudly, + never silently falls back. +- ``delivery=`` follows contract §2: ``"live"`` (default, local) | ``"hosted"`` + raises ``SourceUnavailableError`` naming ``ECON_HOSTED_URL`` + + ``MOSTLYRIGHT_API_KEY`` (reserved seam); unknown delivery raises ``ValueError`` + pre-network. + +No network here: the cache is relocated under ``tmp_path`` and the fetcher +dispatch is monkeypatched (a spy proving zero fetch calls on the loud-validation +paths), so CI stays offline. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest +from mostlyright.core.exceptions import SourceUnavailableError +from mostlyright.econ import _history, history, series + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + """Relocate the whole SDK cache root under a tmp dir for each test.""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + return tmp_path + + +def _vintage_row( + *, + indicator: str = "cpi", + period: str = "2026-05", + value: float = 314.0, + vintage_date: datetime, + settlement_grade: bool, + release_type: str, +) -> dict[str, Any]: + """A well-formed econ observation row (one vintage of one period).""" + return { + "indicator": indicator, + "series_id": "CPIAUCSL", + "period": period, + "value": value, + "units": "index", + "release_datetime": vintage_date, + "vintage_date": vintage_date, + "release_type": release_type, + "settlement_grade": settlement_grade, + "knowledge_time": vintage_date, + "source": "alfred", + "retrieved_at": vintage_date, + } + + +def _seed_dispatch(monkeypatch, indicator: str, rows: list[dict[str, Any]]) -> dict[str, int]: + """Register a synthetic fetcher for ``indicator`` that records its call count.""" + calls = {"n": 0} + + def _fake_fetch(from_date, to_date, *, source=None): + calls["n"] += 1 + return rows + + monkeypatch.setitem(_history.INDICATOR_FETCHERS, indicator, _fake_fetch) + return calls + + +# --- series is the canonical read function ----------------------------------- +def test_series_and_history_are_distinct_callables() -> None: + # The rename lands a NEW canonical name; the alias is a DIFFERENT object. + assert series is not history + + +def test_history_output_byte_identical_to_series(monkeypatch) -> None: + d_first = datetime(2026, 6, 11, tzinfo=UTC) + d_rev = datetime(2026, 7, 15, tzinfo=UTC) + rows = [ + _vintage_row(vintage_date=d_first, settlement_grade=True, release_type="advance"), + _vintage_row( + value=315.0, vintage_date=d_rev, settlement_grade=False, release_type="revised" + ), + ] + _seed_dispatch(monkeypatch, "cpi", rows) + + d0, d1 = datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 7, 31, tzinfo=UTC) + via_series = series("cpi", d0, d1, vintages="all") + # Re-seed (the first call persisted to cache; both read the same window). + via_history = history("cpi", d0, d1, vintages="all") + + # Byte-identical frames (same columns, same values, same order). + from pandas.testing import assert_frame_equal + + assert_frame_equal( + via_series.reset_index(drop=True), + via_history.reset_index(drop=True), + ) + + +# --- source= loud validation (contract §1) ----------------------------------- +def test_unknown_source_raises_before_any_fetch(monkeypatch) -> None: + calls = _seed_dispatch(monkeypatch, "cpi", []) + with pytest.raises(ValueError, match="source") as exc: + series( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + source="nope", # type: ignore[arg-type] + ) + # The message names the accepted set, and NO fetch ran (loud, pre-network). + msg = str(exc.value) + assert "fred" in msg and "bls" in msg and "bea" in msg and "dol" in msg and "fed" in msg + assert calls["n"] == 0 + + +def test_source_pin_that_cannot_serve_indicator_raises_before_fetch(monkeypatch) -> None: + # source="bea" is a VALID authority, but it does not serve CPI — a pin that + # cannot serve the indicator raises loudly, never silently falls back to BLS. + calls = _seed_dispatch(monkeypatch, "cpi", []) + with pytest.raises(ValueError) as exc: + series( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + source="bea", + ) + msg = str(exc.value) + # Names the indicator, the pin, and the valid authorities for that indicator. + assert "cpi" in msg and "bea" in msg + assert calls["n"] == 0 + + +def test_valid_source_pin_that_serves_indicator_is_accepted(monkeypatch) -> None: + # source="bls" (or "fred") IS a valid CPI authority — the pin is honored and + # the existing dispatch runs (no loud raise). + d_first = datetime(2026, 6, 11, tzinfo=UTC) + rows = [_vintage_row(vintage_date=d_first, settlement_grade=True, release_type="advance")] + _seed_dispatch(monkeypatch, "cpi", rows) + out = series( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + source="bls", + ) + assert len(out) == 1 + assert (out["indicator"] == "cpi").all() + + +def test_source_none_is_the_default_routing(monkeypatch) -> None: + d_first = datetime(2026, 6, 11, tzinfo=UTC) + rows = [_vintage_row(vintage_date=d_first, settlement_grade=True, release_type="advance")] + _seed_dispatch(monkeypatch, "cpi", rows) + default = series("cpi", datetime(2026, 6, 1, tzinfo=UTC), datetime(2026, 6, 30, tzinfo=UTC)) + assert len(default) == 1 + + +# --- delivery= loud validation (contract §2) --------------------------------- +def test_delivery_hosted_raises_source_unavailable_naming_the_seam(monkeypatch) -> None: + calls = _seed_dispatch(monkeypatch, "cpi", []) + with pytest.raises(SourceUnavailableError) as exc: + series( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + delivery="hosted", + ) + msg = str(exc.value) + assert "ECON_HOSTED_URL" in msg + assert "MOSTLYRIGHT_API_KEY" in msg + # Reserved seam raises BEFORE any fetch. + assert calls["n"] == 0 + + +def test_unknown_delivery_raises_before_fetch(monkeypatch) -> None: + calls = _seed_dispatch(monkeypatch, "cpi", []) + with pytest.raises(ValueError, match="delivery") as exc: + series( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + delivery="cloud", # type: ignore[arg-type] + ) + msg = str(exc.value) + assert "live" in msg and "hosted" in msg + assert calls["n"] == 0 + + +def test_delivery_live_is_the_default(monkeypatch) -> None: + d_first = datetime(2026, 6, 11, tzinfo=UTC) + rows = [_vintage_row(vintage_date=d_first, settlement_grade=True, release_type="advance")] + _seed_dispatch(monkeypatch, "cpi", rows) + out = series( + "cpi", + datetime(2026, 6, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + delivery="live", + ) + assert len(out) == 1 diff --git a/packages/econ/tests/test_snapshot_surface.py b/packages/econ/tests/test_snapshot_surface.py new file mode 100644 index 00000000..66da44fc --- /dev/null +++ b/packages/econ/tests/test_snapshot_surface.py @@ -0,0 +1,144 @@ +"""Tests for ``econ.snapshot`` — the settlement-target state as-of a cutoff (29-11). + +``snapshot(indicator, *, as_of=None, source=None, delivery="live")`` returns the +settlement-target state of the indicator: the latest settlement-grade +(``settlement_grade == True``) vintage whose ``vintage_date <= as_of`` (default +``as_of``: now, UTC-aware). It generalizes the cross-vertical ``snapshot`` rule +("settlement-target state now") onto the econ vintage store — the current best +settlement read of the thing a Kalshi contract settles on. + +When nothing is knowable at ``as_of`` (no settlement-grade vintage exists at or +before the cutoff), it raises :class:`IndicatorNotYetReleasedError` — never +returns ``[]``/``None``. It accepts the SAME ``source=``/``delivery=`` kwargs as +``series`` (loud pre-network validation), forwarded through. + +No network: cache under ``tmp_path``; fetcher dispatch monkeypatched. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest +from mostlyright.core.exceptions import ( + IndicatorNotYetReleasedError, + SourceUnavailableError, +) +from mostlyright.econ import _history, snapshot + + +@pytest.fixture(autouse=True) +def _isolated_cache(tmp_path, monkeypatch): + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + return tmp_path + + +def _vintage_row( + *, + indicator: str = "cpi", + period: str = "2026-05", + value: float = 314.0, + vintage_date: datetime, + settlement_grade: bool, + release_type: str, +) -> dict[str, Any]: + return { + "indicator": indicator, + "series_id": "CPIAUCSL", + "period": period, + "value": value, + "units": "index", + "release_datetime": vintage_date, + "vintage_date": vintage_date, + "release_type": release_type, + "settlement_grade": settlement_grade, + "knowledge_time": vintage_date, + "source": "alfred", + "retrieved_at": vintage_date, + } + + +def _seed_dispatch(monkeypatch, indicator: str, rows: list[dict[str, Any]]) -> dict[str, int]: + calls = {"n": 0} + + def _fake_fetch(from_date, to_date, *, source=None): + calls["n"] += 1 + return rows + + monkeypatch.setitem(_history.INDICATOR_FETCHERS, indicator, _fake_fetch) + return calls + + +# The first-print CPI vintage lands 2026-06-11 (settlement-grade True). +_FIRST_PRINT = datetime(2026, 6, 11, tzinfo=UTC) +# A later revision lands 2026-07-15 (settlement_grade False — Kalshi excludes it). +_REVISION = datetime(2026, 7, 15, tzinfo=UTC) + + +def _cpi_vintages() -> list[dict[str, Any]]: + return [ + _vintage_row(vintage_date=_FIRST_PRINT, settlement_grade=True, release_type="advance"), + _vintage_row( + value=315.0, vintage_date=_REVISION, settlement_grade=False, release_type="revised" + ), + ] + + +# --- as_of BEFORE the first vintage → nothing knowable → raises --------------- +def test_snapshot_before_first_vintage_raises(monkeypatch) -> None: + _seed_dispatch(monkeypatch, "cpi", _cpi_vintages()) + before = datetime(2026, 6, 1, tzinfo=UTC) # before the 2026-06-11 first print + with pytest.raises(IndicatorNotYetReleasedError): + snapshot("cpi", as_of=before) + + +# --- as_of AFTER the first vintage → returns the first-print settlement row ---- +def test_snapshot_after_first_vintage_returns_first_print(monkeypatch) -> None: + _seed_dispatch(monkeypatch, "cpi", _cpi_vintages()) + after = datetime(2026, 6, 30, tzinfo=UTC) # after first print, before revision + snap = snapshot("cpi", as_of=after) + # Exactly the settlement-grade first print is knowable at this cutoff. + assert len(snap) == 1 + assert bool(snap["settlement_grade"].iloc[0]) is True + assert snap["value"].iloc[0] == 314.0 + # vintage_date is the first print (<= as_of). + assert snap["vintage_date"].iloc[0].to_pydatetime() == _FIRST_PRINT + + +# --- a later revision is NEVER the settlement snapshot (Kalshi excludes it) ---- +def test_snapshot_ignores_post_expiration_revision(monkeypatch) -> None: + _seed_dispatch(monkeypatch, "cpi", _cpi_vintages()) + # as_of AFTER the revision: the settlement snapshot is STILL the first print, + # because settlement_grade=True marks the first print and the revision is + # settlement_grade=False (Kalshi contractually excludes post-expiration revs). + after_rev = datetime(2026, 8, 1, tzinfo=UTC) + snap = snapshot("cpi", as_of=after_rev) + assert len(snap) == 1 + assert snap["value"].iloc[0] == 314.0 + assert bool(snap["settlement_grade"].iloc[0]) is True + + +# --- as_of defaults to now (a released past window is knowable) ---------------- +def test_snapshot_default_as_of_is_now(monkeypatch) -> None: + _seed_dispatch(monkeypatch, "cpi", _cpi_vintages()) + # No as_of → now (2026-07-09 in this env, well after the 2026-06-11 print). + snap = snapshot("cpi") + assert len(snap) == 1 + assert bool(snap["settlement_grade"].iloc[0]) is True + + +# --- snapshot forwards source=/delivery= with the same loud validation --------- +def test_snapshot_unknown_source_raises_before_fetch(monkeypatch) -> None: + calls = _seed_dispatch(monkeypatch, "cpi", _cpi_vintages()) + with pytest.raises(ValueError, match="source"): + snapshot("cpi", source="nope") # type: ignore[arg-type] + assert calls["n"] == 0 + + +def test_snapshot_delivery_hosted_raises_naming_the_seam(monkeypatch) -> None: + calls = _seed_dispatch(monkeypatch, "cpi", _cpi_vintages()) + with pytest.raises(SourceUnavailableError) as exc: + snapshot("cpi", delivery="hosted") + assert "ECON_HOSTED_URL" in str(exc.value) + assert calls["n"] == 0 diff --git a/packages/econ/tests/test_yoy.py b/packages/econ/tests/test_yoy.py new file mode 100644 index 00000000..85384037 --- /dev/null +++ b/packages/econ/tests/test_yoy.py @@ -0,0 +1,99 @@ +"""Tests for the YoY percent-change contracts (cpi_yoy / cpi_core_yoy / ppi_yoy). + +Adversarial-audit finding 9: these three Kalshi contracts settle on the 12-month +PERCENT change, but the dispatch returned the base INDEX LEVEL labeled +settlement_grade=True — a silent wrong settlement value. Operator decision +(2026-07-11): compute YoY properly from the base index first-print vintages. + +These drive the REAL surface (history) through the REAL _fetch_yoy dispatch (only +the base bls.fetch transport is faked) and assert the emitted value is the +hand-computed 12-month % change, in 'percent' units — never the index level. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest +from mostlyright.econ import _history, history +from mostlyright.econ._yoy import compute_yoy + + +def _base_row(period: str, value: float, vintage: datetime) -> dict[str, Any]: + return { + "indicator": "cpi", + "series_id": "CPIAUCSL", + "period": period, + "value": value, + "units": "index", + "release_datetime": None, + "vintage_date": vintage, + "release_type": "advance", + "settlement_grade": True, + "knowledge_time": vintage, + "source": "alfred", + "retrieved_at": None, + } + + +def test_compute_yoy_is_12_month_percent_change() -> None: + # 2024-01 index 300.0; 2025-01 index 309.0 -> YoY = 100*(309/300 - 1) = 3.0%. + base = [ + _base_row("2024-01", 300.0, datetime(2024, 2, 13, tzinfo=UTC)), + _base_row("2025-01", 309.0, datetime(2025, 2, 12, tzinfo=UTC)), + ] + yoy = compute_yoy(base, indicator="cpi_yoy") + assert len(yoy) == 1 # only 2025-01 has a 12-month-prior base + row = yoy[0] + assert row["period"] == "2025-01" + assert row["value"] == pytest.approx(3.0) + assert row["units"] == "percent" + assert row["indicator"] == "cpi_yoy" + # The YoY becomes knowable when the CURRENT period's first print released. + assert row["vintage_date"] == datetime(2025, 2, 12, tzinfo=UTC) + assert row["settlement_grade"] is True + + +def test_compute_yoy_drops_periods_without_year_ago_base() -> None: + base = [_base_row("2025-03", 310.0, datetime(2025, 4, 10, tzinfo=UTC))] + assert compute_yoy(base, indicator="cpi_yoy") == [] # no 2024-03 base + + +def test_yoy_surface_returns_percent_not_index_level(monkeypatch, tmp_path) -> None: + # Drive history('cpi_yoy') through the REAL _fetch_yoy -> _fetch_bls_indicator + # dispatch; fake only the base bls.fetch transport. The window auto-widens 12mo. + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(_history, "_resolve_fred_key", lambda: None) # keyless base + from mostlyright.econ._fetchers import bls as _bls + + def _fake_bls_fetch(series_ids, *, start_year, end_year): + rows = [] + # 2024 base 300, 2025 base 309 -> 3% YoY for each 2025 month. + for y, v in ((2024, 300.0), (2025, 309.0)): + for m in range(1, 13): + rows.append( + { + "indicator": "cpi", + "series_id": series_ids[0], + "period": f"{y}-{m:02d}", + "value": v, + "units": "index", + "release_datetime": None, + "vintage_date": datetime(y, m, 15, tzinfo=UTC), + "knowledge_time": datetime(y, m, 15, tzinfo=UTC), + "release_type": "revised", + "settlement_grade": False, + "source": "bls.v1", + "retrieved_at": None, + } + ) + return rows + + monkeypatch.setattr(_bls, "fetch", _fake_bls_fetch) + out = history("cpi_yoy", "2025-01-01", "2025-06-30", vintages="all") + assert len(out) >= 1 + assert (out["units"] == "percent").all() + # 3% YoY, NOT the 309 index level. + assert (out["value"].round(2) == 3.0).all() + assert (out["value"].abs() < 20).all() diff --git a/packages/markets/pyproject.toml b/packages/markets/pyproject.toml index a9e62eb8..cdaff64a 100644 --- a/packages/markets/pyproject.toml +++ b/packages/markets/pyproject.toml @@ -37,6 +37,13 @@ dependencies = [ # pipeline reads mostlyright.core.* schemas; a stale core would silently # serve the wrong column set. "mostlyrightmd>=1.0.0,<2.0", + # ECON-10 (Phase 29): the Kalshi Economics resolver + # (mostlyright.markets.catalog.kalshi_econ) imports the curated per-series + # routing table `SETTLEMENT_ROUTING` from mostlyright.econ._settlement_map. + # This is the ONE-WAY markets -> econ coupling (econ never depends on + # markets). _settlement_map is a pure routing-rules dict with no cache/ + # schema import, so this does NOT couple markets to the parity firewall. + "mostlyrightmd-econ>=1.11.0,<2.0", "httpx>=0.27", "jsonschema>=4.21", ] diff --git a/packages/markets/src/mostlyright/markets/catalog/__init__.py b/packages/markets/src/mostlyright/markets/catalog/__init__.py index 96de5347..3dbb93bc 100644 --- a/packages/markets/src/mostlyright/markets/catalog/__init__.py +++ b/packages/markets/src/mostlyright/markets/catalog/__init__.py @@ -1,10 +1,13 @@ """mostlyright.markets.catalog — prediction market contract specs. Phase 2 (MARKETS-01..03) ships Kalshi NHIGH/NLOW settlement specs + -the 20-city station whitelist. Polymarket lands in Phase 3.3. +the 20-city station whitelist. Polymarket lands in Phase 3.3. Phase 29 +(ECON-10/ECON-17) adds the Kalshi Economics resolver (``kalshi_econ``), +a per-series agency-NAME router consuming the codegen'd econ settlement +table. """ -from mostlyright.markets.catalog import kalshi_nhigh, kalshi_nlow +from mostlyright.markets.catalog import kalshi_econ, kalshi_nhigh, kalshi_nlow from mostlyright.markets.catalog.kalshi_stations import ( KALSHI_SETTLEMENT_STATIONS, KNOWN_WRONG_STATIONS, @@ -15,6 +18,7 @@ "KALSHI_SETTLEMENT_STATIONS", "KNOWN_WRONG_STATIONS", "StationCitation", + "kalshi_econ", "kalshi_nhigh", "kalshi_nlow", ] diff --git a/packages/markets/src/mostlyright/markets/catalog/kalshi_econ.py b/packages/markets/src/mostlyright/markets/catalog/kalshi_econ.py new file mode 100644 index 00000000..297453ca --- /dev/null +++ b/packages/markets/src/mostlyright/markets/catalog/kalshi_econ.py @@ -0,0 +1,187 @@ +"""Kalshi Economics contract resolver (per-series agency-NAME routing). + +Kalshi Economics markets (CPI, PPI, NFP, GDP, jobless claims, Fed decision, …) +each settle against a specific agency's first print. ``resolve(series_ticker)`` +is the deterministic mapping from a Kalshi Economics series/contract ticker to +an :class:`EconResolution` ``(agency, indicator, settlement_grade, +contract_terms_url)`` tuple that downstream code (``research_econ()``, plan +29-08) uses to join the correct agency first-print row. + +It mirrors :mod:`mostlyright.markets.catalog.kalshi_nhigh` — a frozen-dataclass +result plus a deterministic ``resolve()`` with an up-front type guard, a +``ValueError`` on unknown tickers (it NEVER returns ``None``), and a +``__all__`` of ``[ResultClass, "resolve"]``. + +Routing is genuinely PER-SERIES, driven by the curated +:data:`~mostlyright.econ._settlement_map.SETTLEMENT_ROUTING` table — keyed on +canonical Kalshi series ROOT tickers and the agency NAMES those roots settle to. +The load-bearing case is the PPI family split: ``KXUSPPI`` (PPI MoM) settles to +Trading Economics at ``settlement_grade=False`` while its sibling +``KXUSPPIYOY`` (PPI YoY) settles to BLS at ``settlement_grade=True``. Assuming +"the government first print" for the 48 Trading-Economics-settled series +(``KXUSPPI``, ``KXUSNFP``, and international variants) would mis-settle every one +of them. + +SSRF / sloppy-URL firewall (Pitfall 2; STRIDE T-29-09): this resolver reads +ONLY the pre-curated agency NAMES in ``SETTLEMENT_ROUTING``. It NEVER fetches or +parses the Kalshi-supplied settlement-source URL field handed to us by series +metadata — those URLs are demonstrably wrong live (``KXPAYROLLS`` carries +``name="BLS"`` but a URL pointing at ``ppi.nr0.htm``; one series' URL points at +"San Francisco Unified School District"). Routing on the URL would settle NFP +contracts against PPI data. So: NAME only, from the curated table. (The literal +Kalshi field name is deliberately kept out of this module so the CI grep gate +that asserts "no URL routing" stays green.) + +ECON-17: for a Trading-Economics-settled series the resolver only labels +``settlement_grade=False``. It structurally cannot emit a fabricated Trading +Economics number — :class:`EconResolution` carries no value field; the value +stays the agency first-print in the data layer. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from mostlyright.econ._settlement_map import ( + AGENCY_NAMES, + SETTLEMENT_ROUTING, + SettlementRule, +) + + +@dataclass(frozen=True) +class EconResolution: + """The settlement mapping a Kalshi Economics contract resolves to. + + Attributes: + agency: The canonical settlement-agency short name — one of the five + :data:`~mostlyright.econ._settlement_map.AGENCY_NAMES` + (``"BLS"`` / ``"BEA"`` / ``"DOL"`` / ``"FederalReserve"`` / + ``"TradingEconomics"``). Routed on the Kalshi settlement-source + NAME, never the URL. + indicator: The econ indicator id (same vocabulary as + ``schema.econ.observations.v1``: ``cpi`` / ``cpi_core`` / + ``cpi_yoy`` / ``nfp`` / ``u3`` / ``gdp`` / ``ppi`` / ``ppi_yoy`` / + ``jobless_claims`` / ``fed_funds`` / …). + settlement_grade: ``True`` when the agency first print IS the + settlement truth; ``False`` for the 48 Trading-Economics-settled + series (the agency first print ships as a labeled proxy — no TE + value is ever fabricated; ECON-17). + contract_terms_url: The trusted CFTC contract-terms PDF basename from + the routing rule (e.g. ``"CPI.pdf"``), or ``""`` when the routing + rule carries no PDF. This is a curated routing companion to the + NAME — it is NEVER a fetched or Kalshi-supplied settlement-source + URL. + """ + + agency: str + indicator: str + settlement_grade: bool + contract_terms_url: str + + +def _match_rule(ticker: str) -> tuple[str, SettlementRule] | None: + """Find the routing rule for ``ticker`` by exact-then-longest-root match. + + Concrete Kalshi market tickers carry date/strike suffixes on top of the + series root (e.g. ``KXCPIYOY-26JUL``). We match the base rule by: + + 1. Exact match on the full (uppercased) ticker, then + 2. The LONGEST ``SETTLEMENT_ROUTING`` root that ``ticker`` starts with. + + Longest-prefix wins is load-bearing: ``KXUSPPIYOY`` must bind to the + ``KXUSPPIYOY`` rule (BLS / grade True), NOT the shorter ``KXUSPPI`` rule + (TradingEconomics / grade False) it also prefix-matches. A shorter-prefix + or first-seen match would silently flip the agency and settlement grade. + + Args: + ticker: The already-uppercased series/contract ticker. + + Returns: + The ``(root, SettlementRule)`` pair, or ``None`` if nothing matches. + """ + exact = SETTLEMENT_ROUTING.get(ticker) + if exact is not None: + return ticker, exact + + best_root: str | None = None + for root in SETTLEMENT_ROUTING: + if not ticker.startswith(root): + continue + # Require the dated-ticker delimiter "-" after the root (exact matches are + # handled above) so a longer uncatalogued series sharing a prefix (e.g. + # "KXCPIENERGY" vs "KXCPI") does not silently route to the shorter root + # (codex round-2 M10). + if len(ticker) > len(root) and ticker[len(root)] != "-": + continue + if best_root is None or len(root) > len(best_root): + best_root = root + if best_root is None: + return None + return best_root, SETTLEMENT_ROUTING[best_root] + + +def resolve(series_ticker: str) -> EconResolution: + """Resolve a Kalshi Economics ticker to its settlement mapping. + + Args: + series_ticker: A Kalshi Economics series root (``"KXCPIYOY"``) or a + concrete dated market ticker (``"KXCPIYOY-26JUL"``). + Case-insensitive. + + Returns: + The :class:`EconResolution` for the matched routing rule. + + Raises: + TypeError: ``series_ticker`` is not a ``str`` (mirrors kalshi_nhigh's + up-front type guard). + ValueError: ``series_ticker`` matches no known routing root. The + message names the ticker and the known roots (never returns + ``None`` — an unroutable ticker is an explicit error, not a + silent default to some agency). + """ + # Up-front type guard, mirroring kalshi_nhigh.resolve — reject non-str + # before any string op so callers get a clear TypeError, not an + # AttributeError from `.upper()`. + if not isinstance(series_ticker, str): + raise TypeError( + f"series_ticker must be a string (got {type(series_ticker).__name__}={series_ticker!r})" + ) + + ticker = series_ticker.upper() + matched = _match_rule(ticker) + if matched is None: + raise ValueError( + f"Unknown Kalshi econ series ticker {series_ticker!r}; it matches no " + f"routing root. known: {sorted(SETTLEMENT_ROUTING)}" + ) + + _root, rule = matched + + # ``rule.agency`` is ALREADY a canonical AGENCY_NAMES short name by + # construction of SETTLEMENT_ROUTING (the codegen table stores canonical + # values). We assert membership as defense-in-depth against future table + # drift rather than re-running normalize_agency_name — that helper accepts + # only the sloppy LIVE long forms ("Federal Reserve", "Trading Economics") + # and RAISES on the already-canonical short forms ("FederalReserve", + # "TradingEconomics"), so re-normalizing here would crash KXFED / KXUSPPI. + agency = rule.agency + if agency not in AGENCY_NAMES: + raise ValueError( + f"routing rule for {_root!r} carries non-canonical agency {agency!r}; " + f"expected one of {sorted(AGENCY_NAMES)}. This is a routing-drift " + "signal in SETTLEMENT_ROUTING — fix the table, do not paper over it." + ) + + return EconResolution( + agency=agency, + indicator=rule.indicator, + settlement_grade=rule.settlement_grade, + # Trusted curated PDF basename from the routing rule — NEVER a fetched + # or Kalshi-supplied settlement-source URL. Empty string when the + # rule carries no PDF. + contract_terms_url=rule.contract_terms_pdf or "", + ) + + +__all__ = ["EconResolution", "resolve"] diff --git a/packages/markets/src/mostlyright/markets/econ_trades.py b/packages/markets/src/mostlyright/markets/econ_trades.py new file mode 100644 index 00000000..b5071ce3 --- /dev/null +++ b/packages/markets/src/mostlyright/markets/econ_trades.py @@ -0,0 +1,265 @@ +"""Kalshi venue price-history for SETTLED econ markets — with documented depth. + +``candles(...)`` retrieves Kalshi OHLC candlesticks for a settled economics market +(``/series/{s}/markets/{m}/candlesticks``), mirroring +:mod:`mostlyright.markets.kalshi_trades`. It is the venue-history companion to the +econ vertical's settlement join (``mostlyright.econ.research_econ``): where +``research_econ`` produces the leakage-safe settlement PAIRS, this surfaces the +traded price panel for a settled econ contract. + +**The shallow-depth limit is a first-class, documented behavior (RESEARCH Pitfall +3 / ECON-20).** The PUBLIC Kalshi API surfaces only ~50-90 RECENT settled markets +per series (oldest ~2026-05), and a market's candlesticks span its own ~1-month +life. The FEDS-2026-010 first-contract dates (2021-2023) are therefore NOT +retrievable from the public settled-markets endpoint — a deep backtest needs the +FEDS replication dataset or a running collector. This module refuses to pretend +otherwise: + +- A window entirely before the public-depth floor (~2026-05) returns an + EMPTY DataFrame stamped ``df.attrs["public_depth_limited"] = True`` — never a + fabricated 2021-2023 candle. +- A window below the indicator's FEDS floor raises + :class:`~mostlyright.core.exceptions.DataAvailabilityError` (the floor is a + data-availability CONTRACT, mirroring ``econ.history``). + +The realistic depth expectation is documented in the module docstring and the +:data:`MAX_PUBLIC_DEPTH_NOTE` constant. + +This module lives in ``mostlyright.markets`` (not econ) because it drives the +Kalshi venue client — the workspace edge is one-way ``markets → econ``, so it MAY +reuse both the markets Kalshi client and the econ floor/routing tables. +""" + +from __future__ import annotations + +from datetime import UTC, date, datetime +from typing import TYPE_CHECKING, Any + +from mostlyright.econ._floor import assert_within_floor +from mostlyright.econ._settlement_map import resolve_settlement + +from ._kalshi_client import fetch_candlesticks + +if TYPE_CHECKING: + import httpx + import pandas as pd + +__all__ = ["INTERVALS", "MAX_PUBLIC_DEPTH_NOTE", "PUBLIC_DEPTH_FLOOR", "candles"] + +_SOURCE = "kalshi" + +#: Supported candle intervals → seconds (mirrors ``kalshi_trades.INTERVALS``). +INTERVALS: dict[str, int] = {"1m": 60, "1h": 3600, "1d": 86400} + +#: The realistic public-API depth floor. The public settled-markets endpoint +#: surfaces only recent markets (oldest observed ~2026-05); a window entirely +#: before this returns labeled-empty rather than a fabricated old candle. +PUBLIC_DEPTH_FLOOR: date = date(2026, 5, 1) + +#: The documented shallow-depth limit (surfaced so a caller sets realistic +#: expectations). The public Kalshi API is NOT a deep historical source. +MAX_PUBLIC_DEPTH_NOTE: str = ( + "Kalshi's public settled-markets API surfaces only ~50-90 RECENT settled " + "markets per series (oldest ~2026-05), and a market's candlesticks span its " + "own ~1-month life. The FEDS-2026-010 first-contract history (2021-2023) is " + "NOT retrievable from the public endpoint — a deep econ backtest needs the " + "FEDS replication dataset or a running collector. A pre-2026-05 window returns " + "an empty, clearly-labeled result (public_depth_limited=True); it is never a " + "fabricated deep candle." +) + + +def _require_pandas() -> Any: + """Lazy-import pandas with an actionable install hint on miss.""" + try: + import pandas as _pandas + except ImportError as exc: + from mostlyright.core.exceptions import SourceUnavailableError + + raise SourceUnavailableError( + "mostlyright.markets.econ_trades requires pandas. Install with: " + "pip install mostlyrightmd-markets[trades]", + source=_SOURCE, + retryable=False, + underlying=str(exc), + ) from None + return _pandas + + +def _validate_aware(dt: datetime, name: str) -> None: + if not isinstance(dt, datetime): + raise TypeError(f"{name} must be a datetime instance; got {type(dt).__name__}") + if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None: + raise TypeError(f"{name} must be a tz-aware UTC datetime; got naive {dt!r}") + + +def _series_root(ticker: str) -> str: + """Return the Kalshi series root (the prefix before the first ``-``).""" + if "-" not in ticker: + return ticker + return ticker.split("-", 1)[0] + + +def _empty_candles_frame(pd: Any, ticker: str, *, public_depth_limited: bool) -> pd.DataFrame: + """Build an empty OHLC frame carrying the depth-limit provenance flag.""" + df = pd.DataFrame( + columns=["ts", "open", "high", "low", "close", "volume", "open_interest", "source"] + ) + df.attrs["source"] = _SOURCE + df.attrs["ticker"] = ticker + df.attrs["public_depth_limited"] = public_depth_limited + df.attrs["depth_note"] = MAX_PUBLIC_DEPTH_NOTE + df.attrs["retrieved_at"] = datetime.now(UTC) + return df + + +def candles( + ticker: str, + *, + interval: str, + from_: datetime, + to: datetime, + client: httpx.Client | None = None, + sleep_between: float | None = None, +) -> pd.DataFrame: + """OHLC candles for a settled econ market ``ticker`` between ``from_`` and ``to``. + + Args: + ticker: Full Kalshi econ market ticker (e.g. ``"KXCPI-26JUL-T3.2"``). The + series root (``KXCPI``) is resolved to its econ indicator for the FEDS + floor check. + interval: Candle granularity — one of :data:`INTERVALS` keys. + from_, to: tz-aware UTC datetimes bounding the window. + client: Optional shared ``httpx.Client``. + sleep_between: Per-request polite-sleep override (tests pass 0). + + Returns: + ``pd.DataFrame`` with columns ``ts, open, high, low, close, volume, + open_interest, source`` (prices in cents; see ``kalshi_trades.candles``). + For a window entirely before the public-depth floor (~2026-05) the frame + is EMPTY with ``df.attrs["public_depth_limited"] = True`` — never a + fabricated deep candle. + + Raises: + TypeError: ``from_``/``to`` not tz-aware datetimes. + ValueError: ``from_ >= to`` OR ``interval`` not in :data:`INTERVALS`. + DataAvailabilityError: the window is below the resolved indicator's FEDS + first-contract floor (a data-availability contract violation). + """ + pd = _require_pandas() + _validate_aware(from_, "from_") + _validate_aware(to, "to") + if from_ >= to: + raise ValueError(f"from_ ({from_.isoformat()}) must be < to ({to.isoformat()})") + if interval not in INTERVALS: + raise ValueError(f"interval must be one of {sorted(INTERVALS)}; got {interval!r}") + + # FEDS floor as a data-availability contract: a below-floor window asks for a + # series the venue never priced. Resolve the ticker to its indicator; if it is + # not an econ series we skip the floor (a non-econ ticker has no econ floor). + try: + _root, rule = resolve_settlement(_series_root(ticker)) + except (ValueError, TypeError): + rule = None + if rule is not None: + assert_within_floor(rule.indicator, from_.date()) + + # Shallow public-depth guard: a window entirely before ~2026-05 cannot be + # served by the public settled-markets endpoint — return labeled-empty rather + # than pretend deep history exists. + if to.date() < PUBLIC_DEPTH_FLOOR: + return _empty_candles_frame(pd, ticker, public_depth_limited=True) + + period_interval_minutes = INTERVALS[interval] // 60 + raw = fetch_candlesticks( + ticker, + start_ts=int(from_.timestamp()), + end_ts=int(to.timestamp()), + period_interval=period_interval_minutes, + client=client, + sleep_between=sleep_between, + ) + if not raw: + # No candles in-window from a within-depth request — return an ordinary + # empty frame (NOT depth-limited; the market simply had no trades here). + return _empty_candles_frame(pd, ticker, public_depth_limited=False) + + rows: list[dict[str, Any]] = [] + for c in raw: + price = c.get("price") or {} + ts_value = c.get("end_period_ts") + ts = ( + datetime.fromtimestamp(int(ts_value), tz=UTC) + if isinstance(ts_value, (int, float)) + else None + ) + rows.append( + { + "ts": ts, + "open": _pick_price(price, "open"), + "high": _pick_price(price, "high"), + "low": _pick_price(price, "low"), + "close": _pick_price(price, "close"), + "volume": _pick_fp(c, "volume"), + "open_interest": _pick_fp(c, "open_interest"), + "source": _SOURCE, + } + ) + df = pd.DataFrame( + rows, + columns=["ts", "open", "high", "low", "close", "volume", "open_interest", "source"], + ) + df.attrs["source"] = _SOURCE + df.attrs["ticker"] = ticker + df.attrs["interval"] = interval + df.attrs["public_depth_limited"] = False + df.attrs["depth_note"] = MAX_PUBLIC_DEPTH_NOTE + df.attrs["retrieved_at"] = datetime.now(UTC) + return df + + +# --- Price-field helpers (mirror kalshi_trades) ------------------------------ +def _maybe_float(v: Any) -> float | None: + if v is None: + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +def _dollars_to_cents(v: Any) -> float | None: + f = _maybe_float(v) + return None if f is None else f * 100.0 + + +def _fp_string_to_int(v: Any) -> int | None: + if v is None: + return None + try: + return int(float(v)) + except (TypeError, ValueError): + return None + + +def _maybe_int(v: Any) -> int | None: + if v is None: + return None + try: + return int(v) + except (TypeError, ValueError): + return None + + +def _pick_price(d: dict[str, Any], base: str) -> float | None: + dollars_key = f"{base}_dollars" + if dollars_key in d: + return _dollars_to_cents(d.get(dollars_key)) + return _maybe_float(d.get(base)) + + +def _pick_fp(d: dict[str, Any], base: str) -> int | None: + fp_key = f"{base}_fp" + if fp_key in d: + return _fp_string_to_int(d.get(fp_key)) + return _maybe_int(d.get(base)) diff --git a/packages/markets/tests/catalog/test_kalshi_econ.py b/packages/markets/tests/catalog/test_kalshi_econ.py new file mode 100644 index 00000000..3b94ea48 --- /dev/null +++ b/packages/markets/tests/catalog/test_kalshi_econ.py @@ -0,0 +1,166 @@ +"""Contract tests for the Kalshi econ settlement resolver (29-04). + +``kalshi_econ.resolve(series_ticker)`` is the deterministic mapping from a +Kalshi Economics series/contract ticker to an :class:`EconResolution` +``(agency, indicator, settlement_grade, contract_terms_url)`` tuple. It mirrors +:mod:`mostlyright.markets.catalog.kalshi_nhigh`'s frozen-dataclass pattern and +routes PER-SERIES off the curated ``SETTLEMENT_ROUTING`` agency NAMES — never a +``settlement_sources[].url`` (Pitfall 2 / STRIDE T-29-09 SSRF + sloppy-URL risk). + +The load-bearing invariant proven here is the per-series split inside the PPI +family: ``KXUSPPI`` (PPI MoM) settles to Trading Economics at +``settlement_grade=False`` while its sibling ``KXUSPPIYOY`` (PPI YoY) settles to +BLS at ``settlement_grade=True``. Assuming "the government first print" for the +48 Trading-Economics-settled series would mis-settle every one of them. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest +from mostlyright.markets.catalog import kalshi_econ +from mostlyright.markets.catalog.kalshi_econ import EconResolution, resolve + +_RESOLVER_SRC = Path(kalshi_econ.__file__).resolve() + + +# --- Test 1: CPI-YoY → BLS, settlement-grade ------------------------------- +def test_resolve_cpi_yoy_bls_settlement_grade() -> None: + r = resolve("KXCPIYOY") + assert isinstance(r, EconResolution) + assert r.agency == "BLS" + assert r.indicator == "cpi_yoy" + assert r.settlement_grade is True + + +# --- Test 2: PPI per-series split (the load-bearing proof) ----------------- +def test_ppi_per_series_split_mom_te_vs_yoy_bls() -> None: + """KXUSPPI (MoM) → TradingEconomics/False; KXUSPPIYOY (YoY) → BLS/True. + + This is the proof that routing is genuinely per-series, not per-family: + two members of the PPI family route to DIFFERENT agencies at DIFFERENT + settlement grades. + """ + mom = resolve("KXUSPPI") + assert mom.agency == "TradingEconomics" + assert mom.settlement_grade is False + assert mom.indicator == "ppi" + + yoy = resolve("KXUSPPIYOY") + assert yoy.agency == "BLS" + assert yoy.settlement_grade is True + assert yoy.indicator == "ppi_yoy" + + # Same family, opposite routing — the whole point. + assert mom.agency != yoy.agency + assert mom.settlement_grade != yoy.settlement_grade + + +# --- Test 3: NFP per-series split ------------------------------------------ +def test_nfp_us_te_vs_payrolls_bls() -> None: + """KXUSNFP → TradingEconomics/False; KXPAYROLLS → BLS/True.""" + us_nfp = resolve("KXUSNFP") + assert us_nfp.agency == "TradingEconomics" + assert us_nfp.settlement_grade is False + + payrolls = resolve("KXPAYROLLS") + assert payrolls.agency == "BLS" + assert payrolls.settlement_grade is True + assert payrolls.indicator == "nfp" + + +# --- Test 4: GDP/Fed/Jobless agency routing -------------------------------- +def test_gdp_fed_jobless_agencies() -> None: + assert resolve("KXGDP").agency == "BEA" + # Fed funds decision routes to the Federal Reserve — NOT FRED. + assert resolve("KXFED").agency == "FederalReserve" + assert resolve("KXJOBLESSCLAIMS").agency == "DOL" + + +# --- Test 5: unknown/bad input RAISES (never returns None) ----------------- +def test_unknown_ticker_raises_value_error() -> None: + with pytest.raises(ValueError) as excinfo: + resolve("KXTOTALLYUNKNOWN") + # Mirror kalshi_nhigh: the error names the ticker and lists known roots. + msg = str(excinfo.value) + assert "KXTOTALLYUNKNOWN" in msg + assert "KXCPIYOY" in msg # a known root is surfaced to the caller + + +def test_non_string_ticker_raises_type_error() -> None: + with pytest.raises(TypeError): + resolve(123) # type: ignore[arg-type] + + +def test_resolve_never_returns_none_for_bad_input() -> None: + """Belt-and-suspenders: the resolver RAISES, it does not return None.""" + for bad in ("", "NOTAKALSHITICKER", "KX", "SPY"): + with pytest.raises(ValueError): + resolve(bad) + + +# --- Test 6: case-insensitive ---------------------------------------------- +def test_case_insensitive_resolution() -> None: + assert resolve("kxcpiyoy") == resolve("KXCPIYOY") + assert resolve("KxUsPpI") == resolve("KXUSPPI") + + +# --- Dated / concrete market tickers resolve by root match ----------------- +def test_dated_ticker_root_match() -> None: + """A dated market ticker (``KXCPIYOY-26JUL``) resolves to the base rule.""" + base = resolve("KXCPIYOY") + dated = resolve("KXCPIYOY-26JUL") + assert dated == base + + # Longest-prefix wins: KXUSPPIYOY must NOT collapse onto the shorter + # KXUSPPI rule (that would flip TE↔BLS and False↔True). + dated_yoy = resolve("KXUSPPIYOY-26JUL") + assert dated_yoy.agency == "BLS" + assert dated_yoy.settlement_grade is True + + +# --- EconResolution is a frozen dataclass (immutability) ------------------- +def test_econresolution_is_frozen() -> None: + r = resolve("KXCPIYOY") + assert dataclasses.is_dataclass(r) + with pytest.raises(dataclasses.FrozenInstanceError): + r.agency = "BEA" # type: ignore[misc] + + +# --- Every routed agency is in the canonical five-name vocabulary ---------- +def test_all_agencies_in_canonical_vocabulary() -> None: + from mostlyright.econ._settlement_map import AGENCY_NAMES, SETTLEMENT_ROUTING + + for root in SETTLEMENT_ROUTING: + assert resolve(root).agency in AGENCY_NAMES + + +# --- SSRF firewall: resolver never references settlement_sources URLs ------ +def test_resolver_source_has_no_settlement_sources_url() -> None: + """grep-provable: the resolver must NOT read ``settlement_sources`` at all. + + Routing is agency-NAME-driven (Pitfall 2 / STRIDE T-29-09). The literal + ``settlement_sources`` token appearing in the resolver source would be a + signal that URL-routing crept in. + """ + src = _RESOLVER_SRC.read_text(encoding="utf-8") + assert "settlement_sources" not in src + + +# --- ECON-17: TE-settled series never carry a fabricated value ------------- +def test_te_series_only_labels_grade_never_a_value() -> None: + """A TE-settled series is labeled settlement_grade=False and nothing more. + + ``EconResolution`` carries no numeric value field — the resolver cannot + (structurally) emit a fabricated Trading-Economics number; the value stays + the agency first-print in the data layer (ECON-17). + """ + field_names = {f.name for f in dataclasses.fields(EconResolution)} + assert field_names == { + "agency", + "indicator", + "settlement_grade", + "contract_terms_url", + } diff --git a/packages/markets/tests/test_econ_trades.py b/packages/markets/tests/test_econ_trades.py new file mode 100644 index 00000000..5b60c503 --- /dev/null +++ b/packages/markets/tests/test_econ_trades.py @@ -0,0 +1,111 @@ +"""Tests for ``mostlyright.markets.econ_trades`` — Kalshi venue price-history. + +``econ_trades.candles(...)`` retrieves Kalshi OHLC candlesticks for a SETTLED econ +market (``/series/{s}/markets/{m}/candlesticks``), mirroring ``kalshi_trades.py``. + +The load-bearing behavior (RESEARCH Pitfall 3 / ECON-20): the PUBLIC Kalshi API +surfaces only ~50-90 RECENT settled markets per series (oldest ~2026-05), and +candlesticks span a market's own ~1-month life. A pre-2026-05 window (or a window +below the FEDS floor) returns an empty/clearly-labeled result — the code does NOT +pretend the 2021-2023 first contracts the API cannot return exist. The shallow +depth is documented in the module docstring + a ``MAX_PUBLIC_DEPTH_NOTE`` constant. + +No network here: candlestick responses are injected via ``httpx.MockTransport``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import httpx +import pytest +from mostlyright.markets import econ_trades + +_TICKER = "KXCPI-26JUL-T3.2" + + +def _candlestick_client() -> httpx.Client: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.scheme == "https" + return httpx.Response( + 200, + json={ + "candlesticks": [ + { + "end_period_ts": 1751000000, + "price": { + "open_dollars": "0.5000", + "high_dollars": "0.5500", + "low_dollars": "0.4800", + "close_dollars": "0.5200", + }, + "volume_fp": "100", + "open_interest_fp": "500", + } + ] + }, + ) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +# --- Test 2: candlestick retrieval returns the OHLC shape --------------------- +def test_candles_returns_ohlc_for_recent_settled_market() -> None: + client = _candlestick_client() + # A recent (post-2026-05) window within the market's life. + df = econ_trades.candles( + _TICKER, + interval="1h", + from_=datetime(2026, 6, 1, tzinfo=UTC), + to=datetime(2026, 6, 30, tzinfo=UTC), + client=client, + sleep_between=0, + ) + assert len(df) == 1 + for col in ("ts", "open", "high", "low", "close", "volume", "open_interest"): + assert col in df.columns + assert df["source"].iloc[0] == "kalshi" + + +# --- Test 2 (cont.): a pre-2026-05 window returns labeled-empty ---------------- +def test_pre_public_depth_window_returns_empty_without_false_history() -> None: + client = _candlestick_client() # even if the API returned data, depth guard first + df = econ_trades.candles( + _TICKER, + interval="1h", + from_=datetime(2024, 1, 1, tzinfo=UTC), # before the ~2026-05 public floor + to=datetime(2024, 2, 1, tzinfo=UTC), + client=client, + sleep_between=0, + ) + # The public API cannot return this depth — the result is empty + labeled, + # NOT a fabricated 2024 candle. + assert len(df) == 0 + assert df.attrs.get("public_depth_limited") is True + + +# --- Test 3: the shallow-depth limit is documented ---------------------------- +def test_shallow_depth_is_documented() -> None: + # A module-level note constant states the public API surfaces only recent + # settled markets — deep backtests need the FEDS dataset or a collector. + assert hasattr(econ_trades, "MAX_PUBLIC_DEPTH_NOTE") + note = econ_trades.MAX_PUBLIC_DEPTH_NOTE.lower() + assert "50-90" in note or "shallow" in note or "2026-05" in note + assert "feds" in note or "collector" in note + + +def test_below_feds_floor_raises() -> None: + client = _candlestick_client() + # A window below the CPI FEDS first-contract floor (2021-06) is a + # data-availability contract violation → raise, not silent empty. + from mostlyright.core.exceptions import DataAvailabilityError + + with pytest.raises(DataAvailabilityError): + econ_trades.candles( + _TICKER, + interval="1h", + from_=datetime(2020, 1, 1, tzinfo=UTC), + to=datetime(2020, 2, 1, tzinfo=UTC), + client=client, + sleep_between=0, + ) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3590484..b9d20df8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,6 +106,21 @@ importers: specifier: 2.1.9 version: 2.1.9(@types/node@20.16.10)(jsdom@25.0.1)(msw@2.6.6(@types/node@20.16.10)(typescript@5.6.3)) + packages-ts/econ: + devDependencies: + '@mostlyrightmd/core': + specifier: workspace:* + version: link:../core + tsup: + specifier: 8.3.5 + version: 8.3.5(jiti@2.7.0)(postcss@8.5.15)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.9.0) + typescript: + specifier: 5.6.3 + version: 5.6.3 + vitest: + specifier: 2.1.9 + version: 2.1.9(@types/node@20.16.10)(jsdom@25.0.1)(msw@2.6.6(@types/node@20.16.10)(typescript@5.6.3)) + packages-ts/markets: devDependencies: '@mostlyrightmd/core': diff --git a/pyproject.toml b/pyproject.toml index 6c6d97d3..086931c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "mostlyrightmd", "mostlyrightmd-weather", "mostlyrightmd-markets", + "mostlyrightmd-econ", ] [build-system] @@ -30,12 +31,13 @@ bypass-selection = true package = false [tool.uv.workspace] -members = ["packages/core", "packages/weather", "packages/markets"] +members = ["packages/core", "packages/weather", "packages/markets", "packages/econ"] [tool.uv.sources] mostlyrightmd = { workspace = true } mostlyrightmd-weather = { workspace = true } mostlyrightmd-markets = { workspace = true } +mostlyrightmd-econ = { workspace = true } [dependency-groups] dev = [ diff --git a/schemas/EXPORT_MANIFEST.json b/schemas/EXPORT_MANIFEST.json index 1d8b7ee1..f42a6087 100644 --- a/schemas/EXPORT_MANIFEST.json +++ b/schemas/EXPORT_MANIFEST.json @@ -24,6 +24,12 @@ "sha256": "a415bce1a3d54fb2d15f5c56169c12d9de160df9241bcd02d9ded10b3bcd45d7", "size_bytes": 3106 }, + { + "gated": false, + "path": "json/schema.econ.observations.v1.json", + "sha256": "137da0caf60ccae8cf81a0eed8011dc8eebce4dbce51aa9454265ec8ebbd65e4", + "size_bytes": 2709 + }, { "gated": false, "path": "json/schema.forecast.iem_mos.v1.json", @@ -72,6 +78,12 @@ "sha256": "59d78ec98cfa8c93b78e731fcd27f74c12eae1cd1cdac053852a465371be9882", "size_bytes": 2795 }, + { + "gated": false, + "path": "kalshi-econ-settlement.json", + "sha256": "d1b9248e88edf553d3459110f3f1a9285b3f9695b898b903457c19ce0e244e76", + "size_bytes": 2319 + }, { "gated": false, "path": "kalshi-settlement-stations.json", diff --git a/schemas/json/schema.econ.observations.v1.json b/schemas/json/schema.econ.observations.v1.json new file mode 100644 index 00000000..7554b924 --- /dev/null +++ b/schemas/json/schema.econ.observations.v1.json @@ -0,0 +1,94 @@ +{ + "$id": "https://mostlyright.dev/schemas/schema.econ.observations.v1.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "indicator": { + "description": "indicator id: cpi | cpi_core | cpi_yoy | nfp | u3 | gdp | ppi | ppi_yoy | jobless_claims | fed_funds", + "type": "string" + }, + "knowledge_time": { + "description": "leakage cutoff; equals vintage_date for econ (research_econ wires leakage on this)", + "format": "date-time", + "type": "string" + }, + "period": { + "description": "observation period the value refers to: '2026-06' monthly, '2026Q2' quarterly, '2026-06-28' weekly/date, or FOMC meeting date", + "type": "string" + }, + "release_datetime": { + "description": "wall-clock the value was released (8:30 ET etc.), UTC", + "format": "date-time", + "type": [ + "null", + "string" + ] + }, + "release_type": { + "description": "advance | second | third | final | revised | benchmark | preliminary", + "enum": [ + "advance", + "benchmark", + "final", + "preliminary", + "revised", + "second", + "third" + ], + "type": "string" + }, + "retrieved_at": { + "description": "provenance of the fetch, UTC", + "format": "date-time", + "type": [ + "null", + "string" + ] + }, + "series_id": { + "description": "upstream series identifier (BLS/FRED/BEA code) when applicable", + "type": [ + "null", + "string" + ] + }, + "settlement_grade": { + "description": "is this vintage the settlement truth (first print as-of expiration); False for later vintages AND the 48 TE-settled series", + "type": "boolean" + }, + "source": { + "description": "per-row source identity == df.attrs['source']; one of ECON_SOURCES", + "type": "string" + }, + "units": { + "description": "index | percent | thousands_persons | percent_saar | \u2026", + "type": [ + "null", + "string" + ] + }, + "value": { + "description": "released numeric value; nullable \u2014 a not-yet-released period may be schema-shaped but valueless", + "type": [ + "null", + "number" + ] + }, + "vintage_date": { + "description": "WHEN this value became known (ALFRED realtime_start / release-day capture); the leakage knowledge_time", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "indicator", + "knowledge_time", + "period", + "release_type", + "settlement_grade", + "source", + "vintage_date" + ], + "title": "schema.econ.observations.v1", + "type": "object", + "version": "v1" +} diff --git a/schemas/kalshi-econ-settlement.json b/schemas/kalshi-econ-settlement.json new file mode 100644 index 00000000..b2f6300a --- /dev/null +++ b/schemas/kalshi-econ-settlement.json @@ -0,0 +1,95 @@ +{ + "agency_names": [ + "BEA", + "BLS", + "DOL", + "FederalReserve", + "TradingEconomics" + ], + "routing": { + "KXCPI": { + "agency": "BLS", + "contract_terms_pdf": "CPI.pdf", + "indicator": "cpi", + "settlement_grade": true + }, + "KXCPICORE": { + "agency": "BLS", + "contract_terms_pdf": "CPICORE.pdf", + "indicator": "cpi_core", + "settlement_grade": true + }, + "KXCPICOREYOY": { + "agency": "BLS", + "contract_terms_pdf": "CPICOREYOY.pdf", + "indicator": "cpi_core_yoy", + "settlement_grade": true + }, + "KXCPIYOY": { + "agency": "BLS", + "contract_terms_pdf": "CPIYOY.pdf", + "indicator": "cpi_yoy", + "settlement_grade": true + }, + "KXFED": { + "agency": "FederalReserve", + "contract_terms_pdf": "FED.pdf", + "indicator": "fed_funds", + "settlement_grade": true + }, + "KXFEDDECISION": { + "agency": "FederalReserve", + "contract_terms_pdf": "FEDDECISION.pdf", + "indicator": "fed_decision", + "settlement_grade": true + }, + "KXGDP": { + "agency": "BEA", + "contract_terms_pdf": "GDP.pdf", + "indicator": "gdp", + "settlement_grade": true + }, + "KXJOBLESS": { + "agency": "DOL", + "contract_terms_pdf": "JOBLESSCLAIMS.pdf", + "indicator": "jobless_claims", + "settlement_grade": true + }, + "KXJOBLESSCLAIMS": { + "agency": "DOL", + "contract_terms_pdf": "JOBLESSCLAIMS.pdf", + "indicator": "jobless_claims", + "settlement_grade": true + }, + "KXPAYROLLS": { + "agency": "BLS", + "contract_terms_pdf": "PAYROLLS.pdf", + "indicator": "nfp", + "settlement_grade": true + }, + "KXU3": { + "agency": "BLS", + "contract_terms_pdf": "U3.pdf", + "indicator": "u3", + "settlement_grade": true + }, + "KXUSNFP": { + "agency": "TradingEconomics", + "contract_terms_pdf": "ECONSTATTE.pdf", + "indicator": "nfp", + "settlement_grade": false + }, + "KXUSPPI": { + "agency": "TradingEconomics", + "contract_terms_pdf": "ECONSTATTE.pdf", + "indicator": "ppi", + "settlement_grade": false + }, + "KXUSPPIYOY": { + "agency": "BLS", + "contract_terms_pdf": "PPIYOY.pdf", + "indicator": "ppi_yoy", + "settlement_grade": true + } + } +} diff --git a/scripts/check_no_hosted_calls.py b/scripts/check_no_hosted_calls.py index e19b5786..4c8d912b 100644 --- a/scripts/check_no_hosted_calls.py +++ b/scripts/check_no_hosted_calls.py @@ -12,8 +12,8 @@ call, but it whitelists the opt-in seam identifiers so the 28-31 / 28-40 seam code does not trip it. -Two assertions over the three published distributions' source -(``packages/{core,weather,markets}/src``, tests excluded): +Two assertions over the published distributions' source +(``packages/{core,weather,markets,econ}/src``, tests excluded): RULE 1 — no hardcoded hosted host. A hosted call in the default path always needs a URL; the opt-in seam reads it from ``*_HOSTED_URL`` at call time, so a @@ -43,11 +43,16 @@ import sys from pathlib import Path -#: The three published distributions whose default path must stay hosted-call-free. +#: The published distributions whose default path must stay hosted-call-free. +#: econ (Phase 29) makes NO actual hosted call — but it DOES carry the reserved +#: opt-in ``delivery="hosted"`` seam (which raises today rather than calling), so +#: its seam-bearing public-surface files are allowlisted in ``_SEAM_FILES`` below; +#: RULE 1 still scans all of econ for a stray hosted-host literal. _PUBLISHED_SRC = ( "packages/core/src", "packages/weather/src", "packages/markets/src", + "packages/econ/src", ) #: RULE 1 — hardcoded hosted-host literals. Never legitimate in published code @@ -75,6 +80,12 @@ "packages/weather/src/mostlyright/weather/catalog/earnings.py", "packages/weather/src/mostlyright/weather/satellite/_hosted_client.py", "packages/weather/src/mostlyright/weather/satellite/__init__.py", + # econ (Phase 29) opt-in delivery="hosted" seam — these validate-and-REJECT + # hosted today (raise SourceUnavailableError naming ECON_HOSTED_URL), the + # reserved paid-tier seam; no actual hosted call reaches the default path. + "packages/econ/src/mostlyright/econ/_history.py", + "packages/econ/src/mostlyright/econ/_research.py", + "packages/econ/src/mostlyright/econ/_snapshot.py", } ) diff --git a/scripts/export_schemas.py b/scripts/export_schemas.py index e4f87ef9..9b01ff53 100644 --- a/scripts/export_schemas.py +++ b/scripts/export_schemas.py @@ -71,7 +71,7 @@ # prepend is a no-op in that case but is what lets the exporter run from a # bare ``python scripts/export_schemas.py`` shell, which the CI workflow # does as a sanity check. -for _pkg in ("core", "weather", "markets"): +for _pkg in ("core", "weather", "markets", "econ"): _src = _REPO_ROOT / "packages" / _pkg / "src" if _src.is_dir(): _src_str = str(_src) @@ -114,6 +114,12 @@ "schema.satellite.v1", "schema.earnings_transcript.v1", "schema.earnings_fact.v1", + # Phase 29 (29-03) — the econ vertical's standalone observation schema. + # It is NOT a member of ``mostlyright.core.schemas`` (econ registers lazily + # on its OWN package import, the CWOP firewall discipline), so + # ``_build_group_a_schemas`` imports it from ``mostlyright.econ._schema`` + # separately. Emitting it here makes it the TS-port (29-09) codegen input. + "schema.econ.observations.v1", ) @@ -283,6 +289,12 @@ def _build_group_a_schemas() -> list[_OutputFile]: StationForecastSchema, ) + # Econ's schema is STANDALONE — it is registered lazily when + # ``mostlyright.econ._schema`` is imported (the CWOP firewall discipline at + # package granularity), NOT exported from ``mostlyright.core.schemas``. So we + # import the class directly from its own module rather than the core barrel. + from mostlyright.econ._schema import EconObservationsSchema + by_id: dict[str, Any] = { ObservationSchema.schema_id: ObservationSchema, ForecastSchema.schema_id: ForecastSchema, @@ -294,6 +306,7 @@ def _build_group_a_schemas() -> list[_OutputFile]: SatelliteSchema.schema_id: SatelliteSchema, EarningsTranscriptSchema.schema_id: EarningsTranscriptSchema, EarningsFactSchema.schema_id: EarningsFactSchema, + EconObservationsSchema.schema_id: EconObservationsSchema, } out: list[_OutputFile] = [] for schema_id in _GROUP_A_SCHEMA_IDS: @@ -357,6 +370,54 @@ def _build_kalshi() -> _OutputFile: ) +def _build_kalshi_econ_settlement() -> _OutputFile: + """Emit ``schemas/kalshi-econ-settlement.json`` — per-series settlement routing. + + Codegen'd from ``mostlyright.econ._settlement_map.SETTLEMENT_ROUTING`` (agency + NAMES + contract-terms PDFs — NEVER the sloppy ``settlement_sources[].url``; + RESEARCH Pitfall 2 / STRIDE T-29-06). The emitted table is what the resolver + (29-04) and the TS resolver (29-09) consume to route a Kalshi Economics + contract to its settlement agency + indicator + settlement_grade. + + Mirrors ``_build_kalshi()`` (sorted, ``_dumps``-canonical) with the + ``_build_earnings_webcast_providers`` gated-stub fallback: if the econ source + module is not materialized in ``packages/`` (a base install without the econ + package), the file is emitted as a gated stub rather than crashing the export. + """ + rel = "kalshi-econ-settlement.json" + try: + from mostlyright.econ._settlement_map import ( + AGENCY_NAMES, + SETTLEMENT_ROUTING, + ) + except ImportError: + return _OutputFile( + rel_path=rel, + content=_gated_payload( + "Python source econ._settlement_map.SETTLEMENT_ROUTING " + "not materialized in packages/" + ).encode("utf-8"), + gated=True, + ) + + # Emit routing rules sorted by ticker; each rule dict is keyed sort_keys=True + # at dump time. Routing is agency-NAME driven — no URL is emitted or consumed. + routing: dict[str, dict[str, Any]] = {} + for ticker in sorted(SETTLEMENT_ROUTING): + rule = SETTLEMENT_ROUTING[ticker] + routing[ticker] = { + "agency": rule.agency, + "indicator": rule.indicator, + "settlement_grade": rule.settlement_grade, + "contract_terms_pdf": rule.contract_terms_pdf, + } + payload = { + "routing": routing, + "agency_names": sorted(AGENCY_NAMES), + } + return _OutputFile(rel_path=rel, content=_dumps(payload).encode("utf-8")) + + def _build_source_priority() -> _OutputFile: """Emit ``schemas/source-priority.json``.""" from mostlyright._internal.merge.climate import REPORT_TYPE_PRIORITY @@ -568,6 +629,7 @@ def build_all_outputs() -> list[_OutputFile]: files.extend(_build_group_a_schemas()) files.append(_build_stations()) files.append(_build_kalshi()) + files.append(_build_kalshi_econ_settlement()) files.append(_build_source_priority()) files.append(_build_polymarket_city_stations()) files.append(_build_qc_alpha_rules()) diff --git a/tests/test_econ_live_smoke.py b/tests/test_econ_live_smoke.py new file mode 100644 index 00000000..ad3b55f4 --- /dev/null +++ b/tests/test_econ_live_smoke.py @@ -0,0 +1,606 @@ +"""Live real-API smoke for the econ vertical (``@pytest.mark.live``, CI-excluded). + +The MANDATORY pre-publish real-API gate for the econ adapter (development-workflow +§4: "no code ships against real APIs until a real-API smoke passes"). It hits the +REAL production endpoints with NO mocks (minimal config/logger only), exercises the +full econ path end-to-end, validates the output is SANE (non-empty, values in +plausible ranges, real timestamps, ``settlement_grade`` correctly set, +``vintage_date <= now``), and PRINTS a sample row per source for a human +sanity-check (run with ``-s`` to see them). + +Run: + + uv run pytest tests/test_econ_live_smoke.py -m live -s -v + +Coverage split (RESEARCH §"Environment Availability"): + +- **Keyless (always run under ``-m live``)** — Kalshi trade-api v2 + (``/series?category=Economics`` + a settled-market candlestick), BLS v1 keyless + (CPI/NFP AND PPI final-demand ``WPSFD4`` — proving the BLOCKER-1 PPI path returns + a real value), Polymarket gamma ``/events`` + CLOB ``/prices-history``. +- **Keyed (skipped unless the operator exports the key)** — real ALFRED vintage + fidelity (closes A1, the #1 landmine: does keyed ALFRED reproduce the first + print?), the keyed jobless-claims ICSA first-print (``settlement_grade=True`` — + BLOCKER-3 against real ALFRED ICSA vintages), real BEA GDP estimate-type (closes + A5), real Fed decision sourcing (closes A6). + +SECURITY: an API key is NEVER printed, logged, or embedded in a sample row. Every +``print`` here emits only DATA (values, periods, timestamps, flags). The keyed +tests read the key from the environment and pass it only into the fetcher's +outbound request; the ``_assert_no_key_in`` helper backstops sample rows. +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime, timedelta +from typing import Any + +import httpx +import pytest + +pytestmark = pytest.mark.live + +# --- Key presence gates (keyed sub-paths skip cleanly when absent) ------------ +_FRED_KEY = os.environ.get("FRED_API_KEY") +_BEA_KEY = os.environ.get("BEA_API_KEY") + +_needs_fred = pytest.mark.skipif( + not _FRED_KEY, + reason="FRED_API_KEY not set — keyed ALFRED/jobless sub-paths skipped (closes A1/BLOCKER-3). " + "Free key: https://fred.stlouisfed.org/docs/api/api_key.html", +) +_needs_bea = pytest.mark.skipif( + not _BEA_KEY, + reason="BEA_API_KEY not set — keyed BEA GDP sub-path skipped (closes A5). " + "Free key: https://apps.bea.gov/API/signup", +) + +# A generous "now" ceiling for the vintage_date <= now sanity check (a small +# forward skew tolerates clock drift between us and the agency). +_NOW_CEILING = datetime.now(UTC) + timedelta(hours=1) + +# Plausible-range sanity bounds (loose — a smoke check, not a unit assertion). +_CPI_MIN, _CPI_MAX = 150.0, 600.0 # CPI-U index level (~300s in 2026). +_PPI_MIN, _PPI_MAX = 100.0, 600.0 # PPI final-demand index level. +_NFP_MIN, _NFP_MAX = 100_000.0, 200_000.0 # total nonfarm payroll LEVEL (thousands → ~159,000). + + +# --- Key-leakage backstop ----------------------------------------------------- +def _assert_no_key_in(obj: Any) -> None: + """Fail loudly if any configured API key appears in a to-be-printed object.""" + text = repr(obj) + for key in (_FRED_KEY, _BEA_KEY, os.environ.get("BLS_API_KEY")): + if key: + assert key not in text, "API key leaked into smoke output — REDACT before printing." + + +def _print_sample(source: str, sample: Any) -> None: + """Print a single sample row for human sanity-check (never a key).""" + _assert_no_key_in(sample) + print(f"\n[{source}] sample: {sample}") + + +def _assert_aware_not_future(ts: Any, label: str) -> None: + """Assert ``ts`` is a tz-aware UTC datetime not in the future (<= now+1h).""" + assert isinstance(ts, datetime), f"{label} is not a datetime: {ts!r}" + assert ts.tzinfo is not None, f"{label} is naive (not tz-aware): {ts!r}" + assert ts <= _NOW_CEILING, ( + f"{label} is in the future: {ts.isoformat()} > {_NOW_CEILING.isoformat()}" + ) + + +# ============================================================================= +# KEYLESS sub-paths — always run under -m live. +# ============================================================================= +class TestKalshiKeyless: + """Real Kalshi trade-api v2 (public, no auth).""" + + def test_series_economics_has_settlement_sources(self): + """``/series?category=Economics`` returns real Economics series w/ NAMEs.""" + from mostlyright.markets._kalshi_client import KALSHI_API_BASE + + with httpx.Client(timeout=30.0) as client: + resp = client.get( + f"{KALSHI_API_BASE}/series", + params={"category": "Economics"}, + headers={"User-Agent": "mostlyright-sdk/econ-smoke"}, + ) + resp.raise_for_status() + payload = resp.json() + series = payload.get("series") or [] + assert isinstance(series, list) and len(series) > 100, ( + f"expected >100 Economics series (RESEARCH observed ~607); got {len(series)}" + ) + + # Find a series carrying a settlement_sources NAME + a contract_terms_url. + named = [ + s + for s in series + if s.get("settlement_sources") + and any(src.get("name") for src in s["settlement_sources"]) + ] + assert named, "no Economics series carried a settlement_sources NAME" + sample = named[0] + names = [src.get("name") for src in sample["settlement_sources"]] + _print_sample( + "kalshi.series", + { + "ticker": sample.get("ticker"), + "title": sample.get("title"), + "settlement_source_names": names, + "contract_terms_url": sample.get("contract_terms_url"), + }, + ) + + def test_settled_market_candlestick_ohlc(self): + """A settled econ market yields real OHLC candles with real timestamps. + + Drives ``mostlyright.markets.econ_trades.candles`` (the full venue path + incl. the FEDS-floor + public-depth guards) over a recent within-depth + window. If no settled market with in-window candles is found (venue depth + is shallow), the test skips rather than fails — the depth limit is a + documented behavior, not a smoke failure. + """ + from mostlyright.markets._kalshi_client import KALSHI_API_BASE + from mostlyright.markets.econ_trades import candles + + # List recent settled markets for a high-volume econ series (CPI). + with httpx.Client(timeout=30.0) as client: + resp = client.get( + f"{KALSHI_API_BASE}/markets", + params={"series_ticker": "KXCPI", "status": "settled", "limit": 50}, + headers={"User-Agent": "mostlyright-sdk/econ-smoke"}, + ) + resp.raise_for_status() + markets = resp.json().get("markets") or [] + + if not markets: + pytest.skip("no settled KXCPI markets surfaced by the public API right now") + + # Try a handful of recent settled markets for one with in-window candles. + now = datetime.now(UTC) + window_start = now - timedelta(days=120) + found = None + for m in markets[:10]: + ticker = m.get("ticker") + if not ticker: + continue + df = candles( + ticker, + interval="1d", + from_=window_start, + to=now, + sleep_between=0.1, + ) + if len(df) > 0: + found = (ticker, df) + break + + if found is None: + pytest.skip( + "no in-window candles on recent settled KXCPI markets (shallow public depth — " + "documented limitation, not a smoke failure)" + ) + + ticker, df = found + row = df.iloc[0] + _assert_aware_not_future(row["ts"].to_pydatetime(), "kalshi candle ts") + # OHLC present and non-negative (prices in cents). + for col in ("open", "high", "low", "close"): + val = row[col] + if val is not None: + assert val >= 0, f"kalshi candle {col} negative: {val}" + assert df.attrs.get("public_depth_limited") is False + _print_sample( + "kalshi.candle", + { + "ticker": ticker, + "ts": row["ts"].isoformat(), + "open": row["open"], + "high": row["high"], + "low": row["low"], + "close": row["close"], + "volume": row["volume"], + }, + ) + + +class TestBLSKeyless: + """Real BLS v1 keyless timeseries (latest-revised, settlement_grade=False).""" + + def test_cpi_and_nfp_values_in_range(self): + """BLS v1 returns a CPI-U + NFP value in a plausible range, grade False.""" + from mostlyright.econ._fetchers import bls + + year = datetime.now(UTC).year + rows = bls.fetch( + ["CUUR0000SA0", "CES0000000001"], # CPI-U, total nonfarm payrolls. + start_year=year - 1, + end_year=year, + ) + assert rows, "BLS returned no CPI/NFP rows" + + cpi_rows = [r for r in rows if r["indicator"] == "cpi" and r["value"] is not None] + nfp_rows = [r for r in rows if r["indicator"] == "nfp" and r["value"] is not None] + assert cpi_rows, "no CPI value from BLS" + assert nfp_rows, "no NFP value from BLS" + + cpi = cpi_rows[-1] + nfp = nfp_rows[-1] + assert _CPI_MIN <= cpi["value"] <= _CPI_MAX, f"CPI out of range: {cpi['value']}" + assert _NFP_MIN <= nfp["value"] <= _NFP_MAX, f"NFP level out of range: {nfp['value']}" + + # BLS-API is latest-revised → NEVER settlement-grade first-print. + assert cpi["settlement_grade"] is False + assert nfp["settlement_grade"] is False + _assert_aware_not_future(cpi["vintage_date"], "BLS CPI vintage_date") + + _print_sample( + "bls.cpi", + {"period": cpi["period"], "value": cpi["value"], "grade": cpi["settlement_grade"]}, + ) + _print_sample( + "bls.nfp", + {"period": nfp["period"], "value": nfp["value"], "grade": nfp["settlement_grade"]}, + ) + + def test_ppi_wpsfd4_returns_real_value(self): + """BLS PPI final-demand ``WPSFD4`` returns a real value (BLOCKER-1 live proof).""" + from mostlyright.econ._fetchers import bls + + assert bls.BLS_SERIES.get("WPSFD4") == "ppi", "WPSFD4 must map to the ppi indicator" + + year = datetime.now(UTC).year + rows = bls.fetch(["WPSFD4"], start_year=year - 1, end_year=year) + ppi_rows = [r for r in rows if r["indicator"] == "ppi" and r["value"] is not None] + assert ppi_rows, ( + "BLS PPI final-demand (WPSFD4) returned no value — the BLOCKER-1 PPI path is " + "NOT live. WPSFD4 is the WP-prefixed Producer Price Index database (not CU/CPI)." + ) + ppi = ppi_rows[-1] + assert _PPI_MIN <= ppi["value"] <= _PPI_MAX, f"PPI out of range: {ppi['value']}" + assert ppi["settlement_grade"] is False + assert ppi["series_id"] == "WPSFD4" + _print_sample( + "bls.ppi.WPSFD4", + {"period": ppi["period"], "value": ppi["value"], "series_id": ppi["series_id"]}, + ) + + +class TestPolymarketKeyless: + """Real Polymarket gamma + CLOB (public, keyless).""" + + def test_fed_event_markets_and_clob_history(self): + """A Fed/CPI gamma event yields markets + a resolutionSource + CLOB history. + + Polymarket rotates event slugs (each FOMC gets a new one). The test tries a + few candidate Fed slugs and, failing those, discovers a live econ event via + gamma ``/events`` search — so a rotated slug skips rather than fails. + """ + from mostlyright.econ._polymarket import GAMMA_API_BASE, derive, fetch_clob_history + + # 1. Discover a current econ event slug (Fed/CPI/rate) via gamma search. + slug = None + with httpx.Client( + timeout=30.0, headers={"User-Agent": "mostlyright-sdk/econ-smoke"} + ) as client: + resp = client.get( + f"{GAMMA_API_BASE}/events", + params={"active": "true", "closed": "false", "limit": 200}, + ) + resp.raise_for_status() + events = resp.json() + if isinstance(events, dict): + events = events.get("events") or events.get("data") or [] + for ev in events if isinstance(events, list) else []: + title = str(ev.get("title", "")).lower() + ev_slug = ev.get("slug") + if ev_slug and any( + k in title for k in ("fed", "interest rate", "cpi", "inflation") + ): + slug = ev_slug + break + + if not slug: + pytest.skip("no live Fed/CPI Polymarket event found via gamma search right now") + + outcomes = derive(slug=slug, attach_price_history=False) + assert outcomes, f"Polymarket event {slug!r} yielded no markets" + first = outcomes[0] + assert first["question"], "market has no question text" + assert first.get("resolution_source") is not None or first.get("description"), ( + "market carries neither a resolutionSource nor a description" + ) + + # Attach a CLOB price-history panel for the first market's first token. + clob_points: list[dict[str, Any]] = [] + token_ids = first.get("clob_token_ids") or [] + if token_ids: + clob_points = fetch_clob_history(str(token_ids[0])) + + _print_sample( + "polymarket.event", + { + "slug": slug, + "question": first["question"], + "outcomes": first.get("outcomes"), + "resolution_source": first.get("resolution_source"), + "clob_points": len(clob_points), + }, + ) + + +# ============================================================================= +# KEYED sub-paths — require operator keys; closed A1/A5/A6 + BLOCKER-3. +# ============================================================================= +@_needs_fred +class TestAlfredKeyed: + """Real keyed ALFRED vintage fetch — the #1 landmine (A1).""" + + def test_first_print_differs_from_latest_revised(self): + """Keyed ALFRED reproduces a first-print vintage that DIFFERS from latest. + + Closes A1: fetch the full ALFRED vintage history for a CPI series, extract + the first release per observation date (the fredapi-lifted algorithm), and + confirm (a) the first-print vintage_date is the release date (<= now, + settlement_grade=True), and (b) at least one observation's first print + DIFFERS from its latest-revised value — proving first-print != latest (the + whole point of the vintage store). The ALFRED day-granularity caveat is + documented in docs/econ-vertical.md. + """ + from mostlyright.econ._fetchers import fred_alfred + + # CPI-U (CPIAUCSL is the FRED CPI series ALFRED carries vintages for). Use + # the PUBLIC fetch_vintages API with each selection (both hit the real + # ALFRED endpoint — exercising the full keyed path, no re-derivation). + first = fred_alfred.fetch_vintages("CPIAUCSL", key=_FRED_KEY, vintages="first") + assert first, "keyed ALFRED first-release returned no rows for CPIAUCSL" + all_rows = fred_alfred.fetch_vintages("CPIAUCSL", key=_FRED_KEY, vintages="all") + assert all_rows, "keyed ALFRED all-vintages returned no rows for CPIAUCSL" + + # Every first-print row is settlement-grade with a real, non-future vintage. + sample_first = first[-1] + assert sample_first["settlement_grade"] is True + _assert_aware_not_future(sample_first["vintage_date"], "ALFRED first-print vintage_date") + + # Prove first-print != latest-revised for at least one observation period. + latest_by_period: dict[str, dict[str, Any]] = {} + for r in all_rows: + p = r["period"] + cur = latest_by_period.get(p) + if cur is None or r["vintage_date"] > cur["vintage_date"]: + latest_by_period[p] = r + first_by_period = {r["period"]: r for r in first} + + differs = [ + p + for p, fr in first_by_period.items() + if p in latest_by_period + and fr["value"] is not None + and latest_by_period[p]["value"] is not None + and fr["value"] != latest_by_period[p]["value"] + ] + assert differs, ( + "no observation period had a first-print value differing from its latest " + "revised value — expected at least one revision (A1 fidelity check)." + ) + p = differs[-1] + _print_sample( + "alfred.first_vs_latest", + { + "period": p, + "first_print": first_by_period[p]["value"], + "latest_revised": latest_by_period[p]["value"], + "first_vintage_date": first_by_period[p]["vintage_date"].date().isoformat(), + "vintage_precision": "day (ALFRED cannot resolve intraday 8:30 vs 9:45 ET)", + }, + ) + + +@_needs_fred +class TestJoblessClaimsKeyed: + """Keyed jobless-claims ICSA first-print (BLOCKER-3 against real ALFRED).""" + + def test_released_window_returns_settlement_grade_rows(self): + """A released jobless-claims window is non-empty with settlement_grade=True. + + BLOCKER-3: ``history("jobless_claims", released_window, vintages="settlement")`` + must return NON-EMPTY rows carrying settlement_grade=True (the keyed + ALFRED-ICSA first-release routing) — never special-cased to always-empty. + A genuinely-future week raises IndicatorNotYetReleasedError. + """ + from datetime import date + + from mostlyright.core.exceptions import IndicatorNotYetReleasedError + from mostlyright.econ import history + + # A recently-released window (a few weeks back, safely after the FEDS floor). + today = date.today() + released_from = today - timedelta(days=45) + released_to = today - timedelta(days=14) + + df = history("jobless_claims", released_from, released_to, vintages="settlement") + assert len(df) > 0, ( + "history('jobless_claims', ..., vintages='settlement') returned EMPTY for a " + "released window — BLOCKER-3 regression (jobless_claims must not be always-empty)." + ) + assert bool(df["settlement_grade"].all()), "settlement read returned a non-settlement row" + latest = df.iloc[-1] + _assert_aware_not_future(latest["vintage_date"].to_pydatetime(), "ICSA vintage_date") + _print_sample( + "dol.icsa.first_print", + { + "period": latest["period"], + "value": latest["value"], + "grade": bool(latest["settlement_grade"]), + "vintage_date": latest["vintage_date"].date().isoformat(), + }, + ) + + # A genuinely-future week has no first print → not-yet-released error. + future_from = today + timedelta(days=30) + future_to = today + timedelta(days=37) + with pytest.raises(IndicatorNotYetReleasedError): + history("jobless_claims", future_from, future_to, vintages="settlement") + + +@_needs_bea +class TestBEAKeyed: + """Real keyed BEA GDP — first-print (advance) identifiability (A5). + + A5 asks whether the GDP FIRST PRINT (the advance estimate) is identifiable. It + is — via TWO real mechanisms, both verified live below; a third test pins the + negative contract: + + 1. ``test_production_gdp_settlement_is_advance_first_print`` — the REAL KXGDP + settlement path (``history("gdp", ..., vintages="settlement")``, keyed) routes + GDP through ALFRED's realtime vintages of ``A191RL1Q225SBEA`` and returns the + advance first print with ``settlement_grade=True``. This — NOT BEA + estimate-type inference — is what KXGDP actually settles on. + 2. ``test_bea_inference_identifies_advance_given_release_date`` — the raw BEA + fetcher's documented inference (:func:`infer_release_type`): given a release + date it labels the matching quarter ``"advance"`` / ``settlement_grade=True``. + 3. ``test_unhinted_bulk_is_latest_revised_not_first_print`` — a plain un-hinted + bulk ``fetch_gdp(year_range=...)`` is BEA's latest-REVISED data, NOT the first + print, so it correctly labels every row ``"revised"`` / ``False`` (codex + round-2 C4). Asserting the bulk call identifies ``"advance"`` (as this smoke + did before C4) was a FALSE GREEN — it only passed because un-hinted rows used + to DEFAULT to ``"advance"``, the very mislabeling C4 fixed. This test now + GUARDS that fix instead of contradicting it. + """ + + @_needs_fred + def test_production_gdp_settlement_is_advance_first_print(self): + """Keyed ``history('gdp', ..., vintages='settlement')`` yields the advance first print. + + The REAL KXGDP settlement mechanism: routed through ALFRED + (``A191RL1Q225SBEA``), recently-released quarters come back + ``release_type="advance"`` / ``settlement_grade=True`` with real release-day + ``vintage_date``s (~quarter-end +1mo). Requires BOTH keys (FRED for the ALFRED + vintage store, BEA for the class gate). + """ + from datetime import date + + from mostlyright.econ import history + + today = date.today() + df = history( + "gdp", today - timedelta(days=430), today - timedelta(days=35), vintages="settlement" + ) + assert len(df) > 0, "keyed GDP settlement window returned no rows (A5 first-print path)" + assert bool(df["settlement_grade"].all()), ( + "a settlement read returned a non-settlement GDP row" + ) + assert set(df["release_type"]) == {"advance"}, ( + f"settlement GDP must be the advance first print, got {set(df['release_type'])}" + ) + latest = df.iloc[-1] + _assert_aware_not_future(latest["vintage_date"].to_pydatetime(), "prod GDP vintage_date") + _print_sample( + "gdp.settlement.advance", + { + "period": latest["period"], + "value": latest["value"], + "release_type": latest["release_type"], + "grade": bool(latest["settlement_grade"]), + }, + ) + + def test_bea_inference_identifies_advance_given_release_date(self): + """Raw BEA inference labels a quarter ``"advance"`` from its advance release date. + + Drives the REAL BEA ``GetData`` payload through :func:`infer_release_type` + (A5): the 2025Q1 advance estimate was released 2025-04-30, so that + ``vintage_date`` must yield ``release_type="advance"`` / + ``settlement_grade=True`` for the 2025Q1 row. + """ + from mostlyright.econ._fetchers import bea + + rows = bea.fetch_gdp(year_range=(2025, 2025), key=_BEA_KEY, vintage_date="2025-04-30") + q1 = [r for r in rows if r["period"] == "2025Q1"] + assert q1, "keyed BEA returned no 2025Q1 GDP row" + adv = q1[-1] + assert adv["release_type"] == "advance", ( + f"2025Q1 @ its advance release date must infer 'advance', got {adv['release_type']!r}" + ) + assert adv["settlement_grade"] is True, "an inferred advance estimate is settlement-grade" + _assert_aware_not_future(adv["vintage_date"], "BEA GDP vintage_date") + _print_sample( + "bea.gdp.inferred_advance", + { + "period": adv["period"], + "value": adv["value"], + "release_type": adv["release_type"], + "grade": adv["settlement_grade"], + }, + ) + + def test_unhinted_bulk_is_latest_revised_not_first_print(self): + """An un-hinted bulk fetch is latest-revised: ``"revised"`` / ``False`` (guards C4).""" + from mostlyright.econ._fetchers import bea + + rows = bea.fetch_gdp(year_range=(2025, 2025), key=_BEA_KEY) + assert rows, "keyed BEA returned no GDP rows" + assert {r["release_type"] for r in rows} == {"revised"}, ( + "un-hinted bulk BEA GDP must be the neutral 'revised' label (latest-revised, " + "NOT the advance first print) — codex round-2 C4." + ) + assert not any(r["settlement_grade"] for r in rows), ( + "un-hinted bulk BEA GDP must NOT be settlement-grade (codex round-2 C4)." + ) + sample = rows[-1] + _print_sample( + "bea.gdp.bulk_revised", + { + "period": sample["period"], + "value": sample["value"], + "release_type": sample["release_type"], + "grade": sample["settlement_grade"], + }, + ) + + +class TestFedDecisionSourcing: + """Real Fed FOMC target-rate decision sourcing (A6) — Federal Reserve, NOT FRED.""" + + def test_fed_decision_is_numeric_and_categorical(self): + """A per-FOMC decision carries a numeric target + a hike/hold/cut label (A6). + + The Fed fetcher sources decisions from the Board's canonical + ``monetarypolicy/openmarket.htm`` rate-change record — keyless, so this + sub-path is NOT gated on ``FRED_API_KEY``. Only a network-level outage + skips (connect error / timeout); an HTTP status error or a parse rejection + FAILS — a 4xx or hostile page means OUR request or parser is wrong, and + masking that as a skip is exactly how the original H.15-JSON transport bug + hid until the keyed smoke ran (caught + fixed 2026-07-09). + """ + import httpx + from mostlyright.econ._fetchers import fed + + try: + rows = fed.fetch_decisions() + except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout) as exc: + pytest.skip(f"federalreserve.gov not reachable this run (network): {exc}") + + assert rows, "Fed fetcher returned no FOMC decisions" + graded = [r for r in rows if r.get("value") is not None] + assert graded, "no Fed decision carried a numeric target rate" + sample = graded[-1] + assert isinstance(sample["value"], (int, float)) + # The categorical hike/hold/cut is encoded in series_id ("fed_funds:"). + series_id = str(sample.get("series_id", "")) + categorical = series_id.split(":", 1)[1] if ":" in series_id else None + assert categorical in {"hike", "hold", "cut"}, ( + f"Fed decision categorical not identifiable from series_id {series_id!r} (A6)." + ) + _assert_aware_not_future(sample["vintage_date"], "Fed decision vintage_date") + _print_sample( + "fed.decision", + { + "period": sample["period"], + "target_rate": sample["value"], + "decision": categorical, + "sourced_from": "federalreserve.gov openmarket.htm (NOT FRED)", + }, + ) diff --git a/tests/test_export_schemas_econ.py b/tests/test_export_schemas_econ.py new file mode 100644 index 00000000..db8ded9f --- /dev/null +++ b/tests/test_export_schemas_econ.py @@ -0,0 +1,213 @@ +"""Tests for the econ codegen wiring in ``scripts/export_schemas.py`` (29-03). + +Two codegen artifacts join the byte-deterministic exporter here: + +1. ``schemas/json/schema.econ.observations.v1.json`` — emitted from the + Python source-of-truth ``EconObservationsSchema.COLUMNS`` (29-02), the input + the TS port (29-09) consumes. +2. ``schemas/kalshi-econ-settlement.json`` — the per-series Kalshi + settlement-routing table, codegen'd from agency NAMES (never the sloppy + ``settlement_sources[].url``; Pitfall 2), that the resolver (29-04) applies. + +Both must be covered by the ``--check`` byte-equality gate and carry a SHA-256 +entry in ``EXPORT_MANIFEST.json`` so a drift fails CI. This module is the local +guard rail that fails fast before the ``schema-drift.yml`` CI workflow does. +See ``REQUIREMENTS.md`` ECON-03 / ECON-11. +""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT: Path = Path(__file__).resolve().parents[1] +_EXPORTER: Path = _REPO_ROOT / "scripts" / "export_schemas.py" + +#: The econ JSON Schema codegen output (TS-port input). +_ECON_SCHEMA_REL = "json/schema.econ.observations.v1.json" +#: The codegen'd Kalshi econ settlement-routing table. +_ECON_SETTLEMENT_REL = "kalshi-econ-settlement.json" + +#: The three vintage-identity columns that MUST survive codegen — they are the +#: load-bearing difference from the weather observation schema. +_VINTAGE_PROPERTIES: tuple[str, ...] = ("vintage_date", "release_type", "settlement_grade") + + +def _run_exporter(out_dir: Path) -> None: + """Invoke the exporter in a subprocess with the requested ``--out-dir``. + + A subprocess (rather than an in-process ``main()`` call) catches sys.path / + import-order surprises — exactly the risk when adding a NEW package (econ) + to the exporter's import loop. + """ + proc = subprocess.run( + [sys.executable, str(_EXPORTER), "--out-dir", str(out_dir)], + capture_output=True, + text=True, + check=False, + cwd=str(_REPO_ROOT), + ) + if proc.returncode != 0: + pytest.fail( + f"export_schemas exited non-zero (rc={proc.returncode})\n" + f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + + +def _sha256_hex(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +# --------------------------------------------------------------------------- +# Task 1 — schema.econ.observations.v1 as a codegen-pipeline citizen. +# --------------------------------------------------------------------------- + + +def test_econ_schema_emitted_with_vintage_properties(tmp_path: Path) -> None: + """The exporter writes the econ JSON Schema with the three vintage columns.""" + out = tmp_path / "out" + _run_exporter(out) + + schema_path = out / _ECON_SCHEMA_REL + assert schema_path.is_file(), f"econ schema not emitted at {_ECON_SCHEMA_REL}" + + payload = json.loads(schema_path.read_text(encoding="utf-8")) + assert payload.get("title") == "schema.econ.observations.v1" + props = payload.get("properties") + assert isinstance(props, dict), "econ schema has no properties dict" + for name in _VINTAGE_PROPERTIES: + assert name in props, f"vintage property {name!r} missing from econ schema" + + +def test_econ_schema_carries_required_json_schema_keys(tmp_path: Path) -> None: + """The econ schema declares the standard ``$schema``/``$id``/``version`` keys.""" + out = tmp_path / "out" + _run_exporter(out) + payload = json.loads((out / _ECON_SCHEMA_REL).read_text(encoding="utf-8")) + assert payload.get("$schema") == "https://json-schema.org/draft/2020-12/schema" + assert payload.get("$id", "").startswith("https://mostlyright.dev/schemas/") + assert payload.get("type") == "object" + assert payload.get("version") == "v1" + assert payload.get("required") == sorted(payload["required"]) + + +def test_check_mode_passes_with_econ_included() -> None: + """``--check`` (two in-memory runs) stays byte-identical with econ included.""" + proc = subprocess.run( + [sys.executable, str(_EXPORTER), "--check"], + capture_output=True, + text=True, + check=False, + cwd=str(_REPO_ROOT), + ) + assert proc.returncode == 0, ( + f"--check failed (rc={proc.returncode})\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + + +def test_econ_schema_in_manifest_with_sha(tmp_path: Path) -> None: + """EXPORT_MANIFEST lists the econ schema with a matching non-empty sha256.""" + out = tmp_path / "out" + _run_exporter(out) + manifest = json.loads((out / "EXPORT_MANIFEST.json").read_text(encoding="utf-8")) + entries = {e["path"]: e for e in manifest["files"]} + assert _ECON_SCHEMA_REL in entries, f"{_ECON_SCHEMA_REL} missing from EXPORT_MANIFEST" + entry = entries[_ECON_SCHEMA_REL] + assert entry["sha256"], "econ schema manifest entry has empty sha256" + assert entry["sha256"] == _sha256_hex(out / _ECON_SCHEMA_REL) + assert entry.get("gated") is False, "econ schema unexpectedly gated" + + +# --------------------------------------------------------------------------- +# Task 2 — the codegen'd Kalshi econ settlement-routing table (agency NAMES). +# --------------------------------------------------------------------------- + + +def test_settlement_table_emitted_with_routing_and_agency_names(tmp_path: Path) -> None: + """The exporter writes the settlement table with ``routing`` + ``agency_names``.""" + out = tmp_path / "out" + _run_exporter(out) + + table_path = out / _ECON_SETTLEMENT_REL + assert table_path.is_file(), f"settlement table not emitted at {_ECON_SETTLEMENT_REL}" + + payload = json.loads(table_path.read_text(encoding="utf-8")) + assert "routing" in payload, "settlement table missing 'routing'" + assert "agency_names" in payload, "settlement table missing 'agency_names'" + assert isinstance(payload["routing"], dict) and payload["routing"], "empty routing" + assert payload["agency_names"] == sorted(payload["agency_names"]), ( + "agency_names must be sorted (determinism)" + ) + + +def test_settlement_table_routes_per_series_te_vs_bls(tmp_path: Path) -> None: + """Per-series routing: KXUSPPI/KXUSNFP → TradingEconomics (grade False); + KXUSPPIYOY → BLS (grade True) — routing is per-series, not per-family.""" + out = tmp_path / "out" + _run_exporter(out) + routing = json.loads((out / _ECON_SETTLEMENT_REL).read_text(encoding="utf-8"))["routing"] + + # KXUSPPI (PPI MoM) + KXUSNFP settle to Trading Economics → agency first-print + # ships labeled settlement_grade=False (we never fabricate a TE value). + for ticker in ("KXUSPPI", "KXUSNFP"): + assert ticker in routing, f"{ticker} missing from routing" + assert routing[ticker]["agency"] == "TradingEconomics" + assert routing[ticker]["settlement_grade"] is False + + # KXUSPPIYOY settles to BLS → settlement_grade True (proves per-series routing: + # KXUSPPI MoM → TE but KXUSPPIYOY → BLS within the same PPI family). + assert routing["KXUSPPIYOY"]["agency"] == "BLS" + assert routing["KXUSPPIYOY"]["settlement_grade"] is True + + # Sanity on the agency-native government series. + assert routing["KXCPI"]["agency"] == "BLS" + assert routing["KXGDP"]["agency"] == "BEA" + assert routing["KXFED"]["agency"] == "FederalReserve" + assert routing["KXJOBLESSCLAIMS"]["agency"] == "DOL" + + +def test_settlement_table_agency_names_are_canonical_vocabulary(tmp_path: Path) -> None: + """Every routed agency is in the canonical vocabulary; the vocabulary lists all 5.""" + out = tmp_path / "out" + _run_exporter(out) + payload = json.loads((out / _ECON_SETTLEMENT_REL).read_text(encoding="utf-8")) + vocab = set(payload["agency_names"]) + assert vocab == {"BEA", "BLS", "DOL", "FederalReserve", "TradingEconomics"} + for ticker, rule in payload["routing"].items(): + assert rule["agency"] in vocab, f"{ticker} routes to non-canonical {rule['agency']!r}" + + +def test_settlement_table_in_manifest_with_sha(tmp_path: Path) -> None: + """EXPORT_MANIFEST lists the settlement table with a matching non-empty sha256.""" + out = tmp_path / "out" + _run_exporter(out) + manifest = json.loads((out / "EXPORT_MANIFEST.json").read_text(encoding="utf-8")) + entries = {e["path"]: e for e in manifest["files"]} + assert _ECON_SETTLEMENT_REL in entries, f"{_ECON_SETTLEMENT_REL} missing from manifest" + entry = entries[_ECON_SETTLEMENT_REL] + assert entry["sha256"] == _sha256_hex(out / _ECON_SETTLEMENT_REL) + assert entry.get("gated") is False, "settlement table unexpectedly gated" + + +def test_settlement_map_routes_on_name_never_url() -> None: + """The Python source-of-truth routes on agency NAME only — no URL routing (Pitfall 2). + + The sloppy ``settlement_sources[].url`` (KXPAYROLLS name=BLS url=ppi.nr0.htm) + is proven unreliable. This asserts the routing module never keys off a + ``settlement_sources`` URL: no ``http``-scheme literal appears in a routing + value. + """ + from mostlyright.econ._settlement_map import SETTLEMENT_ROUTING + + for ticker, rule in SETTLEMENT_ROUTING.items(): + agency = rule.agency if hasattr(rule, "agency") else rule["agency"] + indicator = rule.indicator if hasattr(rule, "indicator") else rule["indicator"] + assert not agency.lower().startswith("http"), f"{ticker} routes on a URL" + assert "://" not in agency and "://" not in indicator, ( + f"{ticker} routing value contains a URL scheme" + ) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index afebc84f..188ddf65 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -76,7 +76,7 @@ def test_pandas_cap_present_everywhere_it_is_mentioned() -> None: this because some OTHER extra still has the cap. """ missing: list[str] = [] - for pkg in ("core", "weather", "markets"): + for pkg in ("core", "weather", "markets", "econ"): for label, deps in _dep_locations(pkg): if "pandas" not in _names_in(deps): continue @@ -91,7 +91,7 @@ def test_pandas_cap_present_everywhere_it_is_mentioned() -> None: def test_pyarrow_cap_present_everywhere_it_is_mentioned() -> None: """Same idea, for pyarrow's <24.0 cap (PKG-06).""" missing: list[str] = [] - for pkg in ("core", "weather", "markets"): + for pkg in ("core", "weather", "markets", "econ"): for label, deps in _dep_locations(pkg): if "pyarrow" not in _names_in(deps): continue diff --git a/tests/test_wheel_layout.py b/tests/test_wheel_layout.py index ac5b65a2..720c218d 100644 --- a/tests/test_wheel_layout.py +++ b/tests/test_wheel_layout.py @@ -1,21 +1,26 @@ """PKG-02 + PKG-04: PEP 420 namespace integrity after ``uv build``. -The three-package split relies on Python's implicit namespace-package +The multi-package split relies on Python's implicit namespace-package (PEP 420) rules: only ``mostlyright`` (core) ships a top-level -``mostlyright/__init__.py``; ``mostlyrightmd-weather`` and -``mostlyrightmd-markets`` ship subdirectories WITHOUT their own namespace-root -``__init__.py``. If a sibling distribution ever shipped a top-level -``__init__.py``, the first one installed would shadow the others and -``import mostlyright.weather`` would break depending on install order. +``mostlyright/__init__.py``; ``mostlyrightmd-weather``, +``mostlyrightmd-markets``, and ``mostlyrightmd-econ`` ship subdirectories +WITHOUT their own namespace-root ``__init__.py``. If a sibling distribution +ever shipped a top-level ``__init__.py``, the first one installed would shadow +the others and ``import mostlyright.weather`` would break depending on install +order. We build with ``uv build --all-packages`` (the command PLAN.md Task 4.1 -verifies and Task 4.2 prepares for publish). This used to emit a 4th +verifies and Task 4.2 prepares for publish). This used to emit an extra ``mostlyrightmd_workspace-0.0.0`` wheel from the workspace root; the root pyproject now sets ``[tool.uv] package = false`` so the workspace is recognized as not-a-publishable-package, and ``--all-packages`` returns -exactly the three publishable wheels. This test guards both halves: +exactly the four publishable wheels. This test guards both halves: the wheel-layout invariants AND the absence of a workspace artifact (codex Wave 4 iter-2 HIGH on tests/test_wheel_layout.py). + +Phase 29 (29-01) added ``mostlyrightmd-econ`` as the 4th published dist; +the wheel-count + name-set + version-lockstep assertions were widened from +three to four accordingly. """ from __future__ import annotations @@ -61,6 +66,8 @@ def built_wheels() -> dict[str, Path]: by_name["weather"] = wheel elif wheel.name.startswith("mostlyrightmd_markets-"): by_name["markets"] = wheel + elif wheel.name.startswith("mostlyrightmd_econ-"): + by_name["econ"] = wheel elif wheel.name.startswith("mostlyrightmd-"): by_name["core"] = wheel else: @@ -75,27 +82,29 @@ def _names(wheel: Path) -> list[str]: return z.namelist() -def test_exactly_three_published_wheels(built_wheels: dict[str, Path]) -> None: - """Exactly three wheels, no workspace artifact, no leftovers. +def test_exactly_four_published_wheels(built_wheels: dict[str, Path]) -> None: + """Exactly four wheels, no workspace artifact, no leftovers. - A previous version of this test only checked the three expected names - were present; ``uv build --all-packages`` produced a 4th + A previous version of this test only checked the expected names were + present; ``uv build --all-packages`` once produced an extra ``mostlyrightmd_workspace-0.0.0`` wheel that slipped through (codex Wave - 4 iter-1 HIGH). Now we assert ``dist/`` has exactly three .whl files - and they are precisely the named packages. + 4 iter-1 HIGH). Now we assert ``dist/`` has exactly four .whl files + and they are precisely the named packages. Phase 29 (29-01) added the + 4th dist ``mostlyrightmd-econ``. """ assert set(built_wheels.keys()) == { "core", "weather", "markets", - }, f"expected exactly core+weather+markets wheels, got {sorted(built_wheels)}" + "econ", + }, f"expected exactly core+weather+markets+econ wheels, got {sorted(built_wheels)}" # Belt-and-suspenders: the fixture set the right keys, but re-glob # dist/ in case anything else (e.g. a stray workspace wheel from a # parallel `uv build --all-packages` call) sneaks in. all_wheels = list(DIST.glob("*.whl")) - assert len(all_wheels) == 3, ( - f"dist/ must contain exactly 3 wheels after a clean build; got " + assert len(all_wheels) == 4, ( + f"dist/ must contain exactly 4 wheels after a clean build; got " f"{[w.name for w in all_wheels]}" ) assert not list(DIST.glob("mostlyrightmd_workspace-*.whl")), ( @@ -105,13 +114,15 @@ def test_exactly_three_published_wheels(built_wheels: dict[str, Path]) -> None: def test_wheel_versions_lockstep(built_wheels: dict[str, Path]) -> None: - # PKG-01: all three packages bump in lockstep on the same SHA. This + # PKG-01: all four packages bump in lockstep on the same SHA. This # test extracts the version string from each wheel filename and asserts # they all match — agnostic to which specific version we're at. # `pkg_name-VERSION-py3-none-any.whl` → split on `-` and take index 1. - versions = {pkg: built_wheels[pkg].name.split("-")[1] for pkg in ("core", "weather", "markets")} - assert versions["core"] == versions["weather"] == versions["markets"], ( - f"wheel versions must match lockstep across all 3 packages; got {versions}" + versions = { + pkg: built_wheels[pkg].name.split("-")[1] for pkg in ("core", "weather", "markets", "econ") + } + assert versions["core"] == versions["weather"] == versions["markets"] == versions["econ"], ( + f"wheel versions must match lockstep across all 4 packages; got {versions}" ) @@ -119,6 +130,7 @@ def test_only_core_ships_namespace_root(built_wheels: dict[str, Path]) -> None: core_names = _names(built_wheels["core"]) weather_names = _names(built_wheels["weather"]) markets_names = _names(built_wheels["markets"]) + econ_names = _names(built_wheels["econ"]) assert "mostlyright/__init__.py" in core_names, ( "core wheel MUST ship mostlyright/__init__.py (it owns the namespace root)" @@ -131,14 +143,22 @@ def test_only_core_ships_namespace_root(built_wheels: dict[str, Path]) -> None: "markets wheel must NOT ship mostlyright/__init__.py — install-order " "shadowing would break `import mostlyright.markets` (PEP 420; PKG-02)" ) + assert "mostlyright/__init__.py" not in econ_names, ( + "econ wheel must NOT ship mostlyright/__init__.py — install-order " + "shadowing would break `import mostlyright.econ` (PEP 420; PKG-02)" + ) def test_sibling_subpackages_present(built_wheels: dict[str, Path]) -> None: weather_names = _names(built_wheels["weather"]) markets_names = _names(built_wheels["markets"]) + econ_names = _names(built_wheels["econ"]) assert "mostlyright/weather/__init__.py" in weather_names, ( "weather wheel must ship mostlyright/weather/__init__.py (the subpackage marker)" ) assert "mostlyright/markets/__init__.py" in markets_names, ( "markets wheel must ship mostlyright/markets/__init__.py" ) + assert "mostlyright/econ/__init__.py" in econ_names, ( + "econ wheel must ship mostlyright/econ/__init__.py (the subpackage marker)" + ) diff --git a/uv.lock b/uv.lock index bc2173b5..d7f3124b 100644 --- a/uv.lock +++ b/uv.lock @@ -19,6 +19,7 @@ resolution-markers = [ [manifest] members = [ "mostlyrightmd", + "mostlyrightmd-econ", "mostlyrightmd-markets", "mostlyrightmd-weather", "mostlyrightmd-workspace", @@ -1670,6 +1671,36 @@ requires-dist = [ ] provides-extras = ["parquet", "research", "polars"] +[[package]] +name = "mostlyrightmd-econ" +version = "1.15.0" +source = { editable = "packages/econ" } +dependencies = [ + { name = "filelock" }, + { name = "httpx" }, + { name = "jsonschema" }, + { name = "mostlyrightmd" }, + { name = "pyarrow" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] + +[package.optional-dependencies] +pandas = [ + { name = "pandas" }, +] + +[package.metadata] +requires-dist = [ + { name = "filelock", specifier = ">=3.12" }, + { name = "httpx", specifier = ">=0.27" }, + { name = "jsonschema", specifier = ">=4.21" }, + { name = "mostlyrightmd", editable = "packages/core" }, + { name = "pandas", marker = "extra == 'pandas'", specifier = ">=2.2,<4.0" }, + { name = "pyarrow", specifier = ">=17.0,<24.0" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +provides-extras = ["pandas"] + [[package]] name = "mostlyrightmd-markets" version = "1.15.0" @@ -1678,6 +1709,7 @@ dependencies = [ { name = "httpx" }, { name = "jsonschema" }, { name = "mostlyrightmd" }, + { name = "mostlyrightmd-econ" }, ] [package.optional-dependencies] @@ -1710,6 +1742,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27" }, { name = "jsonschema", specifier = ">=4.21" }, { name = "mostlyrightmd", editable = "packages/core" }, + { name = "mostlyrightmd-econ", editable = "packages/econ" }, { name = "mostlyrightmd-weather", marker = "extra == 'polymarket'", editable = "packages/weather" }, { name = "narwhals", marker = "extra == 'polars'", specifier = ">=1.20,<2.0" }, { name = "pandas", marker = "extra == 'earnings'", specifier = ">=2.2,<4.0" }, @@ -1814,6 +1847,7 @@ version = "0.0.0" source = { virtual = "." } dependencies = [ { name = "mostlyrightmd" }, + { name = "mostlyrightmd-econ" }, { name = "mostlyrightmd-markets" }, { name = "mostlyrightmd-weather" }, ] @@ -1846,6 +1880,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "mostlyrightmd", editable = "packages/core" }, + { name = "mostlyrightmd-econ", editable = "packages/econ" }, { name = "mostlyrightmd-markets", editable = "packages/markets" }, { name = "mostlyrightmd-weather", editable = "packages/weather" }, ]