diff --git a/.github/workflows/release-testpypi.yml b/.github/workflows/release-testpypi.yml index 4f3947bc..addd42a6 100644 --- a/.github/workflows/release-testpypi.yml +++ b/.github/workflows/release-testpypi.yml @@ -10,7 +10,7 @@ name: Release (TestPyPI dry-run on rc tags) # # Setup checklist (one-time, before the first v*rc* tag): # 1. On test.pypi.org, register each of mostlyrightmd / mostlyrightmd-weather / -# mostlyrightmd-markets as a "pending publisher": +# mostlyrightmd-markets / mostlyrightmd-econ as a "pending publisher": # owner = mostlyrightmd (GitHub org that owns the repo, post Phase 12.1) # repo = mostlyright-sdk # workflow filename = release-testpypi.yml @@ -42,63 +42,166 @@ permissions: contents: read jobs: + version-guard: + name: Assert RC tag matches all package versions + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install packaging + run: python -m pip install "packaging>=23" + - name: Compare normalized tag and package versions + env: + REF_NAME: ${{ github.ref_name }} + run: | + python - <<'PY' + import os, sys, tomllib + from packaging.version import InvalidVersion, Version + + ref = os.environ["REF_NAME"] + try: + tag_version = Version(ref.removeprefix("v")) + except InvalidVersion as exc: + sys.exit(f"ERROR: tag {ref!r} is not valid PEP 440: {exc}") + paths = ( + "packages/core/pyproject.toml", + "packages/weather/pyproject.toml", + "packages/markets/pyproject.toml", + "packages/econ/pyproject.toml", + ) + for path in paths: + with open(path, "rb") as handle: + package_version = Version(tomllib.load(handle)["project"]["version"]) + if package_version != tag_version: + sys.exit(f"ERROR: {path} has {package_version}, tag is {tag_version}") + print(f"OK: {path} matches {tag_version}") + PY + + validate-artifacts: + name: Build and validate all RC artifacts + needs: version-guard + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + - run: uv python install 3.12 + - run: uv build --all + - run: uv run python scripts/check_wheel_metadata.py + - uses: actions/upload-artifact@v4 + with: + name: testpypi-core + path: | + dist/mostlyrightmd-*.whl + dist/mostlyrightmd-*.tar.gz + - uses: actions/upload-artifact@v4 + with: + name: testpypi-weather + path: | + dist/mostlyrightmd_weather-*.whl + dist/mostlyrightmd_weather-*.tar.gz + - uses: actions/upload-artifact@v4 + with: + name: testpypi-markets + path: | + dist/mostlyrightmd_markets-*.whl + dist/mostlyrightmd_markets-*.tar.gz + - uses: actions/upload-artifact@v4 + with: + name: testpypi-econ + path: | + dist/mostlyrightmd_econ-*.whl + dist/mostlyrightmd_econ-*.tar.gz + publish-core: name: Publish mostlyrightmd (core) → TestPyPI + needs: validate-artifacts runs-on: ubuntu-latest environment: name: testpypi url: https://test.pypi.org/project/mostlyrightmd/ steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - - name: Set up Python - run: uv python install 3.12 - - name: Build mostlyrightmd wheel + sdist - run: uv build --package mostlyrightmd + - uses: actions/download-artifact@v4 + with: + name: testpypi-core + path: dist + - name: Verify any existing immutable release + env: + PYPI_JSON_BASE_URL: https://test.pypi.org/pypi + run: python scripts/verify_pypi_existing.py mostlyrightmd "${GITHUB_REF_NAME#v}" dist/* - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + skip-existing: true publish-weather: name: Publish mostlyrightmd-weather → TestPyPI + needs: publish-core runs-on: ubuntu-latest environment: name: testpypi url: https://test.pypi.org/project/mostlyrightmd-weather/ steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - - name: Set up Python - run: uv python install 3.12 - - name: Build mostlyrightmd-weather wheel + sdist - run: uv build --package mostlyrightmd-weather - - name: Verify Requires-Dist pin (CI-04 gate) - run: uv run python scripts/check_wheel_metadata.py + - uses: actions/download-artifact@v4 + with: + name: testpypi-weather + path: dist + - name: Verify any existing immutable release + env: + PYPI_JSON_BASE_URL: https://test.pypi.org/pypi + run: python scripts/verify_pypi_existing.py mostlyrightmd-weather "${GITHUB_REF_NAME#v}" dist/* - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + skip-existing: true publish-markets: name: Publish mostlyrightmd-markets → TestPyPI + needs: [publish-weather, publish-econ] runs-on: ubuntu-latest environment: name: testpypi url: https://test.pypi.org/project/mostlyrightmd-markets/ steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - - name: Set up Python - run: uv python install 3.12 - - name: Build mostlyrightmd-markets wheel + sdist - run: uv build --package mostlyrightmd-markets - - name: Verify Requires-Dist pin (CI-04 gate) - run: uv run python scripts/check_wheel_metadata.py + - uses: actions/download-artifact@v4 + with: + name: testpypi-markets + path: dist + - name: Verify any existing immutable release + env: + PYPI_JSON_BASE_URL: https://test.pypi.org/pypi + run: python scripts/verify_pypi_existing.py mostlyrightmd-markets "${GITHUB_REF_NAME#v}" dist/* + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + publish-econ: + name: Publish mostlyrightmd-econ → TestPyPI + needs: publish-core + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/project/mostlyrightmd-econ/ + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: testpypi-econ + path: dist + - name: Verify any existing immutable release + env: + PYPI_JSON_BASE_URL: https://test.pypi.org/pypi + run: python scripts/verify_pypi_existing.py mostlyrightmd-econ "${GITHUB_REF_NAME#v}" dist/* - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + skip-existing: true diff --git a/.github/workflows/release-ts.yml b/.github/workflows/release-ts.yml index 5b777699..01296f75 100644 --- a/.github/workflows/release-ts.yml +++ b/.github/workflows/release-ts.yml @@ -54,9 +54,11 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 24 cache: pnpm - registry-url: "https://registry.npmjs.org" + + - name: Install trusted-publishing-capable npm + run: npm install --global npm@latest - name: Install run: pnpm install --frozen-lockfile @@ -110,12 +112,47 @@ jobs: # (normalize vts-0.1.0rc1 → 0.1.0-rc.1). A stale-commit tag would # otherwise publish whatever's in the files, leaving a partial # release that can't be redone (npm versions are immutable). - # 2. Rewrite weather/markets peerDependencies['@mostlyrightmd/core'] - # from the workspace-time ^0.0.0 placeholder to ^. - # pnpm publish rewrites `workspace:*` but leaves plain - # peerDependencies untouched. + # 2. Rewrite domain peerDependencies['@mostlyrightmd/core'] to the + # matching-minor ~ release boundary. + # pnpm pack rewrites `workspace:*` dependencies but leaves plain + # peerDependencies untouched; npm then publishes the validated tarballs. run: node scripts/release-ts-preflight.mjs "${{ github.ref_name }}" + - name: Pack and validate every npm artifact before publishing + run: | + mkdir -p npm-artifacts + for pkg in core weather markets econ meta; do + pnpm --dir "packages-ts/$pkg" pack --pack-destination "$GITHUB_WORKSPACE/npm-artifacts" + done + python3 - <<'PY' + import json, pathlib, tarfile + + artifacts = sorted(pathlib.Path("npm-artifacts").glob("*.tgz")) + if len(artifacts) != 5: + raise SystemExit(f"expected 5 npm tarballs, found {len(artifacts)}") + + def targets(value): + if isinstance(value, str) and value.startswith("./"): + yield value + elif isinstance(value, dict): + for child in value.values(): + yield from targets(child) + + for artifact in artifacts: + with tarfile.open(artifact, "r:gz") as archive: + names = set(archive.getnames()) + package = json.load(archive.extractfile("package/package.json")) + raw = json.dumps(package) + if "workspace:" in raw: + raise SystemExit(f"{artifact}: unresolved workspace dependency") + for target in targets(package.get("exports", {})): + if f"package/{target[2:]}" not in names: + raise SystemExit(f"{artifact}: missing export target {target}") + if not any(name.startswith("package/dist/") for name in names): + raise SystemExit(f"{artifact}: missing dist/ payload") + print(f"validated {package['name']}@{package['version']}: {artifact}") + PY + - name: Choose dist-tag (rc → next, non-rc → latest) id: dist_tag run: | @@ -127,54 +164,41 @@ jobs: - name: Publish — @mostlyrightmd/core working-directory: packages-ts/core - # Use `pnpm publish` (NOT `npm publish`) because the meta package - # depends on the scoped packages via `workspace:*`. `pnpm publish` - # rewrites those to the actual versions at pack time; `npm publish` - # would emit the literal `workspace:*` token and EUNSUPPORTEDPROTOCOL - # any consumer. Also covers the scoped packages with a uniform path. - run: pnpm publish --access public --tag ${{ steps.dist_tag.outputs.tag }} --no-git-checks - env: - NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + # pnpm pack has already rewritten workspace dependencies and the + # validated tarball is published with npm trusted publishing. + run: | + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + node "$GITHUB_WORKSPACE/scripts/publish-npm-if-needed.mjs" "$NAME" "$VERSION" "$GITHUB_WORKSPACE/npm-artifacts/mostlyrightmd-core-$VERSION.tgz" "${{ steps.dist_tag.outputs.tag }}" - name: Publish — @mostlyrightmd/weather working-directory: packages-ts/weather - # Use `pnpm publish` (NOT `npm publish`) because the meta package - # depends on the scoped packages via `workspace:*`. `pnpm publish` - # rewrites those to the actual versions at pack time; `npm publish` - # would emit the literal `workspace:*` token and EUNSUPPORTEDPROTOCOL - # any consumer. Also covers the scoped packages with a uniform path. - run: pnpm publish --access public --tag ${{ steps.dist_tag.outputs.tag }} --no-git-checks - env: - NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + # Publish the prevalidated pnpm-packed tarball with npm OIDC. + run: | + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + node "$GITHUB_WORKSPACE/scripts/publish-npm-if-needed.mjs" "$NAME" "$VERSION" "$GITHUB_WORKSPACE/npm-artifacts/mostlyrightmd-weather-$VERSION.tgz" "${{ steps.dist_tag.outputs.tag }}" - name: Publish — @mostlyrightmd/markets working-directory: packages-ts/markets - # Use `pnpm publish` (NOT `npm publish`) because the meta package - # depends on the scoped packages via `workspace:*`. `pnpm publish` - # rewrites those to the actual versions at pack time; `npm publish` - # would emit the literal `workspace:*` token and EUNSUPPORTEDPROTOCOL - # any consumer. Also covers the scoped packages with a uniform path. - run: pnpm publish --access public --tag ${{ steps.dist_tag.outputs.tag }} --no-git-checks - env: - NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + # Publish the prevalidated pnpm-packed tarball with npm OIDC. + run: | + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + node "$GITHUB_WORKSPACE/scripts/publish-npm-if-needed.mjs" "$NAME" "$VERSION" "$GITHUB_WORKSPACE/npm-artifacts/mostlyrightmd-markets-$VERSION.tgz" "${{ steps.dist_tag.outputs.tag }}" - name: Publish — @mostlyrightmd/econ working-directory: packages-ts/econ - # pnpm publish (NOT npm publish): econ's @mostlyrightmd/core - # peerDependency is rewritten from the ^0.0.0 placeholder to ^ - # by the preflight above; pnpm also handles any workspace protocol at - # pack time. Mirrors the weather/markets scoped-package publish path. - run: pnpm publish --access public --tag ${{ steps.dist_tag.outputs.tag }} --no-git-checks - env: - NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + # The core peer range was rewritten before pnpm pack; publish the + # prevalidated tarball with npm OIDC. + run: | + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + node "$GITHUB_WORKSPACE/scripts/publish-npm-if-needed.mjs" "$NAME" "$VERSION" "$GITHUB_WORKSPACE/npm-artifacts/mostlyrightmd-econ-$VERSION.tgz" "${{ steps.dist_tag.outputs.tag }}" - name: Publish — mostlyright (meta) working-directory: packages-ts/meta - # Use `pnpm publish` (NOT `npm publish`) because the meta package - # depends on the scoped packages via `workspace:*`. `pnpm publish` - # rewrites those to the actual versions at pack time; `npm publish` - # would emit the literal `workspace:*` token and EUNSUPPORTEDPROTOCOL - # any consumer. Also covers the scoped packages with a uniform path. - run: pnpm publish --access public --tag ${{ steps.dist_tag.outputs.tag }} --no-git-checks - env: - NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + run: | + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + node "$GITHUB_WORKSPACE/scripts/publish-npm-if-needed.mjs" "$NAME" "$VERSION" "$GITHUB_WORKSPACE/npm-artifacts/mostlyright-$VERSION.tgz" "${{ steps.dist_tag.outputs.tag }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d0820955..434d6746 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -104,86 +104,135 @@ jobs: print(f"OK: {pkg} version {pkg_version!s} matches tag") PY - publish-core: - name: Publish mostlyrightmd (core) + validate-wheels: + name: Build and validate all Python wheels needs: version-guard runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/project/mostlyrightmd/ steps: - uses: actions/checkout@v4 - name: Install uv uses: astral-sh/setup-uv@v3 - name: Set up Python run: uv python install 3.12 - - name: Build mostlyrightmd wheel + sdist - run: uv build --package mostlyrightmd + - name: Build all release artifacts + run: uv build --all + - name: Validate every sibling dependency edge + run: uv run python scripts/check_wheel_metadata.py + - name: Preserve core artifacts + uses: actions/upload-artifact@v4 + with: + name: pypi-core + path: | + dist/mostlyrightmd-*.whl + dist/mostlyrightmd-*.tar.gz + - name: Preserve weather artifacts + uses: actions/upload-artifact@v4 + with: + name: pypi-weather + path: | + dist/mostlyrightmd_weather-*.whl + dist/mostlyrightmd_weather-*.tar.gz + - name: Preserve markets artifacts + uses: actions/upload-artifact@v4 + with: + name: pypi-markets + path: | + dist/mostlyrightmd_markets-*.whl + dist/mostlyrightmd_markets-*.tar.gz + - name: Preserve econ artifacts + uses: actions/upload-artifact@v4 + with: + name: pypi-econ + path: | + dist/mostlyrightmd_econ-*.whl + dist/mostlyrightmd_econ-*.tar.gz + + publish-core: + name: Publish mostlyrightmd (core) + needs: validate-wheels + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/mostlyrightmd/ + steps: + - uses: actions/checkout@v4 + - name: Download validated artifacts + uses: actions/download-artifact@v4 + with: + name: pypi-core + path: dist + - name: Verify any existing immutable release + run: python scripts/verify_pypi_existing.py mostlyrightmd "${GITHUB_REF_NAME#v}" dist/* - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true publish-weather: name: Publish mostlyrightmd-weather - needs: version-guard + needs: publish-core runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/project/mostlyrightmd-weather/ steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - - name: Set up Python - run: uv python install 3.12 - - name: Build mostlyrightmd-weather wheel + sdist - run: uv build --package mostlyrightmd-weather - - name: Verify Requires-Dist pin (CI-04 gate) - run: uv run python scripts/check_wheel_metadata.py + - name: Download validated artifacts + uses: actions/download-artifact@v4 + with: + name: pypi-weather + path: dist + - name: Verify any existing immutable release + run: python scripts/verify_pypi_existing.py mostlyrightmd-weather "${GITHUB_REF_NAME#v}" dist/* - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true publish-markets: name: Publish mostlyrightmd-markets - needs: version-guard + needs: [publish-weather, publish-econ] runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/project/mostlyrightmd-markets/ steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - - name: Set up Python - run: uv python install 3.12 - - name: Build mostlyrightmd-markets wheel + sdist - run: uv build --package mostlyrightmd-markets - - name: Verify Requires-Dist pin (CI-04 gate) - run: uv run python scripts/check_wheel_metadata.py + - name: Download validated artifacts + uses: actions/download-artifact@v4 + with: + name: pypi-markets + path: dist + - name: Verify any existing immutable release + run: python scripts/verify_pypi_existing.py mostlyrightmd-markets "${GITHUB_REF_NAME#v}" dist/* - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true - # Phase 29 (29-01): the 4th published distribution. Mirrors the markets job - # verbatim (own trusted-publishing environment url; own `uv build --package`). + # Phase 29 (29-01): the 4th published distribution, using its own trusted- + # publishing environment URL and the centrally validated artifact. # NOTE: `mostlyrightmd-econ` must be registered as a PyPI "pending publisher" # (owner=mostlyrightmd, repo=mostlyright-sdk, workflow=release.yml, # environment=pypi) BEFORE the first tagged release, exactly like the other # three (see the one-time setup checklist at the top of this file). publish-econ: name: Publish mostlyrightmd-econ - needs: version-guard + needs: publish-core runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/project/mostlyrightmd-econ/ steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - - name: Set up Python - run: uv python install 3.12 - - name: Build mostlyrightmd-econ wheel + sdist - run: uv build --package mostlyrightmd-econ - - name: Verify Requires-Dist pin (CI-04 gate) - run: uv run python scripts/check_wheel_metadata.py + - name: Download validated artifacts + uses: actions/download-artifact@v4 + with: + name: pypi-econ + path: dist + - name: Verify any existing immutable release + run: python scripts/verify_pypi_existing.py mostlyrightmd-econ "${GITHUB_REF_NAME#v}" dist/* - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true diff --git a/README.md b/README.md index 4065d045..c50b355d 100644 --- a/README.md +++ b/README.md @@ -10,47 +10,77 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Docs](https://img.shields.io/badge/docs-mostlyright.md-blue.svg)](https://mostlyright.md/docs/sdk/) -`mostlyright` gives you one clean interface to public data sources — **general-first**. Today it ships weather (live observations, forecasts, climate history) as source-identified domain tables, plus `dataset()`, a leakage-safe training-table builder that aligns them to any label you choose. Prediction markets (Kalshi, Polymarket) are a thin convenience layer on top. Local-first, no hosted backend, no API key. +`mostlyright` gives you one clean interface to public data sources — **general-first**. Today it ships weather (live observations, forecasts, climate history) and economic indicators as source-identified domain tables. Weather adds a capped `weather.pairs()` quickstart and `mr.align(spine, *sources)` composition ladder; the equivalent econ ladder is deferred. Prediction markets (Kalshi, Polymarket) are a thin convenience layer on top. Local-first by default; an explicit hosted satellite transport is optional. Most weather paths are keyless, while the reserved EUMETSAT integration and some econ vintage paths require upstream credentials. --- ## Install ```bash -# Python -pip install 'mostlyrightmd[research]' +# Python — core + weather composition, plus the shipped markets/econ domains +pip install 'mostlyrightmd[research]' mostlyrightmd-markets mostlyrightmd-econ -# TypeScript / Node -pnpm add mostlyright +# TypeScript / Node — meta package plus the separately shipped econ domain +pnpm add mostlyright @mostlyrightmd/econ ``` -Python 3.11+. Node 18+. No API key required for any package below. +Python 3.11+. Node 20+. Most weather and keyless/latest-revised econ paths need no +API key; the reserved EUMETSAT integration requires `EUMETSAT_CONSUMER_KEY` and +`EUMETSAT_CONSUMER_SECRET`. Settlement-grade ALFRED vintages require `FRED_API_KEY`. A +`BEA_API_KEY` unlocks the latest-revised GDP fallback only; it is not +settlement-grade. See the econ adapter for per-series requirements. ## Quickstart ```python # Python -import mostlyright +from mostlyright import weather -# dataset() is the alignment engine; research() is a fully working alias. -# Pass an explicit label= (here the NWS CLI settlement extremes). -df = mostlyright.dataset("KNYC", "2025-01-06", "2025-01-12", label="cli") +# pairs() is the capped quickstart (v0.14.1 heritage: client.pairs()). +# label="cli" (the NWS CLI settlement extremes) is the documented default. +df = weather.pairs("KNYC", "2025-01-06", "2025-01-12") print(df.head()) # pandas DataFrame: one row per LST settlement date -# Columns: date, station, cli_high_f, cli_low_f, obs_high_f, obs_low_f, -# obs_mean_f, obs_count, fcst_*, market_close_utc +# Columns include the settlement spine (`date`, `station`, `decision_time`, +# `label_available_time`) plus cli_* and obs_* values. +``` + +When you outgrow the one-liner, you don't add kwargs — you graduate one rung +down to the same call it desugars to: + +```python +import mostlyright as mr +from mostlyright import weather + +spine = weather.label.cli("KNYC", "2025-01-06", "2025-01-12") # the target (y) +frame = mr.align(spine, weather.obs("KNYC")) +# align() applies each source contract's equality/as-of policy and audits +# exact point-in-time sources against the spine decision time. ``` ```ts -// TypeScript +// TypeScript — the TS SDK keeps research()/dataset() unchanged on 1.17 +// (align/spine/pairs are Python-only for now; parity ticket PT-3401). import { research } from "mostlyright"; const rows = await research("KNYC", "2025-01-06", "2025-01-12"); console.log(rows[0]); -// ReadonlyArray: same schema as Python, JSON-serializable +// ReadonlyArray: the legacy TS research() schema, JSON-serializable. +// PT-3401 tracks parity with Python's new spine/align output. ``` -First call writes a parquet (Python) or JSON-envelope (Node) cache to `~/.mostlyright/cache/`. Subsequent calls in the same window are local-only — no network. +> **Python users coming from root `mostlyright.dataset()` / +> `mostlyright.research()`?** Those relocated root exports remain as +> deprecation shims until 2.0; their canonical `mostlyright.weather.*` homes +> continue to work. TypeScript keeps +> `dataset()` as its primary API and deprecates only `research()`. The +> [1.17 migration guide](docs/migration/v117-align-spine-sources.md) maps all +> 26 old kwargs to their new homes. + +Eligible historical calls write a parquet cache under `~/.mostlyright/cache/` +(Python) or a JSON-envelope under `~/.mostlyright/cache-ts/` (Node). Repeated +stable historical windows can then reuse local data; volatile current-LST +months/years deliberately bypass the relevant cache and may fetch again. Full quickstart with concepts at . @@ -60,9 +90,10 @@ Full quickstart with concepts at . | Package | Description | Downloads | Status | |---|---|---|---| -| [`mostlyrightmd`](https://pypi.org/project/mostlyrightmd/) | Core types, schemas, validators, the `research()` join, and snapshot primitives. Imports as `mostlyright`. | [![monthly downloads](https://badgen.net/pypi/dm/mostlyrightmd?label=monthly)](https://pypistats.org/packages/mostlyrightmd) | stable | +| [`mostlyrightmd`](https://pypi.org/project/mostlyrightmd/) | Core types, schemas, validators, the `align()`/`spine()` composition layer, and snapshot primitives. Imports as `mostlyright`. | [![monthly downloads](https://badgen.net/pypi/dm/mostlyrightmd?label=monthly)](https://pypistats.org/packages/mostlyrightmd) | stable | | [`mostlyrightmd-weather`](https://pypi.org/project/mostlyrightmd-weather/) | Weather data fetchers — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). Direct public-API access. | [![monthly downloads](https://badgen.net/pypi/dm/mostlyrightmd-weather?label=monthly)](https://pypistats.org/packages/mostlyrightmd-weather) | stable | | [`mostlyrightmd-markets`](https://pypi.org/project/mostlyrightmd-markets/) | Prediction-market data — Kalshi NHIGH/NLOW weather-contract resolvers, Polymarket discovery + settlement, and Kalshi + Polymarket trade history. | [![monthly downloads](https://badgen.net/pypi/dm/mostlyrightmd-markets?label=monthly)](https://pypistats.org/packages/mostlyrightmd-markets) | stable | +| [`mostlyrightmd-econ`](https://pypi.org/project/mostlyrightmd-econ/) | Economic indicators, releases, and point-in-time vintages. | [![monthly downloads](https://badgen.net/pypi/dm/mostlyrightmd-econ?label=monthly)](https://pypistats.org/packages/mostlyrightmd-econ) | stable | | `mostlyrightmd-edgar` | SEC filings (10-K, 10-Q, 8-K) — direct EDGAR full-text + facts access. | n/a | planned | | `mostlyrightmd-equities` | Equities structured data — fundamentals, corporate actions, normalized XBRL pulls. | n/a | planned | | `mostlyrightmd-fred` | Federal Reserve economic data (FRED series, observations, releases). | n/a | planned | @@ -74,9 +105,10 @@ Full quickstart with concepts at . | Package | Description | Downloads | Status | |---|---|---|---| | [`mostlyright`](https://www.npmjs.com/package/mostlyright) | Meta package — one `import { research } from "mostlyright"` for weather data + prediction-market settlements + the core join. | [![monthly downloads](https://img.shields.io/npm/dm/mostlyright?label=monthly)](https://www.npmjs.com/package/mostlyright) | stable | -| [`@mostlyrightmd/core`](https://www.npmjs.com/package/@mostlyrightmd/core) | Core types, schemas, validators, temporal-safety primitives, and the `research()` join. | [![monthly downloads](https://img.shields.io/npm/dm/%40mostlyrightmd%2Fcore?label=monthly)](https://www.npmjs.com/package/@mostlyrightmd/core) | stable | +| [`@mostlyrightmd/core`](https://www.npmjs.com/package/@mostlyrightmd/core) | Core types, schemas, validators, and temporal-safety primitives (`research()` itself is exported by the `mostlyright` meta package). | [![monthly downloads](https://img.shields.io/npm/dm/%40mostlyrightmd%2Fcore?label=monthly)](https://www.npmjs.com/package/@mostlyrightmd/core) | stable | | [`@mostlyrightmd/weather`](https://www.npmjs.com/package/@mostlyrightmd/weather) | Weather data fetchers — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). | [![monthly downloads](https://img.shields.io/npm/dm/%40mostlyrightmd%2Fweather?label=monthly)](https://www.npmjs.com/package/@mostlyrightmd/weather) | stable | | [`@mostlyrightmd/markets`](https://www.npmjs.com/package/@mostlyrightmd/markets) | Prediction-market data — Kalshi NHIGH/NLOW weather-contract resolvers, Polymarket discovery + settlement, and Kalshi + Polymarket trade history. | [![monthly downloads](https://img.shields.io/npm/dm/%40mostlyrightmd%2Fmarkets?label=monthly)](https://www.npmjs.com/package/@mostlyrightmd/markets) | stable | +| [`@mostlyrightmd/econ`](https://www.npmjs.com/package/@mostlyrightmd/econ) | Economic indicator types, settlement routing, and parity helpers. | [![monthly downloads](https://img.shields.io/npm/dm/%40mostlyrightmd%2Fecon?label=monthly)](https://www.npmjs.com/package/@mostlyrightmd/econ) | stable | | `@mostlyrightmd/edgar` | SEC filings (10-K, 10-Q, 8-K) — direct EDGAR full-text + facts access. | n/a | planned | | `@mostlyrightmd/equities` | Equities structured data — fundamentals, corporate actions, normalized XBRL pulls. | n/a | planned | | `@mostlyrightmd/fred` | Federal Reserve economic data (FRED series, observations, releases). | n/a | planned | @@ -87,9 +119,9 @@ Full quickstart with concepts at . ### Pull source-identified domain tables -The domain tables are the raw ingredients — each row carries a durable **source -identity** so a model trained on one provider never silently reconciles against -another. Fetch them directly and join by hand, or feed them to `dataset()`. +The domain tables are the raw ingredients. Fetch explicit-window DataFrames for +inspection or hand joins; `mr.align()` accepts registered lazy source specs (or +an explicit `(DataFrame, SourceContract)` tuple), not an uncontracted frame. ```python from mostlyright.weather import obs, climate @@ -98,54 +130,72 @@ from mostlyright.weather import obs, climate o = obs("KNYC", "2024-01-01", "2024-12-31", granularity="observation") c = climate("KNYC", "2024-01-01", "2024-12-31") # NWS CLI settlement labels -# Forecasts compose into dataset() via features=["forecasts"]; the raw NWP -# table is mostlyright.forecasts.forecast_nwp (needs the [nwp] extra). +# weather.forecasts(...) is the registered forecast source for align(); the +# raw NWP table is mostlyright.forecasts.forecast_nwp (needs the [nwp] extra). ``` -### Build a leakage-safe training table with `dataset()` +### Compose a leakage-safe training table — the `align` ladder -`dataset()` is the **alignment engine** — a training-table builder, not a fixed -recipe. Give it a station, a window, a **label axis**, and the features you want; -it returns one leakage-safe row per LST settlement day, every column aligned to the -settlement calendar and joined as-of so nothing from the future leaks backward. -`research()` is a fully working alias in this release. +Sources (features / X) and label spines (targets / y) are **orthogonal**; +`mr.align(spine, *sources)` is the one composition operator that joins them. +Materialized raw sources and aligned outputs are plain, inspectable +`pd.DataFrame`s. Dateless rung-2 source calls return inspectable lazy source +specifications that `align()` materializes against the spine window. Every rung +is supported — pick how much you want the SDK to do: ```python -import mostlyright - -# A general feature table — works for ANY catalog station worldwide, no market -# required. label=None composes features only (no settlement label columns). -df = mostlyright.dataset("EGLL", "2025-01-06", "2025-01-12", label=None) - -# Attach the WU/NOAA-WRH daily extremes as the label (native Celsius names). -df = mostlyright.dataset("KNYC", "2025-01-06", "2025-01-12", label="daily_extremes") - -# Or bring your own label frame — the SDK aligns it to the settlement calendar. -df = mostlyright.dataset("KNYC", "2025-01-06", "2025-01-12", label=my_label_frame) +import mostlyright as mr +from mostlyright import weather + +# Rung 3 — the per-domain quickstart one-liner (capped kwargs, never grows). +pairs = weather.pairs("KNYC", "2025-01-01", "2025-01-31") # label="cli" default + +# Rung 2 — spine + align: the SDK does ONLY the dangerous step (the as-of join). +spine = weather.label.daily_extremes("KNYC", "2025-01-01", "2025-01-31") +frame = mr.align(spine, weather.obs("KNYC"), weather.forecasts("KNYC")) + +# Rung 1 — raw source frames for inspection/hand joins; build your own spine. +raw_obs = weather.obs("KNYC", "2025-01-01", "2025-01-31") +# BYO label frame: keep a `date` column (the ISO settlement day) — it is the +# key weather sources attach on — and use the canonical station value emitted +# by the source (`"NYC"` here, not the input alias `"KNYC"`). +my_label_frame = my_label_frame.assign(station="NYC") +# mr.spine() maps YOUR column names explicitly. +spine = mr.spine(my_label_frame, + entity="station", decision_time="cutoff", y="settle_high") +# Re-enter align through the registered lazy source spec: +frame = mr.align(spine, weather.obs("KNYC")) ``` -> Calling `dataset()` **without** an explicit `label=` still returns the NWS CLI -> settlement columns byte-identically, but emits a `FutureWarning`: the default -> flips to `label=None` at 2.0. Pass `label="cli"` or `label=None` to silence it. -> Full walkthrough in [`docs/dataset.md`](docs/dataset.md). +Features-only (no settlement label)? Align sources onto the bare settlement +calendar: `mr.align(weather.days("KSEA", d1, d2), weather.obs("KSEA"))`. + +> **Relocated root surface.** `mostlyright.dataset()` / +> `mostlyright.research()` still delegate byte-identically to their canonical +> weather homes; only these root shims are removed at 2.0. No `FutureWarning` +> fires on the old default anymore in +> Python and TypeScript (the planned label-default flip was cancelled). The +> [1.17 migration guide](docs/migration/v117-align-spine-sources.md) maps +> every old kwarg to its new home; the legacy walkthrough lives in +> [`docs/dataset.md`](docs/dataset.md). ### Train models with train/infer source parity -`dataset()` (and its `research()` alias) stamps a source identity on every row. -Pinning `source=` on `obs()` returns single-source coverage; the validator catches -train/infer mismatches at load time, instead of silently corrupting predictions in -production. +`align()` records its contributing sources in `DataFrame.attrs["provenance"]`. +Legacy composed frames expose source identity only when `source=` is pinned; +unpinned fused frames intentionally have no single frame source. +Pinning `source=` on both historical and live calls makes the train/trade +provenance choice explicit instead of silently switching providers. ```python +import asyncio +from mostlyright import weather from mostlyright.weather import obs -from mostlyright.core import validate_dataframe train = obs("KNYC", "2024-01-01", "2024-12-31", - source="iem.archive", granularity="observation") -# Validator pins the schema's expected source. SourceMismatchError fires if you -# accidentally route a source-blind (fused AWC+IEM+GHCNh) DataFrame through an -# iem.archive-only schema. See the source-identity contract for the full rule. -validate_dataframe(train, schema_id="schema.observation.v1") + source="iem", granularity="observation") +# TRADE uses the same authority as TRAIN. +tick = asyncio.run(weather.live.latest("KNYC", source="iem")) ``` > Pinning `source=` returns a different row composition than the fused default — @@ -155,21 +205,25 @@ validate_dataframe(train, schema_id="schema.observation.v1") ### Backtest prediction markets (thin convenience layer) -The prediction-market entry points are thin delegators to the venue-free -`dataset()` — they own **all** venue knowledge (station resolution, strike parsing, -outcome targets) and route to the **venue-correct** label: Kalshi settles on NWS CLI +The prediction-market quickstarts are thin, venue-aware delegators — they own +**all** venue knowledge (station resolution, strike parsing, outcome targets) +and route to the **venue-correct** label: Kalshi settles on NWS CLI (`label="cli"`), Polymarket on WU/NOAA-WRH extremes (`label="daily_extremes"`). `outcome=True` appends a binary `label_outcome` by binarizing that venue's label -against its parsed strike (inclusive `>=`). +against its parsed strike. Threshold/range direction follows venue terms: +Kalshi high thresholds use inclusive `>=`; Polymarket below markets use `<=`, +and between markets use inclusive bounds. Venue label spines +(`markets.kalshi.label.settlement(...)`) compose with `mr.align()` like any other. ```python from mostlyright.markets import kalshi, polymarket # Kalshi → NWS CLI settlement. outcome=True adds the binary YES/NO target. -k = kalshi.dataset("KXHIGHNY-25MAY26-T79", "2025-05-26", "2025-05-26", outcome=True) +k = kalshi.pairs("KXHIGHNY-25MAY26-T79", "2025-05-26", "2025-05-26", outcome=True) # Polymarket → WU/NOAA-WRH extremes (the correct ground truth — NOT cli_*). -p = polymarket.dataset("", "2025-05-26", "2025-05-26", outcome=True) +# The event id comes from the discovery row's `event_id` field (not `slug`). +p = polymarket.pairs("your-event-id", "2025-05-26", "2025-05-26", outcome=True) ``` ### Feed AI agents structured public data @@ -192,9 +246,9 @@ Native-L2 single-pixel extraction across the **entire populated world**, each instrument family carrying its own leakage-safe **source identity** so a model trained on one never silently reconciles against another. A feature supplement (cloud-mask / land-surface covariates), not a primary Tmax/Tmin settlement -source. Ships as the optional `mostlyrightmd-weather[satellite]` extra -(whole-file S3 reads via `s3fs` + `h5netcdf`/`h5py`; no hosted backend — reads -the same anonymous public buckets as the AWC/IEM/NWP calls). +source. Ships as the optional `mostlyrightmd-weather[satellite]` extra. Its +default `delivery="live"` path reads public buckets directly via `s3fs` + +`h5netcdf`/`h5py`; `delivery="hosted"` is a separate explicit opt-in. | Source identity | Instrument | Coverage | Access | |---|---|---|---| @@ -204,7 +258,7 @@ the same anonymous public buckets as the AWC/IEM/NWP calls). | `eumetsat_meteosat` | Meteosat SEVIRI (MSG-CLM) | Europe/Africa/Indian-Ocean | **key** (EUMETSAT Data Store) | ```bash -pip install mostlyrightmd-weather[satellite] +pip install 'mostlyrightmd-weather[satellite]' ``` ```python @@ -218,18 +272,17 @@ df = satellite("KSEA", start=..., end=...) # -> noaa_goes (GOES-West/PACUS) df = satellite("RJTT", start=..., end=...) # -> jma_himawari df = satellite("EGLL", start=..., end=...) # -> noaa_viirs fill (Meteosat gated from auto-route; see below) -# Meteosat SEVIRI projection landed but its live Data-Store GRIB reader is a -# forward seam, so it is reachable by EXPLICIT satellite= (and gated out of -# auto-routing). An explicit fetch surfaces a clear "not yet wired" signal: +# Meteosat SEVIRI projection and keyed Data-Store transport are registered, but +# the live GRIB navigation reader is not wired end to end. It is gated out of +# automatic routing; this explicit call currently raises the documented seam error: df = satellite("EGLL", "meteosat-0deg", start=..., end=...) # needs a Data-Store key ``` -**Live vs hosted.** `delivery="live"` (default) self-parses the public data -directly. `delivery="hosted"` is the Phase-27 paid-adapter seam and raises a -clear "arrives in Phase 27" error — there is **no** `api.mostlyright.md` call -anywhere. The future paid adapter shares each native source identity -(byte-identical to live self-extraction), distinguished only by the -informational `delivery` lineage column. +**Live vs hosted.** `delivery="live"` (default) self-parses public data directly +and never calls a mostlyright service. `delivery="hosted"` is an explicit opt-in +that calls `${WEATHER_HOSTED_URL}/satellite` with `MOSTLYRIGHT_API_KEY`; its rows +preserve the resolved native source identity and carry informational `delivery` +lineage. The fleet bulk/training path is `python -m mostlyright.weather.satellite backfill` (per-`(satellite,year,month)` slices, crash-safe resume, @@ -241,18 +294,25 @@ table, coverage map, auto-routing, cheap-CONUS steering, DSRF gating, the 28 TB ## Why mostlyright -- **No hosted backend.** Direct calls to public APIs (NOAA, NWS, IEM, Kalshi, Polymarket). No proxy. No vendor account. No rate-limited tier. -- **Local-first cache.** Parquet (Python) or JSON envelope (Node) at `~/.mostlyright/cache/`. Byte-stable across runs — deterministic backtests. +- **Local-first by default.** Live calls go directly to public APIs (NOAA, NWS, + IEM, Kalshi, Polymarket); only the explicit satellite `delivery="hosted"` path + uses the configured mostlyright service. +- **Local-first cache.** Parquet under `~/.mostlyright/cache/` (Python) or JSON + envelopes under `~/.mostlyright/cache-ts/` (Node). Stable historical windows + are reusable; volatile current-LST windows deliberately refresh. - **Schema-versioned outputs.** Every response carries a stable `schema.*.v1` URI. Train/infer source mismatches fail loudly instead of silently corrupting models. -- **Python + TypeScript peers.** Same `research()` shape, byte-equivalent on the parity fixtures. Use whichever runtime your stack prefers. +- **Python + TypeScript peers.** Python 1.17 uses the spine/align shape; TS + `research()` retains the legacy `PairsRow` shape while PT-3401 tracks the + ladder port. Use whichever runtime your stack prefers. - **MIT licensed.** Use it commercially. Fork it. Ship it. ## Documentation - **Quickstart + concepts:** -- **`dataset()` — the alignment engine:** [`docs/dataset.md`](docs/dataset.md) — the label axis, the contributor protocol, panels, and the honest with/without comparison. +- **The `align` / `spine` / `sources` ladder (the 1.17 surface):** [`docs/migration/v117-align-spine-sources.md`](docs/migration/v117-align-spine-sources.md) — the three rungs, per-domain `pairs()`, and the composition model. +- **`weather.dataset()` — the compatibility composer:** [`docs/dataset.md`](docs/dataset.md) — the label axis, contributor protocol, panels, and root-shim migration. - **Source identity — the cross-vertical kwarg contract:** [`docs/source-identity.md`](docs/source-identity.md) — `source=` / `delivery=` / grain, the labels-only firewall, and the fetch-provenance ledger. -- **Migrating to 1.17.0 — the `align` / `spine` / `sources` ladder:** [`docs/migration/v117-align-spine-sources.md`](docs/migration/v117-align-spine-sources.md) — the where-did-each-`dataset()`-kwarg-go table (all 26), `label`'s 5 jobs → 4 call shapes, per-domain `pairs()`, the namespace-inversion symbol map, TypeScript-on-1.17, and the econ-unchanged note. Additive-plus-shims; the D-22 `FutureWarning` is gone. +- **Migrating to 1.17.0 — the `align` / `spine` / `sources` ladder:** [`docs/migration/v117-align-spine-sources.md`](docs/migration/v117-align-spine-sources.md) — the where-did-each-`dataset()`-kwarg-go table (all 26), `label`'s 5 jobs → 4 call shapes, per-domain `pairs()`, the namespace-inversion symbol map, TypeScript-on-1.17, and the econ-unchanged note. Additive-plus-shims; Python removed the D-22 `FutureWarning`, with the paired TS cancellation in #127. - **Migrating to 1.15.0:** [`docs/migration-1.15.md`](docs/migration-1.15.md) — the D-22 default-label `FutureWarning`, `include_*` → `features=`, and `contract=` → the markets layer. - **API reference:** (Python + TypeScript reference auto-generated per release) - **Migration from the legacy hosted-API client:** diff --git a/docs/dataset.md b/docs/dataset.md index 91eb434c..9650f090 100644 --- a/docs/dataset.md +++ b/docs/dataset.md @@ -1,10 +1,20 @@ # `dataset()` — the alignment engine +> **Relocated surface (1.17+).** The root `mostlyright.dataset` and +> `mostlyright.research` exports are deprecation shims removed at 2.0. Their +> canonical `mostlyright.weather.dataset` / `mostlyright.weather.research` +> homes continue to work. The preferred weather surface is `weather.pairs()` +> and the `mr.align(spine, *sources)` ladder; see the +> [1.17 migration guide](migration/v117-align-spine-sources.md) for the +> complete kwarg-by-kwarg mapping. This document covers the canonical +> compatibility composer and its root shims. + `dataset()` is the one thing in mostlyright that is genuinely hard to get right by hand: it takes a station, a window, and a set of feature ingredients, and returns **one leakage-safe row per LST settlement day** — every column aligned to -the settlement calendar, joined as-of so nothing from the future leaks backward, -and stamped with a durable source identity. +the settlement calendar and joined as-of so nothing from the future leaks +backward. A pinned source is recorded in `DataFrame.attrs`; fused defaults +intentionally have no single frame-level source identity. It is **not** a predefined recipe or a fixed Kalshi table. It is a **training-table builder** (an *aligner*): you tell it what label axis to attach and which features @@ -18,8 +28,7 @@ conveniences (`markets.kalshi.dataset` / `markets.polymarket.dataset`) are thin delegators that call this same `dataset()` — see [§ Markets are thin sugar](#markets-are-thin-sugar). > `research()` is a **fully working alias** of `dataset()` in this release. The -> two are distinct thin wrappers over one shared body; they behave identically -> except for one default-label warning (see the [D-22 bridge](#the-label-axis)). +> two are distinct thin wrappers over one shared body and behave identically. --- @@ -38,7 +47,7 @@ byte-stable primitive. Here is what each capability costs you by hand versus wha | One row per settlement day | You dedup + aggregate + guard against fan-out | Row count is unique-per-day by construction | | Attach a label | You pick a label source, join it, null low-coverage days | `label=` routes a named recipe / `None` / a BYO frame | | Multi-station panel | You loop, concat, re-key, preserve order | `stations=[...]` — long format, order preserved | -| Source identity on every row | You track which provider each row came from | Durable per-row `source` column + frame identity | +| Source provenance | You track which provider each input came from | `attrs["source"]` when `source=` is pinned; fused defaults intentionally have no single frame source | `dataset()` is worth reaching for exactly when the day-alignment and the no-leakage join are the parts you'd otherwise get subtly wrong. If you only need @@ -92,36 +101,14 @@ df = mostlyright.dataset("EGLL", "2025-01-06", "2025-01-12", label=None) # No cli_* label columns — just the composed, settlement-aligned features. ``` -### The D-22 bridge — the default label is changing - -Today, calling `dataset()` **without** an explicit `label=` still returns the -`cli_*` columns byte-identically — but it emits a one-time `FutureWarning`: - -> `dataset() label default changes from 'cli' to None in 2.0; pass label='cli' to -> keep settlement labels or label=None to silence` - -This is the **D-22 bridge**: a ≥2-minor deprecation window so no one is surprised -when the default flips to the pure `None` (features-only) table at 2.0. Pass an -explicit label and the warning never fires. - -| Release | `dataset()` with no `label=` | Warning | `research()` with no `label=` | -| --- | --- | --- | --- | -| 1.15 (now) | returns `cli_*` (byte-identical) | `FutureWarning` (once) | returns `cli_*`, **never warns** | -| 1.16 … 1.x | returns `cli_*` (byte-identical) | `FutureWarning` (once) | returns `cli_*`, **never warns** | -| 2.0 | default flips to `label=None` | — (default is now explicit) | *(legacy alias retires at 2.0)* | - -`research()` is the legacy name — it keeps the implicit `cli` label **forever with -no new warning** (it retires at 2.0 anyway, so there is nothing to migrate toward). -Parity tests pin `label="cli"` explicitly, so the parity gate stays warning-free by -construction. +### Default label -> **A note on `-W error`.** mostlyright's own CI and test suites do **not** run -> Python with warnings-as-errors — the D-22 `FutureWarning` is purely a -> **user-facing, opt-in concern**. If *you* run your code under `python -W error` -> (or a `filterwarnings = error` pytest config), the bare-`dataset()` call will -> raise instead of warn; pass an explicit `label="cli"` / `label=None` to keep it -> silent. Parity byte-stability is guaranteed by the explicit `label="cli"` pins, -> not by any warnings filter. +The proposed D-22 default flip was canceled. Calling `dataset()` or `research()` +without `label=` keeps the implicit `"cli"` label and emits no default-label +warning. Only the relocated root shims retire at 2.0; the canonical +`weather.dataset()` / `weather.research()` functions remain. Use +`weather.pairs()` for the capped recipe or construct an explicit spine and +sources with `align()`. ### Bring your own label diff --git a/docs/migration-1.15.md b/docs/migration-1.15.md index 4d0eb665..10f8ad5f 100644 --- a/docs/migration-1.15.md +++ b/docs/migration-1.15.md @@ -1,5 +1,10 @@ # Migrating to 1.15.0 +> **Superseded notice:** The D-22 label-default warning and planned 2.0 flip +> described below were canceled. Current releases keep the implicit `cli` label +> for both `dataset()` and `research()` and emit no label-default warning. This +> page otherwise remains as a historical 1.15 migration guide. + 1.15.0 is the **general-first re-layering** release. The venue-free core `dataset()` becomes an alignment engine with an explicit **label axis**; the prediction-market surfaces (Kalshi, Polymarket) become thin delegators on top of diff --git a/docs/migration/v117-align-spine-sources.md b/docs/migration/v117-align-spine-sources.md index f4757fc5..411feb86 100644 --- a/docs/migration/v117-align-spine-sources.md +++ b/docs/migration/v117-align-spine-sources.md @@ -256,9 +256,10 @@ registry — no fuzzy search) The registry redesign is a **Python-first** landing. For the TypeScript SDK: -- **Unchanged — keep using it as-is:** `research()` and `dataset()` on - `@mostlyrightmd/weather` / `@mostlyrightmd/markets` work exactly as before in - 1.17. No TS code changes are required to upgrade. +- **Unchanged — keep using it as-is:** `research()` and `dataset()` from the + `mostlyright` meta package work exactly as before in 1.17. The scoped weather + and markets packages continue to provide their domain fetchers and resolvers; + they do not export these composition functions. - **Python-only on 1.17:** the ladder verbs `align` / `spine` / `pairs` and the `.label.` axis ship on the **Python** SDK only this release. The async TS `align()` (browser fetch makes the materializing wrapper async) is diff --git a/docs/satellite.md b/docs/satellite.md index 18a29a81..8381888f 100644 --- a/docs/satellite.md +++ b/docs/satellite.md @@ -295,16 +295,15 @@ Until you run it in-region, the constants stay conservative - **`delivery="live"`** — self-parses the public data directly (anonymous S3 for GOES/Himawari/VIIRS; the keyed EUMETSAT Data Store for Meteosat). This is the whole of Phase 25 + 26. -- **`delivery="hosted"`** — the Phase-27 paid-adapter seam. It is a **valid enum - value** but the hosted path does not exist yet, so it raises a clear - *"hosted delivery arrives in Phase 27; use delivery='live'"* error **before any - I/O**. There is **no `api.mostlyright.md` call anywhere** in the SDK this phase - (the CLAUDE.md rule is still in force). - -When the paid adapter lands (Phase 27) it will read a pre-extracted catalog but -SHARE each native source identity (byte-identical to live self-extraction), -distinguished only by the informational `delivery` lineage column — so a model -trained on adapter data reconciles with live self-extraction (no source drift). +- **`delivery="hosted"`** — an explicit opt-in that calls + `${WEATHER_HOSTED_URL}/satellite` with `MOSTLYRIGHT_API_KEY`. It preserves the + resolved native source identity while stamping hosted transport lineage. The + default live path never calls this service. + +The hosted adapter reads a pre-extracted catalog but shares each native source +identity and measurement payload with live self-extraction. The informational +`delivery` lineage column intentionally differs, so transport remains auditable +without creating source drift. ## EUMETSAT Meteosat — key setup diff --git a/docs/source-identity.md b/docs/source-identity.md index fd8ed2e2..ab989923 100644 --- a/docs/source-identity.md +++ b/docs/source-identity.md @@ -48,7 +48,7 @@ before any network call): | --- | --- | | weather observations (`obs()`) | `{None, "awc", "iem", "ghcnh"}` | | weather climate (`climate()`) | `{None, "iem", "cli", "cli.archive"}` | -| satellite (`satellite()`) | `{None, "noaa_goes"}` | +| satellite (`satellite()`) | no `source=` kwarg; select a native source through `satellite=` or omit it for regional auto-routing | | **econ (`series()`/`snapshot()`/`research_econ()`)** | `{None, "fred", "bls", "bea", "dol", "fed"}` | Econ is the contract's own "FRED for macro series" example made pinnable: today @@ -76,21 +76,23 @@ per-source limitation — but it means: --- -## 2. The `delivery=` contract — always local | hosted +## 2. The `delivery=` contract — always live | hosted -`delivery=` **always** means *where the computation runs* — `local` (default; -public APIs hit directly from the SDK) or `hosted` (the opt-in precomputed API, +`delivery=` **always** means *where the computation runs* — `live` (default; +public APIs hit directly from the local SDK) or `hosted` (the opt-in precomputed API, reached only via the documented hosted seams + `MOSTLYRIGHT_API_KEY`). It is a strictly orthogonal axis to `source=`: | axis | question it answers | example values | | ----------- | ------------------- | ------------------------------- | | `source=` | WHO produced it | `None`, `"awc"`, `"iem"`, `"noaa_goes"` | -| `delivery=` | WHERE it is computed| `"local"`, `"hosted"` | +| `delivery=` | WHERE it is computed| `"live"`, `"hosted"` | -Hosted rows are byte-identical to the local path. Satellite already models this -correctly: `source="noaa_goes"` (provenance) × `delivery ∈ {local, hosted}` -(transport). Any vertical that overloads `source=` to mean transport is a bug — +Hosted and live paths preserve the same native measurements and source identity; +their transport-lineage `delivery` values intentionally differ. Satellite models +this correctly: a native `satellite=` identifier resolves provenance such as +`"noaa_goes"`, while `delivery ∈ {live, hosted}` selects transport. Any vertical +that overloads `source=` to mean transport is a bug — see §8 for the one outstanding case (earnings) and its tracked follow-up. **Econ carries the `delivery=` axis with a reserved hosted seam.** `series()` / @@ -243,7 +245,7 @@ protects the LABEL computation only: is the "labels-only" narrowing — the blanket "never composed" rule is withdrawn. -### The label axis + the D-22 default-label bridge +### The label axis `dataset(label=...)` chooses the target the aligner attaches — a named recipe (`"cli"` = the NWS CLI settlement extremes; `"daily_extremes"` = the WU / NOAA-WRH @@ -253,15 +255,12 @@ run through the same LST-alignment + LEFT-join + as-of discipline. The label axi the settlement-firewall's *positive* side: only a label recipe or a BYO frame may occupy the label namespace the contributor firewall forbids everyone else from. -The **D-22 bridge** governs the default: calling `dataset()` **without** an explicit -`label=` still returns `cli_*` byte-identically this release but emits a one-time -`FutureWarning` ("default changes from 'cli' to None in 2.0"). The default flips to -`None` at **2.0** after a ≥2-minor window. `research()` — the legacy alias — keeps -the implicit `cli` label **forever with no new warning** (it retires at 2.0 anyway). -Parity tests pin `label="cli"` explicitly, so the parity gate stays warning-free by -construction — **not** via any warnings filter. mostlyright's own CI does not run -warnings-as-errors; the `FutureWarning` is a user-opt-in concern only. Full bridge -timeline in [`docs/migration-1.15.md`](migration-1.15.md). +Calling `dataset()` without an explicit `label=` continues to return `cli_*` +byte-identically and does not emit a label-default warning. The proposed D-22 +default flip was canceled: both `dataset()` and the legacy `research()` alias keep +the implicit `cli` label. Pass `label=None` explicitly for features-only output. +Parity tests pin `label="cli"` explicitly so their intended contract remains clear. +The historical 1.15 migration guide is retained for context and marked superseded. ### Covariate composition (D-16) @@ -449,8 +448,9 @@ loudly instead of quietly poisoning predictions. The earnings vertical currently declares `source ∈ {live, hosted}`. Under this contract that is a **`delivery=` axis mislabeled as `source=`** — `{live, hosted}` -describes *where* the data is computed, not *who* produced it. (Satellite got this -right: `source="noaa_goes"` × `delivery ∈ {live, hosted}`.) +describes *where* the data is computed, not *who* produced it. Satellite gets this +right: `satellite=` selects a native source whose resolved provenance can be +`"noaa_goes"`, while `delivery ∈ {live, hosted}` remains the transport axis. **This phase changes NO earnings schema or code — it documents the target rule only.** The target rule: earnings should carry `delivery ∈ {live, hosted}`, diff --git a/docs/ts-quickstart.md b/docs/ts-quickstart.md index 18c80136..e143d10c 100644 --- a/docs/ts-quickstart.md +++ b/docs/ts-quickstart.md @@ -1,13 +1,23 @@ # TypeScript SDK quickstart (`@mostlyrightmd/*`) -The TS SDK mirrors the Python public surface and ships four npm packages: +> **TS on 1.17:** the TS SDK surface is unchanged — `research()` and +> `dataset()` keep working exactly as in 1.16. The paired TS cancellation in +> PR #127 removes the D-22 default-label flip and warning. Python's new +> `align`/`spine`/`pairs()` ladder is Python-only for now (parity ticket +> PT-3401); see the +> ["TypeScript on 1.17" section of the migration guide](migration/v117-align-spine-sources.md). + +The TS SDK ships parallel data and legacy composition contracts, with the new +Python align/spine/pairs ladder tracked separately in PT-3401. It has five npm +packages: | npm | Path | Use case | |---|---|---| | [`@mostlyrightmd/core`](../packages-ts/core/) | core types, schemas, temporal/QC primitives, formats | every consumer | | [`@mostlyrightmd/weather`](../packages-ts/weather/) | AWC/IEM/GHCNh/CLI fetchers + parsers | observations + climate | | [`@mostlyrightmd/markets`](../packages-ts/markets/) | Kalshi NHIGH/NLOW + Polymarket discover/settle | settlement logic | -| [`mostlyright`](../packages-ts/meta/) | meta convenience re-export | full-SDK import | +| [`@mostlyrightmd/econ`](../packages-ts/econ/) | economic indicator types + settlement routing | economics | +| [`mostlyright`](../packages-ts/meta/) | core + weather + markets convenience re-export | weather/markets composition | Five-minute path: **Node** + **browser** below. The browser path is for service workers / content scripts in extensions or web apps; the Node path is for scripts, backtests, Workers, and Bun/Deno. @@ -19,6 +29,8 @@ npm install mostlyright ``` `mostlyright` re-exports the three scoped packages (`@mostlyrightmd/core`, `@mostlyrightmd/weather`, `@mostlyrightmd/markets`) so importing `mostlyright` is enough. Install the scoped packages separately only if you want a single subpath (e.g. `@mostlyrightmd/markets/polymarket`) without pulling the meta surface. +Economics is intentionally separate: install and import `@mostlyrightmd/econ` +directly; the `mostlyright` meta package does not re-export it. ```ts import { research } from "mostlyright"; @@ -36,7 +48,10 @@ console.log(rows[0]); // } ``` -`research(station, fromDate, toDate)` returns the same 20-column shape Python emits — byte-equivalent on the 5 canonical parity fixtures. +`research(station, fromDate, toDate)` returns the legacy TS `PairsRow` shape. +PT-3401 tracks parity with Python's new spine/align output. Historical fixture +shapes are retained, but full five-fixture value replay remains pending +committed TS HTTP recordings. ## Cache diff --git a/packages-ts/core/README.md b/packages-ts/core/README.md index a95610d0..2b64702f 100644 --- a/packages-ts/core/README.md +++ b/packages-ts/core/README.md @@ -1,6 +1,6 @@ # @mostlyrightmd/core -The TypeScript core of the [mostlyright](https://github.com/mostlyrightmd/mostlyright-sdk) SDK — for quants, ML pipelines, and AI agents working with public weather data + prediction-market settlements. Provides core types, schemas, validators, temporal-safety primitives, the `research()` join, snapshot primitives, mode-2 source-pinned selectors, transforms, QC, discovery, formats, merge logic, and the full exception hierarchy. Mirrors the Python `mostlyrightmd` distribution. +The TypeScript core of the [mostlyright](https://github.com/mostlyrightmd/mostlyright-sdk) SDK — for quants, ML pipelines, and AI agents working with public weather data + prediction-market settlements. Provides core types, schemas, validators, temporal-safety primitives, snapshot primitives, mode-2 source-pinned selectors, transforms, QC, discovery, formats, merge logic, and the full exception hierarchy. The legacy `research()` join is exported by the `mostlyright` meta package, not this scoped core package. Mirrors the Python `mostlyrightmd` distribution. ## Install diff --git a/packages-ts/core/package.json b/packages-ts/core/package.json index 74a06e68..e67c1095 100644 --- a/packages-ts/core/package.json +++ b/packages-ts/core/package.json @@ -1,7 +1,7 @@ { "name": "@mostlyrightmd/core", "version": "1.17.0", - "description": "TypeScript SDK core for quants, ML pipelines, and AI agents: types, schemas, validators, temporal-safety primitives, and the research() join over weather data + prediction-market settlements. Local-first, no hosted backend.", + "description": "TypeScript SDK core for quants, ML pipelines, and AI agents: types, schemas, validators, temporal-safety primitives, formats, and source contracts. Composition is provided by the mostlyright meta package.", "keywords": [ "weather", "weather-data", diff --git a/packages-ts/core/src/index.ts b/packages-ts/core/src/index.ts index 70670178..de36aaa6 100644 --- a/packages-ts/core/src/index.ts +++ b/packages-ts/core/src/index.ts @@ -1,12 +1,8 @@ // @mostlyrightmd/core — placeholder scaffold for TS-W0 Wave 1. // Real implementation (research, snapshot, mode2, transforms, qc, ...) lands in TS-W1+. -/** - * Placeholder version string from the TS-W0 Wave 1 scaffold. The - * authoritative package version lives in `package.json#version` - * (currently `0.1.0-rc.7`); this constant has not been bumped. - */ -export const version = "1.15.0"; +/** Public package version; kept in lockstep with package.json. */ +export const version = "1.17.0"; /** * Smoke-test export from the TS-W0 Wave 1 scaffold. Returns the literal diff --git a/packages-ts/core/tests/hello.test.ts b/packages-ts/core/tests/hello.test.ts index 8fcea423..19a34def 100644 --- a/packages-ts/core/tests/hello.test.ts +++ b/packages-ts/core/tests/hello.test.ts @@ -3,8 +3,8 @@ import { describe, expect, it } from "vitest"; import { helloCore, version } from "../src/index.js"; describe("@mostlyrightmd/core hello-world scaffold", () => { - it("exports the placeholder version string", () => { - expect(version).toBe("1.15.0"); + it("exports the package version", () => { + expect(version).toBe("1.17.0"); }); it("returns the expected hello string", () => { diff --git a/packages-ts/econ/package.json b/packages-ts/econ/package.json index 65ffe305..0f1a9476 100644 --- a/packages-ts/econ/package.json +++ b/packages-ts/econ/package.json @@ -61,7 +61,7 @@ "codegen": "echo \"@mostlyrightmd/econ — no codegen step at this layer\"" }, "peerDependencies": { - "@mostlyrightmd/core": "^0.0.0" + "@mostlyrightmd/core": "~1.17.0" }, "devDependencies": { "@mostlyrightmd/core": "workspace:*", diff --git a/packages-ts/markets/package.json b/packages-ts/markets/package.json index a9531372..4e8adcd7 100644 --- a/packages-ts/markets/package.json +++ b/packages-ts/markets/package.json @@ -64,7 +64,7 @@ "codegen": "echo \"@mostlyrightmd/markets \u2014 no codegen step at this layer\"" }, "peerDependencies": { - "@mostlyrightmd/core": "^0.0.0" + "@mostlyrightmd/core": "~1.17.0" }, "devDependencies": { "@mostlyrightmd/core": "workspace:*", diff --git a/packages-ts/markets/src/index.ts b/packages-ts/markets/src/index.ts index 45e5dac3..1f0a5f8f 100644 --- a/packages-ts/markets/src/index.ts +++ b/packages-ts/markets/src/index.ts @@ -1,12 +1,8 @@ // @mostlyrightmd/markets — placeholder scaffold for TS-W0 Wave 1. // Real implementation (Kalshi NHIGH/NLOW resolvers, Polymarket discover/settle) lands in TS-W1+. -/** - * Placeholder version string from the TS-W0 Wave 1 scaffold. The - * authoritative package version lives in `package.json#version` - * (currently `0.1.0-rc.7`); this constant has not been bumped. - */ -export const version = "1.15.0"; +/** Public package version; kept in lockstep with package.json. */ +export const version = "1.17.0"; /** * Smoke-test export from the TS-W0 Wave 1 scaffold. Returns the literal diff --git a/packages-ts/markets/tests/hello.test.ts b/packages-ts/markets/tests/hello.test.ts index a01994d5..e54ace76 100644 --- a/packages-ts/markets/tests/hello.test.ts +++ b/packages-ts/markets/tests/hello.test.ts @@ -3,8 +3,8 @@ import { describe, expect, it } from "vitest"; import { helloMarkets, version } from "../src/index.js"; describe("@mostlyrightmd/markets hello-world scaffold", () => { - it("exports the placeholder version string", () => { - expect(version).toBe("1.15.0"); + it("exports the package version", () => { + expect(version).toBe("1.17.0"); }); it("returns the expected hello string", () => { diff --git a/packages-ts/meta/README.md b/packages-ts/meta/README.md index 482491d2..a737bc44 100644 --- a/packages-ts/meta/README.md +++ b/packages-ts/meta/README.md @@ -2,11 +2,18 @@ **The public-data SDK for quants, ML pipelines, and AI agents.** -`mostlyright` is the convenience meta-package for the TypeScript SDK. A single `import { research } from "mostlyright"` re-exports the surfaces of `@mostlyrightmd/core`, `@mostlyrightmd/weather`, and `@mostlyrightmd/markets` — weather data (METAR, ASOS, GHCNh, NWS CLI), prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket), and the core `research()` join. Direct calls to public APIs. No hosted backend, no API key. +`mostlyright` is the convenience meta-package for the TypeScript SDK. A single +`import { research } from "mostlyright"` provides the legacy weather composition +join plus a selected convenience surface from core, weather, and markets. It is +not an exhaustive barrel for every scoped-package export; import specialized +features such as `forecastNwp` from their scoped package. Public-API paths are +local and keyless by default; hosted satellite/stream clients are explicit +credentialed opt-ins. Weather + prediction-markets adapters are live today. SEC filings (EDGAR), equities structured data, Federal Reserve series (FRED), court filings, and FDA approvals are next — and the architecture is built to ship an adapter for any public data source. -If you only need one slice of the SDK, depend on the scoped packages directly. If you want everything in one import, this is the package. +Depend on scoped packages directly for a complete domain surface; use this meta +package for the legacy composition join and common conveniences. ## Install diff --git a/packages-ts/meta/package.json b/packages-ts/meta/package.json index 3efbfff5..fb7588be 100644 --- a/packages-ts/meta/package.json +++ b/packages-ts/meta/package.json @@ -1,7 +1,7 @@ { "name": "mostlyright", "version": "1.17.0", - "description": "Public-data SDK for TypeScript — one import for quants, ML pipelines, and AI agents. Adapters ship weather (METAR, ASOS, GHCNh, NWS CLI) and prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket) today; SEC filings, Federal Reserve series, court filings, FDA approvals, and equities are next. Local-first, no hosted backend.", + "description": "Public-data SDK for TypeScript — one import for weather and prediction-market research. Local-first and keyless by default, with explicit credentialed hosted satellite/stream clients.", "keywords": [ "weather", "weather-data", diff --git a/packages-ts/meta/src/index.ts b/packages-ts/meta/src/index.ts index cb2aff0e..05a49575 100644 --- a/packages-ts/meta/src/index.ts +++ b/packages-ts/meta/src/index.ts @@ -189,11 +189,5 @@ export { preprocessing }; */ export const Preprocessing = preprocessing; -/** - * Placeholder version string for the meta package. The authoritative - * package version lives in `package.json#version` (currently - * `0.1.0-rc.7`); this constant has not been bumped. Sibling packages - * (`@mostlyrightmd/core` / `weather` / `markets`) each export their own - * `version` constant, exposed here via the namespaced module objects. - */ -export const version = "1.15.0"; +/** Public package version; kept in lockstep with package.json. */ +export const version = "1.17.0"; diff --git a/packages-ts/meta/tests/hello.test.ts b/packages-ts/meta/tests/hello.test.ts index 04154ecc..03205fc9 100644 --- a/packages-ts/meta/tests/hello.test.ts +++ b/packages-ts/meta/tests/hello.test.ts @@ -11,8 +11,8 @@ import { } from "../src/index.js"; describe("mostlyright (meta) hello-world scaffold", () => { - it("exports the placeholder version string", () => { - expect(version).toBe("1.15.0"); + it("exports the package version", () => { + expect(version).toBe("1.17.0"); }); it("re-exports helloCore from @mostlyrightmd/core", () => { @@ -28,8 +28,8 @@ describe("mostlyright (meta) hello-world scaffold", () => { }); it("namespaces each underlying package's version constant", () => { - expect(core.version).toBe("1.15.0"); - expect(weather.version).toBe("1.15.0"); - expect(markets.version).toBe("1.15.0"); + expect(core.version).toBe("1.17.0"); + expect(weather.version).toBe("1.17.0"); + expect(markets.version).toBe("1.17.0"); }); }); diff --git a/packages-ts/weather/README.md b/packages-ts/weather/README.md index fca32a69..ec0f2144 100644 --- a/packages-ts/weather/README.md +++ b/packages-ts/weather/README.md @@ -1,6 +1,6 @@ # @mostlyrightmd/weather -Weather data fetchers and parsers for the [mostlyright](https://github.com/mostlyrightmd/mostlyright-sdk) TypeScript SDK — live METAR (AWC), ASOS archive (IEM), IEM CLI, historical observations (GHCNh), NWS climate text products (CLI), plus the local-first cache layer — for quants, ML training pipelines, and weather-bot agents. Direct public-API access; no hosted backend, no API key. Mirrors the Python `mostlyrightmd-weather` distribution. Declares `@mostlyrightmd/core` as a peer dependency. +Weather data fetchers and parsers for the [mostlyright](https://github.com/mostlyrightmd/mostlyright-sdk) TypeScript SDK — live METAR (AWC), ASOS archive (IEM), IEM CLI, historical observations (GHCNh), NWS climate text products (CLI), plus the local-first cache layer — for quants, ML training pipelines, and weather-bot agents. Public-API paths are local and keyless by default; hosted satellite/stream clients are explicit opt-ins and require their documented service credentials. Mirrors the Python `mostlyrightmd-weather` distribution. Declares `@mostlyrightmd/core` as a peer dependency. ## Install @@ -38,7 +38,8 @@ recommended TS-side workaround. The Python SDK's 1.17 release introduces the `align` / `spine` / `sources` ladder (`mr.align(spine, *sources)`, per-domain `pairs()`, the `.label.` axis). **The TypeScript surface you use is unchanged in 1.17:** -`research()` / `dataset()` keep working exactly as before. The new ladder +`research()` / `dataset()` from the `mostlyright` meta package keep working +exactly as before; they are not exports of this scoped package. The new ladder verbs (`align` / `spine` / `pairs`) are **Python-only on 1.17** — the async TS `align()` is deferred and tracked as parity ticket **PT-3401** (see `.planning/CROSS-SDK-SYNC.md`). The shared cross-SDK conformance fixtures diff --git a/packages-ts/weather/package.json b/packages-ts/weather/package.json index fd175633..a2953123 100644 --- a/packages-ts/weather/package.json +++ b/packages-ts/weather/package.json @@ -1,7 +1,7 @@ { "name": "@mostlyrightmd/weather", "version": "1.17.0", - "description": "Weather data for TypeScript / Node — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Direct public-API access, no hosted backend.", + "description": "Weather data for TypeScript / Node — local-first public weather APIs with explicit credentialed hosted satellite/stream clients.", "keywords": [ "weather", "weather-data", @@ -83,7 +83,7 @@ "codegen": "echo \"@mostlyrightmd/weather \u2014 no codegen step at this layer\"" }, "peerDependencies": { - "@mostlyrightmd/core": "^0.0.0" + "@mostlyrightmd/core": "~1.17.0" }, "devDependencies": { "@mostlyrightmd/core": "workspace:*", diff --git a/packages-ts/weather/src/index.ts b/packages-ts/weather/src/index.ts index 7f5e1346..4db1bc75 100644 --- a/packages-ts/weather/src/index.ts +++ b/packages-ts/weather/src/index.ts @@ -4,12 +4,8 @@ // (yearly-chunk historical METARs) + the IEM CSV parser. Subsequent TS-W2 // plans add GHCNh + mergeObservations; TS-W3 adds the disk cache. -/** - * Placeholder version string from the TS-W0 Wave 1 scaffold. The - * authoritative package version lives in `package.json#version` - * (currently `0.1.0-rc.7`); this constant has not been bumped. - */ -export const version = "1.15.0"; +/** Public package version; kept in lockstep with package.json. */ +export const version = "1.17.0"; /** * Smoke-test export from the TS-W0 Wave 1 scaffold. Returns the literal diff --git a/packages-ts/weather/tests/hello.test.ts b/packages-ts/weather/tests/hello.test.ts index 9c446424..32cafa4d 100644 --- a/packages-ts/weather/tests/hello.test.ts +++ b/packages-ts/weather/tests/hello.test.ts @@ -3,8 +3,8 @@ import { describe, expect, it } from "vitest"; import { helloWeather, version } from "../src/index.js"; describe("@mostlyrightmd/weather hello-world scaffold", () => { - it("exports the placeholder version string", () => { - expect(version).toBe("1.15.0"); + it("exports the package version", () => { + expect(version).toBe("1.17.0"); }); it("returns the expected hello string", () => { diff --git a/packages/core/README.md b/packages/core/README.md index 63a96939..a62dccc9 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -2,9 +2,15 @@ **The public-data SDK for quants, ML pipelines, and AI agents.** -`mostlyrightmd` is the Python entry point: core types, schemas, validators, temporal-safety primitives, and the `research()` join that ties weather observations × climate into one row per settlement date — ready for prediction-market backtests (Kalshi NHIGH/NLOW, Polymarket), ML training pipelines, and AI-agent tool calls. Direct calls to public APIs (NOAA, NWS, IEM, GHCNh, Kalshi, Polymarket). No hosted backend, no API key. +`mostlyrightmd` is the Python core distribution: types, schemas, validators, +temporal-safety primitives, `align()` / `spine()`, and snapshot helpers. Weather, +markets, and econ adapters ship as separate distributions. The `[research]` extra +adds weather composition; credentialed and hosted options belong to their domain +packages rather than the core-only install. -Weather + prediction-markets adapters are live today. SEC filings (EDGAR), equities structured data, Federal Reserve series (FRED), court filings, and FDA approvals are next — and the architecture is built to ship an adapter for any public data source. +Weather and prediction-market adapters are available from +`mostlyrightmd-weather` and `mostlyrightmd-markets`; economics is +`mostlyrightmd-econ`. ## Install @@ -16,11 +22,12 @@ pip install mostlyrightmd-weather # weather data sources only (brings cor `mostlyrightmd` and `mostlyrightmd-weather` share the `mostlyright.*` Python namespace but ship as separate PyPI distributions, so users who only need the helpers can skip the heavier weather deps. `research()` lazy-imports `mostlyright.weather` and raises a clear error if the weather package is not installed. -## Quickstart +## Compatibility quickstart ```python import mostlyright +# Deprecated root shim; prefer mostlyright.weather.pairs() for new code. df = mostlyright.research("KNYC", "2025-01-06", "2025-01-12") print(df.head()) ``` diff --git a/packages/core/pyproject.toml b/packages/core/pyproject.toml index 3b9cef7d..dffbba97 100644 --- a/packages/core/pyproject.toml +++ b/packages/core/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "mostlyrightmd" version = "1.17.0" -description = "Python SDK for quants, ML engineers, and AI agents — one interface to public data. Adapters ship weather + prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket) today; SEC filings, Federal Reserve series, court filings, FDA approvals, and equities are next. Schema-versioned, leakage-free, local-first. Imports as `mostlyright`." +description = "Python SDK core for quants, ML engineers, and AI agents — schema-versioned types, temporal safety, validation, alignment, and snapshot primitives. Domain adapters ship separately. Imports as `mostlyright`." readme = "README.md" license = "MIT" authors = [{ name = "Robert Tarabcak", email = "tarabcakr@gmail.com" }] @@ -38,6 +38,8 @@ dependencies = [ # Pinned to v0.14.1 mostlyright deps for inherited test coverage (per TODOS.md item 1) "httpx>=0.27", "jsonschema>=4.21", + # align() is a first-class core export and imports pandas eagerly. + "pandas>=2.2,<4.0", "tzdata; sys_platform == 'win32'", # NOTE: `mostlyrightmd-weather` is INTENTIONALLY NOT in runtime deps. # Listing it here creates a runtime cycle (core <-> weather) that defeats @@ -72,12 +74,10 @@ parquet = [ # pandas upper bound aligned with `parquet` extra at <4.0; both backends # are exercised by the dual-pandas CI matrix. research = [ - # >=1.7.0: forecast_nwp() threads member= to the weather impl, a kwarg - # introduced in mostlyrightmd-weather 1.7.0 (#74) — an older weather would - # TypeError on forecast_nwp(..., member=...). (Supersedes the >=1.6.0 floor - # for fetch_open_meteo(variables=...), #64 codex P1; default calls stay - # skew-tolerant via the core wrapper's conditional member threading.) - "mostlyrightmd-weather>=1.7.0,<2.0", + # >=1.17.0: the documented weather.pairs() quickstart and the spine/align + # composition surface ship together. An older weather release can otherwise + # install successfully and fail at the first documented call. + "mostlyrightmd-weather>=1.17.0,<1.18.0", "pyarrow>=17.0,<24.0", "pandas>=2.2,<4.0", ] diff --git a/packages/econ/pyproject.toml b/packages/econ/pyproject.toml index 6ac2abc1..3a2d1c88 100644 --- a/packages/econ/pyproject.toml +++ b/packages/econ/pyproject.toml @@ -43,11 +43,9 @@ dependencies = [ # mostlyright (core) provides the load-bearing shared surface the econ # fetchers/cache/resolver build on: _internal._bounds, _internal._cache_dir, # _internal._http, core.schema, core.validator, core.temporal, core.exceptions. - # Pinned >=1.11.0 (the lockstep floor for THIS release) so a partial upgrade - # (econ==1.11.0 + core==1.10.x) can't ImportError on a symbol econ needs - # that first ships in core 1.11.0 — mirrors the PKG-03 floor discipline the - # weather/markets siblings use across the parity boundary. - "mostlyrightmd>=1.11.0,<2.0", + # Pinned to the matching lockstep release so a partial upgrade cannot import + # an older core missing the current composition and schema contracts. + "mostlyrightmd>=1.17.0,<1.18.0", "httpx>=0.27", "jsonschema>=4.21", "tzdata; sys_platform == 'win32'", diff --git a/packages/markets/pyproject.toml b/packages/markets/pyproject.toml index 5e20019f..5b4a81b0 100644 --- a/packages/markets/pyproject.toml +++ b/packages/markets/pyproject.toml @@ -36,14 +36,14 @@ dependencies = [ # mostlyright.markets.catalog (this package) but the wider settlement # pipeline reads mostlyright.core.* schemas; a stale core would silently # serve the wrong column set. - "mostlyrightmd>=1.0.0,<2.0", + "mostlyrightmd>=1.17.0,<1.18.0", # ECON-10 (Phase 29): the Kalshi Economics resolver # (mostlyright.markets.catalog.kalshi_econ) imports the curated per-series # routing table `SETTLEMENT_ROUTING` from mostlyright.econ._settlement_map. # This is the ONE-WAY markets -> econ coupling (econ never depends on # markets). _settlement_map is a pure routing-rules dict with no cache/ # schema import, so this does NOT couple markets to the parity firewall. - "mostlyrightmd-econ>=1.11.0,<2.0", + "mostlyrightmd-econ>=1.17.0,<1.18.0", "httpx>=0.27", "jsonschema>=4.21", ] @@ -70,7 +70,7 @@ parquet = [ # Users opt in via `pip install mostlyrightmd-markets[polymarket]`. polymarket = [ "pandas>=2.2,<4.0", - "mostlyrightmd-weather>=1.0.0,<2.0", + "mostlyrightmd-weather>=1.17.0,<1.18.0", ] # Phase 9: trade-history surface. kalshi_trades + polymarket_trades return # DataFrames (pandas required); _trades_cache writes parquet (pyarrow diff --git a/packages/weather/README.md b/packages/weather/README.md index c61825c3..ad7781f3 100644 --- a/packages/weather/README.md +++ b/packages/weather/README.md @@ -1,6 +1,6 @@ # mostlyrightmd-weather -Weather data fetchers for the `mostlyright` Python SDK — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI) — for quants, ML training pipelines, and weather-bot agents. Direct public-API access; no hosted backend, no API key. Local parquet cache at `$HOME/.mostlyright/cache/`. +Weather data fetchers for the `mostlyright` Python SDK — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI) — for quants, ML training pipelines, and weather-bot agents. Public-API paths are local and keyless by default. Explicit hosted satellite delivery requires `WEATHER_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`; the reserved EUMETSAT path requires its consumer credentials. Local parquet cache at `$HOME/.mostlyright/cache/`. ## Install diff --git a/packages/weather/pyproject.toml b/packages/weather/pyproject.toml index ce3689fc..11d2fd14 100644 --- a/packages/weather/pyproject.toml +++ b/packages/weather/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "mostlyrightmd-weather" version = "1.17.0" -description = "Weather data for Python — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Direct public-API access, no hosted backend. Imports as `mostlyright.weather`." +description = "Weather data for Python — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Local-first direct public-API access with optional hosted satellite delivery. Imports as `mostlyright.weather`." readme = "README.md" license = "MIT" authors = [{ name = "Robert Tarabcak", email = "tarabcakr@gmail.com" }] @@ -44,13 +44,9 @@ dependencies = [ # mostlyright.core.exceptions, which first ships in core 1.10.0. A partial # upgrade (weather==1.10.0 + core==1.9.0) would ImportError on # `import mostlyright.weather.cwop`; the floor bump forbids that mix. - # >=1.10.1: mostlyright.weather.earnings imports EarningsFactSchema / - # EarningsTranscriptSchema from mostlyright.core.schemas and EarningsError / - # CaptureNotAvailable from mostlyright.core.exceptions, which first ship in - # core 1.10.1 (Phase 27 Wave 1 canonical schemas — NOT present in 1.10.0). A - # partial upgrade (weather==1.10.1 + core==1.10.0) would ImportError on - # `import mostlyright.weather.earnings`; the floor bump forbids that mix. - "mostlyrightmd>=1.10.1,<2.0", + # 1.17 weather.pairs(), lazy sources, and label/spine composition consume + # the matching 1.17 core surface; forbid a resolvable partial upgrade. + "mostlyrightmd>=1.17.0,<1.18.0", # Pinned to v0.14.1 mostlyright deps "httpx>=0.27", "jsonschema>=4.21", @@ -109,7 +105,7 @@ polars = [ # Phase 25-26: NATIVE satellite ring (free local tier). Reads anonymous public # NOAA NODD buckets (GOES E/W s3://noaa-goes16/17/18/19, Himawari # s3://noaa-himawari8/9, VIIRS s3://noaa-nesdis-*-pds, AWS) or the GCS mirror — -# no hosted backend, no api.mostlyright.md. Whole-file reads via `h5netcdf` +# the default live path makes no hosted call. Whole-file reads via `h5netcdf` # (ships `libhdf5` as a wheel; no system install); `h5py` backs direct HDF5 # inspection for the Himawari/VIIRS/SEVIRI L2 products (Phase 26). `xarray` is # the dataset API (shared floor with `[nwp]`). `boto3`/`s3fs` do anonymous diff --git a/scripts/check_wheel_metadata.py b/scripts/check_wheel_metadata.py index a5fe160b..f1732a5e 100644 --- a/scripts/check_wheel_metadata.py +++ b/scripts/check_wheel_metadata.py @@ -1,154 +1,138 @@ -"""Pre-publish METADATA check (Phase 2 Wave 5 PKG-03). - -Every sibling-package wheel under ``dist/`` must declare an explicit -``Requires-Dist: mostlyrightmd >=0.1.0a1,<0.2`` (or stricter) so a downstream -user cannot silently install a core/weather/markets combo across an -incompatible boundary. - -Usage:: - - uv build --all - uv run python scripts/check_wheel_metadata.py - -Exits 0 when every wheel in ``dist/`` passes. Exits 1 with a descriptive -message per offending wheel otherwise. -""" +"""Verify the complete lockstep Python release artifact set.""" from __future__ import annotations import sys +import tarfile import zipfile +from argparse import ArgumentParser +from email.parser import Parser from pathlib import Path -#: Per-package wheel-name-prefix -> list of regex patterns that MUST match in -#: the wheel's METADATA Requires-Dist lines. Multiple patterns are AND-ed. -#: -#: We accept any ``>=0.1.0..., <0.2`` form to leave room for alpha/beta -#: revisions (``>=0.1.0a1``, ``>=0.1.0b0``, etc.). The hard requirement is -#: that the upper bound is ``<0.2`` so a future major bump (0.2.0) does not -#: silently get pulled. -# codex iter-5 HIGH fix: parse Requires-Dist with packaging.requirements -# instead of regex-substring matching. The old regex `<\s*0\.2` matched -# `<0.20` too — letting an incompatible 0.20.x upper bound silently slip -# through. Semantic comparison via SpecifierSet handles PEP 440 -# normalization + bound semantics correctly. from packaging.requirements import InvalidRequirement, Requirement -from packaging.specifiers import SpecifierSet - -#: Distributions we gate on (wheel filename prefix). Map -> the SpecifierSet -#: that the wheel's `Requires-Dist: mostlyrightmd ` line MUST -#: SATISFY. ``"<0.2"`` is the load-bearing upper bound — the gate exists -#: precisely to keep a future 0.2.x core from being pulled by an -#: under-specified weather/markets wheel. -REQUIRED_MOSTLYRIGHT_SPECIFIER: dict[str, SpecifierSet] = { - "mostlyrightmd_weather": SpecifierSet(">=0.1.0a1,<0.2"), - "mostlyrightmd_markets": SpecifierSet(">=0.1.0a1,<0.2"), +from packaging.version import Version + +EXPECTED_EDGES: dict[str, set[str]] = { + "mostlyrightmd": {"mostlyrightmd-weather"}, + "mostlyrightmd-weather": {"mostlyrightmd"}, + "mostlyrightmd-econ": {"mostlyrightmd"}, + "mostlyrightmd-markets": { + "mostlyrightmd", + "mostlyrightmd-econ", + "mostlyrightmd-weather", + }, } -def _find_mostlyright_requirement(metadata: str) -> Requirement | None: - """Return the parsed `Requires-Dist: mostlyrightmd ` from the - wheel's METADATA, or None if absent. - - Iterates Requires-Dist lines, parses each with packaging.Requirement, - and returns the first one whose ``name == "mostlyrightmd"``. Skips - Requires-Dist lines we can't parse (e.g. ones with environment - markers we don't care about; the parser handles markers but a - malformed one would otherwise blow up). - """ - for line in metadata.splitlines(): - if not line.startswith("Requires-Dist:"): - continue - spec_text = line[len("Requires-Dist:") :].strip() +def _metadata(artifact: Path) -> tuple[str, Version, list[Requirement]]: + if artifact.suffix == ".whl": + with zipfile.ZipFile(artifact) as archive: + metadata_path = next( + (name for name in archive.namelist() if name.endswith(".dist-info/METADATA")), + None, + ) + if metadata_path is None: + raise ValueError("no .dist-info/METADATA file") + raw_metadata = archive.read(metadata_path).decode() + elif artifact.name.endswith(".tar.gz"): + with tarfile.open(artifact, "r:gz") as archive: + metadata = next( + (member for member in archive.getmembers() if member.name.endswith("/PKG-INFO")), + None, + ) + if metadata is None: + raise ValueError("no PKG-INFO file") + extracted = archive.extractfile(metadata) + if extracted is None: + raise ValueError("could not read PKG-INFO") + raw_metadata = extracted.read().decode() + else: + raise ValueError("unsupported artifact type") + message = Parser().parsestr(raw_metadata) + + name = message.get("Name", "").lower().replace("_", "-") + version = Version(message.get("Version", "")) + requirements: list[Requirement] = [] + for raw in message.get_all("Requires-Dist", []): try: - req = Requirement(spec_text) - except InvalidRequirement: - continue - if req.name.replace("_", "-") == "mostlyrightmd": - return req - return None - - -def check_wheel(wheel_path: Path) -> list[str]: - """Return a list of error messages (empty when the wheel passes).""" + requirements.append(Requirement(raw)) + except InvalidRequirement as exc: + raise ValueError(f"invalid Requires-Dist {raw!r}: {exc}") from exc + return name, version, requirements + + +def check_artifact(artifact: Path) -> tuple[tuple[str, Version] | None, list[str]]: + """Return whether this is a recognized release wheel and any errors.""" + try: + name, version, requirements = _metadata(artifact) + except (KeyError, ValueError, tarfile.TarError, zipfile.BadZipFile) as exc: + return None, [f"{artifact.name}: {exc}"] + + expected_names = EXPECTED_EDGES.get(name) + if expected_names is None: + return None, [f"{artifact.name}: unrecognized distribution {name!r}"] + + upper = f"{version.major}.{version.minor + 1}.0" + expected_spec = f"<{upper},>={version}" + by_name = {req.name.lower().replace("_", "-"): req for req in requirements} errors: list[str] = [] - # Wheel filenames: ``----.whl``. - # Splitting on "-" gives ["", "", ...]. - pkg_name = wheel_path.name.split("-")[0] - required = REQUIRED_MOSTLYRIGHT_SPECIFIER.get(pkg_name) - if required is None: - # Not a sibling-package wheel we gate on. - return errors - with zipfile.ZipFile(wheel_path) as z: - metadata_path = next((name for name in z.namelist() if name.endswith("METADATA")), None) - if metadata_path is None: - errors.append(f"{wheel_path.name}: no METADATA file in wheel") - return errors - content = z.read(metadata_path).decode() - req = _find_mostlyright_requirement(content) - if req is None: - errors.append(f"{wheel_path.name}: missing Requires-Dist: mostlyrightmd line") - return errors - # The wheel's specifier MUST be at least as strict as the required range - # ``>=0.1.0a1,<0.2``. Two checks via SpecifierSet.contains() on sentinel - # versions chosen to fail loose specifiers that look superficially correct: - # - # - Upper bound: 0.2.0 (and 0.2.0a1 via prereleases=True) MUST NOT satisfy. - # Catches the iter-5 case (<0.20) and the more subtle (<0.3, or no upper). - # - Lower bound: 0.1.0a0 MUST NOT satisfy. Catches the iter-6 case - # (>=0.1.0a0 — alpha-0 is older than alpha-1 per PEP 440 ordering; - # our floor is alpha-1) AND the looser case (>0.0.9 — anything that - # accepts 0.0.9.post1 or 0.1.0a0 is below the parity floor). 0.0.9 - # itself is also checked because >0.0.9 specifically excludes 0.0.9 - # but accepts 0.1.0a0, which still fails this stricter check. - wheel_spec = str(req.specifier) - if req.specifier.contains("0.2.0", prereleases=True): - errors.append( - f"{wheel_path.name}: Requires-Dist: mostlyrightmd {wheel_spec!s} " - f"allows 0.2.0 (upper bound missing or too loose; required " - f"<0.2). Fix the pyproject.toml dep to '>=0.1.0a1,<0.2'." - ) - # codex iter-6 HIGH fix: tighten lower-bound sentinel from 0.0.9 to - # 0.1.0a0. PEP 440 orders 0.1.0a0 < 0.1.0a1, so a wheel pinning - # ``>=0.1.0a0`` previously slipped through (passed the 0.0.9 check). - # Our parity floor is alpha-1; any spec that lets alpha-0 (or any 0.0.x) - # in is a HIGH gate failure. - if req.specifier.contains("0.1.0a0", prereleases=True): - errors.append( - f"{wheel_path.name}: Requires-Dist: mostlyrightmd {wheel_spec!s} " - f"allows mostlyrightmd 0.1.0a0 or older (lower bound missing or " - f"too loose; required >=0.1.0a1). Fix the pyproject.toml dep " - f"to '>=0.1.0a1,<0.2'." - ) - return errors + for dependency in sorted(expected_names): + req = by_name.get(dependency) + if req is None: + errors.append(f"{artifact.name}: missing Requires-Dist: {dependency}") + elif str(req.specifier) != expected_spec: + errors.append( + f"{artifact.name}: {dependency} has {req.specifier}; expected >={version},<{upper}" + ) + return (name, version), errors def main() -> int: + parser = ArgumentParser() + parser.add_argument( + "--package", + choices=sorted(EXPECTED_EDGES), + help="validate exactly one distribution pair (for isolated TestPyPI RC jobs)", + ) + args = parser.parse_args() dist = Path(__file__).resolve().parent.parent / "dist" - if not dist.exists(): - print( - "ERROR: dist/ directory does not exist. Run `uv build --all` first.", - file=sys.stderr, - ) - return 1 - wheels = sorted(dist.glob("*.whl")) - if not wheels: - print("ERROR: dist/ contains no wheels.", file=sys.stderr) + artifacts = sorted([*dist.glob("*.whl"), *dist.glob("*.tar.gz")]) if dist.exists() else [] + if not artifacts: + print("ERROR: dist/ contains no release artifacts; run `uv build --all`.", file=sys.stderr) return 1 - all_errors: list[str] = [] - checked = 0 - for wheel in wheels: - wheel_errors = check_wheel(wheel) - if wheel.name.split("-")[0] in REQUIRED_MOSTLYRIGHT_SPECIFIER: - checked += 1 - all_errors.extend(wheel_errors) - if all_errors: - for err in all_errors: - print(f"ERROR: {err}", file=sys.stderr) + + errors: list[str] = [] + seen: dict[tuple[str, Version], list[str]] = {} + for artifact in artifacts: + identity, artifact_errors = check_artifact(artifact) + errors.extend(artifact_errors) + if identity is not None: + kind = "wheel" if artifact.suffix == ".whl" else "sdist" + seen.setdefault(identity, []).append(kind) + + selected_names = {args.package} if args.package else set(EXPECTED_EDGES) + selected = { + identity: kinds for identity, kinds in seen.items() if identity[0] in selected_names + } + versions = {version for _, version in selected} + if not args.package and len(versions) != 1: + errors.append(f"release artifacts must share exactly one version; found {sorted(versions)}") + for name in sorted(selected_names): + matches = [(identity, kinds) for identity, kinds in selected.items() if identity[0] == name] + if len(matches) != 1: + errors.append(f"expected exactly one {name} wheel/sdist pair; found {len(matches)}") + elif sorted(matches[0][1]) != ["sdist", "wheel"]: + errors.append(f"{name}: expected wheel and sdist; found {sorted(matches[0][1])}") + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) return 1 - print(f"OK: checked {checked} sibling-package wheel(s); all pins present.") + if args.package: + print(f"OK: validated the {args.package} wheel/sdist pair.") + else: + print("OK: validated the complete four-package wheel/sdist release matrix.") return 0 if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) diff --git a/scripts/publish-npm-if-needed.mjs b/scripts/publish-npm-if-needed.mjs new file mode 100644 index 00000000..b093a063 --- /dev/null +++ b/scripts/publish-npm-if-needed.mjs @@ -0,0 +1,39 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; + +const [name, version, tarball, tag] = process.argv.slice(2); +if (!name || !version || !tarball || !tag) { + throw new Error("usage: publish-npm-if-needed.mjs NAME VERSION TARBALL TAG"); +} + +let remoteSha; +try { + remoteSha = execFileSync("npm", ["view", `${name}@${version}`, "dist.shasum"], { + encoding: "utf8", + }).trim(); +} catch { + remoteSha = ""; +} + +if (remoteSha) { + const localSha = createHash("sha1") + .update(await readFile(tarball)) + .digest("hex"); + if (remoteSha !== localSha) { + throw new Error(`existing ${name}@${version} shasum differs: ${remoteSha} != ${localSha}`); + } + const tags = JSON.parse( + execFileSync("npm", ["view", name, "dist-tags", "--json"], { encoding: "utf8" }), + ); + if (tags[tag] !== version) { + throw new Error( + `existing ${name}@${version} is not tagged ${tag} (registry has ${tags[tag] ?? "missing"}); OIDC trusted publishing cannot mutate dist-tags outside npm publish`, + ); + } + console.log(`verified ${name}@${version} and dist-tag ${tag}`); +} else { + execFileSync("npm", ["publish", tarball, "--access", "public", "--tag", tag], { + stdio: "inherit", + }); +} diff --git a/scripts/release-ts-preflight.mjs b/scripts/release-ts-preflight.mjs index c81e9be8..c5a23d52 100644 --- a/scripts/release-ts-preflight.mjs +++ b/scripts/release-ts-preflight.mjs @@ -46,7 +46,7 @@ async function readJson(path) { } async function writeJson(path, value) { - await writeFile(path, JSON.stringify(value, null, 2) + "\n", "utf8"); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } async function main() { @@ -86,14 +86,13 @@ async function main() { const pj = await readJson(path); if (pj.peerDependencies?.[PEER_KEY] === undefined) { console.error( - `Preflight aborted: ${pkg}/package.json missing peerDependencies['${PEER_KEY}']. ` + - "If the package was renamed again, update PEER_KEY in scripts/release-ts-preflight.mjs.", + `Preflight aborted: ${pkg}/package.json missing peerDependencies['${PEER_KEY}']. If the package was renamed again, update PEER_KEY in scripts/release-ts-preflight.mjs.`, ); process.exit(1); } const old = pj.peerDependencies[PEER_KEY]; - pj.peerDependencies[PEER_KEY] = `^${expected}`; - console.log(`Rewrote ${pkg} peerDependencies['${PEER_KEY}']: ${old} → ^${expected}`); + pj.peerDependencies[PEER_KEY] = `~${expected}`; + console.log(`Rewrote ${pkg} peerDependencies['${PEER_KEY}']: ${old} → ~${expected}`); await writeJson(path, pj); } diff --git a/scripts/verify_pypi_existing.py b/scripts/verify_pypi_existing.py new file mode 100644 index 00000000..be2aa246 --- /dev/null +++ b/scripts/verify_pypi_existing.py @@ -0,0 +1,63 @@ +"""Fail if an existing PyPI version differs from locally validated artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path + + +def validate_existing(remote: dict[str, str], local: dict[str, str]) -> None: + """Accept an absent/partial matching upload; reject conflicts or foreign files.""" + unexpected = remote.keys() - local.keys() + if unexpected: + raise ValueError(f"unexpected remote artifacts: {sorted(unexpected)}") + conflicts = { + filename: (remote[filename], local[filename]) + for filename in remote.keys() & local.keys() + if remote[filename] != local[filename] + } + if conflicts: + raise ValueError(f"artifact hash conflicts: {conflicts}") + + +def main() -> int: + if len(sys.argv) < 4: + raise SystemExit("usage: verify_pypi_existing.py DIST VERSION ARTIFACT...") + distribution, version, *raw_paths = sys.argv[1:] + paths = [Path(path) for path in raw_paths] + base_url = os.environ.get("PYPI_JSON_BASE_URL", "https://pypi.org/pypi").rstrip("/") + url = f"{base_url}/{distribution}/{version}/json" + try: + with urllib.request.urlopen(url) as response: + payload = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + print(f"{distribution} {version} is not yet published") + return 0 + raise + + remote = {entry["filename"]: entry["digests"]["sha256"] for entry in payload["urls"]} + local = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in paths} + try: + validate_existing(remote, local) + except ValueError as exc: + print( + f"ERROR: existing PyPI artifacts differ for {distribution} {version}: {exc}", + file=sys.stderr, + ) + return 1 + missing = sorted(local.keys() - remote.keys()) + print( + f"verified existing PyPI artifacts for {distribution} {version}; " + f"missing files will be uploaded: {missing}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 188ddf65..946736fa 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -21,7 +21,6 @@ from __future__ import annotations -import re import tomllib from pathlib import Path @@ -177,25 +176,10 @@ def test_markets_pins_core_to_active_major() -> None: # client with mostlyrightmd-markets. The pin must constrain to the # active major (>=1.0.0,<2.0 at v1.x; >=0.1.0,<0.2 at v0.1.x). runtime_deps = _runtime_deps("markets") - assert any( - d.startswith("mostlyrightmd") - and (">=1.0.0,<2.0" in d.replace(" ", "") or ">=0.1.0,<0.2" in d.replace(" ", "")) - for d in runtime_deps - ), ( - "mostlyrightmd-markets runtime deps must constrain mostlyright to " - "the active major (>=1.0.0,<2.0 or >=0.1.0,<0.2) — see PKG-03" - ) - - -#: A core pin is "active-major-bounded" when it floors anywhere on the active -#: major and caps below the next major: ``>=1.x.y,<2.0`` on the v1 line (the -#: floor may be bumped above 1.0.0 — e.g. weather 1.10.0 raises it to >=1.10.0 -#: so a partial upgrade can't import a symbol that only exists in the newer -#: core), or ``>=0.1.x,<0.2`` on the pre-1.0 line. Either way a legacy 0.0.x -#: client and a cross-major mix are forbidden (PKG-03). -_ACTIVE_MAJOR_CORE_PIN_RE = re.compile( - r"^mostlyrightmd>=1\.\d+\.\d+,<2\.0$|^mostlyrightmd>=0\.1\.\d+,<0\.2$" -) + version = _project("core")["version"] + major, minor, _ = (int(part) for part in version.split(".")) + expected = f"mostlyrightmd>={version},<{major}.{minor + 1}.0" + assert expected in [d.replace(" ", "") for d in runtime_deps] def test_weather_pins_core_to_active_major() -> None: @@ -204,11 +188,11 @@ def test_weather_pins_core_to_active_major() -> None: # sit ABOVE 1.0.0 (weather 1.10.0 bumps it to >=1.10.0 because # mostlyright.weather.cwop imports NoCWOPDataError, which only exists in # core 1.10.0) so long as the cap stays <2.0 (same active major). - runtime_deps = _runtime_deps("weather") - assert any(_ACTIVE_MAJOR_CORE_PIN_RE.match(d.replace(" ", "")) for d in runtime_deps), ( - "mostlyrightmd-weather runtime deps must constrain mostlyright to the " - "active major (>=1.x.y,<2.0 or >=0.1.x,<0.2) — see PKG-03" - ) + runtime_deps = [d.replace(" ", "") for d in _runtime_deps("weather")] + version = _project("core")["version"] + major, minor, _ = (int(part) for part in version.split(".")) + expected = f"mostlyrightmd>={version},<{major}.{minor + 1}.0" + assert expected in runtime_deps def test_core_research_extra_pins_weather_to_active_major() -> None: @@ -225,14 +209,7 @@ def test_core_research_extra_pins_weather_to_active_major() -> None: ] assert weather, "mostlyrightmd[research] must depend on mostlyrightmd-weather" - def _floor_ge_1_6_active_major(dep: str) -> bool: - if ",<2.0" not in dep: - return False - m = re.search(r">=1\.(\d+)\.", dep) - return m is not None and int(m.group(1)) >= 6 - - assert any(_floor_ge_1_6_active_major(d) for d in weather), ( - "mostlyrightmd[research] extra must constrain mostlyrightmd-weather to the " - "active major (<2.0) with a >=1.6.0 floor (fetch_open_meteo variables= kwarg); " - f"got {weather}" - ) + version = _project("core")["version"] + major, minor, _ = (int(part) for part in version.split(".")) + expected = f"mostlyrightmd-weather>={version},<{major}.{minor + 1}.0" + assert expected in weather diff --git a/tests/test_release_integrity.py b/tests/test_release_integrity.py new file mode 100644 index 00000000..51a3b012 --- /dev/null +++ b/tests/test_release_integrity.py @@ -0,0 +1,32 @@ +"""Release recovery must distinguish partial uploads from immutable conflicts.""" + +from __future__ import annotations + +import pytest + +from scripts.verify_pypi_existing import validate_existing + +LOCAL = {"package.whl": "wheel-sha", "package.tar.gz": "sdist-sha"} + + +@pytest.mark.parametrize( + "remote", + [ + {}, + LOCAL, + {"package.whl": "wheel-sha"}, + {"package.tar.gz": "sdist-sha"}, + ], +) +def test_matching_absent_complete_or_partial_release_is_resumable(remote: dict[str, str]) -> None: + validate_existing(remote, LOCAL) + + +def test_hash_conflict_fails_loudly() -> None: + with pytest.raises(ValueError, match="hash conflicts"): + validate_existing({"package.whl": "different"}, LOCAL) + + +def test_unexpected_remote_artifact_fails_loudly() -> None: + with pytest.raises(ValueError, match="unexpected remote artifacts"): + validate_existing({"foreign.whl": "sha"}, LOCAL) diff --git a/uv.lock b/uv.lock index c903dfd1..c416f80f 100644 --- a/uv.lock +++ b/uv.lock @@ -1634,6 +1634,7 @@ source = { editable = "packages/core" } dependencies = [ { name = "httpx" }, { name = "jsonschema" }, + { name = "pandas" }, { name = "tzdata", marker = "sys_platform == 'win32'" }, ] @@ -1660,6 +1661,7 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.21" }, { name = "mostlyrightmd-weather", marker = "extra == 'research'", editable = "packages/weather" }, { name = "narwhals", marker = "extra == 'polars'", specifier = ">=1.20,<2.0" }, + { name = "pandas", specifier = ">=2.2,<4.0" }, { name = "pandas", marker = "extra == 'parquet'", specifier = ">=2.2,<4.0" }, { name = "pandas", marker = "extra == 'polars'", specifier = ">=2.2,<4.0" }, { name = "pandas", marker = "extra == 'research'", specifier = ">=2.2,<4.0" },