diff --git a/.gitattributes b/.gitattributes index 4a0d86f..a0398de 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ -scripts/* linguist-documentation +tests/fixtures/** -text +*.db binary diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 0000000..40b36eb --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,52 @@ +name: Live-source canary + +# The test suite runs fully offline against captured payloads, so upstream +# format or availability changes are invisible to CI. This canary hits the +# real services on a schedule and fails loudly when they drift. + +on: + schedule: + - cron: "0 5 8 * *" + workflow_dispatch: + +jobs: + canary: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install package + run: pip install -e . + - name: Fetch live observations and validate shape + run: | + python - << 'PY' + from eeweather.sources.ghcnh.source import GHCNhSource + from eeweather.build.refresh import STATION_LIST_URL, INVENTORY_URL + import requests + + # one real station-year through the live NCEI api + df = GHCNhSource().fetch_year( + "USW00023234", 2024, ("temperature", "relative_humidity") + ) + assert len(df) > 8000, "unexpectedly sparse response: {} rows".format(len(df)) + assert list(df.columns) == ["temperature", "relative_humidity"] + mean = float(df.temperature.mean()) + assert 5 < mean < 25, "implausible SFO mean temperature: {}".format(mean) + + # refresh inputs + for url in (STATION_LIST_URL, INVENTORY_URL): + response = requests.get(url, stream=True, timeout=120) + response.raise_for_status() + + # one archive file per normals source + from eeweather.sources.tmy3.source import TMY3Source + from eeweather.sources.cz2010.source import CZ2010Source + for source in (TMY3Source(), CZ2010Source()): + ts = source.fetch("USW00023152") + assert len(ts) == 8760, "{}: unexpected typical year length".format( + source.name + ) + print("canary green") + PY diff --git a/.github/workflows/refresh-registry.yml b/.github/workflows/refresh-registry.yml new file mode 100644 index 0000000..bf956cd --- /dev/null +++ b/.github/workflows/refresh-registry.yml @@ -0,0 +1,56 @@ +name: Refresh registry + +on: + schedule: + # monthly; GHCNh inventory updates continuously, ratings decay yearly + - cron: "0 6 2 * *" + workflow_dispatch: + +jobs: + refresh: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install package + run: pip install -e .[dev] + - name: Refresh live registry content + run: python -m eeweather.build + # refresh PRs opened with GITHUB_TOKEN do not trigger pull_request + # workflows, so the suite must pass here, against the refreshed data + - name: Run the test suite against the refreshed data + run: pytest tests/ -o addopts= + # clients' auto-update downloads these assets (CDN-served) instead + # of each rebuilding from NOAA; the wheel snapshot updates via PR + - name: Publish refreshed data to the rolling release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create registry-latest \ + --title "Latest registry data" \ + --notes "Refreshed registry data, published by the scheduled workflow. Client auto-update downloads these files when local data is stale." \ + || true + gh release upload registry-latest \ + eeweather/sources/ghcnh/ghcnh.db \ + eeweather/registry/identifiers.db \ + --clobber + - name: Open pull request with the refreshed registry + uses: peter-evans/create-pull-request@v6 + with: + commit-message: Refresh registry inventory and quality ratings + title: Refresh registry inventory and quality ratings + body: | + Scheduled refresh of the packaged registry database: GHCNh + observation inventory, newly listed stations, and build-time + quality ratings. Static content (zones, places, identifiers, + normals) is untouched. + branch: automation/refresh-registry + delete-branch: true + add-paths: | + eeweather/sources/ghcnh/ghcnh.db + eeweather/registry/identifiers.db diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c656ae6..9176f62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,30 @@ on: - cron: "0 0 1 * *" jobs: + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install tox + run: pip install tox + - name: Enforce the coverage floor + run: tox -e coverage + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install ruff + run: pip install ruff + - name: Lint (logic checks only) + run: ruff check eeweather tests + build: strategy: fail-fast: false diff --git a/.gitignore b/.gitignore index 5a85b27..6daa530 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ __pycache__/ _build/ # packaging and distribution -build/ +/build/ dist/ # coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index b874398..e9305ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,59 +4,124 @@ Changelog Development ----------- +This release is a redesign; the public API is not compatible with 0.3.x. + * Observed weather data is served from NOAA GHCNh; ISD and GSOD stopped - receiving data 2025-08-27. Overlap validation against ISD history shows - median hourly deviation of 0.0001 degrees C - (`scripts/validate_ghcnh_against_isd.py`). -* New api: `load_data(usaf_id, start, end, frequency, variables)` returns a - DataFrame and warnings, replacing the `load_isd_*`/`load_gsod_*` family; - `ISDStation` is renamed `WeatherStation`. GHCNh variables beyond - temperature (dew point, relative humidity, wind speed, ...) are available - through `variables`. -* Station registry maps each station to its GHCNh id - (`get_ghcn_id(usaf_id)` for one station, `get_ghcn_ids()` for the - registry); 349 stations with no - GHCNh counterpart are removed. Each station records the first and last - year its GHCNh record has observations (`ghcn_first_year`, - `ghcn_last_year`); about a fifth of the registry, nearly all low - quality, has no observations after 2023. -* Station quality ratings are computed from the GHCNh inventory (same - rule as before: every month of the last five full years above 600 - observations is high, above 360 is medium). Tiers move for about a - fifth of stations, mostly upgrades of stations the retired ISD - inventory undercounted: 1858 high / 393 medium / 2246 low. -* Quality can be rated for the period being requested: - `get_station_quality(usaf_id, start, end)` and - `rank_stations(..., rating_period=(start, end))` rate stations over - the five calendar years ending two years after the period's last - date (sliding back to end no later than the last full year), so - historical requests rank stations by their reliability in that era. - The packaged database carries the monthly counts (`ghcn_inventory`). -* Caching uses new ghcnh-* keys; ISD-era cache entries are never served. -* Deleted: FTP fetch code, ISD/GSOD parsing, filename helpers and CLI - commands, sphinx docs (documentation moves to opendsm.energy). -* Switched to https api over FTP for temperature data fetching -* Remove deprecated `typing` dependency -* Switch from snapshottest to syrupy for snapshot testing. -* Update internal database (2025-03-12). -* Modernize packaging: pyproject.toml with hatchling replaces setup.py, - Pipfile, and MANIFEST.in; python >=3.10. -* Add test workflow (python/os matrix) and tox environments for current - python versions. -* Rewrite key-value cache on stdlib sqlite3; sqlalchemy is no longer used - or required. Cache urls must have the sqlite:/// form. -* Replace retry and attrs dependencies with stdlib equivalents; fix - pkg_resources import broken on setuptools >=81. -* Warn when loaded data is empty, truncated, or contains a multi-day - internal gap. Requests ending after a station's last available - observation previously returned NaN-padded series silently. -* Fix crash (`No objects to concatenate`) when all years in a requested - range are missing and `error_on_missing_years=False`. -* Loaders always return a series covering the full requested range. -* Add `error_on_missing_years` to daily ISD/GSOD loaders; default is True - for all loaders, matching the ISDStation method default. -* Tests run fully offline against captured NCEI access api payloads; - add coverage floor. + receiving data 2025-08-27. Overlap validation against ISD history + showed a median hourly deviation of 0.0001 degrees C. +* The primary entry point is `WeatherLocation(latitude, longitude)` + (construct from a ZIP code with `from_place("zcta", code)`): weather at + a point, estimated by configurable sources. `location.candidates()` + ranks nearby stations and `location.zones` gives its climate zones. + Provenance is the reproducibility mechanism: a location pins itself to + the stations resolved at first load, and `to_dict`/`to_json` with + `from_dict`/`from_json` carry that state across processes, so later + loads (e.g. a reporting period after a baseline) use the same station + per source. +* `WeatherStation` is keyed by the station's GHCN id. Other identifier + systems are aliases that translate to it: `from_usaf`, + `from_wban` (most-recent mapping wins), `from_icao`, and the generic + `from_id(namespace, external_id)`; `WeatherStation.translate(ids, + from_namespace, to_namespace)` converts in bulk and + `WeatherStation.search(country=, subdivision=, has_sources=)` + enumerates the registry as a DataFrame. +* One loading verb: `load_data(start, end, frequency, variables, + source=)` serves observations and typical years alike, returning + `(DataFrame, warnings)` with per-source provenance on the frame's + attrs and on the station/location object. Each requested variable + routes to the first configured source that serves it; typical-year + sources (`"tmy3"`, `"cz2010"`) are reachable only by explicit + `source=` pin and tile onto requested calendar years by + month-day-hour with NaN leap days. Partial coverage returns NaN plus + warnings; a pinned source with nothing at all raises + `DataNotAvailableError`. +* Variables have a canonical curated vocabulary with fixed units + (temperature, dew point temperature [degC], relative humidity [%], + wind speed [m/s], station-level pressure [hPa], visibility [km]) and + a per-variable aggregation governing resampling. `load_data`'s + `frequency` uses pandas' own offset nomenclature, bound by delegation + (pandas parses the alias, so its spellings and deprecations apply + automatically): coarser-than-hourly frequencies (`D`, `W`, `MS`, + `YS`, ...) aggregate each variable by its declared aggregation — + point-in-time variables average, accumulation variables + (`aggregation="sum"`) add up within each period and are never + gap-interpolated — and finer-than-hourly frequencies (`30min`, + `20min`, ...; they must divide the hour evenly) interpolate + point-in-time variables linearly between hourly values and spread + accumulations evenly, never crossing a missing hour. `eeweather.sources.variables()` lists them + and which sources serve each. Arbitrary native GHCNh dataTypes are no + longer passed through. +* Custom data plugs in through public protocols: station-keyed feeds + (`eeweather.sources.Feed` — keyed by any translatable id system, with + opt-out caching), typical-year sources (`eeweather.sources. + NormalsSource`), and location-keyed gridded sources + (`eeweather.sources.Source`). `eeweather.sources.register(source)` + makes a custom feed or normals source nameable — usable as a string + in preference tuples and `source=` pins, and rebuildable by location + serialization — and accepts vocabulary entries + (`register(..., vocabulary=(Variable(name, unit, description, + aggregation),))`) + for variables the canonical vocabulary lacks, which then route and + validate like canonical ones. Canonical names are reserved and + definitions must agree across sources, so a registered variable's + unit is frozen at first registration. +* Packaged data is split by ownership: the registry holds the + identifier crosswalk (`identifiers.db`) and regional geography packs + (`geography_us.db`: zone geometries, ZCTA places, zone assignments); + each source packages its own facts beside its adapter (the GHCNh + station catalog with zone assignments, observation inventory, and + quality ratings; TMY3/CZ2010 archive station lists). 349 ISD + stations with no GHCNh counterpart were removed during the migration. +* The live packaged data (station catalog, observation inventory, + quality ratings, identifier aliases) keeps itself current without + package releases: when it is more than six months old — quality + rating windows advance when a calendar year completes, so six months + caps the lag behind that yearly step — loading data starts a + background update into the platform user data directory, which takes + precedence over the wheel's copies in new processes + (`EEWEATHER_AUTO_UPDATE=0` disables; `python -m + eeweather.registry.update` updates on demand, `--clear` reverts). + Updates prefer the ready-made pack a scheduled workflow publishes to + the repository's rolling release — CDN-served, so fleets of any size + cost NOAA nothing and clients skip the local rebuild — and fall back + to rebuilding from the live NOAA files when the channel is + unreachable, implausible, stale, or no newer than the local data. + Concurrent workers coordinate through an atomic claim file (one + attempt per machine per day) and the update thread defers network + traffic for a minute, so short-lived pipeline workers exit without + fetching anything. + New stations must sit within 10 km of a known zone geometry and show + recent observations to be appended; implausibly small upstream files + abort an update, leaving the previous data in place. A scheduled + workflow keeps the wheel's snapshot current via the same refresh. +* Station quality ratings come from the GHCNh inventory (every month of + the rating window above 600 observations is high, above 360 medium); + `rank_stations(..., rating_period=(start, end))` rates stations over + the five calendar years ending two years after the period's last date + (sliding back to the last full year), so historical requests rank + stations by their reliability in that era. +* Exceptions live in `eeweather.exceptions` (not re-exported at the + root), carry their messages through `str()`, and cover distinct + recovery paths: `UnrecognizedStationError`, `UnrecognizedPlaceError`, + `AmbiguousIdentifierError`, `DataNotAvailableError`, + `NoQualifiedStationError`. Non-UTC datetimes raise `ValueError`. +* The GHCNh fetch reuses one session, retries only connection and + server errors with backoff, and raises client errors immediately. + Loads warn when data is empty, starts late, ends early, or contains a + multi-day internal gap. +* The shared cache is a stdlib-sqlite store at the platform user cache + dir by default (`EEWEATHER_CACHE_URL` env var and + `eeweather.cache.set_path()` override; `eeweather.cache.clear()` + empties it). ISD-era and 0.3.x cache entries are never served. +* Public type hints ship with a `py.typed` marker. +* Modernized packaging: pyproject.toml with hatchling replaces setup.py, + Pipfile, and MANIFEST.in; python >=3.10; sqlalchemy, click, and pytz + are no longer dependencies; platformdirs and shapely are. The CLI, + plotting helpers, sphinx docs (documentation moves to opendsm.energy), + and FTP-era fetch code are deleted. +* Tests run fully offline against captured NCEI access api payloads, + with a python/os matrix workflow, tox environments, ruff lint, and a + 97% coverage floor. 0.3.29 ------ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e271246..ad28b1b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,8 @@ Guidelines ---------- * Make sure you follow PEP 008 style guide conventions. +* Adding a weather-data source: see `docs/adding-a-source.md`. +* Adding a variable to the vocabulary: see `docs/expanding-the-vocabulary.md`. * Commit messages should start with a capital letter ("Updated models", not "updated models"). * Write new tests and run old tests! Make sure that % test coverage does not decrease. @@ -30,26 +32,19 @@ Releasing Updating the internal db ------------------------ -The interal source file database needs to be periodically updated. - -TODO: automate this! - -Here are is the current process to use to update the internal db. +The scheduled `refresh-registry` workflow updates the live parts of the +packaged registry (GHCNh station list, observation inventory, quality +ratings) monthly and opens a pull request; review its diff, run the test +suite (some registry pins move as the registry ages), update snapshots as +warranted, and merge. To run the refresh by hand: ``` -git checkout master -git pull -git checkout -b feature/update-db-YYYY-MM-DD - -# Rebuild the internal DB -docker-compose build -docker-compose run --rm eeweather rebuild-db -git commit -am "Rebuild db YYYY-MM-DD" -vim CHANGELOG.md # write "Update internal database (YYYY-MM-DD)." - -# Update the tests and snapshots if everything looks good -docker-compose run --rm test -docker-compose run --rm test --snapshot-update +python -m eeweather.build +pytest ``` -Then make a PR and review as normal. Rinse and repeat as necessary. +Static content (zone geometries and assignments, places, archive +station lists, identifier mappings) is carried forward from previously +packaged data; `python -m eeweather.build --migrate OLD DESTDIR` builds +the per-ownership data files (identifiers, geography pack, one per +source) from a historical single-file database. diff --git a/Dockerfile b/Dockerfile index f548d42..448ba54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,23 @@ -FROM python:3.12-bookworm +# syntax=docker/dockerfile:1.7 +FROM python:3.12-slim AS app -RUN apt-get update \ - && apt-get install -yqq \ - # sphinxcontrib-spelling dependency - libenchant-2-dev \ - # geo libraries - binutils libproj-dev gdal-bin libgeos-dev \ - # unzip for rebuilding metadata.db - unzip \ - # node for mapshaper - nodejs npm \ - # for access to metadata.db - sqlite3 libsqlite3-dev +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* -RUN npm install -g mapshaper +# uv (fast installer) +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -COPY pyproject.toml README.md LICENSE /app/ -COPY eeweather/ /app/eeweather -RUN set -ex && pip install -e /app[dev] +ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy \ + PYTHONUNBUFFERED=1 PIP_DISABLE_PIP_VERSION_CHECK=1 WORKDIR /app + +# deps layer (cacheable): resolve and install dependencies only +COPY pyproject.toml README.md LICENSE /app/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip compile pyproject.toml --extra dev -o /tmp/requirements.txt && \ + uv pip install --system -r /tmp/requirements.txt + +COPY eeweather/ /app/eeweather +RUN uv pip install --system --no-deps -e /app diff --git a/README.md b/README.md index 4640579..8f8faff 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,71 @@ -# EEweather: Weather station wrangling for EEmeter +# EEweather: Weather for energy-efficiency modeling [![License](https://img.shields.io/github/license/opendsm/eeweather.svg)](https://github.com/opendsm/eeweather) [![PyPI Version](https://img.shields.io/pypi/v/eeweather.svg)](https://pypi.python.org/pypi/eeweather) --- -**EEweather** — tools for matching to and fetching data from NCEI GHCNh, TMY3, or CZ2010 weather stations. +**EEweather** answers "what was the weather here?" — it matches locations to +weather stations, fetches observed and typical-year data, and returns +analysis-ready frames with warnings and provenance. -EEweather comes with a database of weather station metadata, ZCTA metadata, and GIS data that makes it easier to find the right weather station to use for a particular ZIP code or lat/long coordinate. - -[Documentation lives at opendsm.energy.](https://opendsm.energy) +Documentation lives in [docs/](docs/) (publishing to +[opendsm.energy](https://opendsm.energy)). ## Usage ```python -import datetime +from datetime import datetime, timezone + import eeweather -station = eeweather.WeatherStation("722880") +location = eeweather.WeatherLocation(34.2, -118.4) # or .from_place("zcta", "91104") +df, warnings = location.load_data( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 12, 31, tzinfo=timezone.utc), +) +location.provenance # which station served each variable, and how far away + +saved = location.to_json() # coordinates, sources, and the stations resolved at load +location = eeweather.WeatherLocation.from_json(saved) # later loads (e.g. a reporting + # period) reuse the same stations + +station = eeweather.WeatherStation("USW00023152") # or by registry id +station = eeweather.WeatherStation.from_usaf("722880") # or by historical ids df, warnings = station.load_data( - datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), - datetime.datetime(2024, 12, 31, tzinfo=datetime.timezone.utc), - frequency="hourly", - variables=("temperature",), + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 12, 31, tzinfo=timezone.utc), + variables=("temperature", "relative_humidity"), +) +typical, _ = station.load_data( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 12, 31, tzinfo=timezone.utc), + source="tmy3", # typical-year data by explicit pin ) ``` ## Installation -EEweather is a python package and can be installed with pip. - ``` $ pip install eeweather ``` -## Supported Sources of Weather Data - -- NOAA Global Historical Climatology Network hourly (GHCNh) -- NREL Typical Meteorological Year 3 (TMY3) -- California Energy Commission 1998-2009 Weather Normals (CZ2010) - ## Features -- Match by ZIP code (ZCTA) or by lat/long coordinates -- Use user-supplied weather station mappings -- Match within climate zones - - IECC Climate Zones - - IECC Moisture Regimes - - Building America Climate Zones - - California Building Climate Zone Areas -- User-friendly SQLite database of metadata compiled from primary sources - - US Census Bureau (ZCTAs, county shapefiles) - - Building America climate zone county lists - - NOAA NCEI Integrated Surface Database Station History - - NOAA GHCNh station list - - NREL TMY3 site -- Plot maps of outputs +- One loading verb over pluggable sources: GHCNh observations (NOAA), TMY3 + (NREL) and CZ2010 (CEC) typical years; every load returns an aligned UTC + frame plus data-quality warnings and per-source provenance +- A packaged station registry with opaque ids and identifier translation + (USAF, WBAN, ICAO), precomputed climate-zone assignments (IECC, Building + America, California), ZCTA place codes, and per-source observation + inventory +- Station matching: ranked candidates for any point with zone, quality, + distance, and availability filters (`location.candidates()`), plus + data-sufficiency selection +- Extension protocols for custom data: station-keyed feeds (e.g. a BigQuery + table keyed by any translatable id system) and location-keyed gridded + sources +- A shared sqlite weather cache (`eeweather.cache.set_path`/`clear`) ## Contributing @@ -79,33 +89,38 @@ Run tests on multiple python versions: $ tox ``` -## Use with Docker - -To use with docker-compose, use the following: - -Run a tutorial notebook (copy link w/ token, open tutorial.ipynb): +Or with Docker: ``` -$ docker-compose up jupyter +$ docker compose run --rm test ``` -Open a shell: +## Registry updates -``` -$ docker-compose run --rm shell -``` - -Run tests: +The packaged registry (GHCNh station list, observation inventory, quality +ratings, identifier aliases) keeps itself current: when the live data is +more than six months old, loading data starts a background update into +the platform user data directory, and those copies take precedence over +the wheel's in new processes. Updates download the ready-made files a +scheduled workflow publishes to the repository's rolling release +(CDN-served, so any number of clients costs NOAA nothing and no client +rebuilds locally) and fall back to rebuilding directly from the live +NOAA files when that channel is unreachable or stale, so they keep +working even if the repository goes dormant. Concurrent workers on a +machine coordinate through an atomic claim file (one attempt per day, +however many race), and the update thread waits a minute before touching +the network, so short-lived pipeline workers exit without generating +traffic. Set `EEWEATHER_AUTO_UPDATE=0` to suppress the traffic entirely, +or manage it directly with: ``` -$ docker-compose run --rm test +$ python -m eeweather.registry.update # refresh on demand +$ python -m eeweather.registry.update --clear # revert to packaged data ``` -Run the CLI: - -``` -$ docker-compose run --rm eeweather --help -``` +A scheduled workflow runs the same refresh monthly and opens a pull +request so the wheel's snapshot stays current; maintainers can rebuild +all packaged data with `python -m eeweather.build`. ## Notice Regarding CZ2010 Data @@ -115,11 +130,3 @@ The non-U.S. data cannot be redistributed for commercial purposes. Re-distribution of these data by others must provide this same notification. See [further explanation](http://weather.whiteboxtechnologies.com/faq#Q12/) here. - -## Metadata Yearly Updates - -Every year, the metadata database needs to be updated. This can be done by running: - -``` -docker-compose run --rm eeweather rebuild-db -``` diff --git a/docker-compose.yml b/docker-compose.yml index a1c706d..dcda298 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,7 @@ -version: '3' services: shell: build: . - image: eeweather_shell + image: eeweather stdin_open: true tty: true entrypoint: /bin/sh @@ -10,26 +9,14 @@ services: - .:/app test: - image: eeweather_shell + image: eeweather entrypoint: py.test -n0 volumes: - .:/app - /app/tests/__pycache__/ - eeweather: - image: eeweather_shell - entrypoint: eeweather - volumes: - - .:/app - - python: - image: eeweather_shell - entrypoint: python - volumes: - - .:/app - - blacken: - image: eeweather_shell - entrypoint: black . + uv: + image: eeweather + entrypoint: uv volumes: - .:/app diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..29ed91b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,21 @@ +# Documentation + +User documentation (source material for the eeweather section of +[opendsm.energy](https://opendsm.energy/)): + +- [methodology.md](methodology.md) — motivation and how the library works: + locations and stations, sources and routing, the variable vocabulary, + provenance and reproducibility, the packaged registry and its updates. +- [matching.md](matching.md) — station matching and data quality: ranking, + filters, the distance cap and disqualification, quality ratings and rating + periods. +- [example.md](example.md) — an end-to-end walkthrough with real outputs. +- [api.md](api.md) — the public API, surface by surface. +- [references.md](references.md) — data sources, methods, related projects. + +Contributor documentation: + +- [adding-a-source.md](adding-a-source.md) — the source contracts, built-in + package anatomy, and the no-fork path for external sources. +- [expanding-the-vocabulary.md](expanding-the-vocabulary.md) — adding a + canonical variable. diff --git a/docs/adding-a-source.md b/docs/adding-a-source.md new file mode 100644 index 0000000..9620275 --- /dev/null +++ b/docs/adding-a-source.md @@ -0,0 +1,164 @@ +# Adding a weather-data source + +A *source* is a provider of weather data. This guide covers the three +kinds the engine understands, the package anatomy a built-in source +follows, and the contracts every source must honor. `sources/ghcnh/`, +`sources/tmy3/`, and `sources/cz2010/` are the reference +implementations. + +## Pick the kind + +| kind | contract (in `sources/base.py`) | when | +|---|---|---| +| observations | `Feed` — `fetch_year(external_id, year, variables)` | station-keyed time series (an API, a warehouse table, a file mirror) | +| normals | `NormalsSource` — `fetch(station_id)` returning one typical year | typical-year products; served only by explicit `source=` pin, never routed | +| grid | `Source` — `estimate(latitude, longitude, start, end, ...)` | gridded/reanalysis products sampled at a point; no station involved. A grid source declares `name`, `kind`, and `variables` and is registerable via `eeweather.sources.register`, making it nameable in preference tuples, `source=` pins, and `to_dict`/`from_dict`; it participates in routing when `kind == "observations"`. Its `Provenance` carries `kind`, `source`, and `variables` with an open `payload` dict and no `station_id`. No built-in grid adapter ships yet, and `candidates()` remains station-only | + +The engine supplies everything around the fetch: per-year caching with +variable-union refresh, typical-year tiling (Feb 29 is NaN), range +alignment to an identical UTC index per request, resampling to any +pandas frequency by each variable's declared aggregation, gap warnings, +provenance records, and variable routing. A source +implements transport and parsing, nothing else. + +## The contracts every source honors + +- **Canonical variables and units.** Users never see native names or + units. Declare the canonical variables you serve (see + `sources/vocabulary.py`: name, unit, definition, aggregation) and + translate both + directions at ingest — request canonical → your native names, response + native → canonical columns in canonical units. If you serve a variable + the vocabulary lacks, add the vocabulary entry (one `Variable` line) + in the same change. Unit validation against real payloads is part of + acceptance. +- **Station identity.** Station ids are GHCN ids; every other id system + is an alias in the identifier crosswalk (`registry/identifiers.db`). + Never parse an id's format. If your data is keyed by another system, + declare `id_namespace` and the engine translates per fetch; if it uses + a system the crosswalk lacks, add alias rows for it. A source's + catalog, inventory, quality, and availability tables are keyed by + station id alone, and enumeration unions every registered catalog, so + a source covering a disjoint set of stations from the built-ins needs + no repo change to integrate. +- **Frames** are indexed by UTC timestamps; values the station did not + report are NaN; duplicate timestamps are averaged (accumulation + variables are instead summed within the hour). + +## Anatomy of a built-in source package + +``` +sources// + __init__.py docstring, DATA_PATH, one register_attachment call, + re-export of the class (no classes or functions here) + source.py class Source() + .db packaged sqlite data, if the source ships any +``` + +Registration attaches the data file to every registry connection under +the source's name, with roles describing its tables: + +- `catalog=True` — a `stations` table of station facts (station_id, + name, latitude, longitude, elevation, country, subdivision) and a + `station_zone` table of zone assignments. Facts for a station are + read from the first registered catalog that lists it, and enumeration + is the union across catalogs; both behaviors are pinned in + `tests/registry/test_db.py`. +- `inventory=True` — an `inventory` table of monthly observation counts + per (station_id, year); powers quality ratings. +- `quality=True` — a `quality` table of (station_id, rating). +- `availability=True` — a `stations` table listing which stations the + source can serve (archive lists); powers the `has_sources` ranking + filter, exposed as an `is_` column. + +Invariant (pinned in `tests/registry/test_schema.py`): every station id +in the crosswalk appears in at least one catalog. If your source +introduces stations the crosswalk lacks, its build step must insert +their identity rows and catalog facts together. + +## Steps + +1. Create the package with the anatomy above; subclass the kind's + contract and declare only what is distinctive (compare + `GHCNhSource`: `name`, `id_namespace`, `variables`; the normals + classes: `name` and an archive `_url`). +2. Register the built-in: add the class to `BUILTIN_SOURCES` in + `sources/engine.py` so the string name resolves. +3. Data: extend `build/schema.py` and `build/migrate.py` (or your own + build step) to produce `.db`; if its content decays, extend + `build/refresh.py` and the `refresh-registry` workflow's `add-paths`. +4. Vocabulary: add any new canonical variables, with units and + aggregation, to `sources/vocabulary.py`. +5. Tests, mirroring the package layout (`tests/sources/test_.py`): + transport-level tests against real captured payloads (fixtures in + `tests/fixtures/`, recorded from the live service — never + synthesized; synthetic data is for edge cases only), unit pins with + hardcoded expected values, error paths (empty responses, unknown + stations), and a schema entry in `tests/registry/test_schema.py` for + your data file. Add the source's expected rows to the + `sources_serving`/`variables()` pins in `tests/sources/test_sources.py`. +6. CHANGELOG entry describing the source and its variables. + +## External sources (no repo changes) + +Users plug in their own data without touching this repo: + +- Station-keyed (e.g. a BigQuery table): implement the `Feed` protocol + and pass the instance — `StationSource(dataset=my_feed)` or directly + in a location's `sources=(...)` tuple. Declare `cacheable = False` + for fast backends. The contract test pattern is `FixtureFeed` in + `tests/sources/test_sources.py`. +- Gridded: implement `Source.estimate` returning + `(DataFrame, warnings, provenance)` and declare `name`, `kind`, and + `variables` on the object (routing reads them). Register the instance + the same way as a feed to make it nameable in preference tuples, + `source=` pins, and location serialization. No built-in grid adapter + ships yet, and `candidates()` remains station-only. + +To make an external source nameable — usable as a string in preference +tuples and `load_data(source=...)`, and rebuildable by +`WeatherLocation.from_dict`/`from_json` — register the instance once at +import of your package: + +```python +eeweather.sources.register(MyFeed()) +location = eeweather.WeatherLocation(lat, lon, sources=("ghcnh", "my-feed")) +``` + +Registration accepts feeds and normals sources; built-in names cannot +be replaced. Optionally, register a sqlite file of your own +(`eeweather.registry.db.metadata_db_connection_proxy.register_attachment`, +same roles as built-in sources) to give your source availability +filtering, inventory-based quality, and ranking columns. + +A source serving variables the vocabulary lacks declares them at +registration; they become requestable, routable, and visible in +`eeweather.sources.variables()` like canonical entries: + +```python +eeweather.sources.register( + MyIrradianceFeed(), + vocabulary=( + eeweather.sources.Variable( + "ghi", "W/m2", "Global horizontal irradiance.", "mean" + ), + ), +) +``` + +The one-definition rule still holds: canonical names are reserved, and +when several sources register the same variable their definitions must +agree, so the first registration freezes the unit and aggregation. A +variable's ``aggregation`` ("mean", "sum", "min", or "max") controls +its resampling to other frequencies; +accumulation variables (``aggregation="sum"``, e.g. precipitation +depth) add up within each period and are never gap-interpolated. + +What an external source cannot do, by design: + +- Serve stations the registry does not know. Station-keyed loads + translate registry ids; a private sensor network enters as a + location-keyed `Source`, not a feed. + +Dependencies for external backends (cloud SDKs and the like) belong in +the user's package, never in eeweather's. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..c303c35 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,198 @@ +# API Reference + +The public surface is deliberately small: two classes at the root, plus the +`eeweather.sources`, `eeweather.cache`, and `eeweather.exceptions` namespaces. +Anything not documented here is internal and may change without notice. + +--- + +## `eeweather.WeatherLocation` + +Weather at a latitude/longitude. The location is the primary abstraction; +weather is estimated by the configured sources, and each requested variable +routes to the first source in the preference tuple that serves it. + +```python +WeatherLocation(latitude, longitude, sources=("ghcnh",), pins=None) +``` + +- `sources` — ordered source preference: built-in names (`"ghcnh"`), custom + source objects, or registered names. +- `pins` — station id per source name, bypassing matching. Normally not + written by hand: pins are captured automatically from provenance at load + time and round-trip through `to_dict`/`from_dict`. + +**`from_place(kind, code, sources=("ghcnh",))`** — construct from a coded +place, e.g. `from_place("zcta", "91104")` for a ZIP code tabulation area. + +**`load_data(start, end, frequency="h", variables=None, source=None, +ignore_disqualification=False, **load_kwargs)`** — weather between two +explicit-UTC datetimes (inclusive). Returns `(DataFrame, warnings)`. + +- `frequency` — any pandas offset alias (`"30min"`, `"h"`, `"D"`, `"W"`, + `"MS"`, `"YS"`). Coarser-than-hourly aggregates by each variable's + vocabulary aggregation; finer-than-hourly (must divide the hour evenly) + interpolates. +- `variables` — canonical names; defaults to `("temperature",)`; `"all"` is + every variable the configured sources serve. +- `source` — pin every variable to one source, bypassing routing. + Typical-year sources (`"tmy3"`, `"cz2010"`) are reachable only this way. +- `ignore_disqualification` — serve the best available station with a warning + instead of raising when none qualifies (see + [matching.md](matching.md)). +- `load_kwargs` — cache and network controls: `read_from_cache`, + `write_to_cache`, `fetch_from_web` (all default `True`). + +Routed loads never raise for missing data (NaN plus warnings); a pinned load +with nothing at all for the dates raises `DataNotAvailableError`. Records +provenance on the frame's `attrs["provenance"]` and on `self.provenance`, and +pins the location to the resolved stations. + +**`candidates(**filters)`** — ranked candidate stations as a DataFrame. +Filters: `match_zones`, `match_subdivision` (with `site_subdivision`), +`has_sources`, `minimum_quality`, `rating_period`, `max_distance_meters`, +`max_difference_elevation_meters` (with `site_elevation`). + +**`zones`** — `{system: zone_id}` for the point, e.g. +`{"iecc_climate_zone": "3", "ba_climate_zone": "Hot-Dry", ...}`. + +**`provenance`** — `{source_name: Provenance}` from the most recent load; +`None` before any load. Each `Provenance` records `kind` +(`"observations"`/`"normals"`), `source`, `variables`, `station_id`, +`distance_meters`, and a `payload` dict (empty unless a source adds +source-specific detail; the station fields are `None` for non-station +sources). + +**`pins`** — `{source_name: station_id}` the location replays. + +**`to_dict()` / `from_dict(data)` / `to_json()` / `from_json(text)`** — the +location's replay state (coordinates, source names, pins). A location rebuilt +from this state loads from the same station per source. Custom source objects +must be registered (`eeweather.sources.register`) to serialize. + +--- + +## `eeweather.WeatherStation` + +Weather at a known station, keyed by its registry id. All other identifier +systems are aliases that translate to it. + +```python +WeatherStation(station_id, sources=("ghcnh",), load_metadata=True) +``` + +- `sources` — ordered source preference for routing; every entry must be an + observation source (built-in name, custom source object, or registered + name). Duplicate names or a normals source (e.g. `"tmy3"`) raise at + construction — pin a normals source through `load_data(source=...)` instead. + +**Constructors** — `from_id(namespace, external_id)` plus the conveniences +`from_usaf(usaf_id)`, `from_wban(wban_id)`, `from_icao(icao_code)`. +Identifiers are normalized (case, zero-padding); a historically reused +identifier resolves to its most recent holder, and raises +`AmbiguousIdentifierError` only when genuinely unresolvable. + +**`search(country=None, subdivision=None, has_sources=())`** *(classmethod)* — +the station registry as a metadata DataFrame indexed by station id. + +**`translate(ids, from_namespace, to_namespace)`** *(static)* — bulk +identifier translation; returns `{input_id: (target_ids, ...)}` (mappings can +be one-to-many; unmapped ids are absent). + +**Attributes** — `id`, `ids` (`{namespace: [values]}`), `name`, `latitude`, +`longitude`, `elevation`, `country`, `subdivision`, `zones`, `quality` +(default-window rating), `inventory_years`; `json()` returns them as a dict. + +**`load_data(start, end, frequency="h", variables=None, source=None, +read_from_cache=True, write_to_cache=True, fetch_from_web=True)`** — same +`frequency`, `variables`, `source`, and cache-control semantics as the +location's, minus resolution: there is no `ignore_disqualification` (the +station is already chosen) and no arbitrary `**load_kwargs` passthrough (a +pinned station has nothing further to configure). Sets `self.provenance`. A +pinned (`source=`) load with nothing at all for the requested dates raises +`DataNotAvailableError`; a routed load (`source` unset) never raises for +missing data, returning NaN plus warnings instead. + +**`get_quality(anchor, source=None)`** — the station's data-quality +rating (`"high"`/`"medium"`/`"low"`) anchored to a date; see +[matching.md](matching.md) for the rating window. + +--- + +## `eeweather.sources` + +**`Source`** — the estimation protocol: `estimate(latitude, longitude, start, +end, frequency=, variables=, **load_kwargs)` returning +`(DataFrame, warnings, provenance)`. Implemented directly for location-keyed +(gridded) data. + +**`Feed`** — the station-keyed data protocol: declare `name`, `id_namespace`, +`variables` (canonical names), optionally `default_variables` and +`cacheable`; implement `fetch_year(external_id, year, variables)`. The engine +supplies caching, id translation, alignment, warnings, and provenance. + +**`NormalsSource`** — the typical-year protocol: `fetch(station_id)` +returning one typical year, tiled onto requested calendar years by +month-day-hour (leap days are NaN). + +**`StationSource`** — the nearest-suitable-station strategy wrapping a feed: +`StationSource(dataset="ghcnh", rank_kwargs=None, select_kwargs=None)`. +Applies the 150 km distance cap and quality/coverage checks by default; +configure via `rank_kwargs`/`select_kwargs`. + +**`Variable(name, unit, description, aggregation)`** — a vocabulary entry; +`aggregation` is one of `"mean"`, `"sum"`, `"min"`, `"max"`. + +**`register(source, vocabulary=())`** — register a custom feed, normals +source, or estimation/grid source under its name, making it usable as a +string in preference tuples and `source=` pins and rebuildable by location +serialization. `vocabulary` +declares any variables the vocabulary lacks; canonical names are reserved and +definitions must agree across sources. + +**`variables()`** — the vocabulary as a DataFrame: name, unit, description, +aggregation, and the sources serving each entry. + +--- + +## `eeweather.cache` + +Observed data caches locally per (source, station, year); current-year +entries refresh as new data arrives. + +**`set_path(path)`** — relocate the cache database (the `EEWEATHER_CACHE_URL` +environment variable does the same). **`clear()`** — empty it. + +--- + +## `eeweather.exceptions` + +All errors subclass **`EEWeatherError`**: + +- **`UnrecognizedStationError`** — an id no identifier system knows. +- **`UnrecognizedPlaceError`** — a place code missing from the geography. +- **`AmbiguousIdentifierError`** — an external id with multiple holders and + no recent one to prefer. +- **`DataNotAvailableError`** — a pinned source with nothing at all for the + requested dates (carries `source` and `station_id`). +- **`NoQualifiedStationError`** — no station passed qualification for a + location (see `ignore_disqualification`). + +**`EEWeatherWarning`** — the structured warning returned by loads (not a +Python `warnings` category): `qualified_name` (e.g. +`"eeweather.data_gap"`), human-readable `description`, and a `data` dict. + +--- + +## Registry maintenance + +Not an import surface, but part of the public contract: + +``` +python -m eeweather.registry.update # refresh registry data on demand +python -m eeweather.registry.update --clear # revert to packaged data +``` + +Updates also run automatically in the background when the installed registry +data is more than six months old; set `EEWEATHER_AUTO_UPDATE=0` to disable. +See [methodology.md](methodology.md) for the update channel's design. diff --git a/docs/example.md b/docs/example.md new file mode 100644 index 0000000..d0471b8 --- /dev/null +++ b/docs/example.md @@ -0,0 +1,198 @@ +# Example + +An end-to-end walkthrough: weather for a building, provenance, reproducible +reporting-period loads, station-level access, and typical years. Outputs shown +are real (Burbank, CA; calendar 2024), captured against live NOAA services. + +## Weather at a location + +Analysts have coordinates, not station ids, so the primary entry point is a +location: + +```python +from datetime import datetime, timezone + +import eeweather + +location = eeweather.WeatherLocation(34.2, -118.365) +# or from a ZIP code tabulation area: +location = eeweather.WeatherLocation.from_place("zcta", "91505") + +df, warnings = location.load_data( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 12, 31, tzinfo=timezone.utc), +) +``` + +``` + temperature +2024-01-01 00:00:00+00:00 13.890610 +2024-01-01 01:00:00+00:00 13.240167 +2024-01-01 02:00:00+00:00 13.098333 +... +[8761 rows x 1 columns] +``` + +The frame covers the full requested range at the requested frequency (hourly +by default) in UTC; hours the station did not report are NaN, and `warnings` +lists anything worth knowing (late starts, gaps, truncation) as structured +`EEWeatherWarning` objects — this load returned none. + +Everything about *how* the values were produced is on the provenance record, +stamped on both the frame (`df.attrs["provenance"]`) and the location: + +```python +location.provenance +``` + +``` +{'ghcnh': Provenance(kind='observations', source='ghcnh', + variables=('temperature',), station_id='USW00023152', + distance_meters=0.0, payload={})} +``` + +The location also knows its climate zones (from packaged geometries — no +network involved): + +```python +location.zones +``` + +``` +{'ba_climate_zone': 'Hot-Dry', 'ca_climate_zone': 'CA_09', + 'iecc_climate_zone': '3', 'iecc_moisture_regime': 'B'} +``` + +## Reproducibility across periods + +The first load pinned the location to the station it resolved. Serialize that +state with the baseline, and the reporting-period load — months later, in a +different process — is guaranteed the same station per source: + +```python +saved = location.to_json() +``` + +``` +{"latitude": 34.2, "longitude": -118.365, "sources": ["ghcnh"], + "pins": {"ghcnh": "USW00023152"}} +``` + +```python +location = eeweather.WeatherLocation.from_json(saved) +reporting, warnings = location.load_data(reporting_start, reporting_end) +``` + +## Auditing the match + +The ranked candidate frame shows the full field the resolution chose from, +with the metadata to defend (or override) the choice: + +```python +location.candidates(minimum_quality="high").head(4) +``` + +``` + distance_meters quality iecc_climate_zone is_tmy3 +station_id +USW00023152 0.0 high 3 True +USC00041194 2159.8 high 3 False +USW00023130 11688.4 high 3 True +USW00093197 21761.5 high 3 True +``` + +Stations more than 150 km away are disqualified by default (a station that far +is not a defensible temperature proxy); see +[matching.md](matching.md) for the cap, quality ratings, and the +`ignore_disqualification` escape hatch. + +## Weather at a known station + +When the station is already decided, key on it directly — by registry id or +through any historical identifier system: + +```python +station = eeweather.WeatherStation("USW00023152") +station = eeweather.WeatherStation.from_usaf("722880") # same station +station.ids +``` + +``` +{'ghcn': ['USW00023152'], 'icao': ['KBUR'], 'usaf': ['722880'], + 'wban': ['23152'], 'wmo': ['72288']} +``` + +Multiple variables come back as columns of one aligned frame, each in its +canonical unit: + +```python +df, warnings = station.load_data( + start, end, variables=("temperature", "relative_humidity", "wind_speed") +) +df.describe().loc[["mean", "min", "max"]].round(1) +``` + +``` + temperature relative_humidity wind_speed +mean 17.7 59.9 2.3 +min 2.8 4.0 0.0 +max 44.8 100.0 17.6 +``` + +`eeweather.sources.variables()` lists the full vocabulary — canonical names, +units, aggregations, and which sources serve each. + +## Other frequencies + +`frequency` takes any pandas offset alias. Coarser frequencies aggregate each +variable by its declared aggregation (temperature averages; an accumulation +variable would sum); finer-than-hourly frequencies interpolate: + +```python +monthly, _ = location.load_data(start, end, frequency="MS") +``` + +``` + temperature +2024-01-01 00:00:00+00:00 12.5 +2024-02-01 00:00:00+00:00 12.4 +2024-03-01 00:00:00+00:00 13.6 +2024-04-01 00:00:00+00:00 15.5 +... +``` + +```python +halfhourly, _ = station.load_data(july_1, july_2, frequency="30min") +``` + +``` + temperature +2024-07-01 00:00:00+00:00 31.464583 +2024-07-01 00:30:00+00:00 30.516958 +2024-07-01 01:00:00+00:00 29.569333 +2024-07-01 01:30:00+00:00 28.895125 +``` + +## Typical years + +Typical-year products (TMY3, CZ2010) are synthetic, so they never mix into +observed data implicitly — they are reachable only by an explicit `source=` +pin, and the typical year tiles onto the calendar years you request: + +```python +typical, _ = station.load_data(start, end, source="tmy3") +station.provenance +``` + +``` +{'tmy3': Provenance(kind='normals', source='tmy3', + variables=('temperature',), station_id='USW00023152', + distance_meters=None, payload={})} +``` + +## Caching + +Observed data caches locally per station-year, so repeated loads (a portfolio +of sites sharing stations, re-runs of an analysis) do not refetch. +`eeweather.cache.set_path(...)` relocates the cache; +`EEWEATHER_CACHE_URL` does the same via the environment. diff --git a/docs/expanding-the-vocabulary.md b/docs/expanding-the-vocabulary.md new file mode 100644 index 0000000..43487b3 --- /dev/null +++ b/docs/expanding-the-vocabulary.md @@ -0,0 +1,61 @@ +# Expanding the variable vocabulary + +> External sources do not need this document: a variable served only by +> a private source is declared at runtime via +> `eeweather.sources.register(source, vocabulary=...)`. This document +> covers adding a *canonical* entry — one built-in sources serve. + +Every variable a source can serve has exactly one canonical name, unit, +definition, and aggregation, owned by `sources/vocabulary.py`. Users only ever see +canonical forms; sources translate their native names and units at +ingest. The vocabulary is curated: un-vetted native fields are never +passed through, so adding a variable is a deliberate, verified change. + +## Rules + +- **One canonical unit per variable, forever.** Pick it when the + variable is added (SI-leaning, ascii-spelled: `degC`, `m/s`, `hPa`, + `W/m2`) and never change it — downstream analyses depend on a column + name implying its unit. There are no unit options in the API; sources + convert at ingest, users convert downstream if they must. +- **Names are snake_case, owned by eeweather.** The initial set adopted + GHCNh's spellings because they were sensible, but the name belongs to + the vocabulary, not to any source. +- **Declare the aggregation.** A variable's `aggregation` ("mean", + "sum", "min", or "max") governs its resampling to coarser frequencies, + and is always stated explicitly — a silent default on an accumulation + variable would produce wrong values with no error. Point-in-time + quantities declare "mean"; accumulation variables (precipitation) + declare "sum" — the engine adds them up within each period, spreads + them evenly across sub-hourly slots, and never gap-interpolates them. +- **Reserved suffixes.** Columns ending in `_imputed_fraction` or + `_is_imputed` are engine-produced companions of a canonical variable + and are exempt from vocabulary validation; never name a variable to + collide with them. + +## Steps + +1. Add the `Variable(name, unit, description, aggregation)` entry to + `VARIABLES` in `sources/vocabulary.py`. +2. Declare it in the `variables` tuple of every source that serves it, + with translation from the source's native name and conversion into + the canonical unit inside that source's `fetch_year`/`fetch`. +3. Verify units against real payloads — capture a live response as a + fixture and pin expected values with hardcoded numbers. Unit + validation against real data is the acceptance bar for the + declaration; a plausible-looking column in the wrong unit is worse + than no column. +4. Update the pins in `tests/sources/test_sources.py` + (`sources_serving`, the `variables()` accessor frame) and add + fetch-level tests for the new variable in the serving sources' test + modules. +5. CHANGELOG entry naming the variable, its unit, and which sources + serve it. + +## What routing does with it automatically + +Nothing else is required: a requested variable routes to the first +source in the preference tuple that declares it, `variables="all"` +expands to the union of the configured sources' vocabularies, an +unroutable request raises a `ValueError` naming the sources that do +serve it, and provenance records which source supplied it. diff --git a/docs/matching.md b/docs/matching.md new file mode 100644 index 0000000..c708f5c --- /dev/null +++ b/docs/matching.md @@ -0,0 +1,88 @@ +# Station Matching and Quality + +How EEweather decides which station represents a location, and how it judges +whether that station's data is good enough. Matching runs automatically inside +`WeatherLocation.load_data`; everything here is also inspectable directly +through `location.candidates()`. + +## Ranking + +Candidates are ranked by geodesic distance from the site (WGS84). The ranked +frame carries everything a reviewer needs to audit the choice: distance, +coordinates and elevation, climate-zone assignments (IECC zone and moisture +regime, Building America, California Building Climate Zone), subdivision, +quality rating, and per-source availability columns (`is_tmy3`, `is_cz2010`, +`tmy3_class`). + +Filters restrict the pool before ranking: + +- `match_zones=("iecc_climate_zone", ...)` — candidates must share the site's + zone in the named systems (a site with no zone matches only stations with + none). +- `match_subdivision=True` — restrict to the site's state/subdivision. +- `has_sources=("tmy3",)` — candidates must be able to serve the named + sources. Station-backed sources apply this for their own dataset + automatically. +- `minimum_quality="high"` — see quality, below. +- `max_distance_meters`, `max_difference_elevation_meters` — hard caps. + +## The distance cap + +**A station 150 km from a building is not a defensible temperature proxy**, so +station-backed sources apply `max_distance_meters=150_000` by default when +resolving a location. The cap is generous by construction: every US ZCTA has a +GHCNh station within 124 km (half are within 19 km), so no ordinary location is +stranded — the cap exists to catch the extraordinary ones (offshore +coordinates, geocoding errors, sites outside the station network) rather than +to constrain normal matching. + +When no station qualifies, the load raises `NoQualifiedStationError` rather +than silently serving a bad proxy. To proceed anyway — mirroring OpenDSM's +disqualification pattern, the flag sits on the operation that would raise — +pass `ignore_disqualification=True` to `load_data`: the best available station +serves, and an `eeweather.station_disqualified` warning records which +qualifications it failed and by how much. `rank_stations` itself applies no +default cap; as the audit surface it shows the full field. + +## Quality ratings + +Quality is computed from the station's observation inventory — monthly +observation counts per station-year, packaged and refreshed from NOAA's +inventory file. A station rates **high** when every month of the rating window +has more than 600 observations (roughly ~20 per day), **medium** above 360, +and **low** otherwise. + +Because networks change, quality is a function of *when*: a station that is +excellent today may have been sparse in 2012. The default `quality` column +rates the last five full calendar years; anchored ratings use the same +counts over a window matched to the request — the five calendar years ending +two years after the anchor date (sliding back so the window never ends +after the last full year). Two entry points expose it: + +- `location.candidates(minimum_quality="high", rating_period=anchor)` — + rank and filter by quality as of an era. +- `station.get_quality(anchor)` — one station's rating anchored to a date. + +Ratings are computed at query time from the inventory, so registry updates +extend them to recent periods without ever changing what a historical period +rates. + +## Coverage and selection + +Ranking answers "which stations are plausible"; selection confirms the winner +can actually serve the request. When a coverage check is configured, the +selected station's data for the requested range is validated against a +minimum-fraction threshold before it is accepted, and selection falls through +to the next candidate when it fails. Custom feeds validate against their own +data, not the default source's. + +## Memoization and pinning + +Within a process, a source memoizes its resolution per location and period +year-span — repeated loads do not re-rank, and a different era re-resolves +(station suitability is era-dependent). Across periods and processes, the +guarantee is stronger: a location pins itself to the stations recorded in its +first load's provenance, and serialized locations +(`to_json`/`from_json`) replay those pins exactly. An explicit pin is its own +acceptance: pinned resolution skips qualification, exactly as loading through +`WeatherStation` directly does. diff --git a/docs/methodology.md b/docs/methodology.md new file mode 100644 index 0000000..0ab1f00 --- /dev/null +++ b/docs/methodology.md @@ -0,0 +1,142 @@ +# Methodology + +EEweather answers one question for energy-efficiency measurement: **what was the +weather at this building?** Meters are at buildings; weather is observed at +stations. Between the two sit every problem this library exists to solve — +finding a defensible station for a location, judging whether its data is good +enough, fetching and aligning the observations, and being able to prove, years +later, exactly where every value came from. + +## Motivation + +Savings measurement compares energy use across periods that can be years apart, +normalized for weather. That places demands on weather data that general-purpose +weather APIs do not meet: + +- **Defensible attribution.** A weather-normalized savings number is only as + credible as its weather. The station serving a site must be near enough to be + a valid proxy, its data quality must be assessable, and the choice must be + explainable. +- **Reproducibility across periods.** A baseline fit in one year and a + reporting-period prediction in another must use the *same* station. A silent + switch to a different station between periods contaminates the comparison. +- **Analysis-ready alignment.** Models want complete, regular, UTC-indexed + frames with honest missing-data semantics — not raw observation streams with + duplicate timestamps, gaps, and native units. +- **Longevity.** Programs run for decades. Station networks change, identifier + systems come and go, and data products are retired (EEweather itself + migrated when the ISD feed stopped receiving data in 2025). The library must + absorb that churn without breaking the analyses built on it. + +## The two abstractions + +The public API is two classes and a handful of namespaces. + +**`WeatherLocation`** is weather at a latitude/longitude — the common case, +since analysts have building coordinates, not station ids. A location resolves +itself to the nearest suitable station per source, loads data, and records how. + +**`WeatherStation`** is weather at a known station — the pinned case, keyed by +the station's registry id, with constructors that translate historical +identifier systems (USAF, WBAN, ICAO, WMO). + +Both share one loading verb: `load_data(start, end, frequency, variables, +source=)` returns `(DataFrame, warnings)` for observed data and typical years +alike. There are no per-product methods to learn. + +## Station identity + +Every station has exactly one registry id (its GHCN id — the broadest station +identifier system in use, covering roughly ten times as many stations as USAF +or WMO numbering). Every other identifier is an *alias* recorded in a packaged +crosswalk, so `WeatherStation.from_usaf("722880")`, +`from_wban("23152")`, `from_icao("KBUR")`, and +`WeatherStation("USW00023152")` all reach the same station. Aliases carry a +recency marker: when an identifier was reused over history (retired WBAN +numbers were), the most recent holder wins, and a genuinely unresolvable +identifier raises rather than guessing. Id formats are never parsed; ids are +opaque keys into the registry. + +## Sources and routing + +A *source* is a provider of weather data. Built-ins: **GHCNh** (NOAA's hourly +observation network — the default), **TMY3** and **CZ2010** (typical-year +products, for normals-based analysis). Custom sources — a BigQuery mirror, an +internal archive — implement small public protocols and participate as equals +(see [adding-a-source.md](adding-a-source.md)). + +Locations and stations carry an ordered source preference +(`sources=("ghcnh",)` by default). Each requested variable routes to the +first source in the preference that serves it, and the results join into one +frame — a user asking for temperature from observations and irradiance from a +gridded product names variables, not plumbing. Two rules keep routing honest: + +- **One variable, one source, per call.** Values from different sources are + never stitched together inside a single column. +- **Typical years never route.** Measured and synthetic data are never mixed + silently; TMY3/CZ2010 are reachable only by an explicit `source=` pin. + +## The variable vocabulary + +Every variable has exactly one canonical name, unit, definition, and +aggregation, owned by the vocabulary. Sources translate their native fields +and units at ingest; users never see native forms, and un-vetted fields are +never passed through. The 1.0 set is temperature, dew point temperature +[degC], relative humidity [%], wind speed [m/s], station-level pressure +[hPa], and visibility [km]; `eeweather.sources.variables()` lists the current +vocabulary and which sources serve each entry. External sources may register +additional variables at runtime under the same one-definition rules. + +## Frequencies and aggregation + +Data is fetched hourly and served at any pandas frequency — +`load_data(frequency=...)` takes a pandas offset alias (`"30min"`, `"h"`, +`"D"`, `"W"`, `"MS"`, `"YS"`), parsed and validated by pandas itself. +Resampling honors each variable's declared aggregation: point-in-time +variables (temperature) average into coarser periods, while accumulation +variables sum, are never gap-interpolated, and report NaN (never a silent +zero) for periods with no data. Frequencies finer than hourly interpolate +point-in-time variables linearly between hourly values and spread +accumulations evenly, never crossing a missing hour. Frames for one request +range and frequency share an identical UTC index regardless of source, so +multi-source results join safely by construction. + +## Missing data + +Partial coverage surfaces as NaN values plus structured warnings (late starts, +early ends, internal gaps) — routed requests never raise for missing data. A +source explicitly pinned with `source=` that has nothing at all for the +requested dates raises `DataNotAvailableError`. Sub-hourly gaps up to an hour +are filled by linear interpolation at the minute level (the CalTRACK 2.3.3 +rule) for point-in-time variables. + +## Provenance and reproducibility + +Every load records provenance — per source: what kind of data, which station, +how far from the site, which variables — on the returned frame's +`attrs["provenance"]` and on the location or station object. Provenance is +also the reproducibility mechanism: a location **pins itself to the stations +resolved at its first load**, so later loads on the same location reuse them, +and `to_dict`/`to_json` (with `from_dict`/`from_json`) carry that state across +processes. A baseline load, a saved JSON blob, and a reporting-period load a +year later are guaranteed the same station per source. See +[matching.md](matching.md) for how the station is chosen in the first place. + +## The packaged registry, and keeping it current + +EEweather ships its registry — station catalog, identifier crosswalk, +climate-zone geometries and assignments, ZCTA places, observation inventory, +quality ratings — as packaged data, so matching and metadata work offline and +identically for everyone on a given install. Weather data itself is always +fetched live (and cached locally; see `eeweather.cache`). + +The live parts of the registry (inventory, quality, newly commissioned +stations, identifier aliases) decay on a yearly timescale. EEweather keeps +them current automatically: when the installed data is more than six months +old, a load starts a background update into the platform user data directory, +preferring ready-made files published by the repository's scheduled refresh +(CDN-served, so fleets of any size cost the upstream agencies nothing) and +falling back to rebuilding directly from the live NOAA files. Updates are +atomic, abort on implausible upstream content, and take effect in new +processes. `EEWEATHER_AUTO_UPDATE=0` disables them; +`python -m eeweather.registry.update` runs one on demand. diff --git a/docs/references.md b/docs/references.md new file mode 100644 index 0000000..d282293 --- /dev/null +++ b/docs/references.md @@ -0,0 +1,47 @@ +# References + +## Data sources + +- **GHCNh — Global Historical Climatology Network, hourly** (NOAA NCEI). + The observed-weather source: hourly surface observations, served through + the [NCEI Access Data Service](https://www.ncei.noaa.gov/access/services/data/v1) + and described on the + [GHCNh product page](https://www.ncei.noaa.gov/products/global-historical-climatology-network-hourly). + The packaged station catalog and observation inventory derive from the + GHCNh station list and inventory files under + `https://www.ncei.noaa.gov/oa/global-historical-climatology-network/`. + GHCNh supersedes the ISD/GSOD feeds earlier EEweather versions used (those + stopped receiving data 2025-08-27); overlap validation against ISD history + showed a median hourly deviation of 0.0001 °C. +- **TMY3 — Typical Meteorological Year 3** (NREL). Typical-year data for US + stations, from the archived NREL TMY3 collection; EEweather serves a + mirrored copy of the archive. +- **CZ2010** (California Energy Commission). Typical-year data for + California's building-climate-zone reference stations, used in Title 24 + work; EEweather serves a mirrored copy. +- **Identifier crosswalk.** Station aliases (USAF, WBAN, ICAO, WMO) derive + from the GHCNh station list and the historical ISD station history file, + audited against both (every alias verified by coordinate agreement; + catch-all and reused identifiers resolved by recency). +- **Geography.** ZCTA centroids from the US Census Bureau; IECC climate zones + and moisture regimes and Building America climate zones from DOE/PNNL + county assignments; California Building Climate Zone Areas from the CEC. + +## Methods + +- **Sub-hourly gap interpolation** follows CalTRACK 2.3.3: linear + interpolation at the minute level, limited to 60 minutes, for + point-in-time variables. See the + [CalTRACK methods](https://docs.caltrack.org/) and their successor, + maintained under [OpenDSM](https://opendsm.energy/). +- **Station matching** ranks by WGS84 geodesic distance with optional + climate-zone, subdivision, quality, and availability constraints; see + [matching.md](matching.md). + +## Related projects + +- [OpenDSM](https://opendsm.energy/) (formerly OpenEEmeter) — the + measurement models EEweather was built to feed; its documentation site + hosts the published version of these pages. +- [eemeter](https://github.com/opendsm/opendsm) — model implementations + consuming EEweather frames. diff --git a/eeweather/__init__.py b/eeweather/__init__.py index 070fe62..52988e5 100644 --- a/eeweather/__init__.py +++ b/eeweather/__init__.py @@ -1,8 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" +"""Weather for energy-efficiency modeling. -Copyright 2018-2023 OpenEEmeter contributors +Copyright 2018-2026 OpenDSM contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,77 +13,16 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - """ - import logging +from logging import NullHandler -from .__version__ import __title__, __description__, __url__, __version__ -from .__version__ import __author__, __author_email__, __license__ -from .__version__ import __copyright__ -from .geo import get_lat_long_climate_zones, get_zcta_metadata, zcta_to_lat_long -from .database import build_metadata_db -from .exceptions import ( - EEWeatherError, - UnrecognizedUSAFIDError, - UnrecognizedZCTAError, - DataNotAvailableError, -) -from .summaries import get_zcta_ids, get_isd_station_usaf_ids -from .ranking import rank_stations, combine_ranked_stations, select_station -from .stations import ( - WeatherStation, - get_ghcn_id, - get_isd_station_metadata, - get_station_quality, - get_station_qualities, - get_isd_file_metadata, - fetch_hourly_data, - fetch_tmy3_hourly_temp_data, - fetch_cz2010_hourly_temp_data, - get_hourly_data_cache_key, - get_tmy3_hourly_temp_data_cache_key, - get_cz2010_hourly_temp_data_cache_key, - cached_hourly_data_is_expired, - validate_hourly_data_cache, - validate_tmy3_hourly_temp_data_cache, - validate_cz2010_hourly_temp_data_cache, - serialize_hourly_data, - serialize_tmy3_hourly_temp_data, - serialize_cz2010_hourly_temp_data, - deserialize_hourly_data, - deserialize_tmy3_hourly_temp_data, - deserialize_cz2010_hourly_temp_data, - read_hourly_data_from_cache, - read_tmy3_hourly_temp_data_from_cache, - read_cz2010_hourly_temp_data_from_cache, - write_hourly_data_to_cache, - write_tmy3_hourly_temp_data_to_cache, - write_cz2010_hourly_temp_data_to_cache, - destroy_cached_hourly_data, - destroy_cached_tmy3_hourly_temp_data, - destroy_cached_cz2010_hourly_temp_data, - load_hourly_data_cached_proxy, - load_tmy3_hourly_temp_data_cached_proxy, - load_cz2010_hourly_temp_data_cached_proxy, - load_data, - load_tmy3_hourly_temp_data, - load_cz2010_hourly_temp_data, - load_cached_hourly_data, - load_cached_tmy3_hourly_temp_data, - load_cached_cz2010_hourly_temp_data, -) -from .utils import get_ghcn_ids -from .visualization import plot_station_mapping, plot_station_mappings - +from .__version__ import __version__ +from .location import WeatherLocation +from .station import WeatherStation -def get_version(): - return __version__ -# Set default logging handler to avoid "No handler found" warnings. -# will not work below py2.7 -# workaround: https://docs.python.org/release/2.6/library/logging.html#configuring-logging-for-a-library -from logging import NullHandler +__all__ = ("WeatherLocation", "WeatherStation", "__version__") logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/eeweather/__version__.py b/eeweather/__version__.py index c8ad4d7..ab1ae1e 100644 --- a/eeweather/__version__.py +++ b/eeweather/__version__.py @@ -1,27 +1 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -__title__ = "eeweather" -__description__ = "Weather for Open Energy Efficiency Meter" -__url__ = "http://github.com/openeemeter/eeweather" __version__ = "0.3.30" -__author__ = "Phil Ngo" -__author_email__ = "phil@openee.io" -__license__ = "Apache 2.0" -__copyright__ = "Copyright 2018-2023 OpenEEmeter contributors" diff --git a/eeweather/build/__init__.py b/eeweather/build/__init__.py new file mode 100644 index 0000000..bba35e2 --- /dev/null +++ b/eeweather/build/__init__.py @@ -0,0 +1 @@ +"""Registry build and refresh tooling.""" diff --git a/eeweather/build/__main__.py b/eeweather/build/__main__.py new file mode 100644 index 0000000..0e6c1f7 --- /dev/null +++ b/eeweather/build/__main__.py @@ -0,0 +1,34 @@ +"""Packaged-data maintenance entry point. + +``python -m eeweather.build`` refreshes the live parts of the packaged +data (GHCNh catalog, observation inventory, quality ratings, identity +rows for new stations) in place. ``python -m eeweather.build --migrate +OLD DESTDIR`` builds the packaged data files from a single-file +ghcn-keyed database. +""" +import argparse + +from .migrate import migrate +from .refresh import refresh + + + +def main(): + parser = argparse.ArgumentParser(prog="python -m eeweather.build") + parser.add_argument( + "--migrate", + nargs=2, + metavar=("OLD", "DESTDIR"), + help="build the packaged data files from a single-file db", + ) + args = parser.parse_args() + + if args.migrate: + counts = migrate(args.migrate[0], args.migrate[1]) + else: + counts = refresh() + print(counts) + + +if __name__ == "__main__": + main() diff --git a/eeweather/build/migrate.py b/eeweather/build/migrate.py new file mode 100644 index 0000000..f6b30b7 --- /dev/null +++ b/eeweather/build/migrate.py @@ -0,0 +1,278 @@ +"""Migrate a single-file ghcn-keyed database to the packaged data files. + +Produces the identifier crosswalk, the US geography pack, and one +database per built-in source (the GHCNh catalog with inventory and +quality; TMY3/CZ2010 station lists). Static content originates from +primary sources that are partly retired, so it is carried forward from +the existing packaged data rather than rebuilt; live content is updated +separately by ``refresh``. +""" +import os +import sqlite3 +from datetime import datetime, timezone + +from ..registry.db import MONTH_COLUMNS +from .schema import ( + CZ2010_SCHEMA, + GEOGRAPHY_SCHEMA, + GHCNH_SCHEMA, + IDENTIFIERS_SCHEMA, + TMY3_SCHEMA, + create_schema, +) + + + +# GHCNh ids begin with a FIPS 10-4 country code; the registry stores ISO +# 3166-1 alpha-2. Codes below are the full set present in the registry. +FIPS_TO_ISO = { + "US": "US", + "AS": "AU", + "RQ": "PR", + "AY": "AQ", + "GM": "DE", + "VQ": "VI", + "CQ": "MP", + "AQ": "AS", + "GQ": "GU", + "WQ": "UM", + "MQ": "UM", + "JQ": "UM", + "LQ": "UM", + "FQ": "UM", + "DQ": "UM", + "TT": "TT", + "PP": "PG", + "AV": "AI", +} + +WBAN_SENTINEL = "99999" + +# isd-history's catch-all id, reused for hundreds of distinct Australian +# sites; aliases derived from it reference no particular station +USAF_CATCHALL = "949999" + +# alias rows that contradict station coordinates in other primary sources +# (e.g. the GHCNh list assigns Wasilla's ICAO to Yakataga, whose isd +# records say PACY); excluded on build and on refresh +EXCLUDED_ALIASES = { + ("icao", "PAWS", "USW00026445"), +} + +# recency demotions where two records legitimately reference one site +# (TJSJ: the San Juan airport's COOP record next to its airport record) +DEMOTED_ALIASES = { + ("icao", "TJSJ", "RQC00668814"), +} + +ZONE_SYSTEMS = ( + "iecc_climate_zone", + "iecc_moisture_regime", + "ba_climate_zone", + "ca_climate_zone", +) + + +def _connect_fresh(path, schema): + if os.path.exists(path): + os.remove(path) + conn = sqlite3.connect(path) + create_schema(conn, schema) + + return conn + + +def _float_or_none(value): + try: + return float(value) + except (TypeError, ValueError): + return None + + +# GHCN's id-encoding knowledge lives here: this is the only place a +# station id is sliced to derive an attribute. Generic row-insert code +# receives country as the value this returns, never by parsing the id. +def _country(ghcn_id): + return FIPS_TO_ISO.get(ghcn_id[:2]) + + +def _migrate_stations(src, identifiers, ghcnh): + rows = src.execute( + """ + select ghcn_id, name, latitude, longitude, elevation, state, quality, + usaf_ids, wban_ids, recent_wban_id, icao_code, + iecc_climate_zone, iecc_moisture_regime, + ba_climate_zone, ca_climate_zone + from station_metadata + """ + ).fetchall() + for ( + ghcn_id, name, latitude, longitude, elevation, state, quality, + usaf_ids, wban_ids, recent_wban_id, icao_code, *zones + ) in rows: + ghcnh.execute( + "insert into stations values (?, ?, ?, ?, ?, ?, ?)", + ( + ghcn_id, + name, + _float_or_none(latitude), + _float_or_none(longitude), + _float_or_none(elevation), + _country(ghcn_id), + state, + ), + ) + ghcnh.execute( + "insert into quality values (?, ?)", (ghcn_id, quality) + ) + + alias_rows = [("ghcn", ghcn_id, 1)] + real_usafs = [ + u for u in (usaf_ids or "").split(",") if u and u != USAF_CATCHALL + ] + for usaf_id in real_usafs: + alias_rows.append(("usaf", usaf_id, 0)) + # a station whose only usaf id is the catch-all inherited its wban + # list from the catch-all's unrelated records + if real_usafs: + for wban_id in set((wban_ids or "").split(",")): + if wban_id and wban_id != WBAN_SENTINEL: + alias_rows.append( + ("wban", wban_id, int(wban_id == recent_wban_id)) + ) + if icao_code: + alias_rows.append(("icao", icao_code, 1)) + rows = [ + (namespace, external_id, ghcn_id, + 0 if (namespace, external_id, ghcn_id) in DEMOTED_ALIASES else recent) + for namespace, external_id, recent in alias_rows + if (namespace, external_id, ghcn_id) not in EXCLUDED_ALIASES + ] + identifiers.executemany( + "insert into station_identifier values (?, ?, ?, ?)", rows + ) + + for system, zone_id in zip(ZONE_SYSTEMS, zones): + if zone_id is not None: + ghcnh.execute( + "insert into station_zone values (?, ?, ?)", + (ghcn_id, system, str(zone_id)), + ) + + +def _migrate_inventory(src, ghcnh): + months = ", ".join(MONTH_COLUMNS) + rows = src.execute( + "select ghcn_id, year, {} from ghcn_inventory".format(months) + ).fetchall() + placeholders = ", ".join("?" for _ in range(2 + len(MONTH_COLUMNS))) + ghcnh.executemany( + "insert into inventory values ({})".format(placeholders), rows + ) + + +def _migrate_places(src, geography): + rows = src.execute( + """ + select zcta_id, state, latitude, longitude, + iecc_climate_zone, iecc_moisture_regime, + ba_climate_zone, ca_climate_zone + from zcta_metadata + """ + ).fetchall() + for zcta_id, state, latitude, longitude, *zones in rows: + geography.execute( + "insert into place values (?, ?, ?, ?, ?, ?)", + ("zcta", zcta_id, "US", state, + _float_or_none(latitude), _float_or_none(longitude)), + ) + for system, zone_id in zip(ZONE_SYSTEMS, zones): + if zone_id is not None: + geography.execute( + "insert into place_zone values (?, ?, ?, ?)", + ("zcta", zcta_id, system, str(zone_id)), + ) + + +def _migrate_zones(src, geography): + for system in ZONE_SYSTEMS: + if system == "ca_climate_zone": + name_col = "name" + else: + name_col = "null" + rows = src.execute( + "select {}, {}, geometry from {}_metadata".format(system, name_col, system) + ).fetchall() + geography.executemany( + "insert into zone values (?, ?, ?, ?)", + [(system, str(zone_id), name, geometry) + for zone_id, name, geometry in rows], + ) + + +def _migrate_normals(src, tmy3, cz2010): + tmy3.executemany( + "insert into stations values (?, ?, ?)", + src.execute( + "select ghcn_id, usaf_id, class from tmy3_station_metadata" + ).fetchall(), + ) + cz2010.executemany( + "insert into stations values (?, ?)", + src.execute( + "select ghcn_id, usaf_id from cz2010_station_metadata" + ).fetchall(), + ) + + +def _stamp_refreshed(ghcnh): + ghcnh.execute( + "create table if not exists meta (key text primary key, value text)" + " without rowid" + ) + ghcnh.execute( + "insert or replace into meta values ('refreshed_at', ?)", + (datetime.now(timezone.utc).strftime("%Y-%m-%d"),), + ) + + +def migrate(source_path, dest_dir): + """Build the packaged data files from a single-file ghcn-keyed + database, writing identifiers.db, geography_us.db, ghcnh.db, tmy3.db, + and cz2010.db into ``dest_dir``.""" + src = sqlite3.connect(source_path) + identifiers = _connect_fresh( + os.path.join(dest_dir, "identifiers.db"), IDENTIFIERS_SCHEMA + ) + geography = _connect_fresh( + os.path.join(dest_dir, "geography_us.db"), GEOGRAPHY_SCHEMA + ) + ghcnh = _connect_fresh(os.path.join(dest_dir, "ghcnh.db"), GHCNH_SCHEMA) + tmy3 = _connect_fresh(os.path.join(dest_dir, "tmy3.db"), TMY3_SCHEMA) + cz2010 = _connect_fresh(os.path.join(dest_dir, "cz2010.db"), CZ2010_SCHEMA) + + _migrate_stations(src, identifiers, ghcnh) + _migrate_inventory(src, ghcnh) + _migrate_places(src, geography) + _migrate_zones(src, geography) + _migrate_normals(src, tmy3, cz2010) + _stamp_refreshed(ghcnh) + + counts = {} + for name, conn, tables in ( + ("identifiers", identifiers, ("station_identifier",)), + ("geography_us", geography, ("place", "place_zone", "zone")), + ("ghcnh", ghcnh, ("stations", "station_zone", "inventory", "quality")), + ("tmy3", tmy3, ("stations",)), + ("cz2010", cz2010, ("stations",)), + ): + for table in tables: + counts["{}.{}".format(name, table)] = conn.execute( + "select count(*) from {}".format(table) + ).fetchone()[0] + conn.commit() + conn.execute("vacuum") + conn.close() + src.close() + + return counts diff --git a/eeweather/build/refresh.py b/eeweather/build/refresh.py new file mode 100644 index 0000000..3e58bbe --- /dev/null +++ b/eeweather/build/refresh.py @@ -0,0 +1,304 @@ +"""Refresh the live parts of the packaged data files in place. + +Updates the GHCNh catalog (newly listed stations), its observation +inventory and quality ratings, identity rows for appended stations, and +WMO/ICAO alias rows the GHCNh station list carries. Geography packs, +archive station lists, and existing identifier rows are never touched. +Station ids are permanent: existing rows are never re-keyed or deleted. + +Stations append only when they fall within the geography the package +serves (within APPEND_BUFFER_KM of a zone geometry of an attached +geography pack; offshore platforms self-exclude) and have observations +within the last APPEND_ACTIVITY_YEARS full years; appended stations +arrive with their zone assignments. Implausibly small upstream files +abort the refresh rather than wiping packaged content. +""" +import csv +import io +import json +import sqlite3 +from collections import defaultdict +from datetime import datetime, timezone + +import requests +from shapely.geometry import Point, shape + +from ..registry.db import ( + GHCNH_DB_PATH, + IDENTIFIERS_DB_PATH, + MONTH_COLUMNS, + metadata_db_connection_proxy, +) +from ..registry.quality import QUALITY_WINDOW_YEARS, _quality_from_minimum +from .migrate import EXCLUDED_ALIASES, _country, _float_or_none, _stamp_refreshed + + + +STATION_LIST_URL = ( + "https://www.ncei.noaa.gov/oa/global-historical-climatology-network/" + "hourly/doc/ghcnh-station-list.csv" +) +INVENTORY_URL = ( + "https://www.ncei.noaa.gov/oa/global-historical-climatology-network/" + "hourly/doc/ghcnh-inventory.txt" +) + +REQUEST_TIMEOUT_SECONDS = 120 + +# admission buffer around zone geometries: coastal stations sit just +# outside imprecise polygon edges; offshore platforms sit far outside +APPEND_BUFFER_KM = 10.0 + +APPEND_ACTIVITY_YEARS = 2 + + +def _fetch_station_list(): + """GHCNh station list as {ghcn_id: (name, lat, lon, elev, wmo, icao, iso_code)}.""" + response = requests.get(STATION_LIST_URL, timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + stations = {} + reader = csv.DictReader(io.StringIO(response.text)) + for row in reader: + ghcn_id = (row.get("GHCN_ID") or "").strip() + if not ghcn_id: + continue + stations[ghcn_id] = ( + (row.get("NAME") or "").strip(), + row.get("LATITUDE"), + row.get("LONGITUDE"), + row.get("ELEVATION"), + (row.get("WMO_ID") or "").strip(), + (row.get("ICAO") or "").strip(), + (row.get("ISO_CODE") or "").strip(), + ) + + return stations + + +def _fetch_inventory(): + """GHCNh inventory rows as (ghcn_id, year, jan..dec observation counts).""" + response = requests.get(INVENTORY_URL, timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + rows = [] + for line in response.text.splitlines(): + parts = line.split() + if len(parts) != 2 + len(MONTH_COLUMNS): + continue + ghcn_id, year = parts[0], parts[1] + if not year.isdigit(): # skip the header row + continue + rows.append((ghcn_id, int(year)) + tuple(int(n) for n in parts[2:])) + + return rows + + +def _zone_geometries(): + """(system, zone_id, shape) triples from every attached geography pack.""" + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + geometries = [] + for alias in proxy.geography_aliases: + for system, zone_id, geometry in conn.execute( + "select system, zone_id, geometry from {}.zone".format(alias) + ): + geometries.append((system, zone_id, shape(json.loads(geometry)))) + + return geometries + + +def _recently_active_stations(inventory): + """Station ids with any observations in the last full years.""" + last_full_year = datetime.now(timezone.utc).year - 1 + threshold = last_full_year - APPEND_ACTIVITY_YEARS + 1 + active = set() + for row in inventory: + if row[1] >= threshold and any(row[2:]): + active.add(row[0]) + + return active + + +def _append_new_stations(ghcnh, identifiers, station_list, inventory): + """Insert stations newly present in the GHCNh list. + + A station appends only when its coordinates fall within + APPEND_BUFFER_KM of a zone geometry of an attached geography pack and + it has observations within the last APPEND_ACTIVITY_YEARS full years. + Appended stations get their zone assignments (strict containment) and + a ghcn identity row. Existing stations are never modified.""" + known = {row[0] for row in ghcnh.execute("select station_id from stations")} + geometries = _zone_geometries() + buffer_degrees = APPEND_BUFFER_KM / 111.0 + active = _recently_active_stations(inventory) + added = 0 + for ghcn_id, (name, latitude, longitude, elevation, _wmo, _icao, iso_code) in station_list.items(): + if ghcn_id in known or ghcn_id not in active: + continue + lat = _float_or_none(latitude) + lon = _float_or_none(longitude) + if lat is None or lon is None: + continue + point = Point(lon, lat) + if not any( + geometry.distance(point) <= buffer_degrees + for _, _, geometry in geometries + ): + continue + country = iso_code or _country(ghcn_id) + ghcnh.execute( + "insert into stations values (?, ?, ?, ?, ?, ?, ?)", + ( + ghcn_id, + name, + lat, + lon, + _float_or_none(elevation), + country, + None, + ), + ) + zones = {} + for system, zone_id, geometry in geometries: + if system not in zones and geometry.contains(point): + zones[system] = zone_id + ghcnh.executemany( + "insert into station_zone values (?, ?, ?)", + [(ghcn_id, system, zone_id) for system, zone_id in zones.items()], + ) + identifiers.execute( + "insert into station_identifier values (?, ?, ?, ?)", + ("ghcn", ghcn_id, ghcn_id, 1), + ) + added += 1 + + return added + + +def _refresh_aliases(ghcnh, identifiers, station_list): + """Insert WMO and ICAO alias rows the GHCNh list carries for cataloged + stations. Existing rows are never modified, and an id that already + has a recent holder on another station is never claimed (the upstream + list disagreeing with the packaged crosswalk is a conflict to review, + not to auto-resolve).""" + known = {row[0] for row in ghcnh.execute("select station_id from stations")} + existing = set() + recent_holders = defaultdict(set) + for namespace, external_id, station_id, recent in identifiers.execute( + "select namespace, external_id, station_id, recent" + " from station_identifier where namespace in ('wmo', 'icao')" + ): + existing.add((namespace, external_id, station_id)) + if recent: + recent_holders[(namespace, external_id)].add(station_id) + added = 0 + for ghcn_id, (_name, _lat, _lon, _elev, wmo, icao, _iso) in station_list.items(): + if ghcn_id not in known: + continue + for namespace, external_id in (("wmo", wmo), ("icao", icao)): + if not external_id: + continue + if (namespace, external_id, ghcn_id) in existing: + continue + if (namespace, external_id, ghcn_id) in EXCLUDED_ALIASES: + continue + if recent_holders[(namespace, external_id)] - {ghcn_id}: + continue + identifiers.execute( + "insert into station_identifier values (?, ?, ?, ?)", + (namespace, external_id, ghcn_id, 1), + ) + recent_holders[(namespace, external_id)].add(ghcn_id) + added += 1 + + return added + + +def _replace_inventory(ghcnh, inventory): + """Replace inventory rows for cataloged stations.""" + known = {row[0] for row in ghcnh.execute("select station_id from stations")} + ghcnh.execute("delete from inventory") + placeholders = ", ".join("?" for _ in range(2 + len(MONTH_COLUMNS))) + rows = [row for row in inventory if row[0] in known] + ghcnh.executemany( + "insert into inventory values ({})".format(placeholders), rows + ) + + return len(rows) + + +def _recompute_quality(ghcnh): + """Recompute the quality ratings over the last full years.""" + last_full_year = datetime.now(timezone.utc).year - 1 + window_start = last_full_year - QUALITY_WINDOW_YEARS + 1 + months = ", ".join(MONTH_COLUMNS) + counts_by_station = {} + for row in ghcnh.execute( + "select station_id, year, {} from inventory" + " where year between ? and ?".format(months), + (window_start, last_full_year), + ): + counts_by_station.setdefault(row[0], {})[row[1]] = row[2:] + + ghcnh.execute("delete from quality") + for station_id, in ghcnh.execute("select station_id from stations").fetchall(): + years = counts_by_station.get(station_id, {}) + minimum = None + for year in range(window_start, last_full_year + 1): + year_minimum = min(years.get(year, (0,) * 12)) + if minimum is None or year_minimum < minimum: + minimum = year_minimum + ghcnh.execute( + "insert into quality values (?, ?)", + (station_id, _quality_from_minimum(minimum)), + ) + + +def _plausible_or_raise(ghcnh, station_list, inventory): + """Abort when an upstream file is implausibly small: a renamed header + or format change must fail the refresh, not wipe packaged content.""" + n_stations = ghcnh.execute("select count(*) from stations").fetchone()[0] + n_inventory = ghcnh.execute("select count(*) from inventory").fetchone()[0] + listed = sum(1 for row in ghcnh.execute("select station_id from stations") + if row[0] in station_list) + if listed < 0.9 * n_stations: + raise RuntimeError( + "Implausible station list: covers {} of {} cataloged" + " stations.".format(listed, n_stations) + ) + covered = sum(1 for row in inventory if row[0] in + {r[0] for r in ghcnh.execute("select station_id from stations")}) + if covered < 0.9 * n_inventory: + raise RuntimeError( + "Implausible inventory: {} rows for cataloged stations," + " packaged has {}.".format(covered, n_inventory) + ) + + +def refresh(ghcnh_path=GHCNH_DB_PATH, identifiers_path=IDENTIFIERS_DB_PATH): + """Refresh live packaged content in place; returns change counts.""" + station_list = _fetch_station_list() + inventory = _fetch_inventory() + + ghcnh = sqlite3.connect(ghcnh_path) + identifiers = sqlite3.connect(identifiers_path) + try: + _plausible_or_raise(ghcnh, station_list, inventory) + added = _append_new_stations(ghcnh, identifiers, station_list, inventory) + aliases_added = _refresh_aliases(ghcnh, identifiers, station_list) + inventory_rows = _replace_inventory(ghcnh, inventory) + _recompute_quality(ghcnh) + _stamp_refreshed(ghcnh) + for conn in (ghcnh, identifiers): + conn.commit() + conn.execute("vacuum") + finally: + ghcnh.close() + identifiers.close() + + result = { + "stations_added": added, + "aliases_added": aliases_added, + "inventory_rows": inventory_rows, + } + + return result diff --git a/eeweather/build/schema.py b/eeweather/build/schema.py new file mode 100644 index 0000000..42fb7f5 --- /dev/null +++ b/eeweather/build/schema.py @@ -0,0 +1,131 @@ +"""DDL for the packaged data files. + +The registry holds two kinds of files, each named for exactly what it +contains: the identifier crosswalk (identifiers.db — global; station ids +are GHCN ids, other id systems are aliases), and regional geography +packs (geography_us.db — zone geometries and coded places with their +zone assignments). Each source packages its own database beside its +adapter: the GHCNh catalog (station facts and their zone assignments, +as GHCNh's material claims them) with observation inventory and quality +ratings, and the TMY3/CZ2010 archive station lists. Connections ATTACH +geography packs and registered source databases so fact, availability, +and quality queries can join across files. Composite-key tables are WITHOUT ROWID so +the primary key is the storage order and no shadow index is needed. +""" +from ..registry.db import MONTH_COLUMNS + + + +IDENTIFIERS_SCHEMA = ( + """ + create table station_identifier ( + namespace text not null, + external_id text not null, + station_id text not null, + recent integer not null default 0, + primary key (namespace, external_id, station_id) + ) without rowid + """, + "create index ix_identifier_station on station_identifier (station_id)", +) + +GEOGRAPHY_SCHEMA = ( + """ + create table place ( + kind text not null, + code text not null, + country text, + subdivision text, + latitude real, + longitude real, + primary key (kind, code) + ) without rowid + """, + """ + create table place_zone ( + kind text not null, + code text not null, + system text not null, + zone_id text not null, + primary key (kind, code, system) + ) without rowid + """, + """ + create table zone ( + system text not null, + zone_id text not null, + name text, + geometry text, + primary key (system, zone_id) + ) without rowid + """, +) + +GHCNH_SCHEMA = ( + """ + create table stations ( + station_id text primary key, + name text, + latitude real, + longitude real, + elevation real, + country text, + subdivision text + ) without rowid + """, + "create index ix_stations_country on stations (country, subdivision)", + """ + create table station_zone ( + station_id text not null, + system text not null, + zone_id text not null, + primary key (station_id, system) + ) without rowid + """, + """ + create table inventory ( + station_id text not null, + year integer not null, + {}, + primary key (station_id, year) + ) without rowid + """.format(", ".join("{} integer".format(m) for m in MONTH_COLUMNS)), + """ + create table quality ( + station_id text primary key, + quality text not null + ) without rowid + """, + """ + create table meta ( + key text primary key, + value text + ) without rowid + """, +) + +# normals sources: which stations have archives, and the USAF id naming +# each station's archive file +TMY3_SCHEMA = ( + """ + create table stations ( + station_id text primary key, + usaf_id text not null, + class text + ) without rowid + """, +) + +CZ2010_SCHEMA = ( + """ + create table stations ( + station_id text primary key, + usaf_id text not null + ) without rowid + """, +) + + +def create_schema(conn, statements): + for statement in statements: + conn.execute(statement) diff --git a/eeweather/cache.py b/eeweather/cache.py index cf04880..81c71dc 100644 --- a/eeweather/cache.py +++ b/eeweather/cache.py @@ -1,21 +1,9 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +"""The shared weather-data cache. +One sqlite-backed JSON key-value store serves every station and source. +Its location is taken from ``set_path``, the EEWEATHER_CACHE_URL +environment variable, or the platform user cache directory, in that +order. """ import contextlib import datetime @@ -23,12 +11,18 @@ import os import sqlite3 -import pytz +import platformdirs SQLITE_URL_PREFIX = "sqlite:///" +DATA_EXPIRATION_DAYS = 1 + +# a data year keeps arriving for a while after it ends (publication lag), +# so entries written shortly after year end stay refreshable +YEAR_END_GRACE_DAYS = 14 + def _sqlite_path_from_url(url): if not url.startswith(SQLITE_URL_PREFIX): @@ -41,12 +35,7 @@ def _sqlite_path_from_url(url): class KeyValueStore(object): - """JSON key-value store on a local sqlite database. - - The database location is taken from the url argument, the - EEWEATHER_CACHE_URL environment variable, or ~/.eeweather/cache.db, - in that order. Urls have the form sqlite:///path/to/cache.db. - """ + """JSON key-value store on a local sqlite database.""" def __init__(self, url=None): self._prepare_db(url) @@ -57,9 +46,8 @@ def __repr__(self): def _get_url(self): # pragma: no cover (tests always provide url) url = os.environ.get("EEWEATHER_CACHE_URL") if url is None: - directory = "{}/.eeweather".format(os.path.expanduser("~")) - if not os.path.exists(directory): - os.makedirs(directory) + directory = platformdirs.user_cache_dir("eeweather") + os.makedirs(directory, exist_ok=True) url = "sqlite:///{}/cache.db".format(directory) return url @@ -73,26 +61,38 @@ def _prepare_db(self, url=None): with self._connect() as conn, conn: conn.execute( "create table if not exists items (" - " key text unique," + " key text primary key," " data text," " updated text)" ) - conn.execute("create index if not exists ix_items_key on items (key)") def _connect(self): # closes the connection on exit; writes commit via the inner # transaction context in each caller return contextlib.closing(sqlite3.connect(self._path)) - def key_exists(self, key): + def key_exists(self, key: str) -> bool: with self._connect() as conn: row = conn.execute("select 1 from items where key = ?", (key,)).fetchone() return row is not None + def keys(self, prefix: str) -> list: + """All keys beginning with a prefix, sorted.""" + escaped = ( + prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + ) + with self._connect() as conn: + rows = conn.execute( + "select key from items where key like ? escape '\\' order by key", + (escaped + "%",), + ).fetchall() + + return [row[0] for row in rows] + def save_json(self, key, data): data = json.dumps(data, separators=(",", ":")) - updated = datetime.datetime.now(pytz.UTC).isoformat() + updated = datetime.datetime.now(datetime.timezone.utc).isoformat() with self._connect() as conn, conn: conn.execute( "insert into items (key, data, updated) values (?, ?, ?)" @@ -128,3 +128,55 @@ def clear(self, key=None): conn.execute("delete from items") else: conn.execute("delete from items where key = ?", (key,)) + + +class KeyValueStoreProxy(object): + def __init__(self): + self._store = None + + def get_store(self): # pragma: no cover + if self._store is None: + self._store = KeyValueStore() + + return self._store + + def set_store(self, store): + self._store = store + + +key_value_store_proxy = KeyValueStoreProxy() + + +def set_path(path: str) -> None: + """Point the shared weather cache at a specific sqlite file, creating + its directory if needed. + + All stations and sources share this one store; overrides the + EEWEATHER_CACHE_URL environment variable and the default location. + """ + parent = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent, exist_ok=True) + key_value_store_proxy.set_store(KeyValueStore("sqlite:///{}".format(path))) + + +def clear() -> None: + """Drop all cached weather data from the shared store.""" + key_value_store_proxy.get_store().clear() + + +def _expired(last_updated, year): + """Whether a cache entry for a data year is stale: entries written + while the year's data was still arriving (during the year, or within + YEAR_END_GRACE_DAYS after it ends) expire after + DATA_EXPIRATION_DAYS.""" + if last_updated is None: + return True + expiration_limit = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + days=DATA_EXPIRATION_DAYS + ) + volatile_until = datetime.datetime( + year + 1, 1, 1, tzinfo=datetime.timezone.utc + ) + datetime.timedelta(days=YEAR_END_GRACE_DAYS) + still_volatile = last_updated < volatile_until + + return expiration_limit > last_updated and still_volatile diff --git a/eeweather/cli.py b/eeweather/cli.py deleted file mode 100644 index fa929c2..0000000 --- a/eeweather/cli.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import json -import subprocess - -import click - -from . import ( - get_isd_station_metadata as _get_isd_station_metadata, - get_isd_file_metadata as _get_isd_file_metadata, -) -from .exceptions import UnrecognizedUSAFIDError - -from .database import build_metadata_db, inspect_metadata_db - - -@click.group() -def cli(): - """Example usage - - See station metadata: - - \b - $ eeweather inspect-isd-station 722880 - { - "usaf_id": "722880", - "wban_ids": "23152,99999", - "recent_wban_id": "23152", - "name": "BURBANK-GLENDALE-PASA ARPT", - "latitude": "+34.201", - "longitude": "-118.358", - "elevation": "+0236.2" - } - - See station file data: - - \b - $ eeweather inspect-isd-file-years 722880 - {...} - - See station file names for ISD: - - \b - $ eeweather inspect-isd-filenames 722880 2017 - ftp://ftp.ncei.noaa.gov/pub/data/noaa/2017/722880-23152-2017.gz - - Or for GSOD: - - \b - $ eeweather inspect-gsod-filenames 722880 2017 - ftp://ftp.ncei.noaa.gov/pub/data/gsod/2017/722880-23152-2017.op.gz - - Rebuild metadata db from primary source files: - - \b - $ eeweather rebuild-db - - Inspect metadata db using sqlite3 CLI: - - \b - $ eeweather inspect-db - - - """ - pass # pragma: no cover - - -@cli.command() -@click.argument("usaf_id") -def inspect_isd_station(usaf_id): - metadata = _get_isd_station_metadata(usaf_id) - click.echo(json.dumps(metadata, indent=2)) - - -@cli.command() -@click.argument("usaf_id") -def inspect_isd_file_years(usaf_id): - metadata = _get_isd_file_metadata(usaf_id) - click.echo(json.dumps(metadata, indent=2)) - - -@cli.command() -@click.option("--zcta-geometry/--no-zcta-geometry", default=False) -@click.option( - "--iecc-climate-zone-geometry/--no-iecc-climate-zone-geometry", default=True -) -@click.option( - "--iecc-moisture-regime-geometry/--no-iecc-moisture-regime-geometry", default=True -) -@click.option("--ba-climate-zone-geometry/--no-ba-climate-zone-geometry", default=True) -@click.option("--ca-climate-zone-geometry/--no-ca-climate-zone-geometry", default=True) -def rebuild_db( - zcta_geometry, - iecc_climate_zone_geometry, - iecc_moisture_regime_geometry, - ba_climate_zone_geometry, - ca_climate_zone_geometry, -): - build_metadata_db( # pragma: no cover - zcta_geometry, - iecc_climate_zone_geometry, - iecc_moisture_regime_geometry, - ba_climate_zone_geometry, - ca_climate_zone_geometry, - ) - - -@cli.command() -def inspect_db(): - inspect_metadata_db() # pragma: no cover diff --git a/eeweather/connections.py b/eeweather/connections.py deleted file mode 100644 index 841a22b..0000000 --- a/eeweather/connections.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import os -import sqlite3 -import threading - -from .cache import KeyValueStore - -logger = logging.getLogger(__name__) - -__all__ = ("metadata_db_connection_proxy",) - - -class MetadataDBConnectionProxy(object): - """Serves a cached read connection to the packaged metadata database. - - Connections are cached per thread and reused, so callers do not close - them. - """ - - def __init__(self): - root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - path = os.path.join(root_dir, "eeweather", "resources") - self.db_path = os.path.join(path, "metadata.db") - self._local = threading.local() - - def get_connection(self): - connection = getattr(self._local, "connection", None) - if connection is None: - connection = sqlite3.connect(self.db_path) - self._local.connection = connection - - return connection - - def reset_database(self): # pragma: no cover - connection = getattr(self._local, "connection", None) - if connection is not None: - connection.close() - self._local.connection = None - os.remove(self.db_path) - - return self.get_connection() - - -class KeyValueStoreProxy(object): - def __init__(self): - self._store = None - - def get_store(self): # pragma: no cover - if self._store is None: - self._store = KeyValueStore() - return self._store - - -# Use proxies for lazy loading, abstraction -metadata_db_connection_proxy = MetadataDBConnectionProxy() -key_value_store_proxy = KeyValueStoreProxy() diff --git a/eeweather/database.py b/eeweather/database.py deleted file mode 100644 index eb0723c..0000000 --- a/eeweather/database.py +++ /dev/null @@ -1,1465 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from collections import defaultdict -from datetime import datetime, timedelta -import json -import logging -import os -import shutil -import subprocess -import tempfile - -import pandas as pd -import numpy as np - -from .connections import metadata_db_connection_proxy -from .stations import QUALITY_WINDOW_YEARS, _quality_from_minimum - - - -logger = logging.getLogger(__name__) - -__all__ = ("build_metadata_db", "inspect_metadata_db") - -CZ2010_LIST = [ - "725958", - "725945", - "723840", - "724837", - "724800", - "725845", - "747188", - "722880", - "723926", - "722926", - "722927", - "746120", - "722899", - "724936", - "725946", - "723815", - "723810", - "722810", - "725940", - "723890", - "722976", - "724935", - "747185", - "722909", - "723826", - "722956", - "725847", - "723816", - "747020", - "724927", - "722895", - "722970", - "722975", - "722874", - "722950", - "724815", - "724926", - "722953", - "725955", - "724915", - "725957", - "724955", - "723805", - "724930", - "723927", - "722868", - "747187", - "723820", - "724937", - "723965", - "723910", - "723895", - "725910", - "725920", - "722860", - "722869", - "724830", - "724839", - "724917", - "724938", - "722925", - "722907", - "722900", - "722903", - "722906", - "724940", - "724945", - "724946", - "722897", - "722910", - "723830", - "722977", - "723925", - "723940", - "722885", - "724957", - "724920", - "722955", - "745160", - "725846", - "690150", - "725905", - "722886", - "723930", - "723896", - "724838", -] - - -class PrettyFloat(float): - def __repr__(self): - return "%.7g" % self - - -def pretty_floats(obj): - if isinstance(obj, float): - return PrettyFloat(round(obj, 4)) - elif isinstance(obj, dict): - return dict((k, pretty_floats(v)) for k, v in obj.items()) - elif isinstance(obj, (list, tuple)): - return list(map(pretty_floats, obj)) - return obj - - -def to_geojson(polygon): - import simplejson - from shapely.geometry import mapping - - return simplejson.dumps(pretty_floats(mapping(polygon)), separators=(",", ":")) - - -def _download_primary_sources(): - root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - scripts_path = os.path.join(root_dir, "scripts", "download_primary_sources.sh") - - download_path = tempfile.mkdtemp() - subprocess.call([scripts_path, download_path]) - return download_path - - -def _load_isd_station_metadata(download_path): - """Collect metadata for US isd stations.""" - from shapely.geometry import Point - - # load ISD history which contains metadata - isd_history = pd.read_csv( - os.path.join(download_path, "isd-history.csv"), - dtype=str, - parse_dates=["BEGIN", "END"], - ) - - hasGEO = ( - isd_history.LAT.notnull() & isd_history.LON.notnull() & (isd_history.LAT != 0) - ) - hasUSAF = isd_history.USAF != "999999" - - isUS = ( - ((isd_history.CTRY == "US") & (isd_history.STATE.notnull())) - # AQ = American Samoa, GQ = Guam, RQ = Peurto Rico, VQ = Virgin Islands - | (isd_history.CTRY.str[1] == "Q") - ) - - isAus = isd_history.CTRY == "AS" - - metadata = {} - for usaf_station, group in isd_history[hasGEO & hasUSAF & (isUS | isAus)].groupby( - "USAF" - ): - # find most recent - recent = group.loc[group.END.idxmax()] - wban_stations = list(group.WBAN) - metadata[usaf_station] = { - "usaf_id": usaf_station, - "wban_ids": wban_stations, - "recent_wban_id": recent.WBAN, - "name": recent["STATION NAME"], - "icao_code": recent.ICAO, - "latitude": recent.LAT if recent.LAT not in ("+00.000",) else None, - "longitude": recent.LON if recent.LON not in ("+000.000",) else None, - "point": Point(float(recent.LON), float(recent.LAT)), - "elevation": ( - recent["ELEV(M)"] - if not str(float(recent["ELEV(M)"])).startswith("-999") - else None - ), - "state": recent.STATE, - } - - for usaf_id, (lat, lon) in ISD_COORDINATE_CORRECTIONS.items(): - if usaf_id in metadata: - metadata[usaf_id]["latitude"] = lat - metadata[usaf_id]["longitude"] = lon - metadata[usaf_id]["point"] = Point(float(lon), float(lat)) - - return metadata - - -# Corrections to known-bad coordinates in the upstream isd-history registry, -# verified against the physical site location and the GHCNh station list. -ISD_COORDINATE_CORRECTIONS = { - # Ann Arbor Municipal (KARB): isd-history longitude is off by 4 degrees - "725374": ("+42.223", "-083.740"), -} - - -GHCN_MATCH_SANITY_KM = 50.0 -GHCN_NEAREST_NEIGHBOR_KM = 5.0 - - -def _haversine_km(lat1, lon1, lat2, lon2): - earth_radius_km = 6371.0 - p1, p2 = np.radians(lat1), np.radians(lat2) - dp, dl = np.radians(lat2 - lat1), np.radians(lon2 - lon1) - a = np.sin(dp / 2) ** 2 + np.cos(p1) * np.cos(p2) * np.sin(dl / 2) ** 2 - - return 2 * earth_radius_km * np.arcsin(np.sqrt(a)) - - -GHCNH_INVENTORY_MONTHS = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", - "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"] - - -def _load_ghcnh_inventory(download_path): - """Monthly observation counts per GHCNh station and year.""" - inventory = pd.read_csv( - os.path.join(download_path, "ghcnh-inventory.txt"), sep=r"\s+" - ) - - return inventory - - -def _ghcn_data_years(inventory): - """First and last year with observations per GHCNh station.""" - with_data = inventory[inventory[GHCNH_INVENTORY_MONTHS].sum(axis=1) > 0] - years = with_data.groupby("GHCNh_ID").YEAR.agg(["min", "max"]) - - return {gid: (int(row["min"]), int(row["max"])) for gid, row in years.iterrows()} - - -def _load_registry_ghcn_inventory(isd_station_metadata, inventory): - """Monthly observation counts per registry station and year.""" - ghcn_to_usaf = {} - for usaf_id, metadata in isd_station_metadata.items(): - ghcn_to_usaf.setdefault(metadata["ghcn_id"], []).append(usaf_id) - - rows = [] - for record in inventory.itertuples(index=False): - for usaf_id in ghcn_to_usaf.get(record.GHCNh_ID, ()): - rows.append( - (usaf_id, int(record.YEAR)) - + tuple(int(getattr(record, month)) for month in GHCNH_INVENTORY_MONTHS) - ) - - return rows - - -def _write_ghcn_inventory_table(conn, ghcn_inventory): - cur = conn.cursor() - cur.executemany( - """ - insert into ghcn_inventory( - usaf_id, year, jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec - ) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) - """, - ghcn_inventory, - ) - cur.execute( - """ - create index ghcn_inventory_usaf_id_year on ghcn_inventory(usaf_id, year) - """ - ) - cur.close() - conn.commit() - - -def _compute_station_quality_from_ghcnh( - isd_station_metadata, inventory, end_year=None, years_back=None -): - """Rate each station by its GHCNh observation counts. - - A station is high quality when every month of the last years_back - full years has more than 600 observations, medium above 360, low - otherwise or when any year is absent. - """ - if end_year is None: - end_year = datetime.now().year - 1 # last full year - if years_back is None: - years_back = QUALITY_WINDOW_YEARS - - year_range = set(range(end_year - (years_back - 1), end_year + 1)) - window = inventory[inventory.YEAR.isin(year_range)] - grouped = {gid: group for gid, group in window.groupby("GHCNh_ID")} - - def quality(ghcn_id): - group = grouped.get(ghcn_id) - if group is None or set(group.YEAR) != year_range: - return "low" - minimum = group.groupby("YEAR")[GHCNH_INVENTORY_MONTHS].sum().to_numpy().min() - - return _quality_from_minimum(minimum) - - for usaf_id, metadata in isd_station_metadata.items(): - metadata["quality"] = quality(metadata["ghcn_id"]) - - -def _map_isd_stations_to_ghcn(isd_station_metadata, download_path): - """Assign each station its GHCNh id; stations without one are removed. - - Match priority: shared ICAO code (nearest candidate, within - GHCN_MATCH_SANITY_KM), GHCNh id following the USW000{wban} pattern - (within GHCN_MATCH_SANITY_KM), then nearest GHCNh station within - GHCN_NEAREST_NEIGHBOR_KM. Each mapped station records the first and - last year its GHCNh record has observations. - """ - ghcn = pd.read_csv( - os.path.join(download_path, "ghcnh-station-list.csv"), dtype=str - ) - ghcn["lat"] = pd.to_numeric(ghcn.LATITUDE, errors="coerce") - ghcn["lon"] = pd.to_numeric(ghcn.LONGITUDE, errors="coerce") - ghcn = ghcn.dropna(subset=["lat", "lon"]).reset_index(drop=True) - ghcn_by_id = ghcn.set_index("GHCN_ID", drop=False) - data_years = _ghcn_data_years(_load_ghcnh_inventory(download_path)) - ghcn_by_icao = {icao: group for icao, group in ghcn.dropna(subset=["ICAO"]).groupby("ICAO")} - - unmapped = [] - for usaf_id, metadata in isd_station_metadata.items(): - lat = pd.to_numeric(metadata["latitude"], errors="coerce") - lon = pd.to_numeric(metadata["longitude"], errors="coerce") - ghcn_id, method = None, None - - if not pd.isna(lat): - icao_candidates = ghcn_by_icao.get(metadata["icao_code"]) - if icao_candidates is not None: - d = _haversine_km(lat, lon, icao_candidates.lat.values, icao_candidates.lon.values) - i = int(np.argmin(d)) - if d[i] <= GHCN_MATCH_SANITY_KM: - ghcn_id, method = icao_candidates.GHCN_ID.values[i], "icao" - - if ghcn_id is None: - wban_guess = "USW000" + str(metadata["recent_wban_id"]).zfill(5) - if wban_guess in ghcn_by_id.index: - row = ghcn_by_id.loc[wban_guess] - if _haversine_km(lat, lon, row.lat, row.lon) <= GHCN_MATCH_SANITY_KM: - ghcn_id, method = wban_guess, "wban" - - if ghcn_id is None: - d = _haversine_km(lat, lon, ghcn.lat.values, ghcn.lon.values) - i = int(np.argmin(d)) - if d[i] <= GHCN_NEAREST_NEIGHBOR_KM: - ghcn_id, method = ghcn.GHCN_ID.values[i], "latlon" - - if ghcn_id is None: - unmapped.append(usaf_id) - else: - metadata["ghcn_id"] = ghcn_id - metadata["ghcn_map_method"] = method - first_year, last_year = data_years.get(ghcn_id, (None, None)) - metadata["ghcn_first_year"] = first_year - metadata["ghcn_last_year"] = last_year - - for usaf_id in unmapped: - del isd_station_metadata[usaf_id] - - print("Removed {} stations with no GHCNh counterpart".format(len(unmapped))) - - -def _load_isd_file_metadata(download_path, isd_station_metadata): - """Collect data counts for isd files.""" - - isd_inventory = pd.read_csv( - os.path.join(download_path, "isd-inventory.csv"), dtype=str - ) - - # filter to stations with metadata - station_keep = [usaf in isd_station_metadata for usaf in isd_inventory.USAF] - isd_inventory = isd_inventory[station_keep] - # filter by year - year_keep = isd_inventory.YEAR > "2005" - isd_inventory = isd_inventory[year_keep] - - metadata = {} - for (usaf_station, year), group in isd_inventory.groupby(["USAF", "YEAR"]): - if usaf_station not in metadata: - metadata[usaf_station] = {"usaf_id": usaf_station, "years": {}} - metadata[usaf_station]["years"][year] = [ - { - "wban_id": row.WBAN, - "counts": [ - row.JAN, - row.FEB, - row.MAR, - row.APR, - row.MAY, - row.JUN, - row.JUL, - row.AUG, - row.SEP, - row.OCT, - row.NOV, - row.DEC, - ], - } - for i, row in group.iterrows() - ] - return metadata - - -def _load_zcta_metadata(download_path): - from shapely.geometry import shape - - # load zcta geojson - geojson_path = os.path.join(download_path, "cb_2016_us_zcta510_500k.json") - with open(geojson_path, "r") as f: - geojson = json.load(f) - - # load ZIP code prefixes by state - zipcode_prefixes_path = os.path.join(download_path, "zipcode_prefixes.json") - with open(zipcode_prefixes_path, "r") as f: - zipcode_prefixes = json.load(f) - prefix_to_zipcode = { - zipcode_prefix: state - for state, zipcode_prefix_list in zipcode_prefixes.items() - for zipcode_prefix in zipcode_prefix_list - } - - def _get_state(zcta): - prefix = zcta[:3] - return prefix_to_zipcode.get(prefix) - - metadata = {} - for feature in geojson["features"]: - zcta = feature["properties"]["GEOID10"] - geometry = feature["geometry"] - polygon = shape(geometry) - centroid = polygon.centroid - state = _get_state(zcta) - metadata[zcta] = { - "zcta": zcta, - "polygon": polygon, - "geometry": to_geojson(polygon), - "centroid": centroid, - "latitude": centroid.coords[0][1], - "longitude": centroid.coords[0][0], - "state": state, - } - return metadata - - -def _load_county_metadata(download_path): - from shapely.geometry import shape - - # load county geojson - geojson_path = os.path.join(download_path, "cb_2016_us_county_500k.json") - with open(geojson_path, "r") as f: - geojson = json.load(f) - - metadata = {} - for feature in geojson["features"]: - county = feature["properties"]["GEOID"] - geometry = feature["geometry"] - polygon = shape(geometry) - centroid = polygon.centroid - metadata[county] = { - "county": county, - "polygon": polygon, - "geometry": to_geojson(polygon), - "centroid": centroid, - "latitude": centroid.coords[0][1], - "longitude": centroid.coords[0][0], - } - - # load county climate zones - county_climate_zones = pd.read_csv( - os.path.join(download_path, "climate_zones.csv"), - dtype=str, - usecols=[ - "State FIPS", - "County FIPS", - "IECC Climate Zone", - "IECC Moisture Regime", - "BA Climate Zone", - "County Name", - ], - ) - - for i, row in county_climate_zones.iterrows(): - county = row["State FIPS"] + row["County FIPS"] - if county not in metadata: - logger.warn( - "Could not find geometry for county {}, skipping.".format(county) - ) - continue - - metadata[county].update( - { - "name": row["County Name"], - "iecc_climate_zone": row["IECC Climate Zone"], - "iecc_moisture_regime": ( - row["IECC Moisture Regime"] - if not pd.isnull(row["IECC Moisture Regime"]) - else None - ), - "ba_climate_zone": row["BA Climate Zone"], - } - ) - - return metadata - - -def _load_CA_climate_zone_metadata(download_path): - from shapely.geometry import shape, mapping - - ca_climate_zone_names = { - "01": "Arcata", - "02": "Santa Rosa", - "03": "Oakland", - "04": "San Jose-Reid", - "05": "Santa Maria", - "06": "Torrance", - "07": "San Diego-Lindbergh", - "08": "Fullerton", - "09": "Burbank-Glendale", - "10": "Riverside", - "11": "Red Bluff", - "12": "Sacramento", - "13": "Fresno", - "14": "Palmdale", - "15": "Palm Spring-Intl", - "16": "Blue Canyon", - } - geojson_path = os.path.join( - download_path, "CA_Building_Standards_Climate_Zones.json" - ) - with open(geojson_path, "r") as f: - geojson = json.load(f) - - metadata = {} - for feature in geojson["features"]: - zone = "{:02d}".format(int(feature["properties"]["Zone"])) - geometry = feature["geometry"] - polygon = shape(geometry) - metadata[zone] = { - "ca_climate_zone": "CA_{}".format(zone), - "name": ca_climate_zone_names[zone], - "polygon": polygon, - "geometry": to_geojson(polygon), - } - return metadata - - -def _load_tmy3_station_metadata(download_path): - from bs4 import BeautifulSoup - - path = os.path.join(download_path, "tmy3-stations.html") - with open(path, "r") as f: - soup = BeautifulSoup(f.read(), "html.parser") - tmy3_station_elements = soup.select("td .hide") - - metadata = {} - for station_el in tmy3_station_elements: - station_name_el = station_el.findNext("td").findNext("td") - station_class_el = station_name_el.findNext("td") - usaf_id = station_el.text.strip() - name = ( - "".join(station_name_el.text.split(",")[:-1]) - .replace("\n", "") - .replace("\t", "") - .strip() - ) - metadata[usaf_id] = { - "usaf_id": usaf_id, - "name": name, - "state": station_name_el.text.split(",")[-1].strip(), - "class": station_class_el.text.split()[1].strip(), - } - return metadata - - -def _load_cz2010_station_metadata(): - return {usaf_id: {"usaf_id": usaf_id} for usaf_id in CZ2010_LIST} - - -def _create_merged_climate_zones_metadata(county_metadata): - from shapely.ops import unary_union - - iecc_climate_zone_polygons = defaultdict(list) - iecc_moisture_regime_polygons = defaultdict(list) - ba_climate_zone_polygons = defaultdict(list) - - for county in county_metadata.values(): - polygon = county["polygon"] - iecc_climate_zone = county.get("iecc_climate_zone") - iecc_moisture_regime = county.get("iecc_moisture_regime") - ba_climate_zone = county.get("ba_climate_zone") - if iecc_climate_zone is not None: - iecc_climate_zone_polygons[iecc_climate_zone].append(polygon) - if iecc_moisture_regime is not None: - iecc_moisture_regime_polygons[iecc_moisture_regime].append(polygon) - if ba_climate_zone is not None: - ba_climate_zone_polygons[ba_climate_zone].append(polygon) - - iecc_climate_zone_metadata = {} - for iecc_climate_zone, polygons in iecc_climate_zone_polygons.items(): - polygon = unary_union(polygons) - polygon = polygon.simplify(0.01) - iecc_climate_zone_metadata[iecc_climate_zone] = { - "iecc_climate_zone": iecc_climate_zone, - "polygon": polygon, - "geometry": to_geojson(polygon), - } - - iecc_moisture_regime_metadata = {} - for iecc_moisture_regime, polygons in iecc_moisture_regime_polygons.items(): - polygon = unary_union(polygons) - polygon = polygon.simplify(0.01) - iecc_moisture_regime_metadata[iecc_moisture_regime] = { - "iecc_moisture_regime": iecc_moisture_regime, - "polygon": polygon, - "geometry": to_geojson(polygon), - } - - ba_climate_zone_metadata = {} - for ba_climate_zone, polygons in ba_climate_zone_polygons.items(): - polygon = unary_union(polygons) - polygon = polygon.simplify(0.01) - ba_climate_zone_metadata[ba_climate_zone] = { - "ba_climate_zone": ba_climate_zone, - "polygon": polygon, - "geometry": to_geojson(polygon), - } - - return ( - iecc_climate_zone_metadata, - iecc_moisture_regime_metadata, - ba_climate_zone_metadata, - ) - - -def _compute_containment( - point_metadata, point_id_field, polygon_metadata, polygon_metadata_field -): - from shapely.vectorized import contains - - points, lats, lons = zip( - *[ - ( - point, - float(point["latitude"]), - float(point["longitude"]), - ) - for point in point_metadata.values() - if point["latitude"] and point["longitude"] - ] - ) - - for i, polygon in enumerate(polygon_metadata.values()): - containment = contains(polygon["polygon"], lons, lats) - for point, c in zip(points, containment): - if c: - point[polygon_metadata_field] = polygon[polygon_metadata_field] - # fill in with None - for point in point_metadata.values(): - point[polygon_metadata_field] = point.get(polygon_metadata_field, None) - - -def _map_zcta_to_climate_zones( - zcta_metadata, - iecc_climate_zone_metadata, - iecc_moisture_regime_metadata, - ba_climate_zone_metadata, - ca_climate_zone_metadata, -): - _compute_containment( - zcta_metadata, "zcta", iecc_climate_zone_metadata, "iecc_climate_zone" - ) - - _compute_containment( - zcta_metadata, "zcta", iecc_moisture_regime_metadata, "iecc_moisture_regime" - ) - - _compute_containment( - zcta_metadata, "zcta", ba_climate_zone_metadata, "ba_climate_zone" - ) - - _compute_containment( - zcta_metadata, "zcta", ca_climate_zone_metadata, "ca_climate_zone" - ) - - -def _map_isd_station_to_climate_zones( - isd_station_metadata, - iecc_climate_zone_metadata, - iecc_moisture_regime_metadata, - ba_climate_zone_metadata, - ca_climate_zone_metadata, -): - _compute_containment( - isd_station_metadata, "usaf_id", iecc_climate_zone_metadata, "iecc_climate_zone" - ) - - _compute_containment( - isd_station_metadata, - "usaf_id", - iecc_moisture_regime_metadata, - "iecc_moisture_regime", - ) - - _compute_containment( - isd_station_metadata, "usaf_id", ba_climate_zone_metadata, "ba_climate_zone" - ) - - _compute_containment( - isd_station_metadata, "usaf_id", ca_climate_zone_metadata, "ca_climate_zone" - ) - - -def _find_zcta_closest_isd_stations(zcta_metadata, isd_station_metadata, limit=None): - if limit is None: - limit = 10 - import pyproj - - geod = pyproj.Geod(ellps="WGS84") - - isd_usaf_ids, isd_lats, isd_lngs = zip( - *[ - ( - isd_station["usaf_id"], - float(isd_station["latitude"]), - float(isd_station["longitude"]), - ) - for isd_station in isd_station_metadata.values() - ] - ) - - isd_lats = np.array(isd_lats) - isd_lngs = np.array(isd_lngs) - - for zcta in zcta_metadata.values(): - zcta_lats = np.tile(zcta["latitude"], isd_lats.shape) - zcta_lngs = np.tile(zcta["longitude"], isd_lngs.shape) - - dists = geod.inv(zcta_lngs, zcta_lats, isd_lngs, isd_lats)[2] - sorted_dists = np.argsort(dists)[:limit] - - closest_isd_stations = [] - for i, idx in enumerate(sorted_dists): - usaf_id = isd_usaf_ids[idx] - isd_station = isd_station_metadata[usaf_id] - closest_isd_stations.append( - { - "usaf_id": usaf_id, - "distance_meters": int(round(dists[idx])), - "rank": i + 1, - "iecc_climate_zone_match": ( - zcta.get("iecc_climate_zone") - == isd_station.get("iecc_climate_zone") - ), - "iecc_moisture_regime_match": ( - zcta.get("iecc_moisture_regime") - == isd_station.get("iecc_moisture_regime") - ), - "ba_climate_zone_match": ( - zcta.get("ba_climate_zone") - == isd_station.get("ba_climate_zone") - ), - "ca_climate_zone_match": ( - zcta.get("ca_climate_zone") - == isd_station.get("ca_climate_zone") - ), - } - ) - zcta["closest_isd_stations"] = closest_isd_stations - - -def _create_table_structures(conn): - cur = conn.cursor() - cur.execute( - """ - create table isd_station_metadata ( - usaf_id text not null - , wban_ids text not null - , recent_wban_id text not null - , name text not null - , icao_code text - , latitude text - , longitude text - , elevation text - , state text - , ghcn_id text not null - , ghcn_map_method text not null - , ghcn_first_year integer - , ghcn_last_year integer - , quality text default 'low' - , iecc_climate_zone text - , iecc_moisture_regime text - , ba_climate_zone text - , ca_climate_zone text - ) - """ - ) - - cur.execute( - """ - create table isd_file_metadata ( - usaf_id text not null - , year text not null - , wban_id text not null - ) - """ - ) - - cur.execute( - """ - create table zcta_metadata ( - zcta_id text not null - , geometry text - , latitude text not null - , longitude text not null - , state text - , iecc_climate_zone text - , iecc_moisture_regime text - , ba_climate_zone text - , ca_climate_zone text - ) - """ - ) - - cur.execute( - """ - create table iecc_climate_zone_metadata ( - iecc_climate_zone text not null - , geometry text - ) - """ - ) - - cur.execute( - """ - create table iecc_moisture_regime_metadata ( - iecc_moisture_regime text not null - , geometry text - ) - """ - ) - - cur.execute( - """ - create table ba_climate_zone_metadata ( - ba_climate_zone text not null - , geometry text - ) - """ - ) - - cur.execute( - """ - create table ca_climate_zone_metadata ( - ca_climate_zone text not null - , name text not null - , geometry text - ) - """ - ) - - cur.execute( - """ - create table tmy3_station_metadata ( - usaf_id text not null - , name text not null - , state text not null - , class text not null - ) - """ - ) - - cur.execute( - """ - create table cz2010_station_metadata ( - usaf_id text not null - ) - """ - ) - - cur.execute( - """ - create table ghcn_inventory ( - usaf_id text not null - , year integer not null - , jan integer not null - , feb integer not null - , mar integer not null - , apr integer not null - , may integer not null - , jun integer not null - , jul integer not null - , aug integer not null - , sep integer not null - , oct integer not null - , nov integer not null - , dec integer not null - ) - """ - ) - - -def _write_isd_station_metadata_table(conn, isd_station_metadata): - cur = conn.cursor() - - rows = [ - ( - metadata["usaf_id"], - ",".join(metadata["wban_ids"]), - metadata["recent_wban_id"], - metadata["name"], - metadata["icao_code"], - metadata["latitude"], - metadata["longitude"], - metadata["elevation"], - metadata["state"], - metadata["ghcn_id"], - metadata["ghcn_map_method"], - metadata["ghcn_first_year"], - metadata["ghcn_last_year"], - metadata["quality"], - metadata["iecc_climate_zone"], - metadata["iecc_moisture_regime"], - metadata["ba_climate_zone"], - metadata["ca_climate_zone"], - ) - for station, metadata in sorted(isd_station_metadata.items()) - ] - cur.executemany( - """ - insert into isd_station_metadata( - usaf_id - , wban_ids - , recent_wban_id - , name - , icao_code - , latitude - , longitude - , elevation - , state - , ghcn_id - , ghcn_map_method - , ghcn_first_year - , ghcn_last_year - , quality - , iecc_climate_zone - , iecc_moisture_regime - , ba_climate_zone - , ca_climate_zone - ) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - """, - rows, - ) - cur.execute( - """ - create index isd_station_metadata_usaf_id on isd_station_metadata(usaf_id) - """ - ) - cur.execute( - """ - create index isd_station_metadata_state on isd_station_metadata(state) - """ - ) - cur.execute( - """ - create index isd_station_metadata_iecc_climate_zone on - isd_station_metadata(iecc_climate_zone) - """ - ) - cur.execute( - """ - create index isd_station_metadata_iecc_moisture_regime on - isd_station_metadata(iecc_moisture_regime) - """ - ) - cur.execute( - """ - create index isd_station_metadata_ba_climate_zone on - isd_station_metadata(ba_climate_zone) - """ - ) - cur.execute( - """ - create index isd_station_metadata_ca_climate_zone on - isd_station_metadata(ca_climate_zone) - """ - ) - cur.close() - conn.commit() - - -def _write_isd_file_metadata_table(conn, isd_file_metadata): - cur = conn.cursor() - - rows = [ - (metadata["usaf_id"], year, station_data["wban_id"]) - for isd_station, metadata in sorted(isd_file_metadata.items()) - for year, year_data in sorted(metadata["years"].items()) - for station_data in year_data - ] - - cur.executemany( - """ - insert into isd_file_metadata( - usaf_id - , year - , wban_id - ) values (?,?,?) - """, - rows, - ) - - cur.execute( - """ - create index isd_file_metadata_usaf_id on - isd_file_metadata(usaf_id) - """ - ) - cur.execute( - """ - create index isd_file_metadata_year on - isd_file_metadata(year) - """ - ) - cur.execute( - """ - create index isd_file_metadata_usaf_id_year on - isd_file_metadata(usaf_id, year) - """ - ) - cur.execute( - """ - create index isd_file_metadata_wban_id on - isd_file_metadata(wban_id) - """ - ) - - cur.close() - conn.commit() - - -def _write_zcta_metadata_table(conn, zcta_metadata, geometry=False): - cur = conn.cursor() - - rows = [ - ( - metadata["zcta"], - metadata["geometry"] if geometry else None, - metadata["latitude"], - metadata["longitude"], - metadata["state"], - metadata["iecc_climate_zone"], - metadata["iecc_moisture_regime"], - metadata["ba_climate_zone"], - metadata["ca_climate_zone"], - ) - for zcta, metadata in sorted(zcta_metadata.items()) - ] - cur.executemany( - """ - insert into zcta_metadata( - zcta_id - , geometry - , latitude - , longitude - , state - , iecc_climate_zone - , iecc_moisture_regime - , ba_climate_zone - , ca_climate_zone - ) values (?,?,?,?,?,?,?,?,?) - """, - rows, - ) - cur.execute( - """ - create index zcta_metadata_zcta_id on zcta_metadata(zcta_id) - """ - ) - cur.execute( - """ - create index zcta_metadata_state on zcta_metadata(state) - """ - ) - cur.execute( - """ - create index zcta_metadata_iecc_climate_zone on zcta_metadata(iecc_climate_zone) - """ - ) - cur.execute( - """ - create index zcta_metadata_iecc_moisture_regime on zcta_metadata(iecc_moisture_regime) - """ - ) - cur.execute( - """ - create index zcta_metadata_ba_climate_zone on zcta_metadata(ba_climate_zone) - """ - ) - cur.execute( - """ - create index zcta_metadata_ca_climate_zone on zcta_metadata(ca_climate_zone) - """ - ) - cur.close() - conn.commit() - - -def _write_iecc_climate_zone_metadata_table( - conn, iecc_climate_zone_metadata, geometry=True -): - cur = conn.cursor() - - rows = [ - (metadata["iecc_climate_zone"], metadata["geometry"] if geometry else None) - for iecc_climate_zone, metadata in sorted(iecc_climate_zone_metadata.items()) - ] - cur.executemany( - """ - insert into iecc_climate_zone_metadata( - iecc_climate_zone - , geometry - ) values (?,?) - """, - rows, - ) - cur.execute( - """ - create index iecc_climate_zone_metadata_iecc_climate_zone on - iecc_climate_zone_metadata(iecc_climate_zone) - """ - ) - cur.close() - conn.commit() - - -def _write_iecc_moisture_regime_metadata_table( - conn, iecc_moisture_regime_metadata, geometry=True -): - cur = conn.cursor() - - rows = [ - (metadata["iecc_moisture_regime"], metadata["geometry"] if geometry else None) - for iecc_moisture_regime, metadata in sorted( - iecc_moisture_regime_metadata.items() - ) - ] - cur.executemany( - """ - insert into iecc_moisture_regime_metadata( - iecc_moisture_regime - , geometry - ) values (?,?) - """, - rows, - ) - cur.execute( - """ - create index iecc_moisture_regime_metadata_iecc_moisture_regime on - iecc_moisture_regime_metadata(iecc_moisture_regime) - """ - ) - cur.close() - conn.commit() - - -def _write_ba_climate_zone_metadata_table( - conn, ba_climate_zone_metadata, geometry=True -): - cur = conn.cursor() - - rows = [ - (metadata["ba_climate_zone"], metadata["geometry"] if geometry else None) - for ba_climate_zone, metadata in sorted(ba_climate_zone_metadata.items()) - ] - cur.executemany( - """ - insert into ba_climate_zone_metadata( - ba_climate_zone - , geometry - ) values (?,?) - """, - rows, - ) - cur.execute( - """ - create index ba_climate_zone_metadata_ba_climate_zone on - ba_climate_zone_metadata(ba_climate_zone) - """ - ) - cur.close() - conn.commit() - - -def _write_ca_climate_zone_metadata_table( - conn, ca_climate_zone_metadata, geometry=True -): - cur = conn.cursor() - - rows = [ - ( - metadata["ca_climate_zone"], - metadata["name"], - metadata["geometry"] if geometry else None, - ) - for ca_climate_zone, metadata in sorted(ca_climate_zone_metadata.items()) - ] - cur.executemany( - """ - insert into ca_climate_zone_metadata( - ca_climate_zone - , name - , geometry - ) values (?,?,?) - """, - rows, - ) - cur.execute( - """ - create index ca_climate_zone_metadata_ca_climate_zone on - ca_climate_zone_metadata(ca_climate_zone) - """ - ) - cur.close() - conn.commit() - - -def _write_tmy3_station_metadata_table(conn, tmy3_station_metadata): - cur = conn.cursor() - rows = [ - (metadata["usaf_id"], metadata["name"], metadata["state"], metadata["class"]) - for tmy3_station, metadata in sorted(tmy3_station_metadata.items()) - ] - cur.executemany( - """ - insert into tmy3_station_metadata( - usaf_id - , name - , state - , class - ) values (?,?,?,?) - """, - rows, - ) - cur.execute( - """ - create index tmy3_station_metadata_usaf_id on - tmy3_station_metadata(usaf_id) - """ - ) - cur.close() - conn.commit() - - -def _write_cz2010_station_metadata_table(conn, cz2010_station_metadata): - cur = conn.cursor() - rows = [ - (metadata["usaf_id"],) - for cz2010_station, metadata in sorted(cz2010_station_metadata.items()) - ] - cur.executemany( - """ - insert into cz2010_station_metadata( - usaf_id - ) values (?) - """, - rows, - ) - cur.execute( - """ - create index cz2010_station_metadata_usaf_id on - cz2010_station_metadata(usaf_id) - """ - ) - cur.close() - conn.commit() - - -def build_metadata_db( - zcta_geometry=False, - iecc_climate_zone_geometry=True, - iecc_moisture_regime_geometry=True, - ba_climate_zone_geometry=True, - ca_climate_zone_geometry=True, -): - """Build database of metadata from primary sources. - - Downloads primary sources, clears existing DB, and rebuilds from scratch. - - Parameters - ---------- - zcta_geometry : bool, optional - Whether or not to include ZCTA geometry in database. - iecc_climate_zone_geometry : bool, optional - Whether or not to include IECC Climate Zone geometry in database. - iecc_moisture_regime_geometry : bool, optional - Whether or not to include IECC Moisture Regime geometry in database. - ba_climate_zone_geometry : bool, optional - Whether or not to include Building America Climate Zone geometry in database. - ca_climate_zone_geometry : bool, optional - Whether or not to include California Building Climate Zone Area geometry in database. - """ - - try: - import shapely - except ImportError: - raise ImportError("Loading polygons requires shapely.") - - try: - from bs4 import BeautifulSoup - except ImportError: - raise ImportError("Scraping TMY3 station data requires beautifulsoup4.") - - try: - import pyproj - except ImportError: - raise ImportError("Computing distances requires pyproj.") - - try: - import simplejson - except ImportError: - raise ImportError("Writing geojson requires simplejson.") - - download_path = _download_primary_sources() - conn = metadata_db_connection_proxy.reset_database() - - # Load data into memory - print("Loading ZCTAs") - zcta_metadata = _load_zcta_metadata(download_path) - - print("Loading counties") - county_metadata = _load_county_metadata(download_path) - - print("Merging county climate zones") - ( - iecc_climate_zone_metadata, - iecc_moisture_regime_metadata, - ba_climate_zone_metadata, - ) = _create_merged_climate_zones_metadata(county_metadata) - - print("Loading CA climate zones") - ca_climate_zone_metadata = _load_CA_climate_zone_metadata(download_path) - - print("Loading ISD station metadata") - isd_station_metadata = _load_isd_station_metadata(download_path) - - print("Mapping ISD stations to GHCNh ids") - _map_isd_stations_to_ghcn(isd_station_metadata, download_path) - - print("Loading ISD station file metadata") - isd_file_metadata = _load_isd_file_metadata(download_path, isd_station_metadata) - - print("Loading TMY3 station metadata") - tmy3_station_metadata = _load_tmy3_station_metadata(download_path) - - print("Loading CZ2010 station metadata") - cz2010_station_metadata = _load_cz2010_station_metadata() - - # Augment data in memory - print("Loading GHCNh inventory for registry stations") - ghcn_inventory = _load_registry_ghcn_inventory( - isd_station_metadata, _load_ghcnh_inventory(download_path) - ) - - print("Computing station quality from the GHCNh inventory") - # rough station quality: all months in the last 5 full years have - # more than 600 observations - _compute_station_quality_from_ghcnh( - isd_station_metadata, _load_ghcnh_inventory(download_path) - ) - - print("Mapping ZCTAs to climate zones") - # add county and ca climate zone mappings - _map_zcta_to_climate_zones( - zcta_metadata, - iecc_climate_zone_metadata, - iecc_moisture_regime_metadata, - ba_climate_zone_metadata, - ca_climate_zone_metadata, - ) - - print("Mapping ISD stations to climate zones") - # add county and ca climate zone mappings - _map_isd_station_to_climate_zones( - isd_station_metadata, - iecc_climate_zone_metadata, - iecc_moisture_regime_metadata, - ba_climate_zone_metadata, - ca_climate_zone_metadata, - ) - - # Write tables - print("Creating table structures") - _create_table_structures(conn) - - print("Writing ZCTA data") - _write_zcta_metadata_table(conn, zcta_metadata, geometry=zcta_geometry) - - print("Writing IECC climate zone data") - _write_iecc_climate_zone_metadata_table( - conn, iecc_climate_zone_metadata, geometry=iecc_climate_zone_geometry - ) - - print("Writing IECC moisture regime data") - _write_iecc_moisture_regime_metadata_table( - conn, iecc_moisture_regime_metadata, geometry=iecc_moisture_regime_geometry - ) - - print("Writing BA climate zone data") - _write_ba_climate_zone_metadata_table( - conn, ba_climate_zone_metadata, geometry=ba_climate_zone_geometry - ) - - print("Writing CA climate zone data") - _write_ca_climate_zone_metadata_table( - conn, ca_climate_zone_metadata, geometry=ca_climate_zone_geometry - ) - - print("Writing ISD station metadata") - _write_isd_station_metadata_table(conn, isd_station_metadata) - - print("Writing ISD file metadata") - _write_isd_file_metadata_table(conn, isd_file_metadata) - - print("Writing GHCNh inventory") - _write_ghcn_inventory_table(conn, ghcn_inventory) - - print("Writing TMY3 station metadata") - _write_tmy3_station_metadata_table(conn, tmy3_station_metadata) - - print("Writing CZ2010 station metadata") - _write_cz2010_station_metadata_table(conn, cz2010_station_metadata) - - print("Cleaning up...") - shutil.rmtree(download_path) - - print("\u2728 Completed! \u2728") - - -def inspect_metadata_db(): - subprocess.call(["sqlite3", metadata_db_connection_proxy.db_path]) diff --git a/eeweather/exceptions.py b/eeweather/exceptions.py index 0e0c6c6..2daf59f 100644 --- a/eeweather/exceptions.py +++ b/eeweather/exceptions.py @@ -1,136 +1,151 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +"""Exceptions and the warning type returned by data loads. +These are data-dependent conditions pipelines legitimately branch on. +Caller bugs (e.g. non-UTC datetimes) raise plain ValueError instead. """ class EEWeatherError(Exception): """Base class for exceptions in the eeweather package.""" - pass + def __init__(self, message): + super().__init__(message) + self.message = message -class UnrecognizedUSAFIDError(EEWeatherError): - """Raised when an unrecognized USAF station id is encountered. +class UnrecognizedStationError(EEWeatherError): + """Raised when a station id is not in the registry. Attributes ---------- value : str - the value which is not a valid USAF ID - message : str - a message describing the error + the value which is not a recognized station id """ def __init__(self, value): - self.value = value - self.message = ( - 'The value "{}" was not recognized as a valid USAF weather station' + super().__init__( + 'The value "{}" was not recognized as a valid weather station' " identifier.".format(value) ) + self.value = value -class UnrecognizedZCTAError(EEWeatherError): - """Raised when an unrecognized ZCTA is encountered. +class UnrecognizedPlaceError(EEWeatherError): + """Raised when a place code is not in the registry. Attributes ---------- - value : str - the value which is not a valid ZCTA - message : str - a message describing the error + kind : str + the place kind, e.g. ``'zcta'`` + code : str + the code which is not a recognized place of that kind """ - def __init__(self, value): - self.value = value - self.message = ( - 'The value "{}" was not recognized as a valid ZCTA identifier.'.format( - value - ) + def __init__(self, kind, code): + super().__init__( + 'The value "{}" was not recognized as a valid place of kind' + ' "{}".'.format(code, kind) ) + self.kind = kind + self.code = code -class DataNotAvailableError(EEWeatherError): - """Raised when data is not available for a particular station and year. +class AmbiguousIdentifierError(EEWeatherError): + """Raised when an external identifier maps to multiple stations and no + mapping is marked recent. Attributes ---------- - usaf_id : str - the USAF ID for which data does not exist. - year : int - the year for which data does not exist. - message : str - a message describing the error + namespace : str + the identifier system, e.g. ``'wban'`` + external_id : str + the ambiguous identifier + station_ids : tuple of str + the registry stations the identifier maps to """ - def __init__(self, usaf_id, year): - self.usaf_id = usaf_id - self.year = year - self.message = 'Data does not exist for station "{}" in year {}.'.format( - usaf_id, year + def __init__(self, namespace, external_id, station_ids): + super().__init__( + 'The {} id "{}" maps to multiple stations ({}) and cannot be' + " resolved.".format(namespace, external_id, ", ".join(station_ids)) ) + self.namespace = namespace + self.external_id = external_id + self.station_ids = tuple(station_ids) + +class DataNotAvailableError(EEWeatherError): + """Raised when a source has no data at all for a request. -class TMY3DataNotAvailableError(EEWeatherError): - """Raised when TMY3 data is not available for a particular station. + Partial coverage never raises; it surfaces as NaN values plus warnings. Attributes ---------- - usaf_id : str - the USAF ID for which TMY3 data does not exist. - message : str - a message describing the error + source : str + the source that had no data + station_id : str or None + the station requested; None for non-station sources + year : int or None + the year requested; None when the whole request had no data """ - def __init__(self, usaf_id): - self.usaf_id = usaf_id - self.message = 'TMY3 data does not exist for station "{}".'.format(usaf_id) + def __init__(self, source, *, station_id=None, year=None): + super().__init__( + "No {} data available for station={} year={}.".format( + source, station_id, year + ) + ) + self.source = source + self.station_id = station_id + self.year = year -class CZ2010DataNotAvailableError(EEWeatherError): - """Raised when CZ2010 data is not available for a particular station. +class NoQualifiedStationError(EEWeatherError): + """Raised when no station in the registry qualifies to estimate weather + at a location under the configured filters. Attributes ---------- - usaf_id : str - the USAF ID for which CZ2010 data does not exist. - message : str - a message describing the error + latitude, longitude : float + the location that could not be served """ - def __init__(self, usaf_id): - self.usaf_id = usaf_id - self.message = 'CZ2010 data does not exist for station "{}".'.format(usaf_id) + def __init__(self, latitude, longitude): + super().__init__( + "No qualified station found for location ({}, {}).".format( + latitude, longitude + ) + ) + self.latitude = latitude + self.longitude = longitude -class NonUTCTimezoneInfoError(EEWeatherError): - """Raised when input start and end date aren't explicitly defined - to have a UTC timezone. +class EEWeatherWarning(object): + """A warning describing a data condition, returned alongside loaded data. Attributes ---------- - usaf_id : str - the USAF ID for which CZ2010 data does not exist. - message : str - a message describing the error + qualified_name : str + Qualified name, e.g. ``'eeweather.data_gap'``. + description : str + Prose describing the nature of the warning. + data : dict + Data that reproducibly shows why the warning was issued. """ - def __init__(self, this_date): - self.message = ( - '"{}" does not have a UTC timezone. If using the datetime package, it should be' - " in the format datetime(1,1,1,tzinfo=pytz.UTC).".format(this_date) - ) + def __init__(self, qualified_name, description, data): + self.qualified_name = qualified_name + self.description = description + self.data = data + + def __repr__(self): + return "EEWeatherWarning(qualified_name={})".format(self.qualified_name) + + def json(self): + serialized = { + "qualified_name": self.qualified_name, + "description": self.description, + "data": self.data, + } + + return serialized diff --git a/eeweather/geo.py b/eeweather/geo.py deleted file mode 100644 index e212ee2..0000000 --- a/eeweather/geo.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import json - -from .connections import metadata_db_connection_proxy -from .exceptions import UnrecognizedUSAFIDError, UnrecognizedZCTAError - -from .utils import lazy_property -from .validation import valid_zcta_or_raise - - -__all__ = ("get_lat_long_climate_zones", "get_zcta_metadata", "zcta_to_lat_long") - - -class CachedData(object): - @lazy_property - def climate_zone_geometry(self): - try: - from shapely.geometry import shape - except ImportError: # pragma: no cover - raise ImportError( - "Matching by lat/lng within climate zone requires shapely" - ) - - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - - cur.execute( - """ - select - iecc_climate_zone, geometry - from - iecc_climate_zone_metadata - """ - ) - iecc_climate_zones = [ - (cz_id, shape(json.loads(geometry))) for (cz_id, geometry) in cur.fetchall() - ] - - cur.execute( - """ - select - iecc_moisture_regime, geometry - from - iecc_moisture_regime_metadata - """ - ) - iecc_moisture_regimes = [ - (cz_id, shape(json.loads(geometry))) for (cz_id, geometry) in cur.fetchall() - ] - - cur.execute( - """ - select - ba_climate_zone, geometry - from - ba_climate_zone_metadata - """ - ) - ba_climate_zones = [ - (cz_id, shape(json.loads(geometry))) for (cz_id, geometry) in cur.fetchall() - ] - - cur.execute( - """ - select - ca_climate_zone, geometry - from - ca_climate_zone_metadata - """ - ) - ca_climate_zones = [ - (cz_id, shape(json.loads(geometry))) for (cz_id, geometry) in cur.fetchall() - ] - - return ( - iecc_climate_zones, - iecc_moisture_regimes, - ba_climate_zones, - ca_climate_zones, - ) - - -cached_data = CachedData() - - -def get_lat_long_climate_zones(latitude, longitude): - """Get climate zones that contain lat/long coordinates. - - Parameters - ---------- - latitude : float - Latitude of point. - longitude : float - Longitude of point. - - Returns - ------- - climate_zones: dict of str - Region ids for each climate zone type. - """ - try: - from shapely.geometry import Point - except ImportError: # pragma: no cover - raise ImportError("Finding climate zone of lat/long points requires shapely.") - - ( - iecc_climate_zones, - iecc_moisture_regimes, - ba_climate_zones, - ca_climate_zones, - ) = cached_data.climate_zone_geometry - - point = Point(longitude, latitude) # x,y - climate_zones = {} - for iecc_climate_zone, shape in iecc_climate_zones: - if shape.contains(point): - climate_zones["iecc_climate_zone"] = iecc_climate_zone - break - else: - climate_zones["iecc_climate_zone"] = None - - for iecc_moisture_regime, shape in iecc_moisture_regimes: - if shape.contains(point): - climate_zones["iecc_moisture_regime"] = iecc_moisture_regime - break - else: - climate_zones["iecc_moisture_regime"] = None - - for ba_climate_zone, shape in ba_climate_zones: - if shape.contains(point): - climate_zones["ba_climate_zone"] = ba_climate_zone - break - else: - climate_zones["ba_climate_zone"] = None - - for ca_climate_zone, shape in ca_climate_zones: - if shape.contains(point): - climate_zones["ca_climate_zone"] = ca_climate_zone - break - else: - climate_zones["ca_climate_zone"] = None - - return climate_zones - - -def get_zcta_metadata(zcta): - """Get metadata about a ZIP Code Tabulation Area (ZCTA). - - Parameters - ---------- - zcta : str - ID of ZIP Code Tabulation Area - - Returns - ------- - metadata : dict - Dict of data about the ZCTA, including lat/long coordinates. - """ - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select - * - from - zcta_metadata - where - zcta_id = ? - """, - (zcta,), - ) - row = cur.fetchone() - if row is None: - raise UnrecognizedZCTAError(zcta) - return {col[0]: row[i] for i, col in enumerate(cur.description)} - - -def zcta_to_lat_long(zcta): - """Get location of ZCTA centroid - - Retrieves latitude and longitude of centroid of ZCTA - to use for matching with weather station. - - Parameters - ---------- - zcta : str - ID of the target ZCTA. - - Returns - ------- - latitude : float - Latitude of centroid of ZCTA. - longitude : float - Target Longitude of centroid of ZCTA. - """ - valid_zcta_or_raise(zcta) - - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - - cur.execute( - """ - select - latitude - , longitude - from - zcta_metadata - where - zcta_id = ? - """, - (zcta,), - ) - # match existence checked in validate_zcta_or_raise(zcta) - latitude, longitude = cur.fetchone() - - return float(latitude), float(longitude) diff --git a/eeweather/location.py b/eeweather/location.py new file mode 100644 index 0000000..3da9e5a --- /dev/null +++ b/eeweather/location.py @@ -0,0 +1,303 @@ +"""Weather at a geographic location.""" +from __future__ import annotations + +import json +from datetime import datetime + +import pandas as pd + +from .registry.summaries import get_place +from .registry.zones import zones_at +from .sources import StationSource +from .sources.engine import known_source_names, resolve_source, _route, _validate_requested +from .sources.matching import rank_stations + + + +__all__ = ("WeatherLocation",) + + +def _normalize_sources(sources): + if isinstance(sources, str): + sources = (sources,) + normalized = [] + for entry in sources: + if isinstance(entry, str): + resolved = resolve_source(entry) + if not hasattr(resolved, "estimate"): + resolved = StationSource(dataset=resolved) + entry = resolved + elif hasattr(entry, "fetch_year"): + entry = StationSource(dataset=entry) + normalized.append(entry) + names = [source.name for source in normalized] + if len(set(names)) != len(names): + raise ValueError( + "Duplicate source names in preference tuple: {}.".format( + ", ".join(names) + ) + ) + + return tuple(normalized) + + +class WeatherLocation(object): + """Weather at a latitude/longitude. + + The location is the primary abstraction; weather is estimated by the + configured sources. Each requested variable routes to the first + source in the preference tuple that serves it; the default resolves + the point to the nearest suitable GHCNh station. After a load, + ``provenance`` records how each value was produced (which source, + which station or cell, how far away). + + Parameters + ---------- + latitude, longitude : float + sources : str, source object, or tuple of these + Ordered source preference. Strings name built-in sources; Feed + objects and configured sources (e.g. ``StationSource``) are used + as given. + pins : dict of str to str, optional + Station id to use per source name, bypassing matching. Pins are + captured automatically from provenance at load time, so once a + location has loaded data, later loads reuse the same station per + source; :meth:`to_dict`/:meth:`to_json` carry them across + processes for consistency between baseline and reporting + periods. + """ + + def __init__( + self, latitude: float, longitude: float, sources=("ghcnh",), pins=None + ): + self.latitude = latitude + self.longitude = longitude + self.pins = dict(pins or {}) + self.sources = _normalize_sources(sources) + normals = [s.name for s in self.sources if getattr(s, "kind", None) == "normals"] + if normals: + raise ValueError( + "Normals sources are not preference sources: {}. Pin one" + " with load_data(source=...) instead.".format(", ".join(normals)) + ) + + self.provenance = None + + @classmethod + def from_place(cls, kind: str, code: str, sources=("ghcnh",)) -> "WeatherLocation": + """Construct a location from a coded place, e.g. + ``from_place("zcta", "91104")`` for a ZIP code tabulation area.""" + place = get_place(kind, code) + + return cls(place["latitude"], place["longitude"], sources=sources) + + def to_dict(self) -> dict: + """The location's replay state: coordinates, source names, and + station pins. A location rebuilt with :meth:`from_dict` loads from + the same station per source. Source configuration (rank/select + kwargs) is not captured; pins bypass matching, so replay does not + depend on it. + + To make replay faithful, any default :class:`StationSource` + without a captured pin is resolved and pinned here (registry-only, + no fetch) — so this mutates ``self.pins`` and, for a location with + no station within the distance cap, may raise + ``NoQualifiedStationError``. A source configured with + ``coverage_range`` or ``rating_period`` is pinned at load, not + here (resolving it depends on the request and may fetch), so + serializing one before any load raises ``ValueError``. Grid and + other non-station sources need no pin: coordinates plus source + name already round-trip deterministically. + """ + unrebuildable = [ + source.name for source in self.sources + if source.name not in known_source_names() + ] + if unrebuildable: + raise ValueError( + "Custom sources cannot be serialized: {}. Register them" + " with eeweather.sources.register to make them" + " nameable.".format(", ".join(unrebuildable)) + ) + + self._pin_unloaded_station_sources() + state = { + "latitude": self.latitude, + "longitude": self.longitude, + "sources": [source.name for source in self.sources], + "pins": dict(self.pins), + } + + return state + + def _pin_unloaded_station_sources(self) -> None: + """Capture a station pin for each configured StationSource that + lacks one, so serialized replay resolves to the same station.""" + for source in self.sources: + if not isinstance(source, StationSource): + continue + if source.name in self.pins: + continue + if source.period_dependent: + raise ValueError( + "Source {!r} is configured with coverage_range or" + " rating_period and has no captured pin; load once" + " before serializing it.".format(source.name) + ) + station, _distance, _warnings = source.resolve( + self.latitude, self.longitude + ) + self.pins[source.name] = station.id + + @classmethod + def from_dict(cls, data: dict) -> "WeatherLocation": + """Rebuild a location from :meth:`to_dict` output.""" + return cls( + data["latitude"], + data["longitude"], + sources=tuple(data["sources"]), + pins=data.get("pins"), + ) + + def to_json(self) -> str: + """:meth:`to_dict` as a JSON string.""" + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, text: str) -> "WeatherLocation": + """Rebuild a location from :meth:`to_json` output.""" + return cls.from_dict(json.loads(text)) + + def __repr__(self) -> str: + return "WeatherLocation({}, {})".format(self.latitude, self.longitude) + + @property + def zones(self) -> dict[str, str | None]: + """The zone containing this location, by zone system.""" + return zones_at(self.latitude, self.longitude) + + def candidates(self, **filters) -> pd.DataFrame: + """Ranked candidate stations for this location. + + Filters are those of + :func:`eeweather.sources.matching.rank_stations`: ``match_zones``, + ``match_subdivision`` (with ``site_subdivision``), ``has_sources``, + ``minimum_quality``, ``rating_period``, ``max_distance_meters``, + and ``max_difference_elevation_meters`` (with ``site_elevation``). + """ + return rank_stations(self.latitude, self.longitude, **filters) + + def _station_source(self, source=None): + if source is not None: + for candidate in self.sources: + if isinstance(candidate, StationSource) and ( + candidate is source + or candidate.name == source + or candidate.adapter is source + ): + return candidate + + return _normalize_sources((source,))[0] + for candidate in self.sources: + if isinstance(candidate, StationSource): + return candidate + + raise ValueError("No station-backed source is configured.") + + def _pinned_kwargs(self, estimation_source, load_kwargs): + """Load kwargs with this source's station pin applied.""" + kwargs = dict(load_kwargs) + if isinstance(estimation_source, StationSource): + kwargs["station"] = self.pins.get(estimation_source.name) + + return kwargs + + def _capture_pins(self, provenance): + """First resolution wins: record each source's station so later + loads reuse it.""" + for name, record in provenance.items(): + if record.station_id is not None: + self.pins.setdefault(name, record.station_id) + + def load_data( + self, + start: datetime, + end: datetime, + frequency="h", + variables=None, + source=None, + ignore_disqualification: bool = False, + **load_kwargs, + ): + """Weather at this location between two dates (inclusive). + + Each requested variable routes to the first source in the + preference tuple that serves it; ``source=`` pins every variable + to one source instead (typical-year sources are only reachable + this way). Routed requests never raise for missing data (NaN + values plus warnings); a pinned request with nothing at all for + the dates raises DataNotAvailableError. Records provenance on + ``self.provenance`` and the frame's ``attrs["provenance"]``, + keyed by source name. + + Returns + ------- + tuple of (pandas.DataFrame, list of EEWeatherWarning) + """ + if "sources" in load_kwargs: + raise TypeError( + "load_data() got an unexpected keyword argument 'sources'" + " (did you mean the constructor's sources=, or source= to" + " pin a single source for this load?)" + ) + if source is not None: + estimation_source = self._station_source(source) + df, warnings, provenance = estimation_source.estimate( + self.latitude, + self.longitude, + start, + end, + frequency=frequency, + variables=variables, + raise_when_empty=True, + ignore_disqualification=ignore_disqualification, + **self._pinned_kwargs(estimation_source, load_kwargs), + ) + df.attrs["provenance"] = provenance + self.provenance = provenance + self._capture_pins(provenance) + + return df, warnings + + # routed: the engine's routing groups variables by the first + # source serving them; typical-year sources never route + if variables is None: + variables = ("temperature",) + groups, requested = _route(variables, self.sources) + _validate_requested(requested) + + frames = [] + warnings = [] + provenance = {} + for estimation_source, group_variables in groups.items(): + df, group_warnings, group_provenance = estimation_source.estimate( + self.latitude, + self.longitude, + start, + end, + frequency=frequency, + variables=tuple(group_variables), + raise_when_empty=False, + ignore_disqualification=ignore_disqualification, + **self._pinned_kwargs(estimation_source, load_kwargs), + ) + frames.append(df) + warnings.extend(group_warnings) + provenance.update(group_provenance) + + df = pd.concat(frames, axis=1)[list(requested)] + df.attrs["provenance"] = provenance + self.provenance = provenance + self._capture_pins(provenance) + + return df, warnings diff --git a/eeweather/mockable.py b/eeweather/mockable.py deleted file mode 100644 index 7a79d86..0000000 --- a/eeweather/mockable.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -""" -Usage: - - # module.py - import mockable - - @mockable.mockable - def expensive_function(): - pass - - def function_a(): - mockable.expensive_function() - - - # test_module.py - import pytest - - @pytest.fixture - def mock_expensive_function(monkeypatch): - def cheap_func(): - pass - monkeypatch.setattr( - 'mockable.expensive_function', - cheap_func - ) - - def test_function_a(mock_expensive_function): - function_a() - -""" - -import sys - - -def mockable(): - def decorator(func): - setattr(sys.modules[__name__], func.__name__, func) - return func - - return decorator diff --git a/eeweather/py.typed b/eeweather/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/eeweather/ranking.py b/eeweather/ranking.py deleted file mode 100644 index 533ddf1..0000000 --- a/eeweather/ranking.py +++ /dev/null @@ -1,461 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import pandas as pd -import numpy as np -import pyproj - -import eeweather.mockable -from .exceptions import DataNotAvailableError -from .connections import metadata_db_connection_proxy -from .geo import get_lat_long_climate_zones -from .stations import WeatherStation, get_station_qualities -from .utils import lazy_property -from .warnings import EEWeatherWarning - -__all__ = ("rank_stations", "combine_ranked_stations", "select_station") - - -class CachedData(object): - @lazy_property - def all_station_metadata(self): - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select - isd.usaf_id - , isd.latitude - , isd.longitude - , isd.iecc_climate_zone - , isd.iecc_moisture_regime - , isd.ba_climate_zone - , isd.ca_climate_zone - , isd.quality as rough_quality - , isd.elevation - , isd.state - , tmy3.class as tmy3_class - , tmy3.usaf_id is not null as is_tmy3 - , cz2010.usaf_id is not null as is_cz2010 - from - isd_station_metadata as isd - left join cz2010_station_metadata as cz2010 on - isd.usaf_id = cz2010.usaf_id - left join tmy3_station_metadata as tmy3 on - isd.usaf_id = tmy3.usaf_id - order by - isd.usaf_id - """ - ) - - df = pd.DataFrame( - [ - {col[0]: val for col, val in zip(cur.description, row)} - for row in cur.fetchall() - ], - columns=[ - "usaf_id", - "latitude", - "longitude", - "iecc_climate_zone", - "iecc_moisture_regime", - "ba_climate_zone", - "ca_climate_zone", - "rough_quality", - "elevation", - "state", - "tmy3_class", - "is_tmy3", - "is_cz2010", - ], - ).set_index("usaf_id") - - df["latitude"] = df.latitude.astype(float) - df["longitude"] = df.longitude.astype(float) - df["elevation"] = df.elevation.astype(float) - df["is_tmy3"] = df.is_tmy3.astype(bool) - df["is_cz2010"] = df.is_cz2010.astype(bool) - return df - - -cached_data = CachedData() - - -def _combine_filters(filters, index): - combined_filters = pd.Series(True, index=index) - for f in filters: - combined_filters &= f - return combined_filters - - -def rank_stations( - site_latitude, - site_longitude, - site_state=None, - site_elevation=None, - match_iecc_climate_zone=False, - match_iecc_moisture_regime=False, - match_ba_climate_zone=False, - match_ca_climate_zone=False, - match_state=False, - minimum_quality=None, - rating_period=None, - minimum_tmy3_class=None, - max_distance_meters=None, - max_difference_elevation_meters=None, - is_tmy3=None, - is_cz2010=None, -): - """Get a ranked, filtered set of candidate weather stations and metadata - for a particular site. - - Parameters - ---------- - site_latitude : float - Latitude of target site for which to find candidate weather stations. - site_longitude : float - Longitude of target site for which to find candidate weather stations. - site_state : str, 2 letter abbreviation - US state of target site, used optionally to filter potential candidate - weather stations. Ignored unless ``match_state=True``. - site_elevation : float - Elevation of target site in meters, used optionally to filter potential - candidate weather stations. Ignored unless - ``max_difference_elevation_meters`` is set. - match_iecc_climate_zone : bool - If ``True``, filter candidate weather stations to those - matching the IECC climate zone of the target site. - match_iecc_moisture_regime : bool - If ``True``, filter candidate weather stations to those - matching the IECC moisture regime of the target site. - match_ca_climate_zone : bool - If ``True``, filter candidate weather stations to those - matching the CA climate zone of the target site. - match_ba_climate_zone : bool - If ``True``, filter candidate weather stations to those - matching the Building America climate zone of the target site. - match_state : bool - If ``True``, filter candidate weather stations to those - matching the US state of the target site, as specified by - ``site_state=True``. - rating_period : tuple of (datetime.datetime, datetime.datetime), optional - When given, station quality is rated from GHCNh monthly - observation counts over the five calendar years ending two years - after the period's last date (sliding back to end no later than - the last full year), and the ``rough_quality`` column and - ``minimum_quality`` filter use that rating. When None, the rating - covers the last five full years. - minimum_quality : str, ``'high'``, ``'medium'``, ``'low'`` - If given, filter candidate weather stations to those meeting or - exceeding the given quality, as summarized by the frequency and - availability of observations in the NOAA Integrated Surface Database. - minimum_tmy3_class : str, ``'I'``, ``'II'``, ``'III'`` - If given, filter candidate weather stations to those meeting or - exceeding the given class, as reported in the NREL TMY3 metadata. - max_distance_meters : float - If given, filter candidate weather stations to those within the - ``max_distance_meters`` of the target site location. - max_difference_elevation_meters : float - If given, filter candidate weather stations to those with elevations - within ``max_difference_elevation_meters`` of the target site elevation. - is_tmy3 : bool - If given, filter candidate weather stations to those for which TMY3 - normal year temperature data is available. - is_cz2010 : bool - If given, filter candidate weather stations to those for which CZ2010 - normal year temperature data is available. - - Returns - ------- - ranked_filtered_candidates : :any:`pandas.DataFrame` - Index is ``usaf_id``. Each row contains a potential weather station - match and metadata. Contains the following columns: - - - ``rank``: Rank of weather station match for the target site. - - ``distance_meters``: Distance from target site to weather station site. - - ``latitude``: Latitude of weather station site. - - ``longitude``: Longitude of weather station site. - - ``iecc_climate_zone``: IECC Climate Zone ID (1-8) - - ``iecc_moisture_regime``: IECC Moisture Regime ID (A-C) - - ``ba_climate_zone``: Building America climate zone name - - ``ca_climate_zone``: Califoria climate zone number - - ``rough_quality``: Approximate measure of frequency of GHCNh - observations data at weather station. - - ``elevation``: Elevation of weather station site, if available. - - ``state``: US state of weather station site, if applicable. - - ``tmy3_class``: Weather station class as reported by NREL TMY3, if - available - - ``is_tmy3``: Weather station has associated TMY3 data. - - ``is_cz2010``: Weather station has associated CZ2010 data. - - ``difference_elevation_meters``: Absolute difference in meters - between target site elevation and weather station elevation, if - available. - - """ - candidates = cached_data.all_station_metadata.copy() - - # compute distances - candidates_defined_lat_long = candidates[ - candidates.latitude.notnull() & candidates.longitude.notnull() - ] - candidates_latitude = candidates_defined_lat_long.latitude - candidates_longitude = candidates_defined_lat_long.longitude - tiled_site_latitude = np.tile(site_latitude, candidates_latitude.shape) - tiled_site_longitude = np.tile(site_longitude, candidates_longitude.shape) - geod = pyproj.Geod(ellps="WGS84") - dists = geod.inv( - tiled_site_longitude, - tiled_site_latitude, - candidates_longitude.values, - candidates_latitude.values, - )[2] - distance_meters = pd.Series(dists, index=candidates_defined_lat_long.index).reindex( - candidates.index - ) - candidates["distance_meters"] = distance_meters - - if site_elevation is not None: - difference_elevation_meters = (candidates.elevation - site_elevation).abs() - else: - difference_elevation_meters = None - candidates["difference_elevation_meters"] = difference_elevation_meters - - site_climate_zones = get_lat_long_climate_zones(site_latitude, site_longitude) - site_iecc_climate_zone = site_climate_zones["iecc_climate_zone"] - site_iecc_moisture_regime = site_climate_zones["iecc_moisture_regime"] - site_ca_climate_zone = site_climate_zones["ca_climate_zone"] - site_ba_climate_zone = site_climate_zones["ba_climate_zone"] - - # create filters - filters = [] - - if match_iecc_climate_zone: - if site_iecc_climate_zone is None: - filters.append(candidates.iecc_climate_zone.isnull()) - else: - filters.append(candidates.iecc_climate_zone == site_iecc_climate_zone) - if match_iecc_moisture_regime: - if site_iecc_moisture_regime is None: - filters.append(candidates.iecc_moisture_regime.isnull()) - else: - filters.append(candidates.iecc_moisture_regime == site_iecc_moisture_regime) - if match_ba_climate_zone: - if site_ba_climate_zone is None: - filters.append(candidates.ba_climate_zone.isnull()) - else: - filters.append(candidates.ba_climate_zone == site_ba_climate_zone) - if match_ca_climate_zone: - if site_ca_climate_zone is None: - filters.append(candidates.ca_climate_zone.isnull()) - else: - filters.append(candidates.ca_climate_zone == site_ca_climate_zone) - - if match_state: - if site_state is None: - filters.append(candidates.state.isnull()) - else: - filters.append(candidates.state == site_state) - - if is_tmy3 is not None: - filters.append(candidates.is_tmy3.isin([is_tmy3])) - if is_cz2010 is not None: - filters.append(candidates.is_cz2010.isin([is_cz2010])) - - if rating_period is not None: - start, end = rating_period - period_qualities = get_station_qualities(start, end) - candidates["rough_quality"] = period_qualities.reindex( - candidates.index, fill_value="low" - ) - - if minimum_quality == "low": - filters.append(candidates.rough_quality.isin(["high", "medium", "low"])) - elif minimum_quality == "medium": - filters.append(candidates.rough_quality.isin(["high", "medium"])) - elif minimum_quality == "high": - filters.append(candidates.rough_quality.isin(["high"])) - - if minimum_tmy3_class == "III": - filters.append(candidates.tmy3_class.isin(["I", "II", "III"])) - elif minimum_tmy3_class == "II": - filters.append(candidates.tmy3_class.isin(["I", "II"])) - elif minimum_tmy3_class == "I": - filters.append(candidates.tmy3_class.isin(["I"])) - - if max_distance_meters is not None: - filters.append(candidates.distance_meters <= max_distance_meters) - - if max_difference_elevation_meters is not None and site_elevation is not None: - filters.append( - candidates.difference_elevation_meters <= max_difference_elevation_meters - ) - - combined_filters = _combine_filters(filters, candidates.index) - filtered_candidates = candidates[combined_filters] - ranked_filtered_candidates = filtered_candidates.sort_values(by=["distance_meters"]) - - # add rank column - ranks = range(1, 1 + len(ranked_filtered_candidates)) - ranked_filtered_candidates.insert(0, "rank", ranks) - - return ranked_filtered_candidates[ - [ - "rank", - "distance_meters", - "latitude", - "longitude", - "iecc_climate_zone", - "iecc_moisture_regime", - "ba_climate_zone", - "ca_climate_zone", - "rough_quality", - "elevation", - "state", - "tmy3_class", - "is_tmy3", - "is_cz2010", - "difference_elevation_meters", - ] - ] - - -def combine_ranked_stations(rankings): - """Combine :any:`pandas.DataFrame` s of candidate weather stations to form - a hybrid ranking dataframe. - - Parameters - ---------- - rankings : list of :any:`pandas.DataFrame` - Dataframes of ranked weather station candidates and metadata. - All ranking dataframes should have the same columns and must be - sorted by rank. - - Returns - ------- - ranked_filtered_candidates : :any:`pandas.DataFrame` - Dataframe has a rank column and the same columns given in the source - dataframes. - """ - - if len(rankings) == 0: - raise ValueError("Requires at least one ranking.") - - combined_ranking = rankings[0] - for ranking in rankings[1:]: - filtered_ranking = ranking[~ranking.index.isin(combined_ranking.index)] - combined_ranking = pd.concat([combined_ranking, filtered_ranking]) - - combined_ranking["rank"] = range(1, 1 + len(combined_ranking)) - return combined_ranking - - -@eeweather.mockable.mockable() -def load_hourly_temp_data( - station, start_date, end_date, fetch_from_web -): # pragma: no cover - df, warnings = station.load_data( - start_date, end_date, fetch_from_web=fetch_from_web, - error_on_missing_years=False, - ) - - return df["temperature"], warnings - - -def select_station( - candidates, - coverage_range=None, - min_fraction_coverage=0.9, - distance_warnings=(50000, 200000), - rank=1, - fetch_from_web=True, -): - """Select a station from a list of candidates that meets given data - quality criteria. - - Parameters - ---------- - candidates : :any:`pandas.DataFrame` - A dataframe of the form given by :any:`eeweather.rank_stations` or - :any:`eeweather.combine_ranked_stations`, specifically having at least - an index with ``usaf_id`` values and the column ``distance_meters``. - - Returns - ------- - station, warnings : tuple of (:any:`eeweather.WeatherStation`, list of str) - A qualified weather station. ``None`` if no station meets criteria. - """ - - def _test_station(station): - if coverage_range is None: - return True, [] - else: - start_date, end_date = coverage_range - try: - tempC, warnings = eeweather.mockable.load_hourly_temp_data( - station, start_date, end_date, fetch_from_web - ) - except DataNotAvailableError: - return False, [] # reject - - # TODO(philngo): also need to incorporate within-day limits - if len(tempC) > 0: - fraction_coverage = tempC.notnull().sum() / float(len(tempC)) - return (fraction_coverage > min_fraction_coverage), warnings - else: - return False, [] # reject - - def _station_warnings(station, distance_meters): - return [ - EEWeatherWarning( - qualified_name="eeweather.exceeds_maximum_distance", - description=( - "Distance from target to weather station is greater" - "than the specified km." - ), - data={ - "distance_meters": distance_meters, - "max_distance_meters": d, - "rank": rank, - }, - ) - for d in distance_warnings - if distance_meters > d - ] - - n_stations_passed = 0 - for usaf_id, row in candidates.iterrows(): - station = WeatherStation(usaf_id) - test_result, warnings = _test_station(station) - if test_result: - n_stations_passed += 1 - if n_stations_passed == rank: - if not warnings: - warnings = [] - warnings.extend(_station_warnings(station, row.distance_meters)) - return station, warnings - - no_station_warning = EEWeatherWarning( - qualified_name="eeweather.no_weather_station_selected", - description=( - "No weather station found with the specified rank and" - " minimum fracitional coverage." - ), - data={"rank": rank, "min_fraction_coverage": min_fraction_coverage}, - ) - return None, [no_station_warning] diff --git a/eeweather/registry/__init__.py b/eeweather/registry/__init__.py new file mode 100644 index 0000000..76324be --- /dev/null +++ b/eeweather/registry/__init__.py @@ -0,0 +1 @@ +"""The packaged station and place registry.""" diff --git a/eeweather/registry/db.py b/eeweather/registry/db.py new file mode 100644 index 0000000..276d8ab --- /dev/null +++ b/eeweather/registry/db.py @@ -0,0 +1,174 @@ +"""Access to the packaged data files. + +Every connection opens the identifier crosswalk (identifiers.db) and +ATTACHes the regional geography packs (geography_*.db) plus the data +files source packages register at import, so identity, geography, fact, +availability, and quality queries can join across files. Connections are +cached per thread and reused; callers do not close them. +""" +import glob +import os +import sqlite3 +import threading +from collections import namedtuple + +import platformdirs + +from ..exceptions import UnrecognizedStationError, UnrecognizedPlaceError + + + +_REGISTRY_DIR = os.path.dirname(os.path.abspath(__file__)) +_SOURCES_DIR = os.path.join(os.path.dirname(_REGISTRY_DIR), "sources") + +UPDATED_DATA_DIR = os.path.join(platformdirs.user_data_dir("eeweather"), "registry") + +# an update writes this marker only after both files swap in, so its +# presence certifies a complete (non-torn) updated pair +COMMIT_MARKER = ".committed" + + +def data_path(packaged_path): + """The live path for a packaged data file: the downloaded update in + the user data directory when a completed update is present, else the + packaged copy. The updated copy is used only alongside the commit + marker that certifies the swap finished, so a torn pair is ignored. + Resolved at import, so an update applies to new processes.""" + updated = os.path.join(UPDATED_DATA_DIR, os.path.basename(packaged_path)) + marker = os.path.join(UPDATED_DATA_DIR, COMMIT_MARKER) + if os.path.exists(marker) and os.path.exists(updated): + return updated + + return packaged_path + + +PACKAGED_IDENTIFIERS_DB_PATH = os.path.join(_REGISTRY_DIR, "identifiers.db") +PACKAGED_GHCNH_DB_PATH = os.path.join(_SOURCES_DIR, "ghcnh", "ghcnh.db") + +IDENTIFIERS_DB_PATH = data_path(PACKAGED_IDENTIFIERS_DB_PATH) +GHCNH_DB_PATH = data_path(PACKAGED_GHCNH_DB_PATH) + +MONTH_COLUMNS = ( + "jan", "feb", "mar", "apr", "may", "jun", + "jul", "aug", "sep", "oct", "nov", "dec", +) + +Attachment = namedtuple( + "Attachment", ["alias", "path", "catalog", "inventory", "quality", "availability"] +) + + +def _geography_packs(): + packs = [] + for path in sorted(glob.glob(os.path.join(_REGISTRY_DIR, "geography_*.db"))): + alias = os.path.splitext(os.path.basename(path))[0] + packs.append((alias, path)) + + return packs + + +class MetadataDBConnectionProxy(object): + def __init__(self): + self.db_path = IDENTIFIERS_DB_PATH + self._local = threading.local() + self._attachments = [] + self.geography_aliases = [alias for alias, _ in _geography_packs()] + + def register_attachment( + self, alias, path, + catalog=False, inventory=False, quality=False, availability=False, + ): + """Register a source database to attach to every connection. + + ``catalog`` marks a station catalog (facts and zone assignments); + ``inventory``/``quality`` mark observation-count and rating + tables; ``availability`` marks an archive station list. Sources + register at import, before any connection is made. + """ + if any(a.alias == alias for a in self._attachments): + return + self._attachments.append( + Attachment(alias, path, catalog, inventory, quality, availability) + ) + + @property + def catalogs(self): + return [a.alias for a in self._attachments if a.catalog] + + @property + def inventory_sources(self): + return [a.alias for a in self._attachments if a.inventory] + + @property + def quality_sources(self): + return [a.alias for a in self._attachments if a.quality] + + @property + def availability_sources(self): + return [a.alias for a in self._attachments if a.availability] + + def get_connection(self): + connection = getattr(self._local, "connection", None) + if connection is None: + connection = sqlite3.connect(self.db_path) + for alias, path in _geography_packs(): + connection.execute( + "attach database ? as {}".format(alias), (path,) + ) + self._local.connection = connection + self._local.attached = set() + # attach any source registered since this thread's connection was + # made, so late registration (e.g. importing a custom source + # after a first query) is not silently missing + for attachment in self._attachments: + if attachment.alias not in self._local.attached: + connection.execute( + "attach database ? as {}".format(attachment.alias), + (attachment.path,), + ) + self._local.attached.add(attachment.alias) + + return connection + + def close(self): + """Close and drop this thread's cached connection, if any; the next + get_connection opens a fresh one. Attachments are re-applied then.""" + connection = getattr(self._local, "connection", None) + if connection is not None: + connection.close() + self._local.connection = None + self._local.attached = set() + + +metadata_db_connection_proxy = MetadataDBConnectionProxy() + + +def valid_station_id_or_raise(station_id): + """Raise UnrecognizedStationError unless the id is in the registry.""" + conn = metadata_db_connection_proxy.get_connection() + row = conn.execute( + "select exists (select 1 from station_identifier where station_id = ?)", + (station_id,), + ).fetchone() + if not row[0]: + raise UnrecognizedStationError(station_id) + + return True + + +def valid_place_or_raise(kind, code): + """Raise UnrecognizedPlaceError unless the place code is in a geography + pack.""" + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + for alias in proxy.geography_aliases: + row = conn.execute( + "select exists (select 1 from {}.place where kind = ? and code = ?)".format( + alias + ), + (kind, code), + ).fetchone() + if row[0]: + return True + + raise UnrecognizedPlaceError(kind, code) diff --git a/eeweather/registry/geography_us.db b/eeweather/registry/geography_us.db new file mode 100644 index 0000000..a68ff85 Binary files /dev/null and b/eeweather/registry/geography_us.db differ diff --git a/eeweather/registry/identifiers.db b/eeweather/registry/identifiers.db new file mode 100644 index 0000000..ccb5957 Binary files /dev/null and b/eeweather/registry/identifiers.db differ diff --git a/eeweather/registry/identifiers.py b/eeweather/registry/identifiers.py new file mode 100644 index 0000000..f76b96f --- /dev/null +++ b/eeweather/registry/identifiers.py @@ -0,0 +1,86 @@ +"""External identifier translation over the station_identifier table.""" +from ..exceptions import AmbiguousIdentifierError, UnrecognizedStationError +from .db import metadata_db_connection_proxy + + + +def translate(ids, from_namespace, to_namespace): + """Translate external ids between identifier systems. + + Parameters + ---------- + ids : iterable of str + External ids in ``from_namespace``. Ids with no mapping are absent + from the result. + from_namespace, to_namespace : str + Identifier systems, e.g. ``'usaf'``, ``'ghcn'``, ``'wban'``, + ``'icao'``. + + Returns + ------- + dict of str to tuple of str + Target ids keyed by input id. Values are always tuples; mappings + can be one-to-many in either direction. + """ + ids = list(dict.fromkeys(ids)) + conn = metadata_db_connection_proxy.get_connection() + placeholders = ", ".join("?" for _ in ids) + rows = conn.execute( + """ + select src.external_id, dst.external_id + from station_identifier as src + join station_identifier as dst on src.station_id = dst.station_id + where src.namespace = ? and dst.namespace = ? + and src.external_id in ({}) + order by src.external_id, dst.external_id + """.format(placeholders), + [from_namespace, to_namespace] + ids, + ).fetchall() + + mapping = {} + for source_id, target_id in rows: + mapping.setdefault(source_id, []) + if target_id not in mapping[source_id]: + mapping[source_id].append(target_id) + mapping = {key: tuple(values) for key, values in mapping.items()} + + return mapping + + +_NORMALIZERS = { + "icao": lambda v: v.strip().upper(), + "wban": lambda v: v.strip().zfill(5), + "usaf": lambda v: v.strip().zfill(6), + "wmo": lambda v: v.strip().zfill(5), +} + + +def resolve_station(namespace, external_id): + """The registry station id an external id maps to. + + Ids are normalized to the registry's uniform forms (icao uppercased; + wban/wmo zero-padded to 5, usaf to 6). When the id maps to several + stations, the mapping marked recent wins; with no recent mapping, + AmbiguousIdentifierError is raised. + """ + if namespace in _NORMALIZERS: + external_id = _NORMALIZERS[namespace](external_id) + conn = metadata_db_connection_proxy.get_connection() + rows = conn.execute( + "select station_id, recent from station_identifier" + " where namespace = ? and external_id = ?" + " order by station_id", + (namespace, external_id), + ).fetchall() + if not rows: + raise UnrecognizedStationError(external_id) + if len(rows) == 1: + return rows[0][0] + + recent = [station_id for station_id, is_recent in rows if is_recent] + if len(recent) == 1: + return recent[0] + + raise AmbiguousIdentifierError( + namespace, external_id, [station_id for station_id, _ in rows] + ) diff --git a/eeweather/registry/metadata.py b/eeweather/registry/metadata.py new file mode 100644 index 0000000..331d94b --- /dev/null +++ b/eeweather/registry/metadata.py @@ -0,0 +1,72 @@ +"""Station metadata assembled from the crosswalk and source catalogs.""" +from ..exceptions import UnrecognizedStationError +from .db import metadata_db_connection_proxy + + + +def get_station_metadata(station_id): + """Registry metadata for a station. + + Identity comes from the crosswalk; facts (name, coordinates, + elevation, country, subdivision), zone assignments, and the quality + rating come from the first registered source catalog that lists the + station; first/last inventory years come from every source that + keeps an observation inventory. + """ + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + + ids = {} + for namespace, external_id in conn.execute( + "select namespace, external_id from station_identifier" + " where station_id = ? order by namespace, external_id", + (station_id,), + ): + ids.setdefault(namespace, []).append(external_id) + if not ids: + raise UnrecognizedStationError(station_id) + + metadata = {"station_id": station_id, "ids": ids} + for alias in proxy.catalogs: + cur = conn.cursor() + cur.execute( + "select * from {}.stations where station_id = ?".format(alias), + (station_id,), + ) + row = cur.fetchone() + if row is None: + continue + for column, value in zip(cur.description, row): + metadata.setdefault(column[0], value) + metadata.setdefault( + "zones", + dict( + conn.execute( + "select system, zone_id from {}.station_zone" + " where station_id = ?".format(alias), + (station_id,), + ).fetchall() + ), + ) + metadata.setdefault("zones", {}) + + for alias in proxy.quality_sources: + row = conn.execute( + "select quality from {}.quality where station_id = ?".format(alias), + (station_id,), + ).fetchone() + if row is not None: + metadata.setdefault("quality", row[0]) + metadata.setdefault("quality", None) + + metadata["inventory_years"] = {} + for alias in proxy.inventory_sources: + row = conn.execute( + "select min(year), max(year) from {}.inventory" + " where station_id = ?".format(alias), + (station_id,), + ).fetchone() + if row[0] is not None: + metadata["inventory_years"][alias] = (row[0], row[1]) + + return metadata diff --git a/eeweather/registry/quality.py b/eeweather/registry/quality.py new file mode 100644 index 0000000..8d4f070 --- /dev/null +++ b/eeweather/registry/quality.py @@ -0,0 +1,113 @@ +"""Station quality ratings from per-source observation inventory.""" +from datetime import datetime, timezone + +import pandas as pd + +from .db import MONTH_COLUMNS, metadata_db_connection_proxy + + + +QUALITY_WINDOW_YEARS = 5 + +# every month of the rating window above these observation counts +HIGH_MONTHLY_OBSERVATIONS = 600 +MEDIUM_MONTHLY_OBSERVATIONS = 360 + + +def _quality_rating_window(anchor): + """Calendar years rating a request: five years ending two years after + the anchor date, sliding back to end no later than the last full + year.""" + last_full_year = datetime.now(timezone.utc).year - 1 + window_end = min(anchor.year + 2, last_full_year) + window_start = window_end - (QUALITY_WINDOW_YEARS - 1) + + return window_start, window_end + + +def _quality_from_minimum(minimum): + if minimum > HIGH_MONTHLY_OBSERVATIONS: + return "high" + elif minimum > MEDIUM_MONTHLY_OBSERVATIONS: + return "medium" + + return "low" + + +def _inventory_source_or_raise(source): + proxy = metadata_db_connection_proxy + if source is None: + return proxy.inventory_sources[0] + if source not in proxy.inventory_sources: + raise ValueError( + "Unknown inventory source: '{}'. Sources with inventory:" + " {}.".format(source, ", ".join(proxy.inventory_sources)) + ) + + return source + + +def get_station_quality(station_id, anchor, source=None): + """Station quality anchored to a date, from a source's observation + counts. + + Rates the five calendar years ending two years after the anchor date + (sliding back so the window ends no later than the last full year): + every month over 600 observations is high, over 360 is medium; + anything less, including absent months or years, is low. + """ + source = _inventory_source_or_raise(source) + window_start, window_end = _quality_rating_window(anchor) + conn = metadata_db_connection_proxy.get_connection() + cur = conn.cursor() + cur.execute( + """ + select year, {} + from {}.inventory + where station_id = ? and year between ? and ? + """.format(", ".join(MONTH_COLUMNS), source), + (station_id, window_start, window_end), + ) + counts_by_year = {row[0]: row[1:] for row in cur.fetchall()} + + minimum = None + for year in range(window_start, window_end + 1): + year_counts = counts_by_year.get(year, (0,) * 12) + year_minimum = min(year_counts) + if minimum is None or year_minimum < minimum: + minimum = year_minimum + + return _quality_from_minimum(minimum) + + +def get_station_qualities(anchor, source=None): + """Quality anchored to a date for every station, as a + station_id-indexed Series. Same rating as get_station_quality, computed + for the whole registry in one query.""" + source = _inventory_source_or_raise(source) + window_start, window_end = _quality_rating_window(anchor) + conn = metadata_db_connection_proxy.get_connection() + inventory = pd.read_sql_query( + """ + select station_id, year, {} + from {}.inventory + where year between ? and ? + """.format(", ".join(MONTH_COLUMNS), source), + conn, + params=(window_start, window_end), + ) + + months = list(MONTH_COLUMNS) + year_min = inventory[months].min(axis=1) + observed_min = year_min.groupby(inventory.station_id).min() + + # a station must have a row for every year of the window + n_years = window_end - window_start + 1 + year_counts = inventory.groupby("station_id").year.nunique() + observed_min = observed_min.where(year_counts >= n_years, 0) + + qualities = pd.Series("low", index=observed_min.index) + qualities[observed_min > MEDIUM_MONTHLY_OBSERVATIONS] = "medium" + qualities[observed_min > HIGH_MONTHLY_OBSERVATIONS] = "high" + + return qualities diff --git a/eeweather/registry/summaries.py b/eeweather/registry/summaries.py new file mode 100644 index 0000000..843fdeb --- /dev/null +++ b/eeweather/registry/summaries.py @@ -0,0 +1,175 @@ +"""Registry enumeration and place lookups.""" +import pandas as pd + +from ..exceptions import UnrecognizedPlaceError +from .db import metadata_db_connection_proxy + + + +def get_station_ids(state=None): + """Registry ids of all stations, optionally filtered by subdivision. + + Stations are enumerated from the registered source catalogs. + """ + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + station_ids = set() + for alias in proxy.catalogs: + if state is None: + cur = conn.execute( + "select station_id from {}.stations".format(alias) + ) + else: + cur = conn.execute( + "select station_id from {}.stations where subdivision = ?".format( + alias + ), + (state,), + ) + station_ids.update(row[0] for row in cur.fetchall()) + + return sorted(station_ids) + + +def get_zcta_ids(state=None): + """Codes of all ZCTA places, optionally filtered by subdivision.""" + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + zcta_ids = set() + for alias in proxy.geography_aliases: + if state is None: + cur = conn.execute( + "select code from {}.place where kind = 'zcta'".format(alias) + ) + else: + cur = conn.execute( + "select code from {}.place where kind = 'zcta'" + " and subdivision = ?".format(alias), + (state,), + ) + zcta_ids.update(row[0] for row in cur.fetchall()) + + return sorted(zcta_ids) + + +def get_place(kind, code): + """Registry metadata for a place: the place row fields plus ``zones``.""" + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + for alias in proxy.geography_aliases: + cur = conn.cursor() + cur.execute( + "select * from {}.place where kind = ? and code = ?".format(alias), + (kind, code), + ) + row = cur.fetchone() + if row is None: + continue + place = {col[0]: row[i] for i, col in enumerate(cur.description)} + place["zones"] = dict( + conn.execute( + "select system, zone_id from {}.place_zone" + " where kind = ? and code = ?".format(alias), + (kind, code), + ).fetchall() + ) + + return place + + raise UnrecognizedPlaceError(kind, code) + + +def search_stations(country=None, subdivision=None, has_sources=()): + """Registry stations as a DataFrame indexed by station id. + + One row per station from the registered source catalogs (first + catalog listing a station wins): name, latitude, longitude, + elevation, country, subdivision, quality, one column per zone + system, and per-source availability flags. + """ + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + + frames = [] + for alias in proxy.catalogs: + frames.append(_catalog_frame(conn, alias, proxy)) + df = pd.concat(frames) + df = df[~df.index.duplicated(keep="first")].sort_index() + + if country is not None: + df = df[df.country == country] + if subdivision is not None: + df = df[df.subdivision == subdivision] + for source in has_sources: + if source in proxy.catalogs: + continue + column = "is_{}".format(source) + if column not in df.columns: + raise ValueError("Unknown source: {}".format(source)) + df = df[df[column]] + + return df + + +def _catalog_frame(conn, alias, proxy): + availability_selects = "".join( + """ + , max({avail}.station_id) is not null as is_{avail}""".format(avail=avail) + for avail in proxy.availability_sources + ) + availability_joins = "".join( + """ + left join {avail}.stations as {avail} on + s.station_id = {avail}.station_id""".format(avail=avail) + for avail in proxy.availability_sources + ) + quality_select = ", null as quality" + quality_join = "" + if alias in proxy.quality_sources: + quality_select = ", max(q.quality) as quality" + quality_join = ( + "\n left join {alias}.quality as q on" + " s.station_id = q.station_id".format(alias=alias) + ) + + df = pd.read_sql_query( + """ + select + s.station_id + , s.name + , s.latitude + , s.longitude + , s.elevation + , s.country + , s.subdivision + {quality_select} + , max(case when z.system = 'iecc_climate_zone' then z.zone_id end) + as iecc_climate_zone + , max(case when z.system = 'iecc_moisture_regime' then z.zone_id end) + as iecc_moisture_regime + , max(case when z.system = 'ba_climate_zone' then z.zone_id end) + as ba_climate_zone + , max(case when z.system = 'ca_climate_zone' then z.zone_id end) + as ca_climate_zone + {availability_selects} + from + {alias}.stations as s + left join {alias}.station_zone as z on s.station_id = z.station_id + {quality_join} + {availability_joins} + group by s.station_id + order by s.station_id + """.format( + alias=alias, + quality_select=quality_select, + quality_join=quality_join, + availability_selects=availability_selects, + availability_joins=availability_joins, + ), + conn, + ).set_index("station_id") + for avail in proxy.availability_sources: + column = "is_{}".format(avail) + df[column] = df[column].astype(bool) + + return df diff --git a/eeweather/registry/update.py b/eeweather/registry/update.py new file mode 100644 index 0000000..9b497c3 --- /dev/null +++ b/eeweather/registry/update.py @@ -0,0 +1,361 @@ +"""Keep the live registry data current without package releases. + +The live packaged data (GHCNh station catalog, observation inventory, +quality ratings, identifier aliases) is updated into the platform user +data directory, where it takes precedence over the wheel's copies in new +processes. Updates prefer the repository's rolling-release channel — the +scheduled refresh publishes ready-made files served from GitHub's CDN, +so any number of clients updating costs NOAA nothing and no client +rebuilds locally. When the channel is unreachable, stale, or no newer +than the local data, the update falls back to rebuilding from the live +NOAA files directly, so updates keep working even if the repository goes +dormant. Static content (geography packs, archive station lists) never +changes and is not part of the update. Implausible files from either +channel abort an update, leaving the previous data in place. + +Updates run automatically: loading data or ranking stations starts a +background refresh when the live data is older than ``STALE_AFTER_DAYS`` +(quality rating windows advance when a calendar year completes, so six +months caps the lag behind that yearly step). Set +``EEWEATHER_AUTO_UPDATE=0`` to disable. ``python -m +eeweather.registry.update`` refreshes on demand; ``--clear`` removes +updated copies, returning to the packaged data. +""" +import argparse +import os +import shutil +import sqlite3 +import threading +import time +import warnings +from datetime import datetime, timedelta, timezone + +import requests + +from ..build.refresh import refresh +from .db import ( + COMMIT_MARKER, + GHCNH_DB_PATH, + PACKAGED_GHCNH_DB_PATH, + PACKAGED_IDENTIFIERS_DB_PATH, + UPDATED_DATA_DIR, + data_path, +) + + + +UPDATABLE = { + "ghcnh.db": PACKAGED_GHCNH_DB_PATH, + "identifiers.db": PACKAGED_IDENTIFIERS_DB_PATH, +} + +# source dbs whose meta(refreshed_at) stamp drives staleness; a db not +# listed here (e.g. identifiers.db, which carries no meta table) never +# forces the registry stale +AUTO_UPDATING_SOURCES = { + "ghcnh.db": GHCNH_DB_PATH, +} + +RELEASE_URL = ( + "https://github.com/opendsm/eeweather/releases/download/registry-latest/{}" +) + +DOWNLOAD_TIMEOUT_SECONDS = 300 + +# a downloaded pack must be at least this fraction as populated as the +# data it replaces +PLAUSIBILITY_FLOOR = 0.9 + +_FLOOR_TABLES = { + "ghcnh.db": ("stations", "inventory"), + "identifiers.db": ("station_identifier",), +} + +STALE_AFTER_DAYS = 183 + +RETRY_AFTER_DAYS = 1 + +# the update thread waits before touching the network, so short-lived +# processes (pipeline workers) exit before generating any traffic +UPDATE_DELAY_SECONDS = 60 + +_ATTEMPT_STAMP = "last_attempt" + +_process_checked = False + + +def refreshed_at(path=GHCNH_DB_PATH): + """When the data at ``path`` was last refreshed, or None if unstamped.""" + conn = sqlite3.connect(path) + try: + row = conn.execute( + "select value from meta where key = 'refreshed_at'" + ).fetchone() + except sqlite3.OperationalError: + row = None + finally: + conn.close() + if row is None: + return None + + return datetime.strptime(row[0], "%Y-%m-%d").replace(tzinfo=timezone.utc) + + +def _stale(now): + for path in AUTO_UPDATING_SOURCES.values(): + stamp = refreshed_at(path) + if stamp is None or now - stamp > timedelta(days=STALE_AFTER_DAYS): + return True + + return False + + +def _claim_attempt(now): + """Atomically claim the machine-wide right to attempt an update. + + The claim file doubles as the attempt record: exclusive creation + means exactly one process wins per retry window, however many + workers race for it. + """ + path = os.path.join(UPDATED_DATA_DIR, _ATTEMPT_STAMP) + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + try: + attempted = datetime.fromtimestamp( + os.path.getmtime(path), tz=timezone.utc + ) + except OSError: + return False + if now - attempted < timedelta(days=RETRY_AFTER_DAYS): + return False + try: + os.remove(path) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except OSError: + return False + os.close(fd) + + return True + + +def maybe_update(): + """Start a background update when the live data is stale. + + Checks at most once per process; across processes on a machine, an + atomic claim file limits attempts to one per ``RETRY_AFTER_DAYS`` + however many workers race for it, and the update thread waits + ``UPDATE_DELAY_SECONDS`` before touching the network, so short-lived + processes exit without generating traffic. Updates prefer the + CDN-served published pack, so even uncoordinated fleets of ephemeral + machines cost NOAA nothing; ``EEWEATHER_AUTO_UPDATE=0`` suppresses + the traffic entirely. The running process keeps the data it + imported; the updated copies serve subsequent processes. Returns the + update thread, or None when no update starts. + """ + global _process_checked + if _process_checked: + return None + _process_checked = True + if os.environ.get("EEWEATHER_AUTO_UPDATE", "1").lower() in ("0", "false"): + return None + now = datetime.now(timezone.utc) + if not _stale(now): + return None + + os.makedirs(UPDATED_DATA_DIR, exist_ok=True) + if not _claim_attempt(now): + return None + thread = threading.Thread(target=_update_or_warn, daemon=True) + thread.start() + + return thread + + +def _update_or_warn(): + try: + time.sleep(UPDATE_DELAY_SECONDS) + update() + except Exception as error: + warnings.warn( + "eeweather registry auto-update failed; using existing data:" + " {}".format(error) + ) + + +def _download(url, dest): + response = requests.get(url, timeout=DOWNLOAD_TIMEOUT_SECONDS, stream=True) + response.raise_for_status() + with open(dest, "wb") as f: + for chunk in response.iter_content(chunk_size=1 << 20): + f.write(chunk) + + +def _plausible_pack_or_raise(staged): + """Abort when a downloaded file is implausibly small next to the data + it would replace.""" + for filename, stage in staged.items(): + live = data_path(UPDATABLE[filename]) + stage_conn = sqlite3.connect(stage) + live_conn = sqlite3.connect(live) + try: + for table in _FLOOR_TABLES[filename]: + query = "select count(*) from {}".format(table) + staged_count = stage_conn.execute(query).fetchone()[0] + live_count = live_conn.execute(query).fetchone()[0] + if staged_count < PLAUSIBILITY_FLOOR * live_count: + raise RuntimeError( + "Implausible published {}: {} has {} rows, live data" + " has {}.".format(filename, table, staged_count, live_count) + ) + finally: + stage_conn.close() + live_conn.close() + + +def _write_marker(): + """Certify that a complete updated pair is installed. Written via an + atomic rename so the marker only ever appears whole; data_path serves + the updated pair only when it is present, so a torn swap is ignored. + Only the marker's presence is consulted; its contents are unused.""" + marker = os.path.join(UPDATED_DATA_DIR, COMMIT_MARKER) + stage = marker + ".staging" + with open(stage, "w"): + pass + os.replace(stage, marker) + + +def _clear_marker(): + """Retract certification before a swap begins, so an interrupted swap + (first update or any re-update) is never certified and reads fall back + to the packaged pair until _write_marker completes.""" + marker = os.path.join(UPDATED_DATA_DIR, COMMIT_MARKER) + if os.path.exists(marker): + os.remove(marker) + + +def _install_published(now): + """Install the published registry pack; returns its stamp, or None + when the channel is unreachable, implausible, stale, or no newer + than the live data (the NOAA rebuild is the fallback). A failed + replace leaves no marker, so reads fall back to the packaged pair.""" + staged = {} + try: + for filename in UPDATABLE: + stage = os.path.join(UPDATED_DATA_DIR, filename + ".download") + _download(RELEASE_URL.format(filename), stage) + staged[filename] = stage + published = refreshed_at(staged["ghcnh.db"]) + if published is None or now - published > timedelta(days=STALE_AFTER_DAYS): + return None + current = refreshed_at() + if current is not None and published <= current: + return None + _plausible_pack_or_raise(staged) + _clear_marker() + for filename, stage in staged.items(): + os.replace(stage, os.path.join(UPDATED_DATA_DIR, filename)) + staged[filename] = None + _write_marker() + except (requests.RequestException, sqlite3.Error, RuntimeError, OSError): + return None + finally: + for stage in staged.values(): + if stage is not None and os.path.exists(stage): + os.remove(stage) + + return published + + +def _rebuild(): + """Refresh staged copies of the live data from NOAA and swap them in.""" + staged = {} + for filename, packaged in UPDATABLE.items(): + dest = os.path.join(UPDATED_DATA_DIR, filename) + stage = dest + ".staging" + if os.path.exists(dest): + shutil.copy(dest, stage) + else: + shutil.copy(packaged, stage) + staged[filename] = stage + + try: + counts = refresh( + ghcnh_path=staged["ghcnh.db"], + identifiers_path=staged["identifiers.db"], + ) + try: + _clear_marker() + for filename, stage in staged.items(): + os.replace(stage, os.path.join(UPDATED_DATA_DIR, filename)) + _write_marker() + except OSError: + return None + finally: + for stage in staged.values(): + if os.path.exists(stage): + os.remove(stage) + + return counts + + +def update(): + """Update the registry data on this machine; returns a summary. + + Prefers the published rolling-release pack (CDN-served, no local + rebuild); falls back to rebuilding from the live NOAA files when the + channel is unreachable, implausible, stale, or no newer than the + live data. Takes effect in new processes. + + Readers never observe a torn pair: the commit marker is retracted + before the swap and rewritten only after both files are in place, so + an interrupted swap falls back to the packaged pair. Concurrent + updaters are not serialized across processes — the background path is + gated once per process and per machine, and a manual update racing it + is the unguarded case; readers stay consistent regardless. + """ + os.makedirs(UPDATED_DATA_DIR, exist_ok=True) + published = _install_published(datetime.now(timezone.utc)) + if published is not None: + summary = { + "channel": "published", + "refreshed_at": published.strftime("%Y-%m-%d"), + } + + return summary + + counts = _rebuild() + + return counts + + +def clear(): + """Remove updated registry data, returning to the packaged copies.""" + removed = [] + # retract the marker first so a partial clear is never certified + for filename in [COMMIT_MARKER] + list(UPDATABLE) + [_ATTEMPT_STAMP]: + path = os.path.join(UPDATED_DATA_DIR, filename) + if os.path.exists(path): + os.remove(path) + removed.append(path) + + return removed + + +def main(): + parser = argparse.ArgumentParser(prog="python -m eeweather.registry.update") + parser.add_argument( + "--clear", + action="store_true", + help="remove updated registry data, returning to packaged copies", + ) + args = parser.parse_args() + + if args.clear: + print({"removed": clear()}) + else: + print(update()) + + +if __name__ == "__main__": + main() diff --git a/eeweather/registry/zones.py b/eeweather/registry/zones.py new file mode 100644 index 0000000..a5cdf54 --- /dev/null +++ b/eeweather/registry/zones.py @@ -0,0 +1,47 @@ +"""Point-in-zone lookups against the packaged geography packs.""" +import json +from functools import cached_property + +from shapely.geometry import Point, shape + +from .db import metadata_db_connection_proxy + + + +class _ZoneGeometries(object): + @cached_property + def by_system(self): + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + geometries = {} + for alias in proxy.geography_aliases: + for system, zone_id, geometry in conn.execute( + "select system, zone_id, geometry from {}.zone" + " order by system, zone_id".format(alias) + ): + geometries.setdefault(system, []).append( + (zone_id, shape(json.loads(geometry))) + ) + + return geometries + + +_zone_geometries = _ZoneGeometries() + + +def zones_at(latitude, longitude): + """The zone containing a point, by zone system. + + Returns a dict with one entry per zone system in the geography packs; + the value is None when no zone of that system contains the point. + """ + point = Point(longitude, latitude) + zones = {} + for system, geometries in _zone_geometries.by_system.items(): + zones[system] = None + for zone_id, geometry in geometries: + if geometry.contains(point): + zones[system] = zone_id + break + + return zones diff --git a/eeweather/resources/__init__.py b/eeweather/resources/__init__.py deleted file mode 100644 index a5ebcd3..0000000 --- a/eeweather/resources/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" diff --git a/eeweather/resources/metadata.db b/eeweather/resources/metadata.db deleted file mode 100644 index 4fab6fc..0000000 Binary files a/eeweather/resources/metadata.db and /dev/null differ diff --git a/eeweather/sources/__init__.py b/eeweather/sources/__init__.py index 84ba50c..7d259cd 100644 --- a/eeweather/sources/__init__.py +++ b/eeweather/sources/__init__.py @@ -1,24 +1,24 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +"""Weather-data sources and estimation strategies. +A *source* is a provider of weather data. Strings name the built-in +sources (``"ghcnh"``, ``"tmy3"``, ``"cz2010"``); configured or custom +sources are objects. Station-keyed external data plugs in through the +:class:`Feed` protocol; location-keyed (gridded) data implements +:class:`Source` directly. """ -from .ghcnh import DEFAULT_VARIABLES, fetch_ghcnh_hourly - - - -__all__ = ("DEFAULT_VARIABLES", "fetch_ghcnh_hourly") +from .base import Feed, NormalsSource, Source +from .engine import register, variables +from .station_source import StationSource +from .vocabulary import Variable + + + +__all__ = ( + "Source", + "StationSource", + "Feed", + "NormalsSource", + "Variable", + "register", + "variables", +) diff --git a/eeweather/sources/base.py b/eeweather/sources/base.py new file mode 100644 index 0000000..d408aa2 --- /dev/null +++ b/eeweather/sources/base.py @@ -0,0 +1,205 @@ +"""The contracts weather-data sources implement. + +``Source`` is the location-level estimation contract; ``Feed`` is the +station-keyed observation contract; ``NormalsSource`` is the +typical-year contract, including the machinery its implementations +share. The engine and location layers interpret these kinds; adapters +subclass them. +""" +from __future__ import annotations + +import time +from datetime import datetime, timedelta, timezone + +import pandas as pd +import requests + +from ..exceptions import DataNotAvailableError +from ..registry.db import metadata_db_connection_proxy + + + +class Source(object): + """Estimates weather at a geographic location. + + Subclasses implement ``estimate`` for one estimation strategy: a + station-based source resolves the point to a nearby weather station, + a grid-based source samples a gridded product at the point. All + return the same ``(DataFrame, warnings, provenance)`` contract so the + strategy is interchangeable behind a location; ``provenance`` maps + each source name used to a Provenance record. + """ + + def estimate( + self, + latitude: float, + longitude: float, + start: datetime, + end: datetime, + frequency="h", + variables=("temperature",), + **load_kwargs, + ): + raise NotImplementedError + + +class Feed(object): + """The protocol for station-keyed observation data. + + Implementations declare how their data is keyed and fetched; the + engine supplies caching, alignment, warnings, and provenance, and the + registry supplies station choice. Frames returned by ``fetch_year`` + carry canonical variable names and units. + + Attributes + ---------- + name : str + Unique source name; appears in provenance and cache keys. + kind : str + ``"observations"``. + id_namespace : str + The identifier system the data is keyed by (``"ghcn"``, + ``"usaf"``, ...); translated from registry ids by the engine. + variables : tuple of str + Canonical variables the feed serves. + default_variables : tuple of str + Variables a bare load requests. + cacheable : bool + Whether the engine should cache fetched blocks; declare False + for fast local backends. + """ + + kind = "observations" + cacheable = True + default_variables = ("temperature",) + + def fetch_year(self, external_id: str, year: int, variables) -> pd.DataFrame: + raise NotImplementedError + + +REQUEST_TRIES = 3 + +REQUEST_RETRY_BACKOFF_SECONDS = 5 + +REQUEST_TIMEOUT_SECONDS = 120 + + +def request_text(url): + """Fetch a url's text, retrying connection errors and server errors + with growing backoff; client errors raise immediately.""" + for attempt in range(REQUEST_TRIES): + try: + response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + except requests.HTTPError: + if response.status_code < 500 or attempt == REQUEST_TRIES - 1: + raise + time.sleep(REQUEST_RETRY_BACKOFF_SECONDS * (attempt + 1)) + except requests.RequestException: + if attempt == REQUEST_TRIES - 1: + raise + time.sleep(REQUEST_RETRY_BACKOFF_SECONDS * (attempt + 1)) + else: + return response.text + + +def parse_normals_csv(text): + """Parse a TMY-format CSV into an hourly temperature series over the + year 1900 in UTC.""" + index = pd.date_range( + "1900-01-01 00:00", "1900-12-31 23:00", freq="h", tz=timezone.utc + ) + ts = pd.Series(None, index=index, dtype=float) + + lines = text.splitlines() + + utc_offset_str = lines[0].split(",")[3] + utc_offset = timedelta(seconds=3600 * float(utc_offset_str)) + + for line in lines[2:]: + row = line.split(",") + month = row[0][0:2] + day = row[0][3:5] + hour = int(row[1][0:2]) - 1 + + # YYYYMMDDHH + date_string = "1900{}{}{:02d}".format(month, day, hour) + + dt = datetime.strptime(date_string, "%Y%m%d%H") - utc_offset + + # The offset can push the first or last few hours of the year + # into an adjacent year; fold them back into 1900. + dt = dt.replace(year=1900, tzinfo=timezone.utc) + temp_C = float(row[31]) + + ts[dt] = temp_C + + return ts + + +class NormalsSource(object): + """A typical-year (normals) source. + + One typical year per station, served from a USAF-named archive of + TMY-format CSVs listed in the source's packaged station table. The + engine tiles it onto requested calendar years (Feb 29 is NaN), + caches it, and serves it through ``load_data(source=...)`` — never + through routing, so typical values cannot silently mix with + observations. Subclasses declare ``name`` and the archive + ``_url(usaf_id)``. + + Attributes + ---------- + name : str + Unique source name; appears in provenance and cache keys. + kind : str + ``"normals"``. + variables : tuple of str + Canonical variables the source serves. + cacheable : bool + Whether the engine should cache fetched blocks. + """ + + kind = "normals" + cacheable = True + variables = ("temperature",) + default_variables = ("temperature",) + + def _url(self, usaf_id): + raise NotImplementedError + + def archive_station(self, station_id): + """The archive row for a station, raising DataNotAvailableError + when the station has no data in this normals source.""" + conn = metadata_db_connection_proxy.get_connection() + cur = conn.cursor() + cur.execute( + "select * from {}.stations where station_id = ?".format(self.name), + (station_id,), + ) + row = cur.fetchone() + if row is None: + raise DataNotAvailableError(self.name, station_id=station_id) + archive = {col[0]: row[i] for i, col in enumerate(cur.description)} + + return archive + + def fetch(self, station_id): + """The station's typical-year hourly temperature series (year 1900). + + Raises DataNotAvailableError when the station has no data in this + normals source. + """ + archive = self.archive_station(station_id) + url = self._url(archive["usaf_id"]) + try: + text = request_text(url) + except requests.HTTPError as error: + if error.response is not None and error.response.status_code == 404: + raise DataNotAvailableError( + self.name, station_id=station_id + ) from error + raise + ts = parse_normals_csv(text) + + return ts diff --git a/eeweather/sources/cz2010/__init__.py b/eeweather/sources/cz2010/__init__.py new file mode 100644 index 0000000..c04f98d --- /dev/null +++ b/eeweather/sources/cz2010/__init__.py @@ -0,0 +1,15 @@ +"""California Energy Commission CZ2010 typical-year temperatures.""" +import os + +from ...registry.db import metadata_db_connection_proxy +from .source import CZ2010Source + + + +__all__ = ("CZ2010Source", "DATA_PATH") + +# packaged archive station list +DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cz2010.db") +metadata_db_connection_proxy.register_attachment( + "cz2010", DATA_PATH, availability=True +) diff --git a/eeweather/sources/cz2010/cz2010.db b/eeweather/sources/cz2010/cz2010.db new file mode 100644 index 0000000..a345e49 Binary files /dev/null and b/eeweather/sources/cz2010/cz2010.db differ diff --git a/eeweather/sources/cz2010/source.py b/eeweather/sources/cz2010/source.py new file mode 100644 index 0000000..354dbfa --- /dev/null +++ b/eeweather/sources/cz2010/source.py @@ -0,0 +1,14 @@ +"""California Energy Commission CZ2010 typical-year temperatures.""" +from ..base import NormalsSource + + + +class CZ2010Source(NormalsSource): + name = "cz2010" + + def _url(self, usaf_id): + url = "https://storage.googleapis.com/oee-cz2010/csv/{}_CZ2010.CSV".format( + usaf_id + ) + + return url diff --git a/eeweather/sources/engine.py b/eeweather/sources/engine.py new file mode 100644 index 0000000..5dea37f --- /dev/null +++ b/eeweather/sources/engine.py @@ -0,0 +1,813 @@ +"""The source-agnostic load engine. + +Owns everything between an adapter's fetch and the frame a user receives: +variable routing, per-year caching with variable-union refresh, +typical-year tiling, range alignment, frequency aggregation, gap +warnings, and provenance. Frames for one request range and frequency +always share an identical UTC index regardless of source, so they join +safely. + +Missing-data semantics: partial coverage (missing years, station gaps) +surfaces as NaN values plus warnings. A pinned source with nothing at all +for the request raises DataNotAvailableError; a routed request never +raises for missing data. +""" +from __future__ import annotations + +from collections import namedtuple +from datetime import datetime, timedelta, timezone + +import pandas as pd + +import eeweather.cache +from ..exceptions import DataNotAvailableError, EEWeatherWarning +from ..registry.identifiers import translate +from ..registry.update import maybe_update +from .cz2010 import CZ2010Source +from .ghcnh import GHCNhSource +from .tmy3 import TMY3Source +from .vocabulary import ( + aggregation_for, + all_variables, + register_variables, + valid_variables_or_raise, +) + + + +BUILTIN_SOURCES = { + adapter.name: adapter + for adapter in (GHCNhSource(), TMY3Source(), CZ2010Source()) +} + +_registered_sources = {} + + +def register(source, vocabulary=()): + """Register a custom source under its name. + + Registration makes the source nameable everywhere a built-in name + works — preference tuples, ``load_data(source=...)``, and location + serialization (``to_dict``/``from_dict``) — without any change to + this repository. Accepts station-keyed feeds (``fetch_year``), + normals sources (``fetch``), and estimation/grid sources + (``estimate``). Registering the same name again replaces the + previous object; built-in names cannot be replaced. Returns the + source. + + A source serving variables outside the vocabulary passes their + definitions as ``vocabulary`` (an iterable of + ``eeweather.sources.Variable``); they become requestable and + routable like canonical entries. Canonical names are reserved, and + definitions must agree across sources, so the first registration of + a name freezes its unit. + """ + name = getattr(source, "name", None) + if not isinstance(name, str) or not name: + raise ValueError("A source must declare a non-empty string name.") + if name in BUILTIN_SOURCES: + raise ValueError("'{}' is a built-in source name.".format(name)) + is_estimation = hasattr(source, "estimate") + is_fetch_year = hasattr(source, "fetch_year") + if not is_fetch_year and not hasattr(source, "fetch") and not is_estimation: + raise ValueError( + "A registered source must declare one of: fetch_year" + " (station-keyed feeds), fetch (normals sources), or" + " estimate (estimation/grid sources)." + ) + if is_estimation: + if not hasattr(source, "variables"): + raise ValueError("An estimation source must declare 'variables'.") + if not hasattr(source, "kind"): + raise ValueError("An estimation source must declare 'kind'.") + elif is_fetch_year: + if not hasattr(source, "variables"): + raise ValueError("A fetch_year source must declare 'variables'.") + if not hasattr(source, "kind"): + raise ValueError("A fetch_year source must declare 'kind'.") + if not hasattr(source, "id_namespace"): + raise ValueError("A fetch_year source must declare 'id_namespace'.") + else: + if not hasattr(source, "variables"): + raise ValueError("A normals source must declare 'variables'.") + register_variables(vocabulary) + known = all_variables() + undeclared = [v for v in source.variables if v not in known] + if undeclared: + raise ValueError( + "Source '{}' serves variables the vocabulary lacks: {}. Pass" + " their definitions via register(..., vocabulary=...).".format( + name, ", ".join(undeclared) + ) + ) + _registered_sources[name] = source + + return source + + +def known_source_names(): + """Names resolvable to a source: built-in plus registered.""" + return set(BUILTIN_SOURCES) | set(_registered_sources) + + +def resolve_source(source): + """The adapter for a source name or source object.""" + if isinstance(source, str): + if source in BUILTIN_SOURCES: + return BUILTIN_SOURCES[source] + if source in _registered_sources: + return _registered_sources[source] + + raise ValueError( + "Unknown source: '{}'. Known sources: {}.".format( + source, ", ".join(sorted(known_source_names())) + ) + ) + + return source + + +def sources_serving(variable): + """Names of built-in and registered sources that serve a canonical + variable.""" + catalog = {**BUILTIN_SOURCES, **_registered_sources} + names = [ + name + for name, adapter in sorted(catalog.items()) + if variable in adapter.variables + ] + + return names + + + +TRAILING_GAP_WARNING_THRESHOLD = timedelta(days=1) +LEADING_GAP_WARNING_THRESHOLD = timedelta(days=1) +INTERNAL_GAP_WARNING_THRESHOLD = timedelta(days=7) + +_ProvenanceFields = namedtuple( + "Provenance", + ["kind", "source", "variables", "station_id", "distance_meters", "payload"], +) + + +class Provenance(_ProvenanceFields): + """How a value was produced. Station fields are None for non-station + sources; ``payload`` is always a dict, empty unless the source adds + source-specific detail (e.g. a grid cell or interpolation method).""" + __slots__ = () + + def __new__( + cls, kind, source, variables, + station_id=None, distance_meters=None, payload=None, + ): + if payload is None: + payload = {} + + record = super().__new__( + cls, kind, source, variables, station_id, distance_meters, payload + ) + + return record + + +def _store(): + return eeweather.cache.key_value_store_proxy.get_store() + + +def _datetime_is_utc(dt): + if dt.tzinfo is None: + return False + + return dt.utcoffset().total_seconds() == 0 + + +def _validate_range(start, end): + if not _datetime_is_utc(start): + raise ValueError( + "start must be an explicit-UTC datetime, got: {}".format(start) + ) + if not _datetime_is_utc(end): + raise ValueError("end must be an explicit-UTC datetime, got: {}".format(end)) + if start > end: + raise ValueError( + "start must not be after end, got: {} > {}".format(start, end) + ) + + +def _validate_requested(variables): + if len(variables) == 0: + raise ValueError("At least one variable must be requested.") + if len(set(variables)) != len(variables): + raise ValueError( + "Duplicate variables requested: {}".format(", ".join(variables)) + ) + + +def _external_id(adapter, station_id): + """The id the adapter fetches by, translated from the registry id.""" + namespace = adapter.id_namespace + if namespace == "ghcn": + return station_id + mapping = translate([station_id], "ghcn", namespace) + external_ids = mapping.get(station_id) + if not external_ids: + raise DataNotAvailableError(adapter.name, station_id=station_id) + + return external_ids[0] + + +def _data_gap_warnings(ts, source, variable): + """EEWeatherWarnings for requested ranges the returned data does not + cover: entirely empty series, late-starting or early-ending data, and + long internal gaps.""" + warnings = [] + if len(ts) == 0: + return warnings + + if ts.isna().all(): + warnings.append( + EEWeatherWarning( + qualified_name="eeweather.no_data_in_requested_range", + description="No data was available within the requested range.", + data={ + "source": source, + "variable": variable, + "requested_start": ts.index[0].isoformat(), + "requested_end": ts.index[-1].isoformat(), + }, + ) + ) + + return warnings + + first_valid = ts.first_valid_index() + leading_gap = first_valid - ts.index[0] + if leading_gap > LEADING_GAP_WARNING_THRESHOLD: + warnings.append( + EEWeatherWarning( + qualified_name="eeweather.data_starts_late", + description=( + "Data begins {} after the start of the requested" + " range.".format(leading_gap) + ), + data={ + "source": source, + "variable": variable, + "first_valid": first_valid.isoformat(), + "requested_start": ts.index[0].isoformat(), + }, + ) + ) + + last_valid = ts.last_valid_index() + trailing_gap = ts.index[-1] - last_valid + if trailing_gap > TRAILING_GAP_WARNING_THRESHOLD: + warnings.append( + EEWeatherWarning( + qualified_name="eeweather.data_truncated", + description=( + "Data ends {} before the end of the requested range.".format( + trailing_gap + ) + ), + data={ + "source": source, + "variable": variable, + "last_valid": last_valid.isoformat(), + "requested_end": ts.index[-1].isoformat(), + }, + ) + ) + + interior = ts.loc[first_valid:last_valid] + if len(interior) > 1: + period = interior.index[1] - interior.index[0] + is_missing = interior.isna() + max_gap_periods = int(is_missing.groupby((~is_missing).cumsum()).sum().max()) + max_gap = max_gap_periods * period + if max_gap > INTERNAL_GAP_WARNING_THRESHOLD: + warnings.append( + EEWeatherWarning( + qualified_name="eeweather.data_gap", + description=( + "Data contains an internal gap of {}.".format(max_gap) + ), + data={ + "source": source, + "variable": variable, + "max_gap_days": max_gap / timedelta(days=1), + }, + ) + ) + + return warnings + + +# observation caching: one JSON block per (source, station, year) + + +def observation_cache_key(source_name, station_id, year): + return "{}-hourly-{}-{}".format(source_name, station_id, year) + + +def serialize_hourly_data(df): + rows = [ + [index.strftime("%Y%m%d%H")] + values + for index, values in zip( + df.index, df.astype(object).where(df.notna(), None).values.tolist() + ) + ] + serialized = {"columns": list(df.columns), "rows": rows} + + return serialized + + +def deserialize_hourly_data(data): + index = pd.to_datetime( + [row[0] for row in data["rows"]], format="%Y%m%d%H", utc=True + ) + df = pd.DataFrame( + [row[1:] for row in data["rows"]], + index=index, + columns=data["columns"], + dtype=float, + ) + + return df.sort_index().resample("h").mean() + + +def _read_cached_year(adapter, station_id, year): + """The fresh cached block for a station-year, or None.""" + store = _store() + key = observation_cache_key(adapter.name, station_id, year) + if not store.key_exists(key): + return None + if eeweather.cache._expired(store.key_updated(key), year): + store.clear(key) + + return None + + return deserialize_hourly_data(store.retrieve_json(key)) + + +def _fetch_year(adapter, station_id, external_id, year, variables): + """One year of observations resampled to an hourly frame. + + Raises DataNotAvailableError when the station has no observations at + all for the year. + """ + raw = adapter.fetch_year(external_id, year, variables) + if len(raw) == 0: + raise DataNotAvailableError(adapter.name, station_id=station_id, year=year) + + point = [c for c in raw.columns if aggregation_for(c) != "sum"] + accumulation = [c for c in raw.columns if aggregation_for(c) == "sum"] + parts = [] + if point: + # CalTRACK 2.3.3 + parts.append( + raw[point] + .resample("min") + .mean() + .interpolate(method="linear", limit=60, limit_direction="both") + .resample("h") + .mean() + ) + if accumulation: + # accumulations add up within the hour and are never fabricated + # by interpolation + parts.append(raw[accumulation].resample("h").sum(min_count=1)) + df = pd.concat(parts, axis=1)[list(raw.columns)] + + return df + + +def _load_observation_year( + adapter, station_id, external_id, year, variables, + read_from_cache, write_to_cache, fetch_from_web, +): + """One year of hourly data, from cache when it covers the request. + + A cache entry serves the request when it is fresh and holds every + requested variable. Fetches request the union of the requested and + already-cached variables so a cache refresh never drops columns. + """ + cached = None + if adapter.cacheable: + cached = _read_cached_year(adapter, station_id, year) + + cache_covers_request = cached is not None and set(variables) <= set(cached.columns) + if read_from_cache and cache_covers_request: + return cached[list(variables)] + + if not fetch_from_web: + raise DataNotAvailableError(adapter.name, station_id=station_id, year=year) + + if cached is None: + cached_columns = () + else: + cached_columns = tuple(cached.columns) + fetch_variables = tuple(dict.fromkeys(variables + cached_columns)) + df = _fetch_year(adapter, station_id, external_id, year, fetch_variables) + if adapter.cacheable and write_to_cache: + _store().save_json( + observation_cache_key(adapter.name, station_id, year), + serialize_hourly_data(df), + ) + + return df.reindex(columns=list(variables)) + + +def _load_observations( + adapter, station_id, start, end, variables, + read_from_cache, write_to_cache, fetch_from_web, +): + """Hourly observations over the requested years; missing years surface + as warnings, and NaN rows after alignment.""" + warnings = [] + data = [] + try: + external_id = _external_id(adapter, station_id) + except DataNotAvailableError: + # station-level condition: the id has no alias in the adapter's + # namespace; one warning, not one per year + warnings.append( + EEWeatherWarning( + qualified_name="eeweather.data_not_available", + description="Station has no {} identifier".format( + adapter.id_namespace + ), + data={"source": adapter.name, "station_id": station_id}, + ) + ) + external_id = None + if external_id is None: + years = [] + else: + years = range(start.year, end.year + 1) + for year in years: + try: + data.append( + _load_observation_year( + adapter, station_id, external_id, year, variables, + read_from_cache, write_to_cache, fetch_from_web, + ) + ) + except DataNotAvailableError: + warnings.append( + EEWeatherWarning( + qualified_name="eeweather.data_not_available", + description="Data not available", + data={ + "source": adapter.name, + "station_id": station_id, + "year": year, + }, + ) + ) + + if data: + df = pd.concat(data) + else: + df = pd.DataFrame( + columns=list(variables), + index=pd.DatetimeIndex([], tz=timezone.utc), + dtype=float, + ) + + return df, warnings + + +# normals: one typical-year JSON block per (source, station), tiled onto +# the requested calendar years + + +def normals_cache_key(source_name, station_id): + return "{}-hourly-{}".format(source_name, station_id) + + +def _load_normals_block( + adapter, station_id, read_from_cache, write_to_cache, fetch_from_web +): + store = _store() + key = normals_cache_key(adapter.name, station_id) + cached_ok = adapter.cacheable and store.key_exists(key) + column = adapter.variables[0] + + if read_from_cache and cached_ok: + cached = deserialize_hourly_data(store.retrieve_json(key)) + + return cached[column] + + if not fetch_from_web: + raise DataNotAvailableError(adapter.name, station_id=station_id) + + ts = adapter.fetch(station_id) + if adapter.cacheable and write_to_cache: + store.save_json(key, serialize_hourly_data(ts.to_frame(name=column))) + + return ts + + +def _load_normals( + adapter, station_id, start, end, variables, + read_from_cache, write_to_cache, fetch_from_web, +): + """The typical year tiled onto each requested calendar year by + month-day-hour; Feb 29 is NaN.""" + single_year = _load_normals_block( + adapter, station_id, read_from_cache, write_to_cache, fetch_from_web + ) + + data = [] + for year in range(start.year, end.year + 1): + tiled_index = single_year.index.map(lambda t: t.replace(year=year)) + data.append(pd.Series(single_year.values, index=tiled_index)) + ts = pd.concat(data).resample("h").mean() + df = ts.to_frame(name=adapter.variables[0]) + no_warnings = [] + + return df, no_warnings + + +# routing + + +def _route(variables, adapters): + """Group requested variables by the first adapter declaring them. + + Normals adapters never participate; they are reachable only by + explicit pin. Unroutable variables raise, naming the sources that do + serve them. + """ + routable = [a for a in adapters if a.kind == "observations"] + if variables == "all": + union = [] + for adapter in routable: + for name in adapter.variables: + if name not in union: + union.append(name) + variables = tuple(union) + valid_variables_or_raise(variables) + + groups = {} + for variable in variables: + chosen = None + for adapter in routable: + if variable in adapter.variables: + chosen = adapter + break + if chosen is None: + raise ValueError( + "No configured source serves '{}'. Sources that serve it:" + " {}.".format(variable, ", ".join(sources_serving(variable)) or "none") + ) + groups.setdefault(chosen, []).append(variable) + requested = tuple(variables) + + return groups, requested + + +def load_data( + station_id: str, + start: datetime, + end: datetime, + frequency="h", + variables=None, + source=None, + sources=("ghcnh",), + read_from_cache: bool = True, + write_to_cache: bool = True, + fetch_from_web: bool = True, + raise_when_empty: bool | None = None, +): + """Load a station's weather data between two dates (inclusive). + + Parameters + ---------- + station_id : str + Registry station id. + start, end : datetime.datetime + Request range; must be explicit UTC. + frequency : str or pandas offset + A pandas offset alias, parsed and validated by pandas itself + (e.g. '30min', 'h', 'D', 'W', 'MS', 'YS'). Frequencies coarser + than hourly aggregate the hourly values within each UTC period + by each variable's vocabulary aggregation: means for + point-in-time variables, sums (never gap-interpolated) for + accumulations. Frequencies finer than hourly (they must divide + the hour evenly) interpolate point-in-time variables linearly + between hourly values and spread accumulations evenly, never + crossing a missing hour. + variables : tuple of str, or 'all' + Canonical variable names; 'all' is every variable the configured + sources serve (or, pinned, the pinned source's full vocabulary). + source : str or source object, optional + Pin every requested variable to this source, bypassing routing. + Typical-year sources (tmy3, cz2010) are only reachable this way. + sources : tuple of (str or source object) + Ordered source preference for routing; per variable, the first + source declaring it wins. + read_from_cache, write_to_cache, fetch_from_web : bool + Cache and network controls. + raise_when_empty : bool, optional + Whether a request yielding no data at all raises + DataNotAvailableError; defaults to True for pinned requests and + False for routed ones (partial coverage never raises either way). + + Returns + ------- + tuple of (pandas.DataFrame, list of EEWeatherWarning) + One column per requested variable, indexed over the full + requested range at the requested frequency in UTC; periods + without data are NaN. The frame's ``attrs["provenance"]`` maps + each source used to a Provenance record. + """ + maybe_update() + _validate_range(start, end) + + # the frequency vocabulary is pandas', bound by delegation: pandas + # parses the alias, so its spellings, deprecations, and renames apply + # here automatically + offset = pd.tseries.frequencies.to_offset(frequency) + + pinned = source is not None + if raise_when_empty is None: + raise_when_empty = pinned + if pinned: + adapter = resolve_source(source) + if variables is None: + variables = tuple(adapter.default_variables) + elif variables == "all": + variables = tuple(adapter.variables) + _validate_requested(tuple(variables)) + unservable = [v for v in variables if v not in adapter.variables] + if unservable: + raise ValueError( + "Source '{}' does not serve: {}. It serves: {}.".format( + adapter.name, + ", ".join(unservable), + ", ".join(adapter.variables), + ) + ) + groups = {adapter: list(variables)} + requested = tuple(variables) + else: + if variables is None: + variables = ("temperature",) + adapters = [resolve_source(entry) for entry in sources] + groups, requested = _route(variables, adapters) + _validate_requested(requested) + + warnings = [] + frames = [] + provenance = {} + for adapter, group_variables in groups.items(): + if adapter.kind == "observations": + loader = _load_observations + elif adapter.kind == "normals": + loader = _load_normals + else: + raise ValueError("Unknown source kind: {}".format(adapter.kind)) + group_df, group_warnings = loader( + adapter, station_id, start, end, tuple(group_variables), + read_from_cache, write_to_cache, fetch_from_web, + ) + warnings.extend(group_warnings) + frames.append(group_df) + provenance[adapter.name] = Provenance( + kind=adapter.kind, + source=adapter.name, + variables=tuple(group_variables), + station_id=station_id, + distance_meters=None, + payload={}, + ) + + df = pd.concat(frames, axis=1)[list(requested)] + if offset != pd.tseries.frequencies.to_offset("h"): + df = _resample_by_vocabulary(df, offset) + df = df[start:end] + + # start and end dates need to fall exactly on period boundaries; a + # period partially before start is excluded, end's period is included + if isinstance(offset, pd.tseries.offsets.Tick): + range_start = pd.Timestamp(start).ceil(offset) + range_end = pd.Timestamp(end).floor(offset) + else: + range_start = offset.rollforward(pd.Timestamp(start).normalize()) + if range_start < pd.Timestamp(start): + range_start = range_start + offset + range_end = offset.rollback(pd.Timestamp(end).normalize()) + + # cover the full requested range even when no data loaded + df = df.reindex(pd.date_range(range_start, range_end, freq=offset)) + + # nothing at all for the requested dates (not merely the requested + # calendar years) raises for pinned requests + if raise_when_empty and len(df) > 0 and df.isna().all().all(): + source_name = ", ".join(sorted(a.name for a in groups)) + raise DataNotAvailableError(source_name, station_id=station_id) + + for adapter, group_variables in groups.items(): + for variable in group_variables: + warnings.extend(_data_gap_warnings(df[variable], adapter.name, variable)) + + df.attrs["provenance"] = provenance + + return df, warnings + + +def load_cached_data(station_id, source_name="ghcnh"): + """All fresh cached hourly observations for a station from one + source, or None when nothing is cached. Applies the same staleness + rule as loads.""" + store = _store() + prefix = "{}-hourly-{}-".format(source_name, station_id) + data = [] + for key in store.keys(prefix): + year = int(key.rsplit("-", 1)[1]) + if eeweather.cache._expired(store.key_updated(key), year): + continue + data.append(deserialize_hourly_data(store.retrieve_json(key))) + if not data: + return None + df = pd.concat(data).resample("h").mean() + + return df + + +def _resample_by_vocabulary(df, offset): + """Hourly values at the requested frequency, column by column. + + Coarser periods roll up by the column's vocabulary aggregation; a + period with no data at all is NaN regardless of aggregation. Every + label is its period's start. Sub-hourly slots interpolate + point-in-time columns linearly between hourly values and spread + accumulations evenly, never crossing a missing hour. + """ + if isinstance(offset, pd.tseries.offsets.Tick) and ( + pd.Timedelta(offset) < pd.Timedelta(hours=1) + ): + return _upsample(df, offset) + + resampled = df.resample(offset, label="left", closed="left") + columns = {} + for column in df.columns: + aggregation = aggregation_for(column) + if aggregation == "sum": + columns[column] = resampled[column].sum(min_count=1) + else: + columns[column] = getattr(resampled[column], aggregation)() + aggregated = pd.DataFrame(columns) + + return aggregated + + +def _upsample(df, offset): + """Hourly values at a finer frequency; the offset must divide the + hour evenly.""" + step = pd.Timedelta(offset) + if pd.Timedelta(hours=1) % step != pd.Timedelta(0): + raise ValueError( + "A sub-hourly frequency must divide the hour evenly," + " got: {}".format(offset.freqstr) + ) + slots = int(pd.Timedelta(hours=1) / step) + + up = df.resample(offset).asfreq() + columns = {} + for column in df.columns: + if aggregation_for(column) == "sum": + spread = df[column].reindex(up.index, method="ffill", limit=slots - 1) + columns[column] = spread / slots + else: + filled = up[column].interpolate(method="linear", limit_area="inside") + valid = df[column].notna() + left_valid = valid.reindex(up.index, method="ffill") + right_valid = valid.reindex(up.index, method="bfill") + columns[column] = filled.where(left_valid & right_valid) + upsampled = pd.DataFrame(columns) + + return upsampled + + +def variables() -> pd.DataFrame: + """The canonical variable vocabulary and which sources serve each + entry, as a DataFrame indexed by variable name.""" + catalog = all_variables() + rows = [] + for name, variable in sorted(catalog.items()): + rows.append( + { + "unit": variable.unit, + "description": variable.description, + "aggregation": variable.aggregation, + "sources": tuple(sources_serving(name)), + } + ) + df = pd.DataFrame(rows, index=sorted(catalog)) + df.index.name = "variable" + + return df diff --git a/eeweather/sources/ghcnh.py b/eeweather/sources/ghcnh.py deleted file mode 100644 index ef15f38..0000000 --- a/eeweather/sources/ghcnh.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import io -import time - -import pandas as pd -import requests - - - -API_URL = "https://www.ncei.noaa.gov/access/services/data/v1" - -API_REQUEST_TRIES = 3 - -API_RETRY_BACKOFF_SECONDS = 5 - -DATASET = "global-historical-climatology-network-hourly" - -DEFAULT_VARIABLES = ("temperature",) - - -def _get_api_request_params(ghcn_id, year, variables): - params = { - "dataset": DATASET, - # DATE is only included in the response when explicitly requested - "dataTypes": ",".join(("DATE",) + tuple(variables)), - "stations": ghcn_id, - "startDate": "{}-01-01".format(year), - "endDate": "{}-12-31".format(year), - } - - return params - - -def fetch_ghcnh_hourly(ghcn_id, year, variables=DEFAULT_VARIABLES): - """Fetch one year of GHCNh observations for a station. - - The request is retried on connection and http errors, as the api has - been shown to intermittently fail. - - Parameters - ---------- - ghcn_id : str - GHCNh station id, e.g. ``'USW00023234'``. - year : int - Calendar year to fetch. - variables : tuple of str - GHCNh variable names, e.g. ``('temperature', 'wind_speed')``. - Temperatures are degrees Celsius; units for other variables follow - the GHCNh documentation. - - Returns - ------- - pandas.DataFrame - One column per requested variable, indexed by UTC observation time. - Observations sharing a timestamp are averaged. Values the station - did not report are NaN. Empty when the station has no data for the - year. - """ - params = _get_api_request_params(ghcn_id, year, variables) - - for attempt in range(API_REQUEST_TRIES): - try: - resp = requests.get(url=API_URL, params=params) - resp.raise_for_status() - except requests.RequestException: - if attempt == API_REQUEST_TRIES - 1: - raise - # the api intermittently returns 5xx bursts; immediate retries - # land inside the burst - time.sleep(API_RETRY_BACKOFF_SECONDS * (attempt + 1)) - else: - break - - empty_index = pd.DatetimeIndex([], tz="UTC") - if resp.text.strip() == "": - return pd.DataFrame(columns=list(variables), index=empty_index, dtype=float) - - raw = pd.read_csv(io.StringIO(resp.text), dtype=str) - if len(raw) == 0: - return pd.DataFrame(columns=list(variables), index=empty_index, dtype=float) - - index = pd.to_datetime(raw["DATE"]).dt.tz_localize("UTC").rename(None) - df = pd.DataFrame(index=index) - for variable in variables: - if variable in raw.columns: - df[variable] = pd.to_numeric(raw[variable], errors="coerce").values - else: - df[variable] = float("nan") - - df = df.groupby(df.index).mean() - df = df.sort_index() - - return df diff --git a/eeweather/sources/ghcnh/__init__.py b/eeweather/sources/ghcnh/__init__.py new file mode 100644 index 0000000..a9ec0c5 --- /dev/null +++ b/eeweather/sources/ghcnh/__init__.py @@ -0,0 +1,14 @@ +"""GHCNh hourly observations served through the NCEI access API.""" +from ...registry.db import GHCNH_DB_PATH, metadata_db_connection_proxy +from .source import GHCNhSource + + + +__all__ = ("GHCNhSource", "DATA_PATH") + +# packaged catalog: station facts and zone assignments as GHCNh's material +# claims them, observation inventory, and quality ratings +DATA_PATH = GHCNH_DB_PATH +metadata_db_connection_proxy.register_attachment( + "ghcnh", DATA_PATH, catalog=True, inventory=True, quality=True +) diff --git a/eeweather/sources/ghcnh/ghcnh.db b/eeweather/sources/ghcnh/ghcnh.db new file mode 100644 index 0000000..7499dcd Binary files /dev/null and b/eeweather/sources/ghcnh/ghcnh.db differ diff --git a/eeweather/sources/ghcnh/source.py b/eeweather/sources/ghcnh/source.py new file mode 100644 index 0000000..5918ada --- /dev/null +++ b/eeweather/sources/ghcnh/source.py @@ -0,0 +1,113 @@ +"""The GHCNh observation source served through the NCEI access API.""" +import io +import time + +import pandas as pd +import requests + +from ..base import Feed + + + +API_URL = "https://www.ncei.noaa.gov/access/services/data/v1" + +API_REQUEST_TRIES = 3 + +API_RETRY_BACKOFF_SECONDS = 5 + +API_TIMEOUT_SECONDS = 120 + +DATASET = "global-historical-climatology-network-hourly" + +# one session for connection reuse across the many per-station-year requests +_session = requests.Session() + + +def _get(url, params): # pragma: no cover (mocked in tests via this seam) + return _session.get(url=url, params=params, timeout=API_TIMEOUT_SECONDS) + + +class GHCNhSource(Feed): + """Observation source for the GHCNh dataset. + + GHCNh's names and units for the canonical vocabulary are already + canonical, so no translation is applied. + """ + + name = "ghcnh" + id_namespace = "ghcn" + variables = ( + "temperature", + "dew_point_temperature", + "relative_humidity", + "wind_speed", + "station_level_pressure", + "visibility", + ) + + def fetch_year(self, external_id, year, variables): + """Fetch one year of observations for a station. + + Retried on connection errors and server errors, which the api + intermittently returns in bursts; client errors raise immediately. + + Returns + ------- + pandas.DataFrame + One column per requested variable, indexed by UTC observation + time. Observations sharing a timestamp are averaged. Values + the station did not report are NaN. Empty when the station + has no data for the year. + """ + params = { + "dataset": DATASET, + # DATE is only included in the response when explicitly requested + "dataTypes": ",".join(("DATE",) + tuple(variables)), + "stations": external_id, + "startDate": "{}-01-01".format(year), + "endDate": "{}-12-31".format(year), + } + + for attempt in range(API_REQUEST_TRIES): + try: + resp = _get(API_URL, params) + resp.raise_for_status() + except requests.HTTPError: + if resp.status_code < 500 or attempt == API_REQUEST_TRIES - 1: + raise + time.sleep(API_RETRY_BACKOFF_SECONDS * (attempt + 1)) + except requests.RequestException: + if attempt == API_REQUEST_TRIES - 1: + raise + time.sleep(API_RETRY_BACKOFF_SECONDS * (attempt + 1)) + else: + break + + empty_index = pd.DatetimeIndex([], tz="UTC") + empty = pd.DataFrame(columns=list(variables), index=empty_index, dtype=float) + if resp.text.strip() == "": + return empty + + raw = pd.read_csv(io.StringIO(resp.text), dtype=str) + if len(raw) == 0: + return empty + if "DATE" not in raw.columns: + raise ValueError( + "Malformed GHCNh response for station {} year {}: no DATE" + " column (the api returned a non-csv body).".format( + external_id, year + ) + ) + + index = pd.to_datetime(raw["DATE"]).dt.tz_localize("UTC").rename(None) + df = pd.DataFrame(index=index) + for variable in variables: + if variable in raw.columns: + df[variable] = pd.to_numeric(raw[variable], errors="coerce").values + else: + df[variable] = float("nan") + + df = df.groupby(df.index).mean() + df = df.sort_index() + + return df diff --git a/eeweather/sources/matching.py b/eeweather/sources/matching.py new file mode 100644 index 0000000..714aa4b --- /dev/null +++ b/eeweather/sources/matching.py @@ -0,0 +1,446 @@ +"""Candidate-station ranking and data-sufficiency selection.""" +from functools import cached_property + +import numpy as np +import pandas as pd +import pyproj + +from ..exceptions import DataNotAvailableError, EEWeatherWarning +from ..registry.db import metadata_db_connection_proxy +from ..registry.quality import get_station_qualities +from ..registry.update import maybe_update +from ..registry.zones import zones_at +from ..station import WeatherStation + + + +__all__ = ("rank_stations", "combine_ranked_stations", "select_station") + +def _ranking_columns(): + """Output columns: the fixed set plus one availability flag (and any + class column) per registered archive source.""" + proxy = metadata_db_connection_proxy + columns = [ + "rank", + "distance_meters", + "latitude", + "longitude", + "iecc_climate_zone", + "iecc_moisture_regime", + "ba_climate_zone", + "ca_climate_zone", + "quality", + "elevation", + "subdivision", + ] + frame_columns = cached_data.all_station_metadata.columns + for avail in sorted(proxy.availability_sources): + class_column = "{}_class".format(avail) + if class_column in frame_columns: + columns.append(class_column) + columns.append("is_{}".format(avail)) + columns.append("difference_elevation_meters") + + return columns + + +class CachedData(object): + @cached_property + def all_station_metadata(self): + proxy = metadata_db_connection_proxy + conn = proxy.get_connection() + frames = [] + for alias in proxy.catalogs: + frames.append(self._catalog_frame(conn, proxy, alias)) + df = pd.concat(frames) + df = df[~df.index.duplicated(keep="first")].sort_index() + + return df + + def _catalog_frame(self, conn, proxy, alias): + availability_selects = "".join( + """ + , max({avail}.station_id) is not null as is_{avail} + , max({avail}.class) as {avail}_class""".format(avail=avail) + if self._has_class(conn, avail) else + """ + , max({avail}.station_id) is not null as is_{avail}""".format(avail=avail) + for avail in proxy.availability_sources + ) + availability_joins = "".join( + """ + left join {avail}.stations as {avail} on + s.station_id = {avail}.station_id""".format(avail=avail) + for avail in proxy.availability_sources + ) + quality_select = ", null as quality" + quality_join = "" + if alias in proxy.quality_sources: + quality_select = ", max(q.quality) as quality" + quality_join = ( + "\n left join {alias}.quality as q on" + " s.station_id = q.station_id".format(alias=alias) + ) + df = pd.read_sql_query( + """ + select + s.station_id + , s.latitude + , s.longitude + , max(case when z.system = 'iecc_climate_zone' then z.zone_id end) + as iecc_climate_zone + , max(case when z.system = 'iecc_moisture_regime' then z.zone_id end) + as iecc_moisture_regime + , max(case when z.system = 'ba_climate_zone' then z.zone_id end) + as ba_climate_zone + , max(case when z.system = 'ca_climate_zone' then z.zone_id end) + as ca_climate_zone + {quality_select} + , s.elevation + , s.subdivision + {availability_selects} + from + {alias}.stations as s + left join {alias}.station_zone as z on s.station_id = z.station_id + {quality_join} + {availability_joins} + group by s.station_id + order by s.station_id + """.format( + alias=alias, + quality_select=quality_select, + quality_join=quality_join, + availability_selects=availability_selects, + availability_joins=availability_joins, + ), + conn, + ).set_index("station_id") + for avail in proxy.availability_sources: + df["is_{}".format(avail)] = df["is_{}".format(avail)].astype(bool) + + return df + + @staticmethod + def _has_class(conn, alias): + columns = [ + row[1] + for row in conn.execute("pragma {}.table_info(stations)".format(alias)) + ] + + return "class" in columns + + +cached_data = CachedData() + + +def source_availability_columns(): + """Availability filters served by the packaged data, by source name. + + Catalog sources map to None (every cataloged station serves them); + archive sources map to their availability flag column. + """ + proxy = metadata_db_connection_proxy + columns = {alias: None for alias in proxy.catalogs} + for alias in proxy.availability_sources: + columns[alias] = "is_{}".format(alias) + + return columns + + +def _combine_filters(filters, index): + combined_filters = pd.Series(True, index=index) + for f in filters: + combined_filters &= f + + return combined_filters + + +def rank_stations( + site_latitude, + site_longitude, + site_elevation=None, + site_subdivision=None, + match_zones=(), + match_subdivision=False, + has_sources=(), + minimum_quality=None, + rating_period=None, + quality_source="ghcnh", + max_distance_meters=None, + max_difference_elevation_meters=None, +): + """Get a ranked, filtered set of candidate weather stations and metadata + for a site. + + Parameters + ---------- + site_latitude : float + Latitude of the target site. + site_longitude : float + Longitude of the target site. + site_elevation : float, optional + Elevation of the target site in meters. Ignored unless + ``max_difference_elevation_meters`` is set. + site_subdivision : str, optional + Country subdivision of the target site (e.g. a US state + abbreviation). Ignored unless ``match_subdivision=True``. + match_zones : tuple of str + Zone systems the candidates must match the site in, e.g. + ``('iecc_climate_zone',)``. Candidates match a system the site has + no zone for only when they also have none. + match_subdivision : bool + If True, filter candidates to the site's subdivision. + has_sources : tuple of str + Sources the candidates must be able to serve, e.g. ``('tmy3',)``. + minimum_quality : {'high', 'medium', 'low'}, optional + Filter candidates to those meeting or exceeding this quality. + rating_period : datetime, optional + When given, quality is rated from observation counts over the five + calendar years ending two years after this anchor date (sliding + back to end no later than the last full year), and the ``quality`` + column and ``minimum_quality`` filter use that rating. When None, + the build-time rating over the last five full years is used. + quality_source : str + Source whose observation counts rate quality for + ``rating_period``. + max_distance_meters : float, optional + Filter candidates to those within this distance of the site. + max_difference_elevation_meters : float, optional + Filter candidates to those with elevations within this difference + of ``site_elevation``. + + Returns + ------- + pandas.DataFrame + Indexed by station id, sorted by distance, one row per candidate: + rank, distance_meters, latitude, longitude, one column per zone + system, quality, elevation, subdivision, tmy3_class, + is_tmy3/is_cz2010, difference_elevation_meters. + """ + maybe_update() + candidates = cached_data.all_station_metadata.copy() + + candidates_defined_lat_long = candidates[ + candidates.latitude.notnull() & candidates.longitude.notnull() + ] + candidates_latitude = candidates_defined_lat_long.latitude + candidates_longitude = candidates_defined_lat_long.longitude + tiled_site_latitude = np.tile(site_latitude, candidates_latitude.shape) + tiled_site_longitude = np.tile(site_longitude, candidates_longitude.shape) + geod = pyproj.Geod(ellps="WGS84") + dists = geod.inv( + tiled_site_longitude, + tiled_site_latitude, + candidates_longitude.values, + candidates_latitude.values, + )[2] + distance_meters = pd.Series( + dists, index=candidates_defined_lat_long.index + ).reindex(candidates.index) + candidates["distance_meters"] = distance_meters + + if site_elevation is not None: + difference_elevation_meters = (candidates.elevation - site_elevation).abs() + else: + difference_elevation_meters = None + candidates["difference_elevation_meters"] = difference_elevation_meters + + filters = [] + + if match_zones: + site_zones = zones_at(site_latitude, site_longitude) + for system in match_zones: + if system not in candidates.columns: + raise ValueError("Unknown zone system: {}".format(system)) + site_zone = site_zones.get(system) + if site_zone is None: + filters.append(candidates[system].isnull()) + else: + filters.append(candidates[system] == site_zone) + + if match_subdivision: + if site_subdivision is None: + filters.append(candidates.subdivision.isnull()) + else: + filters.append(candidates.subdivision == site_subdivision) + + availability_columns = source_availability_columns() + for source in has_sources: + column = availability_columns.get(source, "unknown") + if column == "unknown": + raise ValueError("Unknown source: {}".format(source)) + if column is not None: + filters.append(candidates[column]) + + if rating_period is not None: + period_qualities = get_station_qualities(rating_period, source=quality_source) + candidates["quality"] = period_qualities.reindex( + candidates.index, fill_value="low" + ) + + if minimum_quality is None: + pass + elif minimum_quality == "low": + filters.append(candidates.quality.isin(["high", "medium", "low"])) + elif minimum_quality == "medium": + filters.append(candidates.quality.isin(["high", "medium"])) + elif minimum_quality == "high": + filters.append(candidates.quality.isin(["high"])) + else: + raise ValueError("Unknown minimum_quality: {}".format(minimum_quality)) + + if max_distance_meters is not None: + filters.append(candidates.distance_meters <= max_distance_meters) + + if max_difference_elevation_meters is not None and site_elevation is not None: + filters.append( + candidates.difference_elevation_meters <= max_difference_elevation_meters + ) + + combined_filters = _combine_filters(filters, candidates.index) + filtered_candidates = candidates[combined_filters] + ranked_filtered_candidates = filtered_candidates.sort_values( + by=["distance_meters"] + ) + + ranks = range(1, 1 + len(ranked_filtered_candidates)) + ranked_filtered_candidates.insert(0, "rank", ranks) + + return ranked_filtered_candidates[_ranking_columns()] + + +def combine_ranked_stations(rankings): + """Combine ranked candidate frames into one fallback-ordered ranking. + + Parameters + ---------- + rankings : list of pandas.DataFrame + Ranked candidate frames of the form given by rank_stations, each + sorted by rank. Stations already present in an earlier frame are + dropped from later ones. + + Returns + ------- + pandas.DataFrame + One frame with a recomputed rank column. + """ + if len(rankings) == 0: + raise ValueError("Requires at least one ranking.") + + combined_ranking = rankings[0] + for ranking in rankings[1:]: + filtered_ranking = ranking[~ranking.index.isin(combined_ranking.index)] + combined_ranking = pd.concat([combined_ranking, filtered_ranking]) + + combined_ranking["rank"] = range(1, 1 + len(combined_ranking)) + + return combined_ranking + + +def load_hourly_temp_data( + station, start_date, end_date, fetch_from_web, source=None +): # pragma: no cover + df, warnings = station.load_data( + start_date, end_date, fetch_from_web=fetch_from_web, source=source + ) + temps = df["temperature"] + + return temps, warnings + + +def select_station( + candidates, + coverage_range=None, + min_fraction_coverage=0.9, + distance_warnings=(50000, 200000), + rank=1, + fetch_from_web=True, + coverage_source=None, +): + """Select a station from ranked candidates that meets data-sufficiency + criteria. + + Parameters + ---------- + candidates : pandas.DataFrame + A frame of the form given by rank_stations, having at least a + station-id index and a ``distance_meters`` column. + coverage_range : tuple of (datetime, datetime), optional + When given, candidates must serve temperature data covering at + least ``min_fraction_coverage`` of this period. + coverage_source : str or source object, optional + The source coverage is tested against; defaults to the + candidate's default routing (the source actually being served + should be passed, so a station rich in one source but absent + from another cannot pass selection wrongly). + + Returns + ------- + tuple of (WeatherStation or None, list of EEWeatherWarning) + The first candidate passing the criteria; None if none passes. + """ + + def _test_station(station): + no_warnings = [] + if coverage_range is None: + return True, no_warnings + + start_date, end_date = coverage_range + try: + tempC, warnings = load_hourly_temp_data( + station, start_date, end_date, fetch_from_web, + source=coverage_source, + ) + except DataNotAvailableError: + return False, no_warnings + + if len(tempC) == 0: + return False, no_warnings + fraction_coverage = tempC.notnull().sum() / float(len(tempC)) + passed = fraction_coverage > min_fraction_coverage + + return passed, warnings + + def _station_warnings(station, distance_meters): + warnings = [ + EEWeatherWarning( + qualified_name="eeweather.exceeds_maximum_distance", + description=( + "Distance from target to weather station is greater" + " than the specified km." + ), + data={ + "distance_meters": distance_meters, + "max_distance_meters": d, + "rank": rank, + }, + ) + for d in distance_warnings + if distance_meters > d + ] + + return warnings + + n_stations_passed = 0 + for station_id, row in candidates.iterrows(): + station = WeatherStation(station_id) + test_result, warnings = _test_station(station) + if test_result: + n_stations_passed += 1 + if n_stations_passed == rank: + warnings.extend(_station_warnings(station, row.distance_meters)) + + return station, warnings + + no_station_warnings = [ + EEWeatherWarning( + qualified_name="eeweather.no_weather_station_selected", + description=( + "No weather station found with the specified rank and" + " minimum fractional coverage." + ), + data={"rank": rank, "min_fraction_coverage": min_fraction_coverage}, + ) + ] + + return None, no_station_warnings diff --git a/eeweather/sources/station_source.py b/eeweather/sources/station_source.py new file mode 100644 index 0000000..f41aa9c --- /dev/null +++ b/eeweather/sources/station_source.py @@ -0,0 +1,204 @@ +"""The nearest-suitable-station estimation strategy.""" +from __future__ import annotations + +from datetime import datetime + +import pyproj + +from ..exceptions import NoQualifiedStationError +from .base import Source +from .engine import load_data as _engine_load_data +from .engine import resolve_source +from ..exceptions import EEWeatherWarning +from .matching import rank_stations, select_station, source_availability_columns +from ..station import WeatherStation + + + +# beyond this, a station is not a defensible temperature proxy for a +# location without explicit acknowledgment; every US ZCTA has a station +# within 124 km, so no ordinary location is stranded by the default +DEFAULT_MAX_DISTANCE_METERS = 150_000 + + +class StationSource(Source): + """Estimates weather at a point from the nearest suitable station. + + Parameters + ---------- + dataset : str or Feed + The observation source served: a built-in name or a Feed object. + rank_kwargs, select_kwargs : dict, optional + Forwarded to rank_stations and select_station to control + filtering and coverage requirements. + """ + + def __init__(self, dataset="ghcnh", rank_kwargs=None, select_kwargs=None): + self.adapter = resolve_source(dataset) + self.name = self.adapter.name + self.kind = self.adapter.kind + self.variables = self.adapter.variables + self.default_variables = self.adapter.default_variables + rank_kwargs = dict(rank_kwargs or {}) + if self.name in source_availability_columns(): + # only consider stations that can serve this dataset + rank_kwargs.setdefault("has_sources", (self.name,)) + rank_kwargs.setdefault("max_distance_meters", DEFAULT_MAX_DISTANCE_METERS) + self.rank_kwargs = rank_kwargs + self.select_kwargs = select_kwargs or {} + self._resolutions = {} + + @property + def period_dependent(self): + """Whether a coverage or rating-period filter is configured, so + resolution depends on more than coordinates. Such a source is + pinned at load (where the request is known and any coverage fetch + is expected), not at serialization.""" + configured = ( + "coverage_range" in self.select_kwargs + or "rating_period" in self.rank_kwargs + ) + + return configured + + def resolve( + self, + latitude: float, + longitude: float, + period: tuple[datetime, datetime] | None = None, + ignore_disqualification: bool = False, + station: str | None = None, + ): + """The station serving this point, with its distance and any + selection warnings. + + Resolutions (including their warnings) are memoized per location + and period year-span; the same request reuses its station, a + different era re-resolves. Call :meth:`reset` to force + re-resolution. + + With ``ignore_disqualification``, a point no station qualifies + for (distance cap, coverage test) gets the best available + station anyway, plus an ``eeweather.station_disqualified`` + warning carrying what failed; NoQualifiedStationError then only + remains for a point with no candidates at all. + """ + if station is not None: + return self._resolve_pinned(latitude, longitude, station) + + key = (round(latitude, 6), round(longitude, 6), ignore_disqualification) + if period is not None: + start, end = period + key = key + (start.year, end.year) + if key in self._resolutions: + station, distance_meters, select_warnings = self._resolutions[key] + + return station, distance_meters, list(select_warnings) + + select_kwargs = dict(self.select_kwargs) + if "coverage_range" in select_kwargs: + select_kwargs.setdefault("coverage_source", self.adapter) + candidates = rank_stations(latitude, longitude, **self.rank_kwargs) + station, select_warnings = select_station(candidates, **select_kwargs) + + if station is None and ignore_disqualification: + station, select_warnings, candidates = self._resolve_disqualified( + latitude, longitude + ) + if station is None: + raise NoQualifiedStationError(latitude, longitude) + distance_meters = float(candidates.loc[station.id, "distance_meters"]) + self._resolutions[key] = (station, distance_meters, list(select_warnings)) + + return station, distance_meters, list(select_warnings) + + def _resolve_pinned(self, latitude, longitude, station_id): + """The pinned station, with its true distance to the point; an + explicit pin is its own acceptance, so qualification is skipped.""" + station = WeatherStation(station_id) + geod = pyproj.Geod(ellps="WGS84") + distance_meters = float( + geod.inv(longitude, latitude, station.longitude, station.latitude)[2] + ) + no_warnings = [] + + return station, distance_meters, no_warnings + + def _resolve_disqualified(self, latitude, longitude): + """The best station ignoring the distance cap and coverage test, + with a warning stating which qualifications it failed.""" + rank_kwargs = dict(self.rank_kwargs) + max_distance = rank_kwargs.pop("max_distance_meters", None) + candidates = rank_stations(latitude, longitude, **rank_kwargs) + select_kwargs = { + k: v for k, v in self.select_kwargs.items() + if k not in ("coverage_range", "min_fraction_coverage") + } + station, select_warnings = select_station(candidates, **select_kwargs) + if station is not None: + failed = [] + distance_meters = float(candidates.loc[station.id, "distance_meters"]) + if max_distance is not None and distance_meters > max_distance: + failed.append("distance") + if "coverage_range" in self.select_kwargs: + failed.append("coverage") + select_warnings = list(select_warnings) + select_warnings.append( + EEWeatherWarning( + qualified_name="eeweather.station_disqualified", + description=( + "No station qualified; using the best available" + " with qualification ignored." + ), + data={ + "station_id": station.id, + "failed": failed, + "distance_meters": distance_meters, + "max_distance_meters": max_distance, + }, + ) + ) + + return station, select_warnings, candidates + + def reset(self) -> None: + """Drop memoized station resolutions.""" + self._resolutions = {} + + def estimate( + self, + latitude: float, + longitude: float, + start: datetime, + end: datetime, + frequency="h", + variables=None, + raise_when_empty=True, + ignore_disqualification=False, + station=None, + **load_kwargs, + ): + station, distance_meters, select_warnings = self.resolve( + latitude, longitude, period=(start, end), + ignore_disqualification=ignore_disqualification, + station=station, + ) + + df, load_warnings = _engine_load_data( + station.id, + start, + end, + frequency=frequency, + variables=variables, + source=self.adapter, + raise_when_empty=raise_when_empty, + **load_kwargs, + ) + provenance = { + name: record._replace(distance_meters=distance_meters) + for name, record in df.attrs["provenance"].items() + } + df.attrs["provenance"] = provenance + warnings = select_warnings + load_warnings + + return df, warnings, provenance diff --git a/eeweather/sources/tmy3/__init__.py b/eeweather/sources/tmy3/__init__.py new file mode 100644 index 0000000..b9155c2 --- /dev/null +++ b/eeweather/sources/tmy3/__init__.py @@ -0,0 +1,15 @@ +"""NREL TMY3 typical-year temperatures.""" +import os + +from ...registry.db import metadata_db_connection_proxy +from .source import TMY3Source + + + +__all__ = ("TMY3Source", "DATA_PATH") + +# packaged archive station list +DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tmy3.db") +metadata_db_connection_proxy.register_attachment( + "tmy3", DATA_PATH, availability=True +) diff --git a/eeweather/sources/tmy3/source.py b/eeweather/sources/tmy3/source.py new file mode 100644 index 0000000..29efe7b --- /dev/null +++ b/eeweather/sources/tmy3/source.py @@ -0,0 +1,15 @@ +"""NREL TMY3 typical-year temperatures.""" +from ..base import NormalsSource + + + +class TMY3Source(NormalsSource): + name = "tmy3" + + def _url(self, usaf_id): + url = ( + "https://storage.googleapis.com/openeemeter-public-resources/" + "tmy3_archive/{}TYA.CSV".format(usaf_id) + ) + + return url diff --git a/eeweather/sources/tmy3/tmy3.db b/eeweather/sources/tmy3/tmy3.db new file mode 100644 index 0000000..cf6ebce Binary files /dev/null and b/eeweather/sources/tmy3/tmy3.db differ diff --git a/eeweather/sources/vocabulary.py b/eeweather/sources/vocabulary.py new file mode 100644 index 0000000..d2a41a2 --- /dev/null +++ b/eeweather/sources/vocabulary.py @@ -0,0 +1,111 @@ +"""The variable vocabulary. + +Every variable a source can serve has exactly one name, unit, and +definition. The canonical entries are owned here; external sources add +entries at runtime through ``eeweather.sources.register``, with the same +one-definition rule (canonical names are reserved, and the first +registration of a name freezes its unit). Adapters translate their +native names and units into vocabulary forms at ingest; users never see +native forms. Columns ending in the reserved suffixes are +engine-produced companions of a variable and are exempt from vocabulary +validation. +""" +from collections import namedtuple + + + +Variable = namedtuple( + "Variable", ["name", "unit", "description", "aggregation"] +) + +# how hourly values roll up to daily: point-in-time variables average; +# accumulation variables ("sum") add up and are never gap-interpolated +AGGREGATIONS = ("mean", "sum", "min", "max") + +VARIABLES = { + variable.name: variable + for variable in ( + Variable("temperature", "degC", "Air temperature.", "mean"), + Variable( + "dew_point_temperature", "degC", "Dew point temperature.", "mean" + ), + Variable("relative_humidity", "%", "Relative humidity.", "mean"), + Variable("wind_speed", "m/s", "Wind speed.", "mean"), + Variable( + "station_level_pressure", + "hPa", + "Atmospheric pressure at station level.", + "mean", + ), + Variable("visibility", "km", "Horizontal visibility distance.", "mean"), + ) +} + +RESERVED_SUFFIXES = ("_imputed_fraction", "_is_imputed") + +_registered_variables = {} + + +def register_variables(entries): + """Add external vocabulary entries; called through sources.register. + + All entries are validated before any is added: canonical names are + reserved, every entry needs a name, unit, and description, and a + name registered twice must carry an identical definition, so a + variable's unit is frozen by whichever source registers it first. + """ + validated = {} + for entry in entries: + entry = Variable(*entry) + if not entry.name or not entry.unit or not entry.description: + raise ValueError( + "A variable needs a name, unit, description, and" + " aggregation, got: {}.".format(tuple(entry)) + ) + if entry.aggregation not in AGGREGATIONS: + raise ValueError( + "Unknown aggregation '{}' for '{}'; one of: {}.".format( + entry.aggregation, entry.name, ", ".join(AGGREGATIONS) + ) + ) + if entry.name in VARIABLES: + raise ValueError( + "'{}' is canonical; serve it in its canonical unit" + " ({}).".format(entry.name, VARIABLES[entry.name].unit) + ) + existing = _registered_variables.get(entry.name) or validated.get(entry.name) + if existing is not None and existing != entry: + raise ValueError( + "'{}' is already registered as {}; definitions must" + " agree.".format(entry.name, tuple(existing)) + ) + validated[entry.name] = entry + _registered_variables.update(validated) + + +def all_variables(): + """The full vocabulary, canonical plus registered, by name.""" + return {**VARIABLES, **_registered_variables} + + +def valid_variables_or_raise(variables): + """Raise ValueError when a requested variable is not in the + vocabulary.""" + known = all_variables() + unknown = [name for name in variables if name not in known] + if unknown: + raise ValueError( + "Unknown variables: {}. Known variables: {}.".format( + ", ".join(sorted(unknown)), ", ".join(sorted(known)) + ) + ) + + +def aggregation_for(column): + """The daily-rollup aggregation for a frame column; engine-produced + companion columns and anything else outside the vocabulary average.""" + entry = all_variables().get(column) + if entry is None: + return "mean" + + return entry.aggregation diff --git a/eeweather/station.py b/eeweather/station.py new file mode 100644 index 0000000..7d375a8 --- /dev/null +++ b/eeweather/station.py @@ -0,0 +1,265 @@ +"""The pinned-station handle.""" +from __future__ import annotations + +from datetime import datetime + +from .registry.db import valid_station_id_or_raise +from .registry.identifiers import resolve_station +from .registry.identifiers import translate as _translate +from .registry.metadata import get_station_metadata +from .registry.quality import get_station_quality +from .registry.summaries import search_stations +from .sources.engine import load_data as _engine_load_data +from .sources.engine import resolve_source + + + +__all__ = ("WeatherStation",) + + +def _validate_sources(sources): + """Reject a preference tuple with a duplicate or non-observation + source; routing only ever considers observation sources.""" + adapters = [resolve_source(source) for source in sources] + names = [adapter.name for adapter in adapters] + if len(set(names)) != len(names): + raise ValueError( + "Duplicate source names in preference tuple: {}.".format( + ", ".join(names) + ) + ) + wrong_kind = [adapter.name for adapter in adapters if adapter.kind != "observations"] + if wrong_kind: + raise ValueError( + "Sources preference only routes observation sources; pin" + " normals sources (e.g. 'tmy3', 'cz2010') with load_data(" + "source=...) instead: {}.".format(", ".join(wrong_kind)) + ) + + +class WeatherStation(object): + """A known weather station in the registry. + + Stations are keyed by their GHCN id; other identifier systems are + aliases — use the ``from_id``, ``from_usaf``, ``from_wban``, or + ``from_icao`` constructors to look a station up by one. + + Attributes + ---------- + id : str + The station's GHCN id, the registry key. + ids : dict of str to list of str + External identifiers by namespace, e.g. + ``{"usaf": ["722880"], "wban": ["23152"], "icao": ["KBUR"]}``. + name : str + Station name. + latitude, longitude, elevation : float + Station location; elevation in meters. + coords : tuple of (float, float) + Latitude/longitude pair. + country : str + ISO 3166-1 alpha-2 country code. + subdivision : str or None + Country subdivision (e.g. a US state abbreviation); None where the + registry has none. + quality : str + Build-time data-quality rating: "high", "medium", or "low". + zones : dict of str to str + Zone id by zone system, e.g. ``{"iecc_climate_zone": "3"}``. + inventory_years : dict of str to tuple of (int, int) + First and last inventory year by source. + """ + + def __init__(self, station_id: str, sources=("ghcnh",), load_metadata: bool = True): + self.id = station_id + if isinstance(sources, str): + sources = (sources,) + sources = tuple(sources) + _validate_sources(sources) + self.sources = sources + self.provenance = None + + if load_metadata: + self._load_metadata() + else: + valid_station_id_or_raise(station_id) + self.ids = None + self.name = None + self.latitude = None + self.longitude = None + self.elevation = None + self.coords = None + self.country = None + self.subdivision = None + self.quality = None + self.zones = {} + self.inventory_years = {} + + @classmethod + def from_id(cls, namespace: str, external_id: str, load_metadata: bool = True) -> "WeatherStation": + """Construct a station from an external identifier. + + When the identifier maps to several stations, the mapping marked + recent wins; unresolvable identifiers raise + AmbiguousIdentifierError. + """ + station_id = resolve_station(namespace, external_id) + + return cls(station_id, load_metadata=load_metadata) + + @classmethod + def from_usaf(cls, usaf_id: str, load_metadata: bool = True) -> "WeatherStation": + """Construct a station from a historical ISD USAF id.""" + return cls.from_id("usaf", usaf_id, load_metadata=load_metadata) + + @classmethod + def from_wban(cls, wban_id: str, load_metadata: bool = True) -> "WeatherStation": + """Construct a station from a historical WBAN id.""" + return cls.from_id("wban", wban_id, load_metadata=load_metadata) + + @classmethod + def from_icao(cls, icao_code: str, load_metadata: bool = True) -> "WeatherStation": + """Construct a station from an ICAO airport code.""" + return cls.from_id("icao", icao_code, load_metadata=load_metadata) + + @classmethod + def search(cls, country: str | None = None, subdivision: str | None = None, has_sources=()): + """Registry stations as a DataFrame indexed by station id. + + Parameters + ---------- + country : str, optional + ISO 3166-1 alpha-2 country code, e.g. ``'US'``. + subdivision : str, optional + Country subdivision, e.g. a US state abbreviation. + has_sources : tuple of str + Sources the stations must be able to serve, e.g. + ``('tmy3',)``. + """ + stations = search_stations( + country=country, subdivision=subdivision, has_sources=has_sources + ) + + return stations + + @staticmethod + def translate(ids, from_namespace: str, to_namespace: str) -> dict[str, tuple[str, ...]]: + """Translate external station ids between identifier systems. + + Values are always tuples; mappings can be one-to-many in either + direction. Ids with no mapping are absent from the result. + """ + return _translate(ids, from_namespace, to_namespace) + + def __str__(self) -> str: + return self.id + + def __repr__(self) -> str: + return "WeatherStation('{}')".format(self.id) + + def _load_metadata(self): + metadata = get_station_metadata(self.id) + + self.ids = metadata["ids"] + self.name = metadata["name"] + self.latitude = metadata["latitude"] + self.longitude = metadata["longitude"] + self.elevation = metadata["elevation"] + self.coords = (self.latitude, self.longitude) + self.country = metadata["country"] + self.subdivision = metadata["subdivision"] + self.quality = metadata["quality"] + self.zones = metadata["zones"] + self.inventory_years = metadata["inventory_years"] + + def json(self) -> dict: + """Return a JSON-serializeable object containing station metadata.""" + serialized = { + "id": self.id, + "ids": self.ids, + "name": self.name, + "latitude": self.latitude, + "longitude": self.longitude, + "elevation": self.elevation, + "country": self.country, + "subdivision": self.subdivision, + "quality": self.quality, + "zones": self.zones, + "inventory_years": self.inventory_years, + } + + return serialized + + def load_data( + self, + start: datetime, + end: datetime, + frequency="h", + variables=None, + source=None, + read_from_cache: bool = True, + write_to_cache: bool = True, + fetch_from_web: bool = True, + ): + """Load this station's weather data between two dates (inclusive). + + Parameters + ---------- + start : datetime.datetime + The earliest date from which to load data. Must be UTC. + end : datetime.datetime + The latest date until which to load data. Must be UTC. + frequency : str or pandas offset + A pandas offset alias (e.g. ``'30min'``, ``'h'``, ``'D'``, + ``'W'``, ``'MS'``, ``'YS'``). Coarser-than-hourly + frequencies aggregate the hourly values within each UTC + period by each variable's vocabulary aggregation; + finer-than-hourly frequencies interpolate. + variables : tuple of str, or 'all' + Canonical variable names; defaults to ``('temperature',)``. + Each is routed to the first source in this station's + ``sources`` preference that serves it; ``'all'`` is every + variable those sources serve. + source : str or source object, optional + Pin every requested variable to this source, bypassing + routing. Typical-year sources (``'tmy3'``, ``'cz2010'``) are + only reachable this way. A pinned load with nothing at all + for the requested dates raises ``DataNotAvailableError``; a + routed load (``source`` unset) never raises for missing + data, returning NaN plus warnings instead. + read_from_cache : bool + Whether or not to load data from cache. + write_to_cache : bool + Whether or not to write newly loaded data to cache. + fetch_from_web : bool + Whether or not to fetch data from the web. + + Returns + ------- + tuple of (pandas.DataFrame, list of EEWeatherWarning) + One column per requested variable, indexed over the full + requested range at the requested frequency; periods without + data are NaN. The frame's ``attrs["provenance"]`` (also + recorded on ``self.provenance``) maps each source used to a + Provenance record. + """ + df, warnings = _engine_load_data( + self.id, + start, + end, + frequency=frequency, + variables=variables, + source=source, + sources=self.sources, + read_from_cache=read_from_cache, + write_to_cache=write_to_cache, + fetch_from_web=fetch_from_web, + ) + self.provenance = df.attrs["provenance"] + + return df, warnings + + def get_quality(self, anchor: datetime, source: str | None = None) -> str: + """Station quality anchored to a date, from a source's observation + counts; defaults to the first registered source with inventory.""" + return get_station_quality(self.id, anchor, source=source) diff --git a/eeweather/stations.py b/eeweather/stations.py deleted file mode 100644 index 2158127..0000000 --- a/eeweather/stations.py +++ /dev/null @@ -1,1272 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from datetime import datetime, timedelta, timezone - -import numpy as np -import pandas as pd -import pytz - -import requests - -from .exceptions import ( - UnrecognizedUSAFIDError, - DataNotAvailableError, - TMY3DataNotAvailableError, - CZ2010DataNotAvailableError, - NonUTCTimezoneInfoError, -) -from .validation import valid_usaf_id_or_raise -from .warnings import EEWeatherWarning -import eeweather.connections -from eeweather.connections import metadata_db_connection_proxy -import eeweather.mockable -from .sources.ghcnh import DEFAULT_VARIABLES, fetch_ghcnh_hourly - -DATA_EXPIRATION_DAYS = 1 - -__all__ = ( - "WeatherStation", - "get_ghcn_id", - "get_isd_station_metadata", - "get_station_quality", - "get_station_qualities", - "get_isd_file_metadata", - "fetch_hourly_data", - "get_hourly_data_cache_key", - "get_tmy3_hourly_temp_data_cache_key", - "get_cz2010_hourly_temp_data_cache_key", - "cached_hourly_data_is_expired", - "validate_hourly_data_cache", - "validate_tmy3_hourly_temp_data_cache", - "validate_cz2010_hourly_temp_data_cache", - "serialize_hourly_data", - "serialize_tmy3_hourly_temp_data", - "serialize_cz2010_hourly_temp_data", - "deserialize_hourly_data", - "deserialize_tmy3_hourly_temp_data", - "deserialize_cz2010_hourly_temp_data", - "read_hourly_data_from_cache", - "read_tmy3_hourly_temp_data_from_cache", - "read_cz2010_hourly_temp_data_from_cache", - "write_hourly_data_to_cache", - "write_tmy3_hourly_temp_data_to_cache", - "write_cz2010_hourly_temp_data_to_cache", - "destroy_cached_hourly_data", - "destroy_cached_tmy3_hourly_temp_data", - "destroy_cached_cz2010_hourly_temp_data", - "load_hourly_data_cached_proxy", - "load_tmy3_hourly_temp_data_cached_proxy", - "load_cz2010_hourly_temp_data_cached_proxy", - "load_data", - "load_tmy3_hourly_temp_data", - "load_cz2010_hourly_temp_data", - "load_cached_hourly_data", - "load_cached_tmy3_hourly_temp_data", - "load_cached_cz2010_hourly_temp_data", -) - - -def _datetime_is_utc(dt): - orig_tzinfo = dt.tzinfo - return False if orig_tzinfo is None else dt.utcoffset().seconds == 0 - - -TRAILING_GAP_WARNING_THRESHOLD = timedelta(days=1) -INTERNAL_GAP_WARNING_THRESHOLD = timedelta(days=7) - - -def _data_gap_warnings(ts, variable): - """EEWeatherWarnings for requested ranges the returned data does not cover. - - Emitted when the series is entirely empty, when it ends more than - TRAILING_GAP_WARNING_THRESHOLD before the end of the requested range, or - when it contains an internal gap longer than INTERNAL_GAP_WARNING_THRESHOLD. - """ - warnings = [] - if len(ts) == 0: - return warnings - - if ts.isna().all(): - warnings.append( - EEWeatherWarning( - qualified_name="eeweather.no_data_in_requested_range", - description="No data was available within the requested range.", - data={ - "variable": variable, - "requested_start": ts.index[0].isoformat(), - "requested_end": ts.index[-1].isoformat(), - }, - ) - ) - - return warnings - - last_valid = ts.last_valid_index() - trailing_gap = ts.index[-1] - last_valid - if trailing_gap > TRAILING_GAP_WARNING_THRESHOLD: - warnings.append( - EEWeatherWarning( - qualified_name="eeweather.data_truncated", - description=( - "Data ends {} before the end of the requested range.".format( - trailing_gap - ) - ), - data={ - "variable": variable, - "last_valid": last_valid.isoformat(), - "requested_end": ts.index[-1].isoformat(), - }, - ) - ) - - interior = ts.loc[ts.first_valid_index() : last_valid] - if len(interior) > 1: - period = interior.index[1] - interior.index[0] - is_missing = interior.isna() - max_gap_periods = int(is_missing.groupby((~is_missing).cumsum()).sum().max()) - max_gap = max_gap_periods * period - if max_gap > INTERNAL_GAP_WARNING_THRESHOLD: - warnings.append( - EEWeatherWarning( - qualified_name="eeweather.data_gap", - description=( - "Data contains an internal gap of {}.".format(max_gap) - ), - data={ - "variable": variable, - "max_gap_days": max_gap / timedelta(days=1), - }, - ) - ) - - return warnings - - -def get_ghcn_id(usaf_id): - """GHCNh station id mapped to this USAF id.""" - metadata = get_isd_station_metadata(usaf_id) - - return metadata["ghcn_id"] - - -def fetch_hourly_data(usaf_id, year, variables=DEFAULT_VARIABLES): - """Fetch one year of GHCNh observations resampled to an hourly frame. - - Raises DataNotAvailableError when the station has no observations at - all for the year. - """ - ghcn_id = get_ghcn_id(usaf_id) - raw = fetch_ghcnh_hourly(ghcn_id, year, variables) - if len(raw) == 0: - raise DataNotAvailableError(usaf_id, year) - - # CalTRACK 2.3.3 - df = ( - raw.resample("min") - .mean() - .interpolate(method="linear", limit=60, limit_direction="both") - .resample("h") - .mean() - ) - - return df - - -def get_hourly_data_cache_key(usaf_id, year): - return "ghcnh-hourly-{}-{}".format(usaf_id, year) - - -def cached_hourly_data_is_expired(usaf_id, year): - key = get_hourly_data_cache_key(usaf_id, year) - store = eeweather.connections.key_value_store_proxy.get_store() - last_updated = store.key_updated(key) - - return _expired(last_updated, year) - - -def validate_hourly_data_cache(usaf_id, year): - key = get_hourly_data_cache_key(usaf_id, year) - store = eeweather.connections.key_value_store_proxy.get_store() - - # fail if no key - if not store.key_exists(key): - return False - - # check for expired data, fail if so - if cached_hourly_data_is_expired(usaf_id, year): - store.clear(key) - return False - - return True - - -def serialize_hourly_data(df): - rows = [ - [index.strftime("%Y%m%d%H")] + values - for index, values in zip( - df.index, df.astype(object).where(df.notna(), None).values.tolist() - ) - ] - - return {"columns": list(df.columns), "rows": rows} - - -def deserialize_hourly_data(data): - index = pd.to_datetime( - [row[0] for row in data["rows"]], format="%Y%m%d%H", utc=True - ) - df = pd.DataFrame( - [row[1:] for row in data["rows"]], - index=index, - columns=data["columns"], - dtype=float, - ) - - return df.sort_index().resample("h").mean() - - -def read_hourly_data_from_cache(usaf_id, year): - key = get_hourly_data_cache_key(usaf_id, year) - store = eeweather.connections.key_value_store_proxy.get_store() - - return deserialize_hourly_data(store.retrieve_json(key)) - - -def write_hourly_data_to_cache(usaf_id, year, df): - key = get_hourly_data_cache_key(usaf_id, year) - store = eeweather.connections.key_value_store_proxy.get_store() - - return store.save_json(key, serialize_hourly_data(df)) - - -def destroy_cached_hourly_data(usaf_id, year): - key = get_hourly_data_cache_key(usaf_id, year) - store = eeweather.connections.key_value_store_proxy.get_store() - - return store.clear(key) - - -def load_hourly_data_cached_proxy( - usaf_id, - year, - variables=DEFAULT_VARIABLES, - read_from_cache=True, - write_to_cache=True, - fetch_from_web=True, -): - """One year of hourly data, from cache when it covers the request. - - A cache entry serves the request when it is fresh and holds every - requested variable. Fetches request the union of the requested and - already-cached variables so a cache refresh never drops columns. - """ - variables = tuple(variables) - cached = None - if validate_hourly_data_cache(usaf_id, year): - cached = read_hourly_data_from_cache(usaf_id, year) - - cache_covers_request = cached is not None and set(variables) <= set(cached.columns) - if read_from_cache and cache_covers_request: - return cached[list(variables)] - - if not fetch_from_web: - raise DataNotAvailableError(usaf_id, year) - - cached_columns = () if cached is None else tuple(cached.columns) - fetch_variables = tuple(dict.fromkeys(variables + cached_columns)) - df = fetch_hourly_data(usaf_id, year, fetch_variables) - if write_to_cache: - write_hourly_data_to_cache(usaf_id, year, df) - - return df[list(variables)] - - -def load_data( - usaf_id, - start, - end, - frequency="hourly", - variables=DEFAULT_VARIABLES, - read_from_cache=True, - write_to_cache=True, - fetch_from_web=True, - error_on_missing_years=True, -): - """Load a station's weather data between two dates (inclusive). - - This is the primary interface for loading observed weather data. - - Parameters - ---------- - usaf_id : str - Station USAF id. - start : datetime.datetime - The earliest date from which to load data. Must be UTC. - end : datetime.datetime - The latest date until which to load data. Must be UTC. - frequency : str - ``'hourly'`` or ``'daily'``. Daily values are means of the hourly - values within each day. - variables : tuple of str - GHCNh variable names. Temperatures are degrees Celsius; units for - other variables follow the GHCNh documentation. - read_from_cache : bool - Whether or not to load data from cache. - write_to_cache : bool - Whether or not to write newly loaded data to cache. - fetch_from_web : bool - Whether or not to fetch data from the web. - error_on_missing_years : bool - Whether to raise when data is unavailable for a year in the range, - or to warn and fill that year with NaN. - - Returns - ------- - tuple of (pandas.DataFrame, list of EEWeatherWarning) - One column per requested variable, indexed over the full requested - range at the requested frequency; periods without data are NaN. - Warnings describe years with no data and gaps in the returned data. - """ - # CalTRACK 2.3.3 - if not _datetime_is_utc(start): - raise NonUTCTimezoneInfoError(start) - if not _datetime_is_utc(end): - raise NonUTCTimezoneInfoError(end) - - if frequency == "hourly": - freq = "h" - elif frequency == "daily": - freq = "D" - else: - raise ValueError( - "frequency must be 'hourly' or 'daily', got: {}".format(frequency) - ) - - variables = tuple(variables) - warnings = [] - data = [] - for year in range(start.year, end.year + 1): - try: - data.append( - load_hourly_data_cached_proxy( - usaf_id, - year, - variables=variables, - read_from_cache=read_from_cache, - write_to_cache=write_to_cache, - fetch_from_web=fetch_from_web, - ) - ) - except DataNotAvailableError: - if error_on_missing_years: - raise - warnings.append( - EEWeatherWarning( - qualified_name="eeweather.data_not_available", - description="Data not available", - data={"usaf_id": usaf_id, "year": year}, - ) - ) - - if data: - df = pd.concat(data) - if frequency == "daily": - df = df.resample("D").mean() - df = df[start:end] - else: - empty_index = pd.DatetimeIndex([], tz=pytz.UTC) - df = pd.DataFrame(columns=list(variables), index=empty_index, dtype=float) - - # because start and end dates need to fall exactly on period boundaries - if frequency == "hourly": - range_start = datetime( - start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC - ) - if range_start < start: - range_start += timedelta(hours=1) - range_end = datetime(end.year, end.month, end.day, end.hour, tzinfo=pytz.UTC) - else: - range_start = datetime(start.year, start.month, start.day, tzinfo=pytz.UTC) - if range_start < start: - range_start += timedelta(days=1) - range_end = datetime(end.year, end.month, end.day, tzinfo=pytz.UTC) - - # fill in gaps, covering the full requested range even when no data loaded - df = df.reindex(pd.date_range(range_start, range_end, freq=freq, tz=pytz.UTC)) - for variable in variables: - warnings.extend(_data_gap_warnings(df[variable], variable)) - - return df, warnings - - -def load_cached_hourly_data(usaf_id): - """All cached hourly data for a station, or None when none is cached.""" - store = eeweather.connections.key_value_store_proxy.get_store() - - data = [ - read_hourly_data_from_cache(usaf_id, year) - for year in range(2000, datetime.now().year + 1) - if store.key_exists(get_hourly_data_cache_key(usaf_id, year)) - ] - if data == []: - return None - - return pd.concat(data).resample("h").mean() - - -def get_isd_station_metadata(usaf_id): - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select - * - from - isd_station_metadata - where - usaf_id = ? - """, - (usaf_id,), - ) - row = cur.fetchone() - if row is None: - raise UnrecognizedUSAFIDError(usaf_id) - return {col[0]: row[i] for i, col in enumerate(cur.description)} - - -GHCN_INVENTORY_MONTH_COLUMNS = ( - "jan", "feb", "mar", "apr", "may", "jun", - "jul", "aug", "sep", "oct", "nov", "dec", -) - - -QUALITY_WINDOW_YEARS = 5 - -# every month of the rating window above these observation counts -HIGH_MONTHLY_OBSERVATIONS = 600 -MEDIUM_MONTHLY_OBSERVATIONS = 360 - - -def _quality_rating_window(start, end): - """Calendar years rating a request: five years ending two years after - the request's last date, sliding back to end no later than the last - full year.""" - last_full_year = datetime.now().year - 1 - window_end = min(end.year + 2, last_full_year) - window_start = window_end - (QUALITY_WINDOW_YEARS - 1) - - return window_start, window_end - - -def _quality_from_minimum(minimum): - if minimum > HIGH_MONTHLY_OBSERVATIONS: - return "high" - elif minimum > MEDIUM_MONTHLY_OBSERVATIONS: - return "medium" - - return "low" - - -def get_station_quality(usaf_id, start, end): - """Station quality for a request period, from GHCNh observation counts. - - Rates the five calendar years ending two years after the request's - last date (sliding back so the window ends no later than the last - full year): every month over 600 observations is high, over 360 is - medium; anything less, including absent months or years, is low. - """ - window_start, window_end = _quality_rating_window(start, end) - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select year, {} - from ghcn_inventory - where usaf_id = ? and year between ? and ? - """.format( - ", ".join(GHCN_INVENTORY_MONTH_COLUMNS) - ), - (usaf_id, window_start, window_end), - ) - counts_by_year = {row[0]: row[1:] for row in cur.fetchall()} - - minimum = None - for year in range(window_start, window_end + 1): - year_counts = counts_by_year.get(year, (0,) * 12) - year_minimum = min(year_counts) - if minimum is None or year_minimum < minimum: - minimum = year_minimum - - return _quality_from_minimum(minimum) - - -def get_station_qualities(start, end): - """Quality for a request period for every station, as a usaf_id-indexed - Series. - - Same rating as get_station_quality, computed for the whole registry in - one query. - """ - window_start, window_end = _quality_rating_window(start, end) - conn = metadata_db_connection_proxy.get_connection() - inventory = pd.read_sql_query( - """ - select usaf_id, year, {} - from ghcn_inventory - where year between ? and ? - """.format( - ", ".join(GHCN_INVENTORY_MONTH_COLUMNS) - ), - conn, - params=(window_start, window_end), - ) - - months = list(GHCN_INVENTORY_MONTH_COLUMNS) - year_min = inventory[months].min(axis=1) - observed_min = year_min.groupby(inventory.usaf_id).min() - - # a station must have a row for every year of the window - n_years = window_end - window_start + 1 - year_counts = inventory.groupby("usaf_id").year.nunique() - observed_min = observed_min.where(year_counts >= n_years, 0) - - qualities = pd.Series("low", index=observed_min.index) - qualities[observed_min > MEDIUM_MONTHLY_OBSERVATIONS] = "medium" - qualities[observed_min > HIGH_MONTHLY_OBSERVATIONS] = "high" - - return qualities - - -def get_isd_file_metadata(usaf_id): - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select - * - from - isd_file_metadata - where - usaf_id = ? - """, - (usaf_id,), - ) - rows = cur.fetchall() - if rows == []: - raise UnrecognizedUSAFIDError(usaf_id) - return [{col[0]: row[i] for i, col in enumerate(cur.description)} for row in rows] - - -def get_tmy3_station_metadata(usaf_id): - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select - * - from - tmy3_station_metadata - where - usaf_id = ? - """, - (usaf_id,), - ) - row = cur.fetchone() - if row is None: - raise TMY3DataNotAvailableError(usaf_id) - return {col[0]: row[i] for i, col in enumerate(cur.description)} - - -def get_cz2010_station_metadata(usaf_id): - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - cur.execute( - """ - select - * - from - cz2010_station_metadata - where - usaf_id = ? - """, - (usaf_id,), - ) - row = cur.fetchone() - if row is None: - raise CZ2010DataNotAvailableError(usaf_id) - return {col[0]: row[i] for i, col in enumerate(cur.description)} - - -def fetch_tmy3_hourly_temp_data(usaf_id): - url = ( - "https://storage.googleapis.com/openeemeter-public-resources/" - "tmy3_archive/{}TYA.CSV".format(usaf_id) - ) - - # checks that the station has TMY3 data associated with it. - tmy3_metadata = get_tmy3_station_metadata(usaf_id) - - return fetch_hourly_normalized_temp_data(usaf_id, url, "TMY3") - - -def fetch_cz2010_hourly_temp_data(usaf_id): - url = "https://storage.googleapis.com/oee-cz2010/csv/{}_CZ2010.CSV".format(usaf_id) - - # checks that the station has CZ2010 data associated with it. - cz2010_metadata = get_cz2010_station_metadata(usaf_id) - - return fetch_hourly_normalized_temp_data(usaf_id, url, "CZ2010") - - -@eeweather.mockable.mockable() -def request_text(url): # pragma: no cover - response = requests.get(url) - if response.ok: - return response.text - else: - raise RuntimeError("Could not find {}.".format(url)) - - -def fetch_hourly_normalized_temp_data(usaf_id, url, source_name): - index = pd.date_range("1900-01-01 00:00", "1900-12-31 23:00", freq="h", tz=pytz.UTC) - ts = pd.Series(None, index=index, dtype=float) - - lines = eeweather.mockable.request_text(url).splitlines() - - utc_offset_str = lines[0].split(",")[3] - utc_offset = timedelta(seconds=3600 * float(utc_offset_str)) - - for line in lines[2:]: - row = line.split(",") - month = row[0][0:2] - day = row[0][3:5] - hour = int(row[1][0:2]) - 1 - - # YYYYMMDDHH - date_string = "1900{}{}{:02d}".format(month, day, hour) - - dt = datetime.strptime(date_string, "%Y%m%d%H") - utc_offset - - # Only a little redundant to make year 1900 again - matters for - # first or last few hours of the year depending UTC on offset - dt = pytz.UTC.localize(dt.replace(year=1900)) - temp_C = float(row[31]) - - ts[dt] = temp_C - - return ts - - -def get_tmy3_hourly_temp_data_cache_key(usaf_id): - return "tmy3-hourly-{}".format(usaf_id) - - -def get_cz2010_hourly_temp_data_cache_key(usaf_id): - return "cz2010-hourly-{}".format(usaf_id) - - -def _expired(last_updated, year): - if last_updated is None: - return True - expiration_limit = pytz.UTC.localize( - datetime.now() - timedelta(days=DATA_EXPIRATION_DAYS) - ) - updated_during_data_year = year == last_updated.year - return expiration_limit > last_updated and updated_during_data_year - - -def validate_tmy3_hourly_temp_data_cache(usaf_id): - key = get_tmy3_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - - # fail if no key - if not store.key_exists(key): - return False - - return True - - -def validate_cz2010_hourly_temp_data_cache(usaf_id): - key = get_cz2010_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - - # fail if no key - if not store.key_exists(key): - return False - - return True - - -def _serialize(ts, freq): - if freq == "h": - dt_format = "%Y%m%d%H" - elif freq == "D": - dt_format = "%Y%m%d" - else: # pragma: no cover - raise ValueError('Unrecognized frequency "{}"'.format(freq)) - - return [ - [d.strftime(dt_format), round(temp, 4) if pd.notnull(temp) else None] - for d, temp in ts.items() - ] - - -def serialize_tmy3_hourly_temp_data(ts): - return _serialize(ts, "h") - - -def serialize_cz2010_hourly_temp_data(ts): - return _serialize(ts, "h") - - -def _deserialize(data, freq): - if freq == "h": - dt_format = "%Y%m%d%H" - elif freq == "D": - dt_format = "%Y%m%d" - else: # pragma: no cover - raise ValueError('Unrecognized frequency "{}"'.format(freq)) - - dates, values = zip(*data) - index = pd.to_datetime(dates, format=dt_format, utc=True) - return ( - pd.Series(values, index=index, dtype=float).sort_index().resample(freq).mean() - ) - - -def deserialize_tmy3_hourly_temp_data(data): - return _deserialize(data, "h") - - -def deserialize_cz2010_hourly_temp_data(data): - return _deserialize(data, "h") - - -def read_tmy3_hourly_temp_data_from_cache(usaf_id): - key = get_tmy3_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - return deserialize_tmy3_hourly_temp_data(store.retrieve_json(key)) - - -def read_cz2010_hourly_temp_data_from_cache(usaf_id): - key = get_cz2010_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - return deserialize_cz2010_hourly_temp_data(store.retrieve_json(key)) - - -def write_tmy3_hourly_temp_data_to_cache(usaf_id, ts): - key = get_tmy3_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - return store.save_json(key, serialize_tmy3_hourly_temp_data(ts)) - - -def write_cz2010_hourly_temp_data_to_cache(usaf_id, ts): - key = get_cz2010_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - return store.save_json(key, serialize_cz2010_hourly_temp_data(ts)) - - -def destroy_cached_tmy3_hourly_temp_data(usaf_id): - key = get_tmy3_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - return store.clear(key) - - -def destroy_cached_cz2010_hourly_temp_data(usaf_id): - key = get_cz2010_hourly_temp_data_cache_key(usaf_id) - store = eeweather.connections.key_value_store_proxy.get_store() - return store.clear(key) - - -def load_tmy3_hourly_temp_data_cached_proxy( - usaf_id, read_from_cache=True, write_to_cache=True, fetch_from_web=True -): - # take from cache? - data_ok = validate_tmy3_hourly_temp_data_cache(usaf_id) - - if not fetch_from_web and not data_ok: - raise TMY3DataNotAvailableError(usaf_id) - elif fetch_from_web and (not read_from_cache or not data_ok): - # need to actually fetch the data - ts = fetch_tmy3_hourly_temp_data(usaf_id) - if write_to_cache: - write_tmy3_hourly_temp_data_to_cache(usaf_id, ts) - else: - # read_from_cache=True and data_ok=True - ts = read_tmy3_hourly_temp_data_from_cache(usaf_id) - return ts - - -def load_cz2010_hourly_temp_data_cached_proxy( - usaf_id, read_from_cache=True, write_to_cache=True, fetch_from_web=True -): - # take from cache? - data_ok = validate_cz2010_hourly_temp_data_cache(usaf_id) - - if not fetch_from_web and not data_ok: - raise CZ2010DataNotAvailableError(usaf_id) - elif fetch_from_web and (not read_from_cache or not data_ok): - # need to actually fetch the data - ts = fetch_cz2010_hourly_temp_data(usaf_id) - if write_to_cache: - write_cz2010_hourly_temp_data_to_cache(usaf_id, ts) - else: - # read_from_cache=True and data_ok=True - ts = read_cz2010_hourly_temp_data_from_cache(usaf_id) - return ts - - -def load_tmy3_hourly_temp_data( - usaf_id, start, end, read_from_cache=True, write_to_cache=True, fetch_from_web=True -): - # CalTRACK 2.3.3 - if not _datetime_is_utc(start): - raise NonUTCTimezoneInfoError(start) - if not _datetime_is_utc(end): - raise NonUTCTimezoneInfoError(end) - single_year_data = load_tmy3_hourly_temp_data_cached_proxy( - usaf_id, - read_from_cache=read_from_cache, - write_to_cache=write_to_cache, - fetch_from_web=fetch_from_web, - ) - - # dealing with year replacement - data = [] - for year in range(start.year, end.year + 1): - single_year_index = single_year_data.index.map(lambda t: t.replace(year=year)) - - data.append(pd.Series(single_year_data.values, index=single_year_index)) - - # get raw data - ts = pd.concat(data).resample("h").mean() - - # whittle down - ts = ts[start:end] - - # fill in gaps - ts = ts.reindex(pd.date_range(start, end, freq="h", tz=pytz.UTC)) - return ts - - -def load_cz2010_hourly_temp_data( - usaf_id, start, end, read_from_cache=True, write_to_cache=True, fetch_from_web=True -): - # CalTRACK 2.3.3 - if not _datetime_is_utc(start): - raise NonUTCTimezoneInfoError(start) - if not _datetime_is_utc(end): - raise NonUTCTimezoneInfoError(end) - single_year_data = load_cz2010_hourly_temp_data_cached_proxy( - usaf_id, - read_from_cache=read_from_cache, - write_to_cache=write_to_cache, - fetch_from_web=fetch_from_web, - ) - - # dealing with year replacement - data = [] - for year in range(start.year, end.year + 1): - single_year_index = single_year_data.index.map(lambda t: t.replace(year=year)) - - data.append(pd.Series(single_year_data.values, index=single_year_index)) - - # get raw data - ts = pd.concat(data).resample("h").mean() - - # whittle down - ts = ts[start:end] - - # fill in gaps - ts = ts.reindex(pd.date_range(start, end, freq="h", tz=pytz.UTC)) - return ts - - -def load_cached_tmy3_hourly_temp_data(usaf_id): - store = eeweather.connections.key_value_store_proxy.get_store() - - if store.key_exists(get_tmy3_hourly_temp_data_cache_key(usaf_id)): - return read_tmy3_hourly_temp_data_from_cache(usaf_id) - else: - return None - - -def load_cached_cz2010_hourly_temp_data(usaf_id): - store = eeweather.connections.key_value_store_proxy.get_store() - - if store.key_exists(get_cz2010_hourly_temp_data_cache_key(usaf_id)): - return read_cz2010_hourly_temp_data_from_cache(usaf_id) - else: - return None - - -class WeatherStation(object): - """A representation of a weather station. - - Contains data about a particular weather station, as well as methods to - pull data for this station. Stations are keyed by their ISD-registry - USAF id; observed data is served from the station's GHCNh record. - - Parameters - ---------- - usaf_id : str - Station USAF ID - load_metatdata : bool, optional - Whether or not to auto-load metadata for this station - - Attributes - ---------- - usaf_id : str - Station USAF ID - ghcn_id : str - GHCNh station id observed data is fetched with - iecc_climate_zone : str - IECC Climate Zone - iecc_moisture_regime : str - IECC Moisture Regime - ba_climate_zone : str - Building America Climate Zone - ca_climate_zone : str - California Building Climate Zone - elevation : float - elevation of station - latitude : float - latitude of station - longitude : float - longitude of station - coords : tuple of (float, float) - lat/long coordinates of station - name : str - name of the station - quality : str - "high", "medium", "low" - wban_ids : list of str - list of WBAN IDs, or "99999" which have been used to identify the station. - recent_wban_id = None - WBAN ID most recently used to identify the station. - climate_zones = {} - dict of all climate zones. - """ - - def __init__(self, usaf_id, load_metadata=True): - self.usaf_id = usaf_id - - if load_metadata: - self._load_metadata() - else: - valid_usaf_id_or_raise(usaf_id) - self.iecc_climate_zone = None - self.iecc_moisture_regime = None - self.ba_climate_zone = None - self.ca_climate_zone = None - self.elevation = None - self.latitude = None - self.longitude = None - self.coords = None - self.name = None - self.quality = None - self.wban_ids = None - self.recent_wban_id = None - self.ghcn_id = None - self.ghcn_first_year = None - self.ghcn_last_year = None - self.climate_zones = {} - - def __str__(self): - return self.usaf_id - - def __repr__(self): - return "WeatherStation('{}')".format(self.usaf_id) - - def _load_metadata(self): - metadata = get_isd_station_metadata(self.usaf_id) - - def _float_or_none(field): - value = metadata.get(field) - return None if value is None else float(value) - - self.iecc_climate_zone = metadata.get("iecc_climate_zone") - self.iecc_moisture_regime = metadata.get("iecc_moisture_regime") - self.ba_climate_zone = metadata.get("ba_climate_zone") - self.ca_climate_zone = metadata.get("ca_climate_zone") - self.icao_code = metadata.get("icao_code") - self.elevation = _float_or_none("elevation") # meters - self.latitude = _float_or_none("latitude") - self.longitude = _float_or_none("longitude") - self.coords = (self.latitude, self.longitude) - self.name = metadata.get("name") - self.quality = metadata.get("quality") - self.wban_ids = metadata.get("wban_ids", "").split(",") - self.recent_wban_id = metadata.get("recent_wban_id") - self.ghcn_id = metadata.get("ghcn_id") - self.ghcn_first_year = metadata.get("ghcn_first_year") - self.ghcn_last_year = metadata.get("ghcn_last_year") - self.climate_zones = { - "iecc_climate_zone": metadata.get("iecc_climate_zone"), - "iecc_moisture_regime": metadata.get("iecc_moisture_regime"), - "ba_climate_zone": metadata.get("ba_climate_zone"), - "ca_climate_zone": metadata.get("ca_climate_zone"), - } - - def json(self): - """Return a JSON-serializeable object containing station metadata.""" - return { - "elevation": self.elevation, - "latitude": self.latitude, - "longitude": self.longitude, - "icao_code": self.icao_code, - "name": self.name, - "quality": self.quality, - "wban_ids": self.wban_ids, - "recent_wban_id": self.recent_wban_id, - "ghcn_id": self.ghcn_id, - "ghcn_first_year": self.ghcn_first_year, - "ghcn_last_year": self.ghcn_last_year, - "climate_zones": { - "iecc_climate_zone": self.iecc_climate_zone, - "iecc_moisture_regime": self.iecc_moisture_regime, - "ba_climate_zone": self.ba_climate_zone, - "ca_climate_zone": self.ca_climate_zone, - }, - } - - def load_data( - self, - start, - end, - frequency="hourly", - variables=DEFAULT_VARIABLES, - read_from_cache=True, - write_to_cache=True, - fetch_from_web=True, - error_on_missing_years=True, - ): - """Load this station's weather data between two dates (inclusive). - - This is the primary interface for loading observed weather data. - - Parameters - ---------- - start : datetime.datetime - The earliest date from which to load data. Must be UTC. - end : datetime.datetime - The latest date until which to load data. Must be UTC. - frequency : str - ``'hourly'`` or ``'daily'``. Daily values are means of the - hourly values within each day. - variables : tuple of str - GHCNh variable names. Temperatures are degrees Celsius; units - for other variables follow the GHCNh documentation. - read_from_cache : bool - Whether or not to load data from cache. - write_to_cache : bool - Whether or not to write newly loaded data to cache. - fetch_from_web : bool - Whether or not to fetch data from the web. - error_on_missing_years : bool - Whether to raise when data is unavailable for a year in the - range, or to warn and fill that year with NaN. - - Returns - ------- - tuple of (pandas.DataFrame, list of EEWeatherWarning) - One column per requested variable, indexed over the full - requested range at the requested frequency; periods without - data are NaN. - """ - return load_data( - self.usaf_id, - start, - end, - frequency=frequency, - variables=variables, - read_from_cache=read_from_cache, - write_to_cache=write_to_cache, - fetch_from_web=fetch_from_web, - error_on_missing_years=error_on_missing_years, - ) - - def get_quality(self, start, end): - """Station quality over a period, from GHCNh observation counts.""" - return get_station_quality(self.usaf_id, start, end) - - def load_cached_data(self): - """Load all cached hourly data for this station.""" - return load_cached_hourly_data(self.usaf_id) - - def destroy_cached_hourly_data(self, year): - """Remove cached hourly data for this station for the given year.""" - return destroy_cached_hourly_data(self.usaf_id, year) - - def get_isd_file_metadata(self): - """Get raw file metadata for the station.""" - return get_isd_file_metadata(self.usaf_id) - - # fetch raw data - # fetch raw data then frequency-normalize - def fetch_tmy3_hourly_temp_data(self): - """Pull hourly TMY3 temperature hourly time series directly from NREL.""" - return fetch_tmy3_hourly_temp_data(self.usaf_id) - - def fetch_cz2010_hourly_temp_data(self): - """Pull hourly CZ2010 temperature hourly time series from URL.""" - return fetch_cz2010_hourly_temp_data(self.usaf_id) - - # get key-value store key - def get_tmy3_hourly_temp_data_cache_key(self): - """Get key used to cache TMY3 weather-normalized temperature data.""" - return get_tmy3_hourly_temp_data_cache_key(self.usaf_id) - - def get_cz2010_hourly_temp_data_cache_key(self): - """Get key used to cache CZ2010 weather-normalized temperature data.""" - return get_cz2010_hourly_temp_data_cache_key(self.usaf_id) - - # is cached data expired? boolean. true if expired or not in cache - # check if data is available and delete data in the cache if it's expired - def validate_tmy3_hourly_temp_data_cache(self): - """Check if TMY3 data exists in cache.""" - return validate_tmy3_hourly_temp_data_cache(self.usaf_id) - - def validate_cz2010_hourly_temp_data_cache(self): - """Check if CZ2010 data exists in cache.""" - return validate_cz2010_hourly_temp_data_cache(self.usaf_id) - - # pandas time series to json - def serialize_tmy3_hourly_temp_data(self, ts): - """Serialize hourly TMY3 pandas time series as JSON for caching.""" - return serialize_tmy3_hourly_temp_data(ts) - - def serialize_cz2010_hourly_temp_data(self, ts): - """Serialize hourly CZ2010 pandas time series as JSON for caching.""" - return serialize_cz2010_hourly_temp_data(ts) - - # json to pandas time series - def deserialize_tmy3_hourly_temp_data(self, data): - """Deserialize JSON representation of hourly TMY3 into pandas time series.""" - return deserialize_tmy3_hourly_temp_data(data) - - def deserialize_cz2010_hourly_temp_data(self, data): - """Deserialize JSON representation of hourly CZ2010 into pandas time series.""" - return deserialize_cz2010_hourly_temp_data(data) - - # return pandas time series of data from cache - def read_tmy3_hourly_temp_data_from_cache(self): - """Get cached version of hourly TMY3 temperature data.""" - return read_tmy3_hourly_temp_data_from_cache(self.usaf_id) - - def read_cz2010_hourly_temp_data_from_cache(self): - """Get cached version of hourly TMY3 temperature data.""" - return read_cz2010_hourly_temp_data_from_cache(self.usaf_id) - - # write pandas time series of data to cache for a particular year - def write_tmy3_hourly_temp_data_to_cache(self, ts): - """Write hourly TMY3 temperature data to cache for given year.""" - return write_tmy3_hourly_temp_data_to_cache(self.usaf_id, ts) - - def write_cz2010_hourly_temp_data_to_cache(self, ts): - """Write hourly CZ2010 temperature data to cache for given year.""" - return write_cz2010_hourly_temp_data_to_cache(self.usaf_id, ts) - - # delete cached data for a particular year - def destroy_cached_tmy3_hourly_temp_data(self): - """Remove cached hourly TMY3 temperature data to cache.""" - return destroy_cached_tmy3_hourly_temp_data(self.usaf_id) - - def destroy_cached_cz2010_hourly_temp_data(self): - """Remove cached hourly CZ2010 temperature data to cache.""" - return destroy_cached_cz2010_hourly_temp_data(self.usaf_id) - - # load data either from cache if valid or directly from source - def load_tmy3_hourly_temp_data_cached_proxy(self, fetch_from_web=True): - """Load hourly TMY3 temperature data from cache, or if it is expired or hadn't been cached, fetch from NREL.""" - return load_tmy3_hourly_temp_data_cached_proxy(self.usaf_id, fetch_from_web) - - def load_cz2010_hourly_temp_data_cached_proxy(self, fetch_from_web=True): - """Load hourly CZ2010 temperature data from cache, or if it is expired or hadn't been cached, fetch from URL.""" - return load_cz2010_hourly_temp_data_cached_proxy(self.usaf_id, fetch_from_web) - - # main interface: load data from start date to end date - def load_tmy3_hourly_temp_data( - self, start, end, read_from_cache=True, write_to_cache=True, fetch_from_web=True - ): - """Load hourly TMY3 temperature data from start date to end date (inclusive). - - This is the primary convenience method for loading hourly TMY3 temperature data. - - Parameters - ---------- - start : datetime.datetime - The earliest date from which to load data. - end : datetime.datetime - The latest date until which to load data. - read_from_cache : bool - Whether or not to load data from cache. - write_to_cache : bool - Whether or not to write newly loaded data to cache. - fetch_from_web : bool - Whether or not to fetch data from ftp. - """ - return load_tmy3_hourly_temp_data( - self.usaf_id, - start, - end, - read_from_cache=read_from_cache, - write_to_cache=write_to_cache, - fetch_from_web=fetch_from_web, - ) - - def load_cz2010_hourly_temp_data( - self, start, end, read_from_cache=True, write_to_cache=True, fetch_from_web=True - ): - """Load hourly CZ2010 temperature data from start date to end date (inclusive). - - This is the primary convenience method for loading hourly CZ2010 temperature data. - - Parameters - ---------- - start : datetime.datetime - The earliest date from which to load data. - end : datetime.datetime - The latest date until which to load data. - read_from_cache : bool - Whether or not to load data from cache. - write_to_cache : bool - Whether or not to write newly loaded data to cache. - fetch_from_web : bool - Whether or not to fetch data from ftp. - """ - return load_cz2010_hourly_temp_data( - self.usaf_id, - start, - end, - read_from_cache=read_from_cache, - write_to_cache=write_to_cache, - fetch_from_web=fetch_from_web, - ) - - # load all cached data for this station - def load_cached_tmy3_hourly_temp_data(self): - """Load all cached hourly TMY3 temperature data (the year is set to 1900)""" - return load_cached_tmy3_hourly_temp_data(self.usaf_id) - - def load_cached_cz2010_hourly_temp_data(self): - """Load all cached hourly TMY3 temperature data (the year is set to 1900)""" - return load_cached_cz2010_hourly_temp_data(self.usaf_id) diff --git a/eeweather/summaries.py b/eeweather/summaries.py deleted file mode 100644 index a97fd33..0000000 --- a/eeweather/summaries.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from .connections import metadata_db_connection_proxy - -__all__ = ("get_zcta_ids", "get_isd_station_usaf_ids") - - -def get_zcta_ids(state=None): - """Get ids of all supported ZCTAs, optionally by state. - - Parameters - ---------- - state : str, optional - Select zipcodes only from this state or territory, given as 2-letter - abbreviation (e.g., ``'CA'``, ``'PR'``). - - Returns - ------- - results : list of str - List of all supported selected ZCTA IDs. - """ - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - - if state is None: - cur.execute( - """ - select zcta_id from zcta_metadata - """ - ) - else: - cur.execute( - """ - select zcta_id from zcta_metadata where state = ? - """, - (state,), - ) - return [row[0] for row in cur.fetchall()] - - -def get_isd_station_usaf_ids(state=None): - """Get USAF IDs of all supported ISD stations, optionally by state. - - Parameters - ---------- - state : str, optional - Select ISD station USAF IDs only from this state or territory, given - as 2-letter abbreviation (e.g., ``'CA'``, ``'PR'``). - - Returns - ------- - results : list of str - List of all supported selected ISD station USAF IDs. - """ - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - - if state is None: - cur.execute( - """ - select usaf_id from isd_station_metadata - """ - ) - else: - cur.execute( - """ - select usaf_id from isd_station_metadata where state = ? - """, - (state,), - ) - return [row[0] for row in cur.fetchall()] diff --git a/eeweather/utils.py b/eeweather/utils.py deleted file mode 100644 index 71a4cfc..0000000 --- a/eeweather/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import pandas as pd - -from .connections import metadata_db_connection_proxy - - - -class lazy_property(object): - """ - Meant to be used for lazy evaluation of an object attribute. - Property should represent non-mutable data, as it replaces itself. - - e.g., - - class Test(object): - - @lazy_property - def results(self): - calcs = 1 # Do a lot of calculation here - return calcs - - from https://stackoverflow.com/a/6849299/1965736 - """ - - def __init__(self, fget): - self.fget = fget - self.func_name = fget.__name__ - - def __get__(self, obj, cls): - value = self.fget(obj) - setattr(obj, self.func_name, value) - return value - - -def get_ghcn_ids(usaf_ids=None): - """GHCNh station ids mapped to ISD USAF ids, as a usaf_id-indexed Series. - - Parameters - ---------- - usaf_ids : list of str, optional - USAF ids to map. When None, the whole registry is returned. - Unrecognized ids are absent from the result. - """ - conn = metadata_db_connection_proxy.get_connection() - mapping = pd.read_sql_query( - "select usaf_id, ghcn_id from isd_station_metadata", conn - ).set_index("usaf_id")["ghcn_id"] - if usaf_ids is not None: - mapping = mapping[mapping.index.isin(list(usaf_ids))] - - return mapping diff --git a/eeweather/validation.py b/eeweather/validation.py deleted file mode 100644 index b0195a9..0000000 --- a/eeweather/validation.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from .connections import metadata_db_connection_proxy -from .exceptions import UnrecognizedZCTAError, UnrecognizedUSAFIDError - - -__all__ = ("valid_zcta_or_raise", "valid_usaf_id_or_raise") - - -def valid_zcta_or_raise(zcta): - """Check if ZCTA is valid and raise eeweather.UnrecognizedZCTAError if not.""" - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - - cur.execute( - """ - select exists ( - select - zcta_id - from - zcta_metadata - where - zcta_id = ? - ) - """, - (zcta,), - ) - (exists,) = cur.fetchone() - if exists: - return True - else: - raise UnrecognizedZCTAError(zcta) - - -def valid_usaf_id_or_raise(usaf_id): - """Check if USAF ID is valid and raise eeweather.UnrecognizedUSAFIDError if not.""" - conn = metadata_db_connection_proxy.get_connection() - cur = conn.cursor() - - cur.execute( - """ - select exists ( - select - usaf_id - from - isd_station_metadata - where - usaf_id = ? - ) - """, - (usaf_id,), - ) - (exists,) = cur.fetchone() - if exists: - return True - else: - raise UnrecognizedUSAFIDError(usaf_id) diff --git a/eeweather/visualization.py b/eeweather/visualization.py deleted file mode 100644 index 7643114..0000000 --- a/eeweather/visualization.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import numpy as np - -from .connections import metadata_db_connection_proxy -from .exceptions import UnrecognizedUSAFIDError -from .stations import WeatherStation - -__all__ = ("plot_station_mapping", "plot_station_mappings") - - -def plot_station_mapping( - target_latitude, - target_longitude, - isd_station, - distance_meters, - target_label="target", -): # pragma: no cover - """Plots this mapping on a map.""" - try: - import matplotlib.pyplot as plt - except ImportError: - raise ImportError("Plotting requires matplotlib.") - - try: - import cartopy.crs as ccrs - import cartopy.feature as cfeature - import cartopy.io.img_tiles as cimgt - except ImportError: - raise ImportError("Plotting requires cartopy.") - - lat, lng = isd_station.coords - t_lat, t_lng = float(target_latitude), float(target_longitude) - - # fiture - fig = plt.figure(figsize=(16, 8)) - - # axes - tiles = cimgt.Stamen("terrain-background") - ax = plt.subplot(1, 1, 1, projection=tiles.crs) - - # offsets for labels - x_max = max([lng, t_lng]) - x_min = min([lng, t_lng]) - x_diff = x_max - x_min - - y_max = max([lat, t_lat]) - y_min = min([lat, t_lat]) - y_diff = y_max - y_min - - xoffset = x_diff * 0.05 - yoffset = y_diff * 0.05 - - # minimum - left = x_min - x_diff * 0.5 - right = x_max + x_diff * 0.5 - bottom = y_min - y_diff * 0.3 - top = y_max + y_diff * 0.3 - - width_ratio = 2.0 - height_ratio = 1.0 - - if (right - left) / (top - bottom) > width_ratio / height_ratio: - # too short - goal = (right - left) * height_ratio / width_ratio - diff = goal - (top - bottom) - bottom = bottom - diff / 2.0 - top = top + diff / 2.0 - else: - # too skinny - goal = (top - bottom) * width_ratio / height_ratio - diff = goal - (right - left) - left = left - diff / 2.0 - right = right + diff / 2.0 - - ax.set_extent([left, right, bottom, top]) - - # determine zoom level - # tile size at level 1 = 64 km - # level 2 = 32 km, level 3 = 16 km, etc, i.e. 128/(2^n) km - N_TILES = 600 # (how many tiles approximately fit in distance) - km = distance_meters / 1000.0 - zoom_level = int(np.log2(128 * N_TILES / km)) - - ax.add_image(tiles, zoom_level) - - # line between - plt.plot( - [lng, t_lng], - [lat, t_lat], - linestyle="-", - dashes=[2, 2], - transform=ccrs.Geodetic(), - ) - - # station - ax.plot(lng, lat, "ko", markersize=7, transform=ccrs.Geodetic()) - - # target - ax.plot(t_lng, t_lat, "ro", markersize=7, transform=ccrs.Geodetic()) - - # station label - station_label = "{} ({})".format(isd_station.usaf_id, isd_station.name) - ax.text(lng + xoffset, lat + yoffset, station_label, transform=ccrs.Geodetic()) - - # target label - ax.text(t_lng + xoffset, t_lat + yoffset, target_label, transform=ccrs.Geodetic()) - - # distance labels - mid_lng = (lng + t_lng) / 2 - mid_lat = (lat + t_lat) / 2 - dist_text = "{:.01f} km".format(km) - ax.text(mid_lng + xoffset, mid_lat + yoffset, dist_text, transform=ccrs.Geodetic()) - - plt.show() - - -def plot_station_mappings(mapping_results): # pragma: no cover - """Plot a list of mapping results on a map. - - Requires matplotlib and cartopy. - - Parameters - ---------- - mapping_results : list of MappingResult objects - Mapping results to plot - """ - try: - import matplotlib.pyplot as plt - except ImportError: - raise ImportError("Plotting requires matplotlib.") - - try: - import cartopy.crs as ccrs - import cartopy.feature as cfeature - except ImportError: - raise ImportError("Plotting requires cartopy.") - - lats = [] - lngs = [] - t_lats = [] - t_lngs = [] - n_discards = 0 - for mapping_result in mapping_results: - if not mapping_result.is_empty(): - lat, lng = mapping_result.isd_station.coords - t_lat, t_lng = map(float, mapping_result.target_coords) - lats.append(lat) - lngs.append(lng) - t_lats.append(t_lat) - t_lngs.append(t_lng) - else: - n_discards += 1 - - print("Discarded {} empty mappings".format(n_discards)) - - # figure - fig = plt.figure(figsize=(60, 60)) - - # axes - ax = plt.subplot(1, 1, 1, projection=ccrs.Mercator()) - - # offsets for labels - all_lngs = lngs + t_lngs - all_lats = lats + t_lats - x_max = max(all_lngs) # lists - x_min = min(all_lngs) - x_diff = x_max - x_min - - y_max = max(all_lats) - y_min = min(all_lats) - y_diff = y_max - y_min - - # minimum - x_pad = 0.1 * x_diff - y_pad = 0.1 * y_diff - left = x_min - x_pad - right = x_max + x_pad - bottom = y_min - y_pad - top = y_max + y_pad - - width_ratio = 2.0 - height_ratio = 1.0 - - if (right - left) / (top - bottom) > height_ratio / width_ratio: - # too short - goal = (right - left) * height_ratio / width_ratio - diff = goal - (top - bottom) - bottom = bottom - diff / 2.0 - top = top + diff / 2.0 - else: - # too skinny - goal = (top - bottom) * width_ratio / height_ratio - diff = goal - (right - left) - left = left - diff / 2.0 - right = right + diff / 2.0 - - left = max(left, -179.9) - right = min(right, 179.9) - bottom = max([bottom, -89.9]) - top = min([top, 89.9]) - - ax.set_extent([left, right, bottom, top]) - - # OCEAN - ax.add_feature( - cfeature.NaturalEarthFeature( - "physical", - "ocean", - "50m", - edgecolor="face", - facecolor=cfeature.COLORS["water"], - ) - ) - - # LAND - ax.add_feature( - cfeature.NaturalEarthFeature( - "physical", - "land", - "50m", - edgecolor="face", - facecolor=cfeature.COLORS["land"], - ) - ) - - # BORDERS - ax.add_feature( - cfeature.NaturalEarthFeature( - "cultural", - "admin_0_boundary_lines_land", - "50m", - edgecolor="black", - facecolor="none", - ) - ) - - # LAKES - ax.add_feature( - cfeature.NaturalEarthFeature( - "physical", - "lakes", - "50m", - edgecolor="face", - facecolor=cfeature.COLORS["water"], - ) - ) - - # COASTLINE - ax.add_feature( - cfeature.NaturalEarthFeature( - "physical", "coastline", "50m", edgecolor="black", facecolor="none" - ) - ) - - # lines between - # for lat, t_lat, lng, t_lng in zip(lats, t_lats, lngs, t_lngs): - ax.plot( - [lngs, t_lngs], - [lats, t_lats], - color="k", - linestyle="-", - transform=ccrs.Geodetic(), - linewidth=0.3, - ) - - # stations - ax.plot(lngs, lats, "bo", markersize=1, transform=ccrs.Geodetic()) - - plt.title("Location to weather station mapping") - - plt.show() diff --git a/eeweather/warnings.py b/eeweather/warnings.py deleted file mode 100644 index 0e3399c..0000000 --- a/eeweather/warnings.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - - -class EEWeatherWarning(object): - """An object representing a warning and data associated with it. - - Attributes - ---------- - qualified_name : :any:`str` - Qualified name, e.g., `'eeweather.method_abc.missing_data'`. - description : :any:`str` - Prose describing the nature of the warning. - data : :any:`dict` - Data that reproducibly shows why the warning was issued. - """ - - def __init__(self, qualified_name, description, data): - self.qualified_name = qualified_name - self.description = description - self.data = data - - def __repr__(self): - return "EEWeatherWarning(qualified_name={})".format(self.qualified_name) - - def json(self): - """Return a JSON-serializable representation of this result. - - The output of this function can be converted to a serialized string - with :any:`json.dumps`. - """ - return { - "qualified_name": self.qualified_name, - "description": self.description, - "data": self.data, - } diff --git a/pyproject.toml b/pyproject.toml index 049f632..eaa2480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "eeweather" dynamic = ["version"] -description = "Weather station wrangling for EEmeter" +description = "Obtain weather station data for OpenDSM" readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" @@ -20,10 +20,10 @@ classifiers = [ ] dependencies = [ - "click", - "pandas>=1.0.0", + "numpy", + "pandas>=2.2", + "platformdirs", "pyproj>=1.9.6", - "pytz", "requests", "shapely", ] @@ -33,15 +33,14 @@ dev = [ "black", "coverage", "pytest", + "pytz", "pytest-cov", "pytest-xdist", + "ruff", "syrupy>=5,<6", "tox", ] -[project.scripts] -eeweather = "eeweather.cli:cli" - [project.urls] Homepage = "https://github.com/opendsm/eeweather" Documentation = "https://opendsm.energy" @@ -64,12 +63,20 @@ addopts = [ filterwarnings = [ "error", ] +# pytest's default norecursedirs includes "build", which would silently +# skip tests/build/; keep the rest of the defaults +norecursedirs = [".*", "*.egg", "dist", "venv"] + +[tool.ruff.lint] +# logic checks only; the project is deliberately not auto-formatted +select = ["F"] [tool.coverage.run] omit = [ ".tox/*", - # metadata-db rebuild code; runs against live primary sources, not unit-testable - "eeweather/database.py", + # registry maintenance runs against live primary sources or the packaged + # database of a previous schema; not unit-testable + "eeweather/build/*", ] [tool.coverage.report] @@ -78,6 +85,5 @@ exclude_lines = [ "if __name__ == .__main__.:", "raise NotImplementedError", ] -# Conservative floor for `tox -e coverage`. Raise it as test coverage is -# added. The full suite measures 95.3%. -fail_under = 94 +# Floor for `tox -e coverage`; the full suite measures 98.2%. +fail_under = 97 diff --git a/scripts/create_ca_climate_zone_geojson.sh b/scripts/create_ca_climate_zone_geojson.sh deleted file mode 100755 index e845dff..0000000 --- a/scripts/create_ca_climate_zone_geojson.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -DATA_DIR=${1:-data} - -mkdir -p $DATA_DIR - -# download and install CA climate zone raw data -wget -N https://community.esri.com/ccqpr47374/attachments/ccqpr47374/coordinate-reference-systemsforum-board/1814/1/CA_Building_Standards_Climate_Zones.zip -P $DATA_DIR -q --show-progress -unzip -q -o $DATA_DIR/CA_Building_Standards_Climate_Zones.zip -d $DATA_DIR - -# reproject to ESRI Shapefile -ogr2ogr -q -f "ESRI Shapefile" -t_srs EPSG:4326 $DATA_DIR/CA_Building_Standards_Climate_Zones_reprojected.shp $DATA_DIR/CA_Building_Standards_Climate_Zones.shp - -# convert to geojson -mapshaper -quiet -i $DATA_DIR/CA_Building_Standards_Climate_Zones_reprojected.shp -o format=geojson $DATA_DIR/CA_Building_Standards_Climate_Zones.json - -# clean up -rm -f $DATA_DIR/CA\ Building\ Standards\ Climate\ Zones.lyr -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.cpg -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.dbf -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.prj -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.sbn -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.sbx -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.shp -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.shp.xml -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.shx -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones.zip -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones_reprojected.dbf -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones_reprojected.prj -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones_reprojected.shp -rm -f $DATA_DIR/CA_Building_Standards_Climate_Zones_reprojected.shx diff --git a/scripts/create_county_geojson.sh b/scripts/create_county_geojson.sh deleted file mode 100755 index 988e8bb..0000000 --- a/scripts/create_county_geojson.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -DATA_DIR=${1:-data} - -mkdir -p $DATA_DIR - -# download and unzip county shapefiles -wget -N http://www2.census.gov/geo/tiger/GENZ2016/shp/cb_2016_us_county_500k.zip -P $DATA_DIR -q --show-progress -unzip -q -o $DATA_DIR/cb_2016_us_county_500k.zip -d $DATA_DIR - -# convert to GEOJSON - requires mapshaper (https://github.com/mbloch/mapshaper) -mapshaper -quiet -i $DATA_DIR/cb_2016_us_county_500k.shp -o format=geojson $DATA_DIR/cb_2016_us_county_500k.json - -# clean up -rm $DATA_DIR/cb_2016_us_county_500k.cpg -rm $DATA_DIR/cb_2016_us_county_500k.dbf -rm $DATA_DIR/cb_2016_us_county_500k.prj -rm $DATA_DIR/cb_2016_us_county_500k.shp -rm $DATA_DIR/cb_2016_us_county_500k.shp.ea.iso.xml -rm $DATA_DIR/cb_2016_us_county_500k.shp.iso.xml -rm $DATA_DIR/cb_2016_us_county_500k.shp.xml -rm $DATA_DIR/cb_2016_us_county_500k.shx -rm $DATA_DIR/cb_2016_us_county_500k.zip diff --git a/scripts/create_zipcode_geojson.sh b/scripts/create_zipcode_geojson.sh deleted file mode 100755 index d8ab27c..0000000 --- a/scripts/create_zipcode_geojson.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -DATA_DIR=${1:-data} - -mkdir -p $DATA_DIR - -# Download and unzip -wget -N http://www2.census.gov/geo/tiger/GENZ2016/shp/cb_2016_us_zcta510_500k.zip -P $DATA_DIR -q --show-progress -unzip -q -o $DATA_DIR/cb_2016_us_zcta510_500k.zip -d $DATA_DIR - -# Convert to GEOJSON - requires mapshaper (https://github.com/mbloch/mapshaper) -mapshaper -quiet -i $DATA_DIR/cb_2016_us_zcta510_500k.shp -o format=geojson $DATA_DIR/cb_2016_us_zcta510_500k.json - -# Clean up -rm $DATA_DIR/cb_2016_us_zcta510_500k.cpg -rm $DATA_DIR/cb_2016_us_zcta510_500k.dbf -rm $DATA_DIR/cb_2016_us_zcta510_500k.prj -rm $DATA_DIR/cb_2016_us_zcta510_500k.shp -rm $DATA_DIR/cb_2016_us_zcta510_500k.shp.ea.iso.xml -rm $DATA_DIR/cb_2016_us_zcta510_500k.shp.iso.xml -rm $DATA_DIR/cb_2016_us_zcta510_500k.shp.xml -rm $DATA_DIR/cb_2016_us_zcta510_500k.shx -rm $DATA_DIR/cb_2016_us_zcta510_500k.zip diff --git a/scripts/data/tmy3-stations.html b/scripts/data/tmy3-stations.html deleted file mode 100644 index c00ffe9..0000000 --- a/scripts/data/tmy3-stations.html +++ /dev/null @@ -1,6238 +0,0 @@ - - - - - - - - - - -NSRDB Update - TMY3: Numerical List by USAF# - - - - - - - - - -
-
The Wayback Machine - -https://web.archive.org/web/20181119091712/https://rredc.nrel.gov/solar/old_data/nsrdb/1991-2005/tmy3/by_USAFN.html
- - - - -
- - - - - -
National - Solar Radiation Data Base -
- 1991- 2005 Update: - - Typical Meteorological Year 3
-
-
-

- - - - - -
  - - List of Sites in Numerical Order by USAF Number -

- Click on a site number to access the Typical Meteorological Year version 3 files for that site. - -

-

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    690150.............. Twentynine Palms, CA   Class I
    690190.............. Abilene Dyess AFB, TX   Class II
    690230.............. Whidbey Island NAS, WA   Class II
    699604.............. Yuma MCAS, AZ   Class II
    700197.............. Selawik, AK   Class III
    700260.............. Barrow W Post-W Rogers Arpt [NSA - ARM], AK   Class II
    700637.............. Deadhorse, AK   Class II
    701043.............. Point Hope (AWOS), AK   Class II
    701195.............. Shishmaref (AWOS), AK   Class II
    701330.............. Kotzebue Ralph Wein Memorial, AK   Class II
    701625.............. Anaktuvuk Pass, AK   Class III
    701718.............. Ambler, AK   Class II
    701740.............. Bettles Field, AK   Class II
    701780.............. Tanana Ralph M Calhoun Mem AP, AK   Class II
    701940.............. Fort Yukon, AK   Class III
    702000.............. Nome Municipal Arpt, AK   Class II
    702005.............. Saint Mary's (AWOS), AK   Class II
    702035.............. Savoonga, AK   Class III
    702040.............. Gambell, AK   Class III
    702070.............. Unalakleet Field, AK   Class III
    702075.............. Anvik, AK   Class III
    702084.............. Emmonak, AK   Class II
    702185.............. Mekoryuk, AK   Class II
    702186.............. Hooper Bay, AK   Class II
    702190.............. Bethel Airport, AK   Class II
    702225.............. Huslia, AK   Class III
    702310.............. McGrath Arpt, AK   Class II
    702320.............. Aniak Airport, AK   Class III
    702460.............. Minchumina, AK   Class II
    702495.............. Hayes River, AK   Class II
    702510.............. Talkeetna State Arpt, AK   Class II
    702590.............. Kenai Municipal AP, AK   Class II
    702595.............. Soldotna, AK   Class III
    702600.............. Nenana Municipal AP, AK   Class II
    702606.............. Chulitna, AK   Class II
    702607.............. Hoonah, AK   Class II
    702610.............. Fairbanks Intl Arpt, AK   Class II
    702647.............. Healy River Airport, AK   Class II
    702650.............. Fairbanks/Eielson A, AK   Class II
    702670.............. Big Delta Allen AAF, AK   Class II
    702710.............. Gulkana Intermediate Field, AK   Class II
    702720.............. Anchorage/Elmendorf, AK   Class II
    702725.............. Lake Hood Seaplane, AK   Class III
    702730.............. Anchorage Intl AP, AK   Class II
    702735.............. Anchorage Merrill Field, AK   Class II
    702740.............. Palmer Municipal, AK   Class II
    702746.............. Birchwood, AK   Class III
    702750.............. Valdez Wso, AK   Class II
    702756.............. Valdez Pioneer Fiel, AK   Class II
    702757.............. Whittier, AK   Class II
    702770.............. Seward, AK   Class II
    702910.............. Northway Airport, AK   Class II
    702960.............. Cordova, AK   Class II
    702986.............. Big River Lake, AK   Class II
    703080.............. St Paul Island Arpt, AK   Class II
    703160.............. Cold Bay Arpt, AK   Class II
    703165.............. Sand Point, AK   Class II
    703210.............. Dillingham (AMOS), AK   Class II
    703260.............. King Salmon Arpt, AK   Class II
    703330.............. Port Heiden, AK   Class II
    703400.............. Iliamna Arpt, AK   Class II
    703407.............. Sleetmute, AK   Class III
    703410.............. Homer Arpt, AK   Class II
    703430.............. Middleton Island Aut, AK   Class II
    703500.............. Kodiak Airport, AK   Class II
    703606.............. Togiac Village AWOS, AK   Class II
    703610.............. Yakutat State Arpt, AK   Class II
    703620.............. Skagway Airport, AK   Class II
    703670.............. Gustavus, AK   Class II
    703710.............. Sitka Japonski AP, AK   Class II
    703810.............. Juneau Int'l Arpt, AK   Class II
    703855.............. Kake Seaplane Base, AK   Class II
    703860.............. Petersburg, AK   Class II
    703870.............. Wrangell, AK   Class II
    703884.............. Hydaburg Seaplane, AK   Class III
    703950.............. Ketchikan Intl AP, AK   Class II
    703980.............. Annette Island AP, AK   Class II
    704140.............. Shemya AFB, AK   Class II
    704540.............. Adak NAS, AK   Class III
    704890.............. Dutch Harbor, AK   Class II
    722010.............. Key West Intl Arpt, FL   Class I
    722015.............. Key West NAS, FL   Class II
    722016.............. Marathon Airport, FL   Class II
    722020.............. Miami Intl AP, FL   Class I
    722024.............. Miami/Opa Locka, FL   Class II
    722025.............. Fort Lauderdale Hollywood Int, FL   Class II
    722026.............. Homestead AFB, FL   Class II
    722029.............. Miami/Kendall-Tamia, FL   Class II
    722030.............. West Palm Beach Intl Arpt, FL   Class II
    722038.............. Naples Municipal, FL   Class III
    722039.............. Fort Lauderdale, FL   Class II
    722040.............. Melbourne Regional AP, FL   Class II
    722045.............. Vero Beach Municipal Arpt, FL   Class I
    722050.............. Orlando Intl Arpt, FL   Class I
    722053.............. Orlando Executive AP, FL   Class II
    722055.............. Ocala Muni (AWOS), FL   Class II
    722056.............. Daytona Beach Intl AP, FL   Class I
    722057.............. Orlando Sanford Airport, FL   Class II
    722060.............. Jacksonville Intl Arpt, FL   Class I
    722065.............. Jacksonville NAS, FL   Class II
    722066.............. Mayport NS, FL   Class II
    722068.............. Jacksonville/Craig, FL   Class II
    722070.............. Savannah Intl AP, GA   Class I
    722080.............. Charleston Intl Arpt, SC   Class I
    722085.............. Beaufort MCAS, SC   Class II
    722103.............. St Lucie Co Intl, FL   Class II
    722104.............. St Petersburg Albert Whitted, FL   Class II
    722106.............. Fort Myers Page Field, FL   Class I
    722108.............. Southwest Florida I, FL   Class II
    722110.............. Tampa International AP, FL   Class I
    722115.............. Sarasota Bradenton, FL   Class II
    722116.............. St Petersburg Clear, FL   Class II
    722119.............. Lakeland Linder Rgn, FL   Class II
    722135.............. Alma Bacon County AP, GA   Class II
    722136.............. Brunswick Golden Is, GA   Class II
    722137.............. Brunswick Malcolm McKinnon AP, GA   Class II
    722140.............. Tallahassee Regional AP [ISIS], FL   Class I
    722146.............. Gainesville Regional AP, FL   Class I
    722160.............. Albany Dougherty County AP, GA   Class II
    722166.............. Valdosta WB Airport, GA   Class II
    722170.............. Macon Middle Ga Regional AP, GA   Class I
    722175.............. Warner Robins AFB, GA   Class II
    722180.............. Augusta Bush Field, GA   Class I
    722190.............. Atlanta Hartsfield Intl AP, GA   Class I
    722195.............. Fulton Co Arpt Brow, GA   Class II
    722196.............. Dekalb Peachtree, GA   Class II
    722210.............. Valparaiso Elgin AFB, FL   Class II
    722215.............. Crestview Bob Sikes AP, FL   Class II
    722223.............. Pensacola Regional AP, FL   Class I
    722225.............. Pensacola Forest Sherman NAS, FL   Class II
    722226.............. Whiting Field NAAS, FL   Class II
    722230.............. Mobile Regional AP, AL   Class I
    722235.............. Mobile Downtown AP, AL   Class II
    722245.............. Panama City Bay Co, FL   Class II
    722250.............. Fort Benning Lawson, GA   Class II
    722255.............. Columbus Metropolitan Arpt, GA   Class I
    722260.............. Montgomery Dannelly Field, AL   Class I
    722265.............. Maxwell AFB, AL   Class II
    722267.............. Troy AF, AL   Class II
    722268.............. Dothan Municipal AP, AL   Class II
    722269.............. Cairns Field Fort Rucker, AL   Class II
    722270.............. Marietta Dobbins AFB, GA   Class II
    722280.............. Birmingham Municipal AP, AL   Class I
    722284.............. Auburn-Opelika Apt, AL   Class III
    722285.............. Gadsen Muni (AWOS), AL   Class III
    722286.............. Tuscaloosa Municipal AP, AL   Class II
    722287.............. Anniston Metropolitan AP, AL   Class II
    722310.............. New Orleans Intl Arpt, LA   Class I
    722314.............. New Iberia NAAS, LA   Class II
    722315.............. New Orleans Lakefront AP, LA   Class II
    722316.............. New Orleans Alvin Callender F, LA   Class II
    722317.............. Baton Rouge Ryan Arpt, LA   Class II
    722329.............. Patterson Memorial, LA   Class II
    722340.............. Meridian Key Field, MS   Class I
    722345.............. Meridian NAAS, MS   Class II
    722348.............. Hattiesburg Laurel, MS   Class II
    722350.............. Jackson International AP, MS   Class I
    722356.............. Greenville Municipal, MS   Class II
    722357.............. Natchez/Hardy (AWOS), MS   Class II
    722358.............. McComb Pike County AP, MS   Class II
    722359.............. Greenwood Leflore Arpt, MS   Class II
    722390.............. Fort Polk AAF, LA   Class II
    722400.............. Lake Charles Regional Arpt, LA   Class I
    722404.............. Lake Charles WB Airp, LA   Class III
    722405.............. Lafayette Regional AP, LA   Class II
    722406.............. Houma-Terrebonne, LA   Class II
    722410.............. Port Arthur Jefferson County, TX   Class I
    722420.............. Galveston/Scholes, TX   Class II
    722429.............. Houston/D.W. Hooks, TX   Class II
    722430.............. Houston Bush Intercontinental, TX   Class I
    722435.............. Houston William P Hobby AP, TX   Class II
    722436.............. Houston Ellington AFB, [Clear Lake - UT], TX   Class II
    722445.............. College Station Easterwood Fl, TX   Class II
    722446.............. Lufkin Angelina Co, TX   Class I
    722448.............. Tyler/Pounds Fld, TX   Class II
    722470.............. Longview Gregg County AP [Overton - UT], TX   Class II
    722480.............. Shreveport Regional Arpt, LA   Class I
    722484.............. Shreveport Downtown, LA   Class III
    722485.............. Barksdale AFB, LA   Class II
    722486.............. Monroe Regional AP, LA   Class II
    722487.............. Alexandria Esler Regional AP, LA   Class II
    722499.............. Nacogdoches (AWOS), TX   Class III
    722500.............. Brownsville S Padre Isl Intl, TX   Class I
    722505.............. Harlingen Rio Grande Valley I, TX   Class II
    722506.............. McAllen Miller Intl AP [Edinburg - UT], TX   Class II
    722510.............. Corpus Christi Intl Arpt [UT], TX   Class I
    722515.............. Corpus Christi NAS, TX   Class II
    722516.............. Kingsville, TX   Class II
    722517.............. Alice Intl AP, TX   Class II
    722520.............. Laredo Intl AP [UT], TX   Class II
    722523.............. San Antonio/Stinson, TX   Class III
    722524.............. Rockport/Aransas Co, TX   Class II
    722526.............. Cotulla FAA AP, TX   Class II
    722530.............. San Antonio Intl AP, TX   Class I
    722533.............. Hondo Municipal AP, TX   Class III
    722535.............. San Antonio Kelly Field AFB, TX   Class II
    722536.............. Randolph AFB, TX   Class II
    722540.............. Austin Mueller Municipal AP [UT], TX   Class I
    722544.............. Camp Mabry, TX   Class III
    722547.............. Georgetown (AWOS), TX   Class III
    722550.............. Victoria Regional AP, TX   Class I
    722555.............. Palacios Municipal AP, TX   Class II
    722560.............. Waco Regional AP, TX   Class I
    722563.............. McGregor (AWOS), TX   Class III
    722570.............. Fort Hood, TX   Class II
    722575.............. Killeen Muni (AWOS), TX   Class II
    722576.............. Robert Gray AAF, TX   Class II
    722577.............. Draughon Miller Cen, TX   Class II
    722583.............. Dallas Love Field, TX   Class II
    722587.............. Cox Fld, TX   Class III
    722588.............. Greenville/Majors, TX   Class II
    722590.............. Dallas-Fort Worth Intl AP, TX   Class I
    722594.............. Fort Worth Alliance, TX   Class II
    722595.............. Fort Worth NAS, TX   Class II
    722596.............. Fort Worth Meacham, TX   Class II
    722597.............. Mineral Wells Municipal AP, TX   Class II
    722598.............. Dallas/Addison Arpt, TX   Class II
    722599.............. Dallas/Redbird Arpt, TX   Class II
    722610.............. Del Rio [UT], TX   Class II
    722615.............. Del Rio Laughlin AFB, TX   Class II
    722630.............. San Angelo Mathis Field, TX   Class I
    722636.............. Dalhart Municipal AP, TX   Class II
    722640.............. Marfa AP, TX   Class III
    722650.............. Midland International AP, TX   Class I
    722656.............. Wink Winkler County AP, TX   Class II
    722660.............. Abilene Regional AP [UT], TX   Class I
    722670.............. Lubbock International AP, TX   Class I
    722680.............. Roswell Industrial Air Park, NM   Class I
    722683.............. Sierra Blanca Rgnl, NM   Class III
    722686.............. Clovis Cannon AFB, NM   Class II
    722687.............. Carlsbad Cavern City Air Term, NM   Class II
    722689.............. Clovis Muni (AWOS), NM   Class III
    722695.............. Las Cruces Intl, NM   Class II
    722700.............. El Paso International AP [UT], TX   Class I
    722710.............. Truth Or Consequences Muni AP, NM   Class II
    722725.............. Deming Muni, NM   Class II
    722735.............. Douglas Bisbee-Douglas Intl A, AZ   Class II
    722740.............. Tucson International AP, AZ   Class I
    722745.............. Davis Monthan AFB, AZ   Class II
    722747.............. Safford (AMOS), AZ   Class III
    722748.............. Casa Granda (AWOS), AZ   Class III
    722780.............. Phoenix Sky Harbor Intl AP, AZ   Class I
    722784.............. Deer Valley/Phoenix, AZ   Class II
    722785.............. Luke AFB, AZ   Class II
    722789.............. Scottsdale Muni, AZ   Class II
    722800.............. Yuma Intl Arpt, AZ   Class II
    722860.............. March AFB, CA   Class II
    722868.............. Palm Springs Intl, CA   Class II
    722869.............. Riverside Muni, CA   Class II
    722880.............. Burbank-Glendale-Passadena AP, CA   Class II
    722885.............. Santa Monica Muni, CA   Class II
    722886.............. Van Nuys Airport, CA   Class II
    722895.............. Lompoc (AWOS), CA   Class III
    722897.............. San Luis Co Rgnl, CA   Class II
    722899.............. Chino Airport, CA   Class III
    722900.............. San Diego Lindbergh Field, CA   Class I
    722903.............. San Diego/Montgomer, CA   Class II
    722904.............. Chula Vista Brown Field NAAS, CA   Class II
    722906.............. San Diego North Island NAS, CA   Class II
    722926.............. Camp Pendleton MCAS, CA   Class II
    722927.............. Carlsbad/Palomar, CA   Class II
    722930.............. San Diego Miramar NAS, CA   Class II
    722950.............. Los Angeles Intl Arpt, CA   Class I
    722956.............. Jack Northrop Fld H, CA   Class II
    722970.............. Long Beach Daugherty Fld, CA   Class I
    722976.............. Fullerton Municipal, CA   Class II
    722977.............. Santa Ana John Wayne AP, CA   Class II
    723013.............. Wilmington International Arpt, NC   Class I
    723030.............. Fayetteville Pope AFB, NC   Class II
    723035.............. Fayetteville Rgnl G, NC   Class II
    723040.............. Cape Hatteras Nws Bldg, NC   Class I
    723046.............. Dare Co Rgnl, NC   Class II
    723060.............. Raleigh Durham International, NC   Class I
    723065.............. Pitt Greenville Arp, NC   Class III
    723066.............. Goldsboro Seymour Johnson AFB, NC   Class II
    723067.............. Kinston Stallings AFB, NC   Class II
    723068.............. Rocky Mount Wilson, NC   Class II
    723069.............. Jacksonville (AWOS), NC   Class II
    723075.............. Oceana NAS, VA   Class II
    723080.............. Norfolk International AP, VA   Class I
    723083.............. Franklin NAAS, VA   Class III
    723085.............. Norfolk NAS, VA   Class II
    723086.............. Newport News, VA   Class II
    723090.............. Cherry Point MCAS, NC   Class II
    723095.............. New Bern Craven Co Regl AP, NC   Class II
    723096.............. New River MCAF, NC   Class II
    723100.............. Columbia Metro Arpt, SC   Class I
    723106.............. Florence Regional AP, SC   Class II
    723110.............. Athens Ben Epps AP, GA   Class I
    723119.............. Greenville Downtown AP, SC   Class II
    723120.............. Greer Greenv'l-Spartanbrg AP, SC   Class I
    723125.............. Anderson County AP, SC   Class II
    723140.............. Charlotte Douglas Intl Arpt, NC   Class I
    723143.............. Southern Pines AWOS, NC   Class III
    723145.............. Hickory Regional AP, NC   Class II
    723150.............. Asheville Regional Arpt, NC   Class I
    723170.............. Greensboro Piedmont Triad Int, NC   Class I
    723183.............. Bristol Tri City Airport, TN   Class I
    723193.............. Winston-Salem Reynolds AP, NC   Class II
    723200.............. Rome R B Russell AP, GA   Class III
    723230.............. Huntsville Intl/Jones Field, AL   Class I
    723235.............. Muscle Shoals Regional AP, AL   Class II
    723240.............. Chattanooga Lovell Field AP, TN   Class I
    723260.............. Knoxville McGhee Tyson AP, TN   Class I
    723265.............. Crossville Memorial AP, TN   Class II
    723270.............. NAShville International AP, TN   Class I
    723300.............. Poplar Bluff (AMOS), MO   Class III
    723306.............. Columbus AFB, MS   Class II
    723307.............. Golden Tri (AWOS), MS   Class II
    723320.............. Tupelo C D Lemons Arpt, MS   Class II
    723340.............. Memphis International AP, TN   Class I
    723346.............. Jackson McKellar-Sipes Regl A, TN   Class II
    723347.............. Dyersburg Municipal AP, TN   Class II
    723403.............. Little Rock Adams Field, AR   Class I
    723405.............. Little Rock AFB, AR   Class II
    723406.............. Walnut Ridge (AWOS), AR   Class II
    723407.............. Jonesboro Muni, AR   Class II
    723415.............. Memorial Fld, AR   Class II
    723416.............. Stuttgart (AWOS), AR   Class II
    723417.............. Pine Bluff FAA AP, AR   Class II
    723418.............. Texarkana Webb Field, AR   Class II
    723419.............. El Dorado Goodwin Field, AR   Class II
    723434.............. Springdale Muni, AR   Class III
    723440.............. Fort Smith Regional AP, AR   Class II
    723443.............. Siloam Spring (AWOS), AR   Class III
    723444.............. Bentonville (AWOS), AR   Class III
    723445.............. Fayetteville Drake Field, AR   Class II
    723446.............. Harrison FAA AP, AR   Class II
    723447.............. Flippin (AWOS), AR   Class III
    723448.............. Batesville (AWOS), AR   Class III
    723449.............. Rogers (AWOS), AR   Class III
    723489.............. Cape Girardeau Municipal AP, MO   Class II
    723495.............. Joplin Municipal AP, MO   Class II
    723510.............. Wichita Falls Municipal Arpt, TX   Class II
    723520.............. Altus AFB, OK   Class II
    723525.............. Hobart Municipal AP, OK   Class II
    723526.............. Clinton-Sherman, OK   Class III
    723527.............. Gage Airport, OK   Class II
    723530.............. Oklahoma City Will Rogers Wor, OK   Class II
    723535.............. Vance AFB, OK   Class II
    723540.............. Oklahoma City Tinker AFB, OK   Class II
    723544.............. Oklahoma City/Wiley, OK   Class II
    723545.............. Stillwater Rgnl, OK   Class II
    723546.............. Ponca City Municipal AP [SGP - ARM], OK   Class II
    723550.............. Fort Sill Post Field AF, OK   Class II
    723560.............. Tulsa International Airport, OK   Class II
    723565.............. Bartlesville/Philli, OK   Class II
    723566.............. McAlester Municipal AP, OK   Class II
    723575.............. Lawton Municipal, OK   Class II
    723600.............. Clayton Municipal Airpark, NM   Class II
    723604.............. Childress Municipal AP, TX   Class II
    723627.............. Gallup Sen Clarke Fld, NM   Class I
    723630.............. Amarillo International AP [Canyon - UT], TX   Class II
    723650.............. Albuquerque Intl Arpt [ISIS], NM   Class I
    723656.............. Santa Fe County Municipal AP, NM   Class II
    723658.............. Farmington Four Corners Regl, NM   Class II
    723663.............. Taos Muni Apt (AWOS), NM   Class III
    723676.............. Tucumcari FAA AP, NM   Class II
    723677.............. Las Vegas Municipal Arpt, NM   Class II
    723700.............. Kingman (AMOS), AZ   Class II
    723710.............. Page Muni (AMOS), AZ   Class III
    723723.............. Prescott Love Field, AZ   Class I
    723740.............. Winslow Municipal AP, AZ   Class II
    723747.............. Show Low Municipal, AZ   Class II
    723755.............. Flagstaff Pulliam Arpt, AZ   Class III
    723783.............. Grand Canyon Natl P, AZ   Class II
    723805.............. Needles Airport, CA   Class II
    723810.............. Edwards AFB, CA   Class II
    723815.............. Daggett Barstow-Daggett AP, CA   Class I
    723816.............. Lancaster Gen Wm Fox Field, CA   Class II
    723820.............. Palmdale Airport, CA   Class II
    723830.............. Sandberg, CA   Class III
    723840.............. Bakersfield Meadows Field, CA   Class I
    723860.............. Las Vegas McCarran Intl AP, NV   Class I
    723865.............. Nellis AFB, NV   Class II
    723870.............. Mercury Desert Rock AP [SURFRAD], NV   Class I
    723890.............. Fresno Yosemite Intl AP, CA   Class I
    723895.............. Porterville (AWOS), CA   Class III
    723896.............. Visalia Muni (AWOS), CA   Class II
    723910.............. Point Mugu Nf, CA   Class II
    723925.............. Santa Barbara Municipal AP, CA   Class I
    723926.............. Camarillo (AWOS), CA   Class III
    723927.............. Oxnard Airport, CA   Class II
    723940.............. Santa Maria Public Arpt, CA   Class I
    723965.............. Paso Robles Municipal Arpt, CA   Class II
    724010.............. Richmond International AP, VA   Class I
    724014.............. Dinwiddie Co, VA   Class III
    724016.............. Charlottesville FAA, VA   Class II
    724017.............. Farmville, VA   Class III
    724026.............. Melfa/Accomack Arpt, VA   Class III
    724030.............. Washington Dc Dulles Int'l Ar [Sterling - - ISIS], VA   Class I
    724033.............. Shannon Arpt, VA   Class III
    724035.............. Quantico MCAS, VA   Class II
    724036.............. Manassas Muni (AWOS), VA   Class III
    724037.............. Davison AAF, VA   Class II
    724040.............. Patuxent River NAS, MD   Class II
    724045.............. Salisbury Wicomico Co AP, MD   Class II
    724050.............. Washington Dc Reagan AP, VA   Class I
    724053.............. Winchester Rgnl, VA   Class III
    724055.............. Leesburg/Godfrey, VA   Class III
    724056.............. Marion / Wytheville, VA   Class III
    724058.............. Abingdon, VA   Class III
    724060.............. Baltimore Blt-Washngtn Int'l, MD   Class I
    724066.............. Hagerstown Rgnl Ric, MD   Class II
    724070.............. Atlantic City Intl AP, NJ   Class I
    724075.............. Millville Municipal AP, NJ   Class II
    724080.............. Philadelphia International AP, PA   Class I
    724084.............. Belmar Asc, NJ   Class III
    724085.............. Philadelphia Ne Philadelphia, PA   Class II
    724086.............. Willow Grove NAS, PA   Class II
    724088.............. Dover AFB, DE   Class II
    724089.............. Wilmington New Castle Cnty AP, DE   Class I
    724094.............. Caldwell/Essex Co., NJ   Class II
    724095.............. Trenton Mercer County AP, NJ   Class II
    724096.............. McGuire AFB, NJ   Class II
    724100.............. Lynchburg Regional Arpt, VA   Class I
    724105.............. Staunton/Shenandoah, VA   Class II
    724106.............. Danville FAA AP, VA   Class II
    724107.............. Hillsville, VA   Class III
    724110.............. Roanoke Regional AP, VA   Class I
    724113.............. Virginia Tech Arpt, VA   Class III
    724115.............. Hot Springs/Ingalls, VA   Class III
    724116.............. Pulaski, VA   Class III
    724117.............. Wise/Lonesome Pine, VA   Class III
    724120.............. Beckley Raleigh Co Mem AP, WV   Class I
    724125.............. Bluefield/Mercer Co [NREL], WV   Class II
    724127.............. Lewisburg/Greenbrie, WV   Class II
    724140.............. Charleston Yeager Arpt, WV   Class I
    724170.............. Elkins Elkins-Randolph Co Arp, WV   Class I
    724175.............. Harrison Marion Rgn, WV   Class II
    724176.............. Morgantown Hart Field, WV   Class II
    724177.............. Martinsburg Eastern Wv Reg AP, WV   Class II
    724210.............. Cincinnati Northern Ky AP, KY   Class I
    724220.............. Lexington Bluegrass AP, KY   Class I
    724230.............. Louisville Standiford Field, KY   Class I
    724235.............. Louisville Bowman Field, KY   Class II
    724236.............. Jackson Julian Carroll AP, KY   Class I
    724238.............. Henderson City, KY   Class III
    724240.............. Fort Knox Godman AAF, KY   Class II
    724243.............. London-Corbin AP, KY   Class II
    724250.............. Huntington Tri-State Arpt, WV   Class I
    724273.............. Parkersburg Wood County AP, WV   Class II
    724275.............. Wheeling Ohio County AP, WV   Class II
    724280.............. Columbus Port Columbus Intl A, OH   Class I
    724286.............. Zanesville Municipal AP, OH   Class II
    724288.............. Ohio State Universi, OH   Class II
    724290.............. Dayton International Airport, OH   Class I
    724297.............. Cincinnati Municipal AP Lunki, OH   Class II
    724320.............. Evansville Regional AP, IN   Class I
    724335.............. Mount Vernon (AWOS), IL   Class II
    724336.............. Southern Illinois, IL   Class II
    724338.............. Belleville Scott AFB, IL   Class II
    724339.............. Marion Regional, IL   Class II
    724340.............. St Louis Lambert Int'l Arpt, MO   Class I
    724345.............. St Louis Spirit Of St Louis A, MO   Class I
    724350.............. Paducah Barkley Regional AP, KY   Class I
    724354.............. Somerset (AWOS), KY   Class III
    724365.............. Huntingburg, IN   Class III
    724373.............. Terre Haute Hulman Regional A, IN   Class II
    724375.............. Monroe Co, IN   Class II
    724380.............. Indianapolis Intl AP, IN   Class I
    724386.............. Lafayette Purdue Univ AP, IN   Class II
    724390.............. Springfield Capital AP, IL   Class I
    724396.............. Quincy Muni Baldwin Fld, IL   Class II
    724397.............. Central Illinois Rg, IL   Class II
    724400.............. Springfield Regional Arpt, MO   Class I
    724450.............. Columbia Regional Airport, MO   Class I
    724454.............. Farmington, MO   Class III
    724455.............. Kirksville Regional AP, MO   Class II
    724456.............. Vichy Rolla Natl Arpt, MO   Class II
    724457.............. Ft Lnrd Wd AAF, MO   Class II
    724458.............. Jefferson City Mem, MO   Class II
    724459.............. Kaiser Mem (AWOS), MO   Class II
    724460.............. Kansas City Int'l Arpt, MO   Class I
    724463.............. Kansas City Downtown AP, MO   Class II
    724467.............. Whiteman AFB, MO   Class II
    724468.............. Olathe/Johnson Co., KS   Class II
    724475.............. Olathe Johnson Co Industrial, KS   Class II
    724490.............. St Joseph Rosecrans Memorial, MO   Class II
    724500.............. Wichita Mid-Continent AP, KS   Class II
    724504.............. Wichita/Col. Jabara, KS   Class III
    724505.............. McConnell AFB, KS   Class II
    724506.............. Hutchinson Municipal AP, KS   Class II
    724507.............. Chanute Martin Johnson AP, KS   Class II
    724509.............. Newton (AWOS), KS   Class III
    724510.............. Dodge City Regional AP, KS   Class II
    724515.............. Garden City Municipal AP, KS   Class II
    724516.............. Liberal Muni, KS   Class II
    724517.............. Great Bend (AWOS), KS   Class II
    724518.............. Hays Muni (AWOS), KS   Class II
    724550.............. Fort Riley Marshall AAF, KS   Class III
    724555.............. Manhattan Rgnl, KS   Class II
    724556.............. Emporia Municipal AP, KS   Class II
    724560.............. Topeka Municipal AP, KS   Class II
    724565.............. Topeka Forbes Field, KS   Class II
    724580.............. Concordia Blosser Muni AP, KS   Class II
    724585.............. Russell Municipal AP, KS   Class I
    724586.............. Salina Municipal AP, KS   Class II
    724620.............. Alamosa San Luis Valley Rgnl, CO   Class II
    724625.............. Durango/La Plata Co, CO   Class II
    724635.............. La Junta Municipal AP, CO   Class II
    724636.............. Lamar Municipal, CO   Class III
    724640.............. Pueblo Memorial AP, CO   Class II
    724645.............. Trinidad Las Animas County AP, CO   Class II
    724650.............. Goodland Renner Field, KS   Class II
    724655.............. Hill City Municipal AP, KS   Class II
    724660.............. Colorado Springs Muni AP, CO   Class II
    724665.............. Limon, CO   Class I
    724666.............. Denver/Centennial [Golden - NREL], CO   Class II
    724673.............. Leadville/Lake Co., CO   Class II
    724675.............. Eagle County AP, CO   Class II
    724676.............. Aspen Pitkin Co Sar, CO   Class II
    724677.............. Gunnison Co. (AWOS), CO   Class II
    724695.............. Aurora Buckley Field ANGB, CO   Class II
    724698.............. Akron Washington Co AP, CO   Class II
    724699.............. Broomfield/Jeffco [Boulder - SURFRAD], CO   Class III
    724723.............. Blanding, UT   Class II
    724735.............. Hanksville, UT   Class II
    724754.............. Saint George (AWOS), UT   Class II
    724755.............. Cedar City Municipal AP, UT   Class I
    724756.............. Bryce Cnyn FAA AP, UT   Class II
    724760.............. Grand Junction Walker Field, CO   Class I
    724765.............. Montrose Co. Arpt, CO   Class II
    724767.............. Cortez/Montezuma Co, CO   Class II
    724768.............. Greeley/Weld (AWOS), CO   Class II
    724769.............. Fort Collins (AWOS), CO   Class II
    724776.............. Moab/Canyonlands [UO], UT   Class III
    724795.............. Delta, UT   Class II
    724800.............. Bishop Airport, CA   Class II
    724815.............. Merced/Macready Fld, CA   Class II
    724830.............. Sacramento Executive Arpt, CA   Class I
    724837.............. Beale AFB, CA   Class II
    724838.............. Yuba Co, CA   Class II
    724839.............. Sacramento Metropolitan AP, CA   Class II
    724855.............. Tonopah Airport, NV   Class I
    724860.............. Ely Yelland Field, NV   Class I
    724880.............. Reno Tahoe International AP, NV   Class I
    724885.............. Fallon NAAS, NV   Class II
    724915.............. Monterey NAF, CA   Class II
    724917.............. Salinas Municipal AP, CA   Class II
    724920.............. Stockton Metropolitan Arpt, CA   Class II
    724926.............. Modesto City-County AP, CA   Class II
    724927.............. Livermore Municipal, CA   Class II
    724930.............. Oakland Metropolitan Arpt, CA   Class II
    724935.............. Hayward Air Term, CA   Class II
    724936.............. Concord Concord-Buchanan Fiel, CA   Class II
    724940.............. San Francisco Intl AP, CA   Class I
    724945.............. San Jose Intl AP, CA   Class II
    724955.............. Napa Co. Airport, CA   Class II
    724957.............. Santa Rosa (AWOS), CA   Class II
    725020.............. Newark International Arpt, NJ   Class I
    725025.............. Teterboro Airport, NJ   Class II
    725029.............. Oxford (AWOS), CT   Class III
    725030.............. New York Laguardia Arpt, NY   Class I
    725033.............. New York Central Prk Obs Belv, NY   Class III
    725035.............. Islip Long Isl MacArthur AP, NY   Class I
    725036.............. Poughkeepsie Dutchess Co AP, NY   Class II
    725037.............. White Plains Westchester Co A, NY   Class II
    725038.............. Stewart Field, NY   Class II
    725040.............. Bridgeport Sikorsky Memorial, CT   Class I
    725045.............. New Haven Tweed Airport, CT   Class II
    725046.............. Groton New London AP, CT   Class II
    725054.............. Pawtucket (AWOS), RI   Class II
    725058.............. Block Island State Arpt, RI   Class II
    725060.............. Otis ANGB, MA   Class II
    725063.............. Nantucket Memorial AP, MA   Class II
    725064.............. Plymouth Municipal, MA   Class III
    725065.............. New Bedford Rgnl, MA   Class II
    725066.............. Marthas Vineyard, MA   Class II
    725067.............. Barnstable Muni Boa, MA   Class II
    725070.............. Providence T F Green State Ar, RI   Class I
    725073.............. Provincetown (AWOS), MA   Class II
    725075.............. North Adams, MA   Class III
    725080.............. Hartford Bradley Intl AP, CT   Class I
    725086.............. Danbury Municipal, CT   Class II
    725087.............. Hartford Brainard Fd, CT   Class II
    725088.............. Beverly Muni, MA   Class II
    725090.............. Boston Logan Int'l Arpt, MA   Class I
    725095.............. Worchester Regional Arpt, MA   Class I
    725098.............. Norwood Memorial, MA   Class II
    725103.............. Reading Spaatz Field, PA   Class II
    725115.............. Middletown Harrisburg Intl AP, PA   Class I
    725116.............. Lancaster, PA   Class II
    725117.............. Washington (AWOS), PA   Class II
    725118.............. Harrisburg Capital City Arpt, PA   Class II
    725124.............. Butler Co. (AWOS), PA   Class III
    725125.............. Dubois FAA AP, PA   Class II
    725126.............. Altoona Blair Co Arpt, PA   Class II
    725127.............. Johnstown Cambria County AP, PA   Class II
    725128.............. State College [Penn State - SURFRAD], PA   Class II
    725130.............. Wilkes-Barre Scranton Intl AP, PA   Class I
    725140.............. Williamsport Regional AP, PA   Class I
    725145.............. Monticello (AWOS), NY   Class II
    725150.............. Binghamton Edwin A Link Field, NY   Class I
    725156.............. Elmira Corning Regional AP, NY   Class II
    725165.............. Rutland State, VT   Class II
    725170.............. Allentown Lehigh Valley Intl, PA   Class I
    725180.............. Albany County AP, NY   Class I
    725185.............. Glens Falls AP, NY   Class II
    725190.............. Syracuse Hancock Int'l Arpt, NY   Class I
    725197.............. Utica Oneida County AP, NY   Class II
    725200.............. Pittsburgh International AP, PA   Class I
    725205.............. Pittsburgh Allegheny Co AP, PA   Class II
    725210.............. Akron Akron-Canton Reg AP, OH   Class I
    725235.............. Jamestown (AWOS), NY   Class II
    725240.............. Cleveland Hopkins Intl AP, OH   Class I
    725245.............. Burke Lakefront, OH   Class II
    725246.............. Mansfield Lahm Municipal Arpt, OH   Class I
    725250.............. Youngstown Regional Airport, OH   Class I
    725260.............. Erie International AP, PA   Class I
    725266.............. Bradford Regional AP, PA   Class I
    725267.............. Franklin, PA   Class II
    725280.............. Buffalo Niagara Intl AP, NY   Class I
    725287.............. Niagara Falls AF, NY   Class II
    725290.............. Rochester Greater Rochester I, NY   Class I
    725300.............. Chicago Ohare Intl AP, IL   Class I
    725305.............. W. Chicago/Du Page, IL   Class II
    725314.............. Cahokia/St. Louis, IL   Class II
    725315.............. Univ Of Illinois Wi [Bondville - SURFRAD], IL   Class II
    725316.............. Decatur, IL   Class II
    725320.............. Peoria Greater Peoria AP, IL   Class I
    725326.............. Sterling Rockfalls, IL   Class II
    725330.............. Fort Wayne Intl AP, IN   Class I
    725335.............. Grissom Arb, IN   Class II
    725336.............. Delaware Co Johnson, IN   Class II
    725340.............. Chicago Midway AP, IL   Class II
    725347.............. Chicago/Waukegan, IL   Class II
    725350.............. South Bend Michiana Rgnl AP, IN   Class I
    725360.............. Toledo Express Airport, OH   Class I
    725366.............. Findlay Airport, OH   Class II
    725370.............. Detroit Metropolitan Arpt, MI   Class I
    725374.............. Ann Arbor Municipal, MI   Class II
    725375.............. Detroit City Airport, MI   Class I
    725376.............. Detroit Willow Run AP, MI   Class II
    725377.............. Mount Clemens Selfridge Fld, MI   Class II
    725378.............. Howell, MI   Class III
    725384.............. St.Clair County Int, MI   Class III
    725390.............. Lansing Capital City Arpt, MI   Class I
    725395.............. Jackson Reynolds Field, MI   Class II
    725396.............. Battle Creek Kellogg AP, MI   Class II
    725430.............. Rockford Greater Rockford AP, IL   Class I
    725440.............. Moline Quad City Intl AP, IL   Class I
    725450.............. Cedar Rapids Municipal AP, IA   Class II
    725453.............. Atlantic, IA   Class III
    725454.............. Washington, IA   Class III
    725455.............. Burlington Municipal AP, IA   Class II
    725456.............. Keokuk Muni, IA   Class III
    725457.............. Algona, IA   Class III
    725460.............. Des Moines Intl AP, IA   Class I
    725463.............. Charles City, IA   Class III
    725464.............. Newton Muni, IA   Class III
    725465.............. Ottumwa Industrial AP, IA   Class II
    725467.............. Shenandoah Muni, IA   Class III
    725468.............. Carroll, IA   Class III
    725469.............. Chariton, IA   Class III
    725470.............. Dubuque Regional AP, IA   Class I
    725473.............. Clinton Muni (AWOS), IA   Class II
    725474.............. Creston, IA   Class III
    725475.............. Monticello Muni, IA   Class III
    725476.............. Decorah, IA   Class III
    725477.............. Denison, IA   Class III
    725478.............. Webster City, IA   Class III
    725479.............. Clarinda, IA   Class III
    725480.............. Waterloo Municipal AP, IA   Class I
    725483.............. Fort Madison, IA   Class III
    725484.............. Le Mars, IA   Class III
    725485.............. Mason City Municipal Arpt, IA   Class I
    725486.............. Boone Muni, IA   Class III
    725487.............. Muscatine, IA   Class III
    725488.............. Oelwen, IA   Class III
    725489.............. Orange City, IA   Class III
    725490.............. Fort Dodge (AWOS), IA   Class II
    725493.............. Knoxville, IA   Class III
    725494.............. Red Oak, IA   Class III
    725495.............. Sheldon, IA   Class III
    725496.............. Storm Lake, IA   Class III
    725497.............. Council Bluffs, IA   Class III
    725500.............. Omaha Eppley Airfield, NE   Class I
    725510.............. Lincoln Municipal Arpt, NE   Class II
    725515.............. Beatrice Municipal, NE   Class III
    725520.............. Grand Island Central Ne Regio, NE   Class II
    725524.............. Ord/Sharp Field, NE   Class II
    725525.............. Hastings Municipal, NE   Class II
    725526.............. Kearney Muni (AWOS), NE   Class II
    725527.............. Tekamah (ASOS), NE   Class III
    725530.............. Omaha Wsfo, NE   Class III
    725533.............. Falls City/Brenner, NE   Class II
    725540.............. Bellevue Offutt AFB, NE   Class II
    725555.............. Broken Bow Muni, NE   Class II
    725556.............. Ainsworth Municipal, NE   Class II
    725560.............. Norfolk Karl Stefan Mem Arpt, NE   Class I
    725564.............. Fremont Muni Arpt, NE   Class III
    725565.............. Columbus Muni, NE   Class II
    725566.............. O`Neill/Baker Field, NE   Class III
    725570.............. Sioux City Sioux Gateway AP, IA   Class I
    725610.............. Sidney Municipal AP, NE   Class II
    725620.............. North Platte Regional AP, NE   Class I
    725625.............. McCook Municipal, NE   Class II
    725626.............. Imperial FAA AP, NE   Class II
    725628.............. Brewster Field Arpt, NE   Class III
    725635.............. Alliance Municipal, NE   Class II
    725636.............. Chadron Municipal AP, NE   Class II
    725640.............. Cheyenne Municipal Arpt, WY   Class I
    725645.............. Laramie General Brees Field, WY   Class II
    725650.............. Denver Intl AP, CO   Class I
    725660.............. Scottsbluff W B Heilig Field, NE   Class I
    725670.............. Valentine Miller Field, NE   Class I
    725690.............. Casper Natrona Co Intl AP, WY   Class I
    725700.............. Craig-Moffat, CO   Class II
    725705.............. Vernal, UT   Class II
    725715.............. Hayden/Yampa (AWOS), CO   Class II
    725717.............. Rifle/Garfield Rgnl, CO   Class III
    725720.............. Salt Lake City Int'l Arpt [ISIS], UT   Class I
    725724.............. Provo Muni (AWOS), UT   Class II
    725744.............. Rock Springs Arpt [Green River - UO], WY   Class I
    725745.............. Rawlins Municipal AP, WY   Class II
    725750.............. Ogden Hinkley Airport, UT   Class II
    725755.............. Ogden Hill AFB, UT   Class II
    725760.............. Lander Hunt Field, WY   Class I
    725765.............. Riverton Municipl AP, WY   Class II
    725775.............. Evanston/Burns Fld, WY   Class II
    725776.............. Jackson Hole, WY   Class II
    725780.............. Pocatello Regional AP, ID   Class I
    725785.............. Idaho Falls Fanning Field, ID   Class II
    725786.............. Malad City, ID   Class II
    725805.............. Lovelock Derby Field, NV   Class I
    725810.............. Wendover Usaf Auxiliary Field, UT   Class II
    725825.............. Elko Municipal Arpt, NV   Class II
    725830.............. Winnemucca Municipal Arpt, NV   Class I
    725845.............. Blue Canyon AP, CA   Class II
    725846.............. Truckee-Tahoe, CA   Class II
    725847.............. South Lake Tahoe, CA   Class II
    725865.............. Hailey/Friedman Mem, ID   Class II
    725866.............. Joslin Fld Magic Va [Twin Falls - UO], ID   Class II
    725867.............. Burley Municipal Arpt, ID   Class II
    725868.............. Soda Springs/Tigert, ID   Class II
    725895.............. Klamath Falls Intl AP [UO], OR   Class II
    725905.............. Ukiah Municipal AP, CA   Class II
    725910.............. Red Bluff Municipal Arpt, CA   Class II
    725920.............. Redding Municipal Arpt, CA   Class I
    725945.............. Arcata Airport, CA   Class I
    725946.............. Crescent City FAA Ai, CA   Class II
    725955.............. Montague Siskiyou County AP, CA   Class II
    725958.............. Alturas, CA   Class II
    725970.............. Medford Rogue Valley Intl AP [Ashland - UO], OR   Class I
    725975.............. Sexton Summit, OR   Class II
    725976.............. Lakeview (AWOS), OR   Class II
    726050.............. Concord Municipal Arpt, NH   Class I
    726055.............. Pease Intl Tradepor, NH   Class II
    726060.............. Portland Intl Jetport, ME   Class I
    726064.............. Sanford Muni (AWOS), ME   Class II
    726073.............. Waterville (AWOS), ME   Class II
    726077.............. Bar Harbor (AWOS), ME   Class II
    726079.............. Rockland/Knox (AWOS), ME   Class II
    726083.............. Northern Aroostook, ME   Class III
    726088.............. Bangor International AP, ME   Class I
    726115.............. Springfield/Hartnes, VT   Class III
    726116.............. Lebanon Municipal, NH   Class II
    726130.............. Mount Washington, NH   Class II
    726145.............. Montpelier AP, VT   Class II
    726155.............. Laconia Muni (AWOS), NH   Class II
    726160.............. Berlin Municipal, NH   Class III
    726165.............. Dillant Hopkins, NH   Class II
    726170.............. Burlington International AP, VT   Class I
    726184.............. Auburn-Lewiston, ME   Class II
    726185.............. Augusta Airport, ME   Class II
    726196.............. Millinocket Municipal AP, ME   Class III
    726223.............. Massena AP, NY   Class I
    726227.............. Watertown AP, NY   Class II
    726228.............. Adirondack Rgnl, NY   Class II
    726350.............. Grand Rapids Kent County Int', MI   Class I
    726355.............. Benton Harbor/Ross, MI   Class II
    726357.............. Kalamazoo Battle Cr, MI   Class II
    726360.............. Muskegon County Arpt, MI   Class I
    726370.............. Flint Bishop Intl Arpt, MI   Class I
    726375.............. Oakland Co Intl, MI   Class II
    726379.............. Saginaw Tri City Intl AP, MI   Class II
    726380.............. Houghton Lake Roscommon Co Ar, MI   Class I
    726384.............. Cadillac Wexford Co AP, MI   Class III
    726385.............. Manistee (AWOS), MI   Class II
    726387.............. Traverse City Cherry Capital, MI   Class I
    726390.............. Alpena County Regional AP, MI   Class I
    726395.............. Oscoda Wurtsmith AFB, MI   Class III
    726400.............. Milwaukee Mitchell Intl AP, WI   Class I
    726404.............. Minocqua/Woodruff, WI   Class II
    726410.............. Madison Dane Co Regional Arpt [ISIS], WI   Class I
    726415.............. Janesville/Rock Co., WI   Class III
    726416.............. Lone Rock FAA AP, WI   Class II
    726430.............. La Crosse Municipal Arpt, WI   Class I
    726435.............. Eau Claire County AP, WI   Class I
    726440.............. Rochester International Arpt, MN   Class I
    726450.............. Green Bay Austin Straubel Int, WI   Class I
    726455.............. Manitowac Muni AWOS, WI   Class II
    726456.............. Wittman Rgnl, WI   Class II
    726457.............. Appleton/Outagamie, WI   Class II
    726458.............. Sturgeon Bay, WI   Class II
    726463.............. Wausau Municipal Arpt, WI   Class II
    726464.............. Watertown, WI   Class III
    726465.............. Mosinee/Central Wi, WI   Class II
    726467.............. Rice Lake Municipal, WI   Class III
    726468.............. Phillips/Price Co., WI   Class II
    726480.............. Escanaba (AWOS), MI   Class II
    726487.............. Menominee (AWOS), MI   Class II
    726498.............. Fair Field, IA   Class III
    726499.............. Estherville Muni, IA   Class II
    726500.............. Spencer, IA   Class II
    726510.............. Sioux Falls Foss Field, SD   Class I
    726515.............. Brookings (AWOS), SD   Class II
    726525.............. Chan Gurney Muni, SD   Class II
    726540.............. Huron Regional Arpt, SD   Class I
    726544.............. Orr, MN   Class III
    726545.............. Mitchell (AWOS), SD   Class II
    726546.............. Watertown Municipal AP, SD   Class II
    726547.............. Glenwood ( ), MN   Class III
    726550.............. St Cloud Regional Arpt, MN   Class I
    726555.............. Brainerd/Wieland, MN   Class II
    726556.............. Redwood Falls Muni, MN   Class II
    726557.............. Alexandria Municipal AP, MN   Class II
    726558.............. Cloquet (AWOS), MN   Class III
    726559.............. Marshall/Ryan (AWOS), MN   Class II
    726560.............. Fergus Falls (AWOS), MN   Class II
    726563.............. Faribault Muni AWOS, MN   Class II
    726564.............. Red Wing, MN   Class II
    726565.............. Morris Muni (AWOS), MN   Class II
    726566.............. Pipestone (AWOS), MN   Class II
    726567.............. New Ulm Muni (AWOS), MN   Class II
    726568.............. Owatonna (AWOS), MN   Class III
    726569.............. Hutchinson (AWOS), MN   Class III
    726574.............. Marshfield Muni, WI   Class III
    726575.............. Minneapolis/Crystal, MN   Class II
    726576.............. Willmar, MN   Class II
    726578.............. Little Falls (AWOS), MN   Class III
    726579.............. Flying Cloud, MN   Class II
    726580.............. Minneapolis-St Paul Int'l Arp, MN   Class I
    726583.............. Litchfield Muni, MN   Class II
    726584.............. St Paul Downtown AP, MN   Class II
    726585.............. Mankato (AWOS), MN   Class II
    726586.............. Fairmont Muni (AWOS), MN   Class II
    726587.............. Worthington (AWOS), MN   Class II
    726588.............. Winona Muni (AWOS), MN   Class II
    726589.............. Albert Lea (AWOS), MN   Class II
    726590.............. Aberdeen Regional Arpt, SD   Class I
    726603.............. South St Paul Muni, MN   Class III
    726620.............. Rapid City Regional Arpt, SD   Class I
    726625.............. Ellsworth AFB, SD   Class II
    726626.............. Antigo/Lang (AWOS), WI   Class III
    726650.............. Gillette/Gillette-C, WY   Class II
    726660.............. Sheridan County Arpt, WY   Class I
    726665.............. Worland Municipal, WY   Class II
    726676.............. Glendive (AWOS), MT   Class II
    726685.............. Mobridge, SD   Class II
    726686.............. Pierre Municipal AP, SD   Class I
    726700.............. Cody Muni (AWOS), WY   Class II
    726770.............. Billings Logan Int'l Arpt, MT   Class I
    726776.............. Lewistown Municipal Arpt, MT   Class II
    726785.............. Butte Bert Mooney Arpt, MT   Class II
    726797.............. Bozeman Gallatin Field, MT   Class II
    726798.............. Livingston Mission Field, MT   Class II
    726810.............. Boise Air Terminal [UO], ID   Class I
    726813.............. Caldwell (AWOS), ID   Class III
    726815.............. Mountain Home AFB,, ID   Class II
    726830.............. Burns Municipal Arpt [UO], OR   Class II
    726835.............. Redmond Roberts Field, OR   Class II
    726865.............. Salmon/Lemhi (AWOS), ID   Class III
    726880.............. Pendleton E Or Regional AP, OR   Class I
    726884.............. La Grande Muni AP, OR   Class II
    726886.............. Baker Municipal AP, OR   Class II
    726904.............. Roseburg Regional AP, OR   Class II
    726917.............. North Bend Muni Airport, OR   Class II
    726930.............. Eugene Mahlon Sweet Arpt [UO], OR   Class I
    726940.............. Salem McNary Field, OR   Class I
    726945.............. Corvallis Muni, OR   Class II
    726959.............. Aurora State, OR   Class III
    726980.............. Portland International AP, OR   Class I
    726985.............. Portland/Troutdale, OR   Class II
    726986.............. Portland/Hillsboro, OR   Class II
    726988.............. The Dalles Municipal Arpt, WA   Class II
    727033.............. Houlton Intl Arpt, ME   Class II
    727120.............. Caribou Municipal Arpt, ME   Class I
    727130.............. Presque Isle Municip, ME   Class II
    727135.............. Wiscasset, ME   Class III
    727340.............. Sault Ste Marie Sanderson Fie, MI   Class I
    727344.............. Chippewa Co Intl, MI   Class II
    727347.............. Pellston Emmet County AP, MI   Class II
    727415.............. Rhinelander Oneida, WI   Class II
    727437.............. Iron Mountain/Ford, MI   Class II
    727440.............. Hancock Houghton Co AP, MI   Class II
    727444.............. Two Harbors, MN   Class III
    727445.............. Ironwood (AWOS), MI   Class II
    727450.............. Duluth International Arpt, MN   Class I
    727452.............. Crookston Muni Fld, MN   Class III
    727453.............. Park Rapids Municipal AP, MN   Class II
    727455.............. Hibbing Chisholm-Hibbing AP, MN   Class II
    727457.............. Detroit Lakes (AWOS), MN   Class II
    727458.............. Grand Rapids (AWOS), MN   Class II
    727459.............. Ely Muni, MN   Class II
    727470.............. International Falls Intl AP, MN   Class I
    727473.............. Crane Lake (AWOS), MN   Class II
    727474.............. Eveleth Muni (AWOS), MN   Class III
    727475.............. Mora Muni (AWOS), MN   Class II
    727476.............. Baudette International AP, MN   Class III
    727477.............. Roseau Muni (AWOS), MN   Class II
    727478.............. Hallock, MN   Class III
    727503.............. Cambridge Muni, MN   Class III
    727504.............. Aitkin Ndb (AWOS), MN   Class III
    727505.............. Fosston (AWOS), MN   Class III
    727507.............. Benson Muni, MN   Class III
    727530.............. Fargo Hector International AP, ND   Class I
    727533.............. Wheaton Ndb (AWOS), MN   Class III
    727535.............. Jamestown Municipal Arpt, ND   Class II
    727550.............. Bemidji Municipal, MN   Class II
    727555.............. Thief River (AWOS), MN   Class II
    727556.............. Silver Bay, MN   Class III
    727566.............. Austin Muni, MN   Class III
    727573.............. Devils Lake (AWOS), ND   Class II
    727575.............. Grand Forks AF, ND   Class II
    727576.............. Grand Forks International AP, ND   Class II
    727640.............. Bismarck Municipal Arpt [ISIS], ND   Class I
    727645.............. Dickinson Municipal AP, ND   Class II
    727670.............. Williston Sloulin Intl AP, ND   Class I
    727675.............. Minot AFB, ND   Class II
    727676.............. Minot - - AP, ND   Class I
    727680.............. Glasgow Intl Arpt, MT   Class I
    727686.............. Wolf Point Intl [Fort Peck - SURFRAD], MT   Class II
    727687.............. Sidney-Richland, MT   Class II
    727720.............. Helena Regional Airport, MT   Class I
    727730.............. Missoula International AP, MT   Class I
    727750.............. Great Falls Intl Arpt, MT   Class I
    727770.............. Havre City-County AP, MT   Class II
    727790.............. Kalispell Glacier Pk Int'l Ar, MT   Class I
    727796.............. Cut Bank Muni AP, MT   Class II
    727810.............. Yakima Air Terminal, WA   Class I
    727815.............. Stampede Pass, WA   Class II
    727825.............. Wenatchee/Pangborn, WA   Class II
    727826.............. Ephrata AP FCWOS, WA   Class II
    727827.............. Moses Lake Grant County AP, WA   Class II
    727830.............. Lewiston Nez Perce Cnty AP, ID   Class I
    727834.............. Coeur D`Alene (AWOS), ID   Class II
    727840.............. Hanford, WA   Class II
    727845.............. Pasco, WA   Class II
    727846.............. Walla Walla City County AP, WA   Class II
    727850.............. Spokane International AP [Cheney - UO], WA   Class I
    727855.............. Fairchild AFB, WA   Class II
    727856.............. Felts Fld, WA   Class II
    727857.............. Pullman/Moscow Rgnl, WA   Class II
    727885.............. William R Fairchild, WA   Class III
    727910.............. Astoria Regional Airport, OR   Class II
    727920.............. Olympia Airport, WA   Class I
    727923.............. Hoquiam AP, WA   Class II
    727924.............. Kelso WB AP, WA   Class II
    727926.............. Toledo-Winlock Mem, WA   Class II
    727928.............. Bremerton National, WA   Class II
    727930.............. Seattle Seattle-Tacoma Intl A, WA   Class I
    727934.............. Renton Muni, WA   Class II
    727935.............. Seattle Boeing Field [ISIS], WA   Class II
    727937.............. Snohomish Co, WA   Class II
    727938.............. Tacoma Narrows, WA   Class II
    727970.............. Quillayute State Airport, WA   Class I
    727976.............. Bellingham Intl AP, WA   Class II
    742060.............. Tacoma McChord AFB, WA   Class II
    742070.............. Gray AAF, WA   Class II
    742300.............. Miles City Municipal Arpt, MT   Class II
    743700.............. Fort Drum/Wheeler-S, NY   Class II
    743920.............. Brunswick NAS, ME   Class II
    743945.............. Manchester Airport, NH   Class II
    744655.............. Aurora Municipal, IL   Class II
    744860.............. New York J F Kennedy Int'l Ar, NY   Class I
    744864.............. Republic, NY   Class II
    744865.............. Westhampton Gabreski AP, NY   Class II
    744904.............. Lawrence Muni, MA   Class II
    744910.............. Chicopee Falls Westo, MA   Class II
    744915.............. Westfield Barnes Muni AP, MA   Class II
    745090.............. Mountain View Moffett Fld NAS, CA   Class III
    745160.............. Travis Field AFB, CA   Class II
    745700.............. Dayton Wright Patterson AFB,, OH   Class II
    745940.............. Andrews AFB, MD   Class II
    745966.............. Cape May Co, NJ   Class III
    745980.............. Langley AFB, VA   Class II
    745985.............. Martinsville, VA   Class III
    746120.............. China Lake NAF, CA   Class II
    746710.............. Fort Campbell AAF, KY   Class II
    746716.............. Bowling Green Warren Co AP, KY   Class II
    746930.............. Fort Bragg Simmons AAF, NC   Class II
    746943.............. Elizabeth City Coast Guard Ai [NREL], NC   Class III
    747020.............. Lemoore Reeves NAS, CA   Class II
    747185.............. Imperial, CA   Class II
    747187.............. Palm Springs Thermal AP, CA   Class II
    747188.............. Blythe Riverside Co Arpt, CA   Class II
    747320.............. Holloman AFB, NM   Class II
    747540.............. England AFB, LA   Class III
    747685.............. Gulfport Biloxi Int, MS   Class II
    747686.............. Keesler AFB, MS   Class II
    747750.............. Tyndall AFB, FL   Class II
    747770.............. Valparaiso Hurlburt, FL   Class II
    747804.............. Hunter AAF, GA   Class II
    747810.............. Moody AFB/Valdosta, GA   Class II
    747880.............. MacDill AFB, FL   Class II
    747900.............. Sumter Shaw AFB, SC   Class II
    747910.............. Myrtle Beach AFB, SC   Class II
    747915.............. North Myrtle Beach Grand Stra, SC   Class II
    747946.............. NASa Shuttle Fclty, FL   Class III
    785140.............. Aquadilla/Borinquen, PR   Class II
    785145.............. Eugenio Maria De Ho, PR   Class II
    785203.............. Mercedita, PR   Class II
    785260.............. San Juan Intl Arpt, PR   Class II
    785263.............. San Juan L M Marin Intl AP, PR   Class III
    785350.............. Roosevelt Roads, PR   Class III
    785430.............. Charlotte Amalie Harry S Trum, VI   Class II
    911650.............. Lihue Airport, HI   Class I
    911760.............. Kaneohe Bay MCAS, HI   Class II
    911780.............. Barbers Point NAS, HI   Class II
    911820.............. Honolulu Intl Arpt, HI   Class I
    911860.............. Molokai (AMOS), HI   Class II
    911900.............. Kahului Airport, HI   Class I
    911904.............. Kapalua, HI   Class II
    911905.............. Lanai, HI   Class II
    911975.............. Kona Intl At Keahol, HI   Class III
    912120.............. Guam WFO, GU   Class II
    912180.............. Andersen AFB, GU   Class II
    912850.............. Hilo International AP, HI   Class I
- - - - - -

- -

- -Return to RReDC home page - ( http://www.nrel.gov/rredc ) - -
- - - - - - - - \ No newline at end of file diff --git a/scripts/download_primary_sources.sh b/scripts/download_primary_sources.sh deleted file mode 100755 index 3cf4036..0000000 --- a/scripts/download_primary_sources.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# requires mapshaper (npm install -g mapshaper) -# requires ogr2ogr (brew install gdal) - -# Reference materials -# http://energy.gov/sites/prod/files/2015/02/f19/ba_climate_guide_2013.pdf -# http://www2.census.gov/geo/docs/reference/codes/files/national_county.txt - - -# make script work independent of working directory -PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) -cd "$PARENT_PATH" - -DATA_DIR=${1:-data} -mkdir -p $DATA_DIR - -echo Downloading primary source files to $DATA_DIR - -echo Downloading cb_2016_us_zcta510_500k.json -./create_zipcode_geojson.sh $DATA_DIR - -echo Downloading cb_2016_us_county_500k.json -./create_county_geojson.sh $DATA_DIR - -echo Downloading CA_Building_Standards_Climate_Zones.json -./create_ca_climate_zone_geojson.sh $DATA_DIR - -# IECC/Building America climate zone csv. Custom; assembled and corrected from primary source -echo Downloading climate_zones.csv -wget -N https://gist.githubusercontent.com/philngo/d3e251040569dba67942/raw/0c98f906f452b9c80d42aec3c8c3e1aafab9add8/climate_zones.csv -P $DATA_DIR -q --show-progress - -# NCEI station lat lngs and metadata -echo Downloading isd-history.csv -wget -N https://www.ncei.noaa.gov/pub/data/noaa/isd-history.csv -P $DATA_DIR -q --show-progress - -# NCEI weather data quality -echo Downloading isd-inventory.csv -wget -N https://www.ncei.noaa.gov/pub/data/noaa/isd-inventory.csv -P $DATA_DIR -q --show-progress - -# Scrape-friendly TMY3 station list -echo Downloading tmy3-stations.html -cp "$PARENT_PATH/data/tmy3-stations.html" $DATA_DIR - -# Add ZIP code prefix mapping -echo Downloading state zipcode prefixes -wget -N https://gist.githubusercontent.com/philngo/247226aa89e5abf5869b981b9b841245/raw/56e25d8d590c001a18a1c0bab3ac69c53c09117c/zipcode_prefixes.json -P $DATA_DIR -q --show-progress - -echo Finished downloading primary source files - -# GHCNh station list for the usaf -> ghcn id mapping -echo Downloading ghcnh-station-list.csv -wget -N https://www.ncei.noaa.gov/oa/global-historical-climatology-network/hourly/doc/ghcnh-station-list.csv -P $DATA_DIR -q --show-progress - -# GHCNh inventory for per-station data availability years -echo Downloading ghcnh-inventory.txt -wget -N https://www.ncei.noaa.gov/oa/global-historical-climatology-network/hourly/doc/ghcnh-inventory.txt -P $DATA_DIR -q --show-progress diff --git a/scripts/validate_ghcnh_against_isd.py b/scripts/validate_ghcnh_against_isd.py deleted file mode 100644 index 6279ff4..0000000 --- a/scripts/validate_ghcnh_against_isd.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -"""Compare GHCNh-served temperatures against the retired ISD dataset. - -Fetches both datasets from the NCEI access api for a stratified station -sample over their overlap years, runs both through the library's hourly -resampling pipeline, and reports per-station-year deviation statistics. -ISD stopped receiving data 2025-08-27 but still serves its history, which -makes this comparison reproducible. - -Run from the repository root: - - python scripts/validate_ghcnh_against_isd.py -""" -import csv -import io -import sqlite3 -import sys -import time - -import pandas as pd -import pytz -import requests - -from eeweather.sources.ghcnh import API_URL, fetch_ghcnh_hourly - - - -YEARS = [2016, 2019, 2022, 2024] - -STATIONS_PER_STATE_QUALITY = {"high": 2, "medium": 1, "low": 1} - - -def _sample_stations(): - """Deterministic stratified sample: per state, up to 2 high-quality - stations plus 1 medium and 1 low, covering every state and territory.""" - conn = sqlite3.connect("eeweather/resources/metadata.db") - cur = conn.cursor() - cur.execute( - """ - select quality, usaf_id, recent_wban_id, ghcn_id, coalesce(state, 'INTL') - from isd_station_metadata - order by state, quality, usaf_id - """ - ) - by_stratum = {} - for quality, usaf_id, wban_id, ghcn_id, state in cur.fetchall(): - by_stratum.setdefault((state, quality), []).append( - (quality, usaf_id, wban_id, ghcn_id, state) - ) - - stations = [] - for (state, quality), rows in sorted(by_stratum.items()): - n = STATIONS_PER_STATE_QUALITY[quality] - step = max(1, len(rows) // n) - stations.extend(rows[::step][:n]) - - return stations - - -def _caltrack_hourly(ts): - # CalTRACK 2.3.3 - resampled = ( - ts.resample("min") - .mean() - .interpolate(method="linear", limit=60, limit_direction="both") - .resample("h") - .mean() - ) - - return resampled - - -def _fetch_isd_hourly(usaf_id, wban_id, year): - resp = requests.get( - API_URL, - params={ - "dataset": "global-hourly", - "dataTypes": "TMP", - "stations": "{}{}".format(usaf_id, wban_id), - "startDate": "{}-01-01".format(year), - "endDate": "{}-12-31".format(year), - }, - timeout=120, - ) - resp.raise_for_status() - - data = [] - for record in csv.DictReader(io.StringIO(resp.text)): - date, temp = record["DATE"], str(record["TMP"]).strip() - temp_val, suffix = temp.split(",") - if suffix == "9" or temp_val == "+9999": - parsed = float("nan") - else: - parsed = float(temp_val) / 10.0 - data.append((pd.Timestamp(date, tz=pytz.UTC), parsed)) - - if not data: - return None - - ts = pd.Series(dict(data)).sort_index() - ts = ts.groupby(ts.index).mean() - - return _caltrack_hourly(ts) - - -def main(): - results = [] - for quality, usaf_id, wban_id, ghcn_id, state in _sample_stations(): - for year in YEARS: - try: - isd = _fetch_isd_hourly(usaf_id, wban_id, year) - ghcnh_raw = fetch_ghcnh_hourly(ghcn_id, year) - except requests.RequestException as e: - print("{} {} {}: fetch failed ({})".format(usaf_id, state, year, e)) - continue - if isd is None or len(ghcnh_raw) == 0: - if isd is None: - n_isd_obs = 0 - else: - n_isd_obs = len(isd) - print( - "{} {} {}: skipped (isd={}, ghcnh={} obs)".format( - usaf_id, state, year, n_isd_obs, len(ghcnh_raw) - ) - ) - continue - ghcnh = _caltrack_hourly(ghcnh_raw["temperature"]) - - both = pd.DataFrame({"isd": isd, "ghcnh": ghcnh}).dropna() - delta = (both.ghcnh - both.isd).abs() - results.append( - { - "usaf_id": usaf_id, "state": state, "quality": quality, - "year": year, "n_isd": int(isd.notna().sum()), - "n_ghcnh": int(ghcnh.notna().sum()), "n_both": len(both), - "mean_abs_delta": delta.mean(), "p99_abs_delta": delta.quantile(0.99), - "max_abs_delta": delta.max(), - "annual_mean_delta": abs(both.ghcnh.mean() - both.isd.mean()), - } - ) - r = results[-1] - print( - "{usaf_id} {state} {quality} {year}: n_isd={n_isd} n_ghcnh={n_ghcnh}" - " mean|d|={mean_abs_delta:.4f} p99|d|={p99_abs_delta:.4f}" - " max|d|={max_abs_delta:.2f} annual|d|={annual_mean_delta:.5f}".format(**r) - ) - - time.sleep(0.2) - pd.DataFrame(results).to_csv("validation_results.csv", index=False) - - df = pd.DataFrame(results) - print("\n=== summary over {} station-years ===".format(len(df))) - for col in ["mean_abs_delta", "p99_abs_delta", "annual_mean_delta"]: - print( - "{}: median={:.5f} p90={:.5f} max={:.5f}".format( - col, df[col].median(), df[col].quantile(0.9), df[col].max() - ) - ) - print("hourly coverage ratio ghcnh/isd: median={:.4f} min={:.4f}".format( - (df.n_ghcnh / df.n_isd).median(), (df.n_ghcnh / df.n_isd).min() - )) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/__snapshots__/test_database.ambr b/tests/__snapshots__/test_database.ambr deleted file mode 100644 index 6cbad81..0000000 --- a/tests/__snapshots__/test_database.ambr +++ /dev/null @@ -1,36 +0,0 @@ -# serializer version: 1 -# name: test_isd_file_metadata_table_content - dict({ - 'usaf_id': '690150', - 'wban_id': '93121', - 'year': '2006', - }) -# --- -# name: test_isd_file_metadata_table_count - 47646 -# --- -# name: test_isd_station_metadata_table_content - dict({ - 'ba_climate_zone': 'Hot-Humid', - 'ca_climate_zone': None, - 'elevation': '+0003.0', - 'ghcn_first_year': 1955, - 'ghcn_id': 'USW00003852', - 'ghcn_last_year': 2026, - 'ghcn_map_method': 'latlon', - 'icao_code': None, - 'iecc_climate_zone': '2', - 'iecc_moisture_regime': 'A', - 'latitude': '+30.433', - 'longitude': '-086.717', - 'name': 'HURLBURT FLD/EXERCIS', - 'quality': 'high', - 'recent_wban_id': '99999', - 'state': 'FL', - 'usaf_id': '690090', - 'wban_ids': '99999', - }) -# --- -# name: test_isd_station_metadata_table_count - 4497 -# --- diff --git a/tests/__snapshots__/test_ranking.ambr b/tests/__snapshots__/test_ranking.ambr deleted file mode 100644 index 6d192aa..0000000 --- a/tests/__snapshots__/test_ranking.ambr +++ /dev/null @@ -1,175 +0,0 @@ -# serializer version: 1 -# name: test_rank_stations_is_cz2010[is_cz2010=False] - tuple( - 4411, - 15, - ) -# --- -# name: test_rank_stations_is_cz2010[is_cz2010=True] - tuple( - 86, - 15, - ) -# --- -# name: test_rank_stations_is_tmy3[is_tmy3=False] - tuple( - 3479, - 15, - ) -# --- -# name: test_rank_stations_is_tmy3[is_tmy3=True] - tuple( - 1018, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_not_null[match_ba_climate_zone] - tuple( - 254, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_not_null[match_ca_climate_zone] - tuple( - 10, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_not_null[match_iecc_climate_zone] - tuple( - 701, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_not_null[match_iecc_moisture_regime] - tuple( - 634, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_null[match_ba_climate_zone] - tuple( - 1364, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_null[match_ca_climate_zone] - tuple( - 4288, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_null[match_iecc_climate_zone] - tuple( - 1364, - 15, - ) -# --- -# name: test_rank_stations_match_climate_zones_null[match_iecc_moisture_regime] - tuple( - 1754, - 15, - ) -# --- -# name: test_rank_stations_match_state[site_state=CA, match_state=False] - tuple( - 4497, - 15, - ) -# --- -# name: test_rank_stations_match_state[site_state=CA, match_state=True] - tuple( - 241, - 15, - ) -# --- -# name: test_rank_stations_match_state[site_state=None, match_state=True] - tuple( - 1116, - 15, - ) -# --- -# name: test_rank_stations_max_difference_elevation_meters[max_difference_elevation_meters=200] - tuple( - 4497, - 15, - ) -# --- -# name: test_rank_stations_max_difference_elevation_meters[site_elevation=0, max_difference_elevation_meters=200] - tuple( - 2554, - 15, - ) -# --- -# name: test_rank_stations_max_difference_elevation_meters[site_elevation=0, max_difference_elevation_meters=50] - tuple( - 1506, - 15, - ) -# --- -# name: test_rank_stations_max_difference_elevation_meters[site_elevation=1000, max_difference_elevation_meters=50] - tuple( - 44, - 15, - ) -# --- -# name: test_rank_stations_max_distance_meters[max_distance_meters=200000] - tuple( - 43, - 15, - ) -# --- -# name: test_rank_stations_max_distance_meters[max_distance_meters=50000] - tuple( - 5, - 15, - ) -# --- -# name: test_rank_stations_minimum_quality[minimum_quality=high] - tuple( - 1858, - 15, - ) -# --- -# name: test_rank_stations_minimum_quality[minimum_quality=low] - tuple( - 4497, - 15, - ) -# --- -# name: test_rank_stations_minimum_quality[minimum_quality=medium] - tuple( - 2251, - 15, - ) -# --- -# name: test_rank_stations_minimum_tmy3_class[minimum_tmy3_class=III] - tuple( - 1018, - 15, - ) -# --- -# name: test_rank_stations_minimum_tmy3_class[minimum_tmy3_class=II] - tuple( - 856, - 15, - ) -# --- -# name: test_rank_stations_minimum_tmy3_class[minimum_tmy3_class=I] - tuple( - 222, - 15, - ) -# --- -# name: test_rank_stations_no_filter - tuple( - 4497, - 15, - ) -# --- -# name: test_select_station_with_empty_tempC - '747020' -# --- -# name: test_select_station_with_second_level_dates - '747020' -# --- diff --git a/tests/__snapshots__/test_summaries.ambr b/tests/__snapshots__/test_summaries.ambr deleted file mode 100644 index 0f17fef..0000000 --- a/tests/__snapshots__/test_summaries.ambr +++ /dev/null @@ -1,13 +0,0 @@ -# serializer version: 1 -# name: test_get_isd_station_usaf_ids - 4497 -# --- -# name: test_get_isd_station_usaf_ids_by_state - 75 -# --- -# name: test_get_zcta_ids - 33144 -# --- -# name: test_get_zcta_ids_by_state - 1763 -# --- diff --git a/tests/build/test_migrate.py b/tests/build/test_migrate.py new file mode 100644 index 0000000..2a08726 --- /dev/null +++ b/tests/build/test_migrate.py @@ -0,0 +1,38 @@ +"""The migrate step's GHCN id parsing lives in one helper only.""" +import inspect +from pathlib import Path + +import pytest + +from eeweather.build import migrate, refresh +from eeweather.build.migrate import _country + + + +@pytest.mark.parametrize( + "ghcn_id, expected", + [ + ("USW00023152", "US"), + ("RQC00668814", "PR"), + ("GME00099902", "DE"), + ("ASN00099901", "AU"), + ("ZZUNKNOWN01", None), + ], +) +def test_country_maps_fips_prefix_to_iso(ghcn_id, expected): + assert _country(ghcn_id) == expected + + +def test_country_is_sole_id_slice_site_in_migrate(): + source = Path(migrate.__file__).read_text() + helper_source = inspect.getsource(_country) + + assert source.count("[:2]") == 1 + assert "[:2]" in helper_source + + +def test_refresh_does_not_parse_station_ids(): + source = Path(refresh.__file__).read_text() + + assert "[:2]" not in source + assert "FIPS_TO_ISO" not in source diff --git a/tests/build/test_refresh.py b/tests/build/test_refresh.py new file mode 100644 index 0000000..626f2b4 --- /dev/null +++ b/tests/build/test_refresh.py @@ -0,0 +1,210 @@ +"""The refresh's safety behaviors, exercised against temporary databases.""" +import sqlite3 +from datetime import datetime, timezone + +import pytest + +from eeweather.build.refresh import ( + _append_new_stations, + _fetch_inventory, + _plausible_or_raise, + _refresh_aliases, + _replace_inventory, +) +from eeweather.build.schema import GHCNH_SCHEMA, IDENTIFIERS_SCHEMA, create_schema + + + +LAST_FULL_YEAR = datetime.now(timezone.utc).year - 1 + + +def _listed(name, lat, lon, wmo="", icao="", iso=""): + listing = (name, str(lat), str(lon), "10.0", wmo, icao, iso) + + return listing + + +def _inventory_row(station_id, year, count=700): + return (station_id, year) + (count,) * 12 + + +@pytest.fixture +def dbs(tmp_path): + ghcnh = sqlite3.connect(tmp_path / "ghcnh.db") + create_schema(ghcnh, GHCNH_SCHEMA) + identifiers = sqlite3.connect(tmp_path / "identifiers.db") + create_schema(identifiers, IDENTIFIERS_SCHEMA) + # one existing cataloged station (Burbank) + ghcnh.execute( + "insert into stations values" + " ('USW00023152', 'BURBANK', 34.2, -118.365, 236.0, 'US', 'CA')" + ) + identifiers.execute( + "insert into station_identifier values ('ghcn', 'USW00023152', 'USW00023152', 1)" + ) + identifiers.execute( + "insert into station_identifier values ('icao', 'KBUR', 'USW00023152', 1)" + ) + + yield ghcnh, identifiers + + ghcnh.close() + identifiers.close() + + +def test_fetch_inventory_parses_current_noaa_format(monkeypatch): + inventory_text = ( + "GHCNh_ID YEAR JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC\n" + "AFA00409951 1979 0 45 36 34 28 35 47 15 11 0 0 0\n" + "USW00023152 2024 700 700 700 700 700 700 700 700 700 700 700 700\n" + ) + + class _Response: + text = inventory_text + + def raise_for_status(self): + return None + + monkeypatch.setattr( + "eeweather.build.refresh.requests.get", lambda *args, **kwargs: _Response() + ) + + rows = _fetch_inventory() + + assert rows == [ + ("AFA00409951", 1979, 0, 45, 36, 34, 28, 35, 47, 15, 11, 0, 0, 0), + ("USW00023152", 2024, 700, 700, 700, 700, 700, 700, 700, 700, 700, 700, 700, 700), + ] + + +def test_append_admits_active_station_inside_geography_with_zones(dbs): + ghcnh, identifiers = dbs + station_list = {"USW00099901": _listed("PASADENA TEST", 34.15, -118.14)} + inventory = [_inventory_row("USW00099901", LAST_FULL_YEAR)] + + added = _append_new_stations(ghcnh, identifiers, station_list, inventory) + + assert added == 1 + zones = dict(ghcnh.execute( + "select system, zone_id from station_zone where station_id='USW00099901'" + ).fetchall()) + assert zones["ca_climate_zone"] == "CA_09" + assert identifiers.execute( + "select count(*) from station_identifier" + " where namespace='ghcn' and station_id='USW00099901'" + ).fetchone()[0] == 1 + + +def test_append_uses_iso_code_for_country(dbs): + ghcnh, identifiers = dbs + # a "CA" id prefix has no FIPS_TO_ISO entry, so country would be null; + # the station list's ISO_CODE supplies it + station_list = {"CAN01013998": _listed("KELP REEF", 34.15, -118.14, iso="CA")} + inventory = [_inventory_row("CAN01013998", LAST_FULL_YEAR)] + + added = _append_new_stations(ghcnh, identifiers, station_list, inventory) + + assert added == 1 + country = ghcnh.execute( + "select country from stations where station_id = 'CAN01013998'" + ).fetchone()[0] + assert country == "CA" + + +def test_append_rejects_station_outside_served_geography(dbs): + ghcnh, identifiers = dbs + # Hamburg: active, but no geography pack covers Germany + station_list = {"GME00099902": _listed("HAMBURG TEST", 53.55, 9.99)} + inventory = [_inventory_row("GME00099902", LAST_FULL_YEAR)] + + added = _append_new_stations(ghcnh, identifiers, station_list, inventory) + + assert added == 0 + + +def test_append_rejects_offshore_platform(dbs): + ghcnh, identifiers = dbs + # Gulf of Mexico, ~200 km off the Louisiana coast + station_list = {"USW00099903": _listed("PLATFORM TEST", 27.2, -90.0)} + inventory = [_inventory_row("USW00099903", LAST_FULL_YEAR)] + + added = _append_new_stations(ghcnh, identifiers, station_list, inventory) + + assert added == 0 + + +def test_append_rejects_inactive_station(dbs): + ghcnh, identifiers = dbs + station_list = {"USW00099904": _listed("DEAD TEST", 34.15, -118.14)} + inventory = [_inventory_row("USW00099904", 2015)] + + added = _append_new_stations(ghcnh, identifiers, station_list, inventory) + + assert added == 0 + + +def test_alias_refresh_never_claims_an_id_with_another_recent_holder(dbs): + ghcnh, identifiers = dbs + # a second cataloged station whose list row claims Burbank's icao + ghcnh.execute( + "insert into stations values" + " ('USW00099905', 'IMPOSTER', 34.0, -118.0, 10.0, 'US', 'CA')" + ) + station_list = { + "USW00099905": _listed("IMPOSTER", 34.0, -118.0, icao="KBUR"), + } + + added = _refresh_aliases(ghcnh, identifiers, station_list) + + assert added == 0 + holders = identifiers.execute( + "select station_id from station_identifier" + " where namespace='icao' and external_id='KBUR'" + ).fetchall() + assert holders == [("USW00023152",)] + + +def test_alias_refresh_adds_unclaimed_aliases(dbs): + ghcnh, identifiers = dbs + station_list = { + "USW00023152": _listed("BURBANK", 34.2, -118.365, wmo="72288", icao="KBUR"), + } + + added = _refresh_aliases(ghcnh, identifiers, station_list) + + assert added == 1 # wmo added; icao already present + assert identifiers.execute( + "select station_id from station_identifier" + " where namespace='wmo' and external_id='72288'" + ).fetchone() == ("USW00023152",) + + +def test_implausibly_small_station_list_aborts(dbs): + ghcnh, _ = dbs + with pytest.raises(RuntimeError, match="Implausible station list"): + _plausible_or_raise(ghcnh, {}, [_inventory_row("USW00023152", 2024)]) + + +def test_implausibly_small_inventory_aborts(dbs): + ghcnh, _ = dbs + placeholders = ", ".join("?" for _ in range(14)) + ghcnh.executemany( + "insert into inventory values ({})".format(placeholders), + [_inventory_row("USW00023152", y) for y in (2022, 2023, 2024)], + ) + station_list = {"USW00023152": _listed("BURBANK", 34.2, -118.365)} + + with pytest.raises(RuntimeError, match="Implausible inventory"): + _plausible_or_raise(ghcnh, station_list, []) + + +def test_replace_inventory_keeps_only_cataloged_stations(dbs): + ghcnh, _ = dbs + inventory = [ + _inventory_row("USW00023152", 2024), + _inventory_row("ZZNOTINCAT1", 2024), + ] + + n = _replace_inventory(ghcnh, inventory) + + assert n == 1 diff --git a/tests/conftest.py b/tests/conftest.py index 192f6f1..0c21eca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,23 +1,5 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" import gzip +import os import re import tempfile @@ -25,6 +7,9 @@ import pytest + +os.environ["EEWEATHER_AUTO_UPDATE"] = "0" + from eeweather.cache import KeyValueStore @@ -69,7 +54,10 @@ def mock_get(url, params=None, **kwargs): return MockAccessAPIResponse(text) - monkeypatch.setattr("eeweather.sources.ghcnh.requests.get", mock_get) + def mock_session_get(url, params=None, **kwargs): + return mock_get(url, params=params, **kwargs) + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", mock_session_get) def _fixture_ascii(name): @@ -99,3 +87,28 @@ def __init__(self): def get_store(self): return self.store + + +@pytest.fixture +def monkeypatch_key_value_store(monkeypatch): + """A fresh temporary cache store patched into the shared proxy.""" + key_value_store_proxy = MockKeyValueStoreProxy() + monkeypatch.setattr( + "eeweather.cache.key_value_store_proxy", key_value_store_proxy + ) + + return key_value_store_proxy.get_store() + + +@pytest.fixture +def monkeypatch_tmy3_request(monkeypatch): + monkeypatch.setattr( + "eeweather.sources.base.request_text", mock_request_text_tmy3 + ) + + +@pytest.fixture +def monkeypatch_cz2010_request(monkeypatch): + monkeypatch.setattr( + "eeweather.sources.base.request_text", mock_request_text_cz2010 + ) diff --git a/tests/registry/test_db.py b/tests/registry/test_db.py new file mode 100644 index 0000000..5646514 --- /dev/null +++ b/tests/registry/test_db.py @@ -0,0 +1,132 @@ +import sqlite3 + +import pytest + +from eeweather.exceptions import UnrecognizedPlaceError, UnrecognizedStationError +from eeweather.registry.db import ( + metadata_db_connection_proxy, + valid_place_or_raise, + valid_station_id_or_raise, +) + + +def test_valid_station_id_or_raise(): + assert valid_station_id_or_raise("USW00023152") is True + + with pytest.raises(UnrecognizedStationError) as excinfo: + valid_station_id_or_raise("INVALID") + assert excinfo.value.value == "INVALID" + + +def test_valid_place_or_raise(): + assert valid_place_or_raise("zcta", "90210") is True + + with pytest.raises(UnrecognizedPlaceError) as excinfo: + valid_place_or_raise("zcta", "INVALID") + assert excinfo.value.code == "INVALID" + + +# attachment machinery and multi-catalog behavior + + +def test_register_attachment_ignores_duplicate_aliases(): + proxy = metadata_db_connection_proxy + before = list(proxy._attachments) + + proxy.register_attachment("ghcnh", "/nonexistent/other.db", catalog=True) + + assert proxy._attachments == before + + +def test_attachment_role_properties(): + proxy = metadata_db_connection_proxy + + assert proxy.catalogs == ["ghcnh"] + assert proxy.inventory_sources == ["ghcnh"] + assert proxy.quality_sources == ["ghcnh"] + assert set(proxy.availability_sources) == {"tmy3", "cz2010"} + assert proxy.geography_aliases == ["geography_us"] + + +@pytest.fixture +def second_catalog(tmp_path): + """A synthetic catalog registered after ghcnh: one station overlapping + the packaged catalog (with conflicting facts) and one of its own.""" + from eeweather.registry.db import Attachment + from eeweather.sources.matching import cached_data + + path = str(tmp_path / "fakecat.db") + conn = sqlite3.connect(path) + conn.execute( + "create table stations (" + " station_id text primary key, name text, latitude real," + " longitude real, elevation real, country text, subdivision text" + ") without rowid" + ) + conn.execute( + "create table station_zone (" + " station_id text not null, system text not null," + " zone_id text not null, primary key (station_id, system)" + ") without rowid" + ) + conn.execute( + "insert into stations values" + " ('USW00023152', 'CONFLICTING NAME', 1.0, 2.0, 3.0, 'US', 'CA')" + ) + conn.execute( + "insert into stations values" + " ('ZZFAKE00001', 'FAKE ONLY', 10.0, 20.0, 30.0, 'ZZ', null)" + ) + conn.commit() + conn.close() + + proxy = metadata_db_connection_proxy + saved = list(proxy._attachments) + proxy._attachments.append( + Attachment("fakecat", path, True, False, False, False) + ) + proxy.close() + cached_data.__dict__.pop("all_station_metadata", None) + + yield + + proxy._attachments[:] = saved + proxy.close() + cached_data.__dict__.pop("all_station_metadata", None) + + +def test_multi_catalog_first_catalog_wins_for_facts(second_catalog): + from eeweather.registry.metadata import get_station_metadata + + metadata = get_station_metadata("USW00023152") + + assert metadata["name"] == "BURBANK-GLENDALE-PASA ARPT" + assert metadata["latitude"] == pytest.approx(34.2) + + +def test_multi_catalog_search_first_catalog_wins(second_catalog): + from eeweather.registry.summaries import search_stations + + df = search_stations() + + assert df.loc["USW00023152", "name"] == "BURBANK-GLENDALE-PASA ARPT" + assert df.loc["ZZFAKE00001", "name"] == "FAKE ONLY" + assert len(df) == 5892 + + +def test_multi_catalog_enumeration_is_a_union(second_catalog): + from eeweather.registry.summaries import get_station_ids + + station_ids = get_station_ids() + + assert len(station_ids) == 5892 + assert "ZZFAKE00001" in station_ids + + +def test_multi_catalog_ranking_keeps_first_catalog_coordinates(second_catalog): + from eeweather.sources.matching import cached_data + + df = cached_data.all_station_metadata + + assert df.loc["USW00023152", "latitude"] == pytest.approx(34.2) + assert "ZZFAKE00001" in df.index diff --git a/tests/registry/test_identifiers.py b/tests/registry/test_identifiers.py new file mode 100644 index 0000000..be9bdf0 --- /dev/null +++ b/tests/registry/test_identifiers.py @@ -0,0 +1,88 @@ +import pytest + +from eeweather.exceptions import AmbiguousIdentifierError, UnrecognizedStationError +from eeweather.registry.identifiers import resolve_station, translate + + +def test_translate_usaf_to_ghcn(): + mapping = translate(["722880", "722874"], "usaf", "ghcn") + + assert mapping == { + "722880": ("USW00023152",), + "722874": ("USW00093134",), + } + + +def test_translate_values_are_tuples_for_one_to_many(): + # this Australian station carries four historical USAF ids + mapping = translate(["ASA00956720"], "ghcn", "usaf") + + assert len(mapping["ASA00956720"]) == 4 + + +def test_translate_unrecognized_ids_are_absent(): + mapping = translate(["722880", "NOT_AN_ID"], "usaf", "ghcn") + + assert "NOT_AN_ID" not in mapping + assert mapping["722880"] == ("USW00023152",) + + +def test_translate_deduplicates_input_ids(): + mapping = translate(["722880", "722880"], "usaf", "ghcn") + + assert mapping == {"722880": ("USW00023152",)} + + +def test_resolve_station_unique(): + assert resolve_station("usaf", "722880") == "USW00023152" + assert resolve_station("icao", "KBUR") == "USW00023152" + + +def test_resolve_station_ambiguous_recent_wins(): + # wban 03935 was reused; only USW00003935 holds it recently + assert resolve_station("wban", "03935") == "USW00003935" + + +@pytest.fixture +def identifiers_db_with_unresolvable_wban(tmp_path, monkeypatch): + """A copy of the packaged crosswalk plus a synthetic wban mapping to + two stations with no recent marker (no such case survives in the + packaged data after the alias audit).""" + import shutil + import sqlite3 + + from eeweather.registry.db import IDENTIFIERS_DB_PATH, metadata_db_connection_proxy + + path = str(tmp_path / "identifiers.db") + shutil.copy(IDENTIFIERS_DB_PATH, path) + conn = sqlite3.connect(path) + conn.execute( + "insert into station_identifier values ('wban', '90001', 'USW00023152', 0)" + ) + conn.execute( + "insert into station_identifier values ('wban', '90001', 'USW00093134', 0)" + ) + conn.commit() + conn.close() + + proxy = metadata_db_connection_proxy + monkeypatch.setattr(proxy, "db_path", path) + proxy.close() + + yield + + proxy.close() + + +def test_resolve_station_ambiguous_without_recent_raises( + identifiers_db_with_unresolvable_wban, +): + with pytest.raises(AmbiguousIdentifierError) as excinfo: + resolve_station("wban", "90001") + assert excinfo.value.station_ids == ("USW00023152", "USW00093134") + + +def test_resolve_station_unrecognized(): + with pytest.raises(UnrecognizedStationError): + resolve_station("usaf", "000000") + diff --git a/tests/registry/test_metadata.py b/tests/registry/test_metadata.py new file mode 100644 index 0000000..0ac6283 --- /dev/null +++ b/tests/registry/test_metadata.py @@ -0,0 +1,45 @@ +import pytest + +from eeweather.exceptions import UnrecognizedStationError +from eeweather.registry.metadata import get_station_metadata + + +def test_get_station_metadata(): + metadata = get_station_metadata("USW00023152") + + assert metadata["station_id"] == "USW00023152" + assert metadata["name"] == "BURBANK-GLENDALE-PASA ARPT" + assert metadata["latitude"] == pytest.approx(34.2) + assert metadata["longitude"] == pytest.approx(-118.365) + assert metadata["elevation"] == pytest.approx(222.7) + assert metadata["country"] == "US" + assert metadata["subdivision"] == "CA" + assert metadata["quality"] == "high" + assert metadata["ids"] == { + "ghcn": ["USW00023152"], + "icao": ["KBUR"], + "usaf": ["722880"], + "wban": ["23152"], + "wmo": ["72288"], + } + assert metadata["zones"] == { + "ba_climate_zone": "Hot-Dry", + "ca_climate_zone": "CA_09", + "iecc_climate_zone": "3", + "iecc_moisture_regime": "B", + } + assert metadata["inventory_years"]["ghcnh"][0] == 1943 + assert metadata["inventory_years"]["ghcnh"][1] >= 2025 + + +def test_get_station_metadata_unrecognized(): + with pytest.raises(UnrecognizedStationError) as excinfo: + get_station_metadata("INVALID") + assert excinfo.value.value == "INVALID" + + +def test_get_station_metadata_australian_station_has_no_subdivision(): + metadata = get_station_metadata("ASA00749455") + + assert metadata["country"] == "AU" + assert metadata["subdivision"] is None diff --git a/tests/registry/test_quality.py b/tests/registry/test_quality.py new file mode 100644 index 0000000..e87a7b2 --- /dev/null +++ b/tests/registry/test_quality.py @@ -0,0 +1,49 @@ +from datetime import datetime, timezone + +from eeweather.registry.quality import ( + _quality_rating_window, + get_station_quality, + get_station_qualities, +) + + + +def test_quality_rating_window_anchors_five_years_two_after_anchor(): + anchor = datetime(2012, 12, 31, tzinfo=timezone.utc) + + assert _quality_rating_window(anchor) == (2010, 2014) + + +def test_quality_rating_window_clamps_to_last_full_year(): + last_full_year = datetime.now(timezone.utc).year - 1 + # an anchor far past the last full year forces the clamp + anchor = datetime(last_full_year + 5, 1, 1, tzinfo=timezone.utc) + + assert _quality_rating_window(anchor) == (last_full_year - 4, last_full_year) + + +def test_get_station_quality_high_during_active_era(): + anchor = datetime(2012, 12, 31, tzinfo=timezone.utc) + + assert get_station_quality("USW00093134", anchor) == "high" + + +def test_get_station_quality_low_after_station_went_quiet(): + anchor = datetime(2024, 12, 31, tzinfo=timezone.utc) + + assert get_station_quality("USW00093134", anchor) == "low" + + +def test_get_station_quality_low_before_any_data(): + anchor = datetime(1851, 1, 1, tzinfo=timezone.utc) + + assert get_station_quality("USW00093134", anchor) == "low" + + +def test_get_station_qualities_matches_single_station_rating(): + anchor = datetime(2014, 12, 31, tzinfo=timezone.utc) + + qualities = get_station_qualities(anchor) + + for station_id in ["USW00093134", "USW00023152", "USW00023149"]: + assert qualities[station_id] == get_station_quality(station_id, anchor) diff --git a/tests/registry/test_schema.py b/tests/registry/test_schema.py new file mode 100644 index 0000000..a3440eb --- /dev/null +++ b/tests/registry/test_schema.py @@ -0,0 +1,192 @@ +"""Integrity pins for the packaged data files.""" +import pytest + +from eeweather.registry.db import IDENTIFIERS_DB_PATH, metadata_db_connection_proxy + + + +EXPECTED_TABLES = { + "identifiers": { + "station_identifier": ["namespace", "external_id", "station_id", "recent"], + }, + "geography_us": { + "place": ["kind", "code", "country", "subdivision", "latitude", "longitude"], + "place_zone": ["kind", "code", "system", "zone_id"], + "zone": ["system", "zone_id", "name", "geometry"], + }, + "ghcnh": { + "stations": [ + "station_id", "name", "latitude", "longitude", "elevation", + "country", "subdivision", + ], + "station_zone": ["station_id", "system", "zone_id"], + "meta": ["key", "value"], + "inventory": [ + "station_id", "year", + "jan", "feb", "mar", "apr", "may", "jun", + "jul", "aug", "sep", "oct", "nov", "dec", + ], + "quality": ["station_id", "quality"], + }, + "tmy3": { + "stations": ["station_id", "usaf_id", "class"], + }, + "cz2010": { + "stations": ["station_id", "usaf_id"], + }, +} + + +@pytest.fixture(scope="module") +def conn(): + # the shared proxy connection, with every packaged file attached + return metadata_db_connection_proxy.get_connection() + + +def test_identifiers_db_is_the_connection_base(): + assert metadata_db_connection_proxy.db_path == IDENTIFIERS_DB_PATH + + +def test_packaged_files_match_expected_schemas(conn): + for alias, tables in EXPECTED_TABLES.items(): + if alias == "identifiers": + master = "sqlite_master" + prefix = "" + else: + master = "{}.sqlite_master".format(alias) + prefix = "{}.".format(alias) + found = { + row[0] for row in conn.execute( + "select name from {} where type = 'table'".format(master) + ) + } + assert found == set(tables), "tables of {}".format(alias) + + for table, expected_columns in tables.items(): + columns = [row[1] for row in conn.execute( + "pragma {}table_info({})".format(prefix, table) + )] + assert columns == expected_columns, "columns of {}.{}".format( + alias, table + ) + + +def test_row_counts(conn): + counts = {} + for alias, tables in EXPECTED_TABLES.items(): + for table in tables: + if alias == "identifiers": + qualified = table + else: + qualified = "{}.{}".format(alias, table) + counts["{}.{}".format(alias, table)] = conn.execute( + "select count(*) from {}".format(qualified) + ).fetchone()[0] + + # static tables pin exactly; refresh-mutable tables pin floors so a + # legitimate refresh PR stays green + assert counts["identifiers.station_identifier"] >= 14456 + assert counts["geography_us.zone"] == 35 + assert counts["geography_us.place"] == 33144 + assert counts["ghcnh.stations"] >= 3990 + assert counts["ghcnh.quality"] == counts["ghcnh.stations"] + assert counts["ghcnh.inventory"] >= 154825 + assert counts["ghcnh.station_zone"] > 0 + assert counts["geography_us.place_zone"] > 0 + assert counts["tmy3.stations"] == 1015 + assert counts["cz2010.stations"] == 86 + + +def test_identifier_namespaces(conn): + namespaces = { + row[0] for row in conn.execute( + "select distinct namespace from station_identifier" + ) + } + + assert namespaces == {"ghcn", "usaf", "wban", "icao", "wmo"} + + +def test_no_wban_sentinel_identifier_rows(conn): + sentinel_rows = conn.execute( + "select count(*) from station_identifier" + " where namespace = 'wban' and external_id = '99999'" + ).fetchone()[0] + + assert sentinel_rows == 0 + + +def test_every_station_has_a_ghcn_identity_row(conn): + orphans = conn.execute( + """ + select count(*) from ghcnh.stations as s + where not exists ( + select 1 from station_identifier as i + where i.station_id = s.station_id and i.namespace = 'ghcn' + ) + """ + ).fetchone()[0] + + assert orphans == 0 + + +def test_every_identity_appears_in_a_catalog(conn): + # the invariant that matters when non-catalog sources contribute + # stations: every minted id must have facts somewhere + homeless = conn.execute( + """ + select count(distinct station_id) from station_identifier as i + where not exists ( + select 1 from ghcnh.stations as s + where s.station_id = i.station_id + ) + """ + ).fetchone()[0] + + assert homeless == 0 + + +def test_every_cataloged_station_has_a_country(conn): + missing = conn.execute( + "select count(*) from ghcnh.stations where country is null" + ).fetchone()[0] + + assert missing == 0 + + +def test_quality_values(conn): + qualities = { + row[0] for row in conn.execute("select distinct quality from ghcnh.quality") + } + + assert qualities == {"high", "medium", "low"} + + +def test_archive_stations_reference_cataloged_stations(conn): + for alias in ("tmy3", "cz2010"): + dangling = conn.execute( + """ + select count(*) from {}.stations as a + where not exists ( + select 1 from ghcnh.stations as s + where s.station_id = a.station_id + ) + """.format(alias) + ).fetchone()[0] + + assert dangling == 0, alias + + +def test_no_identifier_has_multiple_recent_holders(conn): + # an id with two recent holders makes resolve_station raise; the + # refresh must never mint one + offenders = conn.execute( + """ + select namespace, external_id, count(*) from station_identifier + where recent = 1 + group by namespace, external_id + having count(*) > 1 + """ + ).fetchall() + + assert offenders == [] diff --git a/tests/registry/test_summaries.py b/tests/registry/test_summaries.py new file mode 100644 index 0000000..3fcdfc8 --- /dev/null +++ b/tests/registry/test_summaries.py @@ -0,0 +1,56 @@ +import pytest + +from eeweather.exceptions import UnrecognizedPlaceError +from eeweather.registry.summaries import get_place, get_station_ids, get_zcta_ids + + +def test_get_place(): + place = get_place("zcta", "90006") + + assert place["kind"] == "zcta" + assert place["code"] == "90006" + assert place["country"] == "US" + assert place["subdivision"] == "CA" + assert place["latitude"] == pytest.approx(34.048, abs=0.001) + assert place["longitude"] == pytest.approx(-118.294, abs=0.001) + assert place["zones"] == { + "ba_climate_zone": "Hot-Dry", + "ca_climate_zone": "CA_09", + "iecc_climate_zone": "3", + "iecc_moisture_regime": "B", + } + + +def test_get_place_unrecognized(): + with pytest.raises(UnrecognizedPlaceError) as excinfo: + get_place("zcta", "00000") + assert excinfo.value.kind == "zcta" + assert excinfo.value.code == "00000" + + +def test_get_station_ids(): + station_ids = get_station_ids() + + assert len(station_ids) == 5891 + assert station_ids[0] == "AQA00749401" + + +def test_get_station_ids_by_subdivision(): + station_ids = get_station_ids("IL") + + assert len(station_ids) == 69 + assert station_ids[0] == "USA00724394" + + +def test_get_zcta_ids(): + zcta_ids = get_zcta_ids() + + assert len(zcta_ids) == 33144 + + +def test_get_zcta_ids_by_subdivision(): + zcta_ids = get_zcta_ids("CA") + + assert len(zcta_ids) == 1763 + assert zcta_ids[0] == "90001" + diff --git a/tests/registry/test_update.py b/tests/registry/test_update.py new file mode 100644 index 0000000..d8efb8e --- /dev/null +++ b/tests/registry/test_update.py @@ -0,0 +1,580 @@ +"""The client-side registry update, against miniature copies of the +packaged data (full refreshes of the real files are far too slow for a +test suite).""" +import os +import shutil +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest +import requests + +import eeweather.registry.db as registry_db +import eeweather.registry.update as registry_update +from eeweather.registry.db import data_path + + + +N_STATIONS = 50 + + +def _tables_with_station_id(conn): + tables = [ + row[0] + for row in conn.execute("select name from sqlite_master where type = 'table'") + ] + matched = [] + for table in tables: + columns = [row[1] for row in conn.execute(f"pragma table_info({table})")] + if "station_id" in columns: + matched.append(table) + + return matched + + +@pytest.fixture(scope="module") +def mini_packaged(tmp_path_factory): + """Copies of the packaged databases cut down to N_STATIONS stations.""" + directory = str(tmp_path_factory.mktemp("packaged")) + paths = {} + for filename, packaged in registry_update.UPDATABLE.items(): + dest = os.path.join(directory, filename) + shutil.copy(packaged, dest) + paths[filename] = dest + + ghcnh = sqlite3.connect(paths["ghcnh.db"]) + keep = [ + row[0] + for row in ghcnh.execute( + "select station_id from stations order by station_id limit ?", + (N_STATIONS,), + ) + ] + ghcnh.close() + marks = ", ".join("?" for _ in keep) + for path in paths.values(): + conn = sqlite3.connect(path) + for table in _tables_with_station_id(conn): + conn.execute( + f"delete from {table} where station_id not in ({marks})", keep + ) + conn.commit() + conn.execute("vacuum") + conn.close() + + return paths, keep + + +@pytest.fixture +def patched_update(mini_packaged, tmp_path, monkeypatch): + paths, keep = mini_packaged + directory = str(tmp_path / "registry") + monkeypatch.setattr(registry_db, "UPDATED_DATA_DIR", directory) + monkeypatch.setattr(registry_update, "UPDATED_DATA_DIR", directory) + monkeypatch.setattr(registry_update, "UPDATABLE", dict(paths)) + + return directory, paths, keep + + +@pytest.fixture +def live_files_from_mini_catalog(patched_update, monkeypatch): + """Synthetic NOAA responses covering the mini catalog, sized so the + plausibility floors pass.""" + _, paths, keep = patched_update + conn = sqlite3.connect(paths["ghcnh.db"]) + n_inventory = conn.execute("select count(*) from inventory").fetchone()[0] + conn.close() + + station_list = { + sid: ("NAME", "0.0", "0.0", "0.0", "", "", "") for sid in keep + } + years_per_station = int(0.9 * n_inventory / len(keep)) + 2 + inventory = [ + (sid, year) + (700,) * 12 + for sid in keep + for year in range(2025 - years_per_station, 2025) + ] + monkeypatch.setattr( + "eeweather.build.refresh._fetch_station_list", lambda: station_list + ) + monkeypatch.setattr( + "eeweather.build.refresh._fetch_inventory", lambda: inventory + ) + + return len(inventory) + + +@pytest.fixture +def no_release_channel(monkeypatch): + def unreachable(url, dest): + raise requests.ConnectionError("no route to host") + + monkeypatch.setattr(registry_update, "_download", unreachable) + + +@pytest.fixture +def published_pack(patched_update, tmp_path, monkeypatch): + """A rolling-release pack: the mini databases restamped as freshly + refreshed, served through the download seam.""" + _, paths, _ = patched_update + pack_dir = tmp_path / "release" + pack_dir.mkdir() + published = {} + for filename, mini in paths.items(): + dest = str(pack_dir / filename) + shutil.copy(mini, dest) + published[filename] = dest + newer = registry_update.refreshed_at() + timedelta(days=1) + _stamp(published["ghcnh.db"], newer.strftime("%Y-%m-%d")) + + def download_from_pack(url, dest): + filename = url.rsplit("/", 1)[1] + shutil.copy(published[filename], dest) + + monkeypatch.setattr(registry_update, "_download", download_from_pack) + + return published + + +def _stamp(path, value): + conn = sqlite3.connect(path) + conn.execute( + "insert or replace into meta values ('refreshed_at', ?)", (value,) + ) + conn.commit() + conn.close() + + +def test_update_installs_published_pack_without_rebuilding( + patched_update, published_pack, monkeypatch +): + directory, _, keep = patched_update + + def no_rebuild(**kwargs): + raise AssertionError("published channel must not rebuild from NOAA") + + monkeypatch.setattr(registry_update, "refresh", no_rebuild) + + summary = registry_update.update() + + assert summary["channel"] == "published" + installed = os.path.join(directory, "ghcnh.db") + assert registry_update.refreshed_at(installed) is not None + conn = sqlite3.connect(installed) + n_stations = conn.execute("select count(*) from stations").fetchone()[0] + conn.close() + assert n_stations == len(keep) + assert not os.path.exists(installed + ".download") + + +def test_update_falls_back_to_rebuild_when_channel_unreachable( + patched_update, no_release_channel, live_files_from_mini_catalog +): + counts = registry_update.update() + + assert counts["inventory_rows"] == live_files_from_mini_catalog + + +def test_update_falls_back_when_published_pack_is_stale( + patched_update, published_pack, live_files_from_mini_catalog +): + _stamp(published_pack["ghcnh.db"], "2020-01-01") + + counts = registry_update.update() + + assert counts["inventory_rows"] == live_files_from_mini_catalog + + +def test_update_falls_back_when_published_pack_is_not_newer( + patched_update, published_pack, live_files_from_mini_catalog +): + _stamp( + published_pack["ghcnh.db"], + registry_update.refreshed_at().strftime("%Y-%m-%d"), + ) + + counts = registry_update.update() + + assert counts["inventory_rows"] == live_files_from_mini_catalog + + +def test_update_falls_back_when_published_pack_is_implausible( + patched_update, published_pack, live_files_from_mini_catalog +): + conn = sqlite3.connect(published_pack["ghcnh.db"]) + conn.execute( + "delete from stations where station_id not in" + " (select station_id from stations limit 5)" + ) + conn.commit() + conn.close() + + counts = registry_update.update() + + assert counts["inventory_rows"] == live_files_from_mini_catalog + + +def test_update_copies_then_refreshes_in_user_data_dir( + patched_update, no_release_channel, live_files_from_mini_catalog +): + directory, paths, keep = patched_update + + counts = registry_update.update() + + assert counts["stations_added"] == 0 + assert counts["inventory_rows"] == live_files_from_mini_catalog + updated_path = os.path.join(directory, "ghcnh.db") + updated = sqlite3.connect(updated_path) + n_rows = updated.execute("select count(*) from inventory").fetchone()[0] + updated.close() + assert n_rows == live_files_from_mini_catalog + # the refresh restamps the updated copy + assert registry_update.refreshed_at(updated_path) >= ( + registry_update.refreshed_at(paths["ghcnh.db"]) + ) + assert not os.path.exists(updated_path + ".staging") + # the mini "packaged" copies are untouched + packaged = sqlite3.connect(paths["ghcnh.db"]) + n_packaged = packaged.execute("select count(*) from inventory").fetchone()[0] + packaged.close() + assert n_packaged != n_rows + + +def test_data_path_prefers_updated_copy(patched_update): + directory, _, _ = patched_update + os.makedirs(directory, exist_ok=True) + override = os.path.join(directory, "ghcnh.db") + open(override, "w").close() + + # without the commit marker, an uncertified (possibly torn) copy is ignored + assert data_path("/packaged/ghcnh.db") == "/packaged/ghcnh.db" + + open(os.path.join(directory, registry_db.COMMIT_MARKER), "w").close() + assert data_path("/packaged/ghcnh.db") == override + # geography packs never live in the updated dir, so they always resolve + # to the packaged copy regardless of the marker + assert data_path("/packaged/geography_us.db") == "/packaged/geography_us.db" + + +def test_partial_updated_dir_without_marker_is_ignored(patched_update): + directory, paths, _ = patched_update + os.makedirs(directory, exist_ok=True) + for filename, packaged in paths.items(): + shutil.copy(packaged, os.path.join(directory, filename)) + + assert data_path(paths["ghcnh.db"]) == paths["ghcnh.db"] + assert data_path(paths["identifiers.db"]) == paths["identifiers.db"] + + +def test_completed_update_writes_marker_and_serves_updated_pair( + patched_update, published_pack, monkeypatch +): + directory, paths, _ = patched_update + + def no_rebuild(**kwargs): + raise AssertionError("published channel must not rebuild from NOAA") + + monkeypatch.setattr(registry_update, "refresh", no_rebuild) + + registry_update.update() + + assert os.path.exists(os.path.join(directory, registry_db.COMMIT_MARKER)) + assert data_path(paths["ghcnh.db"]) == os.path.join(directory, "ghcnh.db") + assert data_path(paths["identifiers.db"]) == os.path.join( + directory, "identifiers.db" + ) + + +def test_partial_swap_falls_back_then_next_update_heals( + patched_update, published_pack, monkeypatch +): + directory, paths, _ = patched_update + os.makedirs(directory, exist_ok=True) + now = datetime.now(timezone.utc) + + real_replace = os.replace + fail = {"active": True} + + def replace_failing_on_identifiers(src, dst): + if fail["active"] and os.path.basename(dst) == "identifiers.db": + raise OSError("disk full during swap") + + return real_replace(src, dst) + + monkeypatch.setattr( + registry_update.os, "replace", replace_failing_on_identifiers + ) + + assert registry_update._install_published(now) is None + # the second replace failed, so no marker was written and reads resolve + # to the internally consistent packaged pair (no torn ghcnh/identifiers mix) + assert not os.path.exists(os.path.join(directory, registry_db.COMMIT_MARKER)) + assert data_path(paths["ghcnh.db"]) == paths["ghcnh.db"] + assert data_path(paths["identifiers.db"]) == paths["identifiers.db"] + + fail["active"] = False + published = registry_update._install_published(now) + + assert published is not None + assert os.path.exists(os.path.join(directory, registry_db.COMMIT_MARKER)) + assert data_path(paths["ghcnh.db"]) == os.path.join(directory, "ghcnh.db") + assert data_path(paths["identifiers.db"]) == os.path.join( + directory, "identifiers.db" + ) + + +def test_reupdate_partial_swap_does_not_serve_torn_pair( + patched_update, published_pack, monkeypatch +): + directory, paths, _ = patched_update + os.makedirs(directory, exist_ok=True) + now = datetime.now(timezone.utc) + + # first update completes: marker present, updated pair served + assert registry_update._install_published(now) is not None + assert os.path.exists(os.path.join(directory, registry_db.COMMIT_MARKER)) + + # a re-update fails on the second file; the pre-existing marker must be + # retracted before the swap so the torn pair is never certified + real_replace = os.replace + + def replace_failing_on_identifiers(src, dst): + if os.path.basename(dst) == "identifiers.db": + raise OSError("disk full during re-update") + + return real_replace(src, dst) + + monkeypatch.setattr( + registry_update.os, "replace", replace_failing_on_identifiers + ) + + assert registry_update._install_published(now) is None + assert not os.path.exists(os.path.join(directory, registry_db.COMMIT_MARKER)) + assert data_path(paths["ghcnh.db"]) == paths["ghcnh.db"] + assert data_path(paths["identifiers.db"]) == paths["identifiers.db"] + + +def test_clear_removes_updated_copies(patched_update): + directory, paths, _ = patched_update + os.makedirs(directory, exist_ok=True) + for filename, packaged in paths.items(): + shutil.copy(packaged, os.path.join(directory, filename)) + + removed = registry_update.clear() + + assert len(removed) == 2 + assert os.listdir(directory) == [] + assert registry_update.clear() == [] + + +def test_implausible_upstream_aborts_and_preserves_data( + patched_update, no_release_channel, monkeypatch +): + directory, _, keep = patched_update + monkeypatch.setattr( + "eeweather.build.refresh._fetch_station_list", lambda: {} + ) + monkeypatch.setattr( + "eeweather.build.refresh._fetch_inventory", lambda: [] + ) + + with pytest.raises(RuntimeError, match="Implausible"): + registry_update.update() + + # the staged copy is discarded; no partial state lands in the user dir + assert not os.path.exists(os.path.join(directory, "ghcnh.db")) + packaged = sqlite3.connect(registry_update.UPDATABLE["ghcnh.db"]) + n_stations = packaged.execute("select count(*) from stations").fetchone()[0] + packaged.close() + assert n_stations == len(keep) + + +def test_refreshed_at_none_when_unstamped(tmp_path): + path = str(tmp_path / "unstamped.db") + sqlite3.connect(path).close() + + assert registry_update.refreshed_at(path) is None + + +def _db_with_meta_table(path): + conn = sqlite3.connect(path) + conn.execute("create table meta (key text primary key, value text)") + conn.commit() + conn.close() + + +def test_auto_updating_sources_excludes_meta_less_dbs(): + """identifiers.db is refreshed alongside ghcnh.db but carries no meta + table, and tmy3.db/cz2010.db carry none either; only ghcnh.db drives + staleness.""" + assert set(registry_update.AUTO_UPDATING_SOURCES) == {"ghcnh.db"} + + +def test_stamp_round_trips_through_refreshed_at_for_auto_updating_source(tmp_path): + path = str(tmp_path / "ghcnh.db") + _db_with_meta_table(path) + _stamp(path, "2026-01-15") + + assert registry_update.refreshed_at(path) == datetime( + 2026, 1, 15, tzinfo=timezone.utc + ) + + +def test_stale_false_when_auto_updating_source_freshly_stamped(tmp_path, monkeypatch): + path = str(tmp_path / "ghcnh.db") + _db_with_meta_table(path) + now = datetime.now(timezone.utc) + _stamp(path, now.strftime("%Y-%m-%d")) + monkeypatch.setattr(registry_update, "AUTO_UPDATING_SOURCES", {"ghcnh.db": path}) + + assert registry_update._stale(now) is False + + +def test_stale_true_when_auto_updating_source_stamp_too_old(tmp_path, monkeypatch): + path = str(tmp_path / "ghcnh.db") + _db_with_meta_table(path) + now = datetime.now(timezone.utc) + too_old = now - timedelta(days=registry_update.STALE_AFTER_DAYS + 1) + _stamp(path, too_old.strftime("%Y-%m-%d")) + monkeypatch.setattr(registry_update, "AUTO_UPDATING_SOURCES", {"ghcnh.db": path}) + + assert registry_update._stale(now) is True + + +def test_stale_true_when_auto_updating_source_unstamped(tmp_path, monkeypatch): + path = str(tmp_path / "ghcnh.db") + _db_with_meta_table(path) + monkeypatch.setattr(registry_update, "AUTO_UPDATING_SOURCES", {"ghcnh.db": path}) + + assert registry_update._stale(datetime.now(timezone.utc)) is True + + +def test_stale_ignores_source_db_without_meta_table(tmp_path, monkeypatch): + """A source db lacking a meta table (representing tmy3.db/cz2010.db) + is excluded from the auto-updating set, so its absence of a stamp + neither forces staleness nor raises.""" + ghcnh = str(tmp_path / "ghcnh.db") + _db_with_meta_table(ghcnh) + now = datetime.now(timezone.utc) + _stamp(ghcnh, now.strftime("%Y-%m-%d")) + no_meta = str(tmp_path / "tmy3.db") + sqlite3.connect(no_meta).close() + monkeypatch.setattr(registry_update, "AUTO_UPDATING_SOURCES", {"ghcnh.db": ghcnh}) + + assert registry_update._stale(now) is False + assert registry_update.refreshed_at(no_meta) is None + + +@pytest.fixture +def auto_update(patched_update, monkeypatch): + monkeypatch.setenv("EEWEATHER_AUTO_UPDATE", "1") + monkeypatch.setattr(registry_update, "_process_checked", False) + monkeypatch.setattr(registry_update, "UPDATE_DELAY_SECONDS", 0) + + return patched_update + + +def test_auto_update_disabled_by_env(patched_update, monkeypatch): + monkeypatch.setenv("EEWEATHER_AUTO_UPDATE", "0") + monkeypatch.setattr(registry_update, "_process_checked", False) + monkeypatch.setattr( + registry_update, "refreshed_at", + lambda path=None: datetime(2020, 1, 1, tzinfo=timezone.utc), + ) + + assert registry_update.maybe_update() is None + + +def test_auto_update_skips_fresh_data(auto_update, monkeypatch): + monkeypatch.setattr( + registry_update, "refreshed_at", + lambda path=None: datetime.now(timezone.utc), + ) + + assert registry_update.maybe_update() is None + + +def test_auto_update_checks_once_per_process(auto_update, monkeypatch): + calls = [] + monkeypatch.setattr( + registry_update, "refreshed_at", + lambda path=None: datetime(2020, 1, 1, tzinfo=timezone.utc), + ) + monkeypatch.setattr(registry_update, "update", lambda: calls.append(True)) + + first = registry_update.maybe_update() + first.join() + second = registry_update.maybe_update() + + assert calls == [True] + assert second is None + + +def test_auto_update_backs_off_after_recent_attempt(auto_update, monkeypatch): + calls = [] + monkeypatch.setattr( + registry_update, "refreshed_at", + lambda path=None: datetime(2020, 1, 1, tzinfo=timezone.utc), + ) + monkeypatch.setattr(registry_update, "update", lambda: calls.append(True)) + + registry_update.maybe_update().join() + # a fresh process (checked flag reset) still respects the attempt stamp + monkeypatch.setattr(registry_update, "_process_checked", False) + + assert registry_update.maybe_update() is None + assert calls == [True] + + +def test_auto_update_failure_warns_and_keeps_data(auto_update, monkeypatch): + def broken_update(): + raise RuntimeError("upstream offline") + + monkeypatch.setattr(registry_update, "update", broken_update) + + with pytest.warns(UserWarning, match="auto-update failed"): + registry_update._update_or_warn() + + +def test_claim_attempt_admits_exactly_one_racer(auto_update): + directory, _, _ = auto_update + os.makedirs(directory, exist_ok=True) + now = datetime.now(timezone.utc) + + claims = [registry_update._claim_attempt(now) for _ in range(5)] + + assert claims == [True, False, False, False, False] + + +def test_claim_attempt_reopens_after_retry_window(auto_update): + directory, _, _ = auto_update + os.makedirs(directory, exist_ok=True) + now = datetime.now(timezone.utc) + assert registry_update._claim_attempt(now) + stamp = os.path.join(directory, registry_update._ATTEMPT_STAMP) + two_days_ago = (now - timedelta(days=2)).timestamp() + os.utime(stamp, (two_days_ago, two_days_ago)) + + assert registry_update._claim_attempt(now) + assert not registry_update._claim_attempt(now) + + +def test_update_thread_defers_network_for_short_lived_processes( + auto_update, monkeypatch +): + order = [] + monkeypatch.setattr(registry_update, "UPDATE_DELAY_SECONDS", 0.01) + monkeypatch.setattr( + registry_update, "refreshed_at", + lambda path=None: datetime(2020, 1, 1, tzinfo=timezone.utc), + ) + monkeypatch.setattr( + registry_update.time, "sleep", lambda s: order.append(("sleep", s)) + ) + monkeypatch.setattr( + registry_update, "update", lambda: order.append(("update", None)) + ) + + registry_update.maybe_update().join() + + assert order == [("sleep", 0.01), ("update", None)] diff --git a/tests/registry/test_zones.py b/tests/registry/test_zones.py new file mode 100644 index 0000000..3a4bd2b --- /dev/null +++ b/tests/registry/test_zones.py @@ -0,0 +1,24 @@ +from eeweather.registry.zones import zones_at + + +def test_zones_at(): + zones = zones_at(35.1, -119.2) + + assert zones == { + "iecc_climate_zone": "3", + "iecc_moisture_regime": "B", + "ba_climate_zone": "Hot-Dry", + "ca_climate_zone": "CA_13", + } + + +def test_zones_at_point_outside_all_zones(): + zones = zones_at(0, 0) + + assert zones == { + "iecc_climate_zone": None, + "iecc_moisture_regime": None, + "ba_climate_zone": None, + "ca_climate_zone": None, + } + diff --git a/tests/sources/__snapshots__/test_matching.ambr b/tests/sources/__snapshots__/test_matching.ambr new file mode 100644 index 0000000..706c963 --- /dev/null +++ b/tests/sources/__snapshots__/test_matching.ambr @@ -0,0 +1,127 @@ +# serializer version: 1 +# name: test_rank_stations_has_sources[has_sources=cz2010] + tuple( + 86, + 15, + ) +# --- +# name: test_rank_stations_has_sources[has_sources=tmy3] + tuple( + 1015, + 15, + ) +# --- +# name: test_rank_stations_match_subdivision[match_subdivision without subdivision] + tuple( + 2922, + 15, + ) +# --- +# name: test_rank_stations_match_subdivision[match_subdivision=True] + tuple( + 202, + 15, + ) +# --- +# name: test_rank_stations_match_subdivision[site_subdivision only, no filter] + tuple( + 5891, + 15, + ) +# --- +# name: test_rank_stations_match_zones_not_null[match_zones=ba_climate_zone] + tuple( + 307, + 15, + ) +# --- +# name: test_rank_stations_match_zones_not_null[match_zones=ca_climate_zone] + tuple( + 15, + 15, + ) +# --- +# name: test_rank_stations_match_zones_not_null[match_zones=iecc_climate_zone] + tuple( + 954, + 15, + ) +# --- +# name: test_rank_stations_match_zones_not_null[match_zones=iecc_moisture_regime] + tuple( + 966, + 15, + ) +# --- +# name: test_rank_stations_match_zones_null[match_zones without site zones] + tuple( + 1320, + 15, + ) +# --- +# name: test_rank_stations_max_difference_elevation_meters[max_difference_elevation_meters=200] + tuple( + 5891, + 15, + ) +# --- +# name: test_rank_stations_max_difference_elevation_meters[site_elevation=0, max_difference_elevation_meters=200] + tuple( + 2901, + 15, + ) +# --- +# name: test_rank_stations_max_difference_elevation_meters[site_elevation=0, max_difference_elevation_meters=50] + tuple( + 1566, + 15, + ) +# --- +# name: test_rank_stations_max_difference_elevation_meters[site_elevation=1000, max_difference_elevation_meters=50] + tuple( + 70, + 15, + ) +# --- +# name: test_rank_stations_max_distance_meters[max_distance_meters=200000] + tuple( + 70, + 15, + ) +# --- +# name: test_rank_stations_max_distance_meters[max_distance_meters=50000] + tuple( + 6, + 15, + ) +# --- +# name: test_rank_stations_minimum_quality[minimum_quality=high] + tuple( + 2360, + 15, + ) +# --- +# name: test_rank_stations_minimum_quality[minimum_quality=low] + tuple( + 5891, + 15, + ) +# --- +# name: test_rank_stations_minimum_quality[minimum_quality=medium] + tuple( + 2861, + 15, + ) +# --- +# name: test_rank_stations_no_filter + tuple( + 5891, + 15, + ) +# --- +# name: test_select_station_with_empty_tempC + 'USW00023110' +# --- +# name: test_select_station_with_second_level_dates + 'USW00023110' +# --- diff --git a/tests/sources/test_engine.py b/tests/sources/test_engine.py new file mode 100644 index 0000000..4dae6fe --- /dev/null +++ b/tests/sources/test_engine.py @@ -0,0 +1,760 @@ +import contextlib +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pandas as pd +import pytest +import pytz + +from eeweather.exceptions import DataNotAvailableError +from eeweather.sources.engine import ( + _data_gap_warnings, + _datetime_is_utc, + _fetch_year, + _load_normals_block, + _load_observation_year, + _read_cached_year, + deserialize_hourly_data, + load_cached_data, + load_data, + normals_cache_key, + observation_cache_key, + Provenance, + serialize_hourly_data, +) +from eeweather.sources.ghcnh import GHCNhSource +from eeweather.sources.tmy3 import TMY3Source + + + +GHCNH = GHCNhSource() +TMY3 = TMY3Source() + + +def _backdate_cache_key(store, key, updated): + with contextlib.closing(sqlite3.connect(store._path)) as conn, conn: + conn.execute( + "update items set updated = ? where key = ?", (updated.isoformat(), key) + ) + + +# fetch + + +def test_fetch_year(mock_api_transport): + df = _fetch_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",)) + + assert list(df.columns) == ["temperature"] + assert df.shape == (8760, 1) + assert df.index[0] == datetime(2007, 1, 1, tzinfo=pytz.UTC) + assert df.temperature.sum() == pytest.approx(156159.5455, abs=1e-3) + + +def test_fetch_year_multiple_variables(mock_api_transport): + df = _fetch_year( + GHCNH, "USW00093134", "USW00093134", 2007, + ("temperature", "relative_humidity"), + ) + + assert list(df.columns) == ["temperature", "relative_humidity"] + assert df.shape == (8760, 2) + + +def test_fetch_year_missing_year_raises(mock_api_transport): + with pytest.raises(DataNotAvailableError) as excinfo: + _fetch_year(GHCNH, "USW00093134", "USW00093134", 1800, ("temperature",)) + assert excinfo.value.source == "ghcnh" + assert excinfo.value.year == 1800 + + +# cache keys + + +def test_observation_cache_key(): + key = observation_cache_key("ghcnh", "USW00093134", 2007) + + assert key == "ghcnh-hourly-USW00093134-2007" + + +def test_normals_cache_key(): + assert normals_cache_key("tmy3", "USW00023152") == "tmy3-hourly-USW00023152" + + +# cache freshness + + +def test_read_cached_year_empty(monkeypatch_key_value_store): + assert _read_cached_year(GHCNH, "USW00093134", 2007) is None + + +def test_read_cached_year_fresh(mock_api_transport, monkeypatch_key_value_store): + _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, True) + + assert _read_cached_year(GHCNH, "USW00093134", 2007) is not None + + +def test_read_cached_year_expired_entry_is_cleared( + mock_api_transport, monkeypatch_key_value_store +): + _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, True) + + # a cache entry written during its own data year goes stale + key = observation_cache_key("ghcnh", "USW00093134", 2007) + _backdate_cache_key( + monkeypatch_key_value_store, key, pytz.UTC.localize(datetime(2007, 3, 3)) + ) + + assert _read_cached_year(GHCNH, "USW00093134", 2007) is None + assert monkeypatch_key_value_store.key_exists(key) is False + + +def test_load_observation_year_no_cache_no_web_raises(monkeypatch_key_value_store): + with pytest.raises(DataNotAvailableError): + _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, False) + + +# serialization round-trips + + +def test_serialize_deserialize_hourly_data_round_trip(mock_api_transport): + df = _fetch_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",)) + + serialized = serialize_hourly_data(df) + + assert serialized["columns"] == ["temperature"] + assert serialized["rows"][0][0] == "2007010100" + assert len(serialized["rows"]) == len(df) + + round_tripped = deserialize_hourly_data(serialized) + + pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) + + +def test_serialize_hourly_data_nan_round_trips_as_null(mock_api_transport): + df = _fetch_year(GHCNH, "USW00093194", "USW00093194", 2013, ("temperature",)) # ends 2013-11-04 + + serialized = serialize_hourly_data(df) + + assert any(row[1] is None for row in serialized["rows"]) + + round_tripped = deserialize_hourly_data(serialized) + + pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) + + +def test_serialize_multivariable_round_trip(mock_api_transport): + df = _fetch_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature", "wind_speed")) + + round_tripped = deserialize_hourly_data(serialize_hourly_data(df)) + + pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) + + +def test_normals_block_round_trips_through_the_hourly_serializer( + monkeypatch_tmy3_request, monkeypatch_key_value_store +): + fresh = _load_normals_block(TMY3, "USW00023152", True, True, True) + cached = _load_normals_block(TMY3, "USW00023152", True, True, True) + + pd.testing.assert_series_equal( + cached, fresh, check_freq=False, check_names=False + ) + + +# cached proxy behavior + + +def test_load_observation_year_serves_from_cache( + mock_api_transport, monkeypatch_key_value_store +): + df1 = _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, True) + df2 = _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, True) + + pd.testing.assert_frame_equal(df1, df2, check_freq=False) + + +def test_load_observation_year_variable_superset_refetches( + mock_api_transport, monkeypatch_key_value_store +): + df1 = _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, True) + assert list(df1.columns) == ["temperature"] + + # cache holds temperature only, so requesting more refetches the union + df2 = _load_observation_year( + GHCNH, "USW00093134", "USW00093134", 2007, ("temperature", "wind_speed"), + True, True, True, + ) + assert list(df2.columns) == ["temperature", "wind_speed"] + + # the refreshed cache entry now covers both variables + cached = _read_cached_year(GHCNH, "USW00093134", 2007) + assert set(cached.columns) == {"temperature", "wind_speed"} + + # a temperature-only request serves the requested subset from cache + df3 = _load_observation_year(GHCNH, "USW00093134", "USW00093134", 2007, ("temperature",), + True, True, True) + assert list(df3.columns) == ["temperature"] + + +def test_load_normals_block_cached( + monkeypatch_tmy3_request, monkeypatch_key_value_store +): + ts1 = _load_normals_block(TMY3, "USW00023152", True, True, True) + ts2 = _load_normals_block(TMY3, "USW00023152", True, True, True) + + assert int(ts1.sum()) == int(ts2.sum()) == 156194 + assert monkeypatch_key_value_store.key_exists("tmy3-hourly-USW00023152") + + +def test_load_normals_block_no_cache_no_web_raises(monkeypatch_key_value_store): + with pytest.raises(DataNotAvailableError): + _load_normals_block(TMY3, "USW00023152", True, True, False) + + +# load_data: hourly and daily observed + + +def test_load_data_hourly(mock_api_transport, monkeypatch_key_value_store): + start = datetime(2006, 1, 3, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + assert df.index[0] == start + assert df.index[-1] == end + assert len(df) == 10921 + assert int(df.temperature.notna().sum()) == 10893 + assert warnings == [] + + +def test_load_data_hourly_non_normalized_dates( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2006, 1, 3, 11, 12, 13, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, 12, 13, 14, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + assert df.index[0] == datetime(2006, 1, 3, 12, tzinfo=pytz.UTC) + assert df.index[-1] == datetime(2007, 4, 3, 12, tzinfo=pytz.UTC) + + +def test_load_data_daily(mock_api_transport, monkeypatch_key_value_store): + start = datetime(2006, 1, 3, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end, frequency="D") + + assert df.index[0] == start + assert df.index[-1] == end + assert len(df) == 456 + assert int(df.temperature.notna().sum()) == 456 + assert warnings == [] + + +def test_load_data_daily_non_normalized_dates( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2006, 1, 3, 11, 12, 13, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, 12, 13, 14, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end, frequency="D") + + assert df.index[0] == datetime(2006, 1, 4, tzinfo=pytz.UTC) + assert df.index[-1] == datetime(2007, 4, 3, tzinfo=pytz.UTC) + + +def test_load_data_invalid_frequency(monkeypatch_key_value_store): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + with pytest.raises(ValueError, match="frequency"): + load_data("USW00093134", start, end, frequency="fortnight") + + +def test_load_data_non_utc_datetimes_raise(): + with pytest.raises(ValueError, match="UTC"): + load_data( + "USW00093134", + datetime(2007, 1, 1), + datetime(2007, 4, 3, tzinfo=pytz.UTC), + ) + + with pytest.raises(ValueError, match="UTC"): + load_data( + "USW00093134", + datetime(2007, 1, 1, tzinfo=pytz.UTC), + datetime(2007, 4, 3), + ) + + +def test_load_data_multiple_variables(mock_api_transport, monkeypatch_key_value_store): + start = datetime(2007, 6, 1, tzinfo=pytz.UTC) + end = datetime(2007, 6, 30, tzinfo=pytz.UTC) + + df, warnings = load_data( + "USW00093134", + start, + end, + variables=("temperature", "relative_humidity", "wind_speed"), + ) + + assert df.shape == (697, 3) + assert df.temperature.mean() == pytest.approx(19.304219, abs=1e-5) + assert df.relative_humidity.mean() == pytest.approx(67.903771, abs=1e-5) + assert df.wind_speed.mean() == pytest.approx(0.843799, abs=1e-5) + assert warnings == [] + + +def test_load_data_variables_all_is_source_union( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2007, 6, 1, tzinfo=pytz.UTC) + end = datetime(2007, 6, 2, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end, variables="all") + + assert list(df.columns) == list(GHCNH.variables) + + +def test_load_data_unroutable_variable_raises(): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + with pytest.raises(ValueError, match="Unknown variables"): + load_data("USW00093134", start, end, variables=("not_a_variable",)) + + +def test_load_data_pinned_source_never_servable_variable_raises(): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + with pytest.raises(ValueError, match="does not serve"): + load_data( + "USW00093134", start, end, source="tmy3", variables=("wind_speed",) + ) + + +# provenance + + +def test_load_data_provenance_on_frame( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + record = df.attrs["provenance"]["ghcnh"] + assert record.kind == "observations" + assert record.source == "ghcnh" + assert record.station_id == "USW00093134" + assert record.distance_meters is None + assert record.variables == ("temperature",) + + +def test_provenance_station_fields_and_payload_are_optional(): + record = Provenance(kind="observations", source="ghcnh", variables=("temperature",)) + + assert record.station_id is None + assert record.distance_meters is None + assert record.payload == {} + + with_payload = record._replace(payload={"model_version": "v1"}) + + assert with_payload.payload == {"model_version": "v1"} + + +def test_load_data_pinned_normals_provenance( + monkeypatch_tmy3_request, monkeypatch_key_value_store +): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00023152", start, end, source="tmy3") + + record = df.attrs["provenance"]["tmy3"] + assert record.kind == "normals" + assert record.variables == ("temperature",) + + +# regression pins on real captured 2007 data + + +def test_load_data_hourly_2007_regression_values( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 12, 31, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + assert len(df) == 8737 + assert int(df.temperature.notna().sum()) == 8732 + assert df.temperature.mean() == pytest.approx(17.851812, abs=1e-5) + assert warnings == [] + + +def test_load_data_daily_2007_regression_values( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 12, 31, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end, frequency="D") + + assert len(df) == 365 + assert int(df.temperature.notna().sum()) == 365 + assert df.temperature.mean() == pytest.approx(17.835431, abs=1e-5) + assert df.temperature.iloc[0] == pytest.approx(13.27734, abs=1e-5) + + +# typical-year loads through the one verb + + +def test_load_data_tmy3_pinned(monkeypatch_tmy3_request, monkeypatch_key_value_store): + start = datetime(2006, 1, 3, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00023152", start, end, source="tmy3") + + assert df.index[0] == start + assert pd.notnull(df.temperature.iloc[0]) + assert df.index[-1] == end + assert pd.notnull(df.temperature.iloc[-1]) + assert warnings == [] + + +def test_load_data_cz2010_pinned( + monkeypatch_cz2010_request, monkeypatch_key_value_store +): + start = datetime(2006, 1, 3, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00023152", start, end, source="cz2010") + + assert df.index[0] == start + assert df.index[-1] == end + assert int(df.temperature.notna().sum()) == len(df) + + +def test_load_data_normals_tile_by_month_day_hour_with_nan_leap_day( + monkeypatch_tmy3_request, monkeypatch_key_value_store +): + start = datetime(2015, 2, 15, tzinfo=pytz.UTC) + end = datetime(2016, 8, 12, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00023152", start, end, source="tmy3") + ts_orig = TMY3.fetch("USW00023152") + + for i in df.index: + if i.month == 2 and i.day == 29: + assert pd.isnull(df.temperature[i]) + else: + assert df.temperature[i] == ts_orig[i.replace(year=1900)] + + +def test_load_data_normals_excluded_from_routing( + mock_api_transport, monkeypatch_tmy3_request, monkeypatch_key_value_store +): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + # tmy3 leads the preference tuple but normals never route; temperature + # must come from ghcnh + df, warnings = load_data( + "USW00093134", start, end, sources=("tmy3", "ghcnh") + ) + + assert list(df.attrs["provenance"]) == ["ghcnh"] + assert df.attrs["provenance"]["ghcnh"].kind == "observations" + + +# warnings + + +def test_load_data_warns_on_truncated_data( + mock_api_transport, monkeypatch_key_value_store +): + # station 723826 was decommissioned 2013-11-04 + start = datetime(2013, 6, 1, tzinfo=pytz.UTC) + end = datetime(2013, 12, 31, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093194", start, end) + + assert len(df) == 5113 + assert int(df.temperature.notna().sum()) == 1619 + assert df.temperature.last_valid_index() == datetime( + 2013, 11, 4, 19, tzinfo=pytz.UTC + ) + assert [w.qualified_name for w in warnings] == ["eeweather.data_truncated"] + assert warnings[0].data["variable"] == "temperature" + assert warnings[0].data["source"] == "ghcnh" + assert warnings[0].data["last_valid"] == "2013-11-04T19:00:00+00:00" + + +def test_load_data_daily_warns_on_truncated_data( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2013, 6, 1, tzinfo=pytz.UTC) + end = datetime(2013, 12, 31, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093194", start, end, frequency="D") + + assert len(df) == 214 + assert int(df.temperature.notna().sum()) == 156 + assert [w.qualified_name for w in warnings] == ["eeweather.data_truncated"] + + +def test_load_data_warns_on_internal_gap( + mock_api_transport, monkeypatch_key_value_store +): + # station 720193's 2019 data has a real mid-year outage + start = datetime(2019, 1, 1, tzinfo=pytz.UTC) + end = datetime(2019, 12, 31, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00000117", start, end) + + assert len(df) == 8737 + assert int(df.temperature.notna().sum()) == 8106 + assert [w.qualified_name for w in warnings] == ["eeweather.data_gap"] + assert warnings[0].data["max_gap_days"] == pytest.approx(16.125, abs=1e-9) + + +def test_data_gap_warnings_leading_gap(): + # synthetic edge case: a series whose data begins three days late + index = pd.date_range( + "2020-01-01", "2020-01-31", freq="h", tz="UTC" + ) + ts = pd.Series(20.0, index=index) + ts.iloc[: 24 * 3] = float("nan") + + warnings = _data_gap_warnings(ts, "ghcnh", "temperature") + + assert [w.qualified_name for w in warnings] == ["eeweather.data_starts_late"] + assert warnings[0].data["requested_start"] == "2020-01-01T00:00:00+00:00" + assert warnings[0].data["first_valid"] == "2020-01-04T00:00:00+00:00" + + +def test_data_gap_warnings_empty_series_is_silent(): + ts = pd.Series([], dtype=float, index=pd.DatetimeIndex([], tz="UTC")) + + assert _data_gap_warnings(ts, "ghcnh", "temperature") == [] + + +def test_load_data_2025_extends_past_isd_end_of_life( + mock_api_transport, monkeypatch_key_value_store +): + # GHCNh continues past the ISD end-of-life: station 724940's 2025 data + # runs through the year while its ISD record stopped 2025-08-27 + start = datetime(2025, 6, 1, tzinfo=pytz.UTC) + end = datetime(2025, 10, 1, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00023234", start, end) + + assert len(df) == 2929 + assert int(df.temperature.notna().sum()) == 2840 + assert df.temperature.last_valid_index() == end + assert warnings == [] + + +# missing data semantics: routed loads warn and return NaN; pinned loads +# with nothing at all raise + + +def test_load_data_routed_missing_year_returns_nan_range( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2050, 1, 1, tzinfo=pytz.UTC) + end = datetime(2050, 6, 1, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + assert df.index[0] == start + assert df.index[-1] == end + assert df.temperature.isna().all() + assert [w.qualified_name for w in warnings] == [ + "eeweather.data_not_available", + "eeweather.no_data_in_requested_range", + ] + assert warnings[0].data["station_id"] == "USW00093134" + + +def test_load_data_pinned_source_with_no_data_at_all_raises( + mock_api_transport, monkeypatch_key_value_store +): + start = datetime(2050, 1, 1, tzinfo=pytz.UTC) + end = datetime(2050, 6, 1, tzinfo=pytz.UTC) + + with pytest.raises(DataNotAvailableError) as excinfo: + load_data("USW00093134", start, end, source="ghcnh") + assert excinfo.value.source == "ghcnh" + assert excinfo.value.station_id == "USW00093134" + assert excinfo.value.year is None + + +def test_load_data_year_with_no_observations( + mock_api_transport, monkeypatch_key_value_store +): + # station 722874 has no GHCNh observations at all in 2025 + start = datetime(2025, 1, 1, tzinfo=pytz.UTC) + end = datetime(2025, 6, 1, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + assert len(df) == 3625 + assert df.temperature.isna().all() + assert [w.qualified_name for w in warnings] == [ + "eeweather.data_not_available", + "eeweather.no_data_in_requested_range", + ] + + +def test_load_data_sub_hour_range_returns_empty( + mock_api_transport, monkeypatch_key_value_store +): + # a sub-hour range contains no aligned hours; returns empty without warning + start = datetime(2007, 6, 1, 12, 30, tzinfo=pytz.UTC) + end = datetime(2007, 6, 1, 12, 45, tzinfo=pytz.UTC) + + df, warnings = load_data("USW00093134", start, end) + + assert len(df) == 0 + assert warnings == [] + + +# join-safety guarantee: same range and frequency means identical indexes +# across sources + + +def test_frames_from_different_sources_share_an_index( + mock_api_transport, monkeypatch_tmy3_request, monkeypatch_key_value_store +): + start = datetime(2007, 1, 1, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + observed, _ = load_data("USW00093134", start, end) + typical, _ = load_data("USW00023152", start, end, source="tmy3") + + assert observed.index.equals(typical.index) + + joined = observed.join(typical, rsuffix="_typical") + + assert list(joined.columns) == ["temperature", "temperature_typical"] + assert len(joined) == len(observed) + + +# cached data access + + +def test_load_cached_data(mock_api_transport, monkeypatch_key_value_store): + assert load_cached_data("USW00093134") is None + + load_data( + "USW00093134", + datetime(2007, 1, 1, tzinfo=pytz.UTC), + datetime(2007, 4, 3, tzinfo=pytz.UTC), + ) + cached = load_cached_data("USW00093134") + + assert cached is not None + assert list(cached.columns) == ["temperature"] + # the cache holds the full fetched year, not just the requested slice + assert len(cached) == 8760 + +# request-range validation + + +def test_datetime_is_utc_accepts_utc(): + assert _datetime_is_utc(datetime(2020, 1, 1, tzinfo=pytz.UTC)) is True + assert _datetime_is_utc(datetime(2020, 1, 1, tzinfo=timezone.utc)) is True + + +def test_datetime_is_utc_rejects_naive(): + assert _datetime_is_utc(datetime(2020, 1, 1)) is False + + +def test_datetime_is_utc_rejects_offsets(): + assert _datetime_is_utc( + datetime(2020, 1, 1, tzinfo=timezone(timedelta(hours=5))) + ) is False + assert _datetime_is_utc( + datetime(2020, 1, 1, tzinfo=timezone(timedelta(hours=-8))) + ) is False + + +def test_load_data_pinned_raises_when_requested_dates_are_empty( + mock_api_transport, monkeypatch_key_value_store +): + # station 723826 was decommissioned 2013-11-04: its calendar year has + # data, the requested December does not + start = datetime(2013, 12, 1, tzinfo=pytz.UTC) + end = datetime(2013, 12, 31, tzinfo=pytz.UTC) + + with pytest.raises(DataNotAvailableError) as excinfo: + load_data("USW00093194", start, end, source="ghcnh") + assert excinfo.value.source == "ghcnh" + + # the routed equivalent warns and returns NaN + df, warnings = load_data("USW00093194", start, end) + assert df.temperature.isna().all() + assert len(warnings) > 0 + + +def test_load_data_start_after_end_raises(): + with pytest.raises(ValueError, match="start must not be after end"): + load_data( + "USW00093134", + datetime(2007, 6, 1, tzinfo=pytz.UTC), + datetime(2007, 1, 1, tzinfo=pytz.UTC), + ) + + +def test_load_data_duplicate_variables_raise(): + with pytest.raises(ValueError, match="Duplicate variables"): + load_data( + "USW00093134", + datetime(2007, 1, 1, tzinfo=pytz.UTC), + datetime(2007, 2, 1, tzinfo=pytz.UTC), + variables=("temperature", "temperature"), + ) + + +def test_load_data_empty_variables_raise(): + with pytest.raises(ValueError, match="At least one variable"): + load_data( + "USW00093134", + datetime(2007, 1, 1, tzinfo=pytz.UTC), + datetime(2007, 2, 1, tzinfo=pytz.UTC), + variables=(), + ) + + +def test_load_data_untranslatable_station_warns_once( + mock_api_transport, monkeypatch_key_value_store +): + # a fixture feed keyed by usaf, asked about a station with no usaf id, + # warns once for the station-level condition, not once per year + class UsafFeed(GHCNhSource): + name = "usaf-feed" + id_namespace = "usaf" + cacheable = False + + # ASN00026044's usaf alias was removed in the alias audit + df, warnings = load_data( + "ASN00026044", + datetime(2019, 1, 1, tzinfo=pytz.UTC), + datetime(2021, 12, 31, tzinfo=pytz.UTC), + sources=(UsafFeed(),), + ) + + assert df.temperature.isna().all() + names = [w.qualified_name for w in warnings] + assert names.count("eeweather.data_not_available") == 1 diff --git a/tests/sources/test_ghcnh.py b/tests/sources/test_ghcnh.py new file mode 100644 index 0000000..ea0697a --- /dev/null +++ b/tests/sources/test_ghcnh.py @@ -0,0 +1,184 @@ +import gzip +from pathlib import Path + +import pytest +import requests + +from eeweather.sources.ghcnh import GHCNhSource +from eeweather.sources.ghcnh.source import API_REQUEST_TRIES + + + +FIXTURE_DIR = Path(__file__).parent.parent / "fixtures" + + +def _fixture_text(name): + with gzip.open(FIXTURE_DIR / name, "rb") as f: + return f.read().decode() + + +class MockResponse: + def __init__(self, text, status_code=200): + self.text = text + self.status_code = status_code + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError( + "{} error".format(self.status_code), response=self + ) + + +@pytest.fixture +def no_sleep(monkeypatch): + sleeps = [] + monkeypatch.setattr("eeweather.sources.ghcnh.source.time.sleep", sleeps.append) + + return sleeps + + +def test_fetch_year_request_params_include_date(monkeypatch): + seen = {} + + def mock_get(url, params=None): + seen.update(params) + + return MockResponse("") + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", mock_get) + + GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) + + assert seen["dataTypes"] == "DATE,temperature" + assert seen["stations"] == "USW00093134" + assert seen["startDate"] == "2007-01-01" + assert seen["endDate"] == "2007-12-31" + + +def test_fetch_year_retries_connection_errors_with_backoff(monkeypatch, no_sleep): + calls = [] + + def failing_get(url, params=None): + calls.append(url) + raise requests.ConnectionError("refused") + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", failing_get) + + with pytest.raises(requests.ConnectionError): + GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) + + assert len(calls) == API_REQUEST_TRIES + # backs off between attempts, not after the final failure + assert len(no_sleep) == API_REQUEST_TRIES - 1 + assert no_sleep[0] < no_sleep[1] + + +def test_fetch_year_retries_server_errors(monkeypatch, no_sleep): + payload = _fixture_text( + "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz" + ) + calls = [] + + def flaky_get(url, params=None): + calls.append(url) + if len(calls) == 1: + return MockResponse("oops", status_code=502) + + return MockResponse(payload) + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", flaky_get) + + df = GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) + + assert len(calls) == 2 + assert len(df) == 10884 + + +def test_fetch_year_client_errors_raise_immediately(monkeypatch, no_sleep): + calls = [] + + def not_found_get(url, params=None): + calls.append(url) + + return MockResponse("nope", status_code=404) + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", not_found_get) + + with pytest.raises(requests.HTTPError): + GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) + + assert len(calls) == 1 + assert no_sleep == [] + + +def test_fetch_year_succeeds_after_transient_failure(monkeypatch, no_sleep): + payload = _fixture_text( + "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz" + ) + calls = [] + + def flaky_get(url, params=None): + calls.append(url) + if len(calls) == 1: + raise requests.ConnectionError("refused") + + return MockResponse(payload) + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", flaky_get) + + df = GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) + + assert len(calls) == 2 + assert len(df) == 10884 + # first captured 2007 observation is 15.0 C at 00:47 + assert df.temperature.iloc[0] == pytest.approx(15.0, abs=1e-9) + + +def test_fetch_year_empty_year_returns_empty_frame(monkeypatch): + def mock_get(url, params=None): + return MockResponse("") + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", mock_get) + + df = GHCNhSource().fetch_year("USW00093134", 1800, ("temperature",)) + + assert len(df) == 0 + assert list(df.columns) == ["temperature"] + assert str(df.index.tz) == "UTC" + + +def test_fetch_year_unreported_variable_is_nan_column(monkeypatch): + payload = _fixture_text( + "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz" + ) + + def mock_get(url, params=None): + return MockResponse(payload) + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", mock_get) + + df = GHCNhSource().fetch_year( + "USW00093134", 2007, ("temperature", "visibility") + ) + + assert list(df.columns) == ["temperature", "visibility"] + assert df.visibility.isna().all() + assert df.temperature.notna().sum() > 0 + + +def test_fetch_year_averages_duplicate_timestamps(mock_api_transport): + df = GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) + + assert df.index.is_unique + assert df.index.is_monotonic_increasing + + +def test_malformed_payload_raises_meaningful_error(monkeypatch): + # NCEI's under-load failure mode: an error body with HTTP 200 + def mock_get(url, params=None): + return MockResponse("\nservice overloaded\n") + + monkeypatch.setattr("eeweather.sources.ghcnh.source._get", mock_get) + + with pytest.raises(ValueError, match="Malformed GHCNh response"): + GHCNhSource().fetch_year("USW00093134", 2007, ("temperature",)) diff --git a/tests/test_ranking.py b/tests/sources/test_matching.py similarity index 55% rename from tests/test_ranking.py rename to tests/sources/test_matching.py index d831a79..6a05f27 100644 --- a/tests/test_ranking.py +++ b/tests/sources/test_matching.py @@ -1,29 +1,16 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" from datetime import datetime + import pandas as pd import pytest import pytz -from eeweather import rank_stations, combine_ranked_stations, select_station from eeweather.exceptions import DataNotAvailableError +from eeweather.sources.matching import ( + combine_ranked_stations, + rank_stations, + select_station, +) + @pytest.fixture @@ -49,102 +36,105 @@ def test_rank_stations_no_filter(lat_long_fresno, snapshot): "iecc_moisture_regime", "ba_climate_zone", "ca_climate_zone", - "rough_quality", + "quality", "elevation", - "state", + "subdivision", + "is_cz2010", "tmy3_class", "is_tmy3", - "is_cz2010", "difference_elevation_meters", ] assert round(df.distance_meters.iloc[0]) == 2723 - assert round(df.distance_meters.iloc[-10]) == 15057891 + assert round(df.distance_meters.iloc[-10]) == 15046116 # every station in the registry has coordinates, so every distance is real assert pd.notnull(df.distance_meters.iloc[-1]) -def test_rank_stations_match_climate_zones_not_null(lat_long_fresno, snapshot): +def test_rank_stations_match_zones_not_null(lat_long_fresno, snapshot): lat, lng = lat_long_fresno - df = rank_stations(lat, lng, match_iecc_climate_zone=True) - assert df.shape == snapshot(name="match_iecc_climate_zone") - - df = rank_stations(lat, lng, match_iecc_moisture_regime=True) - assert df.shape == snapshot(name="match_iecc_moisture_regime") - - df = rank_stations(lat, lng, match_ba_climate_zone=True) - assert df.shape == snapshot(name="match_ba_climate_zone") - - df = rank_stations(lat, lng, match_ca_climate_zone=True) - assert df.shape == snapshot(name="match_ca_climate_zone") + for system in ( + "iecc_climate_zone", + "iecc_moisture_regime", + "ba_climate_zone", + "ca_climate_zone", + ): + df = rank_stations(lat, lng, match_zones=(system,)) + assert df.shape == snapshot(name="match_zones={}".format(system)) -def test_rank_stations_match_climate_zones_null(lat_long_africa, snapshot): +def test_rank_stations_match_zones_null(lat_long_africa, snapshot): + # the site has no zones, so matching candidates have none either lat, lng = lat_long_africa - df = rank_stations(lat, lng, match_iecc_climate_zone=True) - assert df.shape == snapshot(name="match_iecc_climate_zone") - - df = rank_stations(lat, lng, match_iecc_moisture_regime=True) - assert df.shape == snapshot(name="match_iecc_moisture_regime") + df = rank_stations( + lat, lng, match_zones=("iecc_climate_zone", "iecc_moisture_regime") + ) + assert df.shape == snapshot(name="match_zones without site zones") + assert df.iecc_climate_zone.isnull().all() - df = rank_stations(lat, lng, match_ba_climate_zone=True) - assert df.shape == snapshot(name="match_ba_climate_zone") - df = rank_stations(lat, lng, match_ca_climate_zone=True) - assert df.shape == snapshot(name="match_ca_climate_zone") +def test_rank_stations_match_zones_unknown_system(lat_long_fresno): + lat, lng = lat_long_fresno + with pytest.raises(ValueError): + rank_stations(lat, lng, match_zones=("not_a_system",)) -def test_rank_stations_match_state(lat_long_fresno, snapshot): +def test_rank_stations_match_subdivision(lat_long_fresno, snapshot): lat, lng = lat_long_fresno - df = rank_stations(lat, lng, site_state="CA") - assert df.shape == snapshot(name="site_state=CA, match_state=False") + df = rank_stations(lat, lng, site_subdivision="CA") + assert df.shape == snapshot(name="site_subdivision only, no filter") - df = rank_stations(lat, lng, site_state="CA", match_state=True) - assert df.shape == snapshot(name="site_state=CA, match_state=True") + df = rank_stations( + lat, lng, site_subdivision="CA", match_subdivision=True + ) + assert df.shape == snapshot(name="match_subdivision=True") + assert (df.subdivision == "CA").all() - df = rank_stations(lat, lng, site_state=None, match_state=True) - assert df.shape == snapshot(name="site_state=None, match_state=True") + df = rank_stations(lat, lng, site_subdivision=None, match_subdivision=True) + assert df.shape == snapshot(name="match_subdivision without subdivision") + assert df.subdivision.isnull().all() -def test_rank_stations_is_tmy3(lat_long_fresno, snapshot): +def test_rank_stations_has_sources(lat_long_fresno, snapshot): lat, lng = lat_long_fresno - df = rank_stations(lat, lng, is_tmy3=True) - assert df.shape == snapshot(name="is_tmy3=True") + df = rank_stations(lat, lng, has_sources=("tmy3",)) + assert df.shape == snapshot(name="has_sources=tmy3") + assert df.is_tmy3.all() - df = rank_stations(lat, lng, is_tmy3=False) - assert df.shape == snapshot(name="is_tmy3=False") + df = rank_stations(lat, lng, has_sources=("cz2010",)) + assert df.shape == snapshot(name="has_sources=cz2010") + assert df.is_cz2010.all() + # every registry station serves ghcnh, so this filters nothing + df_ghcnh = rank_stations(lat, lng, has_sources=("ghcnh",)) + df_all = rank_stations(lat, lng) + assert df_ghcnh.shape == df_all.shape -def test_rank_stations_is_cz2010(lat_long_fresno, snapshot): - lat, lng = lat_long_fresno - df = rank_stations(lat, lng, is_cz2010=True) - assert df.shape == snapshot(name="is_cz2010=True") - df = rank_stations(lat, lng, is_cz2010=False) - assert df.shape == snapshot(name="is_cz2010=False") +def test_rank_stations_has_sources_unknown_source(lat_long_fresno): + lat, lng = lat_long_fresno + with pytest.raises(ValueError): + rank_stations(lat, lng, has_sources=("not_a_source",)) def test_rank_stations_minimum_quality(lat_long_fresno, snapshot): lat, lng = lat_long_fresno df = rank_stations(lat, lng, minimum_quality="low") assert df.shape == snapshot(name="minimum_quality=low") + assert df.quality.isin(("low", "medium", "high")).all() df = rank_stations(lat, lng, minimum_quality="medium") assert df.shape == snapshot(name="minimum_quality=medium") + assert df.quality.isin(("medium", "high")).all() df = rank_stations(lat, lng, minimum_quality="high") assert df.shape == snapshot(name="minimum_quality=high") + assert (df.quality == "high").all() -def test_rank_stations_minimum_tmy3_class(lat_long_fresno, snapshot): +def test_rank_stations_minimum_quality_unknown_value(lat_long_fresno): lat, lng = lat_long_fresno - df = rank_stations(lat, lng, minimum_tmy3_class="III") - assert df.shape == snapshot(name="minimum_tmy3_class=III") - - df = rank_stations(lat, lng, minimum_tmy3_class="II") - assert df.shape == snapshot(name="minimum_tmy3_class=II") - - df = rank_stations(lat, lng, minimum_tmy3_class="I") - assert df.shape == snapshot(name="minimum_tmy3_class=I") + with pytest.raises(ValueError): + rank_stations(lat, lng, minimum_quality="excellent") def test_rank_stations_max_distance_meters(lat_long_fresno, snapshot): @@ -152,15 +142,17 @@ def test_rank_stations_max_distance_meters(lat_long_fresno, snapshot): df = rank_stations(lat, lng, max_distance_meters=200000) assert df.shape == snapshot(name="max_distance_meters=200000") + assert (df.distance_meters <= 200000).all() df = rank_stations(lat, lng, max_distance_meters=50000) assert df.shape == snapshot(name="max_distance_meters=50000") + assert (df.distance_meters <= 50000).all() def test_rank_stations_max_difference_elevation_meters(lat_long_fresno, snapshot): lat, lng = lat_long_fresno - # no site_elevation + # no site_elevation, so the filter is inert df = rank_stations(lat, lng, max_difference_elevation_meters=200) assert df.shape == snapshot(name="max_difference_elevation_meters=200") @@ -168,11 +160,13 @@ def test_rank_stations_max_difference_elevation_meters(lat_long_fresno, snapshot assert df.shape == snapshot( name="site_elevation=0, max_difference_elevation_meters=200" ) + assert (df.difference_elevation_meters <= 200).all() df = rank_stations(lat, lng, site_elevation=0, max_difference_elevation_meters=50) assert df.shape == snapshot( name="site_elevation=0, max_difference_elevation_meters=50" ) + assert (df.difference_elevation_meters <= 50).all() df = rank_stations( lat, lng, site_elevation=1000, max_difference_elevation_meters=50 @@ -180,31 +174,37 @@ def test_rank_stations_max_difference_elevation_meters(lat_long_fresno, snapshot assert df.shape == snapshot( name="site_elevation=1000, max_difference_elevation_meters=50" ) + assert (df.difference_elevation_meters <= 50).all() @pytest.fixture def cz_candidates(lat_long_fresno): lat, lng = lat_long_fresno - return rank_stations( + candidates = rank_stations( lat, lng, - match_iecc_climate_zone=True, - match_iecc_moisture_regime=True, - match_ba_climate_zone=True, - match_ca_climate_zone=True, + match_zones=( + "iecc_climate_zone", + "iecc_moisture_regime", + "ba_climate_zone", + "ca_climate_zone", + ), minimum_quality="high", - is_tmy3=True, - is_cz2010=True, + has_sources=("tmy3", "cz2010"), ) + return candidates + @pytest.fixture def naive_candidates(lat_long_fresno): lat, lng = lat_long_fresno - return rank_stations( - lat, lng, minimum_quality="high", is_tmy3=True, is_cz2010=True + candidates = rank_stations( + lat, lng, minimum_quality="high", has_sources=("tmy3", "cz2010") ).head() + return candidates + def test_combine_ranked_stations_empty(): with pytest.raises(ValueError): @@ -213,16 +213,16 @@ def test_combine_ranked_stations_empty(): def test_combine_ranked_stations(cz_candidates, naive_candidates): assert list(cz_candidates.index) == [ - "723890", - "747020", - "723840", + "USW00093193", + "USW00023110", + "USW00023155", ] assert list(naive_candidates.index) == [ - "723890", - "747020", - "724815", - "723965", - "724926", + "USW00093193", + "USW00023110", + "USW00023257", + "USW00093209", + "USW00023258", ] combined_candidates = combine_ranked_stations([cz_candidates, naive_candidates]) @@ -231,23 +231,43 @@ def test_combine_ranked_stations(cz_candidates, naive_candidates): assert combined_candidates["rank"].iloc[0] == 1 assert combined_candidates["rank"].iloc[-1] == 6 assert list(combined_candidates.index) == [ - "723890", - "747020", - "723840", - "724815", - "723965", - "724926", + "USW00093193", + "USW00023110", + "USW00023155", + "USW00023257", + "USW00093209", + "USW00023258", ] def test_select_station_no_coverage_check(cz_candidates): station, warnings = select_station(cz_candidates) - assert station.usaf_id == "723890" + assert station.id == "USW00093193" + + +def test_select_station_real_coverage_path( + monkeypatch_tmy3_request, monkeypatch_key_value_store +): + # exercises the real load_hourly_temp_data -> station.load_data -> tmy3 + # source path (no monkeypatch of load_hourly_temp_data itself), against + # the recorded 722880TYA.CSV fixture for station USW00023152 + candidates = rank_stations(34.200, -118.350, has_sources=("tmy3",)) + assert candidates.index[0] == "USW00023152" + + start = datetime(2006, 1, 3, tzinfo=pytz.UTC) + end = datetime(2007, 4, 3, tzinfo=pytz.UTC) + + station, warnings = select_station( + candidates, coverage_range=(start, end), coverage_source="tmy3" + ) + + assert station.id == "USW00023152" + assert warnings == [] @pytest.fixture def monkeypatch_load_hourly_temp_data(monkeypatch): - def load_hourly_temp_data(station, start, end, fetch_from_web=True): + def load_hourly_temp_data(station, start, end, fetch_from_web=True, source=None): # because result datetimes should fall exactly on hours normalized_start = datetime( start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC @@ -258,12 +278,16 @@ def load_hourly_temp_data(station, start, end, fetch_from_web=True): index = pd.date_range(normalized_start, normalized_end, freq="h", tz="UTC") # simulate missing data - if station.usaf_id in ("723890", "723896"): - return pd.Series(1, index=index)[: -24 * 50].reindex(index), [] - return pd.Series(1, index=index)[: -24 * 10].reindex(index), [] + no_warnings = [] + if station.id in ("USW00093193", "USW00093144"): + temps = pd.Series(1, index=index)[: -24 * 50].reindex(index) + else: + temps = pd.Series(1, index=index)[: -24 * 10].reindex(index) + + return temps, no_warnings monkeypatch.setattr( - "eeweather.mockable.load_hourly_temp_data", load_hourly_temp_data + "eeweather.sources.matching.load_hourly_temp_data", load_hourly_temp_data ) @@ -273,13 +297,13 @@ def test_select_station_full_data(cz_candidates, monkeypatch_load_hourly_temp_da # 1st misses qualification station, warnings = select_station(cz_candidates, coverage_range=(start, end)) - assert station.usaf_id == "747020" + assert station.id == "USW00023110" # 1st meets qualification station, warnings = select_station( cz_candidates, coverage_range=(start, end), min_fraction_coverage=0.8 ) - assert station.usaf_id == "723890" + assert station.id == "USW00093193" # none meet qualification station, warnings = select_station( @@ -290,27 +314,30 @@ def test_select_station_full_data(cz_candidates, monkeypatch_load_hourly_temp_da @pytest.fixture def monkeypatch_load_hourly_temp_data_with_error(monkeypatch): - def load_hourly_temp_data(station, start, end, fetch_from_web=True): + def load_hourly_temp_data(station, start, end, fetch_from_web=True, source=None): index = pd.date_range(start, end, freq="h", tz="UTC") - if station.usaf_id == "723890": + if station.id == "USW00093193": + # first choice not available raise DataNotAvailableError( - "723890", start.year - ) # first choice not available - elif station.usaf_id == "747020": - return pd.Series(1, index=index)[: -24 * 10].reindex(index), [] + "ghcnh", station_id="USW00093193", year=start.year + ) + elif station.id == "USW00023110": + temps = pd.Series(1, index=index)[: -24 * 10].reindex(index) + no_warnings = [] + + return temps, no_warnings else: # pragma: no cover - only for helping to debug failing tests raise ValueError( - "The requested station is not specified in the monkeypatched data: {}.".format( - station - ) + "The requested station is not specified in the monkeypatched" + " data: {}.".format(station) ) monkeypatch.setattr( - "eeweather.mockable.load_hourly_temp_data", load_hourly_temp_data + "eeweather.sources.matching.load_hourly_temp_data", load_hourly_temp_data ) -def test_select_station_with_isd_data_not_available_error( +def test_select_station_with_data_not_available_error( cz_candidates, monkeypatch_load_hourly_temp_data_with_error ): start = datetime(2017, 1, 1, tzinfo=pytz.UTC) @@ -320,26 +347,30 @@ def test_select_station_with_isd_data_not_available_error( station, warnings = select_station( cz_candidates, coverage_range=(start, end), min_fraction_coverage=0.8 ) - assert station.usaf_id == "747020" + assert station.id == "USW00023110" @pytest.fixture def monkeypatch_load_hourly_temp_data_with_empty(monkeypatch): - def load_hourly_temp_data(station, start, end, fetch_from_web=True): + def load_hourly_temp_data(station, start, end, fetch_from_web=True, source=None): index = pd.date_range(start, end, freq="h", tz="UTC") - if station.usaf_id == "723890": - return pd.Series(1, index=index)[:0], [] - elif station.usaf_id == "747020": - return pd.Series(1, index=index)[: -24 * 10].reindex(index), [] + no_warnings = [] + if station.id == "USW00093193": + temps = pd.Series(1, index=index)[:0] + + return temps, no_warnings + elif station.id == "USW00023110": + temps = pd.Series(1, index=index)[: -24 * 10].reindex(index) + + return temps, no_warnings else: # pragma: no cover - only for helping to debug failing tests raise ValueError( - "The requested station is not specified in the monkeypatched data: {}.".format( - station - ) + "The requested station is not specified in the monkeypatched" + " data: {}.".format(station) ) monkeypatch.setattr( - "eeweather.mockable.load_hourly_temp_data", load_hourly_temp_data + "eeweather.sources.matching.load_hourly_temp_data", load_hourly_temp_data ) @@ -353,7 +384,7 @@ def test_select_station_with_empty_tempC( station, warnings = select_station( cz_candidates, coverage_range=(start, end), min_fraction_coverage=0.8 ) - assert station.usaf_id == snapshot + assert station.id == snapshot def test_select_station_distance_warnings_check(lat_long_africa): @@ -382,25 +413,24 @@ def test_select_station_with_second_level_dates( end = datetime(2018, 1, 1, 12, 13, 14, tzinfo=pytz.UTC) station, warnings = select_station(cz_candidates, coverage_range=(start, end)) - assert station.usaf_id == snapshot + assert station.id == snapshot def test_rank_stations_rating_period_uses_era_quality(lat_long_fresno): lat, lng = lat_long_fresno - start = datetime(2010, 1, 1, tzinfo=pytz.UTC) - end = datetime(2014, 12, 31, tzinfo=pytz.UTC) + anchor = datetime(2014, 12, 31, tzinfo=pytz.UTC) df = rank_stations( - lat, lng, minimum_quality="high", is_tmy3=True, is_cz2010=True, - rating_period=(start, end), + lat, lng, minimum_quality="high", has_sources=("tmy3", "cz2010"), + rating_period=anchor, ) # 723895 and 723896 rate high in their active era despite being # medium or low today assert list(df.head().index) == [ - "723890", - "747020", - "723896", - "724815", - "723895", + "USW00093193", + "USW00023110", + "USW00093144", + "USW00023257", + "USW00023149", ] diff --git a/tests/sources/test_normals.py b/tests/sources/test_normals.py new file mode 100644 index 0000000..31aaf9b --- /dev/null +++ b/tests/sources/test_normals.py @@ -0,0 +1,118 @@ +from pathlib import Path + +import pytest +import requests + +from eeweather.exceptions import DataNotAvailableError +from eeweather.sources.cz2010 import CZ2010Source +from eeweather.sources.tmy3 import TMY3Source + + + +FIXTURE_TMY3_TEXT = ( + Path(__file__).parent.parent / "fixtures" / "722880TYA.CSV" +).read_text(encoding="ascii") + + +def test_tmy3_source_fetch(monkeypatch_tmy3_request): + ts = TMY3Source().fetch("USW00023152") + + assert ts.sum() == pytest.approx(156194.3, 0.00001) + assert ts.shape == (8760,) + + +def test_tmy3_source_fetch_station_not_in_archive(): + with pytest.raises(DataNotAvailableError): + TMY3Source().fetch("USW00093134") + + +def test_cz2010_source_fetch_station_not_in_archive(): + with pytest.raises(DataNotAvailableError): + CZ2010Source().fetch("USW00014819") + + +def test_normals_source_fetch_unrecognized_station(): + with pytest.raises(DataNotAvailableError): + TMY3Source().fetch("INVALID") + + +def test_archive_station(): + archive = TMY3Source().archive_station("USW00023152") + + assert archive == { + "station_id": "USW00023152", + "usaf_id": "722880", + "class": "II", + } + + +def test_archive_station_not_available(): + # a German station mapped from isd history has no US normals + with pytest.raises(DataNotAvailableError) as excinfo: + TMY3Source().archive_station("GMMU0010254") + assert excinfo.value.source == "tmy3" + assert excinfo.value.station_id == "GMMU0010254" + + +class _Response: + def __init__(self, status_code, text=""): + self.status_code = status_code + self.text = text + + def raise_for_status(self): + if self.status_code >= 400: + error = requests.HTTPError("{} error".format(self.status_code)) + error.response = self + raise error + + +def test_archive_miss_raises_data_not_available(monkeypatch): + monkeypatch.setattr( + "eeweather.sources.base.requests.get", + lambda url, timeout=None: _Response(404), + ) + + with pytest.raises(DataNotAvailableError) as excinfo: + TMY3Source().fetch("USW00023152") + assert excinfo.value.source == "tmy3" + assert excinfo.value.station_id == "USW00023152" + + +def test_archive_fetch_retries_server_errors(monkeypatch): + calls = [] + sleeps = [] + monkeypatch.setattr("eeweather.sources.base.time.sleep", sleeps.append) + + def flaky_get(url, timeout=None): + calls.append(url) + if len(calls) == 1: + return _Response(503) + + return _Response(200, text=FIXTURE_TMY3_TEXT) + + monkeypatch.setattr("eeweather.sources.base.requests.get", flaky_get) + + ts = TMY3Source().fetch("USW00023152") + + assert len(calls) == 2 + assert len(sleeps) == 1 + assert ts.shape == (8760,) + + +def test_archive_fetch_client_error_raises_immediately(monkeypatch): + calls = [] + monkeypatch.setattr( + "eeweather.sources.base.time.sleep", lambda s: None + ) + + def forbidden_get(url, timeout=None): + calls.append(url) + + return _Response(403) + + monkeypatch.setattr("eeweather.sources.base.requests.get", forbidden_get) + + with pytest.raises(requests.HTTPError): + TMY3Source().fetch("USW00023152") + + assert len(calls) == 1 diff --git a/tests/sources/test_sources.py b/tests/sources/test_sources.py new file mode 100644 index 0000000..c709561 --- /dev/null +++ b/tests/sources/test_sources.py @@ -0,0 +1,564 @@ +import gzip +import io +from datetime import datetime, timezone +from pathlib import Path + +import pandas as pd +import pytest + +from eeweather.sources import Feed, Source, StationSource, Variable, register, variables +from eeweather.sources import engine +from eeweather.sources import vocabulary +from eeweather.sources.engine import ( + known_source_names, + load_data, + resolve_source, + sources_serving, +) +from eeweather.sources.ghcnh import GHCNhSource + + + +FIXTURE_DIR = Path(__file__).parent.parent / "fixtures" + + +def _fixture_text(name): + with gzip.open(FIXTURE_DIR / name, "rb") as f: + return f.read().decode() + + +def test_resolve_source_by_name(): + assert resolve_source("ghcnh").name == "ghcnh" + assert resolve_source("tmy3").name == "tmy3" + assert resolve_source("cz2010").name == "cz2010" + + +def test_resolve_source_passes_objects_through(): + feed = GHCNhSource() + + assert resolve_source(feed) is feed + + +def test_resolve_source_unknown_name(): + with pytest.raises(ValueError, match="Unknown source"): + resolve_source("noaa_ftp") + + +def test_sources_serving(): + assert sources_serving("temperature") == ["cz2010", "ghcnh", "tmy3"] + assert sources_serving("wind_speed") == ["ghcnh"] + assert sources_serving("ghi") == [] + + +def test_variables_accessor(): + df = variables() + + assert df.index.name == "variable" + assert df.loc["temperature", "unit"] == "degC" + assert df.loc["temperature", "sources"] == ("cz2010", "ghcnh", "tmy3") + assert df.loc["wind_speed", "sources"] == ("ghcnh",) + assert set(df.columns) == {"unit", "description", "aggregation", "sources"} + assert (df.aggregation == "mean").all() + + +class FixtureFeed(Feed): + """A user-style feed: keyed by usaf ids, serving captured payloads, + declaring itself uncacheable.""" + + name = "fixture-feed" + id_namespace = "usaf" + variables = ("temperature",) + cacheable = False + + # ghcn fixtures keyed by the usaf ids this feed is asked for + _files = { + "722874": "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz", + } + + def __init__(self): + self.requested_ids = [] + + def fetch_year(self, external_id, year, variables): + self.requested_ids.append(external_id) + payload = _fixture_text(self._files[external_id]) + raw = pd.read_csv(io.StringIO(payload), dtype=str) + index = pd.to_datetime(raw["DATE"]).dt.tz_localize("UTC").rename(None) + df = pd.DataFrame(index=index) + for variable in variables: + df[variable] = pd.to_numeric(raw[variable], errors="coerce").values + df = df.groupby(df.index).mean().sort_index() + + return df + + +def test_custom_feed_through_engine(monkeypatch_key_value_store): + feed = FixtureFeed() + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + df, warnings = load_data("USW00093134", start, end, source=feed) + + # the registry id was translated into the feed's usaf namespace + assert feed.requested_ids == ["722874"] + assert int(df.temperature.notna().sum()) == 8732 + assert df.attrs["provenance"]["fixture-feed"].station_id == "USW00093134" + # cacheable=False feeds leave nothing in the store + assert monkeypatch_key_value_store.keys("") == [] + + +def test_custom_feed_through_station_source(monkeypatch_key_value_store): + feed = FixtureFeed() + source = StationSource(dataset=feed) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + # USC campus coordinates; USW00093134 is the nearest station + df, warnings, provenance = source.estimate(34.024, -118.291, start, end) + + assert feed.requested_ids == ["722874"] + assert provenance["fixture-feed"].kind == "observations" + assert provenance["fixture-feed"].station_id == "USW00093134" + assert provenance["fixture-feed"].distance_meters < 1000 + assert provenance["fixture-feed"].variables == ("temperature",) + + +class PartialFeed(Feed): + """A feed that returns fewer columns than requested, e.g. an upstream + API that has not backfilled a newer variable.""" + + name = "partial-feed" + id_namespace = "ghcn" + variables = ("temperature", "relative_humidity") + default_variables = ("temperature", "relative_humidity") + cacheable = False + + def fetch_year(self, external_id, year, variables): + index = pd.date_range( + "{}-01-01".format(year), "{}-12-31 23:00".format(year), + freq="h", tz="UTC", + ) + df = pd.DataFrame({"temperature": 10.0}, index=index) + + return df + + +@pytest.fixture +def clean_registry(monkeypatch): + monkeypatch.setattr(engine, "_registered_sources", {}) + monkeypatch.setattr(vocabulary, "_registered_variables", {}) + + +def test_register_makes_source_nameable( + clean_registry, monkeypatch_key_value_store +): + feed = register(FixtureFeed()) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + df, warnings = load_data("USW00093134", start, end, source="fixture-feed") + + assert resolve_source("fixture-feed") is feed + assert int(df.temperature.notna().sum()) == 8732 + + +def test_register_rejects_builtin_names(clean_registry): + class Impostor(Feed): + name = "ghcnh" + id_namespace = "ghcn" + variables = ("temperature",) + + def fetch_year(self, external_id, year, variables): + raise NotImplementedError + + with pytest.raises(ValueError, match="built-in source name"): + register(Impostor()) + + +def test_register_rejects_grid_sources(clean_registry): + class GridSource(object): + name = "era5" + kind = "grid" + variables = ("temperature",) + + with pytest.raises(ValueError, match="fetch_year"): + register(GridSource()) + + +GRID_VOCABULARY = ( + Variable("gridded_temperature", "degC", "Grid-cell temperature.", "mean"), +) + + +class GridEstimateSource(Source): + name = "grid-source" + kind = "grid" + variables = ("gridded_temperature",) + default_variables = ("gridded_temperature",) + + +def test_register_accepts_estimation_sources(clean_registry): + source = register(GridEstimateSource(), vocabulary=GRID_VOCABULARY) + + assert source.name == "grid-source" + assert "grid-source" in known_source_names() + assert "grid-source" in sources_serving("gridded_temperature") + + +def test_register_rejects_estimation_source_without_variables(clean_registry): + class NoVariables(Source): + name = "no-variables" + kind = "grid" + + with pytest.raises(ValueError, match="must declare 'variables'"): + register(NoVariables()) + + +def test_register_rejects_estimation_source_without_kind(clean_registry): + class NoKind(Source): + name = "no-kind" + variables = ("temperature",) + + with pytest.raises(ValueError, match="must declare 'kind'"): + register(NoKind()) + + +def test_register_rejects_fetch_year_source_without_kind(clean_registry): + class NoKindFeed(object): + name = "no-kind-feed" + id_namespace = "ghcn" + variables = ("temperature",) + + def fetch_year(self, external_id, year, variables): + raise NotImplementedError + + with pytest.raises(ValueError, match="must declare 'kind'"): + register(NoKindFeed()) + + +def test_register_rejects_fetch_year_source_without_id_namespace(clean_registry): + class NoNamespaceFeed(object): + name = "no-namespace-feed" + kind = "observations" + variables = ("temperature",) + + def fetch_year(self, external_id, year, variables): + raise NotImplementedError + + with pytest.raises(ValueError, match="must declare 'id_namespace'"): + register(NoNamespaceFeed()) + + +def test_register_requires_a_name(clean_registry): + class Nameless(Feed): + name = None + id_namespace = "ghcn" + variables = ("temperature",) + + def fetch_year(self, external_id, year, variables): + raise NotImplementedError + + with pytest.raises(ValueError, match="non-empty string name"): + register(Nameless()) + + +def test_registered_source_appears_in_discovery(clean_registry): + register(FixtureFeed()) + + assert "fixture-feed" in sources_serving("temperature") + assert "fixture-feed" in variables().loc["temperature", "sources"] + + +def test_missing_feed_column_fills_nan_instead_of_raising(clean_registry): + register(PartialFeed()) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + source="partial-feed", variables=("temperature", "relative_humidity"), + ) + + assert (df.temperature == 10.0).all() + assert df.relative_humidity.isna().all() + + +class SoilFeed(Feed): + """A feed serving a variable the canonical vocabulary lacks. + + A private variable has no real payloads to capture, so the fetch is + synthetic by necessity: a constant series over the requested year. + """ + + name = "soil-feed" + id_namespace = "ghcn" + variables = ("soil_temperature",) + default_variables = ("soil_temperature",) + cacheable = False + + def fetch_year(self, external_id, year, variables): + index = pd.date_range( + "{}-01-01".format(year), "{}-12-31 23:00".format(year), + freq="h", tz="UTC", + ) + df = pd.DataFrame({v: 10.0 for v in variables}, index=index) + + return df + + +SOIL_VOCABULARY = ( + Variable("soil_temperature", "degC", "Soil temperature at 10 cm depth.", "mean"), +) + + +def test_registered_vocabulary_serves_new_variable(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 2, 1, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + source="soil-feed", variables=("soil_temperature",), + ) + + assert list(df.columns) == ["soil_temperature"] + assert (df.soil_temperature == 10.0).all() + assert df.attrs["provenance"]["soil-feed"].variables == ("soil_temperature",) + + +def test_registered_variable_routes_to_its_source(clean_registry): + # temperature would route to ghcnh; the private variable must route + # to the private source without touching ghcnh at all + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 2, 1, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + sources=("ghcnh", "soil-feed"), variables=("soil_temperature",), + ) + + assert list(df.attrs["provenance"]) == ["soil-feed"] + + +def test_registered_variable_appears_in_discovery(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + + df = variables() + + assert df.loc["soil_temperature", "unit"] == "degC" + assert df.loc["soil_temperature", "sources"] == ("soil-feed",) + + +def test_register_vocabulary_rejects_canonical_names(clean_registry): + entry = Variable("temperature", "K", "Air temperature in kelvin.", "mean") + + with pytest.raises(ValueError, match="canonical"): + register(SoilFeed(), vocabulary=(entry,)) + + +def test_register_vocabulary_requires_agreement_across_sources(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + + class OtherSoilFeed(SoilFeed): + name = "other-soil" + + disagreeing = ( + Variable("soil_temperature", "K", "Soil temperature at 10 cm depth.", "mean"), + ) + with pytest.raises(ValueError, match="definitions must agree"): + register(OtherSoilFeed(), vocabulary=disagreeing) + + # an identical definition is fine + register(OtherSoilFeed(), vocabulary=SOIL_VOCABULARY) + + assert sources_serving("soil_temperature") == ["other-soil", "soil-feed"] + + +def test_register_rejects_undeclared_variables(clean_registry): + with pytest.raises(ValueError, match="vocabulary lacks"): + register(SoilFeed()) + + +class RainFeed(Feed): + """A feed serving an accumulation variable; synthetic by necessity, + like SoilFeed: constant 1.0 mm per hour, with hour 5 of Jan 1 + missing.""" + + name = "rain-feed" + id_namespace = "ghcn" + variables = ("rainfall",) + default_variables = ("rainfall",) + cacheable = False + + def fetch_year(self, external_id, year, variables): + index = pd.date_range( + "{}-01-01".format(year), "{}-12-31 23:00".format(year), + freq="h", tz="UTC", + ) + df = pd.DataFrame({v: 1.0 for v in variables}, index=index) + df.iloc[5] = float("nan") + + return df + + +RAIN_VOCABULARY = ( + Variable("rainfall", "mm", "Liquid precipitation depth.", "sum"), +) + + +def test_sum_variable_rolls_up_daily_by_sum(clean_registry): + register(RainFeed(), vocabulary=RAIN_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 31, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + frequency="D", source="rain-feed", variables=("rainfall",), + ) + + # Jan 1 is missing one hour; every other day sums its 24 hours + assert df.rainfall.iloc[0] == 23.0 + assert (df.rainfall.iloc[1:] == 24.0).all() + + +def test_sum_variable_hourly_gaps_are_not_interpolated(clean_registry): + register(RainFeed(), vocabulary=RAIN_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, source="rain-feed", variables=("rainfall",), + ) + + assert pd.isna(df.rainfall.iloc[5]) + assert df.rainfall.drop(df.index[5]).notna().all() + + +def test_monthly_frequency_aggregates_by_vocabulary(clean_registry): + register(RainFeed(), vocabulary=RAIN_VOCABULARY) + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + rain, _ = load_data( + "USW00093134", start, end, + frequency="MS", source="rain-feed", variables=("rainfall",), + ) + soil, _ = load_data( + "USW00093134", start, end, + frequency="MS", source="soil-feed", variables=("soil_temperature",), + ) + + assert len(rain) == 12 + assert rain.index[1] == datetime(2007, 2, 1, tzinfo=timezone.utc) + assert rain.rainfall.iloc[1] == 28 * 24.0 + assert (soil.soil_temperature == 10.0).all() + + +def test_annual_frequency_aggregates_by_vocabulary(clean_registry): + register(RainFeed(), vocabulary=RAIN_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2008, 12, 31, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + frequency="YS", source="rain-feed", variables=("rainfall",), + ) + + # one missing hour in each year's Jan 1 + assert list(df.rainfall) == [365 * 24.0 - 1, 366 * 24.0 - 1] + + +def test_monthly_frequency_excludes_partial_start_month(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 15, tzinfo=timezone.utc) + end = datetime(2007, 6, 30, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + frequency="MS", source="soil-feed", variables=("soil_temperature",), + ) + + assert df.index[0] == datetime(2007, 2, 1, tzinfo=timezone.utc) + assert len(df) == 5 + + +def test_register_vocabulary_rejects_unknown_aggregation(clean_registry): + entry = Variable("rainfall", "mm", "Liquid precipitation depth.", "median") + + with pytest.raises(ValueError, match="Unknown aggregation"): + register(RainFeed(), vocabulary=(entry,)) + + +def test_weekly_frequency_sums_accumulations(clean_registry): + register(RainFeed(), vocabulary=RAIN_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 3, 1, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + frequency="W", source="rain-feed", variables=("rainfall",), + ) + + # weeks label at their start; full weeks sum 168 hourly values + assert (df.index.dayofweek == df.index.dayofweek[0]).all() + assert (df.rainfall.iloc[1:] == 168.0).all() + + +def test_subhourly_interpolates_point_in_time_variables(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = load_data( + "USW00093134", start, end, + frequency="30min", source="soil-feed", variables=("soil_temperature",), + ) + + assert df.index.freqstr == "30min" + assert (df.soil_temperature == 10.0).all() + + +def test_subhourly_never_crosses_a_missing_hour(clean_registry): + register(RainFeed(), vocabulary=RAIN_VOCABULARY) + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + # RainFeed's hour 5 is missing: both 20-minute slots inside it stay + # NaN, and each present hour's 1.0 mm spreads evenly + rain, _ = load_data( + "USW00093134", start, end, + frequency="20min", source="rain-feed", variables=("rainfall",), + ) + + hour5 = rain.rainfall.loc["2007-01-01 05:00":"2007-01-01 05:59"] + assert hour5.isna().all() + hour6 = rain.rainfall.loc["2007-01-01 06:00":"2007-01-01 06:59"] + assert (hour6 == 1.0 / 3).all() + + +def test_subhourly_frequency_must_divide_the_hour(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + with pytest.raises(ValueError, match="divide the hour evenly"): + load_data( + "USW00093134", start, end, + frequency="25min", source="soil-feed", + variables=("soil_temperature",), + ) + + +def test_frequency_nomenclature_is_validated_by_pandas(clean_registry): + register(SoilFeed(), vocabulary=SOIL_VOCABULARY) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + with pytest.raises(ValueError, match="Invalid frequency"): + load_data( + "USW00093134", start, end, + frequency="fortnightly", source="soil-feed", + variables=("soil_temperature",), + ) diff --git a/tests/test_cache.py b/tests/test_cache.py index ebba018..d0b1d48 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,25 +1,7 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" import tempfile -from eeweather.cache import KeyValueStore -from datetime import datetime, timezone + +from eeweather.cache import KeyValueStore, _expired, key_value_store_proxy, set_path +from datetime import datetime, timedelta, timezone import pytest @@ -76,3 +58,54 @@ def test_key_value_store_clear_single_key(s): def test_key_value_store_rejects_non_sqlite_url(): with pytest.raises(ValueError, match="sqlite:///"): KeyValueStore("postgresql://user@host/db") + + +def test_set_cache_path_points_shared_store(tmp_path): + original = key_value_store_proxy._store + try: + path = "{}/custom.db".format(tmp_path) + set_path(path) + store = key_value_store_proxy.get_store() + assert store.url == "sqlite:///{}".format(path) + store.save_json("k", {"a": 1}) + assert key_value_store_proxy.get_store().retrieve_json("k") == {"a": 1} + finally: + key_value_store_proxy._store = original + + +def test_expired_when_never_updated(): + assert _expired(None, 2020) is True + + +def test_not_expired_when_updated_after_data_year(): + # data from a completed year never expires once publication settles + last_updated = datetime(2021, 6, 1, tzinfo=timezone.utc) + + assert _expired(last_updated, 2020) is False + + +def test_expired_when_updated_within_the_year_end_grace_window(): + # a year cached days after it ends is still incomplete upstream + last_updated = datetime(2021, 1, 2, tzinfo=timezone.utc) + + assert _expired(last_updated, 2020) is True + + +def test_not_expired_when_updated_after_the_grace_window(): + last_updated = datetime(2021, 1, 20, tzinfo=timezone.utc) + + assert _expired(last_updated, 2020) is False + + +def test_expired_when_updated_during_data_year_and_old(): + now = datetime.now(timezone.utc) + last_updated = now - timedelta(days=2) + + assert _expired(last_updated, last_updated.year) is True + + +def test_not_expired_when_updated_during_data_year_and_fresh(): + now = datetime.now(timezone.utc) + last_updated = now - timedelta(hours=1) + + assert _expired(last_updated, last_updated.year) is False diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 5194e84..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import json -from click.testing import CliRunner - - -from eeweather.cli import ( - cli, - inspect_isd_station, - inspect_isd_file_years, -) - - - -def test_eeweather_cli(): - runner = CliRunner() - - result = runner.invoke(cli, ["--help"]) - assert result.exit_code == 0 - assert len(result.output) > 100 - - -def test_inspect_isd_station(): - runner = CliRunner() - - result = runner.invoke(inspect_isd_station, ["722880"]) - assert result.exit_code == 0 - assert json.loads(result.output) == { - "ba_climate_zone": "Hot-Dry", - "ca_climate_zone": "CA_09", - "elevation": "+0222.7", - "icao_code": "KBUR", - "iecc_climate_zone": "3", - "iecc_moisture_regime": "B", - "latitude": "+34.200", - "longitude": "-118.365", - "name": "BURBANK-GLENDALE-PASA ARPT", - "ghcn_id": "USW00023152", - "ghcn_map_method": "icao", - "ghcn_first_year": 1943, - "ghcn_last_year": 2026, - "quality": "high", - "recent_wban_id": "23152", - "state": "CA", - "usaf_id": "722880", - "wban_ids": "23152,99999", - } - - -def test_inspect_isd_station_unrecognized(): - runner = CliRunner() - result = runner.invoke(inspect_isd_station, ["INVALID"]) - assert result.exit_code == 1 - - -def test_inspect_isd_file_years(): - runner = CliRunner() - - result = runner.invoke(inspect_isd_file_years, ["722880"]) - assert result.exit_code == 0 - assert json.loads(result.output) == [ - {"usaf_id": "722880", "wban_id": "23152", "year": "2006"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2007"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2008"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2009"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2010"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2011"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2012"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2013"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2014"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2015"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2016"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2017"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2018"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2019"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2020"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2021"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2022"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2023"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2024"}, - {"usaf_id": "722880", "wban_id": "23152", "year": "2025"}, - ] - - -def test_inspect_isd_file_years_unrecognized(): - runner = CliRunner() - result = runner.invoke(inspect_isd_file_years, ["INVALID"]) - assert result.exit_code == 1 diff --git a/tests/test_database.py b/tests/test_database.py deleted file mode 100644 index 68c9968..0000000 --- a/tests/test_database.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -""" Note about database tests. - - The database can be rebuilt from scratch using the command: - - $ eeweather rebuild_db - - The database can be inspected using the command (opens sqlite3 CLI): - - $ eeweather inspect_db - - These tests do not cover these main functions - instead, they test the - content of the database. They can be thought of as integration tests. -""" -from eeweather.connections import metadata_db_connection_proxy - - -def test_database_tables(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select name from sqlite_master where type='table' """) - tables = [name for (name,) in cur.fetchall()] - assert tables == [ - "isd_station_metadata", - "isd_file_metadata", - "zcta_metadata", - "iecc_climate_zone_metadata", - "iecc_moisture_regime_metadata", - "ba_climate_zone_metadata", - "ca_climate_zone_metadata", - "tmy3_station_metadata", - "cz2010_station_metadata", - "ghcn_inventory", - ] - - -def test_isd_station_metadata_table_count(snapshot): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from isd_station_metadata """) - (count,) = cur.fetchone() - assert snapshot == count - - -def test_isd_station_metadata_table_content(snapshot): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select * from isd_station_metadata where quality='high' limit 1 """) - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert snapshot == data - - -def test_isd_file_metadata_table_count(snapshot): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from isd_file_metadata """) - (count,) = cur.fetchone() - assert snapshot == count - - -def test_isd_file_metadata_table_content(snapshot): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select * from isd_file_metadata limit 1""") - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert snapshot == data - - -def test_zcta_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from zcta_metadata """) - (count,) = cur.fetchone() - assert count == 33144 - - -def test_zcta_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select * from zcta_metadata limit 1""") - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == { - "ba_climate_zone": "Hot-Humid", - "ca_climate_zone": None, - "geometry": None, - "iecc_climate_zone": "1", - "iecc_moisture_regime": "A", - "latitude": "18.1800455429617", - "longitude": "-66.7521781364081", - "state": "PR", - "zcta_id": "00601", - } - - -def test_iecc_climate_zone_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from iecc_climate_zone_metadata """) - (count,) = cur.fetchone() - assert count == 8 - - -def test_iecc_climate_zone_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select iecc_climate_zone from iecc_climate_zone_metadata limit 1""") - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == {"iecc_climate_zone": "1"} - - -def test_iecc_moisture_regime_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from iecc_moisture_regime_metadata """) - (count,) = cur.fetchone() - assert count == 3 - - -def test_iecc_moisture_regime_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute( - """ select iecc_moisture_regime from iecc_moisture_regime_metadata limit 1""" - ) - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == {"iecc_moisture_regime": "A"} - - -def test_ba_climate_zone_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from ba_climate_zone_metadata """) - (count,) = cur.fetchone() - assert count == 8 - - -def test_ba_climate_zone_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select ba_climate_zone from ba_climate_zone_metadata limit 1""") - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == {"ba_climate_zone": "Cold"} - - -def test_ca_climate_zone_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from ca_climate_zone_metadata """) - (count,) = cur.fetchone() - assert count == 16 - - -def test_ca_climate_zone_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute( - """ select ca_climate_zone, name from ca_climate_zone_metadata limit 1""" - ) - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == {"ca_climate_zone": "CA_01", "name": "Arcata"} - - -def test_tmy3_station_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from tmy3_station_metadata """) - (count,) = cur.fetchone() - assert count == 1020 - - -def test_tmy3_station_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select * from tmy3_station_metadata limit 1""") - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == { - "class": "I", - "name": "Twentynine Palms", - "state": "CA", - "usaf_id": "690150", - } - - -def test_cz2010_station_metadata_table_count(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select count(*) from cz2010_station_metadata """) - (count,) = cur.fetchone() - assert count == 86 - - -def test_cz2010_station_metadata_table_content(): - conn = metadata_db_connection_proxy.get_connection() - - cur = conn.cursor() - cur.execute(""" select * from cz2010_station_metadata limit 1""") - row = cur.fetchone() - data = {desc[0]: value for value, desc in zip(row, cur.description)} - assert data == {"usaf_id": "690150"} diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 243b2e6..d5a66da 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,62 +1,101 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" +import pytest -Copyright 2018-2023 OpenEEmeter contributors +from eeweather.exceptions import ( + AmbiguousIdentifierError, + DataNotAvailableError, + EEWeatherError, + EEWeatherWarning, + NoQualifiedStationError, + UnrecognizedPlaceError, + UnrecognizedStationError, +) -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +def test_eeweather_error_message_reaches_str(): + error = EEWeatherError("message") -""" -import pytest + assert str(error) == "message" + assert error.message == "message" -from eeweather import ( - EEWeatherError, - UnrecognizedUSAFIDError, - UnrecognizedZCTAError, - DataNotAvailableError, -) +def test_unrecognized_station_error(): + error = UnrecognizedStationError("INVALID") + + assert error.value == "INVALID" + assert str(error) == ( + 'The value "INVALID" was not recognized as a valid weather station' + " identifier." + ) + assert isinstance(error, EEWeatherError) -def test_eeweather_error(): - with pytest.raises(EEWeatherError) as excinfo: - raise EEWeatherError("message") - assert excinfo.value.args[0] == "message" +def test_unrecognized_place_error(): + error = UnrecognizedPlaceError("zcta", "00000") -def test_unrecognized_usaf_id_error(): - with pytest.raises(UnrecognizedUSAFIDError) as excinfo: - raise UnrecognizedUSAFIDError("INVALID") - assert excinfo.value.value == "INVALID" - assert excinfo.value.message == ( - 'The value "INVALID" was not recognized as a' - " valid USAF weather station identifier." + assert error.kind == "zcta" + assert error.code == "00000" + assert str(error) == ( + 'The value "00000" was not recognized as a valid place of kind "zcta".' ) -def test_unrecognized_zcta_error(): - with pytest.raises(UnrecognizedZCTAError) as excinfo: - raise UnrecognizedZCTAError("INVALID") - assert excinfo.value.value == "INVALID" - assert excinfo.value.message == ( - 'The value "INVALID" was not recognized as a valid ZCTA identifier.' +def test_ambiguous_identifier_error(): + error = AmbiguousIdentifierError("wban", "00102", ["ASN00026044", "USW00000102"]) + + assert error.namespace == "wban" + assert error.external_id == "00102" + assert error.station_ids == ("ASN00026044", "USW00000102") + assert str(error) == ( + 'The wban id "00102" maps to multiple stations' + " (ASN00026044, USW00000102) and cannot be resolved." ) -def test_data_does_not_exist_error(): - with pytest.raises(DataNotAvailableError) as excinfo: - raise DataNotAvailableError("123456", 1800) - assert excinfo.value.usaf_id == "123456" - assert excinfo.value.year == 1800 - assert excinfo.value.message == ( - 'Data does not exist for station "123456" in year 1800.' +def test_data_not_available_error(): + error = DataNotAvailableError("ghcnh", station_id="USW00023152", year=1800) + + assert error.source == "ghcnh" + assert error.station_id == "USW00023152" + assert error.year == 1800 + assert str(error) == ( + "No ghcnh data available for station=USW00023152 year=1800." ) + + +def test_data_not_available_error_station_and_year_are_keyword_only(): + with pytest.raises(TypeError): + DataNotAvailableError("ghcnh", "USW00023152", 1800) + + +def test_no_qualified_station_error(): + error = NoQualifiedStationError(34.0, -118.2) + + assert error.latitude == 34.0 + assert error.longitude == -118.2 + assert str(error) == ( + "No qualified station found for location (34.0, -118.2)." + ) + + +@pytest.fixture +def generic_eeweather_warning(): + warning = EEWeatherWarning( + qualified_name="qualified_name", description="description", data={} + ) + + return warning + + +def test_warning_repr(generic_eeweather_warning): + assert repr(generic_eeweather_warning) == ( + "EEWeatherWarning(qualified_name=qualified_name)" + ) + + +def test_warning_json(generic_eeweather_warning): + assert generic_eeweather_warning.json() == { + "qualified_name": "qualified_name", + "description": "description", + "data": {}, + } diff --git a/tests/test_geo.py b/tests/test_geo.py deleted file mode 100644 index c586f0b..0000000 --- a/tests/test_geo.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import pytest - -from eeweather import get_version -from eeweather.geo import ( - get_lat_long_climate_zones, - get_zcta_metadata, - zcta_to_lat_long, -) -from eeweather.exceptions import UnrecognizedZCTAError, UnrecognizedUSAFIDError - - -def test_get_version(): - assert get_version() is not None - - -def test_get_zcta_metadata(): - metadata = get_zcta_metadata("90006") - assert metadata == { - "ba_climate_zone": "Hot-Dry", - "ca_climate_zone": "CA_09", - "geometry": None, - "iecc_climate_zone": "3", - "iecc_moisture_regime": "B", - "latitude": "34.0480061236777", - "longitude": "-118.294181179333", - "state": "CA", - "zcta_id": "90006", - } - - with pytest.raises(UnrecognizedZCTAError) as excinfo: - get_zcta_metadata("00000") - assert excinfo.value.value == "00000" - - -def test_get_lat_long_climate_zones(): - climate_zones = get_lat_long_climate_zones(35.1, -119.2) - assert climate_zones == { - "iecc_climate_zone": "3", - "iecc_moisture_regime": "B", - "ba_climate_zone": "Hot-Dry", - "ca_climate_zone": "CA_13", - } - - -def test_get_lat_long_climate_zones_out_of_range(): - climate_zones = get_lat_long_climate_zones(0, 0) - assert climate_zones == { - "iecc_climate_zone": None, - "iecc_moisture_regime": None, - "ba_climate_zone": None, - "ca_climate_zone": None, - } - - -def test_zcta_to_lat_long(): - with pytest.raises(UnrecognizedZCTAError) as excinfo: - zcta_to_lat_long("00000") - - lat, lng = zcta_to_lat_long("70001") - assert round(lat) == 30 - assert round(lng) == -90 - - lat, lng = zcta_to_lat_long("94574") - assert round(lat) == 39 - assert round(lng) == -122 diff --git a/tests/test_location.py b/tests/test_location.py new file mode 100644 index 0000000..ca446c4 --- /dev/null +++ b/tests/test_location.py @@ -0,0 +1,766 @@ +import gzip +import io +from datetime import datetime, timezone +from pathlib import Path + +import pandas as pd +import pytest + +from eeweather import WeatherLocation +from eeweather.exceptions import DataNotAvailableError +from eeweather.sources import Feed, Source, StationSource, Variable, register +from eeweather.sources import engine, vocabulary +from eeweather.sources.engine import ( + Provenance, + known_source_names, + resolve_source, + sources_serving, +) +from eeweather.sources.matching import rank_stations as real_rank_stations + + + +# station USW00093134 (DOWNTOWN L.A./USC CAMPUS) is the nearest to these coords +USC_LATITUDE = 34.024 +USC_LONGITUDE = -118.291 + + +def test_location_resolves_nearest_station(mock_api_transport): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + df, warnings = location.load_data( + start, end, read_from_cache=False, write_to_cache=False + ) + + assert list(df.columns) == ["temperature"] + assert int(df.temperature.notna().sum()) > 8000 + record = location.provenance["ghcnh"] + assert record.kind == "observations" + assert record.station_id == "USW00093134" + assert record.distance_meters < 1000 + assert record.variables == ("temperature",) + assert df.attrs["provenance"] == location.provenance + + +def test_location_default_sources(): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + + assert len(location.sources) == 1 + assert isinstance(location.sources[0], StationSource) + assert location.sources[0].name == "ghcnh" + assert location.provenance is None + + +def test_location_sources_accepts_single_string(): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources="ghcnh") + + assert len(location.sources) == 1 + assert location.sources[0].name == "ghcnh" + + +def test_location_duplicate_source_names_raise(): + with pytest.raises(ValueError, match="Duplicate source names"): + WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=("ghcnh", "ghcnh")) + + +def test_location_uses_custom_source(): + class StubSource(Source): + name = "stub" + kind = "observations" + variables = ("temperature",) + + def estimate(self, latitude, longitude, start, end, **load_kwargs): + frame = pd.DataFrame({"temperature": [1.0, 2.0]}) + provenance = { + "stub": Provenance( + kind="observations", + source="stub", + station_id="USW00000117", + distance_meters=42.0, + variables=("temperature",), + ) + } + warnings = ["stub_warning"] + + return frame, warnings, provenance + + location = WeatherLocation(0.0, 0.0, sources=(StubSource(),)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 2, 1, tzinfo=timezone.utc) + + df, warnings = location.load_data(start, end) + + assert warnings == ["stub_warning"] + assert location.provenance["stub"].station_id == "USW00000117" + assert location.provenance["stub"].distance_meters == 42.0 + + +def test_location_zones(): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + + assert location.zones == { + "iecc_climate_zone": "3", + "iecc_moisture_regime": "B", + "ba_climate_zone": "Hot-Dry", + "ca_climate_zone": "CA_08", + } + + +def test_location_candidates(): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + + candidates = location.candidates(minimum_quality="high") + + assert candidates.index[0] == "USW00003167" + assert (candidates.quality == "high").all() + assert candidates["rank"].iloc[0] == 1 + + +def test_location_pins_bypass_matching(mock_api_transport, monkeypatch): + rank_calls = [] + + def counting_rank_stations(*args, **kwargs): + rank_calls.append(args) + + return real_rank_stations(*args, **kwargs) + + monkeypatch.setattr( + "eeweather.sources.station_source.rank_stations", counting_rank_stations + ) + + location = WeatherLocation( + USC_LATITUDE, USC_LONGITUDE, pins={"ghcnh": "USW00093134"} + ) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + df, warnings = location.load_data( + start, end, read_from_cache=False, write_to_cache=False + ) + + assert rank_calls == [] + record = location.provenance["ghcnh"] + assert record.station_id == "USW00093134" + assert record.distance_meters < 1000 + + +def test_location_load_data_rejects_sources_kwarg(mock_api_transport): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + with pytest.raises(TypeError, match="sources"): + location.load_data(start, end, sources=("ghcnh",)) + + +def test_location_load_data_allows_cache_kwargs(mock_api_transport): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + df, warnings = location.load_data( + start, + end, + read_from_cache=False, + write_to_cache=False, + fetch_from_web=True, + ) + + assert list(df.columns) == ["temperature"] + + +def test_location_serialization_round_trips_pins(mock_api_transport, monkeypatch): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + location.load_data(start, end, read_from_cache=False, write_to_cache=False) + + # pins are captured from provenance automatically + assert location.pins == {"ghcnh": "USW00093134"} + + replay = WeatherLocation.from_json(location.to_json()) + rank_calls = [] + + def counting_rank_stations(*args, **kwargs): + rank_calls.append(args) + + return real_rank_stations(*args, **kwargs) + + monkeypatch.setattr( + "eeweather.sources.station_source.rank_stations", counting_rank_stations + ) + replay.load_data(start, end, read_from_cache=False, write_to_cache=False) + + assert rank_calls == [] + assert replay.provenance["ghcnh"].station_id == "USW00093134" + assert replay.to_dict() == location.to_dict() + + +def test_location_serialization_with_registered_source(monkeypatch): + class PrivateFeed(Feed): + name = "private" + id_namespace = "ghcn" + variables = ("temperature",) + + def fetch_year(self, external_id, year, variables): + raise NotImplementedError + + monkeypatch.setattr(engine, "_registered_sources", {}) + register(PrivateFeed()) + location = WeatherLocation( + USC_LATITUDE, USC_LONGITUDE, sources=("ghcnh", "private") + ) + + replay = WeatherLocation.from_dict(location.to_dict()) + + assert [s.name for s in replay.sources] == ["ghcnh", "private"] + assert replay.sources[1].adapter.name == "private" + + +def test_location_to_dict_rejects_custom_sources(): + class StubSource(Source): + name = "stub" + kind = "observations" + variables = ("temperature",) + + location = WeatherLocation(0.0, 0.0, sources=(StubSource(),)) + + with pytest.raises(ValueError, match="Custom sources"): + location.to_dict() + + +def test_source_resolution_is_memoized_per_period(monkeypatch): + source = StationSource() + rank_calls = [] + + def counting_rank_stations(*args, **kwargs): + rank_calls.append(args) + + return real_rank_stations(*args, **kwargs) + + monkeypatch.setattr( + "eeweather.sources.station_source.rank_stations", counting_rank_stations + ) + + same_period = ( + datetime(2007, 1, 1, tzinfo=timezone.utc), + datetime(2007, 12, 31, tzinfo=timezone.utc), + ) + source.resolve(USC_LATITUDE, USC_LONGITUDE, period=same_period) + source.resolve(USC_LATITUDE, USC_LONGITUDE, period=same_period) + + assert len(rank_calls) == 1 + + # a different era re-resolves + source.resolve( + USC_LATITUDE, + USC_LONGITUDE, + period=( + datetime(2013, 1, 1, tzinfo=timezone.utc), + datetime(2013, 12, 31, tzinfo=timezone.utc), + ), + ) + + assert len(rank_calls) == 2 + + # reset drops the memo + source.reset() + source.resolve(USC_LATITUDE, USC_LONGITUDE, period=same_period) + + assert len(rank_calls) == 3 + + +def test_location_from_place(): + location = WeatherLocation.from_place("zcta", "91104") + + assert location.latitude == pytest.approx(34.168, abs=0.001) + assert location.longitude == pytest.approx(-118.123, abs=0.001) + + +def test_location_pinned_normals( + monkeypatch_tmy3_request, monkeypatch_key_value_store +): + # Burbank coordinates; the nearest tmy3-bearing station is USW00023152 + location = WeatherLocation(34.2, -118.365) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 4, 3, tzinfo=timezone.utc) + + df, warnings = location.load_data(start, end, source="tmy3") + + record = location.provenance["tmy3"] + assert record.kind == "normals" + assert record.station_id == "USW00023152" + assert int(df.temperature.notna().sum()) == len(df) + + +def test_location_rejects_normals_in_preference_sources(): + # a normals source belongs in a source= pin, not the preference tuple; + # WeatherStation rejects the same, so the two entry points now agree + with pytest.raises(ValueError, match="not preference sources"): + WeatherLocation(34.2, -118.365, sources=("tmy3",)) + + +def test_location_unroutable_variable_raises(): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 2, 1, tzinfo=timezone.utc) + + with pytest.raises(ValueError, match="Unknown variables"): + location.load_data(start, end, variables=("not_a_variable",)) + + +def test_location_pinned_variables_all(mock_api_transport, monkeypatch_key_value_store): + from eeweather.sources.ghcnh import GHCNhSource + + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 6, 1, tzinfo=timezone.utc) + end = datetime(2007, 6, 2, tzinfo=timezone.utc) + + df, warnings = location.load_data(start, end, source="ghcnh", variables="all") + + assert list(df.columns) == list(GHCNhSource.variables) + + +def test_location_routed_missing_data_warns_instead_of_raising( + mock_api_transport, monkeypatch_key_value_store +): + # station USW00093134 has no observations at all in 2050 + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2050, 1, 1, tzinfo=timezone.utc) + end = datetime(2050, 6, 1, tzinfo=timezone.utc) + + df, warnings = location.load_data(start, end) + + assert df.temperature.isna().all() + assert "eeweather.data_not_available" in {w.qualified_name for w in warnings} + + +def test_location_memoized_resolution_repeats_selection_warnings(): + # a memo hit must return the same warnings the first resolution did; + # downtown LA sits a few km from the USC station, past the 1 m + # threshold configured here + source = StationSource(select_kwargs={"distance_warnings": (1,)}) + + first = source.resolve(34.05, -118.25)[2] + second = source.resolve(34.05, -118.25)[2] + + assert [w.qualified_name for w in first] == [ + "eeweather.exceeds_maximum_distance" + ] + assert [w.qualified_name for w in second] == [ + w.qualified_name for w in first + ] + + +# distance qualification: no station within 150 km of this mid-Pacific +# point; the nearest US station is over 1000 km away +MID_PACIFIC = (30.0, -140.0) + + +def test_default_distance_cap_disqualifies_remote_locations(): + from eeweather.exceptions import NoQualifiedStationError + from eeweather.sources.station_source import DEFAULT_MAX_DISTANCE_METERS + + assert StationSource().rank_kwargs["max_distance_meters"] == ( + DEFAULT_MAX_DISTANCE_METERS + ) + + location = WeatherLocation(*MID_PACIFIC) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 2, 1, tzinfo=timezone.utc) + with pytest.raises(NoQualifiedStationError): + location.load_data(start, end) + + +def test_ignore_disqualification_serves_best_available_with_warning(): + source = StationSource() + + station, distance, warnings = source.resolve( + *MID_PACIFIC, ignore_disqualification=True + ) + + names = [w.qualified_name for w in warnings] + assert "eeweather.station_disqualified" in names + disqualified = next( + w for w in warnings + if w.qualified_name == "eeweather.station_disqualified" + ) + assert disqualified.data["failed"] == ["distance"] + assert disqualified.data["distance_meters"] > 150_000 + assert disqualified.data["station_id"] == station.id + + +def test_waived_resolution_is_not_reused_by_unwaived_calls(): + from eeweather.exceptions import NoQualifiedStationError + + source = StationSource() + source.resolve(*MID_PACIFIC, ignore_disqualification=True) + + with pytest.raises(NoQualifiedStationError): + source.resolve(*MID_PACIFIC) + + +def test_distance_cap_can_be_disabled_explicitly(): + source = StationSource(rank_kwargs={"max_distance_meters": None}) + + station, distance, warnings = source.resolve(*MID_PACIFIC) + + assert station is not None + + +class StubGridSource(Source): + name = "stub_grid" + kind = "gridded" + variables = ("temperature",) + + def estimate(self, latitude, longitude, start, end, **load_kwargs): + frame = pd.DataFrame({"temperature": [1.0, 2.0]}) + provenance = { + "stub_grid": Provenance( + kind="gridded", + source="stub_grid", + station_id=None, + distance_meters=None, + variables=("temperature",), + ) + } + no_warnings = [] + + return frame, no_warnings, provenance + + +def test_location_resolves_registered_estimation_source_by_name(monkeypatch): + monkeypatch.setattr(engine, "_registered_sources", {}) + grid_source = register(StubGridSource()) + + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=("stub_grid",)) + + assert location.sources[0] is grid_source + assert not isinstance(location.sources[0], StationSource) + + +def test_capture_pins_skips_sources_without_a_station_id(): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + provenance = { + "ghcnh": Provenance( + kind="observations", + source="ghcnh", + station_id="USW00093134", + distance_meters=42.0, + variables=("temperature",), + ), + "stub_grid": Provenance( + kind="gridded", + source="stub_grid", + station_id=None, + distance_meters=None, + variables=("temperature",), + ), + } + + location._capture_pins(provenance) + + assert location.pins == {"ghcnh": "USW00093134"} + + +def test_pinned_kwargs_omits_station_for_non_station_source(monkeypatch): + monkeypatch.setattr(engine, "_registered_sources", {}) + grid_source = register(StubGridSource()) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, pins={"stub_grid": "irrelevant"}) + + kwargs = location._pinned_kwargs(grid_source, {"frequency": "h"}) + + assert "station" not in kwargs + assert kwargs == {"frequency": "h"} + + +# Contract: an in-memory location-keyed ("field") source double drives the +# same routed/pinned/serialization seams a station source does, with no +# catalog and no station in its provenance. + +FIXTURE_DIR = Path(__file__).parent / "fixtures" + +FIELD_VOCABULARY = ( + Variable("field_temperature", "degC", "Point-estimated air temperature.", "mean"), +) + +FIELD_VALUE = 21.5 + + +class FieldDouble(Source): + """A location-keyed source double: synthetic values for any point + inside a small hard-coded box, DataNotAvailableError outside it, no + catalog and no station in its provenance.""" + + name = "field-double" + kind = "observations" + variables = ("field_temperature",) + default_variables = ("field_temperature",) + + def _in_domain(self, latitude, longitude): + return 33.0 <= latitude <= 35.0 and -119.0 <= longitude <= -117.0 + + def estimate( + self, latitude, longitude, start, end, + frequency="h", variables=None, **load_kwargs, + ): + if not self._in_domain(latitude, longitude): + raise DataNotAvailableError(self.name) + if variables: + columns = tuple(variables) + else: + columns = self.default_variables + index = pd.date_range(start, end, freq="h") + frame = pd.DataFrame({column: FIELD_VALUE for column in columns}, index=index) + provenance = { + self.name: Provenance( + kind="observations", + source=self.name, + variables=columns, + station_id=None, + distance_meters=None, + payload={"domain": "socal-box"}, + ) + } + no_warnings = [] + + return frame, no_warnings, provenance + + +class FieldFixtureFeed(Feed): + """A hermetic station feed serving a captured GHCN payload as + temperature, keyed by usaf id; declares itself uncacheable.""" + + name = "field-fixture-feed" + id_namespace = "usaf" + variables = ("temperature",) + cacheable = False + + _files = { + "722874": "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz", + } + + def fetch_year(self, external_id, year, variables): + with gzip.open(FIXTURE_DIR / self._files[external_id], "rb") as f: + payload = f.read().decode() + raw = pd.read_csv(io.StringIO(payload), dtype=str) + index = pd.to_datetime(raw["DATE"]).dt.tz_localize("UTC").rename(None) + df = pd.DataFrame(index=index) + for variable in variables: + df[variable] = pd.to_numeric(raw[variable], errors="coerce").values + df = df.groupby(df.index).mean().sort_index() + + return df + + +@pytest.fixture +def clean_registry(monkeypatch): + monkeypatch.setattr(engine, "_registered_sources", {}) + monkeypatch.setattr(vocabulary, "_registered_variables", {}) + + +def test_field_double_is_nameable_after_register(clean_registry): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + + assert resolve_source("field-double") is field + assert "field-double" in known_source_names() + assert "field-double" in sources_serving("field_temperature") + + +def test_field_double_routes_synthetic_data_by_object(clean_registry): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=(field,)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = location.load_data(start, end, variables=("field_temperature",)) + + assert location.sources[0] is field + assert not isinstance(location.sources[0], StationSource) + assert list(df.columns) == ["field_temperature"] + assert (df.field_temperature == FIELD_VALUE).all() + + +def test_field_double_routes_synthetic_data_by_name(clean_registry): + register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=("field-double",)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = location.load_data(start, end, variables=("field_temperature",)) + + assert not isinstance(location.sources[0], StationSource) + assert (df.field_temperature == FIELD_VALUE).all() + + +def test_field_double_pinned_returns_synthetic_data(clean_registry): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=(field,)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = location.load_data( + start, end, source="field-double", variables=("field_temperature",) + ) + + assert (df.field_temperature == FIELD_VALUE).all() + + +def test_field_double_raises_outside_its_domain(clean_registry): + register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(0.0, 0.0, sources=("field-double",)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + with pytest.raises(DataNotAvailableError): + location.load_data( + start, end, source="field-double", variables=("field_temperature",) + ) + + +def test_field_double_provenance_has_no_station_id(clean_registry): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=(field,)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + location.load_data(start, end, variables=("field_temperature",)) + + record = location.provenance["field-double"] + assert record.kind == "observations" + assert record.source == "field-double" + assert record.station_id is None + assert record.distance_meters is None + assert isinstance(record.payload, dict) + assert record.payload + + +def test_heterogeneous_routing_joins_station_and_field_double( + clean_registry, monkeypatch_key_value_store +): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + station = StationSource(dataset=FieldFixtureFeed()) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=(station, field)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + + df, warnings = location.load_data( + start, end, variables=("temperature", "field_temperature") + ) + + assert list(df.columns) == ["temperature", "field_temperature"] + # temperature comes from the station source; the custom variable from + # the double, aligned onto the one shared UTC index (no NaN introduced + # by the join) + assert df.index.tz == timezone.utc + assert df.temperature.notna().any() + assert (df.field_temperature == FIELD_VALUE).all() + assert location.provenance["field-fixture-feed"].station_id == "USW00093134" + assert location.provenance["field-fixture-feed"].variables == ("temperature",) + assert location.provenance["field-double"].station_id is None + assert location.provenance["field-double"].variables == ("field_temperature",) + + +def test_field_double_location_round_trips_by_name(clean_registry): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=("field-double",)) + + replay = WeatherLocation.from_dict(location.to_dict()) + + assert [s.name for s in replay.sources] == ["field-double"] + assert replay.sources[0] is field + assert not isinstance(replay.sources[0], StationSource) + assert replay.to_dict() == location.to_dict() + + replay_json = WeatherLocation.from_json(location.to_json()) + + assert replay_json.sources[0] is field + assert replay_json.to_dict() == location.to_dict() + + +def test_to_dict_pins_unloaded_default_matcher_and_replays_without_resolving( + mock_api_transport, monkeypatch +): + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + # never loaded, yet to_dict captures the pin registry-only (no fetch) + state = location.to_dict() + + assert state["pins"] == {"ghcnh": "USW00093134"} + + replay = WeatherLocation.from_dict(state) + rank_calls = [] + + def counting_rank_stations(*args, **kwargs): + rank_calls.append(args) + + return real_rank_stations(*args, **kwargs) + + monkeypatch.setattr( + "eeweather.sources.station_source.rank_stations", counting_rank_stations + ) + replay.load_data(start, end, read_from_cache=False, write_to_cache=False) + + assert rank_calls == [] + assert replay.provenance["ghcnh"].station_id == "USW00093134" + + +def test_to_dict_needs_no_pin_for_field_double(clean_registry): + field = register(FieldDouble(), vocabulary=FIELD_VOCABULARY) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=("field-double",)) + + state = location.to_dict() + + assert state["pins"] == {} + + replay = WeatherLocation.from_dict(state) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 1, 2, tzinfo=timezone.utc) + df, warnings = replay.load_data(start, end, variables=("field_temperature",)) + + assert replay.sources[0] is field + assert (df.field_temperature == FIELD_VALUE).all() + + +@pytest.mark.parametrize( + "kwargs", + [ + {"select_kwargs": {"coverage_range": ( + datetime(2007, 1, 1, tzinfo=timezone.utc), + datetime(2007, 12, 31, tzinfo=timezone.utc), + )}}, + {"rank_kwargs": {"rating_period": datetime(2007, 12, 31, tzinfo=timezone.utc)}}, + ], + ids=["coverage_range", "rating_period"], +) +def test_to_dict_raises_for_unloaded_coverage_gated_source(kwargs): + source = StationSource(**kwargs) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=(source,)) + + with pytest.raises(ValueError, match="load once before serializing"): + location.to_dict() + + +def test_to_dict_round_trips_coverage_gated_source_after_load( + mock_api_transport, monkeypatch_key_value_store +): + coverage = ( + datetime(2007, 1, 1, tzinfo=timezone.utc), + datetime(2007, 12, 31, tzinfo=timezone.utc), + ) + source = StationSource(select_kwargs={"coverage_range": coverage}) + location = WeatherLocation(USC_LATITUDE, USC_LONGITUDE, sources=(source,)) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 12, 31, tzinfo=timezone.utc) + + location.load_data(start, end) + + assert location.pins == {"ghcnh": "USW00093134"} + + state = location.to_dict() + + assert state["pins"] == {"ghcnh": "USW00093134"} diff --git a/tests/test_sources_ghcnh.py b/tests/test_sources_ghcnh.py deleted file mode 100644 index e3cc4a8..0000000 --- a/tests/test_sources_ghcnh.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import gzip - -from pathlib import Path - -import pytest -import requests - -from eeweather.sources.ghcnh import ( - API_REQUEST_TRIES, - _get_api_request_params, - fetch_ghcnh_hourly, -) - - - -FIXTURE_DIR = Path(__file__).parent / "fixtures" - - -def _fixture_text(name): - with gzip.open(FIXTURE_DIR / name, "rb") as f: - return f.read().decode() - - -class MockResponse: - def __init__(self, text): - self.text = text - - def raise_for_status(self): - pass - - -@pytest.fixture -def no_sleep(monkeypatch): - sleeps = [] - monkeypatch.setattr("eeweather.sources.ghcnh.time.sleep", sleeps.append) - - return sleeps - - -def test_get_api_request_params_includes_date(): - params = _get_api_request_params("USW00093134", 2007, ("temperature",)) - - assert params["dataTypes"] == "DATE,temperature" - assert params["stations"] == "USW00093134" - assert params["startDate"] == "2007-01-01" - assert params["endDate"] == "2007-12-31" - - -def test_fetch_retries_with_backoff_then_raises(monkeypatch, no_sleep): - calls = [] - - def failing_get(url, params=None): - calls.append(url) - raise requests.ConnectionError("refused") - - monkeypatch.setattr("eeweather.sources.ghcnh.requests.get", failing_get) - - with pytest.raises(requests.ConnectionError): - fetch_ghcnh_hourly("USW00093134", 2007) - - assert len(calls) == API_REQUEST_TRIES - # backs off between attempts, not after the final failure - assert len(no_sleep) == API_REQUEST_TRIES - 1 - assert no_sleep[0] < no_sleep[1] - - -def test_fetch_succeeds_after_transient_failure(monkeypatch, no_sleep): - payload = _fixture_text( - "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz" - ) - calls = [] - - def flaky_get(url, params=None): - calls.append(url) - if len(calls) == 1: - raise requests.ConnectionError("refused") - - return MockResponse(payload) - - monkeypatch.setattr("eeweather.sources.ghcnh.requests.get", flaky_get) - - df = fetch_ghcnh_hourly("USW00093134", 2007) - - assert len(calls) == 2 - assert len(df) == 10884 - # first captured 2007 observation is 15.0 C at 00:47 - assert df.temperature.iloc[0] == pytest.approx(15.0, abs=1e-9) - - -def test_fetch_empty_year_returns_empty_frame(monkeypatch): - def mock_get(url, params=None): - return MockResponse("") - - monkeypatch.setattr("eeweather.sources.ghcnh.requests.get", mock_get) - - df = fetch_ghcnh_hourly("USW00093134", 1800) - - assert len(df) == 0 - assert list(df.columns) == ["temperature"] - assert str(df.index.tz) == "UTC" - - -def test_fetch_unreported_variable_is_nan_column(monkeypatch): - payload = _fixture_text( - "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz" - ) - - def mock_get(url, params=None): - return MockResponse(payload) - - monkeypatch.setattr("eeweather.sources.ghcnh.requests.get", mock_get) - - df = fetch_ghcnh_hourly( - "USW00093134", 2007, ("temperature", "snow_depth") - ) - - assert list(df.columns) == ["temperature", "snow_depth"] - assert df.snow_depth.isna().all() - assert df.temperature.notna().sum() > 0 - - -def test_fetch_averages_duplicate_timestamps(monkeypatch): - payload = _fixture_text( - "global-historical-climatology-network-hourly_USW00093134_2007.csv.gz" - ) - - def mock_get(url, params=None): - return MockResponse(payload) - - monkeypatch.setattr("eeweather.sources.ghcnh.requests.get", mock_get) - - df = fetch_ghcnh_hourly("USW00093134", 2007) - - assert df.index.is_unique - assert df.index.is_monotonic_increasing diff --git a/tests/test_station.py b/tests/test_station.py new file mode 100644 index 0000000..1f86ffd --- /dev/null +++ b/tests/test_station.py @@ -0,0 +1,264 @@ +from datetime import datetime, timezone + +import pandas as pd +import pytest + +from eeweather import WeatherLocation, WeatherStation +from eeweather.exceptions import UnrecognizedStationError +from eeweather.sources import Feed + + + +def test_station_without_metadata(): + station = WeatherStation("USW00023152", load_metadata=False) + + assert station.id == "USW00023152" + assert station.ids is None + assert station.name is None + assert station.latitude is None + assert station.longitude is None + assert station.elevation is None + assert station.coords is None + assert station.country is None + assert station.subdivision is None + assert station.quality is None + assert station.zones == {} + assert station.inventory_years == {} + + assert str(station) == "USW00023152" + assert repr(station) == "WeatherStation('USW00023152')" + + +def test_station_without_metadata_still_validates_id(): + with pytest.raises(UnrecognizedStationError): + WeatherStation("FAKE", load_metadata=False) + + +def test_station_with_metadata(): + station = WeatherStation("USW00023152") + + assert station.id == "USW00023152" + assert station.ids == { + "ghcn": ["USW00023152"], + "icao": ["KBUR"], + "usaf": ["722880"], + "wban": ["23152"], + "wmo": ["72288"], + } + assert station.name == "BURBANK-GLENDALE-PASA ARPT" + assert station.latitude == 34.2 + assert station.longitude == -118.365 + assert station.elevation == 222.7 + assert station.coords == (34.2, -118.365) + assert station.country == "US" + assert station.subdivision == "CA" + assert station.quality == "high" + assert station.zones == { + "ba_climate_zone": "Hot-Dry", + "ca_climate_zone": "CA_09", + "iecc_climate_zone": "3", + "iecc_moisture_regime": "B", + } + assert station.inventory_years["ghcnh"][0] == 1943 + + +def test_station_unrecognized_id(): + with pytest.raises(UnrecognizedStationError): + WeatherStation("FAKE") + + +def test_station_json(): + station = WeatherStation("USW00023152") + serialized = station.json() + + assert serialized["id"] == "USW00023152" + assert serialized["ids"]["usaf"] == ["722880"] + assert serialized["name"] == "BURBANK-GLENDALE-PASA ARPT" + assert serialized["latitude"] == 34.2 + assert serialized["longitude"] == -118.365 + assert serialized["elevation"] == 222.7 + assert serialized["country"] == "US" + assert serialized["subdivision"] == "CA" + assert serialized["quality"] == "high" + assert serialized["zones"]["ca_climate_zone"] == "CA_09" + assert serialized["inventory_years"]["ghcnh"][0] == 1943 + + +def test_station_from_id(): + station = WeatherStation.from_id("usaf", "722880") + + assert station.id == "USW00023152" + + +def test_station_from_usaf(): + station = WeatherStation.from_usaf("722874") + + assert station.id == "USW00093134" + + +def test_station_from_wban(): + station = WeatherStation.from_wban("23152") + + assert station.id == "USW00023152" + + +def test_station_from_wban_recent_mapping_wins(): + # wban 03935 was reused; only one station holds it recently + station = WeatherStation.from_wban("03935") + + assert station.id == "USW00003935" + + +def test_station_from_icao(): + station = WeatherStation.from_icao("KBUR") + + assert station.id == "USW00023152" + + +def test_station_from_id_wmo(): + # WMO aliases come from the GHCNh station list + station = WeatherStation.from_id("wmo", "72494") + + assert station.id == "USW00023234" + + +def test_station_from_id_unrecognized(): + with pytest.raises(UnrecognizedStationError): + WeatherStation.from_id("usaf", "000000") + + +def test_station_with_null_elevation(): + # this station's isd history record carries no elevation + station = WeatherStation("USI0000KEGI") + + assert station.elevation is None + + +def test_station_load_data(mock_api_transport, monkeypatch_key_value_store): + station = WeatherStation("USW00093134") + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 4, 3, tzinfo=timezone.utc) + + df, warnings = station.load_data(start, end) + + assert df.index[0] == start + assert df.index[-1] == end + assert list(df.columns) == ["temperature"] + + +def test_station_load_data_records_provenance( + mock_api_transport, monkeypatch_key_value_store +): + station = WeatherStation("USW00093134") + assert station.provenance is None + + df, warnings = station.load_data( + datetime(2007, 1, 1, tzinfo=timezone.utc), + datetime(2007, 4, 3, tzinfo=timezone.utc), + ) + + assert station.provenance == df.attrs["provenance"] + record = station.provenance["ghcnh"] + assert record.station_id == "USW00093134" + assert record.variables == ("temperature",) + + +def test_station_sources_accepts_single_string(): + station = WeatherStation("USW00093134", sources="ghcnh") + + assert station.sources == ("ghcnh",) + + +def test_station_duplicate_source_names_raise(): + with pytest.raises(ValueError, match="Duplicate source names"): + WeatherStation("USW00093134", sources=("ghcnh", "ghcnh")) + + +def test_station_wrong_kind_source_raises(): + with pytest.raises(ValueError, match="observation sources"): + WeatherStation("USW00093134", sources=("tmy3",)) + + +class _TwoVariableFeed(Feed): + """A test double whose default is two variables, so a pinned load with + variables=None must yield both columns.""" + + name = "twovar-feed" + id_namespace = "ghcn" + variables = ("temperature", "relative_humidity") + default_variables = ("temperature", "relative_humidity") + cacheable = False + + def fetch_year(self, external_id, year, variables): + index = pd.date_range( + "{}-01-01".format(year), "{}-12-31 23:00".format(year), + freq="h", tz="UTC", + ) + df = pd.DataFrame(index=index) + for variable in variables: + df[variable] = 1.0 + + return df + + +def test_station_load_data_default_variables_match_location( + monkeypatch_key_value_store +): + # both entry points must send variables=None through the same + # None -> adapter.default_variables path; a source whose default is + # NOT ("temperature",) makes the parity observable (a reverted + # WeatherStation default would drop relative_humidity). + feed = _TwoVariableFeed() + station = WeatherStation("USW00023152") + location = WeatherLocation(34.2, -118.365) + start = datetime(2007, 1, 1, tzinfo=timezone.utc) + end = datetime(2007, 4, 3, tzinfo=timezone.utc) + + station_df, _ = station.load_data(start, end, source=feed) + location_df, _ = location.load_data(start, end, source=feed) + + assert list(station_df.columns) == ["temperature", "relative_humidity"] + assert list(location_df.columns) == ["temperature", "relative_humidity"] + + +def test_station_get_quality(): + station = WeatherStation("USW00093134") + anchor = datetime(2012, 12, 31, tzinfo=timezone.utc) + + assert station.get_quality(anchor) == "high" + + +def test_station_search(): + df = WeatherStation.search() + + assert len(df) == 5891 + assert df.loc["USW00023152", "name"] == "BURBANK-GLENDALE-PASA ARPT" + assert df.loc["USW00023152", "quality"] == "high" + assert bool(df.loc["USW00023152", "is_tmy3"]) is True + + +def test_station_search_filters(): + df = WeatherStation.search(country="AU") + + assert len(df) == 937 + assert (df.country == "AU").all() + + df = WeatherStation.search(subdivision="CA", has_sources=("cz2010",)) + + assert len(df) == 86 + assert df.is_cz2010.all() + + df = WeatherStation.search(has_sources=("ghcnh",)) + + assert len(df) == 5891 + + +def test_station_search_unknown_source(): + with pytest.raises(ValueError, match="Unknown source"): + WeatherStation.search(has_sources=("not_a_source",)) + + +def test_station_translate(): + mapping = WeatherStation.translate(["722880"], "usaf", "ghcn") + + assert mapping == {"722880": ("USW00023152",)} diff --git a/tests/test_stations.py b/tests/test_stations.py deleted file mode 100644 index 97ac997..0000000 --- a/tests/test_stations.py +++ /dev/null @@ -1,1295 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from datetime import datetime -import pandas as pd -import contextlib -import pytest -import sqlite3 -import pytz - -from eeweather import ( - WeatherStation, - get_ghcn_id, - get_ghcn_ids, - get_isd_station_metadata, - get_station_quality, - get_station_qualities, - get_isd_file_metadata, - fetch_hourly_data, - fetch_tmy3_hourly_temp_data, - fetch_cz2010_hourly_temp_data, - get_hourly_data_cache_key, - get_tmy3_hourly_temp_data_cache_key, - get_cz2010_hourly_temp_data_cache_key, - cached_hourly_data_is_expired, - validate_hourly_data_cache, - validate_tmy3_hourly_temp_data_cache, - validate_cz2010_hourly_temp_data_cache, - serialize_hourly_data, - serialize_tmy3_hourly_temp_data, - serialize_cz2010_hourly_temp_data, - deserialize_hourly_data, - deserialize_tmy3_hourly_temp_data, - deserialize_cz2010_hourly_temp_data, - read_hourly_data_from_cache, - read_tmy3_hourly_temp_data_from_cache, - read_cz2010_hourly_temp_data_from_cache, - write_hourly_data_to_cache, - write_tmy3_hourly_temp_data_to_cache, - write_cz2010_hourly_temp_data_to_cache, - destroy_cached_hourly_data, - destroy_cached_tmy3_hourly_temp_data, - destroy_cached_cz2010_hourly_temp_data, - load_hourly_data_cached_proxy, - load_tmy3_hourly_temp_data_cached_proxy, - load_cz2010_hourly_temp_data_cached_proxy, - load_data, - load_tmy3_hourly_temp_data, - load_cz2010_hourly_temp_data, - load_cached_hourly_data, - load_cached_tmy3_hourly_temp_data, - load_cached_cz2010_hourly_temp_data, -) -from eeweather.exceptions import ( - UnrecognizedUSAFIDError, - DataNotAvailableError, - TMY3DataNotAvailableError, - CZ2010DataNotAvailableError, - NonUTCTimezoneInfoError, -) -from conftest import ( - MockKeyValueStoreProxy, - mock_request_text_tmy3, - mock_request_text_cz2010, -) - - - -@pytest.fixture -def monkeypatch_tmy3_request(monkeypatch): - monkeypatch.setattr("eeweather.mockable.request_text", mock_request_text_tmy3) - - -@pytest.fixture -def monkeypatch_cz2010_request(monkeypatch): - monkeypatch.setattr("eeweather.mockable.request_text", mock_request_text_cz2010) - - -@pytest.fixture -def monkeypatch_key_value_store(monkeypatch): - key_value_store_proxy = MockKeyValueStoreProxy() - monkeypatch.setattr( - "eeweather.connections.key_value_store_proxy", key_value_store_proxy - ) - - return key_value_store_proxy.get_store() - - -def _backdate_cache_key(store, key, updated): - with contextlib.closing(sqlite3.connect(store._path)) as conn, conn: - conn.execute( - "update items set updated = ? where key = ?", (updated.isoformat(), key) - ) - - - -def test_get_isd_station_metadata(): - assert get_isd_station_metadata("722874") == { - "ba_climate_zone": "Hot-Dry", - "ca_climate_zone": "CA_08", - "elevation": "+0054.6", - "icao_code": "KCQT", - "iecc_climate_zone": "3", - "iecc_moisture_regime": "B", - "latitude": "+34.024", - "longitude": "-118.291", - "name": "DOWNTOWN L.A./USC CAMPUS", - "ghcn_id": "USW00093134", - "ghcn_map_method": "icao", - "ghcn_first_year": 1893, - "ghcn_last_year": 2024, - "quality": "low", - "recent_wban_id": "93134", - "state": "CA", - "usaf_id": "722874", - "wban_ids": "93134", - } - - -def test_isd_station_no_load_metadata(): - station = WeatherStation("722880", load_metadata=False) - assert station.usaf_id == "722880" - assert station.iecc_climate_zone is None - assert station.iecc_moisture_regime is None - assert station.ba_climate_zone is None - assert station.ca_climate_zone is None - assert station.elevation is None - assert station.latitude is None - assert station.longitude is None - assert station.coords is None - assert station.name is None - assert station.quality is None - assert station.wban_ids is None - assert station.recent_wban_id is None - assert station.climate_zones == {} - - assert str(station) == "722880" - assert repr(station) == "WeatherStation('722880')" - - -def test_isd_station_no_load_metadata_invalid(): - with pytest.raises(UnrecognizedUSAFIDError): - station = WeatherStation("FAKE", load_metadata=False) - - -def test_isd_station_with_load_metadata(): - station = WeatherStation("722880", load_metadata=True) - assert station.usaf_id == "722880" - assert station.iecc_climate_zone == "3" - assert station.iecc_moisture_regime == "B" - assert station.ba_climate_zone == "Hot-Dry" - assert station.ca_climate_zone == "CA_09" - assert station.elevation == 222.7 - assert station.icao_code == "KBUR" - assert station.latitude == 34.2 - assert station.longitude == -118.365 - assert station.coords == (34.2, -118.365) - assert station.name == "BURBANK-GLENDALE-PASA ARPT" - assert station.quality == "high" - assert station.wban_ids == ["23152", "99999"] - assert station.recent_wban_id == "23152" - assert station.ghcn_id == "USW00023152" - assert station.ghcn_first_year == 1943 - assert station.ghcn_last_year == 2026 - assert station.climate_zones == { - "ba_climate_zone": "Hot-Dry", - "ca_climate_zone": "CA_09", - "iecc_climate_zone": "3", - "iecc_moisture_regime": "B", - } - - -def test_isd_station_json(): - station = WeatherStation("722880", load_metadata=True) - assert station.json() == { - "elevation": 222.7, - "icao_code": "KBUR", - "latitude": 34.2, - "longitude": -118.365, - "name": "BURBANK-GLENDALE-PASA ARPT", - "quality": "high", - "recent_wban_id": "23152", - "ghcn_id": "USW00023152", - "ghcn_first_year": 1943, - "ghcn_last_year": 2026, - "wban_ids": ["23152", "99999"], - "climate_zones": { - "ba_climate_zone": "Hot-Dry", - "ca_climate_zone": "CA_09", - "iecc_climate_zone": "3", - "iecc_moisture_regime": "B", - }, - } - - -def test_isd_station_unrecognized_usaf_id(): - with pytest.raises(UnrecognizedUSAFIDError): - station = WeatherStation("FAKE", load_metadata=True) - - -def test_get_isd_file_metadata(): - assert get_isd_file_metadata("722874") == [ - {"usaf_id": "722874", "wban_id": "93134", "year": "2006"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2007"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2008"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2009"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2010"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2011"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2012"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2013"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2014"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2015"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2016"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2017"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2018"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2019"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2020"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2021"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2022"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2023"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2024"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2025"}, - ] - - with pytest.raises(UnrecognizedUSAFIDError) as excinfo: - get_isd_file_metadata("000000") - assert excinfo.value.value == "000000" - - -def test_isd_station_get_isd_file_metadata(): - station = WeatherStation("722874") - assert station.get_isd_file_metadata() == [ - {"usaf_id": "722874", "wban_id": "93134", "year": "2006"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2007"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2008"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2009"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2010"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2011"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2012"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2013"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2014"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2015"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2016"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2017"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2018"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2019"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2020"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2021"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2022"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2023"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2024"}, - {"usaf_id": "722874", "wban_id": "93134", "year": "2025"}, - ] - - -def test_fetch_tmy3_hourly_temp_data(monkeypatch_tmy3_request): - data = fetch_tmy3_hourly_temp_data("722880") - assert data.sum() == pytest.approx(156194.3, 0.00001) - assert data.shape == (8760,) - - -def test_fetch_cz2010_hourly_temp_data(monkeypatch_cz2010_request): - data = fetch_cz2010_hourly_temp_data("722880") - assert data.sum() == pytest.approx(153430.9, 0.00001) - assert data.shape == (8760,) - - -def test_tmy3_station_hourly_temp_data(monkeypatch_tmy3_request): - station = WeatherStation("722880") - data = station.fetch_tmy3_hourly_temp_data() - assert data.sum() == pytest.approx(156194.3, 0.00001) - assert data.shape == (8760,) - - -def test_cz2010_station_hourly_temp_data(monkeypatch_cz2010_request): - station = WeatherStation("722880") - data = station.fetch_cz2010_hourly_temp_data() - assert data.sum() == pytest.approx(153430.9, 0.00001) - assert data.shape == (8760,) - - -def test_fetch_tmy3_hourly_temp_data_invalid(): - with pytest.raises(TMY3DataNotAvailableError): - fetch_tmy3_hourly_temp_data("INVALID") - - -def test_fetch_cz2010_hourly_temp_data_invalid(): - with pytest.raises(CZ2010DataNotAvailableError): - fetch_cz2010_hourly_temp_data("INVALID") - - -def test_fetch_tmy3_hourly_temp_data_not_in_tmy3_list(): - with pytest.raises(TMY3DataNotAvailableError): - fetch_tmy3_hourly_temp_data("722874") - - -def test_fetch_cz2010_hourly_temp_data_not_in_cz2010_list(monkeypatch_cz2010_request): - data = fetch_cz2010_hourly_temp_data("722880") - assert data.sum() == 153430.90000000002 - assert data.shape == (8760,) - with pytest.raises(CZ2010DataNotAvailableError): - fetch_cz2010_hourly_temp_data("725340") - - -def test_get_tmy3_hourly_temp_data_cache_key(): - assert get_tmy3_hourly_temp_data_cache_key("722880") == "tmy3-hourly-722880" - - -def test_get_cz2010_hourly_temp_data_cache_key(): - assert get_cz2010_hourly_temp_data_cache_key("722880") == "cz2010-hourly-722880" - - -def test_tmy3_station_get_isd_hourly_temp_data_cache_key(): - station = WeatherStation("722880") - assert station.get_tmy3_hourly_temp_data_cache_key() == "tmy3-hourly-722880" - - -def test_cz2010_station_get_isd_hourly_temp_data_cache_key(): - station = WeatherStation("722880") - assert station.get_cz2010_hourly_temp_data_cache_key() == "cz2010-hourly-722880" - - -def test_validate_tmy3_hourly_temp_data_cache_empty(monkeypatch_key_value_store): - assert validate_tmy3_hourly_temp_data_cache("722880") is False - - -def test_validate_cz2010_hourly_temp_data_cache_empty(monkeypatch_key_value_store): - assert validate_cz2010_hourly_temp_data_cache("722880") is False - - -def test_isd_station_validate_tmy3_hourly_temp_data_cache_empty( - monkeypatch_key_value_store, -): - station = WeatherStation("722880") - assert station.validate_tmy3_hourly_temp_data_cache() is False - - -def test_isd_station_validate_cz2010_hourly_temp_data_cache_empty( - monkeypatch_key_value_store, -): - station = WeatherStation("722880") - assert station.validate_cz2010_hourly_temp_data_cache() is False - - -def test_raise_on_missing_tmy3_hourly_temp_data_cache_data_no_web_fetch( - mock_api_transport, monkeypatch_key_value_store -): - with pytest.raises(TMY3DataNotAvailableError): - load_tmy3_hourly_temp_data_cached_proxy("722874", fetch_from_web=False) - - -def test_raise_on_missing_cz2010_hourly_temp_data_cache_data_no_web_fetch( - mock_api_transport, monkeypatch_key_value_store -): - with pytest.raises(CZ2010DataNotAvailableError): - load_cz2010_hourly_temp_data_cached_proxy("722874", fetch_from_web=False) - - -def test_validate_tmy3_hourly_temp_data_cache_updated_recently( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - load_tmy3_hourly_temp_data_cached_proxy("722880") - assert validate_tmy3_hourly_temp_data_cache("722880") is True - - -def test_validate_cz2010_hourly_temp_data_cache_updated_recently( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - load_cz2010_hourly_temp_data_cached_proxy("722880") - assert validate_cz2010_hourly_temp_data_cache("722880") is True - - -def test_serialize_tmy3_hourly_temp_data(): - ts = pd.Series([1], index=[pytz.UTC.localize(datetime(2017, 1, 1))]) - assert serialize_tmy3_hourly_temp_data(ts) == [["2017010100", 1]] - - -def test_serialize_cz2010_hourly_temp_data(): - ts = pd.Series([1], index=[pytz.UTC.localize(datetime(2017, 1, 1))]) - assert serialize_cz2010_hourly_temp_data(ts) == [["2017010100", 1]] - - -def test_isd_station_serialize_tmy3_hourly_temp_data(): - station = WeatherStation("722880") - ts = pd.Series([1], index=[pytz.UTC.localize(datetime(2017, 1, 1))]) - assert station.serialize_tmy3_hourly_temp_data(ts) == [["2017010100", 1]] - - -def test_isd_station_serialize_cz2010_hourly_temp_data(): - station = WeatherStation("722880") - ts = pd.Series([1], index=[pytz.UTC.localize(datetime(2017, 1, 1))]) - assert station.serialize_cz2010_hourly_temp_data(ts) == [["2017010100", 1]] - - -def test_deserialize_tmy3_hourly_temp_data(): - ts = deserialize_tmy3_hourly_temp_data([["2017010100", 1]]) - assert ts.sum() == 1 - assert ts.index.freq.name == "h" - - -def test_deserialize_cz2010_hourly_temp_data(): - ts = deserialize_cz2010_hourly_temp_data([["2017010100", 1]]) - assert ts.sum() == 1 - assert ts.index.freq.name == "h" - - -def test_isd_station_deserialize_tmy3_hourly_temp_data(): - station = WeatherStation("722880") - ts = station.deserialize_tmy3_hourly_temp_data([["2017010100", 1]]) - assert ts.sum() == 1 - assert ts.index.freq.name == "h" - - -def test_isd_station_deserialize_cz2010_hourly_temp_data(): - station = WeatherStation("722880") - ts = station.deserialize_cz2010_hourly_temp_data([["2017010100", 1]]) - assert ts.sum() == 1 - assert ts.index.freq.name == "h" - - -def test_write_read_destroy_tmy3_hourly_temp_data_to_from_cache( - monkeypatch_key_value_store, -): - store = monkeypatch_key_value_store - key = get_tmy3_hourly_temp_data_cache_key("123456") - assert store.key_exists(key) is False - - ts1 = pd.Series([1], index=[pytz.UTC.localize(datetime(1990, 1, 1))]) - write_tmy3_hourly_temp_data_to_cache("123456", ts1) - assert store.key_exists(key) is True - - ts2 = read_tmy3_hourly_temp_data_from_cache("123456") - assert store.key_exists(key) is True - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - destroy_cached_tmy3_hourly_temp_data("123456") - assert store.key_exists(key) is False - - -def test_write_read_destroy_cz2010_hourly_temp_data_to_from_cache( - monkeypatch_key_value_store, -): - store = monkeypatch_key_value_store - key = get_cz2010_hourly_temp_data_cache_key("123456") - assert store.key_exists(key) is False - - ts1 = pd.Series([1], index=[pytz.UTC.localize(datetime(1990, 1, 1))]) - write_cz2010_hourly_temp_data_to_cache("123456", ts1) - assert store.key_exists(key) is True - - ts2 = read_cz2010_hourly_temp_data_from_cache("123456") - assert store.key_exists(key) is True - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - destroy_cached_cz2010_hourly_temp_data("123456") - assert store.key_exists(key) is False - - -def test_isd_station_write_read_destroy_tmy3_hourly_temp_data_to_from_cache( - monkeypatch_key_value_store, -): - station = WeatherStation("722880") - store = monkeypatch_key_value_store - key = station.get_tmy3_hourly_temp_data_cache_key() - assert store.key_exists(key) is False - - ts1 = pd.Series([1], index=[pytz.UTC.localize(datetime(1990, 1, 1))]) - station.write_tmy3_hourly_temp_data_to_cache(ts1) - assert store.key_exists(key) is True - - ts2 = station.read_tmy3_hourly_temp_data_from_cache() - assert store.key_exists(key) is True - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - station.destroy_cached_tmy3_hourly_temp_data() - assert store.key_exists(key) is False - - -def test_isd_station_write_read_destroy_cz2010_hourly_temp_data_to_from_cache( - monkeypatch_key_value_store, -): - station = WeatherStation("722880") - store = monkeypatch_key_value_store - key = station.get_cz2010_hourly_temp_data_cache_key() - assert store.key_exists(key) is False - - ts1 = pd.Series([1], index=[pytz.UTC.localize(datetime(1990, 1, 1))]) - station.write_cz2010_hourly_temp_data_to_cache(ts1) - assert store.key_exists(key) is True - - ts2 = station.read_cz2010_hourly_temp_data_from_cache() - assert store.key_exists(key) is True - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - station.destroy_cached_cz2010_hourly_temp_data() - assert store.key_exists(key) is False - - -def test_load_tmy3_hourly_temp_data_cached_proxy( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - # doesn't yet guarantee that all code paths are taken, - # except that coverage picks it up either here or elsewhere - ts1 = load_tmy3_hourly_temp_data_cached_proxy("722880", 2007) - ts2 = load_tmy3_hourly_temp_data_cached_proxy("722880", 2007) - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - -def test_load_cz2010_hourly_temp_data_cached_proxy( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - # doesn't yet guarantee that all code paths are taken, - # except that coverage picks it up either here or elsewhere - ts1 = load_cz2010_hourly_temp_data_cached_proxy("722880", 2007) - ts2 = load_cz2010_hourly_temp_data_cached_proxy("722880", 2007) - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - -def test_isd_station_load_tmy3_hourly_temp_data_cached_proxy( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - - # doesn't yet guarantee that all code paths are taken, - # except that coverage picks it up either here or elsewhere - ts1 = station.load_tmy3_hourly_temp_data_cached_proxy() - ts2 = station.load_tmy3_hourly_temp_data_cached_proxy() - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - -def test_isd_station_load_cz2010_hourly_temp_data_cached_proxy( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - - # doesn't yet guarantee that all code paths are taken, - # except that coverage picks it up either here or elsewhere - ts1 = station.load_cz2010_hourly_temp_data_cached_proxy() - ts2 = station.load_cz2010_hourly_temp_data_cached_proxy() - assert int(ts1.sum()) == int(ts2.sum()) - assert ts1.shape == ts2.shape - - -def test_load_tmy3_hourly_temp_data( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - start = datetime(2006, 1, 3, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - ts = load_tmy3_hourly_temp_data("722880", start, end) - assert ts.index[0] == start - assert pd.notnull(ts.iloc[0]) - assert ts.index[-1] == end - assert pd.notnull(ts.iloc[-1]) - - -def test_load_cz2010_hourly_temp_data( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - start = datetime(2006, 1, 3, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - ts = load_cz2010_hourly_temp_data("722880", start, end) - assert ts.index[0] == start - assert pd.notnull(ts.iloc[1]) - assert ts.index[-1] == end - assert pd.notnull(ts.iloc[-1]) - - -def test_isd_station_load_tmy3_hourly_temp_data( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - start = datetime(2007, 3, 3, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - ts = station.load_tmy3_hourly_temp_data(start, end) - assert ts.index[0] == start - assert ts.index[-1] == end - - -def test_isd_station_load_cz2010_hourly_temp_data( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - start = datetime(2007, 3, 3, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - ts = station.load_cz2010_hourly_temp_data(start, end) - assert ts.index[0] == start - assert ts.index[-1] == end - - -def test_load_cached_tmy3_hourly_temp_data( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - ts = load_cached_tmy3_hourly_temp_data("722880") - assert ts is None - - # load data - ts = load_tmy3_hourly_temp_data_cached_proxy("722880") - assert int(ts.sum()) == 156194 - assert ts.shape == (8760,) - - ts = load_cached_tmy3_hourly_temp_data("722880") - assert int(ts.sum()) == 156194 - assert ts.shape == (8760,) - - -def test_load_cached_cz2010_hourly_temp_data( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - ts = load_cached_cz2010_hourly_temp_data("722880") - assert ts is None - - # load data - ts = load_cz2010_hourly_temp_data_cached_proxy("722880") - assert int(ts.sum()) == 153430 - assert ts.shape == (8760,) - - ts = load_cached_cz2010_hourly_temp_data("722880") - assert int(ts.sum()) == 153430 - assert ts.shape == (8760,) - - -def test_isd_station_load_cached_tmy3_hourly_temp_data( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - - ts = station.load_cached_tmy3_hourly_temp_data() - assert ts is None - - # load data - ts = station.load_tmy3_hourly_temp_data_cached_proxy() - assert int(ts.sum()) == 156194 - assert ts.shape == (8760,) - - ts = station.load_cached_tmy3_hourly_temp_data() - assert int(ts.sum()) == 156194 - assert ts.shape == (8760,) - - -def test_isd_station_load_cached_cz2010_hourly_temp_data( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - - ts = station.load_cached_cz2010_hourly_temp_data() - assert ts is None - - # load data - ts = station.load_cz2010_hourly_temp_data_cached_proxy() - assert int(ts.sum()) == 153430 - assert ts.shape == (8760,) - - ts = station.load_cached_cz2010_hourly_temp_data() - assert int(ts.sum()) == 153430 - assert ts.shape == (8760,) - - -def test_load_correctly_sliced_tmy3_hourly_temp_data( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - start = datetime(2015, 2, 15, tzinfo=pytz.UTC) - end = datetime(2016, 8, 12, tzinfo=pytz.UTC) - - ts = load_tmy3_hourly_temp_data("722880", start, end) - ts_orig = fetch_tmy3_hourly_temp_data("722880") - - for i in ts.index: - # leap day is null - if i.month == 2 and i.day == 29: - assert pd.isnull(ts[i]) - else: - assert ts[i] == ts_orig[i.replace(year=1900)] - - -def test_load_correctly_sliced_cz2010_hourly_temp_data( - monkeypatch_cz2010_request, monkeypatch_key_value_store -): - start = datetime(2015, 2, 15, tzinfo=pytz.UTC) - end = datetime(2016, 8, 12, tzinfo=pytz.UTC) - - ts = load_cz2010_hourly_temp_data("722880", start, end) - ts_orig = fetch_cz2010_hourly_temp_data("722880") - - for i in ts.index: - # leap day is null - if i.month == 2 and i.day == 29: - assert pd.isnull(ts[i]) - else: - assert ts[i] == ts_orig[i.replace(year=1900)] - - -def test_isd_station_load_tmy3_hourly_temp_data_tz_exception( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - start = datetime(2007, 4, 10) - end = datetime(2007, 4, 12) - with pytest.raises(NonUTCTimezoneInfoError): - ts = station.load_tmy3_hourly_temp_data(start, end) - - start = datetime(2007, 4, 10, tzinfo=pytz.UTC) - end = datetime(2007, 4, 12) - with pytest.raises(NonUTCTimezoneInfoError): - ts = station.load_tmy3_hourly_temp_data(start, end) - - -def test_isd_station_load_cz2010_hourly_temp_data_tz_exception( - monkeypatch_tmy3_request, monkeypatch_key_value_store -): - station = WeatherStation("722880") - start = datetime(2007, 4, 10) - end = datetime(2007, 4, 12) - with pytest.raises(NonUTCTimezoneInfoError): - ts = station.load_cz2010_hourly_temp_data(start, end) - - start = datetime(2007, 4, 10, tzinfo=pytz.UTC) - end = datetime(2007, 4, 12) - with pytest.raises(NonUTCTimezoneInfoError): - ts = station.load_cz2010_hourly_temp_data(start, end) - - -def test_isd_station_metadata_null_elevation(): - usaf_id = "722246" - metadata = get_isd_station_metadata(usaf_id) - assert metadata["elevation"] is None - isd_station = WeatherStation(usaf_id) - assert isd_station.elevation is None - - -# ghcn id lookup -def test_get_ghcn_id(): - assert get_ghcn_id("722874") == "USW00093134" - - -def test_get_ghcn_id_unrecognized(): - with pytest.raises(UnrecognizedUSAFIDError): - get_ghcn_id("FAKE") - - -# fetch -def test_fetch_hourly_data(mock_api_transport): - df = fetch_hourly_data("722874", 2007) - - assert list(df.columns) == ["temperature"] - assert df.shape == (8760, 1) - assert df.index[0] == datetime(2007, 1, 1, tzinfo=pytz.UTC) - assert df.temperature.sum() == pytest.approx(156159.5455, abs=1e-3) - - -def test_fetch_hourly_data_multiple_variables(mock_api_transport): - df = fetch_hourly_data( - "722874", 2007, variables=("temperature", "relative_humidity") - ) - - assert list(df.columns) == ["temperature", "relative_humidity"] - assert df.shape == (8760, 2) - - -def test_fetch_hourly_data_invalid_station(): - with pytest.raises(UnrecognizedUSAFIDError): - fetch_hourly_data("FAKE", 2007) - - -def test_fetch_hourly_data_missing_year_raises(mock_api_transport): - with pytest.raises(DataNotAvailableError): - fetch_hourly_data("722874", 1800) - - -def test_weather_station_fetch_hourly_data(mock_api_transport): - station = WeatherStation("722874") - df = fetch_hourly_data(station.usaf_id, 2007) - - assert df.shape == (8760, 1) - - -# cache keys -def test_get_hourly_data_cache_key(): - assert get_hourly_data_cache_key("722874", 2007) == "ghcnh-hourly-722874-2007" - - -# cache expiry -def test_cached_hourly_data_is_expired_empty(monkeypatch_key_value_store): - assert cached_hourly_data_is_expired("722874", 2007) is True - - -def test_cached_hourly_data_is_expired_false( - mock_api_transport, monkeypatch_key_value_store -): - load_hourly_data_cached_proxy("722874", 2007) - - assert cached_hourly_data_is_expired("722874", 2007) is False - - -def test_cached_hourly_data_is_expired_true( - mock_api_transport, monkeypatch_key_value_store -): - load_hourly_data_cached_proxy("722874", 2007) - - # manually expire key value item - key = get_hourly_data_cache_key("722874", 2007) - _backdate_cache_key( - monkeypatch_key_value_store, key, pytz.UTC.localize(datetime(2007, 3, 3)) - ) - - assert cached_hourly_data_is_expired("722874", 2007) is True - - -# validate cache -def test_validate_hourly_data_cache_empty(monkeypatch_key_value_store): - assert validate_hourly_data_cache("722874", 2007) is False - - -def test_validate_hourly_data_cache_updated_recently( - mock_api_transport, monkeypatch_key_value_store -): - load_hourly_data_cached_proxy("722874", 2007) - - assert validate_hourly_data_cache("722874", 2007) is True - - -def test_validate_hourly_data_cache_expired( - mock_api_transport, monkeypatch_key_value_store -): - load_hourly_data_cached_proxy("722874", 2007) - - key = get_hourly_data_cache_key("722874", 2007) - _backdate_cache_key( - monkeypatch_key_value_store, key, pytz.UTC.localize(datetime(2007, 3, 3)) - ) - - # expired cache entries are cleared on validation - assert validate_hourly_data_cache("722874", 2007) is False - assert monkeypatch_key_value_store.key_exists(key) is False - - -def test_raise_on_missing_hourly_data_cache_no_web_fetch(monkeypatch_key_value_store): - with pytest.raises(DataNotAvailableError): - load_hourly_data_cached_proxy("722874", 2007, fetch_from_web=False) - - -# serialization round-trips -def test_serialize_deserialize_hourly_data_round_trip(mock_api_transport): - df = fetch_hourly_data("722874", 2007) - - serialized = serialize_hourly_data(df) - - assert serialized["columns"] == ["temperature"] - assert serialized["rows"][0][0] == "2007010100" - assert len(serialized["rows"]) == len(df) - - round_tripped = deserialize_hourly_data(serialized) - - pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) - - -def test_serialize_hourly_data_nan_round_trips_as_null(mock_api_transport): - df = fetch_hourly_data("723826", 2013) # data ends 2013-11-04 - - serialized = serialize_hourly_data(df) - - assert any(row[1] is None for row in serialized["rows"]) - - round_tripped = deserialize_hourly_data(serialized) - - pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) - - -def test_serialize_multivariable_round_trip(mock_api_transport): - df = fetch_hourly_data( - "722874", 2007, variables=("temperature", "wind_speed") - ) - - round_tripped = deserialize_hourly_data(serialize_hourly_data(df)) - - pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) - - -# write read destroy -def test_write_read_destroy_hourly_data_to_from_cache( - mock_api_transport, monkeypatch_key_value_store -): - store = monkeypatch_key_value_store - key = get_hourly_data_cache_key("722874", 2007) - assert store.key_exists(key) is False - - df = fetch_hourly_data("722874", 2007) - write_hourly_data_to_cache("722874", 2007, df) - assert store.key_exists(key) is True - - round_tripped = read_hourly_data_from_cache("722874", 2007) - pd.testing.assert_frame_equal(round_tripped, df, check_freq=False) - - destroy_cached_hourly_data("722874", 2007) - assert store.key_exists(key) is False - - -# cached proxy -def test_load_hourly_data_cached_proxy(mock_api_transport, monkeypatch_key_value_store): - # doesn't yet exist in cache, so fetched - df1 = load_hourly_data_cached_proxy("722874", 2007) - - # now exists in cache, so read - df2 = load_hourly_data_cached_proxy("722874", 2007) - - pd.testing.assert_frame_equal(df1, df2, check_freq=False) - - -def test_load_hourly_data_cached_proxy_variable_superset_refetches( - mock_api_transport, monkeypatch_key_value_store -): - df1 = load_hourly_data_cached_proxy("722874", 2007) - assert list(df1.columns) == ["temperature"] - - # cache holds temperature only, so requesting more refetches the union - df2 = load_hourly_data_cached_proxy( - "722874", 2007, variables=("temperature", "wind_speed") - ) - assert list(df2.columns) == ["temperature", "wind_speed"] - - # the refreshed cache entry now covers both variables - cached = read_hourly_data_from_cache("722874", 2007) - assert set(cached.columns) == {"temperature", "wind_speed"} - - # a temperature-only request serves the requested subset from cache - df3 = load_hourly_data_cached_proxy("722874", 2007) - assert list(df3.columns) == ["temperature"] - - -# load data between dates -def test_load_data_hourly(mock_api_transport, monkeypatch_key_value_store): - start = datetime(2006, 1, 3, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end) - - assert df.index[0] == start - assert df.index[-1] == end - assert len(df) == 10921 - assert int(df.temperature.notna().sum()) == 10893 - assert warnings == [] - - -def test_load_data_hourly_non_normalized_dates( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2006, 1, 3, 11, 12, 13, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, 12, 13, 14, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end) - - assert df.index[0] == datetime(2006, 1, 3, 12, tzinfo=pytz.UTC) - assert df.index[-1] == datetime(2007, 4, 3, 12, tzinfo=pytz.UTC) - - -def test_load_data_daily(mock_api_transport, monkeypatch_key_value_store): - start = datetime(2006, 1, 3, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end, frequency="daily") - - assert df.index[0] == start - assert df.index[-1] == end - assert len(df) == 456 - assert int(df.temperature.notna().sum()) == 456 - assert warnings == [] - - -def test_load_data_daily_non_normalized_dates( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2006, 1, 3, 11, 12, 13, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, 12, 13, 14, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end, frequency="daily") - - assert df.index[0] == datetime(2006, 1, 4, tzinfo=pytz.UTC) - assert df.index[-1] == datetime(2007, 4, 3, tzinfo=pytz.UTC) - - -def test_load_data_invalid_frequency(monkeypatch_key_value_store): - start = datetime(2007, 1, 1, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - - with pytest.raises(ValueError, match="frequency"): - load_data("722874", start, end, frequency="weekly") - - -def test_load_data_tz_exception(): - start = datetime(2007, 1, 1) - end = datetime(2007, 4, 3) - - with pytest.raises(NonUTCTimezoneInfoError): - load_data("722874", start, end) - - -def test_load_data_multiple_variables(mock_api_transport, monkeypatch_key_value_store): - start = datetime(2007, 6, 1, tzinfo=pytz.UTC) - end = datetime(2007, 6, 30, tzinfo=pytz.UTC) - - df, warnings = load_data( - "722874", - start, - end, - variables=("temperature", "relative_humidity", "wind_speed"), - ) - - assert df.shape == (697, 3) - assert df.temperature.mean() == pytest.approx(19.304219, abs=1e-5) - assert df.relative_humidity.mean() == pytest.approx(67.903771, abs=1e-5) - assert df.wind_speed.mean() == pytest.approx(0.843799, abs=1e-5) - assert warnings == [] - - -# regression pins on real captured 2007 data -def test_load_data_hourly_2007_regression_values( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2007, 1, 1, tzinfo=pytz.UTC) - end = datetime(2007, 12, 31, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end) - - assert len(df) == 8737 - assert int(df.temperature.notna().sum()) == 8732 - assert df.temperature.mean() == pytest.approx(17.851812, abs=1e-5) - assert warnings == [] - - -def test_load_data_daily_2007_regression_values( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2007, 1, 1, tzinfo=pytz.UTC) - end = datetime(2007, 12, 31, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end, frequency="daily") - - assert len(df) == 365 - assert int(df.temperature.notna().sum()) == 365 - assert df.temperature.mean() == pytest.approx(17.835431, abs=1e-5) - assert df.temperature.iloc[0] == pytest.approx(13.27734, abs=1e-5) - - -# truncated data warns: station 723826 was decommissioned 2013-11-04 -def test_load_data_warns_on_truncated_data( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2013, 6, 1, tzinfo=pytz.UTC) - end = datetime(2013, 12, 31, tzinfo=pytz.UTC) - - df, warnings = load_data("723826", start, end) - - assert len(df) == 5113 - assert int(df.temperature.notna().sum()) == 1619 - assert df.temperature.last_valid_index() == datetime( - 2013, 11, 4, 19, tzinfo=pytz.UTC - ) - assert [w.qualified_name for w in warnings] == ["eeweather.data_truncated"] - assert warnings[0].data["variable"] == "temperature" - assert warnings[0].data["last_valid"] == "2013-11-04T19:00:00+00:00" - - -def test_load_data_daily_warns_on_truncated_data( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2013, 6, 1, tzinfo=pytz.UTC) - end = datetime(2013, 12, 31, tzinfo=pytz.UTC) - - df, warnings = load_data("723826", start, end, frequency="daily") - - assert len(df) == 214 - assert int(df.temperature.notna().sum()) == 156 - assert [w.qualified_name for w in warnings] == ["eeweather.data_truncated"] - - -# station 720193's 2019 data has a real mid-year outage -def test_load_data_warns_on_internal_gap( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2019, 1, 1, tzinfo=pytz.UTC) - end = datetime(2019, 12, 31, tzinfo=pytz.UTC) - - df, warnings = load_data("720193", start, end) - - assert len(df) == 8737 - assert int(df.temperature.notna().sum()) == 8106 - assert [w.qualified_name for w in warnings] == ["eeweather.data_gap"] - assert warnings[0].data["max_gap_days"] == pytest.approx(16.125, abs=1e-9) - - -# GHCNh continues past the ISD end-of-life: station 724940's 2025 data runs -# through the year while its ISD record stopped 2025-08-27 -def test_load_data_2025_extends_past_isd_end_of_life( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2025, 6, 1, tzinfo=pytz.UTC) - end = datetime(2025, 10, 1, tzinfo=pytz.UTC) - - df, warnings = load_data("724940", start, end) - - assert len(df) == 2929 - assert int(df.temperature.notna().sum()) == 2840 - assert df.temperature.last_valid_index() == end - assert warnings == [] - - -# missing years -def test_load_data_missing_year_strict_raises( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2050, 1, 1, tzinfo=pytz.UTC) - end = datetime(2050, 6, 1, tzinfo=pytz.UTC) - - with pytest.raises(DataNotAvailableError): - load_data("722874", start, end) - - -def test_load_data_missing_year_tolerant_returns_nan_range( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2050, 1, 1, tzinfo=pytz.UTC) - end = datetime(2050, 6, 1, tzinfo=pytz.UTC) - - df, warnings = load_data( - "722874", start, end, error_on_missing_years=False - ) - - assert df.index[0] == start - assert df.index[-1] == end - assert df.temperature.isna().all() - assert [w.qualified_name for w in warnings] == [ - "eeweather.data_not_available", - "eeweather.no_data_in_requested_range", - ] - - -# station 722874 has no GHCNh observations at all in 2025 -def test_load_data_year_with_no_observations( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2025, 1, 1, tzinfo=pytz.UTC) - end = datetime(2025, 6, 1, tzinfo=pytz.UTC) - - with pytest.raises(DataNotAvailableError): - load_data("722874", start, end) - - df, warnings = load_data( - "722874", start, end, error_on_missing_years=False - ) - - assert len(df) == 3625 - assert df.temperature.isna().all() - assert [w.qualified_name for w in warnings] == [ - "eeweather.data_not_available", - "eeweather.no_data_in_requested_range", - ] - - -# a sub-hour range contains no aligned hours; returns empty without warning -def test_load_data_sub_hour_range_returns_empty( - mock_api_transport, monkeypatch_key_value_store -): - start = datetime(2007, 6, 1, 12, 30, tzinfo=pytz.UTC) - end = datetime(2007, 6, 1, 12, 45, tzinfo=pytz.UTC) - - df, warnings = load_data("722874", start, end) - - assert len(df) == 0 - assert warnings == [] - - -# station method -def test_weather_station_load_data(mock_api_transport, monkeypatch_key_value_store): - station = WeatherStation("722874") - start = datetime(2007, 1, 1, tzinfo=pytz.UTC) - end = datetime(2007, 4, 3, tzinfo=pytz.UTC) - - df, warnings = station.load_data(start, end) - - assert df.index[0] == start - assert df.index[-1] == end - assert list(df.columns) == ["temperature"] - - -# load cached -def test_load_cached_hourly_data(mock_api_transport, monkeypatch_key_value_store): - assert load_cached_hourly_data("722874") is None - - df1, _ = load_data( - "722874", - datetime(2007, 1, 1, tzinfo=pytz.UTC), - datetime(2007, 4, 3, tzinfo=pytz.UTC), - ) - cached = load_cached_hourly_data("722874") - - assert cached is not None - assert list(cached.columns) == ["temperature"] - # the cache holds the full fetched year, not just the requested slice - assert len(cached) == 8760 - - -def test_weather_station_load_cached_data( - mock_api_transport, monkeypatch_key_value_store -): - station = WeatherStation("722874") - station.load_data( - datetime(2007, 1, 1, tzinfo=pytz.UTC), - datetime(2007, 4, 3, tzinfo=pytz.UTC), - ) - - cached = station.load_cached_data() - - assert cached is not None - assert len(cached) == 8760 - - station.destroy_cached_hourly_data(2007) - - assert station.load_cached_data() is None - - -# request-period quality ratings: five-year window ending two years -# after the request's last date, sliding back to the last full year -def test_get_station_quality_high_during_active_era(): - start = datetime(2008, 1, 1, tzinfo=pytz.UTC) - end = datetime(2012, 12, 31, tzinfo=pytz.UTC) - - assert get_station_quality("722874", start, end) == "high" - - -def test_get_station_quality_low_after_station_went_quiet(): - start = datetime(2023, 1, 1, tzinfo=pytz.UTC) - end = datetime(2024, 12, 31, tzinfo=pytz.UTC) - - assert get_station_quality("722874", start, end) == "low" - - -def test_get_station_quality_low_before_any_data(): - start = datetime(1850, 1, 1, tzinfo=pytz.UTC) - end = datetime(1851, 1, 1, tzinfo=pytz.UTC) - - assert get_station_quality("722874", start, end) == "low" - - -def test_get_station_qualities_matches_single_station_rating(): - start = datetime(2010, 1, 1, tzinfo=pytz.UTC) - end = datetime(2014, 12, 31, tzinfo=pytz.UTC) - - qualities = get_station_qualities(start, end) - - for usaf_id in ["722874", "722880", "723895"]: - assert qualities[usaf_id] == get_station_quality(usaf_id, start, end) - - -def test_weather_station_get_quality(): - station = WeatherStation("722874") - start = datetime(2008, 1, 1, tzinfo=pytz.UTC) - end = datetime(2012, 12, 31, tzinfo=pytz.UTC) - - assert station.get_quality(start, end) == "high" - - -def test_get_ghcn_ids_whole_registry(): - mapping = get_ghcn_ids() - - assert mapping["722874"] == "USW00093134" - assert mapping["722880"] == "USW00023152" - assert len(mapping) == 4497 - - -def test_get_ghcn_ids_subset_skips_unrecognized(): - mapping = get_ghcn_ids(["722874", "FAKE"]) - - assert mapping.to_dict() == {"722874": "USW00093134"} diff --git a/tests/test_summaries.py b/tests/test_summaries.py deleted file mode 100644 index 70806de..0000000 --- a/tests/test_summaries.py +++ /dev/null @@ -1,25 +0,0 @@ -from eeweather import get_zcta_ids, get_isd_station_usaf_ids - - -def test_get_zcta_ids(snapshot): - zcta_ids = get_zcta_ids() - assert snapshot == len(zcta_ids) - assert zcta_ids[0] == "00601" - - -def test_get_zcta_ids_by_state(snapshot): - zcta_ids = get_zcta_ids(state="CA") - assert snapshot == len(zcta_ids) - assert zcta_ids[0] == "90001" - - -def test_get_isd_station_usaf_ids(snapshot): - usaf_ids = get_isd_station_usaf_ids() - assert snapshot == len(usaf_ids) - assert usaf_ids[0] == "102540" - - -def test_get_isd_station_usaf_ids_by_state(snapshot): - usaf_ids = get_isd_station_usaf_ids(state="IL") - assert snapshot == len(usaf_ids) - assert usaf_ids[0] == "720137" diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 394afa8..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from eeweather.utils import lazy_property - - -def test_lazy_property(): - class NotLazy(object): - n_times = 1 - - @property - def tryme(self): - times = self.n_times - self.n_times += 1 - return times - - class Lazy(object): - n_times = 1 - - @lazy_property - def tryme(self): - times = self.n_times - self.n_times += 1 - return times - - not_lazy = NotLazy() - assert not_lazy.tryme == 1 - assert not_lazy.tryme == 2 # called twice - - lazy = Lazy() - assert lazy.tryme == 1 - assert lazy.tryme == 1 # only called once diff --git a/tests/test_validation.py b/tests/test_validation.py deleted file mode 100644 index e3f6011..0000000 --- a/tests/test_validation.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from eeweather.validation import valid_zcta_or_raise, valid_usaf_id_or_raise -from eeweather.exceptions import UnrecognizedZCTAError, UnrecognizedUSAFIDError -import pytest - - -def test_valid_zcta_or_raise_valid(): - assert valid_zcta_or_raise("90210") is True - - -def test_valid_zcta_or_raise_raise_error(): - with pytest.raises(UnrecognizedZCTAError) as excinfo: - valid_zcta_or_raise("INVALID") - assert excinfo.value.value == "INVALID" - - -def test_valid_usaf_id_or_raise_valid(): - assert valid_usaf_id_or_raise("722880") is True - - -def test_valid_usaf_id_or_raise_raise_error(): - with pytest.raises(UnrecognizedUSAFIDError) as excinfo: - valid_usaf_id_or_raise("INVALID") - assert excinfo.value.value == "INVALID" diff --git a/tests/test_warnings.py b/tests/test_warnings.py deleted file mode 100644 index 92bb3c9..0000000 --- a/tests/test_warnings.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - -Copyright 2018-2023 OpenEEmeter contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import pytest - -from eeweather.warnings import EEWeatherWarning - - -@pytest.fixture -def generic_eeweather_warning(): - return EEWeatherWarning( - qualified_name="qualified_name", description="description", data={} - ) - - -def test_str_repr(generic_eeweather_warning): - assert str(generic_eeweather_warning) == ( - "EEWeatherWarning(qualified_name=qualified_name)" - ) - - -def test_json_repr(generic_eeweather_warning): - assert generic_eeweather_warning.json() == { - "qualified_name": "qualified_name", - "description": "description", - "data": {}, - } diff --git a/tox.ini b/tox.ini index 88421b4..0e79478 100644 --- a/tox.ini +++ b/tox.ini @@ -7,11 +7,13 @@ deps = pytest pytest-cov pytest-xdist + pytz syrupy>=5,<6 commands = pytest tests/ [testenv:coverage] -# Runs serially: the tracer can't attach to xdist workers. +# Runs serially: the tracer can't attach to xdist workers, and -p +# no:xdist would leave the -n flag from addopts unparseable. commands = - pytest tests/ -p no:xdist --cov=eeweather --cov-report=term-missing --cov-report=xml + pytest tests/ -o addopts= --cov=eeweather --cov-report=term-missing --cov-report=xml