From 0a22d06cf2f9e53a4d688fe1376e688a184fe74b Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:24:35 +0200 Subject: [PATCH 01/22] feat(data): add PDS ODE and fixed-mosaic endpoint constants The USGS ARD STAC catalog has no LROC, LOLA, or other lunar collections beyond Kaguya TC. Add the PDS Orbital Data Explorer API root plus the LRO WAC mosaic and LOLA/SLDEM2015 DEM URLs as the new external endpoints. --- src/astrofetch/data/endpoints.py | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/astrofetch/data/endpoints.py b/src/astrofetch/data/endpoints.py index e7156a3..151c0e9 100644 --- a/src/astrofetch/data/endpoints.py +++ b/src/astrofetch/data/endpoints.py @@ -9,3 +9,36 @@ STAC_API_ROOT = "https://stac.astrogeology.usgs.gov/api/" """USGS Astrogeology Analysis Ready Data STAC API root (pystac-client entry).""" + +ODE_API_ROOT = "https://oderest.rsl.wustl.edu/live2/" +"""NASA PDS Orbital Data Explorer (ODE) REST API root, Washington Univ. St. +Louis. Used to search PDS3/PDS4 products (LROC, LOLA, M3, ...) by instrument +and bounding box; the USGS ARD STAC catalog does not carry these instruments. +Last verified 2026-07-20.""" + +LROC_WAC_MOSAIC_100M_URL = ( + "https://asc-pds-services.s3.us-west-2.amazonaws.com/mosaic/" + "Lunar_LRO_LROC-WAC_Mosaic_global_100m_June2013.tif" +) +"""LRO LROC WAC global morphology mosaic, 100 m/px, equirectangular. + +Not a Cloud Optimized GeoTIFF (striped, no overviews): windowed reads at +native resolution (100 m) are efficient; heavily downsampled reads are not. +Last verified 2026-07-20. +""" + +LOLA_DEM_128_URL = ( + "https://pds-geosciences.wustl.edu/lro/lro-l-lola-3-rdr-v1/lrolol_1xxx/" + "data/lola_gdr/cylindrical/float_img/ldem_128_float.lbl" +) +"""LOLA global DEM, 128 px/degree (~237 m/px at the equator), float32 metres +above the IAU 2015 Moon reference sphere. Detached PDS3 label; GDAL's PDS +driver resolves the sibling ``.img`` over HTTPS. Last verified 2026-07-20.""" + +SLDEM2015_URL = ( + "https://pds-geosciences.wustl.edu/lro/lro-l-lola-3-rdr-v1/lrolol_1xxx/" + "data/sldem2015/global/float_img/sldem2015_128_60s_60n_000_360_float.lbl" +) +"""SLDEM2015: LOLA + Kaguya Terrain Camera co-registered DEM, 128 px/degree, +float32 metres. Source coverage is 60S-60N only (not a bug); windows outside +that band read back with ``mask`` all ``False``. Last verified 2026-07-20.""" From 1319fa97748cdc948f7b4729b495697033443b50 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:24:40 +0200 Subject: [PATCH 02/22] feat(data): add PDS Orbital Data Explorer REST client Mirrors data/stac.py: a single retrying, backed-off HTTP session, with failures normalized into EndpointError. Handles ODE's JSON quirks (a lone match comes back as a dict instead of a list, an empty result is the string "No Products Found", errors are HTTP 200 responses with an error message in the body) and converts the internal -180..180 longitude convention to ODE's 0-360 westernlon/easternlon at the query boundary. requests becomes a direct dependency (previously only pulled in transitively via pystac-client). --- pyproject.toml | 1 + src/astrofetch/data/ode.py | 265 +++++++++++++++++++++++++++++++++++++ uv.lock | 2 + 3 files changed, 268 insertions(+) create mode 100644 src/astrofetch/data/ode.py diff --git a/pyproject.toml b/pyproject.toml index f7106a7..432da18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "torch>=2.0", "pystac-client>=0.7", "rasterio>=1.3", + "requests>=2.31", ] [project.urls] diff --git a/src/astrofetch/data/ode.py b/src/astrofetch/data/ode.py new file mode 100644 index 0000000..30fd3e1 --- /dev/null +++ b/src/astrofetch/data/ode.py @@ -0,0 +1,265 @@ +"""Product search against the NASA PDS Orbital Data Explorer (ODE), politely. + +Mirrors :mod:`astrofetch.data.stac`: a single retrying, backed-off HTTP session +(AGENTS rule 5) and failures normalized into :class:`EndpointError`, so callers +never have to know ODE's JSON quirks — a lone result comes back as a dict +instead of a list, an empty result is the string ``"No Products Found"`` +instead of an empty list, and errors are HTTP 200 responses with an error +message in the body. +""" + +from __future__ import annotations + +import re +from functools import cache +from typing import Any, NamedTuple + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from astrofetch.data.endpoints import ODE_API_ROOT +from astrofetch.data.grid import BBox +from astrofetch.errors import EndpointError + +_TIMEOUT_S = 30 +"""Per-request timeout; a stuck archive should fail, not hang a dataloader.""" + +_RETRY = Retry( + total=4, + backoff_factor=0.5, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET"}), +) +"""Exponential backoff on rate-limit and transient server errors.""" + +_PAGE_SIZE = 100 +"""ODE ``limit`` per request; keeps any single request small and polite.""" + +_MAX_PAGES = 20 +"""Hard cap on pages fetched regardless of ``max_products``, so a caller +requesting an unreasonably large ``max_products`` cannot loop indefinitely.""" + + +class ODEFile(NamedTuple): + """One file attached to an ODE product.""" + + filename: str + type: str + """ODE file role, e.g. ``"Product"``, ``"Browse"``, ``"Derived"``.""" + url: str + + +class ODEProduct(NamedTuple): + """One ODE product: its id, files, footprint, and raw metadata.""" + + pdsid: str + files: tuple[ODEFile, ...] + bbox: BBox | None + """(west, south, east, north) in degrees, -180 to 180; ``None`` when the + footprint is not representable as a simple bbox (crosses the antimeridian, + or spans exactly 360 degrees of longitude).""" + metadata: dict[str, Any] + """Raw per-product ODE metadata, unmodified.""" + + +@cache +def _session() -> requests.Session: + session = requests.Session() + adapter = HTTPAdapter(max_retries=_RETRY) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _as_list(value: Any) -> list[Any]: + # ODE returns a bare dict instead of a one-element list when exactly one + # item matches; normalize both shapes to a list. + if value is None: + return [] + return value if isinstance(value, list) else [value] + + +def _normalize_lon(lon: float) -> float: + return ((lon + 180.0) % 360.0) - 180.0 + + +def _to_ode_lon_range(west: float, east: float) -> tuple[float, float]: + # ODE wants 0-360 westernlon/easternlon with westernlon < easternlon. + # Taking `% 360` of each bound independently collapses a full-Moon bbox + # like (-180, 180) to (180, 180) -- a zero-width query that silently + # returns nothing. Shifting `west` into 0-360 and adding back the + # original span avoids that, for any span up to a full 360 degrees. + span = east - west + if span >= 360.0: + return 0.0, 360.0 + w = west % 360.0 + return w, w + span + + +def _product_bbox(meta: dict[str, Any]) -> BBox | None: + try: + west = float(meta["Westernmost_longitude"]) + east = float(meta["Easternmost_longitude"]) + south = float(meta["Minimum_latitude"]) + north = float(meta["Maximum_latitude"]) + except (KeyError, TypeError, ValueError): + return None + west, east = _normalize_lon(west), _normalize_lon(east) + if not (west < east and south < north): + return None + return (west, south, east, north) + + +def _parse_product(raw: dict[str, Any]) -> ODEProduct: + file_entries = _as_list(raw.get("Product_files", {}).get("Product_file")) + files = tuple( + ODEFile(f.get("FileName", ""), f.get("Type", ""), f.get("URL", "")) for f in file_entries + ) + return ODEProduct( + pdsid=raw.get("pdsid", ""), files=files, bbox=_product_bbox(raw), metadata=raw + ) + + +def query_products( + ihid: str, + iid: str, + pt: str, + bbox: BBox, + max_products: int = 20, + root: str = ODE_API_ROOT, +) -> list[ODEProduct]: + """Search ODE for products of one instrument and product type in ``bbox``. + + Args: + ihid: ODE instrument host id, e.g. ``"LRO"``. + iid: ODE instrument id, e.g. ``"LROC"``. + pt: ODE product type, e.g. ``"SDNDTM"``. + bbox: (west, south, east, north) in degrees, -180 to 180. + max_products: cap on products returned; bounds request volume and + paging (fetched in pages of up to 100). + root: ODE API root; defaults to the configured endpoint. + + Returns: + Matching products, possibly empty if none overlap ``bbox``. + + Raises: + EndpointError: the query failed or ODE reported an error. + + Example: + >>> from astrofetch.data.ode import query_products + >>> query_products("LRO", "LROC", "SDNDTM", bbox=(3.0, 25.5, 4.5, 26.5)) # doctest: +SKIP + [ODEProduct(pdsid='sdp.nac_dtm.apollo15...', ...), ...] + """ + west, south, east, north = bbox + lon_west, lon_east = _to_ode_lon_range(west, east) + endpoint = f"{ihid}/{iid}/{pt}" + products: list[ODEProduct] = [] + offset = 0 + for _ in range(_MAX_PAGES): + remaining = max_products - len(products) + if remaining <= 0: + break + limit = min(_PAGE_SIZE, remaining) + params = { + "query": "product", + "results": "fmp", + "output": "JSON", + "target": "moon", + "ihid": ihid, + "iid": iid, + "pt": pt, + "westernlon": lon_west, + "easternlon": lon_east, + "minlat": south, + "maxlat": north, + "limit": limit, + "offset": offset, + } + try: + response = _session().get(root, params=params, timeout=_TIMEOUT_S) + response.raise_for_status() + body = response.json() + except (requests.RequestException, ValueError) as exc: + raise EndpointError(endpoint, f"ODE query failed: {exc}") from exc + + results = body.get("ODEResults", {}) + if results.get("Status") == "ERROR": + raise EndpointError(endpoint, f"ODE error: {results.get('Error')}") + + raw_products = results.get("Products", []) + if isinstance(raw_products, str): # "No Products Found" + break + page = [_parse_product(p) for p in _as_list(raw_products.get("Product"))] + products.extend(page) + if len(page) < limit: + break + offset += limit + return products[:max_products] + + +def match_files( + files: tuple[ODEFile, ...], pattern: str, file_type: str | None = "Product" +) -> list[str]: + """Return URLs of ``files`` whose name matches ``pattern`` and ``file_type``. + + Args: + files: files to filter, typically ``product.files``. + pattern: regex, matched against the filename with ``fullmatch`` and + case-insensitively (ODE filenames are inconsistently cased). + file_type: required ODE file role, e.g. ``"Product"``; ``None`` skips + this filter. + + Returns: + Matching URLs, sorted by filename for deterministic ordering. + """ + regex = re.compile(pattern, re.IGNORECASE) + return sorted( + ( + f.url + for f in files + if (file_type is None or f.type == file_type) and regex.fullmatch(f.filename) + ), + key=lambda url: url.rsplit("/", 1)[-1], + ) + + +def find_file_urls( + ihid: str, + iid: str, + pt: str, + pattern: str, + bbox: BBox, + max_products: int = 20, + file_type: str | None = "Product", + root: str = ODE_API_ROOT, +) -> list[str]: + """Return file URLs matching ``pattern`` across products in ``bbox``. + + The ODE analogue of :func:`astrofetch.data.stac.find_asset_hrefs`: a + product bundle can contain many files (data, browse, derived), so this + searches products then filters their files by name and role. A product + with no matching file is not an error — masks report coverage truthfully + instead. + + Args: + ihid: ODE instrument host id, e.g. ``"LRO"``. + iid: ODE instrument id, e.g. ``"LROC"``. + pt: ODE product type, e.g. ``"SDNDTM"``. + pattern: regex matched against filenames, case-insensitive. + bbox: (west, south, east, north) in degrees, -180 to 180. + max_products: cap on products searched. + file_type: required ODE file role; ``None`` skips this filter. + root: ODE API root; defaults to the configured endpoint. + + Returns: + Matching URLs, product order then filename order. + + Raises: + EndpointError: the search failed or ODE reported an error. + """ + products = query_products(ihid, iid, pt, bbox, max_products, root) + hrefs: list[str] = [] + for product in products: + hrefs.extend(match_files(product.files, pattern, file_type)) + return hrefs diff --git a/uv.lock b/uv.lock index c79d7fa..123f1b1 100644 --- a/uv.lock +++ b/uv.lock @@ -28,6 +28,7 @@ dependencies = [ { name = "pystac-client" }, { name = "rasterio", version = "1.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "rasterio", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "requests" }, { name = "torch" }, ] @@ -51,6 +52,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.24" }, { name = "pystac-client", specifier = ">=0.7" }, { name = "rasterio", specifier = ">=1.3" }, + { name = "requests", specifier = ">=2.31" }, { name = "torch", specifier = ">=2.0" }, ] From 6bd35c93277f0cf07f01e5f1202b881700d09f37 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:24:46 +0200 Subject: [PATCH 03/22] feat(data): support non-reprojected reads and nodata overrides read_window gains an optional nodata_override, for the rare PDS product that omits a nodata value from its label even though its raster does not cover its full requested extent (unwarped pixels would otherwise read back as valid zeros). Add read_full: reads a raster's own native pixels with no reprojection onto a TargetGrid, for sources with no map projection to warp to (raw camera-frame swaths). Used by the upcoming raw-granule datasets. --- src/astrofetch/data/raster.py | 57 ++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/astrofetch/data/raster.py b/src/astrofetch/data/raster.py index a9e44e8..3045bbd 100644 --- a/src/astrofetch/data/raster.py +++ b/src/astrofetch/data/raster.py @@ -17,6 +17,7 @@ from rasterio.enums import Resampling from rasterio.errors import RasterioIOError from rasterio.vrt import WarpedVRT +from rasterio.windows import Window from astrofetch.data.grid import TargetGrid from astrofetch.errors import EndpointError @@ -30,6 +31,7 @@ def read_window( grid: TargetGrid, band: int = 1, resampling: Resampling = Resampling.bilinear, + nodata_override: float | None = None, ) -> tuple[np.ndarray, np.ndarray]: """Read one COG band, reprojected onto ``grid``, in physical units. @@ -38,6 +40,12 @@ def read_window( grid: output grid; defines CRS, size, and extent. band: 1-based band index to read. resampling: resampling used when reprojecting to the grid. + nodata_override: nodata value to use in place of the source's own + (which may be unset). Needed for a handful of PDS products that + omit nodata from their label even though the raster does not + cover its full extent (unwarped pixels would otherwise silently + read back as valid zeros); prefer the source's own declared + nodata whenever a product provides one. Returns: ``(image, mask)`` where ``image`` is a ``(grid.height, grid.width)`` @@ -50,7 +58,7 @@ def read_window( """ try: with rasterio.open(href) as src: - nodata = src.nodata + nodata = src.nodata if nodata_override is None else nodata_override scale = float(src.scales[band - 1]) offset = float(src.offsets[band - 1]) with WarpedVRT( @@ -75,3 +83,50 @@ def read_window( image = raw.astype(np.float32) * np.float32(scale) + np.float32(offset) image[~mask] = _FILL return image, mask + + +def read_full( + href: str, + window: Window | None = None, + bands: list[int] | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """Read a raster's own pixels with no reprojection, in physical units. + + Unlike :func:`read_window`, this does not warp onto a :class:`TargetGrid` + -- it reads the source in its own native geometry, for rasters that have + no map projection to warp to (e.g. raw camera-frame swaths). Used by + :mod:`astrofetch.moon.granules`. + + Args: + href: raster URL or local path. + window: pixel window ``(col_off, row_off, width, height)`` to read; + ``None`` reads the full raster. + bands: 1-based band indices to read; ``None`` reads every band. + + Returns: + ``(image, mask)`` where ``image`` is a ``(bands, height, width)`` + float32 array of physical values (per-band scale/offset applied) + with invalid pixels set to 0, and ``mask`` is a same-shaped bool + array, ``True`` where the pixel is valid. + + Raises: + EndpointError: the raster could not be opened or read. + """ + try: + with rasterio.open(href) as src: + band_list = bands if bands is not None else list(range(1, src.count + 1)) + raw = src.read(band_list, window=window) + nodata = src.nodata + scales = np.array([src.scales[b - 1] for b in band_list], dtype=np.float32) + offsets = np.array([src.offsets[b - 1] for b in band_list], dtype=np.float32) + except RasterioIOError as exc: + raise EndpointError(href, f"could not read raster: {exc}") from exc + + if nodata is not None: + mask = raw != nodata + else: + mask = np.ones(raw.shape, dtype=bool) + + image = raw.astype(np.float32) * scales.reshape(-1, 1, 1) + offsets.reshape(-1, 1, 1) + image[~mask] = _FILL + return image, mask From 90e89b987f930d90f820f051e0f607254a42608e Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:24:55 +0200 Subject: [PATCH 04/22] feat(moon): support ODE-backed and fixed-mosaic instrument datasets Split the shared product-dataset machinery (validation, the read loop, caching) out of InstrumentDataset into a new _ProductDataset base, so it can back three source backends instead of one: - InstrumentDataset: USGS ARD STAC search (unchanged behavior) - ODEInstrumentDataset: PDS Orbital Data Explorer search, for instruments the STAC catalog doesn't carry. Adds footprint_sampling for instruments that cover only a handful of named sites rather than the whole Moon, drawing windows from inside real product footprints instead of uniformly over the bbox. - MosaicDataset: reads one well-known archive URL directly, for instruments published as a single global file. Ships four new datasets: LROCNACDTM (LRO LROC NAC stereo DTM sites -- elevation, orthoimage, pixel confidence; the color-coded slope/shade SDP products are rendered visualizations, not quantitative rasters, so are intentionally not offered), LROCWACMosaic (LRO WAC 100m global mosaic), LOLA (global gridded DEM), and SLDEM2015 (LOLA + Kaguya TC DEM). The NAC DTM file patterns target each product's data file (.TIF/.IMG) directly rather than its detached PDS4 .xml label: GDAL's PDS4 driver mis-parses the resolution unit in these labels into a near-zero pixel size, while opening the data file directly reads correct georeferencing, scale, and nodata (verified live against the Apollo 15 site). --- src/astrofetch/moon/datasets.py | 408 +++++++++++++++++++++++++++----- 1 file changed, 347 insertions(+), 61 deletions(-) diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index 850244a..d7b135b 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -1,18 +1,31 @@ """Torch dataset classes over lunar instrument data. -Each instrument dataset samples random windows inside its ``bbox`` and, for each -window, reads its product COGs from the USGS ARD STAC catalog, reprojects them -onto a common target grid, and stacks them into a coregistered ``(C, H, W)`` -tensor. Combine instruments with ``&`` to stack their channels over the region -they share. - -The Phase 1 read path is real: :mod:`astrofetch.data.stac` finds the COG assets -covering a window, :mod:`astrofetch.data.raster` reprojects and reads them, and -:class:`astrofetch.data.cache.WindowCache` memoizes the result on disk. +Each instrument dataset samples random windows inside its ``bbox`` and, for +each window, reads its product rasters, reprojects them onto a common target +grid, and stacks them into a coregistered ``(C, H, W)`` tensor. Combine +instruments with ``&`` to stack their channels over the region they share. + +Three read paths back the products, chosen per instrument by which base class +it subclasses: + +- :class:`InstrumentDataset` -- searches the USGS ARD STAC catalog + (:mod:`astrofetch.data.stac`) for COG items covering a window. +- :class:`ODEInstrumentDataset` -- searches the NASA PDS Orbital Data + Explorer (:mod:`astrofetch.data.ode`) for products covering a window, for + instruments (LROC, LOLA, ...) the STAC catalog does not carry. +- :class:`MosaicDataset` -- reads a single well-known archive URL directly, + for instruments published as one global (or near-global) file rather than + many searchable items. + +All three reproject through :mod:`astrofetch.data.raster` and cache through +:class:`astrofetch.data.cache.WindowCache`; the sample-dict contract and the +``&`` composition operator are identical regardless of which backs a +particular instrument. """ from __future__ import annotations +import logging from collections.abc import Iterator from typing import ClassVar, NamedTuple @@ -20,10 +33,12 @@ import torch from torch.utils.data import Dataset -from astrofetch.data import raster, stac +from astrofetch.data import endpoints, ode, raster, stac from astrofetch.data.cache import WindowCache from astrofetch.data.grid import GEOGRAPHIC_CRS, TargetGrid, meters_to_degrees +logger = logging.getLogger(__name__) + BBox = tuple[float, float, float, float] """(west, south, east, north) in degrees, IAU 2015 Moon.""" @@ -32,10 +47,59 @@ class Product(NamedTuple): - """One user-facing product: its sample-dict layer id and its STAC asset key.""" + """One STAC-backed product: its sample-dict layer id, STAC asset key, + band, and (rarely needed) nodata override.""" layer: str asset: str + band: int = 1 + nodata: float | None = None + + +class ODEAsset(NamedTuple): + """One PDS-ODE-backed product: its sample-dict layer id, ODE product + type, and a filename pattern selecting the right file within each + product (a product bundle can contain many files -- data, browse, + derived -- so the pattern narrows to the one that should be read).""" + + layer: str + pt: str + pattern: str + band: int = 1 + nodata: float | None = None + + +class MosaicAsset(NamedTuple): + """One fixed-URL product: its sample-dict layer id and archive href (an + :mod:`astrofetch.data.endpoints` constant), read directly with no + search.""" + + layer: str + href: str + band: int = 1 + nodata: float | None = None + + +def _random_window( + bbox: BBox, patch_size: int, resolution: float, generator: torch.Generator +) -> BBox: + """Pick one patch_size*resolution-sized window at a random position inside ``bbox``.""" + west, south, east, north = bbox + span_lon, span_lat = east - west, north - south + # Window ground size from patch_size * resolution, converted to degrees + # at the region's centre latitude; clamp to the bbox so it stays inside. + center_lat = (south + north) / 2.0 + win_lon, win_lat = meters_to_degrees(patch_size * resolution, center_lat) + win_lon, win_lat = min(win_lon, span_lon), min(win_lat, span_lat) + u, v = torch.rand(2, generator=generator).tolist() + west0 = west + u * (span_lon - win_lon) + south0 = south + v * (span_lat - win_lat) + return (west0, south0, west0 + win_lon, south0 + win_lat) + + +def _bbox_area(bbox: BBox) -> float: + west, south, east, north = bbox + return (east - west) * (north - south) class _WindowedDataset(Dataset[dict]): @@ -92,48 +156,23 @@ def _seed_base(self) -> int: self._cached_seed_base = int(torch.randint(0, 2**31 - 1, (1,)).item()) return self._cached_seed_base - def _sample_bbox(self, index: int) -> BBox: + def _seeded_generator(self, index: int) -> torch.Generator: generator = torch.Generator() generator.manual_seed((self._seed_base * 1_000_003 + index) % (2**63 - 1)) - west, south, east, north = self.bbox - span_lon, span_lat = east - west, north - south - # Window ground size from patch_size * resolution, converted to degrees - # at the region's centre latitude; clamp to the bbox so it stays inside. - center_lat = (south + north) / 2.0 - win_lon, win_lat = meters_to_degrees(self.patch_size * self.resolution, center_lat) - win_lon, win_lat = min(win_lon, span_lon), min(win_lat, span_lat) - u, v = torch.rand(2, generator=generator).tolist() - west0 = west + u * (span_lon - win_lon) - south0 = south + v * (span_lat - win_lat) - return (west0, south0, west0 + win_lon, south0 + win_lat) - - -class InstrumentDataset(_WindowedDataset): - """Map-style dataset of patches from a single instrument. + return generator - Each index deterministically samples a bounding box within ``bbox`` and - returns a sample dict: ``image`` is a (C, H, W) float tensor with one - channel per requested product (physical values) where ``H = W = - patch_size``, ``mask`` a same-shaped bool validity tensor (orbital swaths - do not cover everything), plus ``layers``/``bbox``/``crs``/``resolution`` - provenance. Combine instruments with ``&`` to stack their channels over the - overlapping region. + def _sample_bbox(self, index: int) -> BBox: + generator = self._seeded_generator(index) + return _random_window(self.bbox, self.patch_size, self.resolution, generator) - Args: - products: product names to stack, e.g. ``["dtm"]``; defaults to all - products the instrument offers. - bbox: region to sample from as (west, south, east, north) degrees. - resolution: target resolution in metres per pixel. - patch_size: output height and width in pixels. - length: number of patches per epoch. - seed: RNG seed for reproducible sampling. - max_items: cap on STAC items mosaicked per layer per window. - cache: window cache; defaults to the shared on-disk cache. - Example: - >>> moondata = KaguyaTC(products=["dtm"], bbox=(-26.4, -50.7, -25.4, -49.6)) - >>> for sample in moondata: # doctest: +SKIP - ... sample["image"] # torch.Tensor (C, H, W) +class _ProductDataset(_WindowedDataset): + """Shared read/validation/cache path for every product-backed dataset. + + Subclasses provide ``all_products`` and implement ``_hrefs`` to resolve + one product's spec and window into the source hrefs to mosaic; see + :class:`InstrumentDataset`, :class:`ODEInstrumentDataset`, and + :class:`MosaicDataset`. """ probe: ClassVar[str] @@ -142,11 +181,8 @@ class InstrumentDataset(_WindowedDataset): instrument: ClassVar[str] """Human-readable instrument name.""" - collection: ClassVar[str] - """USGS ARD STAC collection id backing this instrument's products.""" - - all_products: ClassVar[dict[str, Product]] - """Product name -> (layer id, STAC asset key).""" + all_products: ClassVar[dict[str, Product | ODEAsset | MosaicAsset]] + """Product name -> asset spec.""" def __init__( self, @@ -156,7 +192,6 @@ def __init__( patch_size: int = 256, length: int = 1000, seed: int | None = None, - max_items: int = 20, cache: WindowCache | None = None, ) -> None: if products is None: @@ -179,7 +214,6 @@ def __init__( self.patch_size = patch_size self.length = length self.seed = seed - self.max_items = max_items self.cache = cache if cache is not None else WindowCache() def read(self, bbox: BBox) -> dict: @@ -199,18 +233,192 @@ def read(self, bbox: BBox) -> dict: "resolution": self.resolution, } - def _read_layer(self, spec: Product, grid: TargetGrid) -> tuple[np.ndarray, np.ndarray]: + def _read_layer( + self, spec: Product | ODEAsset | MosaicAsset, grid: TargetGrid + ) -> tuple[np.ndarray, np.ndarray]: cached = self.cache.get(spec.layer, grid) if cached is not None: return cached - hrefs = stac.find_asset_hrefs(self.collection, spec.asset, grid.bbox, self.max_items) - image, mask = _mosaic(hrefs, grid) + hrefs = self._hrefs(spec, grid.bbox) + image, mask = _mosaic(hrefs, grid, band=spec.band, nodata_override=spec.nodata) self.cache.put(spec.layer, grid, image, mask) return image, mask + def _hrefs(self, spec: Product | ODEAsset | MosaicAsset, bbox: BBox) -> list[str]: + raise NotImplementedError + + +class InstrumentDataset(_ProductDataset): + """Map-style dataset of patches from a single USGS-ARD-STAC instrument. + + Each index deterministically samples a bounding box within ``bbox`` and + returns a sample dict: ``image`` is a (C, H, W) float tensor with one + channel per requested product (physical values) where ``H = W = + patch_size``, ``mask`` a same-shaped bool validity tensor (orbital swaths + do not cover everything), plus ``layers``/``bbox``/``crs``/``resolution`` + provenance. Combine instruments with ``&`` to stack their channels over the + overlapping region. + + Args: + products: product names to stack, e.g. ``["dtm"]``; defaults to all + products the instrument offers. + bbox: region to sample from as (west, south, east, north) degrees. + resolution: target resolution in metres per pixel. + patch_size: output height and width in pixels. + length: number of patches per epoch. + seed: RNG seed for reproducible sampling. + max_items: cap on STAC items mosaicked per layer per window. + cache: window cache; defaults to the shared on-disk cache. + + Example: + >>> moondata = KaguyaTC(products=["dtm"], bbox=(-26.4, -50.7, -25.4, -49.6)) + >>> for sample in moondata: # doctest: +SKIP + ... sample["image"] # torch.Tensor (C, H, W) + """ + + collection: ClassVar[str] + """USGS ARD STAC collection id backing this instrument's products.""" + + def __init__( + self, + products: list[str] | None = None, + bbox: BBox = (-180.0, -90.0, 180.0, 90.0), + resolution: float = 100.0, + patch_size: int = 256, + length: int = 1000, + seed: int | None = None, + max_items: int = 20, + cache: WindowCache | None = None, + ) -> None: + super().__init__(products, bbox, resolution, patch_size, length, seed, cache) + self.max_items = max_items + + def _hrefs(self, spec: Product | ODEAsset | MosaicAsset, bbox: BBox) -> list[str]: + assert isinstance(spec, Product) + return stac.find_asset_hrefs(self.collection, spec.asset, bbox, self.max_items) + + +class ODEInstrumentDataset(_ProductDataset): + """Map-style dataset of patches from an instrument searched via PDS ODE. + + Same sample contract as :class:`InstrumentDataset`, but resolves products + through the NASA PDS Orbital Data Explorer (:mod:`astrofetch.data.ode`) + instead of the USGS ARD STAC catalog, for instruments the STAC catalog + does not carry. + + Some instruments cover only a handful of named sites rather than the + whole Moon (e.g. LROC NAC stereo DTMs). Set ``footprint_sampling = True`` + on such a subclass so sampled windows are drawn from inside real product + footprints instead of uniformly over ``bbox`` -- which, for a sparse + instrument, would draw mostly-empty windows. + + Args: + products: product names to stack; defaults to all products offered. + bbox: region to sample from as (west, south, east, north) degrees. + resolution: target resolution in metres per pixel. + patch_size: output height and width in pixels. + length: number of patches per epoch. + seed: RNG seed for reproducible sampling. + max_products: cap on ODE products mosaicked per layer per window. + footprint_sampling: override the class default; ``None`` keeps it. + cache: window cache; defaults to the shared on-disk cache. + """ + + ihid: ClassVar[str] + """ODE instrument host id, e.g. ``"LRO"``.""" + + iid: ClassVar[str] + """ODE instrument id, e.g. ``"LROC"``.""" + + footprint_sampling: bool = False + """Sample windows from inside real product footprints rather than + uniformly over ``bbox``. Set ``True`` for sparse, site-based instruments. + A plain (not ``ClassVar``) attribute: subclasses set the default, and + ``__init__`` may shadow it per instance.""" + + def __init__( + self, + products: list[str] | None = None, + bbox: BBox = (-180.0, -90.0, 180.0, 90.0), + resolution: float = 100.0, + patch_size: int = 256, + length: int = 1000, + seed: int | None = None, + max_products: int = 20, + footprint_sampling: bool | None = None, + cache: WindowCache | None = None, + ) -> None: + super().__init__(products, bbox, resolution, patch_size, length, seed, cache) + self.max_products = max_products + if footprint_sampling is not None: + self.footprint_sampling = footprint_sampling + self._footprints: list[BBox] | None = None + + def _hrefs(self, spec: Product | ODEAsset | MosaicAsset, bbox: BBox) -> list[str]: + assert isinstance(spec, ODEAsset) + return ode.find_file_urls( + self.ihid, self.iid, spec.pt, spec.pattern, bbox, self.max_products + ) + + def _sample_bbox(self, index: int) -> BBox: + if not self.footprint_sampling: + return super()._sample_bbox(index) + footprints = self._product_footprints() + generator = self._seeded_generator(index) + if not footprints: + logger.warning( + "%s: no product footprints found in bbox %s; falling back to " + "uniform sampling over the full bbox", + self.instrument, + self.bbox, + ) + return _random_window(self.bbox, self.patch_size, self.resolution, generator) + # Same seeded generator, drawn from in order: which footprint, then + # where inside it -- so a given (seed, index) always yields the same + # window, exactly like the uniform-sampling path. + areas = torch.tensor([_bbox_area(fp) for fp in footprints]) + choice = int(torch.multinomial(areas, 1, generator=generator).item()) + return _random_window(footprints[choice], self.patch_size, self.resolution, generator) + + def _product_footprints(self) -> list[BBox]: + # Fetched lazily (on first sample, not __init__) so construction never + # touches the network -- tests can build instances hermetically. + if self._footprints is None: + pts: set[str] = set() + for name in self.products: + entry = self.all_products[name] + if isinstance(entry, ODEAsset): + pts.add(entry.pt) + footprints: list[BBox] = [] + for pt in pts: + products = ode.query_products(self.ihid, self.iid, pt, self.bbox, max_products=500) + footprints.extend(product.bbox for product in products if product.bbox is not None) + self._footprints = footprints + return self._footprints + + +class MosaicDataset(_ProductDataset): + """Map-style dataset of patches from a fixed-URL global (or near-global) + product, e.g. a single mosaic or DEM file. + + Same sample contract as :class:`InstrumentDataset`, but each product maps + to one well-known archive URL (an :mod:`astrofetch.data.endpoints` + constant) rather than being searched: there is exactly one item to read + per layer, so no catalog lookup happens on ``read``. + """ + + def _hrefs(self, spec: Product | ODEAsset | MosaicAsset, bbox: BBox) -> list[str]: + assert isinstance(spec, MosaicAsset) + return [spec.href] + -def _mosaic(hrefs: list[str], grid: TargetGrid) -> tuple[np.ndarray, np.ndarray]: - """Reproject each COG onto ``grid`` and fill invalid pixels from later items. +def _mosaic( + hrefs: list[str], + grid: TargetGrid, + band: int = 1, + nodata_override: float | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """Reproject each source onto ``grid`` and fill invalid pixels from later items. Earlier items win where they have data; later items fill only the gaps. With no items, returns an all-invalid (zero) window — the mask tells the @@ -219,7 +427,9 @@ def _mosaic(hrefs: list[str], grid: TargetGrid) -> tuple[np.ndarray, np.ndarray] image = np.zeros((grid.height, grid.width), dtype=np.float32) valid = np.zeros((grid.height, grid.width), dtype=bool) for href in hrefs: - layer_image, layer_mask = raster.read_window(href, grid) + layer_image, layer_mask = raster.read_window( + href, grid, band=band, nodata_override=nodata_override + ) fill = layer_mask & ~valid image[fill] = layer_image[fill] valid |= layer_mask @@ -251,6 +461,82 @@ class KaguyaTCImagery(InstrumentDataset): all_products = {"image": Product("kaguya_tc_image", "image")} +class LROCNACDTM(ODEInstrumentDataset): + """LRO LROC NAC stereo photogrammetric DTM sites: elevation, orthoimage, + and per-pixel confidence, searched via PDS ODE (product type ``SDNDTM``). + + Coverage is a few hundred named sites (Apollo landing sites, craters and + other features of interest) rather than the whole Moon, so + ``footprint_sampling`` is on by default: sampled windows land inside an + actual site instead of drawing uniformly over ``bbox`` and mostly missing. + The color-coded slope and shaded-relief SDP products are rendered 8-bit + visualizations, not quantitative rasters, so they are intentionally not + offered here (AGENTS rule 3: never mix rendered and quantitative data). + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC NAC (stereo DTM sites)" + ihid = "LRO" + iid = "LROC" + footprint_sampling = True + all_products = { + # PDS4 .xml labels for this SDP pipeline carry a broken resolution + # unit that GDAL's PDS4 driver mis-parses into a near-zero pixel + # size; opening each data file directly (GTiff/raw driver) instead + # of through its label reads correct georeferencing, scale, and + # nodata (verified live 2026-07-20). DTM and confidence/shade/slope + # share a stem, so DTM excludes the derived-product suffixes. + "dtm": ODEAsset( + "lroc_nac_dtm", + "SDNDTM", + r"NAC_DTM_(?:(?!_CLRDISC|_CLRGRAD|_CONF|_SHADE|_SLOPE).)+\.TIF", + ), + "ortho": ODEAsset("lroc_nac_ortho", "SDNDTM", r"NAC_DTM_.+_M\d+_(?:50CM|2M)\.IMG"), + "confidence": ODEAsset("lroc_nac_confidence", "SDNDTM", r"NAC_DTM_.+_CONF\.IMG"), + } + + +class LROCWACMosaic(MosaicDataset): + """LRO LROC WAC global morphology mosaic, 100 m/px, equirectangular. + + A single fixed mosaic (see :mod:`astrofetch.data.endpoints`), not a Cloud + Optimized GeoTIFF: it has no overviews, so prefer ``resolution=100`` (its + native resolution) -- coarser resolutions force GDAL to read every + source row under the requested window. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC WAC (global 100 m mosaic)" + all_products = { + "morphology": MosaicAsset("lroc_wac_mosaic", endpoints.LROC_WAC_MOSAIC_100M_URL) + } + + +class LOLA(MosaicDataset): + """LRO LOLA global gridded DEM, 128 px/degree (~237 m/px at the equator). + + Elevation in metres above the IAU 2015 Moon reference sphere. A single + fixed global product (see :mod:`astrofetch.data.endpoints`), not searched. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LOLA (global gridded DEM)" + all_products = {"dem": MosaicAsset("lola_dem", endpoints.LOLA_DEM_128_URL)} + + +class SLDEM2015(MosaicDataset): + """SLDEM2015: LOLA + Kaguya Terrain Camera co-registered DEM, 128 px/degree. + + Elevation in metres above the IAU 2015 Moon reference sphere. Source + coverage is 60S-60N only (not a bug): windows outside that band read back + with ``mask`` all ``False``. A single fixed product, not searched. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "SLDEM2015 (LOLA + Kaguya TC DEM)" + all_products = {"dem": MosaicAsset("sldem2015_dem", endpoints.SLDEM2015_URL)} + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. From b39434606d49fa5512269930111b7cc2aaf94416 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:03 +0200 Subject: [PATCH 05/22] feat(moon): add experimental raw PDS granule datasets A deliberately different, documented contract from the windowed instrument datasets: no bbox windowing, no reprojection, no ISIS/SPICE -- each item is one raw PDS product read in its own native camera/instrument geometry. GranuleDataset does one eager ODE search in __init__ (so len() needs no network call) and guards against reading a whole gigapixel strip by default (max_pixels raises with a rows= hint instead). Ships LROCNACRaw and LROCWACRaw (calibrated NAC/WAC strips) and M3 (Chandrayaan-1 Moon Mineralogy Mapper L1B radiance, with its lon/lat/ elevation geolocation backplane via extra_patterns). M3's ENVI-format data file must be opened directly -- unlike the PDS-labeled NAC/WAC strips, its driver rejects the .HDR header file and wants the .IMG data file itself. --- src/astrofetch/moon/granules.py | 189 ++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 src/astrofetch/moon/granules.py diff --git a/src/astrofetch/moon/granules.py b/src/astrofetch/moon/granules.py new file mode 100644 index 0000000..cd90a30 --- /dev/null +++ b/src/astrofetch/moon/granules.py @@ -0,0 +1,189 @@ +"""EXPERIMENTAL: map-style datasets over raw PDS granules, camera geometry. + +Unlike every dataset in :mod:`astrofetch.moon.datasets`, these are **not** +reprojected onto a common grid: each item is one raw PDS product (an NAC/WAC +calibrated strip, an M3 radiance cube, ...), read in its own native +camera/instrument geometry with no reprojection, resampling, ISIS, or SPICE +processing. This is a deliberate, documented carve-out from AGENTS.md's +normal coregistered-tensor contract: + +- No bbox windowing: ``__getitem__`` returns a whole granule (or a row range, + via ``rows=``), not a patch cropped to a requested extent. +- Ragged shapes across items -- the default ``DataLoader`` collation will not + work; use ``batch_size=None`` or a custom ``collate_fn``. +- No ``&`` composition -- there is no shared grid to stack channels onto. + +``len()`` is the number of PDS ODE products matching a bbox, fetched once and +eagerly in ``__init__`` so ``len()`` never needs a network call. +""" + +from __future__ import annotations + +import logging +from typing import ClassVar + +import rasterio +import torch +from rasterio.windows import Window +from torch.utils.data import Dataset + +from astrofetch.data import ode, raster +from astrofetch.data.grid import BBox +from astrofetch.errors import EndpointError + +logger = logging.getLogger(__name__) + +_DEFAULT_MAX_PIXELS = 512 * 1024 * 1024 // 4 +"""Default guard: about 512 MiB as float32 (bands * width * height * 4 bytes).""" + + +class GranuleDataset(Dataset[dict]): + """EXPERIMENTAL map-style dataset over one instrument's raw PDS granules. + + Args: + bbox: region to search as (west, south, east, north) degrees. + max_products: cap on granules in the dataset (one ODE search, eager). + rows: pixel row range read from every granule, e.g. ``slice(0, + 512)``; ``None`` reads the full granule, subject to + ``max_pixels``. + max_pixels: raise instead of reading a granule (bands * width * + height) larger than this; NAC/WAC strips can be gigapixel. + + Raises: + EndpointError: the ODE search failed. + """ + + probe: ClassVar[str] + """Name of the probe (spacecraft) carrying this instrument.""" + + instrument: ClassVar[str] + """Human-readable instrument name.""" + + ihid: ClassVar[str] + """ODE instrument host id, e.g. ``"LRO"``.""" + + iid: ClassVar[str] + """ODE instrument id, e.g. ``"LROC"``.""" + + pt: ClassVar[str] + """ODE product type, e.g. ``"CDRNAC4"``.""" + + file_pattern: ClassVar[str] + """Regex (fullmatch, case-insensitive) selecting the main data file.""" + + extra_patterns: ClassVar[dict[str, str]] = {} + """Extra sample-dict keys read the same way, e.g. geolocation backplanes.""" + + def __init__( + self, + bbox: BBox = (-180.0, -90.0, 180.0, 90.0), + max_products: int = 100, + rows: slice | None = None, + max_pixels: int = _DEFAULT_MAX_PIXELS, + ) -> None: + west, south, east, north = bbox + if not (west < east and south < north): + raise ValueError(f"invalid bbox (west, south, east, north): {bbox}") + self.bbox = bbox + self.rows = rows + self.max_pixels = max_pixels + self.products = ode.query_products(self.ihid, self.iid, self.pt, bbox, max_products) + + def __len__(self) -> int: + return len(self.products) + + def __getitem__(self, index: int) -> dict: + if index < 0: + index += len(self.products) + if not 0 <= index < len(self.products): + raise IndexError(f"index out of range for length {len(self.products)}") + product = self.products[index] + + main_urls = ode.match_files(product.files, self.file_pattern) + if not main_urls: + raise EndpointError( + product.pdsid, f"no file matched {self.file_pattern!r} in this granule" + ) + image, mask = self._read_granule(main_urls[0]) + sample: dict = { + "image": image, + "mask": mask, + "pdsid": product.pdsid, + "bbox": product.bbox, + "meta": product.metadata, + } + for key, pattern in self.extra_patterns.items(): + urls = ode.match_files(product.files, pattern) + if not urls: + logger.warning("%s: no file matched %r for %r", product.pdsid, pattern, key) + continue + extra_image, extra_mask = self._read_granule(urls[0]) + sample[key] = extra_image + sample[f"{key}_mask"] = extra_mask + return sample + + def _read_granule(self, url: str) -> tuple[torch.Tensor, torch.Tensor]: + image, mask = raster.read_full(url, window=self._window_for(url)) + return torch.from_numpy(image), torch.from_numpy(mask) + + def _window_for(self, url: str) -> Window | None: + if self.rows is not None: + with rasterio.open(url) as src: + width = src.width + return Window.from_slices(self.rows, slice(0, width)) + with rasterio.open(url) as src: + pixel_count = src.width * src.height * src.count + if pixel_count > self.max_pixels: + raise ValueError( + f"{url}: granule has {pixel_count:,} pixels, over " + f"max_pixels={self.max_pixels:,}; pass rows=slice(...) to " + "GranuleDataset(...) to read a row range instead of the whole granule" + ) + return None + + +class LROCNACRaw(GranuleDataset): + """EXPERIMENTAL: LRO LROC NAC calibrated strips, raw camera geometry. + + Radiometrically calibrated (I/F) but not map-projected: no bbox + windowing, reprojection, ISIS, or SPICE. See :class:`GranuleDataset`. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC NAC (raw calibrated strips)" + ihid = "LRO" + iid = "LROC" + pt = "CDRNAC4" + file_pattern = r"M\d+[LR]C\.XML" + + +class LROCWACRaw(GranuleDataset): + """EXPERIMENTAL: LRO LROC WAC monochrome calibrated strips, raw camera + geometry. See :class:`GranuleDataset`. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC WAC (raw calibrated strips, mono)" + ihid = "LRO" + iid = "LROC" + pt = "CDRWAM4" + file_pattern = r"M\d+MC\.XML" + + +class M3(GranuleDataset): + """EXPERIMENTAL: Chandrayaan-1 Moon Mineralogy Mapper (M3) L1B radiance + cubes (85 bands), raw camera geometry, with per-pixel geolocation. + + Not map-projected: no bbox windowing, no reprojection. The ``loc`` key + holds the 3-band (longitude, latitude, elevation) geolocation backplane + at the same pixel grid as ``image``, for georeferencing samples + yourself. See :class:`GranuleDataset`. + """ + + probe = "Chandrayaan-1" + instrument = "Moon Mineralogy Mapper (M3), L1B radiance" + ihid = "CH1-ORB" + iid = "M3" + pt = "CALIMG" + file_pattern = r"M3G\w+_V\d+_RDN\.IMG" + extra_patterns = {"loc": r"M3G\w+_V\d+_LOC\.IMG"} From 0af04ae855550bf8dcb89e5d51de33ff07e90330 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:08 +0200 Subject: [PATCH 06/22] feat(moon): register new datasets in the layer registry and catalog LayerSpec grows a source discriminator ("stac", "ode", or "mosaic") plus the fields each backend needs, dispatched in _spec by asset spec type. Probe grows a granules field for the experimental raw-granule dataset classes, which sit outside the layer/product contract entirely and are not part of LAYERS. Adds an "lro" probe (nac_dtm, wac_mosaic, lola, sldem2015 instruments; nac_raw, wac_raw granules) and a "chandrayaan1" probe (m3 granule only). --- src/astrofetch/moon/layers.py | 122 +++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 23 deletions(-) diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 8e0dd5f..390baf6 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -1,23 +1,34 @@ """Layer registry and discovery catalog for lunar data. The registry (``LAYERS``) is the single place where user-facing layer -identifiers map to their provenance and their backing USGS ARD STAC collection -and asset. The catalog (``MOON``) arranges the same information as a -Body -> Probe -> Instrument hierarchy for discovery; its nodes hold specs and -dataset *classes*, never dataset instances. Flat imports +identifiers map to their provenance and their backing source (USGS ARD STAC, +PDS ODE, or a fixed mosaic URL). The catalog (``MOON``) arranges the same +information as a Body -> Probe -> Instrument hierarchy for discovery; its +nodes hold specs and dataset *classes*, never dataset instances. A probe may +also carry ``granules``: raw, non-map-projected datasets +(:mod:`astrofetch.moon.granules`) that fall outside the layer/product +contract entirely (see that module's docstring). Flat imports (``from astrofetch.moon import KaguyaTC``) remain the primary path. """ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from astrofetch.moon.datasets import ( CRS, - InstrumentDataset, + LOLA, + LROCNACDTM, + SLDEM2015, KaguyaTC, KaguyaTCImagery, + LROCWACMosaic, + MosaicAsset, + ODEAsset, + Product, + _ProductDataset, ) +from astrofetch.moon.granules import M3, GranuleDataset, LROCNACRaw, LROCWACRaw @dataclass(frozen=True) @@ -36,11 +47,30 @@ class LayerSpec: product: str """Product name within the instrument, e.g. ``"dtm"``.""" - collection: str - """USGS ARD STAC collection id backing this layer.""" + source: str + """Where this layer is read from: ``"stac"``, ``"ode"``, or ``"mosaic"``.""" - asset: str - """STAC asset key read from each item, e.g. ``"dtm"``.""" + collection: str = "" + """USGS ARD STAC collection id (``source == "stac"``).""" + + asset: str = "" + """STAC asset key read from each item (``source == "stac"``).""" + + ihid: str = "" + """PDS ODE instrument host id, e.g. ``"LRO"`` (``source == "ode"``).""" + + iid: str = "" + """PDS ODE instrument id, e.g. ``"LROC"`` (``source == "ode"``).""" + + pt: str = "" + """PDS ODE product type, e.g. ``"SDNDTM"`` (``source == "ode"``).""" + + pattern: str = "" + """Filename pattern selecting the file within an ODE product + (``source == "ode"``).""" + + href: str = "" + """Fixed archive URL (``source == "mosaic"``).""" @dataclass(frozen=True) @@ -49,16 +79,18 @@ class Instrument: name: str products: dict[str, LayerSpec] - dataset: type[InstrumentDataset] + dataset: type[_ProductDataset] """The dataset class (not an instance); construct it on demand.""" @dataclass(frozen=True) class Probe: - """Catalog node: one probe and the instrument datasets it carries.""" + """Catalog node: one probe, the instrument datasets it carries, and any + experimental raw-granule datasets (see :mod:`astrofetch.moon.granules`).""" name: str instruments: dict[str, Instrument] + granules: dict[str, type[GranuleDataset]] = field(default_factory=dict) @dataclass(frozen=True) @@ -70,16 +102,35 @@ class Body: probes: dict[str, Probe] -def _spec(dataset: type[InstrumentDataset], product: str) -> LayerSpec: +def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: entry = dataset.all_products[product] - return LayerSpec( - name=entry.layer, - probe=dataset.probe, - instrument=dataset.instrument, - product=product, - collection=dataset.collection, - asset=entry.asset, - ) + common = { + "name": entry.layer, + "probe": dataset.probe, + "instrument": dataset.instrument, + "product": product, + } + if isinstance(entry, Product): + # dataset is an InstrumentDataset subclass here, guaranteed by + # all_products entries always matching the dataset's own asset kind; + # getattr keeps this function's signature at the shared base type. + collection = getattr(dataset, "collection", "") + return LayerSpec(**common, source="stac", collection=collection, asset=entry.asset) + if isinstance(entry, ODEAsset): + # dataset is an ODEInstrumentDataset subclass here, same guarantee. + ihid = getattr(dataset, "ihid", "") + iid = getattr(dataset, "iid", "") + return LayerSpec( + **common, + source="ode", + ihid=ihid, + iid=iid, + pt=entry.pt, + pattern=entry.pattern, + ) + if isinstance(entry, MosaicAsset): + return LayerSpec(**common, source="mosaic", href=entry.href) + raise TypeError(f"unknown asset spec type for {product!r}: {type(entry)!r}") # pragma: no cover LAYERS: dict[str, LayerSpec] = { @@ -88,11 +139,17 @@ def _spec(dataset: type[InstrumentDataset], product: str) -> LayerSpec: _spec(KaguyaTC, "dtm"), _spec(KaguyaTC, "ortho"), _spec(KaguyaTCImagery, "image"), + _spec(LROCNACDTM, "dtm"), + _spec(LROCNACDTM, "ortho"), + _spec(LROCNACDTM, "confidence"), + _spec(LROCWACMosaic, "morphology"), + _spec(LOLA, "dem"), + _spec(SLDEM2015, "dem"), ) } -def _instrument(dataset: type[InstrumentDataset]) -> Instrument: +def _instrument(dataset: type[_ProductDataset]) -> Instrument: return Instrument( name=dataset.instrument, products={product: LAYERS[entry.layer] for product, entry in dataset.all_products.items()}, @@ -111,6 +168,25 @@ def _instrument(dataset: type[InstrumentDataset]) -> Instrument: "tc_imagery": _instrument(KaguyaTCImagery), }, ), + "lro": Probe( + name=LROCNACDTM.probe, + instruments={ + "nac_dtm": _instrument(LROCNACDTM), + "wac_mosaic": _instrument(LROCWACMosaic), + "lola": _instrument(LOLA), + "sldem2015": _instrument(SLDEM2015), + }, + granules={ + "nac_raw": LROCNACRaw, + "wac_raw": LROCWACRaw, + }, + ), + "chandrayaan1": Probe( + name=M3.probe, + instruments={}, + granules={"m3": M3}, + ), }, ) -"""Discovery catalog for the Moon: enumerate probes, instruments, products.""" +"""Discovery catalog for the Moon: enumerate probes, instruments, products, +and (where a probe has any) experimental raw-granule datasets.""" From 8b964355ca3084947d3eaca0a00a53c46c7d755b Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:12 +0200 Subject: [PATCH 07/22] feat(moon): export new dataset classes from package init LROCNACDTM, LROCWACMosaic, LOLA, SLDEM2015, ODEInstrumentDataset, MosaicDataset, and the granule classes (GranuleDataset, LROCNACRaw, LROCWACRaw, M3) become part of the public API in both astrofetch.moon and the top-level astrofetch namespace. --- src/astrofetch/__init__.py | 23 +++++++++++++++++++++-- src/astrofetch/moon/__init__.py | 17 +++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index e93d6e4..b44a471 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -2,19 +2,38 @@ One dataset class per instrument; combine them with ``&`` to receive coregistered multichannel samples. AstroFetch is a thin layer over existing -archive tooling (STAC, COGs), never a mirror. +archive tooling (STAC, COGs, PDS ODE), never a mirror of any archive. """ from astrofetch import moon -from astrofetch.moon import MOON, IntersectionDataset, KaguyaTC, KaguyaTCImagery +from astrofetch.moon import ( + LOLA, + LROCNACDTM, + M3, + MOON, + SLDEM2015, + IntersectionDataset, + KaguyaTC, + KaguyaTCImagery, + LROCNACRaw, + LROCWACMosaic, + LROCWACRaw, +) __version__ = "0.1.0" __all__ = [ + "LOLA", "MOON", + "M3", "IntersectionDataset", "KaguyaTC", "KaguyaTCImagery", + "LROCNACDTM", + "LROCNACRaw", + "LROCWACMosaic", + "LROCWACRaw", + "SLDEM2015", "moon", "__version__", ] diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index 567d295..440a829 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -1,22 +1,39 @@ """Lunar data access: instrument datasets, layer registry, discovery catalog.""" from astrofetch.moon.datasets import ( + LOLA, + LROCNACDTM, + SLDEM2015, InstrumentDataset, IntersectionDataset, KaguyaTC, KaguyaTCImagery, + LROCWACMosaic, + MosaicDataset, + ODEInstrumentDataset, ) +from astrofetch.moon.granules import M3, GranuleDataset, LROCNACRaw, LROCWACRaw from astrofetch.moon.layers import LAYERS, MOON, Body, Instrument, LayerSpec, Probe __all__ = [ "LAYERS", + "LOLA", "MOON", + "M3", "Body", + "GranuleDataset", "Instrument", "InstrumentDataset", "IntersectionDataset", "KaguyaTC", "KaguyaTCImagery", + "LROCNACDTM", + "LROCNACRaw", + "LROCWACMosaic", + "LROCWACRaw", "LayerSpec", + "MosaicDataset", + "ODEInstrumentDataset", "Probe", + "SLDEM2015", ] From 6cafdc9faca6070861f82d6ba2d1f4ea5d5dfaa6 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:17 +0200 Subject: [PATCH 08/22] test(data): add PDS ODE client unit tests Recorded JSON fixtures under tests/fixtures/ode/ cover the response quirks the client normalizes: a single match as a dict instead of a list, an empty result, and an error body. Also covers longitude conversion (including the full-Moon bbox that would otherwise collapse to a zero-width query), pagination, and match_files/find_file_urls filtering. The HTTP session is stubbed; no network in this suite. --- tests/fixtures/ode/error.json | 6 + tests/fixtures/ode/multi_product.json | 36 ++++ tests/fixtures/ode/no_products.json | 6 + tests/fixtures/ode/single_product.json | 21 +++ tests/unit/test_ode.py | 233 +++++++++++++++++++++++++ 5 files changed, 302 insertions(+) create mode 100644 tests/fixtures/ode/error.json create mode 100644 tests/fixtures/ode/multi_product.json create mode 100644 tests/fixtures/ode/no_products.json create mode 100644 tests/fixtures/ode/single_product.json create mode 100644 tests/unit/test_ode.py diff --git a/tests/fixtures/ode/error.json b/tests/fixtures/ode/error.json new file mode 100644 index 0000000..3b96336 --- /dev/null +++ b/tests/fixtures/ode/error.json @@ -0,0 +1,6 @@ +{ + "ODEResults": { + "Status": "ERROR", + "Error": "Invalid IIPT - no combination of Instrument Host Id, Instrument Id, and Product Type exists" + } +} diff --git a/tests/fixtures/ode/multi_product.json b/tests/fixtures/ode/multi_product.json new file mode 100644 index 0000000..d14dd42 --- /dev/null +++ b/tests/fixtures/ode/multi_product.json @@ -0,0 +1,36 @@ +{ + "ODEResults": { + "Status": "Success", + "Products": { + "Product": [ + { + "pdsid": "a", + "Product_files": { + "Product_file": [ + { + "FileName": "A_DTM.TIF", + "Type": "Product", + "URL": "https://x/a_dtm.tif" + }, + { + "FileName": "A_BROWSE.JPG", + "Type": "Browse", + "URL": "https://x/a_browse.jpg" + } + ] + } + }, + { + "pdsid": "b", + "Product_files": { + "Product_file": { + "FileName": "B_DTM.TIF", + "Type": "Product", + "URL": "https://x/b_dtm.tif" + } + } + } + ] + } + } +} diff --git a/tests/fixtures/ode/no_products.json b/tests/fixtures/ode/no_products.json new file mode 100644 index 0000000..4efa162 --- /dev/null +++ b/tests/fixtures/ode/no_products.json @@ -0,0 +1,6 @@ +{ + "ODEResults": { + "Status": "Success", + "Products": "No Products Found" + } +} diff --git a/tests/fixtures/ode/single_product.json b/tests/fixtures/ode/single_product.json new file mode 100644 index 0000000..50757e8 --- /dev/null +++ b/tests/fixtures/ode/single_product.json @@ -0,0 +1,21 @@ +{ + "ODEResults": { + "Status": "Success", + "Products": { + "Product": { + "pdsid": "sdp.nac_dtm.apollo15", + "Westernmost_longitude": 3.5, + "Easternmost_longitude": 4.0, + "Minimum_latitude": 25.8, + "Maximum_latitude": 26.2, + "Product_files": { + "Product_file": { + "FileName": "NAC_DTM_APOLLO15.XML", + "Type": "Product", + "URL": "https://pds.example/NAC_DTM_APOLLO15.xml" + } + } + } + } + } +} diff --git a/tests/unit/test_ode.py b/tests/unit/test_ode.py new file mode 100644 index 0000000..d0bdcde --- /dev/null +++ b/tests/unit/test_ode.py @@ -0,0 +1,233 @@ +"""Unit tests for the PDS ODE client — the HTTP session is stubbed, never +the network. Fixture bodies in ``tests/fixtures/ode/`` mirror ODE's real +response shapes (verified against the live endpoint): a lone match comes +back as a dict instead of a one-element list, an empty result is the string +``"No Products Found"``, and errors are HTTP 200 responses carrying +``Status: "ERROR"`` in the body. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import requests + +from astrofetch.data import ode +from astrofetch.errors import EndpointError + +_FIXTURES = Path(__file__).parent.parent / "fixtures" / "ode" + + +def _fixture(name: str) -> dict: + return json.loads((_FIXTURES / f"{name}.json").read_text()) + + +_SINGLE_PRODUCT_BODY = _fixture("single_product") +_MULTI_PRODUCT_BODY = _fixture("multi_product") +_EMPTY_BODY = _fixture("no_products") +_ERROR_BODY = _fixture("error") + + +class _Response: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return self._payload + + +class _Session: + """Serves ``pages`` in order, one per call; extra calls get an empty result.""" + + def __init__(self, pages: list[dict]) -> None: + self._pages = list(pages) + self.calls: list[dict] = [] + + def get(self, url: str, params: dict | None = None, timeout: float | None = None) -> _Response: + self.calls.append(dict(params or {})) + payload = self._pages.pop(0) if self._pages else _EMPTY_BODY + return _Response(payload) + + +def _page(pdsids: list[str]) -> dict: + return { + "ODEResults": { + "Status": "Success", + "Products": { + "Product": [{"pdsid": pid, "Product_files": {"Product_file": []}} for pid in pdsids] + }, + } + } + + +class _PagingSession: + """Realistically slices a backing id list by the request's offset/limit.""" + + def __init__(self, pdsids: list[str]) -> None: + self._pdsids = pdsids + self.calls: list[dict] = [] + + def get(self, url: str, params: dict | None = None, timeout: float | None = None) -> _Response: + params = dict(params or {}) + self.calls.append(params) + offset, limit = int(params["offset"]), int(params["limit"]) + page_ids = self._pdsids[offset : offset + limit] + return _Response(_page(page_ids) if page_ids else _EMPTY_BODY) + + +def _patch_session(monkeypatch: pytest.MonkeyPatch, session: object) -> None: + monkeypatch.setattr(ode, "_session", lambda: session) + + +# --- query_products: JSON quirk normalization ----------------------------- + + +def test_single_product_dict_is_normalized_to_a_list(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_session(monkeypatch, _Session([_SINGLE_PRODUCT_BODY])) + products = ode.query_products("LRO", "LROC", "SDNDTM", (3.0, 25.5, 4.5, 26.5)) + assert len(products) == 1 + assert products[0].pdsid == "sdp.nac_dtm.apollo15" + assert products[0].bbox == pytest.approx((3.5, 25.8, 4.0, 26.2)) + assert products[0].files == ( + ode.ODEFile("NAC_DTM_APOLLO15.XML", "Product", "https://pds.example/NAC_DTM_APOLLO15.xml"), + ) + + +def test_multi_product_list_preserves_order(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_session(monkeypatch, _Session([_MULTI_PRODUCT_BODY])) + products = ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0)) + assert [p.pdsid for p in products] == ["a", "b"] + assert len(products[0].files) == 2 + assert len(products[1].files) == 1 + + +def test_no_products_found_returns_empty_list(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_session(monkeypatch, _Session([_EMPTY_BODY])) + assert ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0)) == [] + + +def test_error_status_raises_endpoint_error(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_session(monkeypatch, _Session([_ERROR_BODY])) + with pytest.raises(EndpointError, match="Invalid IIPT"): + ode.query_products("LRO", "LROC", "BOGUS", (0.0, 0.0, 1.0, 1.0)) + + +def test_request_exception_becomes_endpoint_error(monkeypatch: pytest.MonkeyPatch) -> None: + class _Boom: + def get(self, *args: object, **kwargs: object) -> _Response: + raise requests.ConnectionError("boom") + + _patch_session(monkeypatch, _Boom()) + with pytest.raises(EndpointError, match="ODE query failed"): + ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0)) + + +# --- longitude conversion --------------------------------------------------- + + +def test_longitude_converted_to_0_360(monkeypatch: pytest.MonkeyPatch) -> None: + session = _Session([_EMPTY_BODY]) + _patch_session(monkeypatch, session) + ode.query_products("LRO", "LROC", "SDNDTM", (-26.3, -50.6, -25.5, -49.7)) + assert session.calls[0]["westernlon"] == pytest.approx(333.7) + assert session.calls[0]["easternlon"] == pytest.approx(334.5) + + +def test_full_moon_bbox_does_not_collapse_to_zero_width(monkeypatch: pytest.MonkeyPatch) -> None: + # A naive `lon % 360` on each bound independently sends (-180, 180) to + # (180, 180): a zero-width query that would silently return nothing. + session = _Session([_EMPTY_BODY]) + _patch_session(monkeypatch, session) + ode.query_products("LRO", "LROC", "SDNDTM", (-180.0, -90.0, 180.0, 90.0)) + assert session.calls[0]["westernlon"] == 0.0 + assert session.calls[0]["easternlon"] == 360.0 + + +# --- pagination -------------------------------------------------------------- + + +def test_pagination_collects_multiple_pages(monkeypatch: pytest.MonkeyPatch) -> None: + session = _PagingSession(["a", "b", "c"]) + _patch_session(monkeypatch, session) + monkeypatch.setattr(ode, "_PAGE_SIZE", 2) + products = ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0), max_products=10) + assert [p.pdsid for p in products] == ["a", "b", "c"] + assert [c["offset"] for c in session.calls] == [0, 2] + + +def test_pagination_stops_at_max_products(monkeypatch: pytest.MonkeyPatch) -> None: + session = _PagingSession(["a", "b", "c", "d", "e", "f"]) + _patch_session(monkeypatch, session) + monkeypatch.setattr(ode, "_PAGE_SIZE", 2) + products = ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0), max_products=3) + assert [p.pdsid for p in products] == ["a", "b", "c"] + assert len(session.calls) == 2 # never asked for a third page + + +# --- footprint bbox parsing -------------------------------------------------- + + +def test_product_bbox_parses_valid_footprint() -> None: + meta = { + "Westernmost_longitude": 3.5, + "Easternmost_longitude": 4.0, + "Minimum_latitude": 25.8, + "Maximum_latitude": 26.2, + } + assert ode._product_bbox(meta) == pytest.approx((3.5, 25.8, 4.0, 26.2)) + + +def test_product_bbox_returns_none_for_global_footprint() -> None: + meta = { + "Westernmost_longitude": 0, + "Easternmost_longitude": 360, + "Minimum_latitude": -90, + "Maximum_latitude": 90, + } + assert ode._product_bbox(meta) is None + + +def test_product_bbox_returns_none_for_missing_fields() -> None: + assert ode._product_bbox({}) is None + + +# --- match_files / find_file_urls -------------------------------------------- + + +def test_match_files_filters_by_pattern_and_type() -> None: + # URLs end in their real filename, as in actual ODE responses -- the sort + # key is the URL's basename, so this also exercises sort-by-filename. + files = ( + ode.ODEFile("A_DTM.TIF", "Product", "https://x/a_dtm.tif"), + ode.ODEFile("A_BROWSE.JPG", "Browse", "https://x/a_browse.jpg"), + ode.ODEFile("A_SHADE.TIF", "Product", "https://x/a_shade.tif"), + ) + assert ode.match_files(files, r"A_DTM\.TIF") == ["https://x/a_dtm.tif"] + assert ode.match_files(files, r"A_\w+\.TIF") == ["https://x/a_dtm.tif", "https://x/a_shade.tif"] + assert ode.match_files(files, r".*", file_type=None) == [ + "https://x/a_browse.jpg", + "https://x/a_dtm.tif", + "https://x/a_shade.tif", + ] + + +def test_match_files_is_case_insensitive() -> None: + files = (ode.ODEFile("a_dtm.tif", "Product", "https://x/a"),) + assert ode.match_files(files, r"A_DTM\.TIF") == ["https://x/a"] + + +def test_find_file_urls_flattens_across_products(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_session(monkeypatch, _Session([_MULTI_PRODUCT_BODY])) + urls = ode.find_file_urls("LRO", "LROC", "SDNDTM", r"\w+_DTM\.TIF", (0.0, 0.0, 1.0, 1.0)) + assert urls == ["https://x/a_dtm.tif", "https://x/b_dtm.tif"] + + +def test_find_file_urls_no_match_is_not_an_error(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_session(monkeypatch, _Session([_MULTI_PRODUCT_BODY])) + urls = ode.find_file_urls("LRO", "LROC", "SDNDTM", r"NOTHING_MATCHES", (0.0, 0.0, 1.0, 1.0)) + assert urls == [] From 633bd0966073ee383088697ea60634eb1cea60a9 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:22 +0200 Subject: [PATCH 09/22] test(moon): cover ODE, mosaic, and footprint-sampling datasets Extends the existing mocked-network fixture to also stub ode.find_file_urls and ode.query_products. New coverage: the ODE and mosaic sample contracts, that MosaicDataset never searches ODE, footprint-sampling determinism and containment, its fallback to uniform sampling with no footprints, and registry/catalog agreement for the new source types. --- tests/unit/test_datasets.py | 244 +++++++++++++++++++++++++++++++++++- 1 file changed, 238 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index 1c0ba5d..e4ea55c 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -1,9 +1,10 @@ """Unit tests for the instrument datasets. -The read path is real (STAC search + COG reproject + cache), so these tests -stub :mod:`astrofetch.data.stac` and :mod:`astrofetch.data.raster` — no network, -no real COGs — and redirect the cache to a temp dir. A deterministic fake read -lets us assert shapes, layer wiring, reproducibility, and composition. +The read path is real (STAC/ODE search + raster reproject + cache), so these +tests stub :mod:`astrofetch.data.stac`, :mod:`astrofetch.data.ode`, and +:mod:`astrofetch.data.raster` — no network, no real rasters — and redirect +the cache to a temp dir. A deterministic fake read lets us assert shapes, +layer wiring, reproducibility, and composition. """ from __future__ import annotations @@ -28,7 +29,7 @@ def _fake_find_asset_hrefs( return [f"{collection}|{asset}"] -def _fake_read_window(href, grid, band=1, resampling=None): +def _fake_read_window(href, grid, band=1, resampling=None, nodata_override=None): # Deterministic in (href, window): identical requests read identical data, # which is what makes seeded sampling reproducible under mocking. value = float(hash((href, grid.bbox)) % 997) @@ -36,11 +37,34 @@ def _fake_read_window(href, grid, band=1, resampling=None): return image, np.ones((grid.height, grid.width), dtype=bool) +def _fake_find_file_urls( + ihid: str, + iid: str, + pt: str, + pattern: str, + bbox: tuple, + max_products: int = 20, + file_type: str | None = "Product", + root: str | None = None, +) -> list[str]: + return [f"{ihid}|{iid}|{pt}"] + + +def _fake_query_products( + ihid: str, iid: str, pt: str, bbox: tuple, max_products: int = 20, root: str | None = None +) -> list: + # No footprints by default: exercises the uniform-sampling fallback unless + # a test overrides this to supply real footprints. + return [] + + @pytest.fixture(autouse=True) def _mock_reads(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("ASTROFETCH_CACHE", str(tmp_path / "cache")) monkeypatch.setattr(ds.stac, "find_asset_hrefs", _fake_find_asset_hrefs) monkeypatch.setattr(ds.raster, "read_window", _fake_read_window) + monkeypatch.setattr(ds.ode, "find_file_urls", _fake_find_file_urls) + monkeypatch.setattr(ds.ode, "query_products", _fake_query_products) def test_instrument_yields_sample_dicts() -> None: @@ -193,7 +217,7 @@ def test_catalog_and_registry_agree() -> None: def test_mosaic_prefers_earlier_items_and_fills_gaps(monkeypatch: pytest.MonkeyPatch) -> None: from astrofetch.data.grid import TargetGrid - def _coverage(href, grid, band=1, resampling=None): + def _coverage(href, grid, band=1, resampling=None, nodata_override=None): image = np.full((grid.height, grid.width), float(href[-1]), dtype=np.float32) mask = np.zeros((grid.height, grid.width), dtype=bool) if href.endswith("1"): # first item covers only the left half @@ -220,3 +244,211 @@ def test_mosaic_with_no_items_is_all_invalid() -> None: image, mask = ds._mosaic([], grid) assert not mask.any() assert (image == 0.0).all() + + +def test_mosaic_passes_band_and_nodata_override() -> None: + from astrofetch.data.grid import TargetGrid + + seen: list[tuple[int, float | None]] = [] + + def _spy(href, grid, band=1, resampling=None, nodata_override=None): + seen.append((band, nodata_override)) + shape = (grid.height, grid.width) + return np.zeros(shape, dtype=np.float32), np.ones(shape, dtype=bool) + + grid = TargetGrid(bbox=(0.0, 0.0, 1.0, 1.0), width=4, height=4) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(ds.raster, "read_window", _spy) + ds._mosaic(["item"], grid, band=2, nodata_override=-999.0) + assert seen == [(2, -999.0)] + + +# --- ODEInstrumentDataset ----------------------------------------------- + + +def test_ode_instrument_yields_sample_dicts() -> None: + moondata = af.LROCNACDTM( + products=["dtm", "ortho"], + bbox=(-60.0, 5.0, -55.0, 10.0), + patch_size=32, + length=3, + seed=0, + footprint_sampling=False, + ) + samples = list(moondata) + assert len(samples) == 3 + for sample in samples: + assert set(sample) == SAMPLE_KEYS + assert sample["image"].shape == (2, 32, 32) + assert sample["image"].dtype == torch.float32 + assert sample["mask"].shape == (2, 32, 32) + assert sample["layers"] == ["lroc_nac_dtm", "lroc_nac_ortho"] + + +def test_ode_read_queries_ode_by_ihid_iid_pt(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, str, str]] = [] + + def _spy(ihid, iid, pt, pattern, bbox, max_products=20, file_type="Product", root=None): + calls.append((ihid, iid, pt)) + return [f"{ihid}|{iid}|{pt}"] + + monkeypatch.setattr(ds.ode, "find_file_urls", _spy) + next( + iter( + af.LROCNACDTM( + products=["dtm"], patch_size=8, length=1, seed=0, footprint_sampling=False + ) + ) + ) + assert calls == [("LRO", "LROC", "SDNDTM")] + + +def test_ode_dataset_default_products_is_quantitative_only() -> None: + # Slope/shade are rendered visualizations (AGENTS rule 3); only + # elevation, orthoimage, and confidence are offered. + assert set(af.LROCNACDTM.all_products) == {"dtm", "ortho", "confidence"} + + +def test_ode_rejects_unknown_product() -> None: + with pytest.raises(ValueError): + af.LROCNACDTM(products=["slope"]) + + +# --- MosaicDataset ------------------------------------------------------- + + +def test_mosaic_dataset_reads_fixed_href_without_search(monkeypatch: pytest.MonkeyPatch) -> None: + seen_hrefs: list[str] = [] + + def _spy(href, grid, band=1, resampling=None, nodata_override=None): + seen_hrefs.append(href) + return _fake_read_window(href, grid, band, resampling, nodata_override) + + monkeypatch.setattr(ds.raster, "read_window", _spy) + monkeypatch.setattr( + ds.ode, + "query_products", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("MosaicDataset must not search ODE")), + ) + moondata = af.LROCWACMosaic(patch_size=8, length=1, seed=0) + sample = moondata[0] + assert set(sample) == SAMPLE_KEYS + assert sample["image"].shape == (1, 8, 8) + assert seen_hrefs == [ds.endpoints.LROC_WAC_MOSAIC_100M_URL] + + +def test_lola_and_sldem_read_their_fixed_hrefs(monkeypatch: pytest.MonkeyPatch) -> None: + seen_hrefs: list[str] = [] + monkeypatch.setattr( + ds.raster, + "read_window", + lambda href, grid, band=1, resampling=None, nodata_override=None: ( + seen_hrefs.append(href), + _fake_read_window(href, grid), + )[1], + ) + af.LOLA(patch_size=8, length=1, seed=0)[0] + af.SLDEM2015(patch_size=8, length=1, seed=0)[0] + assert seen_hrefs == [ds.endpoints.LOLA_DEM_128_URL, ds.endpoints.SLDEM2015_URL] + + +# --- footprint-constrained sampling --------------------------------------- + +_FOOTPRINTS = [(-60.0, 5.0, -59.0, 6.0), (10.0, -20.0, 11.0, -19.0)] + + +def _fake_footprint_products(*_args, **_kwargs) -> list: + return [ + ds.ode.ODEProduct(pdsid=f"p{i}", files=(), bbox=fp, metadata={}) + for i, fp in enumerate(_FOOTPRINTS) + ] + + +def test_footprint_sampling_draws_windows_inside_a_footprint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ds.ode, "query_products", _fake_footprint_products) + moondata = af.LROCNACDTM(products=["dtm"], patch_size=8, length=20, seed=0) + assert moondata.footprint_sampling is True + for sample in moondata: + west, south, east, north = sample["bbox"] + assert any( + fw <= west and east <= fe and fs <= south and north <= fn + for fw, fs, fe, fn in _FOOTPRINTS + ) + + +def test_footprint_sampling_is_reproducible(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ds.ode, "query_products", _fake_footprint_products) + + def sampler() -> af.LROCNACDTM: + return af.LROCNACDTM(products=["dtm"], patch_size=8, length=3, seed=7) + + first = [s["bbox"] for s in sampler()] + second = [s["bbox"] for s in sampler()] + assert first == second + + +def test_footprint_sampling_falls_back_to_uniform_with_no_footprints() -> None: + # Default autouse fixture's fake query_products returns []. + moondata = af.LROCNACDTM( + products=["dtm"], bbox=(-60.0, 5.0, -55.0, 10.0), patch_size=8, length=1, seed=0 + ) + west, south, east, north = moondata[0]["bbox"] + assert -60.0 <= west and east <= -55.0 + assert 5.0 <= south and north <= 10.0 + + +def test_footprint_sampling_can_be_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + calls = [] + monkeypatch.setattr( + ds.ode, "query_products", lambda *a, **k: calls.append(1) or _fake_footprint_products() + ) + moondata = af.LROCNACDTM( + products=["dtm"], + bbox=(-60.0, 5.0, -55.0, 10.0), + patch_size=8, + length=1, + seed=0, + footprint_sampling=False, + ) + _ = moondata[0] + assert calls == [] + + +# --- catalog / registry for the new datasets ------------------------------ + + +def test_catalog_includes_lro_probe() -> None: + lro = MOON.probes["lro"] + assert lro.instruments["nac_dtm"].dataset is af.LROCNACDTM + assert lro.instruments["wac_mosaic"].dataset is af.LROCWACMosaic + assert lro.instruments["lola"].dataset is af.LOLA + assert lro.instruments["sldem2015"].dataset is af.SLDEM2015 + assert lro.granules["nac_raw"] is af.LROCNACRaw + assert lro.granules["wac_raw"] is af.LROCWACRaw + + +def test_catalog_includes_chandrayaan1_granules() -> None: + assert MOON.probes["chandrayaan1"].granules["m3"] is af.M3 + assert MOON.probes["chandrayaan1"].instruments == {} + + +def test_registry_agrees_for_ode_layer() -> None: + spec = MOON.probes["lro"].instruments["nac_dtm"].products["dtm"] + assert spec is LAYERS["lroc_nac_dtm"] + assert spec.source == "ode" + assert spec.ihid == "LRO" + assert spec.iid == "LROC" + assert spec.pt == "SDNDTM" + + +def test_registry_agrees_for_mosaic_layer() -> None: + spec = MOON.probes["lro"].instruments["wac_mosaic"].products["morphology"] + assert spec is LAYERS["lroc_wac_mosaic"] + assert spec.source == "mosaic" + assert spec.href == ds.endpoints.LROC_WAC_MOSAIC_100M_URL + + +def test_registry_marks_stac_layers_with_source() -> None: + assert LAYERS["kaguya_tc_dtm"].source == "stac" From 57a088710d17815046560478798dc96857b28593 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:27 +0200 Subject: [PATCH 10/22] test(moon): add unit tests for raw granule datasets Local GeoTIFFs plus a stubbed ode.query_products, no network. Covers length, sample keys, extra_patterns backplanes, negative/out-of-range indexing, the missing-file error, rows= partial reads, and the max_pixels guard. --- tests/unit/test_granules.py | 152 ++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/unit/test_granules.py diff --git a/tests/unit/test_granules.py b/tests/unit/test_granules.py new file mode 100644 index 0000000..b176743 --- /dev/null +++ b/tests/unit/test_granules.py @@ -0,0 +1,152 @@ +"""Unit tests for the experimental raw-granule datasets — local GeoTIFFs +and a stubbed :mod:`astrofetch.data.ode`, never the network. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +import rasterio +import torch +from rasterio.transform import from_bounds + +from astrofetch.data import ode +from astrofetch.errors import EndpointError +from astrofetch.moon import granules + + +def _write_geotiff(path: Path, data: np.ndarray) -> str: + height, width = data.shape + with rasterio.open( + path, + "w", + driver="GTiff", + height=height, + width=width, + count=1, + dtype=data.dtype, + crs="EPSG:4326", + transform=from_bounds(0.0, 0.0, width, height, width, height), + ) as dst: + dst.write(data, 1) + return str(path) + + +def _fake_product(pdsid: str, main_url: str, extra_url: str | None = None) -> ode.ODEProduct: + files = [ode.ODEFile("MAIN.TIF", "Product", main_url)] + if extra_url is not None: + files.append(ode.ODEFile("EXTRA.TIF", "Product", extra_url)) + return ode.ODEProduct( + pdsid=pdsid, files=tuple(files), bbox=(1.0, 2.0, 3.0, 4.0), metadata={"k": "v"} + ) + + +class _FakeGranules(granules.GranuleDataset): + probe = "Test Probe" + instrument = "Test Instrument" + ihid = "TEST" + iid = "TESTI" + pt = "TESTPT" + file_pattern = r"MAIN\.TIF" + extra_patterns = {"extra": r"EXTRA\.TIF"} + + +def test_len_matches_ode_query(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + main = _write_geotiff(tmp_path / "main.tif", np.zeros((4, 4), dtype=np.float32)) + products = [_fake_product("p1", main), _fake_product("p2", main)] + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: products) + dataset = _FakeGranules(bbox=(-1.0, -1.0, 1.0, 1.0), max_products=10) + assert len(dataset) == 2 + + +def test_getitem_reads_main_file_and_metadata( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + data = np.arange(64, dtype=np.float32).reshape(8, 8) + main = _write_geotiff(tmp_path / "main.tif", data) + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: [_fake_product("p1", main)]) + sample = _FakeGranules()[0] + assert {"image", "mask", "pdsid", "bbox", "meta"} <= set(sample) + assert sample["pdsid"] == "p1" + assert sample["bbox"] == (1.0, 2.0, 3.0, 4.0) + assert sample["meta"] == {"k": "v"} + assert sample["image"].shape == (1, 8, 8) + assert torch.equal(sample["image"][0], torch.from_numpy(data)) + assert sample["mask"].all() + + +def test_getitem_reads_extra_patterns(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + main = _write_geotiff(tmp_path / "main.tif", np.zeros((4, 4), dtype=np.float32)) + extra = _write_geotiff(tmp_path / "extra.tif", np.ones((4, 4), dtype=np.float32)) + product = _fake_product("p1", main, extra_url=extra) + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: [product]) + sample = _FakeGranules()[0] + assert "extra" in sample + assert "extra_mask" in sample + assert sample["extra"].shape == (1, 4, 4) + assert sample["extra"][0, 0, 0] == 1.0 + + +def test_negative_index_and_out_of_range(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + main = _write_geotiff(tmp_path / "main.tif", np.zeros((4, 4), dtype=np.float32)) + products = [_fake_product("p1", main), _fake_product("p2", main)] + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: products) + dataset = _FakeGranules() + assert dataset[-1]["pdsid"] == "p2" + with pytest.raises(IndexError): + dataset[2] + + +def test_no_matching_file_raises_endpoint_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + product = ode.ODEProduct(pdsid="p1", files=(), bbox=None, metadata={}) + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: [product]) + with pytest.raises(EndpointError): + _FakeGranules()[0] + + +def test_rows_reads_only_the_requested_row_range( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + data = np.arange(64, dtype=np.float32).reshape(8, 8) + main = _write_geotiff(tmp_path / "main.tif", data) + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: [_fake_product("p1", main)]) + sample = _FakeGranules(rows=slice(2, 5))[0] + assert sample["image"].shape == (1, 3, 8) + assert torch.equal(sample["image"][0], torch.from_numpy(data[2:5])) + + +def test_max_pixels_guard_raises_without_rows( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + main = _write_geotiff(tmp_path / "main.tif", np.zeros((8, 8), dtype=np.float32)) + monkeypatch.setattr(granules.ode, "query_products", lambda *a, **k: [_fake_product("p1", main)]) + dataset = _FakeGranules(max_pixels=10) # 8*8*1 = 64 > 10 + with pytest.raises(ValueError, match="max_pixels"): + dataset[0] + + +def test_invalid_bbox_raises() -> None: + with pytest.raises(ValueError): + _FakeGranules(bbox=(10.0, 0.0, -10.0, 5.0)) + + +def test_lroc_nac_raw_classvars() -> None: + assert granules.LROCNACRaw.ihid == "LRO" + assert granules.LROCNACRaw.iid == "LROC" + assert granules.LROCNACRaw.pt == "CDRNAC4" + + +def test_lroc_wac_raw_classvars() -> None: + assert granules.LROCWACRaw.ihid == "LRO" + assert granules.LROCWACRaw.pt == "CDRWAM4" + + +def test_m3_classvars_and_extra_patterns() -> None: + assert granules.M3.ihid == "CH1-ORB" + assert granules.M3.iid == "M3" + assert granules.M3.pt == "CALIMG" + assert "loc" in granules.M3.extra_patterns From b49ad81d0305e8e61b4ef59ff6b7f079f952b424 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:32 +0200 Subject: [PATCH 11/22] test(live): add live smoke tests for the new data sources One targeted test per source against real archives: NAC DTM (PDS ODE search plus footprint sampling), the WAC mosaic, LOLA, SLDEM2015, and the NAC-raw/M3 granule paths. Deselected by default; run manually with pytest tests/live -m live. Each also serves as the live verification for that dataset's filename-pattern regexes. --- tests/live/test_pds_ode_live.py | 118 ++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tests/live/test_pds_ode_live.py diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py new file mode 100644 index 0000000..63e8a52 --- /dev/null +++ b/tests/live/test_pds_ode_live.py @@ -0,0 +1,118 @@ +"""Live endpoint tests for the PDS-ODE-backed and fixed-mosaic datasets — +hit real government/archive servers, manual trigger only. + +These never run in CI or in the default ``pytest`` invocation; they are +deselected by the ``-m 'not live'`` default in ``pyproject.toml``. Run them +explicitly, one at a time, when verifying a real endpoint:: + + uv run pytest tests/live -m live + +Each test also doubles as the live verification step for that dataset's +filename-pattern regexes (AGENTS testing rules): if an archive changes its +naming convention, one of these fails loudly instead of a user silently +getting an all-invalid mask. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import astrofetch as af +from astrofetch.data.cache import WindowCache + +# A small box around the Apollo 15 NAC stereo DTM site (known ODE SDNDTM +# coverage), safely within SLDEM2015's 60S-60N extent too. +_APOLLO15_AREA = (3.0, 25.0, 4.5, 26.5) + + +@pytest.mark.live +def test_nac_dtm_fetches_a_real_patch(tmp_path: Path) -> None: + """End to end: PDS ODE search, footprint-constrained sampling, PDS4 + label-over-HTTPS read, and the elevation-product nodata override.""" + moondata = af.LROCNACDTM( + products=["dtm", "ortho"], + bbox=_APOLLO15_AREA, + resolution=5.0, + patch_size=64, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (2, 64, 64) + assert sample["layers"] == ["lroc_nac_dtm", "lroc_nac_ortho"] + assert bool(sample["mask"].any()) + + +@pytest.mark.live +def test_wac_mosaic_fetches_a_real_patch(tmp_path: Path) -> None: + """The fixed-URL WAC global mosaic opens and reads at its native 100 m.""" + moondata = af.LROCWACMosaic( + bbox=_APOLLO15_AREA, + resolution=100.0, + patch_size=64, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 64, 64) + assert bool(sample["mask"].any()) + + +@pytest.mark.live +def test_lola_dem_fetches_a_real_patch(tmp_path: Path) -> None: + """The fixed-URL global LOLA DEM opens (detached PDS3 label) and reads.""" + moondata = af.LOLA( + bbox=_APOLLO15_AREA, + resolution=200.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + + +@pytest.mark.live +def test_sldem2015_fetches_a_real_patch(tmp_path: Path) -> None: + """SLDEM2015 within its 60S-60N coverage band reads valid elevation.""" + moondata = af.SLDEM2015( + bbox=_APOLLO15_AREA, + resolution=200.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + + +@pytest.mark.live +def test_nac_raw_granule_reads_a_row_slice() -> None: + """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" + dataset = af.LROCNACRaw(bbox=_APOLLO15_AREA, max_products=1, rows=slice(0, 64)) + assert len(dataset) >= 1 + sample = dataset[0] + assert sample["image"].shape[-2] == 64 + assert sample["pdsid"] + + +@pytest.mark.live +def test_m3_granule_reads_radiance_and_geolocation() -> None: + """EXPERIMENTAL granule path: ENVI-format M3 radiance cube (opened via + its .IMG data file directly -- unlike NAC/WAC, M3's driver rejects the + .HDR header file) plus its lon/lat/elevation backplane.""" + dataset = af.M3(bbox=(3.0, 20.0, 8.0, 30.0), max_products=1, rows=slice(0, 32)) + assert len(dataset) >= 1 + sample = dataset[0] + assert sample["image"].shape[0] == 85 # M3 L1B band count + assert sample["image"].shape[-2] == 32 + assert sample["loc"].shape[0] == 3 # longitude, latitude, elevation + assert sample["loc"].shape[-2] == 32 From 301f0aaec461e852ce242e84876c8be5983d46b4 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:37 +0200 Subject: [PATCH 12/22] docs: document new data sources and raw granule datasets Add reference/granules.md (the experimental contract, size guidance) and register it in the mkdocs nav. reference/data.md and datasets.md gain an intro distinguishing the three sources (STAC/ODE/fixed mosaic) and entries for the new classes. index.md gets a quickstart example beyond the STAC catalog. roadmap.md reflects Phase 1 as complete and Phase 2's new-source work as delivered, with what's still open called out. --- docs/index.md | 18 ++++++++++++++ docs/reference/catalog.md | 7 ++++-- docs/reference/data.md | 22 +++++++++++++++-- docs/reference/datasets.md | 32 ++++++++++++++++++++++++- docs/reference/granules.md | 48 ++++++++++++++++++++++++++++++++++++++ docs/roadmap.md | 18 ++++++++++---- mkdocs.yml | 1 + 7 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 docs/reference/granules.md diff --git a/docs/index.md b/docs/index.md index f3bf87c..6430f7d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -44,6 +44,24 @@ for batch in loader: batch["image"] # torch.Tensor (16, C, H, W) ``` +## Beyond the STAC catalog + +Some instruments (LROC, LOLA, ...) aren't in the USGS ARD STAC catalog at +all; those datasets search the NASA PDS Orbital Data Explorer instead, or +read a single fixed mosaic URL, behind the exact same interface: + +```python +import astrofetch as af + +# LROC NAC stereo DTM sites are a few hundred named sites, not global +# coverage, so sampled windows are drawn from inside a real site by default. +nac = af.LROCNACDTM(products=["dtm", "ortho"], bbox=(3.0, 25.0, 4.5, 26.5)) +sample = nac[0] + +# A global 100 m WAC mosaic and a global LOLA DEM, channel-stacked with `&`. +terrain = af.LROCWACMosaic(resolution=100) & af.LOLA(resolution=100) +``` + ## Discovering what data exists The `MOON` catalog enumerates probes, instruments, and products, and points at diff --git a/docs/reference/catalog.md b/docs/reference/catalog.md index b00f4d7..74e299c 100644 --- a/docs/reference/catalog.md +++ b/docs/reference/catalog.md @@ -1,8 +1,11 @@ # Catalog & layer registry `MOON` is the discovery catalog — enumerate probes, instruments, and products, -and reach the dataset classes. `LAYERS` maps each layer id to its provenance and -backing STAC collection. +and reach the dataset classes. `LAYERS` maps each layer id to its provenance +and backing source (`LayerSpec.source` is `"stac"`, `"ode"`, or `"mosaic"`). +A probe may also carry `granules`: experimental, non-map-projected dataset +classes (see [Raw granules](granules.md)) that sit outside the layer/product +contract entirely and are not part of `LAYERS`. ::: astrofetch.moon.layers.MOON options: diff --git a/docs/reference/data.md b/docs/reference/data.md index c9f7937..7d35c00 100644 --- a/docs/reference/data.md +++ b/docs/reference/data.md @@ -1,7 +1,21 @@ # Data layer -Body-agnostic building blocks shared by every instrument dataset: STAC search, -windowed COG reads, the target grid, and the disposable cache. +Body-agnostic building blocks shared by every instrument dataset: STAC and +PDS ODE search, windowed raster reads, the target grid, and the disposable +cache. Three sources back instrument datasets — see +[Instrument datasets](datasets.md) for which each instrument uses: + +- **STAC** (`astrofetch.data.stac`): the USGS Astrogeology Analysis Ready + Data catalog, searched via `pystac-client`. +- **PDS ODE** (`astrofetch.data.ode`): the NASA PDS Orbital Data Explorer + REST API, for instruments (LROC, LOLA, M3, ...) the STAC catalog does not + carry. +- **Fixed mosaics**: a handful of instruments are published as one global + (or near-global) file rather than many searchable items; these are read + directly from a well-known URL in `astrofetch.data.endpoints`, no search. + +All three reproject through the same windowed-read path and share the same +sample-dict contract. ## Target grid @@ -15,6 +29,10 @@ windowed COG reads, the target grid, and the disposable cache. ::: astrofetch.data.stac +## PDS ODE search + +::: astrofetch.data.ode + ## Cache ::: astrofetch.data.cache diff --git a/docs/reference/datasets.md b/docs/reference/datasets.md index 2260e55..ac6b9e9 100644 --- a/docs/reference/datasets.md +++ b/docs/reference/datasets.md @@ -2,14 +2,44 @@ One dataset class per instrument, each a map-style `torch` dataset that samples coregistered patches. Combine instruments with `&` to stack their channels over -the overlapping region. +the overlapping region. Every instrument dataset shares the same sample-dict +contract regardless of where its products are read from — see +[Data layer](data.md) for the three sources (STAC, PDS ODE, fixed mosaics). + +For raw, non-map-projected data (camera-geometry strips), see the separate, +experimental [Raw granules](granules.md) page — a deliberately different +contract, not part of the windowed-dataset family below. + +## Kaguya (SELENE) ::: astrofetch.moon.datasets.KaguyaTC ::: astrofetch.moon.datasets.KaguyaTCImagery +## Lunar Reconnaissance Orbiter + +::: astrofetch.moon.datasets.LROCNACDTM + +::: astrofetch.moon.datasets.LROCWACMosaic + +::: astrofetch.moon.datasets.LOLA + +::: astrofetch.moon.datasets.SLDEM2015 + +## Base classes + ::: astrofetch.moon.datasets.InstrumentDataset +::: astrofetch.moon.datasets.ODEInstrumentDataset + +::: astrofetch.moon.datasets.MosaicDataset + ::: astrofetch.moon.datasets.IntersectionDataset +## Product specs + ::: astrofetch.moon.datasets.Product + +::: astrofetch.moon.datasets.ODEAsset + +::: astrofetch.moon.datasets.MosaicAsset diff --git a/docs/reference/granules.md b/docs/reference/granules.md new file mode 100644 index 0000000..704542c --- /dev/null +++ b/docs/reference/granules.md @@ -0,0 +1,48 @@ +# Raw granules (experimental) + +!!! warning "Experimental — a different contract than every other dataset" + Unlike the [instrument datasets](datasets.md), raw granule datasets are + **not** reprojected onto a common grid: each item is one raw PDS product + (an NAC/WAC calibrated strip, an M3 radiance cube, ...), read in its own + native camera/instrument geometry. No reprojection, resampling, ISIS, or + SPICE processing is applied. + + Consequences: + + - **No bbox windowing.** `__getitem__` returns a whole granule (or a row + range, via `rows=`), not a patch cropped to a requested extent. + - **Ragged shapes across items.** The default `DataLoader` collation will + not work; use `batch_size=None` or a custom `collate_fn`. + - **No `&` composition.** There is no shared grid to stack channels onto. + + `len()` is the number of PDS ODE products matching a bbox, fetched once + and eagerly in `__init__`, so `len()` never needs a network call. + +## Reading large strips + +NAC/WAC strips can be gigapixel. Reading a whole granule with no `rows=` +raises once its pixel count exceeds `max_pixels` (about 512 MiB as float32 +by default), naming the granule's size and suggesting a row range: + +```python +import astrofetch as af + +# Read only the first 512 rows of every matching strip instead of the whole +# multi-gigapixel granule. +dataset = af.LROCNACRaw(bbox=(-26.3, -50.6, -25.5, -49.7), rows=slice(0, 512)) +sample = dataset[0] +sample["image"] # (bands, 512, W) float32, physical values +sample["mask"] # same-shaped bool validity +``` + +## Datasets + +::: astrofetch.moon.granules.LROCNACRaw + +::: astrofetch.moon.granules.LROCWACRaw + +::: astrofetch.moon.granules.M3 + +## Base class + +::: astrofetch.moon.granules.GranuleDataset diff --git a/docs/roadmap.md b/docs/roadmap.md index 51e5a21..7ae392e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -7,10 +7,20 @@ this page is the short version. | Phase | Deliverable | Status | |:-----:|:------------|:------:| | 0 | Scaffolding — package, CI, docs, target API | Done | -| 1 | STAC sampler MVP — bbox to coregistered `(C, H, W)` tensor | In progress | -| 2 | Datasets and transforms — grid-tile dataset, spatial splits, transforms, LRO WAC | Planned | +| 1 | STAC sampler MVP — bbox to coregistered `(C, H, W)` tensor | Done | +| 2 | Datasets and transforms — new data sources, grid-tile dataset, spatial splits, transforms | In progress | | 3 | Release and community — PyPI, planetarypy affiliation, paper | Planned | -**Current phase: Phase 1.** `InstrumentDataset.read(bbox)` now fetches real COGs +**Current phase: Phase 2.** `InstrumentDataset.read(bbox)` fetches real COGs from the USGS ARD catalog, reprojects them onto a common geographic grid, -applies scale/offset, mosaics overlapping items, and caches the result. +applies scale/offset, mosaics overlapping items, and caches the result — the +Phase 1 exit criterion. Phase 2 has started delivering new data sources +beyond STAC: the NASA PDS Orbital Data Explorer (`ODEInstrumentDataset`) adds +LROC NAC stereo DTM sites, and fixed-URL mosaics (`MosaicDataset`) add the +LRO WAC global mosaic and the LOLA and SLDEM2015 global DEMs — all behind +the same sample-dict contract and `&` composition as the STAC-backed +datasets. An experimental, separately-contracted raw-granule path +(`astrofetch.moon.granules`) also now exists for camera-geometry NAC/WAC +strips and M3 radiance cubes; see [Raw granules](reference/granules.md). +Still open for Phase 2: `GridTileDataset`, spatial-autocorrelation-aware +train/val/test splitting, transforms, and the WMS/WMTS rendered mode. diff --git a/mkdocs.yml b/mkdocs.yml index 393c83c..d39e75c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - Roadmap: roadmap.md - API Reference: - Instrument datasets: reference/datasets.md + - Raw granules (experimental): reference/granules.md - Catalog & registry: reference/catalog.md - Data layer: reference/data.md From f7b455eb6cbede94383c57bf0a22f419ee17be88 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:25:43 +0200 Subject: [PATCH 13/22] docs: update AGENTS.md for the new data sources and phase status Architecture tree and design rules now reflect data/ode.py and moon/granules.py. Amends the "no PDS granule access" non-goal to describe what actually shipped (raw, camera-geometry only, experimental, excluded from the quantitative registry) rather than leaving it contradicted by the code. Updates the roadmap's current-phase banner and Phase 2 section to separate delivered work from what's still open. --- AGENTS.md | 54 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index de0424e..f9187fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Instructions for AI coding agents (Claude Code, Codex, Cursor, and others) worki ## What this project is -AstroFetch is an open source, PyTorch-friendly library for ML-ready access to planetary science data, starting with the Moon. The core promise: request a bounding box, receive a coregistered multichannel tensor. The core API: one dataset class per instrument (`KaguyaTC`, `LROCWAC`, ...), each yielding TorchGeo-style sample dicts with a coregistered multichannel `"image"` tensor, a validity `"mask"`, and per-channel provenance; instruments compose with `&` (`IntersectionDataset`) to stack channels over their overlapping region. Probes and bodies are discovery-catalog metadata (`MOON`), never dataset boundaries. It is a thin composition layer over existing archive tooling, never a mirror of any archive. +AstroFetch is an open source, PyTorch-friendly library for ML-ready access to planetary science data, starting with the Moon. The core promise: request a bounding box, receive a coregistered multichannel tensor. The core API: one dataset class per instrument (`KaguyaTC`, `LROCNACDTM`, `LROCWACMosaic`, ...), each yielding TorchGeo-style sample dicts with a coregistered multichannel `"image"` tensor, a validity `"mask"`, and per-channel provenance; instruments compose with `&` (`IntersectionDataset`) to stack channels over their overlapping region. Probes and bodies are discovery-catalog metadata (`MOON`), never dataset boundaries. It is a thin composition layer over existing archive tooling, never a mirror of any archive. A separate, explicitly-marked-experimental family (`astrofetch.moon.granules`) deviates from this contract for raw, non-map-projected camera data — see rule 7 and its own module docstring. ## Architecture at a glance @@ -13,27 +13,31 @@ src/astrofetch/ data/ endpoints.py # ALL external URLs live here, nowhere else stac.py # STAC queries (pystac-client) against USGS Astrogeology ARD - raster.py # windowed COG reads (rasterio), scale/offset -> physical values + ode.py # PDS Orbital Data Explorer (ODE) REST queries, for instruments STAC doesn't carry + raster.py # windowed/full raster reads (rasterio), scale/offset -> physical values grid.py # target grid definition, reprojection, channel stacking cache.py # throwaway local cache, keyed by (collection, item, window, res) - tiles.py # secondary rendered mode: USGS WMS, Moon Trek WMTS + tiles.py # secondary rendered mode: USGS WMS, Moon Trek WMTS (not yet built) moon/ - layers.py # layer registry (name -> STAC collection + read config) + Body/Probe/Instrument catalog (MOON) - datasets.py # instrument dataset classes (InstrumentDataset, KaguyaTC, KaguyaTCImagery) + IntersectionDataset + layers.py # layer registry (name -> source config: STAC/ODE/mosaic) + Body/Probe/Instrument catalog (MOON) + datasets.py # windowed dataset classes: InstrumentDataset (STAC), ODEInstrumentDataset (PDS ODE), + # MosaicDataset (fixed URL), concrete instruments, IntersectionDataset + granules.py # EXPERIMENTAL: raw, non-map-projected granule datasets (GranuleDataset and subclasses) tests/ unit/ # network fully mocked, runs in CI live/ # hits real endpoints, manual trigger only + fixtures/ # recorded JSON response fixtures for unit tests ``` ## Non-negotiable design rules -1. **Default to wrapping archive tooling; reimplement only with a measured reason.** pystac-client queries STAC, rasterio reads and windows COGs. Reach for these first — reinventing them is usually wasted effort and a maintenance burden. Two carve-outs: (a) **never** hand-roll domain-specific correctness — map-projection math, COG windowing, scale/offset conversion — the bugs there are subtle and scientific, so always defer to the established library; (b) generic plumbing (discovery helpers, small utilities) *may* be replaced when a dependency is provably a bad trade — too slow on the hot path, a heavy transitive dependency for a sliver of use, etc. Justify any such reimplementation in the PR description with the concrete reason. +1. **Default to wrapping archive tooling; reimplement only with a measured reason.** pystac-client queries STAC, rasterio reads and windows rasters. Reach for these first — reinventing them is usually wasted effort and a maintenance burden. Two carve-outs: (a) **never** hand-roll domain-specific correctness — map-projection math, raster windowing, scale/offset conversion — the bugs there are subtle and scientific, so always defer to the established library; (b) generic plumbing (discovery helpers, small utilities) *may* be replaced when a dependency is provably a bad trade — too slow on the hot path, a heavy transitive dependency for a sliver of use, etc. Justify any such reimplementation in the PR description with the concrete reason. Note that "defer to the library" can still require picking the *right entry point* into that library: for the LROC NAC DTM PDS4 products, opening the data file directly (GDAL's native GTiff/PDS driver) gives correct georeferencing and nodata, while opening the same product through its detached `.xml` label does not (a confirmed GDAL PDS4-driver resolution-unit parsing bug, not an astrofetch reimplementation) — verify a new source's actual behavior live before trusting either path. 2. **All external endpoint URLs go in `data/endpoints.py`.** No URL literals anywhere else in `src/`. Endpoints move (QuickMap changed domains); one module keeps that survivable. -3. **Quantitative vs rendered is a hard boundary.** The STAC/COG path returns physical values and is the only path for quantitative or ML use. The WMS/WMTS tile path returns rendered 8-bit imagery and must be labeled as such in APIs and docs. Never mix them silently. +3. **Quantitative vs rendered is a hard boundary.** The STAC/ODE/mosaic paths return physical values and are the only paths for quantitative or ML use. The WMS/WMTS tile path returns rendered 8-bit imagery and must be labeled as such in APIs and docs. Never mix them silently — this is also why a dataset offers only the quantitative products a source provides: e.g. `LROCNACDTM` deliberately excludes the SDP pipeline's color-coded slope and shaded-relief products, which are rendered 8-bit visualizations, not calibrated rasters. 4. **Cache is disposable.** Nothing in the cache layer may be load-bearing for correctness or reproducibility. Everything fetched on demand must be re-fetchable from the archive and safe to delete; never commit fetched data to the repo. -5. **Be polite to archive servers.** Default concurrency is low, retries use exponential backoff, and any code path that could issue many requests must go through the rate-limited session in `data/stac.py` / `data/tiles.py`. Never write a loop that hammers NASA or USGS servers. -6. **Body-namespaced layout.** Moon-specific code lives under `moon/`. Body-agnostic code (grid math, COG reads, caching) lives under `data/`. Adding Mars must be a new sibling module, not edits scattered through existing files. -7. **Samples are dicts with a fixed contract.** `"image"` is (C, H, W) float32 (physical values, channel i = `layers[i]`), `"mask"` is (C, H, W) bool validity, plus `"layers"`, `"bbox"`, `"crs"`, and `"resolution"` provenance keys. Datasets that deviate must document it. Samples must collate under the default `DataLoader` collation. +5. **Be polite to archive servers.** Default concurrency is low, retries use exponential backoff, and any code path that could issue many requests must go through the rate-limited session in `data/stac.py` / `data/ode.py` / `data/tiles.py`. Never write a loop that hammers NASA, USGS, or PDS-node servers. +6. **Body-namespaced layout.** Moon-specific code lives under `moon/`. Body-agnostic code (grid math, raster reads, caching, archive search) lives under `data/`. Adding Mars must be a new sibling module, not edits scattered through existing files. +7. **Samples are dicts with a fixed contract.** `"image"` is (C, H, W) float32 (physical values, channel i = `layers[i]`), `"mask"` is (C, H, W) bool validity, plus `"layers"`, `"bbox"`, `"crs"`, and `"resolution"` provenance keys. Datasets that deviate must document it — the one deliberate exception is `astrofetch.moon.granules`, whose module docstring documents its different contract (no bbox windowing, ragged shapes, no `&`) up front. Samples must collate under the default `DataLoader` collation; granule datasets are the one documented exception (`batch_size=None` or a custom `collate_fn`). ## Dev environment and commands @@ -76,24 +80,26 @@ Widely-adopted defaults that keep the codebase consistent. When in doubt, match ## Domain notes agents should know -- Coordinates are IAU 2015 Moon (ocentric, longitude 0 to 360 or -180 to 180 must be normalized at the API boundary; internal convention is -180 to 180). +- Coordinates are IAU 2015 Moon (ocentric, longitude 0 to 360 or -180 to 180 must be normalized at the API boundary; internal convention is -180 to 180). PDS ODE's REST API wants 0-360 `westernlon`/`easternlon`; `data/ode.py` converts at that one boundary — shift the west bound into 0-360 and add back the original span, rather than taking `% 360` of each bound independently, or a full-Moon bbox like `(-180, 180)` collapses to a zero-width query. - Equatorial data uses equirectangular projection; polar data uses polar stereographic. `data/grid.py` owns this decision; never assume equirectangular blindly near the poles. -- COGs from the USGS ARD catalog often store 16-bit DN with scale/offset to physical units (for example Kaguya TC radiance). Always apply scale/offset in `raster.py`; downstream code assumes physical values. -- Nodata regions are common (orbital swaths do not cover everything). Every sample carries a boolean validity tensor under its `"mask"` key; do not silently zero-fill. +- COGs and other rasters often store 16-bit DN (or similar) with scale/offset to physical units (for example Kaguya TC radiance). Always apply scale/offset in `raster.py`; downstream code assumes physical values. +- Nodata regions are common (orbital swaths do not cover everything). Every sample carries a boolean validity tensor under its `"mask"` key; do not silently zero-fill. Some PDS products omit a declared nodata value even though their raster does not cover its full requested extent; `raster.read_window`'s `nodata_override` exists for exactly this (see its docstring) — reach for a source's own declared nodata first, and only override when you have live-verified the product genuinely has none. +- Some instruments cover only a handful of named sites, not the whole Moon (e.g. LROC NAC stereo DTMs via PDS ODE). `ODEInstrumentDataset.footprint_sampling` draws windows from inside real product footprints for exactly this case; leave it off for globally-covered instruments. ## What NOT to do -- Do not add dependencies casually. Core deps are: torch, rasterio, pystac-client, numpy. Anything else needs a justification in the PR description. +- Do not add dependencies casually. Core deps are: torch, rasterio, pystac-client, numpy, requests. Anything else needs a justification in the PR description. - Do not commit data files, fetched tiles, or notebooks with executed output containing large images. - Do not target QuickMap's internal tile URLs; they are not a public API. Use USGS WMS or Moon Trek WMTS via `data/endpoints.py`. - Do not "fix" scientific constants or projection parameters without a source; cite the reference in the commit message. - Do not weaken the mocked-network rule in unit tests to make something pass. +- Do not trust a raster driver's georeferencing just because the file opens and reads without error — a mechanically successful open/read is not proof the transform, bounds, or nodata are correct (see rule 1's PDS4-label example). Verify live against a known location before shipping a new source. ## Roadmap Check the current phase before proposing work; for example, do not build Phase 2 datasets and transforms while Phase 1 (STAC sampler MVP) is incomplete. Everything is a thin layer above existing archive tooling, never a mirror of any archive. -**Current phase: Phase 1 (STAC sampler). Phase 0 scaffolding is complete. `InstrumentDataset.read` now fetches the real COGs covering a window from the USGS ARD catalog, reprojects them onto a common geographic grid, applies scale/offset, mosaics overlapping items, and caches the result — no more synthetic tensors.** +**Current phase: Phase 2 (datasets and transforms). Phase 0 and Phase 1 are complete: `InstrumentDataset.read` fetches the real COGs covering a window from the USGS ARD catalog, reprojects them onto a common geographic grid, applies scale/offset, mosaics overlapping items, and caches the result. Phase 2 has started delivering new data sources beyond STAC (see below); `GridTileDataset`, spatial-split samplers, transforms, and the WMS/WMTS rendered mode remain open.** ### Phase 0: Scaffolding (weekend 1) @@ -127,15 +133,24 @@ sample["layers"] # ["kaguya_tc_dtm", "kaguya_tc_image"], plus bbox/crs/resoluti - Local disk cache keyed by (collection, item, window, resolution), transparent and clearable. -Exit criteria: `KaguyaTC(products=["dtm", "ortho"], bbox=...)` fetches a real, coregistered two-layer patch from the USGS ARD catalog on a clean machine (covered by `tests/live`). A plotting quickstart notebook is a nice-to-have follow-up. +Exit criteria (met): `KaguyaTC(products=["dtm", "ortho"], bbox=...)` fetches a real, coregistered two-layer patch from the USGS ARD catalog on a clean machine (covered by `tests/live`). A plotting quickstart notebook is a nice-to-have follow-up, still open. ### Phase 2: Datasets and transforms (2 to 3 weekends) +**New data sources beyond STAC (done):** the USGS ARD STAC catalog has no LROC, LOLA, or other lunar collections beyond Kaguya TC, so growing past it required a second search backend. + +- `astrofetch.data.ode`: query the NASA PDS Orbital Data Explorer (ODE) REST API by instrument host/id and product type, politely (same retry/backoff/timeout posture as `data/stac.py`), normalizing ODE's JSON quirks (single-result dict vs list, `"No Products Found"`, HTTP-200 error bodies). +- `astrofetch.moon.datasets.ODEInstrumentDataset`: the ODE-backed sibling of `InstrumentDataset`, with `footprint_sampling` for instruments that cover only named sites rather than the whole Moon. Ships `LROCNACDTM` (LRO LROC NAC stereo DTM sites: elevation, orthoimage, pixel confidence). +- `astrofetch.moon.datasets.MosaicDataset`: reads one well-known archive URL directly, for instruments published as a single global (or near-global) file. Ships `LROCWACMosaic` (the LRO WAC 100 m global mosaic — the dataset this phase originally named as its LRO WAC deliverable, shipped as `LROCWACMosaic` rather than `LROCWAC` since a raw, non-map-projected `LROCWACRaw` also now exists), `LOLA` (global gridded DEM), and `SLDEM2015` (LOLA + Kaguya TC co-registered DEM). +- `astrofetch.moon.granules` (new, experimental — see its Deliberate non-goals amendment below): raw, camera-geometry PDS granules for instruments that are not map-projected at all (`LROCNACRaw`, `LROCWACRaw`, `M3`). A deliberately different, documented sample contract; not part of the `InstrumentDataset` family. + +**Still open:** + - `astrofetch.moon.datasets`: random-bbox sampling already ships inside the instrument datasets (Phase 0); add `GridTileDataset` (deterministic tiling of an ROI) over the same `read(bbox)` interface, plus region-list sampling for the random path. - Samplers that respect spatial autocorrelation for train/val/test splits (block splitting, not random pixels). - Transforms: per-channel normalization stats, nodata masking, polar/equatorial projection handling made explicit. - Secondary access mode behind the same interface: WMS/WMTS rendered mode, clearly labeled non-quantitative. -- LRO WAC global mosaic: the USGS ARD STAC catalog has no LRO WAC collection, so add it here from a non-STAC source (a public COG mosaic or WMS/WMTS), behind the same instrument-dataset interface as a new `LROCWAC` class. +- The wider PDS ODE roster beyond the four flagship datasets above: Diviner, Mini-RF, Clementine, ShadowCam, Kaguya MI, and further WAC-derived products (TiO2, GLD100, 7-color reflectance) all fit the same `ODEInstrumentDataset`/`MosaicDataset` pattern; each needs its own live-verified product type and filename pattern before shipping (rule 1's PDS4-label lesson applies to every new source, not just the ones already caught). Exit criteria: `DataLoader` trains a toy model on random lunar patches without custom user code. @@ -149,7 +164,7 @@ Exit criteria: `DataLoader` trains a toy model on random lunar patches without c ### Deliberate non-goals for v0.x - No mirroring or rehosting of raw PDS archives. -- No PDS granule / full-fidelity product access in v0.x; the STAC/COG path is the only quantitative source. +- **Amended:** raw PDS granule access was originally ruled out entirely for v0.x ("the STAC/COG path is the only quantitative source"). `astrofetch.moon.granules` now provides it, narrowly: camera-geometry data only, no map projection, no ISIS/SPICE, explicitly marked experimental with its own documented (different) sample contract, and excluded from the `LAYERS` registry. The `InstrumentDataset`/`ODEInstrumentDataset`/`MosaicDataset` windowed-and-reprojected path remains the only *quantitative, coregistered-tensor* source — that guarantee is unchanged. - No pretrained models or frozen benchmarks; AstroFetch delivers ML-ready data, you bring the model. - No GUI or web viewer; QuickMap and Trek exist. - No Earth support; TorchGeo owns that space. @@ -157,6 +172,7 @@ Exit criteria: `DataLoader` trains a toy model on random lunar patches without c ### Risks and mitigations - Endpoint drift (services move, as QuickMap's domain change showed): keep all endpoint URLs in one config module, cover them with the live test suite, and document last-verified dates. -- M3 data quality issues: defer the M3 dataset class until after v0.1 unless a user strictly needs it; budget preprocessing time if so. +- M3 data quality: rather than deferring M3 entirely, it shipped scoped to what's verified reliable — the experimental raw-granule path (`astrofetch.moon.granules.M3`), radiance plus geolocation backplane, no map projection or further calibration claimed. A map-projected, quantitative M3 `InstrumentDataset`/`ODEInstrumentDataset` remains deferred until a user need justifies the preprocessing work. +- Archive driver quirks: a source opening and reading without error is not proof its georeferencing or nodata are correct (see design rule 1's PDS4-label example, caught via `tests/live` before shipping `LROCNACDTM`). Live-verify a new source against a known location, not just that `rasterio.open` succeeds. - Server load courtesy: default to conservative request concurrency, exponential backoff, and a bulk prefetch helper so training never hammers archive servers with random access. - Solo-maintainer bus factor: keep scope small, tests honest, and architecture boring enough that contributors can navigate it without you. From 2709f2c9a3d422dbfd2d4febe3aa00bd50799575 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 08:54:55 +0200 Subject: [PATCH 14/22] feat(data): let ODEAsset select a non-Product ODE file role Every current ODE-backed dataset reads a file typed "Product", but some archives (e.g. ShadowCam's DTM confidence maps) ship the file we need under a different ODE role such as "Referenced". Add a file_type field to ODEAsset (default "Product", so all existing datasets are unaffected) and thread it through ODEInstrumentDataset._hrefs and LayerSpec/_spec so the registry stays consistent with what each dataset actually reads. --- src/astrofetch/moon/datasets.py | 6 +++++- src/astrofetch/moon/layers.py | 5 +++++ tests/unit/test_datasets.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index d7b135b..daffd16 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -67,6 +67,10 @@ class ODEAsset(NamedTuple): pattern: str band: int = 1 nodata: float | None = None + file_type: str = "Product" + """Required ODE file role for ``pattern`` to match against. Almost every + product's actual data file is typed ``"Product"``; a few (e.g. ShadowCam + DTM confidence maps) ship their data under ``"Referenced"`` instead.""" class MosaicAsset(NamedTuple): @@ -357,7 +361,7 @@ def __init__( def _hrefs(self, spec: Product | ODEAsset | MosaicAsset, bbox: BBox) -> list[str]: assert isinstance(spec, ODEAsset) return ode.find_file_urls( - self.ihid, self.iid, spec.pt, spec.pattern, bbox, self.max_products + self.ihid, self.iid, spec.pt, spec.pattern, bbox, self.max_products, spec.file_type ) def _sample_bbox(self, index: int) -> BBox: diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 390baf6..662163b 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -69,6 +69,10 @@ class LayerSpec: """Filename pattern selecting the file within an ODE product (``source == "ode"``).""" + file_type: str = "" + """Required ODE file role for ``pattern`` to match against + (``source == "ode"``); almost always ``"Product"``.""" + href: str = "" """Fixed archive URL (``source == "mosaic"``).""" @@ -127,6 +131,7 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: iid=iid, pt=entry.pt, pattern=entry.pattern, + file_type=entry.file_type, ) if isinstance(entry, MosaicAsset): return LayerSpec(**common, source="mosaic", href=entry.href) diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index e4ea55c..6f3083a 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -303,6 +303,31 @@ def _spy(ihid, iid, pt, pattern, bbox, max_products=20, file_type="Product", roo assert calls == [("LRO", "LROC", "SDNDTM")] +def test_ode_asset_file_type_defaults_to_product() -> None: + assert ds.ODEAsset("layer", "PT", r".+").file_type == "Product" + + +def test_ode_read_passes_asset_file_type(monkeypatch: pytest.MonkeyPatch) -> None: + seen: list[str] = [] + + def _spy(ihid, iid, pt, pattern, bbox, max_products=20, file_type="Product", root=None): + seen.append(file_type) + return [f"{ihid}|{iid}|{pt}"] + + class _Referenced(ds.ODEInstrumentDataset): + probe = "Test Probe" + instrument = "Test Instrument" + ihid = "X" + iid = "Y" + all_products = { + "data": ds.ODEAsset("test_layer", "PT", r".+", file_type="Referenced"), + } + + monkeypatch.setattr(ds.ode, "find_file_urls", _spy) + next(iter(_Referenced(products=["data"], patch_size=8, length=1, seed=0))) + assert seen == ["Referenced"] + + def test_ode_dataset_default_products_is_quantitative_only() -> None: # Slope/shade are rendered visualizations (AGENTS rule 3); only # elevation, orthoimage, and confidence are offered. @@ -452,3 +477,7 @@ def test_registry_agrees_for_mosaic_layer() -> None: def test_registry_marks_stac_layers_with_source() -> None: assert LAYERS["kaguya_tc_dtm"].source == "stac" + + +def test_registry_carries_ode_asset_file_type() -> None: + assert LAYERS["lroc_nac_dtm"].file_type == "Product" From 7839f3e663f679c6dce1ce86c3b1440c54ac1e91 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:02:25 +0200 Subject: [PATCH 15/22] feat(moon): add MiniRF S-band radar global mosaics Search PDS ODE (product type MOSDDR) for the LRO Mini-RF global mosaics: circular polarization ratio and same-/opposite-sense circular received power, 128 px/degree, detached PDS3 label (same read path as LOLA). The label declares MISSING_CONSTANT as a float64 sentinel inside a float32 band; GDAL's overflowing cast leaves src.nodata unset, so out-of-coverage pixels silently read back as a large-negative overflow artifact marked valid. Verified live by reading raw pixels and pinning the exact bit pattern as a nodata_override, the same class of archive quirk as the LROCNACDTM PDS4-label bug. --- src/astrofetch/__init__.py | 2 + src/astrofetch/moon/__init__.py | 2 + src/astrofetch/moon/datasets.py | 38 +++++++++++++++++++ src/astrofetch/moon/layers.py | 5 +++ tests/fixtures/ode/phase_c_listings.json | 11 ++++++ tests/live/test_pds_ode_live.py | 19 ++++++++++ tests/unit/test_datasets.py | 48 ++++++++++++++++++++++++ 7 files changed, 125 insertions(+) create mode 100644 tests/fixtures/ode/phase_c_listings.json diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index b44a471..7fe75f7 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -18,6 +18,7 @@ LROCNACRaw, LROCWACMosaic, LROCWACRaw, + MiniRF, ) __version__ = "0.1.0" @@ -33,6 +34,7 @@ "LROCNACRaw", "LROCWACMosaic", "LROCWACRaw", + "MiniRF", "SLDEM2015", "moon", "__version__", diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index 440a829..03b8f33 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -9,6 +9,7 @@ KaguyaTC, KaguyaTCImagery, LROCWACMosaic, + MiniRF, MosaicDataset, ODEInstrumentDataset, ) @@ -32,6 +33,7 @@ "LROCWACMosaic", "LROCWACRaw", "LayerSpec", + "MiniRF", "MosaicDataset", "ODEInstrumentDataset", "Probe", diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index daffd16..f89b715 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -541,6 +541,44 @@ class SLDEM2015(MosaicDataset): all_products = {"dem": MosaicAsset("sldem2015_dem", endpoints.SLDEM2015_URL)} +_MINIRF_NODATA = -3.4028226550889045e38 +"""Mini-RF global mosaics declare ``MISSING_CONSTANT = -1.7976931E+308`` (a +float64 sentinel) in a float32 band; GDAL's overflowing cast of that constant +into the band's dtype leaves ``src.nodata`` unset (rather than raising), and +out-of-coverage pixels read back as this specific overflow artifact -- not +even the standard float32 minimum -- marked "valid" (verified live +2026-07-21 by reading raw pixels directly and inspecting the exact bit +pattern; same class of issue as the LROCNACDTM PDS4-label bug -- rule 1).""" + + +class MiniRF(ODEInstrumentDataset): + """LRO Mini-RF S-band bistatic radar global mosaics, 128 px/degree, + searched via PDS ODE (product type ``MOSDDR``): circular polarization + ratio, and same- and opposite-sense circular received power. + + Each product is itself a single global mosaic (detached PDS3 label, same + read path as :class:`LOLA`); ODE is still searched rather than reading a + fixed URL, matching the rest of the ODE-backed roster. Verified live + 2026-07-21. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "Mini-RF (S-band radar mosaics)" + ihid = "LRO" + iid = "MRFLRO" + all_products = { + "cpr": ODEAsset( + "lro_minirf_cpr", "MOSDDR", r"GLOBAL_CPR_128PPD_SIMP_0C\.LBL", nodata=_MINIRF_NODATA + ), + "sc": ODEAsset( + "lro_minirf_sc", "MOSDDR", r"GLOBAL_SC_128PPD_SIMP_0C\.LBL", nodata=_MINIRF_NODATA + ), + "oc": ODEAsset( + "lro_minirf_oc", "MOSDDR", r"GLOBAL_OC_128PPD_SIMP_0C\.LBL", nodata=_MINIRF_NODATA + ), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 662163b..333ff8b 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -23,6 +23,7 @@ KaguyaTC, KaguyaTCImagery, LROCWACMosaic, + MiniRF, MosaicAsset, ODEAsset, Product, @@ -150,6 +151,9 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(LROCWACMosaic, "morphology"), _spec(LOLA, "dem"), _spec(SLDEM2015, "dem"), + _spec(MiniRF, "cpr"), + _spec(MiniRF, "sc"), + _spec(MiniRF, "oc"), ) } @@ -180,6 +184,7 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: "wac_mosaic": _instrument(LROCWACMosaic), "lola": _instrument(LOLA), "sldem2015": _instrument(SLDEM2015), + "minirf": _instrument(MiniRF), }, granules={ "nac_raw": LROCNACRaw, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json new file mode 100644 index 0000000..6197138 --- /dev/null +++ b/tests/fixtures/ode/phase_c_listings.json @@ -0,0 +1,11 @@ +{ + "LRO/MRFLRO/MOSDDR": [ + {"FileName": "GLOBAL_CPR_128PPD_SIMP_0C.IMG", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_cpr_128ppd_simp_0c.img"}, + {"FileName": "GLOBAL_CPR_128PPD_SIMP_0C.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_cpr_128ppd_simp_0c.lbl"}, + {"FileName": "GLOBAL_CPR_128PPD_SIMP_0C.XML", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_cpr_128ppd_simp_0c.xml"}, + {"FileName": "GLOBAL_CPR_32PPD_SIMP_0C.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/32ppd/global_cpr_32ppd_simp_0c.lbl"}, + {"FileName": "GLOBAL_SC_128PPD_SIMP_0C.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_sc_128ppd_simp_0c.lbl"}, + {"FileName": "GLOBAL_OC_128PPD_SIMP_0C.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_oc_128ppd_simp_0c.lbl"}, + {"FileName": "GLOBAL_CPR_SIMP_0C_BR.PNG", "Type": "Browse", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/browse/global_cpr_simp_0c_br.png"} + ] +} diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index 63e8a52..01ecf0d 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -94,6 +94,25 @@ def test_sldem2015_fetches_a_real_patch(tmp_path: Path) -> None: assert bool(sample["mask"].any()) +@pytest.mark.live +def test_minirf_fetches_a_real_patch(tmp_path: Path) -> None: + """Mini-RF global CPR mosaic: detached PDS3 label read via PDS ODE search.""" + moondata = af.MiniRF( + products=["cpr"], + bbox=(23.0, 18.0, 25.0, 20.0), + resolution=500.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + valid = sample["image"][sample["mask"]] + assert bool((valid >= 0.0).all()) and bool((valid <= 3.0).all()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index 6f3083a..812b010 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json from pathlib import Path import numpy as np @@ -17,11 +18,26 @@ from torch.utils.data import DataLoader import astrofetch as af +from astrofetch.data import ode from astrofetch.moon import LAYERS, MOON from astrofetch.moon import datasets as ds SAMPLE_KEYS = {"image", "mask", "layers", "bbox", "crs", "resolution"} +_FIXTURES = Path(__file__).parent.parent / "fixtures" / "ode" + + +def _phase_c_files(pt_key: str) -> tuple[ode.ODEFile, ...]: + raw = json.loads((_FIXTURES / "phase_c_listings.json").read_text())[pt_key] + return tuple(ode.ODEFile(f["FileName"], f["Type"], f["URL"]) for f in raw) + + +def _assert_pattern_selects(spec: ds.ODEAsset, pt_key: str, expected_suffix: str) -> None: + files = _phase_c_files(pt_key) + urls = ode.match_files(files, spec.pattern, spec.file_type) + assert len(urls) == 1, f"{spec.layer}: expected exactly one match, got {urls}" + assert urls[0].endswith(expected_suffix) + def _fake_find_asset_hrefs( collection: str, asset: str, bbox: tuple, max_items: int = 20, root: str | None = None @@ -481,3 +497,35 @@ def test_registry_marks_stac_layers_with_source() -> None: def test_registry_carries_ode_asset_file_type() -> None: assert LAYERS["lroc_nac_dtm"].file_type == "Product" + + +# --- Phase C: wider PDS ODE roster ---------------------------------------- + + +def test_minirf_patterns_select_the_right_global_mosaic() -> None: + _assert_pattern_selects( + af.MiniRF.all_products["cpr"], "LRO/MRFLRO/MOSDDR", "128ppd_simp_0c.lbl" + ) + _assert_pattern_selects( + af.MiniRF.all_products["sc"], "LRO/MRFLRO/MOSDDR", "global_sc_128ppd_simp_0c.lbl" + ) + _assert_pattern_selects( + af.MiniRF.all_products["oc"], "LRO/MRFLRO/MOSDDR", "global_oc_128ppd_simp_0c.lbl" + ) + + +def test_minirf_yields_sample_dicts() -> None: + moondata = af.MiniRF(products=["cpr", "sc"], patch_size=16, length=1, seed=0) + sample = moondata[0] + assert set(sample) == SAMPLE_KEYS + assert sample["layers"] == ["lro_minirf_cpr", "lro_minirf_sc"] + + +def test_catalog_includes_minirf() -> None: + assert MOON.probes["lro"].instruments["minirf"].dataset is af.MiniRF + spec = MOON.probes["lro"].instruments["minirf"].products["cpr"] + assert spec is LAYERS["lro_minirf_cpr"] + assert spec.source == "ode" + assert spec.ihid == "LRO" + assert spec.iid == "MRFLRO" + assert spec.pt == "MOSDDR" From f6aac88f57019f060fe8cd8dab03eb99b8befca8 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:39:29 +0200 Subject: [PATCH 16/22] feat(data): let ODE searches narrow by productid, and add DivinerGDR Add ODE's productid wildcard filter to query_products/find_file_urls (and a matching ODEAsset.product_id/LayerSpec.product_id field), for product types where a bbox-only search buries the products actually wanted among thousands of unrelated ones and no reasonable max_products cap would ever reach them. DivinerGDR needed this immediately: LRO Diviner rock abundance and regolith temperature are mission-cumulative global mosaics republished periodically under product type GDR_L3, which also carries per-orbit bolometric temperature outnumbering every other parameter combined. productid narrows the search to the specific dated product wanted (the most complete date verified live, 2016-09-13) instead of paging through the whole product type. TBOL itself is intentionally not offered, since mosaicking single-orbit epochs would misrepresent the data. --- src/astrofetch/__init__.py | 2 + src/astrofetch/data/ode.py | 16 +++++- src/astrofetch/moon/__init__.py | 2 + src/astrofetch/moon/datasets.py | 72 ++++++++++++++++++++++-- src/astrofetch/moon/layers.py | 9 +++ tests/fixtures/ode/phase_c_listings.json | 9 +++ tests/live/test_pds_ode_live.py | 24 ++++++++ tests/unit/test_datasets.py | 58 ++++++++++++++++++- tests/unit/test_ode.py | 21 +++++++ 9 files changed, 204 insertions(+), 9 deletions(-) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index 7fe75f7..f5a4dfb 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -12,6 +12,7 @@ M3, MOON, SLDEM2015, + DivinerGDR, IntersectionDataset, KaguyaTC, KaguyaTCImagery, @@ -27,6 +28,7 @@ "LOLA", "MOON", "M3", + "DivinerGDR", "IntersectionDataset", "KaguyaTC", "KaguyaTCImagery", diff --git a/src/astrofetch/data/ode.py b/src/astrofetch/data/ode.py index 30fd3e1..fc83dda 100644 --- a/src/astrofetch/data/ode.py +++ b/src/astrofetch/data/ode.py @@ -127,6 +127,7 @@ def query_products( pt: str, bbox: BBox, max_products: int = 20, + product_id: str | None = None, root: str = ODE_API_ROOT, ) -> list[ODEProduct]: """Search ODE for products of one instrument and product type in ``bbox``. @@ -138,6 +139,14 @@ def query_products( bbox: (west, south, east, north) in degrees, -180 to 180. max_products: cap on products returned; bounds request volume and paging (fetched in pages of up to 100). + product_id: ODE ``productid`` wildcard filter (``*`` matches any + substring), e.g. ``"*wac_gld100*"``. Some product types mix many + unrelated products (rendered visualizations, per-orbit granules, + other parameters) under one ``pt``, so a bbox-only search can + bury the products actually wanted far past any reasonable + ``max_products`` cap; narrowing server-side with a product id + pattern is what keeps that search small and polite (rule 5) + instead of paging through everything. root: ODE API root; defaults to the configured endpoint. Returns: @@ -176,6 +185,8 @@ def query_products( "limit": limit, "offset": offset, } + if product_id is not None: + params["productid"] = product_id try: response = _session().get(root, params=params, timeout=_TIMEOUT_S) response.raise_for_status() @@ -232,6 +243,7 @@ def find_file_urls( bbox: BBox, max_products: int = 20, file_type: str | None = "Product", + product_id: str | None = None, root: str = ODE_API_ROOT, ) -> list[str]: """Return file URLs matching ``pattern`` across products in ``bbox``. @@ -250,6 +262,8 @@ def find_file_urls( bbox: (west, south, east, north) in degrees, -180 to 180. max_products: cap on products searched. file_type: required ODE file role; ``None`` skips this filter. + product_id: ODE ``productid`` wildcard filter; see + :func:`query_products`. root: ODE API root; defaults to the configured endpoint. Returns: @@ -258,7 +272,7 @@ def find_file_urls( Raises: EndpointError: the search failed or ODE reported an error. """ - products = query_products(ihid, iid, pt, bbox, max_products, root) + products = query_products(ihid, iid, pt, bbox, max_products, product_id, root) hrefs: list[str] = [] for product in products: hrefs.extend(match_files(product.files, pattern, file_type)) diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index 03b8f33..1dfc09d 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -4,6 +4,7 @@ LOLA, LROCNACDTM, SLDEM2015, + DivinerGDR, InstrumentDataset, IntersectionDataset, KaguyaTC, @@ -22,6 +23,7 @@ "MOON", "M3", "Body", + "DivinerGDR", "GranuleDataset", "Instrument", "InstrumentDataset", diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index f89b715..dce60e6 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -71,6 +71,12 @@ class ODEAsset(NamedTuple): """Required ODE file role for ``pattern`` to match against. Almost every product's actual data file is typed ``"Product"``; a few (e.g. ShadowCam DTM confidence maps) ship their data under ``"Referenced"`` instead.""" + product_id: str | None = None + """ODE ``productid`` wildcard filter narrowing the bbox search + server-side, e.g. ``"*wac_gld100*"``. Needed whenever a product type + mixes the wanted product with many unrelated ones (other parameters, + rendered visualizations, per-orbit granules) that would otherwise + dominate the results within any reasonable ``max_products`` cap.""" class MosaicAsset(NamedTuple): @@ -361,7 +367,14 @@ def __init__( def _hrefs(self, spec: Product | ODEAsset | MosaicAsset, bbox: BBox) -> list[str]: assert isinstance(spec, ODEAsset) return ode.find_file_urls( - self.ihid, self.iid, spec.pt, spec.pattern, bbox, self.max_products, spec.file_type + self.ihid, + self.iid, + spec.pt, + spec.pattern, + bbox, + self.max_products, + spec.file_type, + spec.product_id, ) def _sample_bbox(self, index: int) -> BBox: @@ -388,14 +401,22 @@ def _product_footprints(self) -> list[BBox]: # Fetched lazily (on first sample, not __init__) so construction never # touches the network -- tests can build instances hermetically. if self._footprints is None: - pts: set[str] = set() + # Grouped by (pt, product_id): a product type can mix products + # this instrument doesn't offer (e.g. a sibling instrument's + # products under the same pt), so footprints must go through the + # same product_id narrowing as the file search itself, or the + # sampling pool would include sites that never yield this + # instrument's data. + queries: set[tuple[str, str | None]] = set() for name in self.products: entry = self.all_products[name] if isinstance(entry, ODEAsset): - pts.add(entry.pt) + queries.add((entry.pt, entry.product_id)) footprints: list[BBox] = [] - for pt in pts: - products = ode.query_products(self.ihid, self.iid, pt, self.bbox, max_products=500) + for pt, product_id in queries: + products = ode.query_products( + self.ihid, self.iid, pt, self.bbox, max_products=500, product_id=product_id + ) footprints.extend(product.bbox for product in products if product.bbox is not None) self._footprints = footprints return self._footprints @@ -579,6 +600,47 @@ class MiniRF(ODEInstrumentDataset): } +class DivinerGDR(ODEInstrumentDataset): + """LRO Diviner rock abundance and regolith temperature, mission-cumulative + global mosaics, 128 px/degree, searched via PDS ODE (product type + ``GDR_L3``). + + Each parameter is republished periodically as a new cumulative global + mosaic (same footprint, more orbits folded in); this pins the most + complete date verified live, 2016-09-13, rather than a loose pattern + that would otherwise match all ~105 dated products and mosaic redundant + copies of the same coverage. ``GDR_L3`` also carries per-orbit + bolometric temperature (``TBOL``), which alone outnumbers every other + parameter combined, so a bbox-only search would need to page through + thousands of unrelated candidates before ever reaching a dated RA or ST + product; ``product_id`` narrows the ODE search itself to just that + parameter (verified live 2026-07-21 -- see :func:`astrofetch.data.ode.query_products`). + Coverage is -80 to 80 latitude (cylindrical projection, not a bug). + ``TBOL`` is intentionally not offered here: it is not part of this + cumulative-mosaic family and mixing single-orbit epochs into a windowed + read would misrepresent the data. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "Diviner (rock abundance / regolith temperature)" + ihid = "LRO" + iid = "DLRE" + all_products = { + "rock_abundance": ODEAsset( + "lro_diviner_rock_abundance", + "GDR_L3", + r"DGDR_RA_CLC_CYL_20160913N_128_IMG\.LBL", + product_id="*ra_clc_cyl_20160913*", + ), + "regolith_temp": ODEAsset( + "lro_diviner_regolith_temp", + "GDR_L3", + r"DGDR_ST_CLC_CYL_20160913N_128_IMG\.LBL", + product_id="*st_clc_cyl_20160913*", + ), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 333ff8b..f4d536b 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -20,6 +20,7 @@ LOLA, LROCNACDTM, SLDEM2015, + DivinerGDR, KaguyaTC, KaguyaTCImagery, LROCWACMosaic, @@ -74,6 +75,10 @@ class LayerSpec: """Required ODE file role for ``pattern`` to match against (``source == "ode"``); almost always ``"Product"``.""" + product_id: str = "" + """ODE ``productid`` wildcard filter narrowing the search server-side + (``source == "ode"``); empty when the product type needs no narrowing.""" + href: str = "" """Fixed archive URL (``source == "mosaic"``).""" @@ -133,6 +138,7 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: pt=entry.pt, pattern=entry.pattern, file_type=entry.file_type, + product_id=entry.product_id or "", ) if isinstance(entry, MosaicAsset): return LayerSpec(**common, source="mosaic", href=entry.href) @@ -154,6 +160,8 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(MiniRF, "cpr"), _spec(MiniRF, "sc"), _spec(MiniRF, "oc"), + _spec(DivinerGDR, "rock_abundance"), + _spec(DivinerGDR, "regolith_temp"), ) } @@ -185,6 +193,7 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: "lola": _instrument(LOLA), "sldem2015": _instrument(SLDEM2015), "minirf": _instrument(MiniRF), + "diviner": _instrument(DivinerGDR), }, granules={ "nac_raw": LROCNACRaw, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json index 6197138..cfb3d1f 100644 --- a/tests/fixtures/ode/phase_c_listings.json +++ b/tests/fixtures/ode/phase_c_listings.json @@ -7,5 +7,14 @@ {"FileName": "GLOBAL_SC_128PPD_SIMP_0C.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_sc_128ppd_simp_0c.lbl"}, {"FileName": "GLOBAL_OC_128PPD_SIMP_0C.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/data/128ppd/global_oc_128ppd_simp_0c.lbl"}, {"FileName": "GLOBAL_CPR_SIMP_0C_BR.PNG", "Type": "Browse", "URL": "https://pds-geosciences.wustl.edu/lro/lro-l-mrflro-5-global-mosaic-v1/lromrf_1001/browse/global_cpr_simp_0c_br.png"} + ], + "LRO/DLRE/GDR_L3": [ + {"FileName": "DGDR_RA_CLC_CYL_20160913N_128_IMG.IMG", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2016/cylindrical/img/dgdr_ra_clc_cyl_20160913n_128_img.img"}, + {"FileName": "DGDR_RA_CLC_CYL_20160913N_128_IMG.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2016/cylindrical/img/dgdr_ra_clc_cyl_20160913n_128_img.lbl"}, + {"FileName": "DGDR_RA_CLC_CYL_20160913N_128_IMG.XML", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2016/cylindrical/img/dgdr_ra_clc_cyl_20160913n_128_img.xml"}, + {"FileName": "DGDR_RA_CLC_CYL_20090705N_128_IMG.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2009/cylindrical/img/dgdr_ra_clc_cyl_20090705n_128_img.lbl"}, + {"FileName": "DGDR_ST_CLC_CYL_20160913N_128_IMG.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2016/cylindrical/img/dgdr_st_clc_cyl_20160913n_128_img.lbl"}, + {"FileName": "DGDR_TBOL_CLC_CYL_20090705N_128_IMG.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2009/cylindrical/img/dgdr_tbol_clc_cyl_20090705n_128_img.lbl"}, + {"FileName": "DGDR_RA_CLC_CYL_20160913N_128_JPG.JPG", "Type": "Browse", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/browse/gdr_l3/2016/cylindrical/jpg/dgdr_ra_clc_cyl_20160913n_128_jpg.jpg"} ] } diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index 01ecf0d..779aa9c 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -113,6 +113,30 @@ def test_minirf_fetches_a_real_patch(tmp_path: Path) -> None: assert bool((valid >= 0.0).all()) and bool((valid <= 3.0).all()) +@pytest.mark.live +def test_diviner_fetches_a_real_patch(tmp_path: Path) -> None: + """Diviner rock abundance: the latest cumulative global mosaic, detached + PDS3 label with correctly declared nodata. Rock abundance coverage is + real but incomplete even in the latest cumulative mosaic (insufficient + nighttime passes in some regions -- verified live 2026-07-21 that + (23, 18, 25, 20) is a genuine gap), so this uses a bbox confirmed to + have data rather than asserting coverage anywhere.""" + moondata = af.DivinerGDR( + products=["rock_abundance"], + bbox=(60.0, 55.0, 62.0, 57.0), + resolution=1000.0, + patch_size=16, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 16, 16) + assert bool(sample["mask"].any()) + valid = sample["image"][sample["mask"]] + assert bool((valid >= 0.0).all()) and bool((valid <= 1.0).all()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index 812b010..afd7798 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -61,13 +61,20 @@ def _fake_find_file_urls( bbox: tuple, max_products: int = 20, file_type: str | None = "Product", + product_id: str | None = None, root: str | None = None, ) -> list[str]: return [f"{ihid}|{iid}|{pt}"] def _fake_query_products( - ihid: str, iid: str, pt: str, bbox: tuple, max_products: int = 20, root: str | None = None + ihid: str, + iid: str, + pt: str, + bbox: tuple, + max_products: int = 20, + product_id: str | None = None, + root: str | None = None, ) -> list: # No footprints by default: exercises the uniform-sampling fallback unless # a test overrides this to supply real footprints. @@ -304,7 +311,17 @@ def test_ode_instrument_yields_sample_dicts() -> None: def test_ode_read_queries_ode_by_ihid_iid_pt(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[tuple[str, str, str]] = [] - def _spy(ihid, iid, pt, pattern, bbox, max_products=20, file_type="Product", root=None): + def _spy( + ihid, + iid, + pt, + pattern, + bbox, + max_products=20, + file_type="Product", + product_id=None, + root=None, + ): calls.append((ihid, iid, pt)) return [f"{ihid}|{iid}|{pt}"] @@ -326,7 +343,17 @@ def test_ode_asset_file_type_defaults_to_product() -> None: def test_ode_read_passes_asset_file_type(monkeypatch: pytest.MonkeyPatch) -> None: seen: list[str] = [] - def _spy(ihid, iid, pt, pattern, bbox, max_products=20, file_type="Product", root=None): + def _spy( + ihid, + iid, + pt, + pattern, + bbox, + max_products=20, + file_type="Product", + product_id=None, + root=None, + ): seen.append(file_type) return [f"{ihid}|{iid}|{pt}"] @@ -529,3 +556,28 @@ def test_catalog_includes_minirf() -> None: assert spec.ihid == "LRO" assert spec.iid == "MRFLRO" assert spec.pt == "MOSDDR" + + +def test_diviner_patterns_pin_the_latest_cumulative_date() -> None: + _assert_pattern_selects( + af.DivinerGDR.all_products["rock_abundance"], + "LRO/DLRE/GDR_L3", + "20160913n_128_img.lbl", + ) + _assert_pattern_selects( + af.DivinerGDR.all_products["regolith_temp"], + "LRO/DLRE/GDR_L3", + "20160913n_128_img.lbl", + ) + + +def test_diviner_does_not_offer_bolometric_temperature() -> None: + assert "tbol" not in af.DivinerGDR.all_products + assert set(af.DivinerGDR.all_products) == {"rock_abundance", "regolith_temp"} + + +def test_catalog_includes_diviner() -> None: + assert MOON.probes["lro"].instruments["diviner"].dataset is af.DivinerGDR + spec = LAYERS["lro_diviner_rock_abundance"] + assert spec.source == "ode" + assert spec.pt == "GDR_L3" diff --git a/tests/unit/test_ode.py b/tests/unit/test_ode.py index d0bdcde..3073525 100644 --- a/tests/unit/test_ode.py +++ b/tests/unit/test_ode.py @@ -138,6 +138,27 @@ def test_longitude_converted_to_0_360(monkeypatch: pytest.MonkeyPatch) -> None: assert session.calls[0]["easternlon"] == pytest.approx(334.5) +def test_product_id_filter_is_sent_only_when_given(monkeypatch: pytest.MonkeyPatch) -> None: + session = _Session([_EMPTY_BODY]) + _patch_session(monkeypatch, session) + ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0)) + assert "productid" not in session.calls[0] + + session = _Session([_EMPTY_BODY]) + _patch_session(monkeypatch, session) + ode.query_products("LRO", "LROC", "SDNDTM", (0.0, 0.0, 1.0, 1.0), product_id="*wac_gld100*") + assert session.calls[0]["productid"] == "*wac_gld100*" + + +def test_find_file_urls_forwards_product_id(monkeypatch: pytest.MonkeyPatch) -> None: + session = _Session([_EMPTY_BODY]) + _patch_session(monkeypatch, session) + ode.find_file_urls( + "LRO", "LROC", "SDNDTM", r".+", (0.0, 0.0, 1.0, 1.0), product_id="*wac_gld100*" + ) + assert session.calls[0]["productid"] == "*wac_gld100*" + + def test_full_moon_bbox_does_not_collapse_to_zero_width(monkeypatch: pytest.MonkeyPatch) -> None: # A naive `lon % 360` on each bound independently sends (-180, 180) to # (180, 180): a zero-width query that would silently return nothing. From 28ee07689ddfbab321f494eed35203ad7ab07c05 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:42:11 +0200 Subject: [PATCH 17/22] feat(moon): add WACGLD100 and WACTiO2 Search PDS ODE (product types SDWDTM and SDWTIO) for the LROC WAC GLD100 global 100 m DTM and the WAC TiO2 abundance map. GLD100's product type is dominated by WAC_CSHADE, a rendered shaded-relief product (excluded per rule 3) that would otherwise swamp a bbox-only search, so it uses the same productid narrowing added for DivinerGDR. Both open their .IMG data file directly rather than the PDS4 .xml label, following the LROCNACDTM precedent for this im-ldi archive. --- src/astrofetch/__init__.py | 4 +++ src/astrofetch/moon/__init__.py | 4 +++ src/astrofetch/moon/datasets.py | 45 ++++++++++++++++++++++++ src/astrofetch/moon/layers.py | 6 ++++ tests/fixtures/ode/phase_c_listings.json | 15 ++++++++ tests/live/test_pds_ode_live.py | 35 ++++++++++++++++++ tests/unit/test_datasets.py | 18 ++++++++++ 7 files changed, 127 insertions(+) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index f5a4dfb..008b44d 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -12,6 +12,7 @@ M3, MOON, SLDEM2015, + WACGLD100, DivinerGDR, IntersectionDataset, KaguyaTC, @@ -20,6 +21,7 @@ LROCWACMosaic, LROCWACRaw, MiniRF, + WACTiO2, ) __version__ = "0.1.0" @@ -38,6 +40,8 @@ "LROCWACRaw", "MiniRF", "SLDEM2015", + "WACGLD100", + "WACTiO2", "moon", "__version__", ] diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index 1dfc09d..dacb5dd 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -4,6 +4,7 @@ LOLA, LROCNACDTM, SLDEM2015, + WACGLD100, DivinerGDR, InstrumentDataset, IntersectionDataset, @@ -13,6 +14,7 @@ MiniRF, MosaicDataset, ODEInstrumentDataset, + WACTiO2, ) from astrofetch.moon.granules import M3, GranuleDataset, LROCNACRaw, LROCWACRaw from astrofetch.moon.layers import LAYERS, MOON, Body, Instrument, LayerSpec, Probe @@ -40,4 +42,6 @@ "ODEInstrumentDataset", "Probe", "SLDEM2015", + "WACGLD100", + "WACTiO2", ] diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index dce60e6..10489ba 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -641,6 +641,51 @@ class DivinerGDR(ODEInstrumentDataset): } +class WACGLD100(ODEInstrumentDataset): + """LRO LROC WAC GLD100 global DTM, 100 m/px, searched via PDS ODE + (product type ``SDWDTM``): 8 near-global quadrant tiles plus 2 polar + caps, pinned to their 100 m native resolution (coarser 128/256 px-per- + degree copies of the same tiles, and one separate whole-globe file at + even coarser multi-resolution, both exist under the same product type + and are excluded by the pattern). The same product type is dominated by + ``WAC_CSHADE`` shaded-relief products (a rendered visualization, AGENTS + rule 3, and far more numerous than GLD100 itself), so ``product_id`` + narrows the ODE search server-side rather than relying on the filename + pattern alone (verified live 2026-07-21). im-ldi PDS4 archive: opens the + data ``.IMG`` file directly, never its ``.xml`` label (rule 1's + PDS4-label lesson). + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC WAC GLD100 (global DTM)" + ihid = "LRO" + iid = "LROC" + all_products = { + "dtm": ODEAsset( + "lroc_wac_gld100_dtm", + "SDWDTM", + r"WAC_GLD100_.+_100M\.IMG", + product_id="*wac_gld100*", + ), + } + + +class WACTiO2(ODEInstrumentDataset): + """LRO LROC WAC TiO2 abundance map, searched via PDS ODE (product type + ``SDWTIO``), weight-percent TiO2 in the regolith derived from WAC + multispectral photometry. im-ldi PDS4 archive: opens the data ``.IMG`` + file directly, never its ``.xml`` label (rule 1's PDS4-label lesson). + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC WAC TiO2 abundance" + ihid = "LRO" + iid = "LROC" + all_products = { + "tio2": ODEAsset("lroc_wac_tio2", "SDWTIO", r"WAC_TIO2_.+\.IMG"), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index f4d536b..6d51484 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -20,6 +20,7 @@ LOLA, LROCNACDTM, SLDEM2015, + WACGLD100, DivinerGDR, KaguyaTC, KaguyaTCImagery, @@ -28,6 +29,7 @@ MosaicAsset, ODEAsset, Product, + WACTiO2, _ProductDataset, ) from astrofetch.moon.granules import M3, GranuleDataset, LROCNACRaw, LROCWACRaw @@ -162,6 +164,8 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(MiniRF, "oc"), _spec(DivinerGDR, "rock_abundance"), _spec(DivinerGDR, "regolith_temp"), + _spec(WACGLD100, "dtm"), + _spec(WACTiO2, "tio2"), ) } @@ -194,6 +198,8 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: "sldem2015": _instrument(SLDEM2015), "minirf": _instrument(MiniRF), "diviner": _instrument(DivinerGDR), + "wac_gld100": _instrument(WACGLD100), + "wac_tio2": _instrument(WACTiO2), }, granules={ "nac_raw": LROCNACRaw, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json index cfb3d1f..d53ba7b 100644 --- a/tests/fixtures/ode/phase_c_listings.json +++ b/tests/fixtures/ode/phase_c_listings.json @@ -16,5 +16,20 @@ {"FileName": "DGDR_ST_CLC_CYL_20160913N_128_IMG.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2016/cylindrical/img/dgdr_st_clc_cyl_20160913n_128_img.lbl"}, {"FileName": "DGDR_TBOL_CLC_CYL_20090705N_128_IMG.LBL", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/data_derived_gdr_l3/2009/cylindrical/img/dgdr_tbol_clc_cyl_20090705n_128_img.lbl"}, {"FileName": "DGDR_RA_CLC_CYL_20160913N_128_JPG.JPG", "Type": "Browse", "URL": "https://pds-geosciences.wustl.edu/lro/urn-nasa-pds-lro_diviner_derived1/browse/gdr_l3/2016/cylindrical/jpg/dgdr_ra_clc_cyl_20160913n_128_jpg.jpg"} + ], + "LRO/LROC/SDWDTM": [ + {"FileName": "WAC_GLD100_E300N0450_100M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_GLD100/WAC_GLD100_E300N0450_100M.IMG"}, + {"FileName": "WAC_GLD100_E300N0450_128P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_GLD100/WAC_GLD100_E300N0450_128P.IMG"}, + {"FileName": "WAC_GLD100_E300N0450_256P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_GLD100/WAC_GLD100_E300N0450_256P.IMG"}, + {"FileName": "WAC_GLD100_P900N0000_100M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_GLD100/WAC_GLD100_P900N0000_100M.IMG"}, + {"FileName": "WAC_GLD100_E000N1800_004P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_GLD100/WAC_GLD100_E000N1800_004P.IMG"}, + {"FileName": "WAC_GLD100_E300N0450_100M.XML", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_GLD100/WAC_GLD100_E300N0450_100M.xml"}, + {"FileName": "WAC_CSHADE_E000N1800_004P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_CSHADE/WAC_CSHADE_E000N1800_004P.IMG"} + ], + "LRO/LROC/SDWTIO": [ + {"FileName": "WAC_TIO2_E350N0450.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_TIO2/WAC_TIO2_E350N0450.IMG"}, + {"FileName": "WAC_TIO2_E350N0450.XML", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_TIO2/WAC_TIO2_E350N0450.xml"}, + {"FileName": "WAC_TIO2_E350N0450.MASK.TIF", "Type": "Browse", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/EXTRAS/BROWSE/WAC_TIO2/WAC_TIO2_E350N0450.MASK.TIF"}, + {"FileName": "WAC_TIO2_E350N0450.TIF", "Type": "Browse", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/EXTRAS/BROWSE/WAC_TIO2/WAC_TIO2_E350N0450.TIF"} ] } diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index 779aa9c..006f740 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -137,6 +137,41 @@ def test_diviner_fetches_a_real_patch(tmp_path: Path) -> None: assert bool((valid >= 0.0).all()) and bool((valid <= 1.0).all()) +@pytest.mark.live +def test_wac_gld100_fetches_a_real_patch(tmp_path: Path) -> None: + """WAC GLD100: 100 m tiled DTM, searched and opened via its .IMG data + file (never the PDS4 .xml label -- rule 1).""" + moondata = af.WACGLD100( + bbox=(23.0, 18.0, 25.0, 20.0), + resolution=100.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + valid = sample["image"][sample["mask"]] + assert bool((valid >= -9500.0).all()) and bool((valid <= 10800.0).all()) + + +@pytest.mark.live +def test_wac_tio2_fetches_a_real_patch(tmp_path: Path) -> None: + """WAC TiO2 abundance map over a mare region (TiO2-rich basalt).""" + moondata = af.WACTiO2( + bbox=(23.0, 18.0, 25.0, 20.0), + resolution=500.0, + patch_size=16, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 16, 16) + assert bool(sample["mask"].any()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index afd7798..e633e7e 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -581,3 +581,21 @@ def test_catalog_includes_diviner() -> None: spec = LAYERS["lro_diviner_rock_abundance"] assert spec.source == "ode" assert spec.pt == "GDR_L3" + + +def test_wac_gld100_pattern_selects_only_the_100m_family() -> None: + files = _phase_c_files("LRO/LROC/SDWDTM") + spec = af.WACGLD100.all_products["dtm"] + urls = ode.match_files(files, spec.pattern, spec.file_type) + assert len(urls) == 2 + assert all(u.upper().endswith("_100M.IMG") for u in urls) + assert not any("CSHADE" in u.upper() for u in urls) + + +def test_wac_tio2_pattern_selects_the_data_file() -> None: + _assert_pattern_selects(af.WACTiO2.all_products["tio2"], "LRO/LROC/SDWTIO", "E350N0450.IMG") + + +def test_catalog_includes_wac_gld100_and_tio2() -> None: + assert MOON.probes["lro"].instruments["wac_gld100"].dataset is af.WACGLD100 + assert MOON.probes["lro"].instruments["wac_tio2"].dataset is af.WACTiO2 From e6e187e69f3cee95c96707c9db16a955026e3375 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:44:46 +0200 Subject: [PATCH 18/22] feat(moon): add LROCWACGlobal and LROCWACColor Search PDS ODE for two more LROC WAC products: the global morphology mosaic as 8 searched E-family quadrant tiles (product type BDRWGL, the tiled sibling of the fixed-URL LROCWACMosaic -- the same type also carries a whole-globe O-prefixed family, not offered here), and the empirically-normalized 7-color reflectance (product type MDREMP), one product per band pinned to its 64 px/degree tiling. Both open their .IMG data file directly, following the LROCNACDTM precedent for the im-ldi archive. --- src/astrofetch/__init__.py | 4 ++ src/astrofetch/moon/__init__.py | 4 ++ src/astrofetch/moon/datasets.py | 52 ++++++++++++++++++++++++ src/astrofetch/moon/layers.py | 12 ++++++ tests/fixtures/ode/phase_c_listings.json | 13 ++++++ tests/live/test_pds_ode_live.py | 36 ++++++++++++++++ tests/unit/test_datasets.py | 25 ++++++++++++ 7 files changed, 146 insertions(+) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index 008b44d..1ac6a07 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -18,6 +18,8 @@ KaguyaTC, KaguyaTCImagery, LROCNACRaw, + LROCWACColor, + LROCWACGlobal, LROCWACMosaic, LROCWACRaw, MiniRF, @@ -36,6 +38,8 @@ "KaguyaTCImagery", "LROCNACDTM", "LROCNACRaw", + "LROCWACColor", + "LROCWACGlobal", "LROCWACMosaic", "LROCWACRaw", "MiniRF", diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index dacb5dd..e9cf901 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -10,6 +10,8 @@ IntersectionDataset, KaguyaTC, KaguyaTCImagery, + LROCWACColor, + LROCWACGlobal, LROCWACMosaic, MiniRF, MosaicDataset, @@ -34,6 +36,8 @@ "KaguyaTCImagery", "LROCNACDTM", "LROCNACRaw", + "LROCWACColor", + "LROCWACGlobal", "LROCWACMosaic", "LROCWACRaw", "LayerSpec", diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index 10489ba..45060da 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -686,6 +686,58 @@ class WACTiO2(ODEInstrumentDataset): } +class LROCWACGlobal(ODEInstrumentDataset): + """LRO LROC WAC global morphology mosaic, 100 m/px, searched via PDS ODE + (product type ``BDRWGL``) as 8 near-global quadrant tiles rather than one + monolithic file -- the searched sibling of the fixed-URL + :class:`LROCWACMosaic`. Prefer this when only part of the globe is + needed (fetches one small tile instead of opening the ~2 GB monolith); + prefer :class:`LROCWACMosaic` for dense sampling over a wide area, since + its cache reuses one already-open source across reads. The same product + type also carries a whole-globe ``O``-prefixed file family at each + resolution (an alternative to :class:`LROCWACMosaic`, not offered here); + the pattern selects only the tiled ``E``-prefixed family (verified live + 2026-07-21). + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC WAC global mosaic (tiled)" + ihid = "LRO" + iid = "LROC" + all_products = { + "morphology": ODEAsset( + "lroc_wac_global_tiled", "BDRWGL", r"WAC_GLOBAL_E\d{3}[NS]\d{4}_100M\.IMG" + ), + } + + +class LROCWACColor(ODEInstrumentDataset): + """LRO LROC WAC empirically-normalized 7-color reflectance, searched via + PDS ODE (product type ``MDREMP``): one product per band (321, 360, 415, + 566, 604, 643, and 689 nm), each its own tiled family pinned to its + 64 px/degree tiling (other resolutions of the 643 nm band also exist + under the same product type; the pattern excludes them). The composite + ``3BAND`` product and the Hapke-photometrically-normalized ``MDRHAP`` + variant are not offered here. im-ldi PDS4 archive: opens the data + ``.IMG`` file directly, never its ``.xml`` label (rule 1's PDS4-label + lesson). + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC WAC 7-color reflectance" + ihid = "LRO" + iid = "LROC" + all_products = { + "refl_321nm": ODEAsset("lroc_wac_refl_321nm", "MDREMP", r"WAC_EMP_321NM_.+_064P\.IMG"), + "refl_360nm": ODEAsset("lroc_wac_refl_360nm", "MDREMP", r"WAC_EMP_360NM_.+_064P\.IMG"), + "refl_415nm": ODEAsset("lroc_wac_refl_415nm", "MDREMP", r"WAC_EMP_415NM_.+_064P\.IMG"), + "refl_566nm": ODEAsset("lroc_wac_refl_566nm", "MDREMP", r"WAC_EMP_566NM_.+_064P\.IMG"), + "refl_604nm": ODEAsset("lroc_wac_refl_604nm", "MDREMP", r"WAC_EMP_604NM_.+_064P\.IMG"), + "refl_643nm": ODEAsset("lroc_wac_refl_643nm", "MDREMP", r"WAC_EMP_643NM_.+_064P\.IMG"), + "refl_689nm": ODEAsset("lroc_wac_refl_689nm", "MDREMP", r"WAC_EMP_689NM_.+_064P\.IMG"), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 6d51484..1cb27d0 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -24,6 +24,8 @@ DivinerGDR, KaguyaTC, KaguyaTCImagery, + LROCWACColor, + LROCWACGlobal, LROCWACMosaic, MiniRF, MosaicAsset, @@ -166,6 +168,14 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(DivinerGDR, "regolith_temp"), _spec(WACGLD100, "dtm"), _spec(WACTiO2, "tio2"), + _spec(LROCWACGlobal, "morphology"), + _spec(LROCWACColor, "refl_321nm"), + _spec(LROCWACColor, "refl_360nm"), + _spec(LROCWACColor, "refl_415nm"), + _spec(LROCWACColor, "refl_566nm"), + _spec(LROCWACColor, "refl_604nm"), + _spec(LROCWACColor, "refl_643nm"), + _spec(LROCWACColor, "refl_689nm"), ) } @@ -200,6 +210,8 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: "diviner": _instrument(DivinerGDR), "wac_gld100": _instrument(WACGLD100), "wac_tio2": _instrument(WACTiO2), + "wac_global_tiled": _instrument(LROCWACGlobal), + "wac_color": _instrument(LROCWACColor), }, granules={ "nac_raw": LROCNACRaw, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json index d53ba7b..20731b2 100644 --- a/tests/fixtures/ode/phase_c_listings.json +++ b/tests/fixtures/ode/phase_c_listings.json @@ -31,5 +31,18 @@ {"FileName": "WAC_TIO2_E350N0450.XML", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/SDP/WAC_TIO2/WAC_TIO2_E350N0450.xml"}, {"FileName": "WAC_TIO2_E350N0450.MASK.TIF", "Type": "Browse", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/EXTRAS/BROWSE/WAC_TIO2/WAC_TIO2_E350N0450.MASK.TIF"}, {"FileName": "WAC_TIO2_E350N0450.TIF", "Type": "Browse", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/EXTRAS/BROWSE/WAC_TIO2/WAC_TIO2_E350N0450.TIF"} + ], + "LRO/LROC/BDRWGL": [ + {"FileName": "WAC_GLOBAL_E300N0450_100M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_GLOBAL/WAC_GLOBAL_E300N0450_100M.IMG"}, + {"FileName": "WAC_GLOBAL_E300N0450_256P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_GLOBAL/WAC_GLOBAL_E300N0450_256P.IMG"}, + {"FileName": "WAC_GLOBAL_O000N0000_100M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_GLOBAL/WAC_GLOBAL_O000N0000_100M.IMG"}, + {"FileName": "WAC_GLOBAL_E000N0000_004P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_GLOBAL/WAC_GLOBAL_E000N0000_004P.IMG"} + ], + "LRO/LROC/MDREMP": [ + {"FileName": "WAC_EMP_321NM_E300N0450_064P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_321NM_E300N0450_064P.IMG"}, + {"FileName": "WAC_EMP_360NM_E300N0450_064P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_360NM_E300N0450_064P.IMG"}, + {"FileName": "WAC_EMP_643NM_E300N0450_064P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_643NM_E300N0450_064P.IMG"}, + {"FileName": "WAC_EMP_643NM_E300N0450_100M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_643NM_E300N0450_100M.IMG"}, + {"FileName": "WAC_EMP_3BAND_E300N0450_064P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_3BAND_E300N0450_064P.IMG"} ] } diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index 006f740..b5341ca 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -172,6 +172,42 @@ def test_wac_tio2_fetches_a_real_patch(tmp_path: Path) -> None: assert bool(sample["mask"].any()) +@pytest.mark.live +def test_wac_global_tiled_fetches_a_real_patch(tmp_path: Path) -> None: + """LROC WAC global mosaic, tiled/searched sibling of LROCWACMosaic: one + E-family quadrant tile, opened via its .IMG data file.""" + moondata = af.LROCWACGlobal( + bbox=(23.0, 18.0, 25.0, 20.0), + resolution=100.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + + +@pytest.mark.live +def test_wac_color_fetches_a_real_patch(tmp_path: Path) -> None: + """LROC WAC 7-color reflectance, 643 nm band.""" + moondata = af.LROCWACColor( + products=["refl_643nm"], + bbox=(23.0, 18.0, 25.0, 20.0), + resolution=500.0, + patch_size=16, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 16, 16) + assert bool(sample["mask"].any()) + valid = sample["image"][sample["mask"]] + assert bool((valid >= 0.0).all()) and bool((valid <= 1.0).all()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index e633e7e..1a1eb92 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -599,3 +599,28 @@ def test_wac_tio2_pattern_selects_the_data_file() -> None: def test_catalog_includes_wac_gld100_and_tio2() -> None: assert MOON.probes["lro"].instruments["wac_gld100"].dataset is af.WACGLD100 assert MOON.probes["lro"].instruments["wac_tio2"].dataset is af.WACTiO2 + + +def test_wac_global_pattern_selects_only_the_e_family_tile() -> None: + _assert_pattern_selects( + af.LROCWACGlobal.all_products["morphology"], "LRO/LROC/BDRWGL", "E300N0450_100M.IMG" + ) + + +def test_wac_color_patterns_select_the_right_band_and_resolution() -> None: + _assert_pattern_selects( + af.LROCWACColor.all_products["refl_321nm"], "LRO/LROC/MDREMP", "321NM_E300N0450_064P.IMG" + ) + _assert_pattern_selects( + af.LROCWACColor.all_products["refl_643nm"], "LRO/LROC/MDREMP", "643NM_E300N0450_064P.IMG" + ) + + +def test_wac_color_does_not_offer_the_3band_composite() -> None: + assert "3band" not in af.LROCWACColor.all_products + assert len(af.LROCWACColor.all_products) == 7 + + +def test_catalog_includes_wac_global_and_color() -> None: + assert MOON.probes["lro"].instruments["wac_global_tiled"].dataset is af.LROCWACGlobal + assert MOON.probes["lro"].instruments["wac_color"].dataset is af.LROCWACColor From fed75fdcd06f014d51e17e051ba98a4e779935f2 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:48:25 +0200 Subject: [PATCH 19/22] feat(moon): add LROCNACROI Search PDS ODE (product type BDRROI) for LROC NAC region-of-interest mosaics: named sites such as craters and poles, footprint-sampled like LROCNACDTM since coverage is a few hundred sites, not the whole Moon. Native-resolution mosaics can reach ~14 GB, far too large for windowed reads, so only the downsampled 5 m and 20 m products are offered. The same product type also carries unrelated WAC_ROI mosaics from the WAC camera; productid narrows the ODE search to the NAC family. --- src/astrofetch/__init__.py | 2 ++ src/astrofetch/moon/__init__.py | 2 ++ src/astrofetch/moon/datasets.py | 28 ++++++++++++++++++++++++ src/astrofetch/moon/layers.py | 4 ++++ tests/fixtures/ode/phase_c_listings.json | 6 +++++ tests/live/test_pds_ode_live.py | 16 ++++++++++++++ tests/unit/test_datasets.py | 27 +++++++++++++++++++++++ 7 files changed, 85 insertions(+) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index 1ac6a07..61909bf 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -9,6 +9,7 @@ from astrofetch.moon import ( LOLA, LROCNACDTM, + LROCNACROI, M3, MOON, SLDEM2015, @@ -37,6 +38,7 @@ "KaguyaTC", "KaguyaTCImagery", "LROCNACDTM", + "LROCNACROI", "LROCNACRaw", "LROCWACColor", "LROCWACGlobal", diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index e9cf901..111c6de 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -3,6 +3,7 @@ from astrofetch.moon.datasets import ( LOLA, LROCNACDTM, + LROCNACROI, SLDEM2015, WACGLD100, DivinerGDR, @@ -35,6 +36,7 @@ "KaguyaTC", "KaguyaTCImagery", "LROCNACDTM", + "LROCNACROI", "LROCNACRaw", "LROCWACColor", "LROCWACGlobal", diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index 45060da..848197a 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -738,6 +738,34 @@ class LROCWACColor(ODEInstrumentDataset): } +class LROCNACROI(ODEInstrumentDataset): + """LRO LROC NAC region-of-interest mosaics: named sites (craters, poles, + other features), searched via PDS ODE (product type ``BDRROI``). + ``footprint_sampling`` is on by default, matching :class:`LROCNACDTM`: + coverage is a few hundred named sites, not the whole Moon. + + Each site's native-resolution mosaic can reach ~14 GB (verified live + 2026-07-21), unreasonable for windowed reads, so only the downsampled + 5 m and 20 m products are offered. The same product type also carries + unrelated ``WAC_ROI`` mosaics from the WAC camera; ``product_id`` + narrows the ODE search to the NAC family. + """ + + probe = "Lunar Reconnaissance Orbiter" + instrument = "LROC NAC region-of-interest mosaics" + ihid = "LRO" + iid = "LROC" + footprint_sampling = True + all_products = { + "mosaic_5m": ODEAsset( + "lroc_nac_roi_5m", "BDRROI", r"NAC_ROI_.+_5M\.IMG", product_id="*nac_roi*" + ), + "mosaic_20m": ODEAsset( + "lroc_nac_roi_20m", "BDRROI", r"NAC_ROI_.+_20M\.IMG", product_id="*nac_roi*" + ), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 1cb27d0..214d91f 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -19,6 +19,7 @@ CRS, LOLA, LROCNACDTM, + LROCNACROI, SLDEM2015, WACGLD100, DivinerGDR, @@ -176,6 +177,8 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(LROCWACColor, "refl_604nm"), _spec(LROCWACColor, "refl_643nm"), _spec(LROCWACColor, "refl_689nm"), + _spec(LROCNACROI, "mosaic_5m"), + _spec(LROCNACROI, "mosaic_20m"), ) } @@ -212,6 +215,7 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: "wac_tio2": _instrument(WACTiO2), "wac_global_tiled": _instrument(LROCWACGlobal), "wac_color": _instrument(LROCWACColor), + "nac_roi": _instrument(LROCNACROI), }, granules={ "nac_raw": LROCNACRaw, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json index 20731b2..3ac5499 100644 --- a/tests/fixtures/ode/phase_c_listings.json +++ b/tests/fixtures/ode/phase_c_listings.json @@ -44,5 +44,11 @@ {"FileName": "WAC_EMP_643NM_E300N0450_064P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_643NM_E300N0450_064P.IMG"}, {"FileName": "WAC_EMP_643NM_E300N0450_100M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_643NM_E300N0450_100M.IMG"}, {"FileName": "WAC_EMP_3BAND_E300N0450_064P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/MDR/WAC_EMP/WAC_EMP_3BAND_E300N0450_064P.IMG"} + ], + "LRO/LROC/BDRROI": [ + {"FileName": "NAC_ROI_AITKNCTRHIA_E168S1734.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/NAC_ROI/NAC_ROI_AITKNCTRHIA_E168S1734.IMG"}, + {"FileName": "NAC_ROI_AITKNCTRHIA_E168S1734_5M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/NAC_ROI/NAC_ROI_AITKNCTRHIA_E168S1734_5M.IMG"}, + {"FileName": "NAC_ROI_AITKNCTRHIA_E168S1734_20M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/NAC_ROI/NAC_ROI_AITKNCTRHIA_E168S1734_20M.IMG"}, + {"FileName": "WAC_ROI_FARSIDE_DUSK_E000N1800_004P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/WAC_ROI_FARSIDE_DUSK/WAC_ROI_FARSIDE_DUSK_E000N1800_004P.IMG"} ] } diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index b5341ca..ea093e2 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -208,6 +208,22 @@ def test_wac_color_fetches_a_real_patch(tmp_path: Path) -> None: assert bool((valid >= 0.0).all()) and bool((valid <= 1.0).all()) +@pytest.mark.live +def test_nac_roi_fetches_a_real_patch(tmp_path: Path) -> None: + """NAC ROI: footprint-constrained sampling over named sites, 5 m mosaic.""" + moondata = af.LROCNACROI( + products=["mosaic_5m"], + resolution=5.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index 1a1eb92..d27adc0 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -624,3 +624,30 @@ def test_wac_color_does_not_offer_the_3band_composite() -> None: def test_catalog_includes_wac_global_and_color() -> None: assert MOON.probes["lro"].instruments["wac_global_tiled"].dataset is af.LROCWACGlobal assert MOON.probes["lro"].instruments["wac_color"].dataset is af.LROCWACColor + + +def test_nac_roi_patterns_select_nac_not_wac_and_the_right_resolution() -> None: + _assert_pattern_selects( + af.LROCNACROI.all_products["mosaic_5m"], + "LRO/LROC/BDRROI", + "AITKNCTRHIA_E168S1734_5M.IMG", + ) + _assert_pattern_selects( + af.LROCNACROI.all_products["mosaic_20m"], + "LRO/LROC/BDRROI", + "AITKNCTRHIA_E168S1734_20M.IMG", + ) + + +def test_nac_roi_does_not_offer_the_native_resolution() -> None: + # Native-resolution NAC ROI mosaics can reach ~14 GB; too large for + # windowed reads (verified live 2026-07-21). + assert set(af.LROCNACROI.all_products) == {"mosaic_5m", "mosaic_20m"} + + +def test_nac_roi_uses_footprint_sampling() -> None: + assert af.LROCNACROI.footprint_sampling is True + + +def test_catalog_includes_nac_roi() -> None: + assert MOON.probes["lro"].instruments["nac_roi"].dataset is af.LROCNACROI From 6b84c3dbd88b05cad46275bef68ce85c6e4c6439 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:52:30 +0200 Subject: [PATCH 20/22] feat(moon): add ShadowCam and the kplo probe Search PDS ODE (product types CMOS and DTM) for KPLO ShadowCam controlled mosaics and stereo DTMs of permanently shadowed polar regions. Genuine Cloud Optimized GeoTIFFs, no PDS4-label quirk to work around. footprint_sampling is on, matching the other named-site instruments: coverage is a handful of PSR sites, not the whole Moon. The DTM product type also carries rendered slope/shaded-relief/color visualizations (excluded per rule 3); confidence is the one non-elevation product offered and is typed "Referenced" rather than "Product" in ODE (verified live 2026-07-21), exercising the file_type override added earlier in this phase. New kplo probe in the catalog, since this is the first Korea Pathfinder Lunar Orbiter dataset. --- src/astrofetch/__init__.py | 2 ++ src/astrofetch/moon/__init__.py | 2 ++ src/astrofetch/moon/datasets.py | 35 ++++++++++++++++++++++++ src/astrofetch/moon/layers.py | 10 +++++++ tests/fixtures/ode/phase_c_listings.json | 12 ++++++++ tests/live/test_pds_ode_live.py | 16 +++++++++++ tests/unit/test_datasets.py | 32 ++++++++++++++++++++++ 7 files changed, 109 insertions(+) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index 61909bf..956ff8f 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -24,6 +24,7 @@ LROCWACMosaic, LROCWACRaw, MiniRF, + ShadowCam, WACTiO2, ) @@ -46,6 +47,7 @@ "LROCWACRaw", "MiniRF", "SLDEM2015", + "ShadowCam", "WACGLD100", "WACTiO2", "moon", diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index 111c6de..80d5875 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -17,6 +17,7 @@ MiniRF, MosaicDataset, ODEInstrumentDataset, + ShadowCam, WACTiO2, ) from astrofetch.moon.granules import M3, GranuleDataset, LROCNACRaw, LROCWACRaw @@ -48,6 +49,7 @@ "ODEInstrumentDataset", "Probe", "SLDEM2015", + "ShadowCam", "WACGLD100", "WACTiO2", ] diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index 848197a..ed033c0 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -766,6 +766,41 @@ class LROCNACROI(ODEInstrumentDataset): } +class ShadowCam(ODEInstrumentDataset): + """KPLO ShadowCam controlled mosaics and stereo DTMs of permanently + shadowed regions near the lunar poles, searched via PDS ODE (product + types ``CMOS`` and ``DTM``). ``footprint_sampling`` is on by default: + coverage is a handful of named PSR (permanently shadowed region) sites, + not the whole Moon. + + Genuine Cloud Optimized GeoTIFFs (unlike the LROC im-ldi archive, no + PDS4-label quirk to work around). The DTM product type also carries + rendered slope/shaded-relief/color visualizations (excluded per rule 3); + ``confidence`` is the one non-elevation product offered, and is typed + ``"Referenced"`` rather than ``"Product"`` in ODE (verified live + 2026-07-21). Sites are near-polar, so windows reprojected onto the + geographic target grid see the same equirectangular-near-the-pole + distortion as any other source there today (see :mod:`astrofetch.data.grid`); + a dedicated polar target grid remains future work. + """ + + probe = "Korea Pathfinder Lunar Orbiter" + instrument = "ShadowCam (PSR mosaics and DTMs)" + ihid = "KPLO" + iid = "ShadowCam" + footprint_sampling = True + all_products = { + "mosaic": ODEAsset("shadowcam_mosaic", "CMOS", r"SHADOWCAM_CMOSAIC_.+_COG\.TIF"), + "dtm": ODEAsset("shadowcam_dtm", "DTM", r"SHADOWCAM_DTM_.+_DTM_\d+M_COG\.TIF"), + "confidence": ODEAsset( + "shadowcam_confidence", + "DTM", + r"SHADOWCAM_DTM_.+_CONFIDENCE_COG\.TIF", + file_type="Referenced", + ), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index 214d91f..d9a5e5f 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -32,6 +32,7 @@ MosaicAsset, ODEAsset, Product, + ShadowCam, WACTiO2, _ProductDataset, ) @@ -179,6 +180,9 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(LROCWACColor, "refl_689nm"), _spec(LROCNACROI, "mosaic_5m"), _spec(LROCNACROI, "mosaic_20m"), + _spec(ShadowCam, "mosaic"), + _spec(ShadowCam, "dtm"), + _spec(ShadowCam, "confidence"), ) } @@ -227,6 +231,12 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: instruments={}, granules={"m3": M3}, ), + "kplo": Probe( + name=ShadowCam.probe, + instruments={ + "shadowcam": _instrument(ShadowCam), + }, + ), }, ) """Discovery catalog for the Moon: enumerate probes, instruments, products, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json index 3ac5499..8d3537d 100644 --- a/tests/fixtures/ode/phase_c_listings.json +++ b/tests/fixtures/ode/phase_c_listings.json @@ -50,5 +50,17 @@ {"FileName": "NAC_ROI_AITKNCTRHIA_E168S1734_5M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/NAC_ROI/NAC_ROI_AITKNCTRHIA_E168S1734_5M.IMG"}, {"FileName": "NAC_ROI_AITKNCTRHIA_E168S1734_20M.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/NAC_ROI/NAC_ROI_AITKNCTRHIA_E168S1734_20M.IMG"}, {"FileName": "WAC_ROI_FARSIDE_DUSK_E000N1800_004P.IMG", "Type": "Product", "URL": "https://pds.lroc.im-ldi.com/data/LRO-L-LROC-5-RDR-V1.0/LROLRC_2001/DATA/BDR/WAC_ROI/WAC_ROI_FARSIDE_DUSK/WAC_ROI_FARSIDE_DUSK_E000N1800_004P.IMG"} + ], + "KPLO/ShadowCam/CMOS": [ + {"FileName": "SHADOWCAM_CMOSAIC_FAUSTINI04_P871S0833_SUMMER_04PM.CUB", "Type": "Product", "URL": "https://pds.shadowcam.im-ldi.com/derived/cmosaic/faustini04/shadowcam_cmosaic_faustini04_p871s0833_summer_04pm.cub"}, + {"FileName": "SHADOWCAM_CMOSAIC_FAUSTINI04_P871S0833_SUMMER_04PM_COG.TIF", "Type": "Product", "URL": "https://pds.shadowcam.im-ldi.com/derived/cmosaic/faustini04/shadowcam_cmosaic_faustini04_p871s0833_summer_04pm_cog.tif"}, + {"FileName": "SHADOWCAM_CMOSAIC_FAUSTINI04_P871S0833_SUMMER_04PM_RESIDUALS.PNG", "Type": "Browse", "URL": "https://pds.shadowcam.im-ldi.com/derived/cmosaic/faustini04/shadowcam_cmosaic_faustini04_p871s0833_summer_04pm_residuals.png"} + ], + "KPLO/ShadowCam/DTM": [ + {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_DTM_6M.CUB", "Type": "Product", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_dtm_6m.cub"}, + {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_DTM_6M_COG.TIF", "Type": "Product", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_dtm_6m_cog.tif"}, + {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_CONFIDENCE_COG.TIF", "Type": "Referenced", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_confidence_cog.tif"}, + {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_SLOPE_COG.TIF", "Type": "Referenced", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_slope_cog.tif"}, + {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_COLOR-SLOPE_COG.TIF", "Type": "Referenced", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_color-slope_cog.tif"} ] } diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index ea093e2..7d817af 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -224,6 +224,22 @@ def test_nac_roi_fetches_a_real_patch(tmp_path: Path) -> None: assert bool(sample["mask"].any()) +@pytest.mark.live +def test_shadowcam_fetches_a_real_patch(tmp_path: Path) -> None: + """ShadowCam: footprint-constrained sampling over a PSR site mosaic.""" + moondata = af.ShadowCam( + products=["mosaic"], + resolution=5.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index d27adc0..2dd99d6 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -651,3 +651,35 @@ def test_nac_roi_uses_footprint_sampling() -> None: def test_catalog_includes_nac_roi() -> None: assert MOON.probes["lro"].instruments["nac_roi"].dataset is af.LROCNACROI + + +def test_shadowcam_mosaic_pattern_selects_the_cog() -> None: + _assert_pattern_selects( + af.ShadowCam.all_products["mosaic"], + "KPLO/ShadowCam/CMOS", + "summer_04pm_cog.tif", + ) + + +def test_shadowcam_dtm_pattern_excludes_rendered_products() -> None: + _assert_pattern_selects( + af.ShadowCam.all_products["dtm"], "KPLO/ShadowCam/DTM", "p847s3110_dtm_6m_cog.tif" + ) + + +def test_shadowcam_confidence_uses_referenced_file_type() -> None: + spec = af.ShadowCam.all_products["confidence"] + assert spec.file_type == "Referenced" + _assert_pattern_selects(spec, "KPLO/ShadowCam/DTM", "p847s3110_confidence_cog.tif") + + +def test_shadowcam_uses_footprint_sampling() -> None: + assert af.ShadowCam.footprint_sampling is True + + +def test_catalog_includes_shadowcam() -> None: + assert MOON.probes["kplo"].instruments["shadowcam"].dataset is af.ShadowCam + spec = LAYERS["shadowcam_mosaic"] + assert spec.ihid == "KPLO" + assert spec.iid == "ShadowCam" + assert spec.pt == "CMOS" From 9974422d40f1ccd97b72346380b4d7192a4a97f3 Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:56:54 +0200 Subject: [PATCH 21/22] feat(moon): add ClementineUVVIS and ClementineNIR, and the clementine probe Search PDS ODE (product type MDIM) for the Clementine UVVIS 5-band and NIR 6-band basemaps, one product per band indexing the same per-tile file. Sinusoidal-projected, attached PDS3 label -- the first attached-label source in this project, so its georeferencing was verified live before shipping: opened a tile directly and compared its bounds against the product's own ODE footprint, which agreed to within expected rounding (unlike the LROCNACDTM PDS4-label case, this archive's driver checked out). New clementine probe in the catalog, since these are the first Clementine datasets. --- src/astrofetch/__init__.py | 4 +++ src/astrofetch/moon/__init__.py | 4 +++ src/astrofetch/moon/datasets.py | 46 ++++++++++++++++++++++++ src/astrofetch/moon/layers.py | 20 +++++++++++ tests/fixtures/ode/phase_c_listings.json | 8 +++++ tests/live/test_pds_ode_live.py | 42 ++++++++++++++++++++++ tests/unit/test_datasets.py | 34 ++++++++++++++++++ 7 files changed, 158 insertions(+) diff --git a/src/astrofetch/__init__.py b/src/astrofetch/__init__.py index 956ff8f..17fe98c 100644 --- a/src/astrofetch/__init__.py +++ b/src/astrofetch/__init__.py @@ -14,6 +14,8 @@ MOON, SLDEM2015, WACGLD100, + ClementineNIR, + ClementineUVVIS, DivinerGDR, IntersectionDataset, KaguyaTC, @@ -34,6 +36,8 @@ "LOLA", "MOON", "M3", + "ClementineNIR", + "ClementineUVVIS", "DivinerGDR", "IntersectionDataset", "KaguyaTC", diff --git a/src/astrofetch/moon/__init__.py b/src/astrofetch/moon/__init__.py index 80d5875..d81620a 100644 --- a/src/astrofetch/moon/__init__.py +++ b/src/astrofetch/moon/__init__.py @@ -6,6 +6,8 @@ LROCNACROI, SLDEM2015, WACGLD100, + ClementineNIR, + ClementineUVVIS, DivinerGDR, InstrumentDataset, IntersectionDataset, @@ -29,6 +31,8 @@ "MOON", "M3", "Body", + "ClementineNIR", + "ClementineUVVIS", "DivinerGDR", "GranuleDataset", "Instrument", diff --git a/src/astrofetch/moon/datasets.py b/src/astrofetch/moon/datasets.py index ed033c0..0110c8e 100644 --- a/src/astrofetch/moon/datasets.py +++ b/src/astrofetch/moon/datasets.py @@ -801,6 +801,52 @@ class ShadowCam(ODEInstrumentDataset): } +class ClementineUVVIS(ODEInstrumentDataset): + """Clementine UVVIS 5-band basemap, searched via PDS ODE (product type + ``MDIM``): reflectance at 415, 750, 900, 950, and 1000 nm (per mission + documentation; band order verified live 2026-07-21 as 5 bands matching + the product's declared band count). Sinusoidal-projected, attached PDS3 + label -- the first attached-label source in this project; georeferencing + verified live by comparing the opened raster's bounds against the + product's own ODE footprint (rule 1's "verify, don't assume" lesson; + this one checked out, unlike the LROCNACDTM PDS4-label case). + """ + + probe = "Clementine" + instrument = "UVVIS (5-band basemap)" + ihid = "CLEM" + iid = "UVVIS" + all_products = { + "band_415nm": ODEAsset("clem_uvvis_415nm", "MDIM", r"UI\d{2}[NS]\d{3}\.IMG", band=1), + "band_750nm": ODEAsset("clem_uvvis_750nm", "MDIM", r"UI\d{2}[NS]\d{3}\.IMG", band=2), + "band_900nm": ODEAsset("clem_uvvis_900nm", "MDIM", r"UI\d{2}[NS]\d{3}\.IMG", band=3), + "band_950nm": ODEAsset("clem_uvvis_950nm", "MDIM", r"UI\d{2}[NS]\d{3}\.IMG", band=4), + "band_1000nm": ODEAsset("clem_uvvis_1000nm", "MDIM", r"UI\d{2}[NS]\d{3}\.IMG", band=5), + } + + +class ClementineNIR(ODEInstrumentDataset): + """Clementine NIR 6-band basemap, searched via PDS ODE (product type + ``MDIM``): reflectance at 1100, 1250, 1500, 2000, 2600, and 2780 nm (per + mission documentation; band order verified live 2026-07-21 as 6 bands + matching the product's declared band count). Same sinusoidal-projected, + attached-PDS3-label archive as :class:`ClementineUVVIS`. + """ + + probe = "Clementine" + instrument = "NIR (6-band basemap)" + ihid = "CLEM" + iid = "NIR" + all_products = { + "band_1100nm": ODEAsset("clem_nir_1100nm", "MDIM", r"NI\d{2}[NS]\d{3}\.IMG", band=1), + "band_1250nm": ODEAsset("clem_nir_1250nm", "MDIM", r"NI\d{2}[NS]\d{3}\.IMG", band=2), + "band_1500nm": ODEAsset("clem_nir_1500nm", "MDIM", r"NI\d{2}[NS]\d{3}\.IMG", band=3), + "band_2000nm": ODEAsset("clem_nir_2000nm", "MDIM", r"NI\d{2}[NS]\d{3}\.IMG", band=4), + "band_2600nm": ODEAsset("clem_nir_2600nm", "MDIM", r"NI\d{2}[NS]\d{3}\.IMG", band=5), + "band_2780nm": ODEAsset("clem_nir_2780nm", "MDIM", r"NI\d{2}[NS]\d{3}\.IMG", band=6), + } + + class IntersectionDataset(_WindowedDataset): """Coregistered channel stack of two datasets over their overlap. diff --git a/src/astrofetch/moon/layers.py b/src/astrofetch/moon/layers.py index d9a5e5f..3c63197 100644 --- a/src/astrofetch/moon/layers.py +++ b/src/astrofetch/moon/layers.py @@ -22,6 +22,8 @@ LROCNACROI, SLDEM2015, WACGLD100, + ClementineNIR, + ClementineUVVIS, DivinerGDR, KaguyaTC, KaguyaTCImagery, @@ -183,6 +185,17 @@ def _spec(dataset: type[_ProductDataset], product: str) -> LayerSpec: _spec(ShadowCam, "mosaic"), _spec(ShadowCam, "dtm"), _spec(ShadowCam, "confidence"), + _spec(ClementineUVVIS, "band_415nm"), + _spec(ClementineUVVIS, "band_750nm"), + _spec(ClementineUVVIS, "band_900nm"), + _spec(ClementineUVVIS, "band_950nm"), + _spec(ClementineUVVIS, "band_1000nm"), + _spec(ClementineNIR, "band_1100nm"), + _spec(ClementineNIR, "band_1250nm"), + _spec(ClementineNIR, "band_1500nm"), + _spec(ClementineNIR, "band_2000nm"), + _spec(ClementineNIR, "band_2600nm"), + _spec(ClementineNIR, "band_2780nm"), ) } @@ -237,6 +250,13 @@ def _instrument(dataset: type[_ProductDataset]) -> Instrument: "shadowcam": _instrument(ShadowCam), }, ), + "clementine": Probe( + name=ClementineUVVIS.probe, + instruments={ + "uvvis": _instrument(ClementineUVVIS), + "nir": _instrument(ClementineNIR), + }, + ), }, ) """Discovery catalog for the Moon: enumerate probes, instruments, products, diff --git a/tests/fixtures/ode/phase_c_listings.json b/tests/fixtures/ode/phase_c_listings.json index 8d3537d..c295e4f 100644 --- a/tests/fixtures/ode/phase_c_listings.json +++ b/tests/fixtures/ode/phase_c_listings.json @@ -62,5 +62,13 @@ {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_CONFIDENCE_COG.TIF", "Type": "Referenced", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_confidence_cog.tif"}, {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_SLOPE_COG.TIF", "Type": "Referenced", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_slope_cog.tif"}, {"FileName": "SHADOWCAM_DTM_LCROSS1_P847S3110_COLOR-SLOPE_COG.TIF", "Type": "Referenced", "URL": "https://pds.shadowcam.im-ldi.com/derived/dtm/lcross1/shadowcam_dtm_lcross1_p847s3110_color-slope_cog.tif"} + ], + "CLEM/UVVIS/MDIM": [ + {"FileName": "UI73N007.IMG", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/geocopy/imaging/clem1-l-u-5-dim-uvvis-v1.0/cl_4001/data/ui73n007.img"}, + {"FileName": "UI73N007.JPG", "Type": "Browse", "URL": "https://pds-geosciences.wustl.edu/geocopy/imaging/clem1-l-u-5-dim-uvvis-v1.0/cl_4001/browse/750nm/large/ui73n007.jpg"} + ], + "CLEM/NIR/MDIM": [ + {"FileName": "NI73N007.IMG", "Type": "Product", "URL": "https://pds-geosciences.wustl.edu/geocopy/imaging/clem1-l-n-5-dim-nir-v1.0/cl_5001/data/ni73n007.img"}, + {"FileName": "NI73N007.JPG", "Type": "Browse", "URL": "https://pds-geosciences.wustl.edu/geocopy/imaging/clem1-l-n-5-dim-nir-v1.0/cl_5001/browse/2000nm/large/ni73n007.jpg"} ] } diff --git a/tests/live/test_pds_ode_live.py b/tests/live/test_pds_ode_live.py index 7d817af..7e8330d 100644 --- a/tests/live/test_pds_ode_live.py +++ b/tests/live/test_pds_ode_live.py @@ -240,6 +240,48 @@ def test_shadowcam_fetches_a_real_patch(tmp_path: Path) -> None: assert bool(sample["mask"].any()) +_TYCHO_AREA = (-12.5, -44.5, -10.5, -42.5) + + +@pytest.mark.live +def test_clementine_uvvis_fetches_a_real_patch(tmp_path: Path) -> None: + """Clementine UVVIS, sinusoidal-projected attached-label PDS3 archive, + over Tycho (a bright, high-albedo crater).""" + moondata = af.ClementineUVVIS( + products=["band_750nm"], + bbox=_TYCHO_AREA, + resolution=200.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + valid = sample["image"][sample["mask"]] + assert bool((valid >= 0.0).all()) and bool((valid <= 1.0).all()) + + +@pytest.mark.live +def test_clementine_nir_fetches_a_real_patch(tmp_path: Path) -> None: + """Clementine NIR over the same Tycho area.""" + moondata = af.ClementineNIR( + products=["band_1500nm"], + bbox=_TYCHO_AREA, + resolution=200.0, + patch_size=32, + length=1, + seed=1, + cache=WindowCache(tmp_path), + ) + sample = next(iter(moondata)) + assert sample["image"].shape == (1, 32, 32) + assert bool(sample["mask"].any()) + valid = sample["image"][sample["mask"]] + assert bool((valid >= 0.0).all()) and bool((valid <= 1.0).all()) + + @pytest.mark.live def test_nac_raw_granule_reads_a_row_slice() -> None: """EXPERIMENTAL granule path: PDS4 raw NAC strip, partial row read.""" diff --git a/tests/unit/test_datasets.py b/tests/unit/test_datasets.py index 2dd99d6..89da268 100644 --- a/tests/unit/test_datasets.py +++ b/tests/unit/test_datasets.py @@ -683,3 +683,37 @@ def test_catalog_includes_shadowcam() -> None: assert spec.ihid == "KPLO" assert spec.iid == "ShadowCam" assert spec.pt == "CMOS" + + +def test_clementine_uvvis_pattern_selects_the_data_file() -> None: + _assert_pattern_selects( + af.ClementineUVVIS.all_products["band_415nm"], "CLEM/UVVIS/MDIM", "ui73n007.img" + ) + + +def test_clementine_uvvis_bands_index_into_the_same_file() -> None: + urls_and_bands = [ + (entry.pattern, entry.band) for entry in af.ClementineUVVIS.all_products.values() + ] + assert len({pattern for pattern, _ in urls_and_bands}) == 1 + assert sorted(band for _, band in urls_and_bands) == [1, 2, 3, 4, 5] + + +def test_clementine_nir_pattern_selects_the_data_file() -> None: + _assert_pattern_selects( + af.ClementineNIR.all_products["band_1100nm"], "CLEM/NIR/MDIM", "ni73n007.img" + ) + + +def test_clementine_nir_bands_index_into_the_same_file() -> None: + bands = sorted(entry.band for entry in af.ClementineNIR.all_products.values()) + assert bands == [1, 2, 3, 4, 5, 6] + + +def test_catalog_includes_clementine() -> None: + assert MOON.probes["clementine"].instruments["uvvis"].dataset is af.ClementineUVVIS + assert MOON.probes["clementine"].instruments["nir"].dataset is af.ClementineNIR + spec = LAYERS["clem_uvvis_415nm"] + assert spec.ihid == "CLEM" + assert spec.iid == "UVVIS" + assert spec.pt == "MDIM" From 6b7c822f879b0b35fc94bdc1f1fdd5bf53f1c7ff Mon Sep 17 00:00:00 2001 From: Tom Sander Date: Tue, 21 Jul 2026 09:59:35 +0200 Subject: [PATCH 22/22] docs: document the wider PDS ODE dataset roster Add reference pages for the ten new datasets (MiniRF, DivinerGDR, WACGLD100, WACTiO2, LROCWACGlobal, LROCWACColor, LROCNACROI, ShadowCam, ClementineUVVIS, ClementineNIR), a ShadowCam quickstart example, and an updated Phase 2 roadmap summary. AGENTS.md: record the ODEAsset.product_id search-narrowing pattern as a domain note, list the roster items deliberately dropped with their reasons (Kaguya MI's host ignores Range headers, Kaguya LALT superseded by SLDEM2015, Diviner TBOL mixes epochs, and several redundant/ follow-up products), and add the Mini-RF nodata-overflow-artifact and Clementine attached-label-verification live findings to risks. --- AGENTS.md | 15 +++++++++------ docs/index.md | 12 +++++++++--- docs/reference/datasets.md | 24 ++++++++++++++++++++++++ docs/roadmap.md | 17 +++++++++++------ 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f9187fe..d0efabb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,6 +85,7 @@ Widely-adopted defaults that keep the codebase consistent. When in doubt, match - COGs and other rasters often store 16-bit DN (or similar) with scale/offset to physical units (for example Kaguya TC radiance). Always apply scale/offset in `raster.py`; downstream code assumes physical values. - Nodata regions are common (orbital swaths do not cover everything). Every sample carries a boolean validity tensor under its `"mask"` key; do not silently zero-fill. Some PDS products omit a declared nodata value even though their raster does not cover its full requested extent; `raster.read_window`'s `nodata_override` exists for exactly this (see its docstring) — reach for a source's own declared nodata first, and only override when you have live-verified the product genuinely has none. - Some instruments cover only a handful of named sites, not the whole Moon (e.g. LROC NAC stereo DTMs via PDS ODE). `ODEInstrumentDataset.footprint_sampling` draws windows from inside real product footprints for exactly this case; leave it off for globally-covered instruments. +- Some ODE product types mix the products a dataset actually wants with many unrelated ones under the same `pt` — a different parameter, a rendered visualization, or per-orbit granules that vastly outnumber the wanted product (Diviner's `GDR_L3` is ~80% per-orbit `TBOL` files; LROC's `SDWDTM` is dominated by rendered `WAC_CSHADE`). A bbox-only search then has to page through candidates in whatever order ODE returns them before ever reaching the wanted product, which no reasonable `max_products` cap reliably reaches — and paging that deep violates rule 5. `ODEAsset.product_id` (an ODE `productid` wildcard filter, e.g. `"*wac_gld100*"`) narrows the search itself server-side; reach for it whenever a new source's product type isn't cleanly single-purpose. `ODEInstrumentDataset._product_footprints` applies the same per-product `product_id` narrowing, not just `_hrefs`, so footprint-sampled instruments don't draw from an unrelated sibling instrument's sites either. ## What NOT to do @@ -99,7 +100,7 @@ Widely-adopted defaults that keep the codebase consistent. When in doubt, match Check the current phase before proposing work; for example, do not build Phase 2 datasets and transforms while Phase 1 (STAC sampler MVP) is incomplete. Everything is a thin layer above existing archive tooling, never a mirror of any archive. -**Current phase: Phase 2 (datasets and transforms). Phase 0 and Phase 1 are complete: `InstrumentDataset.read` fetches the real COGs covering a window from the USGS ARD catalog, reprojects them onto a common geographic grid, applies scale/offset, mosaics overlapping items, and caches the result. Phase 2 has started delivering new data sources beyond STAC (see below); `GridTileDataset`, spatial-split samplers, transforms, and the WMS/WMTS rendered mode remain open.** +**Current phase: Phase 2 (datasets and transforms). Phase 0 and Phase 1 are complete: `InstrumentDataset.read` fetches the real COGs covering a window from the USGS ARD catalog, reprojects them onto a common geographic grid, applies scale/offset, mosaics overlapping items, and caches the result. Phase 2 has delivered a wide roster of new data sources beyond STAC (see below); `GridTileDataset`, spatial-split samplers, transforms, and the WMS/WMTS rendered mode remain open.** ### Phase 0: Scaffolding (weekend 1) @@ -139,10 +140,11 @@ Exit criteria (met): `KaguyaTC(products=["dtm", "ortho"], bbox=...)` fetches a r **New data sources beyond STAC (done):** the USGS ARD STAC catalog has no LROC, LOLA, or other lunar collections beyond Kaguya TC, so growing past it required a second search backend. -- `astrofetch.data.ode`: query the NASA PDS Orbital Data Explorer (ODE) REST API by instrument host/id and product type, politely (same retry/backoff/timeout posture as `data/stac.py`), normalizing ODE's JSON quirks (single-result dict vs list, `"No Products Found"`, HTTP-200 error bodies). -- `astrofetch.moon.datasets.ODEInstrumentDataset`: the ODE-backed sibling of `InstrumentDataset`, with `footprint_sampling` for instruments that cover only named sites rather than the whole Moon. Ships `LROCNACDTM` (LRO LROC NAC stereo DTM sites: elevation, orthoimage, pixel confidence). +- `astrofetch.data.ode`: query the NASA PDS Orbital Data Explorer (ODE) REST API by instrument host/id and product type, politely (same retry/backoff/timeout posture as `data/stac.py`), normalizing ODE's JSON quirks (single-result dict vs list, `"No Products Found"`, HTTP-200 error bodies). `query_products`/`find_file_urls` also accept a `product_id` wildcard filter for product types that mix the wanted product with many unrelated ones (see the domain note above). +- `astrofetch.moon.datasets.ODEInstrumentDataset`: the ODE-backed sibling of `InstrumentDataset`, with `footprint_sampling` for instruments that cover only named sites rather than the whole Moon. Ships the wider PDS ODE roster: `LROCNACDTM` (NAC stereo DTM sites), `LROCNACROI` (NAC region-of-interest mosaics, 5 m/20 m only — native resolution can reach ~14 GB per site), `MiniRF` (S-band radar global mosaics), `DivinerGDR` (rock abundance / regolith temperature, pinned to the most complete cumulative-mosaic date), `WACGLD100` (WAC global 100 m DTM), `WACTiO2` (WAC TiO2 abundance), `LROCWACGlobal` (WAC global morphology, tiled/searched sibling of `LROCWACMosaic`), `LROCWACColor` (WAC 7-color reflectance), `ShadowCam` (KPLO PSR mosaics and DTMs, genuine COGs), and `ClementineUVVIS`/`ClementineNIR` (5-/6-band basemaps, the project's first attached-PDS3-label source). - `astrofetch.moon.datasets.MosaicDataset`: reads one well-known archive URL directly, for instruments published as a single global (or near-global) file. Ships `LROCWACMosaic` (the LRO WAC 100 m global mosaic — the dataset this phase originally named as its LRO WAC deliverable, shipped as `LROCWACMosaic` rather than `LROCWAC` since a raw, non-map-projected `LROCWACRaw` also now exists), `LOLA` (global gridded DEM), and `SLDEM2015` (LOLA + Kaguya TC co-registered DEM). - `astrofetch.moon.granules` (new, experimental — see its Deliberate non-goals amendment below): raw, camera-geometry PDS granules for instruments that are not map-projected at all (`LROCNACRaw`, `LROCWACRaw`, `M3`). A deliberately different, documented sample contract; not part of the `InstrumentDataset` family. +- Roster items deliberately dropped, with reasons (do not re-propose without addressing the reason): **Kaguya MI** — its JAXA DARTS host ignores HTTP Range headers (a ranged GET returns the full ~90 MB body), so windowed remote reads are impossible without downloading whole files; **Kaguya LALT** — coarse (3 products) and superseded by `SLDEM2015`; **Diviner `TBOL`** — per-orbit bolometric temperature, not part of the cumulative-mosaic family `DivinerGDR` offers, and mosaicking single-orbit epochs would misrepresent the data; **`MDRHAP`** (Hapke-normalized WAC color), **`SDPWMG`** (monthly WAC mosaics), **Mini-RF polar `MOSCDR`**, **Clementine `MDIMG`** basemap and **`HIRES`** — redundant with what's shipped or left as follow-up. **Still open:** @@ -150,7 +152,7 @@ Exit criteria (met): `KaguyaTC(products=["dtm", "ortho"], bbox=...)` fetches a r - Samplers that respect spatial autocorrelation for train/val/test splits (block splitting, not random pixels). - Transforms: per-channel normalization stats, nodata masking, polar/equatorial projection handling made explicit. - Secondary access mode behind the same interface: WMS/WMTS rendered mode, clearly labeled non-quantitative. -- The wider PDS ODE roster beyond the four flagship datasets above: Diviner, Mini-RF, Clementine, ShadowCam, Kaguya MI, and further WAC-derived products (TiO2, GLD100, 7-color reflectance) all fit the same `ODEInstrumentDataset`/`MosaicDataset` pattern; each needs its own live-verified product type and filename pattern before shipping (rule 1's PDS4-label lesson applies to every new source, not just the ones already caught). +- A dedicated polar target grid (`data/grid.py`): `ShadowCam` and other near-polar sources currently reproject onto the same equirectangular geographic grid as everything else, which distorts near the poles; a polar-stereographic target grid for high-latitude requests remains future work. Exit criteria: `DataLoader` trains a toy model on random lunar patches without custom user code. @@ -173,6 +175,7 @@ Exit criteria: `DataLoader` trains a toy model on random lunar patches without c - Endpoint drift (services move, as QuickMap's domain change showed): keep all endpoint URLs in one config module, cover them with the live test suite, and document last-verified dates. - M3 data quality: rather than deferring M3 entirely, it shipped scoped to what's verified reliable — the experimental raw-granule path (`astrofetch.moon.granules.M3`), radiance plus geolocation backplane, no map projection or further calibration claimed. A map-projected, quantitative M3 `InstrumentDataset`/`ODEInstrumentDataset` remains deferred until a user need justifies the preprocessing work. -- Archive driver quirks: a source opening and reading without error is not proof its georeferencing or nodata are correct (see design rule 1's PDS4-label example, caught via `tests/live` before shipping `LROCNACDTM`). Live-verify a new source against a known location, not just that `rasterio.open` succeeds. -- Server load courtesy: default to conservative request concurrency, exponential backoff, and a bulk prefetch helper so training never hammers archive servers with random access. +- Archive driver quirks: a source opening and reading without error is not proof its georeferencing or nodata are correct (see design rule 1's PDS4-label example, caught via `tests/live` before shipping `LROCNACDTM`). Live-verify a new source against a known location, not just that `rasterio.open` succeeds. The same live check can also come back clean: `ClementineUVVIS`/`ClementineNIR` (attached PDS3 label, sinusoidal projection) checked out on the first try, comparing the opened raster's bounds against the product's own ODE footprint — not every new archive hides a bug, but every one needs the check. +- Not every archive quirk is a georeferencing bug: `MiniRF`'s global mosaics declare a valid `MISSING_CONSTANT` in their PDS3 label, but it's a float64 literal inside a float32 band, and GDAL's overflowing cast leaves `src.nodata` unset rather than raising — out-of-coverage pixels silently read back as a specific overflow artifact (not even the standard float32 minimum), marked "valid". Caught by reading raw pixels directly and pinning the exact observed bit pattern as `nodata_override`. The general lesson: a *declared* nodata value is not automatically a *working* one — verify what the library actually did with it, not just that the label mentions it. +- Server load courtesy: default to conservative request concurrency, exponential backoff, and a bulk prefetch helper so training never hammers archive servers with random access. Also watch for product types where a bbox-only search can't stay small on its own (Diviner's `GDR_L3`, LROC's `SDWDTM`) — `ODEAsset.product_id` narrows the search server-side instead of paging through hundreds of irrelevant candidates client-side. - Solo-maintainer bus factor: keep scope small, tests honest, and architecture boring enough that contributors can navigate it without you. diff --git a/docs/index.md b/docs/index.md index 6430f7d..fb32e94 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,9 +46,11 @@ for batch in loader: ## Beyond the STAC catalog -Some instruments (LROC, LOLA, ...) aren't in the USGS ARD STAC catalog at -all; those datasets search the NASA PDS Orbital Data Explorer instead, or -read a single fixed mosaic URL, behind the exact same interface: +Some instruments (LROC, LOLA, Mini-RF, Diviner, ShadowCam, Clementine, ...) +aren't in the USGS ARD STAC catalog at all; those datasets search the NASA +PDS Orbital Data Explorer instead, or read a single fixed mosaic URL, behind +the exact same interface. The full roster is in +[Instrument datasets](reference/datasets.md); two examples: ```python import astrofetch as af @@ -60,6 +62,10 @@ sample = nac[0] # A global 100 m WAC mosaic and a global LOLA DEM, channel-stacked with `&`. terrain = af.LROCWACMosaic(resolution=100) & af.LOLA(resolution=100) + +# ShadowCam mosaics of permanently shadowed polar craters, another +# site-based instrument -- footprint sampling applies the same way. +shadowcam = af.ShadowCam(products=["mosaic"]) ``` ## Discovering what data exists diff --git a/docs/reference/datasets.md b/docs/reference/datasets.md index ac6b9e9..bae81e2 100644 --- a/docs/reference/datasets.md +++ b/docs/reference/datasets.md @@ -26,6 +26,30 @@ contract, not part of the windowed-dataset family below. ::: astrofetch.moon.datasets.SLDEM2015 +::: astrofetch.moon.datasets.MiniRF + +::: astrofetch.moon.datasets.DivinerGDR + +::: astrofetch.moon.datasets.WACGLD100 + +::: astrofetch.moon.datasets.WACTiO2 + +::: astrofetch.moon.datasets.LROCWACGlobal + +::: astrofetch.moon.datasets.LROCWACColor + +::: astrofetch.moon.datasets.LROCNACROI + +## Korea Pathfinder Lunar Orbiter + +::: astrofetch.moon.datasets.ShadowCam + +## Clementine + +::: astrofetch.moon.datasets.ClementineUVVIS + +::: astrofetch.moon.datasets.ClementineNIR + ## Base classes ::: astrofetch.moon.datasets.InstrumentDataset diff --git a/docs/roadmap.md b/docs/roadmap.md index 7ae392e..54726a2 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -14,12 +14,17 @@ this page is the short version. **Current phase: Phase 2.** `InstrumentDataset.read(bbox)` fetches real COGs from the USGS ARD catalog, reprojects them onto a common geographic grid, applies scale/offset, mosaics overlapping items, and caches the result — the -Phase 1 exit criterion. Phase 2 has started delivering new data sources -beyond STAC: the NASA PDS Orbital Data Explorer (`ODEInstrumentDataset`) adds -LROC NAC stereo DTM sites, and fixed-URL mosaics (`MosaicDataset`) add the -LRO WAC global mosaic and the LOLA and SLDEM2015 global DEMs — all behind -the same sample-dict contract and `&` composition as the STAC-backed -datasets. An experimental, separately-contracted raw-granule path +Phase 1 exit criterion. Phase 2 has delivered new data sources beyond STAC: +the NASA PDS Orbital Data Explorer (`ODEInstrumentDataset`) now backs a wide +roster of instruments — LROC NAC stereo DTM sites and region-of-interest +mosaics, Mini-RF S-band radar, Diviner rock abundance and regolith +temperature, WAC GLD100, TiO2, tiled global morphology, and 7-color +reflectance, ShadowCam polar mosaics and DTMs, and the Clementine UVVIS and +NIR basemaps — and fixed-URL mosaics (`MosaicDataset`) add the LRO WAC +global mosaic and the LOLA and SLDEM2015 global DEMs, all behind the same +sample-dict contract and `&` composition as the STAC-backed datasets. See +[Instrument datasets](reference/datasets.md) for the full list. An +experimental, separately-contracted raw-granule path (`astrofetch.moon.granules`) also now exists for camera-geometry NAC/WAC strips and M3 radiance cubes; see [Raw granules](reference/granules.md). Still open for Phase 2: `GridTileDataset`, spatial-autocorrelation-aware