diff --git a/src/earthkit/data/core/fieldlist.py b/src/earthkit/data/core/fieldlist.py index 8b4417ee3..e195f493f 100644 --- a/src/earthkit/data/core/fieldlist.py +++ b/src/earthkit/data/core/fieldlist.py @@ -263,6 +263,8 @@ def data(self, keys=("lat", "lon", "value"), flatten=False, dtype=None, index=No for k in keys: # TODO: convert dtype v = _keys[k](dtype=dtype) + if v is None: + raise ValueError(f"data: {k} not available") v = self._reshape(v, flatten) if index is not None: v = v[index] diff --git a/src/earthkit/data/sources/array_list.py b/src/earthkit/data/sources/array_list.py index 144e614a5..344a51a0a 100644 --- a/src/earthkit/data/sources/array_list.py +++ b/src/earthkit/data/sources/array_list.py @@ -57,6 +57,11 @@ def _values(self, dtype=None): else: return array_namespace(self._array).astype(self._array, dtype, copy=False) + @property + def shape(self): + v = super().shape + return v if v is not None else self._array.shape + def __repr__(self): return self.__class__.__name__ + "(%s,%s,%s,%s,%s,%s)" % ( self._metadata.get("shortName", None), diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py index dc8b7e0b0..fa22d020c 100644 --- a/src/earthkit/data/utils/metadata/dict.py +++ b/src/earthkit/data/utils/metadata/dict.py @@ -9,6 +9,7 @@ import logging from functools import cached_property +from math import prod import numpy as np @@ -35,17 +36,18 @@ def uniform_resolution(vals): def make_geography(metadata, values_shape=None): lat = metadata.get("latitudes", None) lon = metadata.get("longitudes", None) - val = metadata.get("values") - # lat = np.asarray(lat, dtype=float) - # lon = np.asarray(lon, dtype=float) - - val = np.asarray(val, dtype=float) + values_size = prod(values_shape) if values_shape else None distinct = False if lat is None or lon is None: lat = metadata.get("distinctLatitudes", None) lon = metadata.get("distinctLongitudes", None) + + # it is possible to have no geography at all. + if lat is None and lon is None: + return NoGeography(values_shape) + if values_shape is None: if lat is None: raise ValueError("No latitudes or distinctLatitudes found") @@ -59,13 +61,6 @@ def make_geography(metadata, values_shape=None): raise ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported") if len(lon.shape) != 1: raise ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported") - if lat.size * lon.size != val.size: - raise ValueError( - ( - "Distinct latitudes and longitudes do not match number of values. " - f"Expected number=({lat.size * lon.size}), got={val.size}" - ) - ) distinct = True else: if lat is not None and lon is not None: @@ -76,11 +71,11 @@ def make_geography(metadata, values_shape=None): raise ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported") if len(lon.shape) != 1: raise ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported") - if lat.size * lon.size != val.size: + if lat.size * lon.size != values_size: raise ValueError( ( "Distinct latitudes and longitudes do not match number of values. " - f"Expected number=({lat.size * lon.size}), got={val.size}" + f"Expected number=({lat.size * lon.size}), got={values_size}" ) ) distinct = True @@ -91,29 +86,21 @@ def make_geography(metadata, values_shape=None): else: lat = np.asarray(lat, dtype=float) lon = np.asarray(lon, dtype=float) - if lat.size * lon.size == val.size: - if len(lat.shape) != 1: - raise ValueError( - f"latitudes must be a 1D array when holding distinct values! shape={lat.shape} unsupported" - ) - if len(lon.shape) != 1: - raise ValueError( - f"longitudes must be a 1D array when holding distinct values! shape={lon.shape} unsupported" - ) - distinct = True - - no_latlon = lat is None or lon is None - - if no_latlon: - assert values_shape is not None + if values_size is not None: + if lat.size * lon.size == values_size: + if len(lat.shape) != 1: + raise ValueError( + f"latitudes must be a 1D array when holding distinct values! shape={lat.shape} unsupported" + ) + if len(lon.shape) != 1: + raise ValueError( + f"longitudes must be a 1D array when holding distinct values! shape={lon.shape} unsupported" + ) + distinct = True - if values_shape is None: - assert lat is not None - assert lon is not None + assert lat is not None and lon is not None if distinct: - assert not no_latlon - dx = uniform_resolution(lon) dy = uniform_resolution(lat) @@ -123,28 +110,74 @@ def make_geography(metadata, values_shape=None): return RegularDistinctLLGeography(metadata) else: return DistinctLLGeography(metadata) - - if no_latlon: - return UserGeography(metadata, shape=values_shape) else: if lat.shape != lon.shape: raise ValueError(f"latitudes and longitudes must have the same shape. {lat.shape} != {lon.shape}") - if lat.size == val.size: - if lat.shape != val.shape: - shape = lat.shape if lat.ndim > val.ndim else val.shape - else: - shape = lat.shape + if values_shape is not None: + if lat.size == values_size: + if values_shape is not None: + if lat.shape != values_shape: + shape = lat.shape if lat.ndim > len(values_shape) else values_shape + else: + shape = lat.shape + else: + shape = lat.shape - return UserGeography(metadata, shape=shape) + return UserGeography(metadata, shape=shape) - else: - raise ValueError( - ( - "latitudes and longitudes do not match number of values. " - f"Expected number=({lat.size * lon.size}), got={val.size}" + else: + raise ValueError( + ( + "latitudes and longitudes do not match number of values. " + f"Expected number=({lat.size * lon.size}), got={values_size}" + ) ) - ) + else: + shape = lat.shape + return UserGeography(metadata, shape=shape) + + +class NoGeography(Geography): + def __init__(self, shape): + self._shape = shape + + def latitudes(self, dtype=None): + return None + + def longitudes(self, dtype=None): + return None + + def x(self, dtype=None): + raise NotImplementedError("x is not implemented for this geography") + + def y(self, dtype=None): + raise NotImplementedError("y is not implemented for this geography") + + def shape(self): + return self._shape + + def _unique_grid_id(self): + return self.shape() + + def projection(self): + return None + + def bounding_box(self): + return None + + def gridspec(self): + return None + + def resolution(self): + return None + # raise NotImplementedError("resolution is not implemented for this geography") + + def mars_area(self): + return None + + def mars_grid(self): + raise NotImplementedError("mars_grid is not implemented for this geography") class UserGeography(Geography): @@ -219,10 +252,6 @@ def mars_grid(self): raise NotImplementedError("mars_grid is not implemented for this geography") -# class StructuredGeography(UserGeography): -# pass - - class DistinctLLGeography(UserGeography): def __init__(self, metadata): super().__init__(metadata) diff --git a/tests/array_fieldlist/test_array_field_usermetadata.py b/tests/array_fieldlist/test_array_field_usermetadata.py new file mode 100644 index 000000000..2ac6146f8 --- /dev/null +++ b/tests/array_fieldlist/test_array_field_usermetadata.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# (C) Copyright 2020 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + +import datetime + +import numpy as np +import pytest + +from earthkit.data import ArrayField +from earthkit.data.utils.metadata.dict import UserMetadata + + +def test_array_field_usermetadata_nogeom(): + vals = np.linspace(0, 1, 10) + meta = UserMetadata( + { + "shortName": "test", + "longName": "Test", + "date": 20180801, + "time": 300, + } + ) + + f = ArrayField(vals, meta) + + assert f.metadata("shortName") == "test" + assert f.metadata("longName") == "Test" + assert f.metadata("date") == 20180801 + assert f.metadata("time") == 300 + assert f.datetime()["base_time"] == datetime.datetime(2018, 8, 1, 3, 0) + assert f.shape == (10,) + assert np.allclose(f.values, vals) + + with pytest.raises(ValueError): + f.to_latlon() + + +@pytest.mark.parametrize("_kwargs", [{}, {"shape": None}, {"shape": (10,)}]) +def test_array_field_usermetadata_geom(_kwargs): + vals = np.linspace(0, 1, 10) + meta = UserMetadata( + { + "shortName": "test", + "longName": "Test", + "date": 20180801, + "time": 300, + "latitudes": np.linspace(-10.0, 10.0, 10), + "longitudes": np.linspace(20.0, 40.0, 10), + }, + **_kwargs, + ) + + f = ArrayField(vals, meta) + + assert f.metadata("shortName") == "test" + assert f.metadata("longName") == "Test" + assert f.metadata("date") == 20180801 + assert f.metadata("time") == 300 + assert f.datetime()["base_time"] == datetime.datetime(2018, 8, 1, 3, 0) + assert f.shape == (10,) + lat = f.to_latlon()["lat"] + lon = f.to_latlon()["lon"] + assert lat.shape == (10,) + assert lon.shape == (10,) + assert np.allclose(lat, meta["latitudes"]) + assert np.allclose(lon, meta["longitudes"]) + assert np.allclose(f.values, vals)