From e171fe2b889f9893d14cd88378e616c91833aa03 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Mon, 12 May 2025 13:36:20 +0100 Subject: [PATCH 1/2] Fix crash when add_earthkit_attrs=False used in Xarray engine (#696) * Fix crash when add_earthkit_attrs=False used in Xarray engine --- src/earthkit/data/utils/xarray/builder.py | 2 +- tests/xr_engine/test_xr_engine.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/earthkit/data/utils/xarray/builder.py b/src/earthkit/data/utils/xarray/builder.py index a303ae8cf..4d0276c82 100644 --- a/src/earthkit/data/utils/xarray/builder.py +++ b/src/earthkit/data/utils/xarray/builder.py @@ -407,7 +407,7 @@ def prepare_tensor(self, ds, dims, name): elif num > 1 or not self.profile.dims.squeeze or d.name in self.profile.dims.ensure_dims: tensor_dims.append(d) tensor_coords[d.key] = vals[d.key] - if d.key in component_vals: + if component_vals and d.key in component_vals: tensor_coords_component[d.key] = component_vals[d.key] # check if the dims/coords are consistent with the tensors of diff --git a/tests/xr_engine/test_xr_engine.py b/tests/xr_engine/test_xr_engine.py index 0910f783c..f8d9be3e9 100644 --- a/tests/xr_engine/test_xr_engine.py +++ b/tests/xr_engine/test_xr_engine.py @@ -542,7 +542,7 @@ def test_xr_engine_single_field(): @pytest.mark.cache @pytest.mark.parametrize("add", [False, True]) -def test_xr_engine_add_earthkit_attrs(add): +def test_xr_engine_add_earthkit_attrs_1(add): ds_ek = from_source("url", earthkit_remote_test_data_file("test-data/xr_engine/level/pl.grib")) ds_ek = ds_ek[0] @@ -558,3 +558,16 @@ def test_xr_engine_add_earthkit_attrs(add): assert "_earthkit" in ds["t"].attrs else: assert "_earthkit" not in ds["t"].attrs + + +@pytest.mark.cache +def test_xr_engine_add_earthkit_attrs_2(): + ds_ek = from_source("url", earthkit_remote_test_data_file("test-data/xr_engine/level/pl.grib")) + ds_ek = ds_ek[0] + + ds = ds_ek.to_xarray( + add_earthkit_attrs=False, + ) + + assert ds + assert "_earthkit" not in ds["t"].attrs From 3d6953ae4d779adab9d699be7123c60ee34ad3aa Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Wed, 14 May 2025 16:25:24 +0100 Subject: [PATCH 2/2] Fix file-pattern source returning wrong object (#697) --- docs/examples/files.ipynb | 19 +++++++++++++------ docs/release_notes/version_0.14_updates.rst | 9 +++++++++ src/earthkit/data/sources/file_pattern.py | 2 +- tests/sources/test_file_pattern.py | 21 +++++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 tests/sources/test_file_pattern.py diff --git a/docs/examples/files.ipynb b/docs/examples/files.ipynb index 9728ab253..e222bdb5a 100644 --- a/docs/examples/files.ipynb +++ b/docs/examples/files.ipynb @@ -185,6 +185,14 @@ "tags": [] }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/cgr/git/earthkit-data/src/earthkit/data/readers/netcdf/fieldlist.py:318: FutureWarning: In a future version of xarray decode_timedelta will default to False rather than None. To silence this warning, set decode_timedelta to True, False, or a 'CFTimedeltaCoder' instance.\n", + " return xr.open_dataset(self.path_or_url)\n" + ] + }, { "data": { "text/html": [ @@ -508,10 +516,9 @@ } ], "source": [ - "\n", - "fs = ekd.from_source(\"file-pattern\", \"./test{id}.grib\",\n", + "ds = ekd.from_source(\"file-pattern\", \"./test{id}.grib\",\n", " {\"id\": [4, 6]})\n", - "fs.ls()" + "ds.ls()" ] }, { @@ -531,9 +538,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dev_ecc", + "display_name": "dev", "language": "python", - "name": "dev_ecc" + "name": "dev" }, "language_info": { "codemirror_mode": { @@ -545,7 +552,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.11.12" } }, "nbformat": 4, diff --git a/docs/release_notes/version_0.14_updates.rst b/docs/release_notes/version_0.14_updates.rst index d64e35738..309ef6da1 100644 --- a/docs/release_notes/version_0.14_updates.rst +++ b/docs/release_notes/version_0.14_updates.rst @@ -1,6 +1,15 @@ Version 0.14 Updates ///////////////////////// +Version 0.14.2 +=============== + +Fixes ++++++++++++++++++ + +- Fixed issue when the :ref:`data-sources-file-pattern` source did not return the right data object when the ``hive_partitioning`` option was set to ``False`` (:pr:`697`). +- Fixed issue when disabling the ``add_earthkit_attrs`` option in :py:meth:`~data.readers.grib.index.GribFieldList.to_xarray` caused a crash (:pr:`696`). + Version 0.14.1 =============== diff --git a/src/earthkit/data/sources/file_pattern.py b/src/earthkit/data/sources/file_pattern.py index 5c524cb24..2d45b6a18 100644 --- a/src/earthkit/data/sources/file_pattern.py +++ b/src/earthkit/data/sources/file_pattern.py @@ -97,7 +97,7 @@ def mutate(self) -> Union["HiveFilePattern", "FilePattern"]: if self.hive_partitioning: return HiveFilePattern(self.pattern, self.params) else: - return self + return super().mutate() source = FilePattern diff --git a/tests/sources/test_file_pattern.py b/tests/sources/test_file_pattern.py new file mode 100644 index 000000000..77cfa2470 --- /dev/null +++ b/tests/sources/test_file_pattern.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +# (C) Copyright 2020 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + + +from earthkit.data import from_source +from earthkit.data.testing import earthkit_examples_file + + +def test_file_pattern_source_grib(): + ds = from_source("file-pattern", earthkit_examples_file("test{id}.grib"), {"id": [4, 6]}) + + assert len(ds) == 10 + assert ds.metadata("param") == ["t", "z", "t", "z", "t", "u", "v", "t", "u", "v"]