From 2c4400c2c62c6d0a16164f4299a5a7e925d32a80 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 3 Jun 2025 14:59:43 +0200 Subject: [PATCH 01/14] added check is_image_outdated --- cropclassification/calc_periodic_mosaic.py | 8 ++--- .../preprocess/_timeseries_calc_openeo.py | 3 ++ cropclassification/preprocess/timeseries.py | 1 + cropclassification/util/mosaic_util.py | 33 +++++++++++++++++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/cropclassification/calc_periodic_mosaic.py b/cropclassification/calc_periodic_mosaic.py index fe879373..b0d2b1b3 100644 --- a/cropclassification/calc_periodic_mosaic.py +++ b/cropclassification/calc_periodic_mosaic.py @@ -1,6 +1,6 @@ """Generate periodic mosaics.""" -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path import cropclassification.helpers.config_helper as conf @@ -32,11 +32,6 @@ def calc_periodic_mosaic_task(config_paths: list[Path], default_basedir: Path): end_date = datetime(now.year, now.month, now.day) else: end_date = datetime.fromisoformat(conf.period["end_date"]) - images_available_delay = conf.period["images_available_delay"] - if images_available_delay is not None: - now = datetime.now() - today = datetime(now.year, now.month, now.day) - end_date = today - timedelta(int(images_available_delay)) imageprofiles_to_get = list(conf.parse_image_config(conf.images["images"])) imageprofiles = conf._get_image_profiles( @@ -52,5 +47,6 @@ def calc_periodic_mosaic_task(config_paths: list[Path], default_basedir: Path): output_base_dir=conf.paths.getpath("images_periodic_dir"), imageprofiles_to_get=imageprofiles_to_get, imageprofiles=imageprofiles, + images_available_delay=conf.period["images_available_delay"], force=False, ) diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index acdeb873..4b823f3e 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -28,6 +28,7 @@ def calculate_periodic_timeseries( timeseries_periodic_dir: Path, nb_parallel: int, on_missing_image: str, + images_available_delay: Optional[int] = None, ): """Calculate timeseries data for the input parcels. @@ -50,6 +51,7 @@ def calculate_periodic_timeseries( - ignore: ignore that the image, don't try to download it - calculate_raise: calculate the image and raise an error if it fails - calculate_ignore: calculate the image and ignore the error if it fails + images_available_delay (Optional[int]): delay in days for images to be available """ info = gfo.get_layerinfo(input_parcel_path) if info.crs is not None and not info.crs.equals(roi_crs): @@ -68,6 +70,7 @@ def calculate_periodic_timeseries( imageprofiles_to_get=imageprofiles_to_get, imageprofiles=imageprofiles, on_missing_image=on_missing_image, + images_available_delay=conf.period["images_available_delay"], force=False, ) diff --git a/cropclassification/preprocess/timeseries.py b/cropclassification/preprocess/timeseries.py index ae21d6eb..89245a1b 100644 --- a/cropclassification/preprocess/timeseries.py +++ b/cropclassification/preprocess/timeseries.py @@ -67,6 +67,7 @@ def calc_timeseries_data( timeseries_periodic_dir=timeseries_periodic_dir, nb_parallel=conf.general.getint("nb_parallel", -1), on_missing_image=conf.images.get("on_missing_image", "calculate_raise"), + images_available_delay=conf.period["images_available_delay"], ) diff --git a/cropclassification/util/mosaic_util.py b/cropclassification/util/mosaic_util.py index e91bbf64..d551e9d3 100644 --- a/cropclassification/util/mosaic_util.py +++ b/cropclassification/util/mosaic_util.py @@ -161,6 +161,7 @@ def calc_periodic_mosaic( output_base_dir: Path, delete_existing_openeo_jobs: bool = False, on_missing_image: str = "calculate_raise", + images_available_delay: Optional[int] = None, force: bool = False, ) -> list[dict[str, Any]]: """Generate a periodic mosaic. @@ -201,6 +202,8 @@ def calc_periodic_mosaic( - ignore: ignore that the image, don't try to download it - calculate_raise: calculate the image and raise an error if it fails - calculate_ignore: calculate the image and ignore the error if it fails + images_available_delay (int): number of days to wait before the images are + available. force (bool, optional): True to force recreation of existing output files. Defaults to False. @@ -241,9 +244,12 @@ def calc_periodic_mosaic( # Make sure band information is embedded in the image for image in images_from_openeo: if image["path"].exists(): - raster_util.set_band_descriptions( - image["path"], band_descriptions=image["bands"], overwrite=False - ) + if not _is_image_outdated( + image=image, images_available_delay=images_available_delay + ): + raster_util.set_band_descriptions( + image["path"], band_descriptions=image["bands"], overwrite=False + ) # First get all mosaic images from openeo _ = openeo_util.get_images( @@ -268,6 +274,27 @@ def calc_periodic_mosaic( return periodic_mosaic_params +def _is_image_outdated(image, images_available_delay: Optional[int] = None) -> bool: + creation_date = datetime.fromtimestamp(image["path"].stat().st_ctime) + # Check if the creation date is different from the current date + if creation_date != datetime.now(): + # Check if the creation date is greater then or equal to the end date + if images_available_delay is not None: + day = datetime(creation_date.year, creation_date.month, creation_date.day) + new_date = day - timedelta(int(images_available_delay)) + else: + new_date = creation_date + if not new_date >= image["end_date"]: + # Delete the image + logger.info( + f"Image {image['path']} is older than the end date " + f"{image['end_date']}: deleting it." + ) + image["path"].unlink(missing_ok=True) + return True + return False + + def _prepare_periods( start_date: datetime, end_date: datetime, From 78570238f6970ba04a5cfc1f4ba0e9b10ee941b0 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 3 Jun 2025 15:00:06 +0200 Subject: [PATCH 02/14] added test is_image_outdated --- tests/test_mosaic_util.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_mosaic_util.py b/tests/test_mosaic_util.py index 741990aa..26d6dc19 100644 --- a/tests/test_mosaic_util.py +++ b/tests/test_mosaic_util.py @@ -1,5 +1,5 @@ import shutil -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path import pytest @@ -233,3 +233,32 @@ def test_prepare_mosaic_image_path(): "/tmp/s2-agri-weekly/s2-agri-weekly_2024-01-01_2024-01-02_B01-B02_mean.tif" ) assert result_path == expected_path + + +@pytest.mark.parametrize( + "end_date, is_outdated", + [ + ( + datetime(2024, 3, 11, 0, 0), + False, + ), + ( + datetime.now() - timedelta(days=1), + True, + ), + ], +) +def test_is_image_outdated(tmp_path, end_date, is_outdated): + # Prepare test data + image_path = SampleData.image_s2_mean_path + test_dir = tmp_path / image_path.name + shutil.copyfile(image_path, test_dir) + + image = { + "end_date": end_date, + "path": test_dir, + } + + result = mosaic_util._is_image_outdated(image=image, images_available_delay=3) + assert result == is_outdated + assert image["path"].exists() if not is_outdated else not image["path"].exists() From d230b2cd18005ddaa66fcefaeeeca0e229bb434f Mon Sep 17 00:00:00 2001 From: KriWay Date: Wed, 4 Jun 2025 08:11:31 +0200 Subject: [PATCH 03/14] add sleep to be sure timestamps are different --- tests/test_mosaic_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_mosaic_util.py b/tests/test_mosaic_util.py index 26d6dc19..9730f34f 100644 --- a/tests/test_mosaic_util.py +++ b/tests/test_mosaic_util.py @@ -1,6 +1,7 @@ import shutil from datetime import datetime, timedelta from pathlib import Path +from time import sleep import pytest @@ -253,6 +254,7 @@ def test_is_image_outdated(tmp_path, end_date, is_outdated): image_path = SampleData.image_s2_mean_path test_dir = tmp_path / image_path.name shutil.copyfile(image_path, test_dir) + sleep(1) # Ensure the file has a different timestamp image = { "end_date": end_date, From edc6e2f5484af2fcc99ede84d7d4d169bfc90644 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 23 Nov 2025 14:32:50 +0100 Subject: [PATCH 04/14] Fix linter issues --- .../preprocess/_timeseries_calc_openeo.py | 10 ++++------ cropclassification/util/mosaic_util.py | 10 ++++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index 3f7b3967..922dd0ff 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -27,7 +27,7 @@ def calculate_periodic_timeseries( timeseries_periodic_dir: Path, nb_parallel: int, on_missing_image: str, - images_available_delay: Optional[int] = None, + images_available_delay: int | None = None, force: bool = False, ) -> None: """Calculate timeseries data for the input parcels. @@ -53,13 +53,11 @@ def calculate_periodic_timeseries( - calculate_ignore: calculate the image and ignore the error if it fails images_available_delay (Optional[int]): delay in days for images to be available force (bool = False): whether to force recalculation of existing data. + (will not redownload images) """ info = gfo.get_layerinfo(input_parcel_path) if info.crs is not None and not info.crs.equals(roi_crs): - raise ValueError( - f"parcel crs ({info.crs.to_string()}) not equal to roi_crs " - f"({roi_crs.to_string()})" - ) + raise ValueError(f"parcel crs ({info.crs}) <> roi_crs ({roi_crs})") if not shapely.box(*info.total_bounds).within(shapely.box(*roi_bounds)): raise ValueError( f"parcel bounds ({info.total_bounds}) not within roi_bounds ({roi_bounds})" @@ -74,7 +72,7 @@ def calculate_periodic_timeseries( imageprofiles_to_get=imageprofiles_to_get, imageprofiles=imageprofiles, on_missing_image=on_missing_image, - images_available_delay=conf.period["images_available_delay"], + images_available_delay=images_available_delay, force=False, # dont redownload on force ) diff --git a/cropclassification/util/mosaic_util.py b/cropclassification/util/mosaic_util.py index 131b8c20..dac32c82 100644 --- a/cropclassification/util/mosaic_util.py +++ b/cropclassification/util/mosaic_util.py @@ -162,7 +162,7 @@ def calc_periodic_mosaic( output_base_dir: Path, delete_existing_openeo_jobs: bool = False, on_missing_image: str = "calculate_raise", - images_available_delay: Optional[int] = None, + images_available_delay: int | None = None, force: bool = False, ) -> list[dict[str, Any]]: """Generate a periodic mosaic. @@ -203,8 +203,8 @@ def calc_periodic_mosaic( - ignore: ignore that the image, don't try to download it - calculate_raise: calculate the image and raise an error if it fails - calculate_ignore: calculate the image and ignore the error if it fails - images_available_delay (int): number of days to wait before the images are - available. + images_available_delay (int | None, optional): number of days to wait before the + images are available. Defaults to None. force (bool, optional): True to force recreation of existing output files. Defaults to False. @@ -275,7 +275,9 @@ def calc_periodic_mosaic( return periodic_mosaic_params -def _is_image_outdated(image, images_available_delay: Optional[int] = None) -> bool: +def _is_image_outdated( + image: dict[str, Any], images_available_delay: int | None = None +) -> bool: creation_date = datetime.fromtimestamp(image["path"].stat().st_ctime) # Check if the creation date is different from the current date if creation_date != datetime.now(): From f8d67a148a102ac8799c407381d0ebe6dee88c1a Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 30 Apr 2026 07:33:55 +0200 Subject: [PATCH 05/14] expect float16 in stead of float32 --- tests/test_raster_index_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 12e3e0ab..8f6ac23f 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,20 +236,20 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float32", np.nan), + ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), ( "ndvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["B04", "B08"], - "float32", + "float16", np.nan, ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float32", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float32", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), ], ) def test_calc_index_by_gdal_raster( From 0811a8a121561ebcd61f0f3ac324449e0da11fd3 Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 30 Apr 2026 07:56:47 +0200 Subject: [PATCH 06/14] undo the changes --- tests/test_raster_index_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 8f6ac23f..12e3e0ab 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,20 +236,20 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), + ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float32", np.nan), ( "ndvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["B04", "B08"], - "float16", + "float32", np.nan, ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float32", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float32", np.nan), ], ) def test_calc_index_by_gdal_raster( From 6e8b0b97a9a19fa90ca6acb22b019a5121c03fc4 Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 4 May 2026 10:39:45 +0200 Subject: [PATCH 07/14] redo changes --- tests/test_raster_index_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 12e3e0ab..8f6ac23f 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,20 +236,20 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float32", np.nan), + ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), ( "ndvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["B04", "B08"], - "float32", + "float16", np.nan, ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float32", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float32", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), ], ) def test_calc_index_by_gdal_raster( From b4b70b994ea3007a4341f49d6761caf7452a295d Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 4 May 2026 13:15:39 +0200 Subject: [PATCH 08/14] pinned version rasterio and tests with minimal python 3.12 --- .github/workflows/tests.yml | 4 ++-- ci/envs/latest.yml | 2 +- ci/envs/minimal.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e7c2268..17443b7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: matrix: os: [ubuntu-latest] dev: [false] - python: ["3.10", "3.11", "3.12"] + python: ["3.12"] env: ["latest"] # Use openblas instead of mkl saves 600 MB. Linux OK, 50% slower on Windows and OSX! extra: ["nomkl"] @@ -36,7 +36,7 @@ jobs: - env: minimal os: ubuntu-latest dev: false - python: "3.10" + python: "3.12" - env: latest os: macos-latest dev: false diff --git a/ci/envs/latest.yml b/ci/envs/latest.yml index 5cc83c7c..336634fb 100644 --- a/ci/envs/latest.yml +++ b/ci/envs/latest.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio + - rasterio >=1.5 - rasterstats - rioxarray - scikit-learn diff --git a/ci/envs/minimal.yml b/ci/envs/minimal.yml index 9d8f287a..2d5d1f75 100644 --- a/ci/envs/minimal.yml +++ b/ci/envs/minimal.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio + - rasterio =1.5 - rasterstats - rioxarray - scikit-learn From 94645468352ed7dfa057e1c3c07d62566d99f675 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:04:19 +0200 Subject: [PATCH 09/14] use python 3.13 for macos-latest test --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17443b7e..96f92e30 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: matrix: os: [ubuntu-latest] dev: [false] - python: ["3.12"] + python: ["3.10", "3.11", "3.12", "3.13"] env: ["latest"] # Use openblas instead of mkl saves 600 MB. Linux OK, 50% slower on Windows and OSX! extra: ["nomkl"] @@ -40,7 +40,7 @@ jobs: - env: latest os: macos-latest dev: false - python: "3.12" + python: "3.13" - env: latest os: windows-latest dev: false From 16c7b7402e97b648f167c6eaa5a6faff4442c514 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:05:02 +0200 Subject: [PATCH 10/14] skip tests when certain rasterio version --- environment-dev.yml | 2 +- tests/test_raster_index_util.py | 92 +++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index be31ac47..93f95dbc 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -16,7 +16,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio + - rasterio <1.5 - rasterstats - requests - rioxarray diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 8f6ac23f..197e2861 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,8 +236,19 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), - ( + pytest.param( + "ndvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["B04", "B08"], + "float16", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( "ndvi", "FLOAT16", gdal.GDT_Float32, @@ -245,11 +256,84 @@ def test_calc_index_s2(tmp_path, index, pixel_type): ["B04", "B08"], "float16", np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( + "ndvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["B04", "B08"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), + ), + pytest.param( + "ndvi", + "FLOAT16", + gdal.GDT_Float32, + np.nan, + ["B04", "B08"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["VH", "VV"], + "float16", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_Float32, + np.nan, + ["VH", "VV"], + "float16", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["VH", "VV"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), + ), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_Float32, + np.nan, + ["VH", "VV"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), + ), ], ) def test_calc_index_by_gdal_raster( From 77090f6d3e94f8219aa9666497a25880088d638d Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:45:22 +0200 Subject: [PATCH 11/14] rasterio not pinned --- ci/envs/latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/envs/latest.yml b/ci/envs/latest.yml index 336634fb..5cc83c7c 100644 --- a/ci/envs/latest.yml +++ b/ci/envs/latest.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio >=1.5 + - rasterio - rasterstats - rioxarray - scikit-learn From 0b61113902f6fa2da028bd3e3cca4ec2f5073399 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:52:43 +0200 Subject: [PATCH 12/14] set minimal python version to 3.10 --- .github/workflows/tests.yml | 2 +- ci/envs/minimal.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 96f92e30..b5fc907c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: - env: minimal os: ubuntu-latest dev: false - python: "3.12" + python: "3.10" - env: latest os: macos-latest dev: false diff --git a/ci/envs/minimal.yml b/ci/envs/minimal.yml index 2d5d1f75..9d8f287a 100644 --- a/ci/envs/minimal.yml +++ b/ci/envs/minimal.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio =1.5 + - rasterio - rasterstats - rioxarray - scikit-learn From e7507089a14a49cceacfbd68f532c93b98c828f7 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 12:20:36 +0200 Subject: [PATCH 13/14] refactor yml --- .github/workflows/tests.yml | 4 +-- environment-dev.yml | 2 +- tests/test_raster_index_util.py | 45 ++++++++++++++++++++++++++++----- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5fc907c..a010a076 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,11 +40,11 @@ jobs: - env: latest os: macos-latest dev: false - python: "3.13" + python: "3.12" - env: latest os: windows-latest dev: false - python: "3.12" + python: "3.13" steps: - uses: actions/checkout@v6 diff --git a/environment-dev.yml b/environment-dev.yml index 93f95dbc..be31ac47 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -16,7 +16,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio <1.5 + - rasterio - rasterstats - requests - rioxarray diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 197e2861..fd6d8c26 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -126,11 +126,38 @@ def test_calc_index_invalid(tmp_path): "index, pixel_type, process_options, expected_bands", [ ("dprvi", "BYTE", {}, ["dprvi"]), - ("dprvi", "FLOAT16", None, ["dprvi"]), + pytest.param( + "dprvi", + "FLOAT16", + None, + ["dprvi"], + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), ("dprvi", "FLOAT32", {}, ["dprvi"]), ("rvi", "BYTE", {}, ["rvi"]), - ("vvdvh", "FLOAT16", {}, ["vvdvh"]), - ("sarrgb", "FLOAT16", {}, ["vv", "vh", "vvdvh"]), + pytest.param( + "vvdvh", + "FLOAT16", + {}, + ["vvdvh"], + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), + pytest.param( + "sarrgb", + "FLOAT16", + {}, + ["vv", "vh", "vvdvh"], + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), ("sarrgb", "FLOAT32", {"log10": True}, ["vvdb", "vhdb", "vvdvhdb"]), ("sarrgb", "BYTE", {"log10": True}, ["vvdb", "vhdb", "vvdvhdb"]), ( @@ -269,7 +296,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), pytest.param( @@ -281,7 +309,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), @@ -319,7 +348,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), pytest.param( @@ -331,7 +361,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), ], From 6f2bdc6c61b4f6ff4eb8037f5b30ec1977c39c61 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 12:38:41 +0200 Subject: [PATCH 14/14] refactor yml --- tests/test_raster_index_util.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index fd6d8c26..b7e64022 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -238,7 +238,26 @@ def test_calc_index_s1_error( @pytest.mark.parametrize( - "index, pixel_type", [("ndvi", "BYTE"), ("ndvi", "FLOAT16"), ("bsi", "FLOAT16")] + "index, pixel_type", + [ + ("ndvi", "BYTE"), + pytest.param( + "ndvi", + "FLOAT16", + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), + pytest.param( + "bsi", + "FLOAT16", + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), + ], ) def test_calc_index_s2(tmp_path, index, pixel_type): # Prepare test data