From 05f39785e27045481131daab95af08967e210622 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 8 Apr 2025 12:56:54 +0200 Subject: [PATCH 01/40] use exactextract --- cropclassification/preprocess/_timeseries_calc_openeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index 11439b3d..fe900fe5 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -91,6 +91,6 @@ def calculate_periodic_timeseries( rasters_bands=images_bands, output_dir=timeseries_periodic_dir, stats=["count", "mean", "median", "std", "min", "max"], - engine="pyqgis", + engine="exactextract", nb_parallel=nb_parallel, ) From fd1592f05714b2a81913c0e0783f9f2d44091b5b Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 8 Apr 2025 14:40:54 +0200 Subject: [PATCH 02/40] Test --- cropclassification/preprocess/_timeseries_calc_openeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index fe900fe5..6aa18cd9 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -91,6 +91,6 @@ def calculate_periodic_timeseries( rasters_bands=images_bands, output_dir=timeseries_periodic_dir, stats=["count", "mean", "median", "std", "min", "max"], - engine="exactextract", + engine="exactextract", # "pyqgis" nb_parallel=nb_parallel, ) From 5b0d6eda7a83e569f5ec083ac8bb35c77abea259 Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 10 Apr 2025 12:51:51 +0200 Subject: [PATCH 03/40] use exactextract for zonal stats --- .../preprocess/_timeseries_calc_openeo.py | 2 +- cropclassification/preprocess/timeseries.py | 2 +- .../_zonal_stats_bulk_exactextract.py | 40 ++++++++++++++----- tests/test_zonal_stats_bulk.py | 16 ++------ 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index 6aa18cd9..fe900fe5 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -91,6 +91,6 @@ def calculate_periodic_timeseries( rasters_bands=images_bands, output_dir=timeseries_periodic_dir, stats=["count", "mean", "median", "std", "min", "max"], - engine="exactextract", # "pyqgis" + engine="exactextract", nb_parallel=nb_parallel, ) diff --git a/cropclassification/preprocess/timeseries.py b/cropclassification/preprocess/timeseries.py index 7441fe8b..171ac0f0 100644 --- a/cropclassification/preprocess/timeseries.py +++ b/cropclassification/preprocess/timeseries.py @@ -161,7 +161,7 @@ def collect_and_prepare_timeseries_data( continue # Determine the columns to be read from the file and which to rename. - info = gfo.get_layerinfo(curr_path, raise_on_nogeom=False) + info = gfo.get_layerinfo(curr_path, raise_on_nogeom=False, layer="info") columns = [] columns_to_rename = {} for column in info.columns: diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 2ee98f90..cb6a0e22 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -261,6 +261,17 @@ def zonal_stats_band_tofile( output_path.unlink(missing_ok=True) return output_paths + # In stats replace 'std' with 'stdev' for exactextract + stats = [stat.replace("std", "stdev") for stat in stats] + + # Add the operational arguments to the stats + min_coverage_frac = 0.8 + coverage_weight = "none" + operation_arguments = ( + f"(min_coverage_frac={min_coverage_frac},coverage_weight={coverage_weight})" + ) + stats = [stat + operation_arguments for stat in stats] + stats_df = zonal_stats_band( vector_path=vector_path, raster_path=raster_path, @@ -274,17 +285,24 @@ def zonal_stats_band_tofile( for band in bands: index = raster_info.bands[band].band_index band_columns = include_cols.copy() - band_columns.extend( - [f"band_{index}_{stat}" for stat in [stat.split("(")[0] for stat in stats]] - ) - band_stats_df = stats_df[band_columns].copy() - band_stats_df.rename( - columns={ - f"band_{index}_{stat}": stat - for stat in [stat.split("(")[0] for stat in stats] - }, - inplace=True, - ) + if len(bands) == 1: + band_columns.extend(stats) + band_stats_df = stats_df[band_columns].copy() + else: + band_columns.extend( + [ + f"band_{index}_{stat}" + for stat in [stat.split("(")[0] for stat in stats] + ] + ) + band_stats_df = stats_df[band_columns].copy() + band_stats_df.rename( + columns={ + f"band_{index}_{stat}": stat + for stat in [stat.split("(")[0] for stat in stats] + }, + inplace=True, + ) # Add fid column to the beginning of the dataframe band_stats_df.insert(0, "fid", range(len(band_stats_df))) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 6276a859..483eac5b 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -11,18 +11,8 @@ @pytest.mark.parametrize( - "engine, stats", - [ - ("pyqgis", ["mean", "count"]), - ("rasterstats", ["mean", "count"]), - ( - "exactextract", - [ - "mean(min_coverage_frac=0.5,coverage_weight=none)", - "count(min_coverage_frac=0.5,coverage_weight=none)", - ], - ), - ], + "engine", + ["pyqgis", "rasterstats", "exactextract"], ) def test_zonal_stats_bulk(tmp_path, engine, stats): if engine == "pyqgis" and not HAS_QGIS: @@ -58,7 +48,7 @@ def test_zonal_stats_bulk(tmp_path, engine, stats): id_column="UID", rasters_bands=images_bands, output_dir=tmp_path, - stats=stats, + stats=["mean", "count", "std"], engine=engine, ) From 89e7e6612c53a708824c5b3e8ac48d9ed7684731 Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 10 Apr 2025 12:53:02 +0200 Subject: [PATCH 04/40] buffer = 0 --- cropclassification/general.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cropclassification/general.ini b/cropclassification/general.ini index 2ba62990..563bce67 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -139,7 +139,7 @@ on_missing_image = calculate_raise [timeseries] # Negative buffer to apply to input parcels to account for mixels. -buffer = 5 +buffer = 0 # The maximum percentage cloudcover an (S2) image can have to be used. max_cloudcover_pct = 15 From ee6156a190f62459ea53c09ecaf77475d8db5e44 Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 10 Apr 2025 12:57:59 +0200 Subject: [PATCH 05/40] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65d27183..9a754e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ - Add support to generate vvdvh index (#151) - Add support to generate sarrgb images and apply enhanced lee despeckling (#157) - General small improvements, e.g. save randomforest models compressed,.. (#144) +- Use Exactextract for zonalstats calculation (#181) ### Bugs fixed From 46345737c241c70147f9961636415d2b988a9ead Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 10 Apr 2025 13:22:41 +0200 Subject: [PATCH 06/40] removed stats as parameter --- tests/test_zonal_stats_bulk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 483eac5b..b9297a01 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -14,7 +14,7 @@ "engine", ["pyqgis", "rasterstats", "exactextract"], ) -def test_zonal_stats_bulk(tmp_path, engine, stats): +def test_zonal_stats_bulk(tmp_path, engine): if engine == "pyqgis" and not HAS_QGIS: pytest.skip("QGIS is not available on this system.") From 703cbe3e74468c6cf52df5476014a773d29310cc Mon Sep 17 00:00:00 2001 From: KriWay Date: Wed, 7 May 2025 09:50:18 +0200 Subject: [PATCH 07/40] changed example --- .../_inputdata/Prc_BEFL_2023_2023-07-24.gpkg | Bin 122880 -> 122880 bytes tests/test_end2end.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/markers/_inputdata/Prc_BEFL_2023_2023-07-24.gpkg b/markers/_inputdata/Prc_BEFL_2023_2023-07-24.gpkg index 610d5b0503354498fd200aee8a9be4dcb0c5a56d..b792eebc213c0cde89d13be4dc94dc4725cc7b01 100644 GIT binary patch delta 644 zcmZoTz}|3xeS$Qj>O>i5R#gVQw3i!GQuRelbq#>XJjB4<%GlJ()I`tB)S&sU{`9;0 zjJj;RT3iwwf42)5Fn(iV4d`&5VLzSQm{CENZ%x(&&BM%&31;Gwhkmj-f~A0b@r>-( zE4Ote~3=9m6K+FNev8S3gd;%(lDS~OLIV<3%KRw%$ zF_;fk$@VjrjAo213=B*f)7h*U3#`l{f28f+%wtzcNB}(p zw~hxW08=cYE&Y4e^bOXGlVzoOt{=Rx1)&dBNrDaIDb6O>uH!-C49#KFxw#pot@f_` zvi&8JjoJGRb272n$PKg+rug&ol-Wl>Ha5%t-=i!4185^mAF2|djXv8;>=^qTh0Zf@ z@y}!6|HOZS{|Wy={%icZwu>|{F5@?4?p~$O%)rSE40i|>1f=-yGO&Yrzf?fH*HGT^ zU=Z&o1N-*({fzgyr;E*Dl(1TS$wPL&uZQfVB_8Ipy*f7Am#*OXSwoL{@n;gp2rlm>VTS9t$#N0 zeV$&!$rv>K9|xlVAF9IbXDk`b7+HAMFa%6zvt}%?x^>C=^OViZj_ceXR)qj9KlO5b zy~1ZUup=aXB3a4jpYY%1zs7%lyGR4$GJaG3pA4eR44epx56ENg zUZoGBx_+sEspH`gDi}fqZGYd-c%NHn@g)!0ImyfpULcDe&WD%hdGRLTvzoy XWdG!N$Vs Date: Wed, 7 May 2025 11:37:11 +0200 Subject: [PATCH 08/40] add force parameter to zonal_stats_band_tofile --- .../util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index cb6a0e22..851a8d31 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -136,6 +136,7 @@ def zonal_stats( tmp_dir=tmp_dir, include_cols=columns, output_paths=output_paths, + force=force, ) calc_queue[future] = { "vector_path": vector_path, From 6f9ce87cfef6a9b079bd93a85f3f4e79d8f668a9 Mon Sep 17 00:00:00 2001 From: KriWay Date: Wed, 7 May 2025 12:29:32 +0200 Subject: [PATCH 09/40] add force parameter to test --- debug.log | 9 +++++++++ tests/test_zonal_stats_bulk.py | 9 ++++----- 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 debug.log diff --git a/debug.log b/debug.log new file mode 100644 index 00000000..11be91b6 --- /dev/null +++ b/debug.log @@ -0,0 +1,9 @@ +[0507/122314.187:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122315.996:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122323.090:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122334.963:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122456.210:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122457.711:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122502.946:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122514.849:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) +[0507/122823.970:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index b9297a01..54529fb4 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -10,11 +10,9 @@ from tests.test_helper import SampleData -@pytest.mark.parametrize( - "engine", - ["pyqgis", "rasterstats", "exactextract"], -) -def test_zonal_stats_bulk(tmp_path, engine): +@pytest.mark.parametrize("engine", ["pyqgis", "rasterstats", "exactextract"]) +@pytest.mark.parametrize("force", [True, False]) +def test_zonal_stats_bulk(tmp_path, engine, force): if engine == "pyqgis" and not HAS_QGIS: pytest.skip("QGIS is not available on this system.") @@ -50,6 +48,7 @@ def test_zonal_stats_bulk(tmp_path, engine): output_dir=tmp_path, stats=["mean", "count", "std"], engine=engine, + force=force, ) result_paths = list(tmp_path.glob("*.sqlite")) From 8e717276829bc36914c70f8c37c7927b1a48b028 Mon Sep 17 00:00:00 2001 From: KriWay Date: Wed, 7 May 2025 13:22:30 +0200 Subject: [PATCH 10/40] refactor --- .../util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 851a8d31..023107bb 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -257,10 +257,10 @@ def zonal_stats_band_tofile( ) -> dict[str, Path]: # Init if all(output_path.exists() for output_path in output_paths.values()): - if force: - for output_path in output_paths.values(): - output_path.unlink(missing_ok=True) return output_paths + if force: + for output_path in output_paths.values(): + output_path.unlink(missing_ok=True) # In stats replace 'std' with 'stdev' for exactextract stats = [stat.replace("std", "stdev") for stat in stats] From 2f17bd25da3e87ef19fb765227b8453fc9dfe9d4 Mon Sep 17 00:00:00 2001 From: KriWay Date: Wed, 7 May 2025 16:00:08 +0200 Subject: [PATCH 11/40] refactor --- .../zonal_stats_bulk/_zonal_stats_bulk_exactextract.py | 7 ------- tests/test_zonal_stats_bulk.py | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 023107bb..d78d973c 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -255,13 +255,6 @@ def zonal_stats_band_tofile( include_cols: list[str], force: bool = False, ) -> dict[str, Path]: - # Init - if all(output_path.exists() for output_path in output_paths.values()): - return output_paths - if force: - for output_path in output_paths.values(): - output_path.unlink(missing_ok=True) - # In stats replace 'std' with 'stdev' for exactextract stats = [stat.replace("std", "stdev") for stat in stats] diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 54529fb4..01345452 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -11,8 +11,7 @@ @pytest.mark.parametrize("engine", ["pyqgis", "rasterstats", "exactextract"]) -@pytest.mark.parametrize("force", [True, False]) -def test_zonal_stats_bulk(tmp_path, engine, force): +def test_zonal_stats_bulk(tmp_path, engine): if engine == "pyqgis" and not HAS_QGIS: pytest.skip("QGIS is not available on this system.") @@ -48,7 +47,6 @@ def test_zonal_stats_bulk(tmp_path, engine, force): output_dir=tmp_path, stats=["mean", "count", "std"], engine=engine, - force=force, ) result_paths = list(tmp_path.glob("*.sqlite")) From e38cfdeed8350d10b7bca451619433532af43c23 Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 8 May 2025 16:08:16 +0200 Subject: [PATCH 12/40] refactor + add new test --- .../_zonal_stats_bulk_exactextract.py | 20 ++++++----- ...eekly_2024-03-04_2024-03-10_vvdvh_last.tif | Bin 0 -> 10971 bytes ..._2024-03-04_2024-03-10_vvdvh_last.tif.json | 31 ++++++++++++++++++ ...eekly_2024-03-11_2024-03-17_vvdvh_last.tif | Bin 0 -> 10986 bytes ..._2024-03-11_2024-03-17_vvdvh_last.tif.json | 31 ++++++++++++++++++ ..._2024-03-04_2024-03-10_vvdvh_last.tif.json | 31 ++++++++++++++++++ ..._2024-03-11_2024-03-17_vvdvh_last.tif.json | 31 ++++++++++++++++++ tests/test_helper.py | 10 ++++++ tests/test_zonal_stats_bulk.py | 23 ++++++++++--- 9 files changed, 163 insertions(+), 14 deletions(-) create mode 100644 markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif create mode 100644 markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.json create mode 100644 markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif create mode 100644 markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json create mode 100644 markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.json create mode 100644 markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index d78d973c..3183a08c 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -227,6 +227,14 @@ def zonal_stats_band( dst_dir=tmp_dir, ) + # Add the operational arguments to the stats + min_coverage_frac = 0.8 + coverage_weight = "none" + operation_arguments = ( + f"(min_coverage_frac={min_coverage_frac},coverage_weight={coverage_weight})" + ) + stats = [stat + operation_arguments for stat in stats] + try: import exactextract @@ -258,14 +266,6 @@ def zonal_stats_band_tofile( # In stats replace 'std' with 'stdev' for exactextract stats = [stat.replace("std", "stdev") for stat in stats] - # Add the operational arguments to the stats - min_coverage_frac = 0.8 - coverage_weight = "none" - operation_arguments = ( - f"(min_coverage_frac={min_coverage_frac},coverage_weight={coverage_weight})" - ) - stats = [stat + operation_arguments for stat in stats] - stats_df = zonal_stats_band( vector_path=vector_path, raster_path=raster_path, @@ -280,7 +280,9 @@ def zonal_stats_band_tofile( index = raster_info.bands[band].band_index band_columns = include_cols.copy() if len(bands) == 1: - band_columns.extend(stats) + band_columns.extend( + [f"{stat}" for stat in [stat.split("(")[0] for stat in stats]] + ) band_stats_df = stats_df[band_columns].copy() else: band_columns.extend( diff --git a/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif new file mode 100644 index 0000000000000000000000000000000000000000..5fcf6c23bf58ae7f2645253c41b8fb010f85825c GIT binary patch literal 10971 zcmeI2WmFtZ7obT(0tCX~fdGRA2oT&oxI=IV4#C}>Ns!=7Ah5Sw{R{{-+U%|6<8!sQ+mfj?_#2%WEG{P}Kj+ z7CQ(F<)1oqkYxgcu~1$i*$K&&p)XJs?g^sKqPM<`CI;T6bmF@AouS`Mny^o z69?oG1w}|kQcO|F+|A^xiJOVYYf4HXIX82AN(U2rb75*pDRl`oxz8GMD$3N9t|sPM?4bY1=crYoM931)(5phxP>4`oAs@*z zlo%Qmtbe2@2=zabfksl9f_ZLy6MI(ZQUOlU>>Ge`@_uywZ=>Zbh|EVJK&p z9OBjT@*dQ_;3&Il>NnN$aq=W2RGe{fv9U_B!NV+HiBxaU!$(I)kFFmAh6iC!aQMqz zXJL?+yWuvk!S3pdADz)tOe}1V%S+Kyc2Ikt-BSgY7})HXMY6yP zihn)Zb0D{{^!_s8{gV`6-->Q=y@&td=~I(W$!MK8)v4J+#hJ>)*p0@mqDq1*_1<~z zU2DpUc=^vtel6IvwJ?VSqD656i(_baI1rn`&tgNJHnvcZ2kd)JUKzIJe=pT@vl^VB zg)00Zx8oOL|Sjmp7&i{vf7Pp ztrccmN8?=my!UPLrs;f)@f<}MR)H!SL|~AS`HWlwG4Nq`ZpxHGOSsh;W(8=gnuNLD z9jhwbi+fd{Q9_9L9U)jEf4TnfPxP>+|UEvJesDbu4_T6ty(JdU()BwdtbA(uPrtsOQ!I9F$JrJB@<-ot$d zmt_KUfu#yc5$&7iQq8zA`6TY>Z&YOCu18`Q<_hdt$IG-$xlZ!QmOM(Y~Ed)w` zo>SgrzGB{lzx^I_lmYeeV>s6o3J*m02}Tu~z!jZ5V!QhqGNMM(U_~K@A((Nt;8Y^J zDWiS;H$vd#Z~DyorCMLiMaob?0OCTyh{G%c=L$zp51qVVYtya8qbBJ%VBunK-}v1R z+;`><*!}(|K1&0x5t!TdKSm+V>G6M@hPU`S6(+WwFvP$a=N;@1kO)LHtV`?6F&Yp6 z3NE~xNZd=MIeR-ZeJDqe%U0(hGqV@q6m7 z(pZanD^-MZr9+TJUF&iYbJi^3DcCC3m)?F5SOaPE#aF6aLz~z+BqG5u%g20I97v)P zYhW;YtT!2v-SjOqd*C>SZ}aW~tF9L4yGa-HY&+@b6t(KkGC&`e-YrEs^3LY(vZwxj zy16mDo4k$YZIRr(D-moT`4+*1MWt5 zld~51mCD;`dOw>5>u>f6m6qrO@4%2%H#XJi4nB`I|3t3H8#@#OSU3ywv$3C*mVkQ$ zOVRLb89y7S&XG%rV)iO;;>Q&ZHPAs{X^(UH?EXM9_bwoF**!{3^Tag7&`~+k$E#d@ z4%n_B?p%oOGiZu1H1p7>(ZfI)g?=+0@U)6R*rvng@QMKWwUe=#ZS0d5jNzu%qw@~J zD$-v$G|U4(4kqx~pM?1u^ocnRj=iMc@LskH4#NH(_HcA0bNFKvX^A zQDxffyL9_VSuQqdcKxIL+qzjwHr~)TZeOxoBL{mf#YBE#>Ok^#15^DM;ksHboNFN> z>Jw-~uj~~x!jg3aR`77ICRglJSFG!ElXL=>?ZZC?eYh9Ka262fmv0q!ncj51@6uvr z$KS7Ab9t+MB-@lVUU~$pR=LCp1R^33qMHa`vNO*n!2K=YGdE&@t5SJlO4U1FO}Tvd zHT!W3;(lnP8F~E6mn#q`{o~%S zd$;0fC<5C!H96ZvAMu0VH7v7fEm=EE_ES0{Ur7~e+JMRsQ!z6K)*gdsaPlq>`AJMl zK(icqDeggqa8mxxG1mG8w9hw>w{Hz~DI8dA=4MW^<_Cbu%8e6`KFpBaCA1t_?~Vk3 z#NzaG@u%T7by<@{Ck;!&EkA2*UcfuSxc zlWS|CNs8Pe26=82I>6^i>P)Z8D@30eXc(s@tWM zOiqCuv+5w`$B%bhq7QtF#d`fPhk-z{`;ckn zxTLuYTEauPr~|#B^rXabl#BLWfnbK{*Dt1TD&M?JaXoz}YQ$2m+@w(qrpY7l2e!KF{;gCU^go^|CS_5@REJJX_sc%be-H2^INW!yAlS5^I(Ubkf}`zFQZJ5udAFGR4h$!Pgp zrQNw||5(5SNQ637LLzP4EE8=OuxmJX-<{96LPxp!Y3nVp!#=B-9$pmn=2|}?n*Fe= z+OHT2*enj{`h^2!UNX5YK+-eM-jsM2?A+x#SsHC$r#)3*?8*&uD@B3&xgLtlFyB|-k zDs%^&gQC7i(djiepSO#65Yiyk`ccb`jk85(u0GI8i38zx9My+p=8hF-I6rN{$JN?0 z>g|yxvJsV-pY8X=Z0hb7iqLTIhr@k8$Wc8v$*9_}Dynd|&a3>&m1<|IVv0veuQw_> z6^hq*MkcZM#i6z~TgP7pAF+~T%Zf@=%PtAjcL;@6;^Q}pVaPetPU<0R_fh&0k@Ch) z3*`}RB_q~JVdVpFpsj~)?0R4T&7pD0c@4}ye{7|`xEtlKyl-1wqAe1P{{Gf&oSqpI zFv3qWA-yu}D_e9Hkh6bEAr|jmT+cXOb-wGbWs%BMF0Pv?g2k^;iQTU=t9en!HWA>T z?YQUuDUQAL%D0VU+LePCob+4_d|!QSeb~iXJ{VHXE2nxJXXjlNR}xTjx-GG*tzJ0L zSg~4vRy4<$!u1=@truASZa#M-22W;k+Yt`(^_`g48_g2zf{tiWp#)an-8xi6(s~)` zvRcz_M(q!fVZS5wnV z$b%b|X0M6_u~;657-N8E-3`Ef_Yo>`C`4lFaGI~{N*upMMlF}*k9MZM%NJl(pE9cn zMH{%i!Y-sZG4i2dxMzDd;klY}LQR^esM!nT-@Piq76X8B=QsJgwIQe6;QW4YjGA9s z-0(`${var?KF;P|14o&`yAw#Ko~$jIV~K1RwcBRH^xmGeXKUI%?1oKTY)%L0Rby0)0&ToHe4!2QY2WJKGMMQ-ZWq%HcX9lX6i2++lp#@{UbT0qtiBc;!j zWwibzgFYH+k*}>wZeu&CfoM#d+WGBdWHLK*3yoFG5)2&|N?=p+q1ZMp?tpfZ_;WeL ziSp{RJH$^k5{SkxK|gd=n4+2Kq6&X)kaO1_$e8BF`EkU3)b+(~YIff`k;D!X6P(`%wJNx@T z5G&}9cd2o-x4bPbu+y{B)SlR;LyYL*Xx5P zmoGyQ@lTGsuO{OWrR2`U7-dh=Mu!}re)WAHg)Z|{?2){p@Y|?XYd%It!7@E|Dp_tN zK>@G3R-IP7gjxsomANc*pV`&>9j@d+9a8HJsn-j1SZ>LDw@qp4r})Ps%jRs|1!`{U z=S8P?w#x5bdgQ8WVfR)4V0|TkbH&%)?DR0Z20Y-Sz4mqNUVMbDbiat168&>^LG$~K zchdK~k$4*Sp0%EJo5u(sNz#uxnZ%Z0(m>$Kbq8r)!01g6#U;2G<>r*Bc?%L2o*KZ_ zV&UFj|D=c?^24*g2>T)^w?o8vD<+!9_IDr3KiXr zesVi{q0A!O`Fat}LPzQ-NwZ%X`r656NoNa}E{60VKe6O>OxD>)J?^#u*`yHOIyf~8 zOCXR?6pYX6+n4yJKXR&aXJO5?&a)+rZSm&S_O4D z-X_o?9s&L$yZdr}^jsCHxrKH+91Yks%X0=?w*h1(^Fnf^trsR>?`!Km)+L03$-mmv z@VYBm7wVYt;+;KhX(_T>Q_C%yOB&S<|0Wt-7c}{%=k^Ir3_!I~!2g5u3Se$H3q0hE zC?kVgsEKP{j$M${SmZox)n4i2+hT_ArRU^Lp+b z2s%vkyn`F{llmilQs(@KFDSW;yW@oINeF{g)mVqS1B*&oE&GF-TmGDNW|~T$htop4 zdTA{F-&TGmMLwZCs{@2Y*{WI3AHJeL#AD_(Ia5Qf52Gr5;dsn=-Z0XqTL?l23jo&g z&gJurfIAHdA1H9w&c4Py8AA8j*kk$w1Y|T|a}nl^R_h-t#rMjBDg>b99=hm5}CVmrBUJtj&+;xiu1VC3ZCYjew@fwz`E z0k6&F_3;$@GZH#wrFDgVTom*bO5nfkpF>aEt>2gRLxBA1atz__B5ilqkDB@*dbdvN zHOxOYQR{V-lRz!8Y6qYjnGvAt7-eU^AnVvF%iT^p%rP>mEh9NtWLfzwMQ!^sKe`Ux z$4caWRRd4JmM?mn2~G> z5dHPOM`KrWXiGT{ZXQ)dl|i>M==9Z#Po+P$R^spRD|DZ~@_!^Rp7!3S4YsMokOq+w zf7(eMEu?SBv`!Z&ay2?2qAjfn<%;Pg?+OaOV8-NPY7cb-f+_dd?f*Rb|D4KXkSs30 z6U{Y% zk8au_kcPl9rEVRKV}$A~i&1Tn$_WcGRgN~mDBJ0GJ3l9%V0Y=Kl@56z`$v`^zjTt} z-wp1m1TEbQx@H$}-Ai$}b`aZ|^wDJp<3iy%VK6Cyp}$Y1Io!{ThL`q!PcW+^XuL`> zv}~r|<3Y6(hi>)5_Im^Ld9@g)k;5kLdecNpAl3@Ix-NXuF6*}wED!z|$#rm<#F)lr zy^4lG81d^p*Xp~+kfNF>{=d55;g+x*>nB$xL`Ty(MGd^+<(@UC*2aJ?0>Xx( zG=b1`d^GRgycHEe{7m(ejFY>S6(5%joIe{#@FfVp%MvO7Xj{^@XL%o@-gFY~>Y-cP zs^~o(P9X-~-K2xk@y|&}ZN&z{@noH-LYlUwBh2G(iP}-5Gxh@h*qn}&sQafDfk1H9j?kO!NxYJ` z)QTKyyAH|;chSkU_cPzS3WNmH&rcI8Vl#!m+{t*^vz{?w%G5~~H?ybcQSWW)cNpTT zKDAS+&2FtsjUKHx0+v`0&?@T7qidL|5=yCs4Qw2o;Z@89p1Jx3d`0T((7IYW$~~$} zUuQs#>RxCUTy#tX;eai^1MhHpHK%UfC8}S3HSYd-jD7G&@}L4iFR<5ykX$!)7k-Wr znfKXZ?5I3{Q<%heyR%uXZwr^q8dW$TLqOX{Q1jQ%{XO$VpF+X9W=Mt5#NL9=^1)qed z=Pgh0%f*>F`IVgxJr3R9oy#x_(F=mf#yftUJwr~sDIH@S9(2zUX0CkSnjb+WM@ zKzpX8rSB3=KVhV$KX9TKjUh?RY`tnfda#_3m9c@Y$?U5yE0Ip-TmG|`b<@i00q{G-6p%8TN${ zSU}bDWt*(~`C1v1KO^}?X(hLV83QUO^E&WUO8u#*0RG1B!62sV$?#40{IQ~YP`T)Y;u-!1^XPtlp2}rtlJs&_18~vyR9fCZ%l~4 zA9*~*y2dQ|?P*lL>HnC$sH(9?KY&H@vel2 z;Gyv>E_^w*o?Yv__;{u6ryRLv-GcVJv@+SQOm1GX`n$yI5%k=Lbj zc1AR|bZ#e(yWiM+6j+?YE7@nbmx?%D-0WyN_N1Xb+x4lTr+n3NDT2&XfjKXR+JKA3 zw3CtnF*;I?9@D@6Fnlo3E6Iw3?Yz#UWLg=sw4O@^uXSwdN?MZ7H+bLnd$yBDYl^5A@lDKo1Vwl8tr9th7fO&t! z_yLp{+5H}5SKpToV3Brj(BXw}Ph~9RCo757-^%tsp&1IZvSSYJx&s)4nEMBC`;i zCfp%1GAk}RtL54^n0$Bt=J=v@VyfTCw6Kk;r09n^a(RIswzIeM+#nI4VQY~dP&aUL z#B3;dZ5!enfGzXHf$_(C(c4E?o%hC)aAXtcNUxM}Phd{s!nGr6UcVKv>qgPy&$K9l zg|$@i^3&%z#K=m}eWAgh4nSPdFrGjC6=7}(_vZbTLDa!aHveg5Yrf06Pn6W%Yh82j z4OtjjMaV)wf&{>Ie^`nj?^udIYGU?)feN) zaaR?}Dic*GMs(LcZni)Dt_?XjJ9JdKsN;B+!b64{r^evC<0(?~Eiys7?6sgkGQ9lO zLf1|Bxm8vS_b;a^5X<}a+HN7jV-oCVSMqK#QwrTkPhW(EN)p z%r>fRDRy7n=R7@AV*dQL<|OP_iN=|4lcO9nnqV%X5&lpZu(iuU7zhkf^7eb6)+xHj zv-gjg&U<|@-}{#E(L9Pkt6%`@|;-^?V}^^2l>aL=b;Yi8J0~<-WzQEQ16IrTEkCmuNc~ym2*?dW8ycC ztwUsBHx!%X67PnnIMjLqZXY7<)Q zW!mT;+{6*&s0n6V&n?S$2RKSmrCu0G@qXxNmSHt#3o9IAUH3~0xPHGm*fe0)#EHA= zc(0d;skgl9BL{KU*ax<9O2@JG7lk%nWEaB~ZaTO@g7_fB{d4M{H8kU0M2#Kgs zbJI$Jgqwil21|#RMZ%VB1;#{1J|K-^^)zGef(onp%9{c9@0p48kcBm5rY14KSxH-T zSVvZo#q~n7Z$?%-<(GgYgMfCl-X$>XpzSyp%fA}5|7^%0n(_Hz%gCVaD9#TEO6o0O&t%`_00bXW z?Nv>U+&N|-EHWS-TNE%zP^an~0zdZYK4BA04{&|Y_}y%fHg^5~RHiD1|p)b7}qTt&p7{@6{C9+TO^?-G8sf6QNRLX!hWSS;H|i~XAs-mSOjG5YKs3P~r`Og*vg?|m)I9X@2@c6F>-Ieqqn zYJqK)=)B__@6b?%Z(8za!{#QNGnASr`WVYyd4^Fqt+u8Y=P2K&Gmfqz& z@&rZVt&OtW*UYeR=LLeZL&=C*@;Y`!m0@?}_j;Vqj-V!=KkH0=C;3~ zsvmuh;NHnG9UsozaOvYpl7jd~pVzyEQ$F&8_4cg(dSv|mB>2}&9bfx(An-P>$?v6b z^TfC--~4L2S~RN}jVlg6x%|{{-;W^y4=Gi80lTIkF}-qs|4s(Sxo_9vQq6WKfj}tG z%cdCH=55r4hf#x;aXm&Hmymo{z(FIJO~L%r-DktiisUcHB$d;AlF}dA!?V~K&pw7Q z2Lef9Aojs=u1@F%MP%Q7%ZEQrd|}OZS@j3(IhK>~$A@?Jnj?a&t9nNoyVM@!T+FvV zXZjj(%0czNt_pL#wfD#L^|Cy*Iv&w~{2rZhgI`xVKVj>ggjLyUFMUm-d!x=T9&I_H zTIGvIa_+?Ij-C$lq8bgZVyn*Kn4(LuV9BToj}(E@1p?Lh6{E2DEpBzliNJilboHgb zS|oX`w9o*SSD-#8g%v)=ZOr@@op{|XG>tFs-{|0{2s6$iquiUUegBW+S>GTfPxre4 z*<;M#en)`%zvq)M3QSD}U@2Yc&Gq8F824HwPM{dq?4gh*_N5>rOQ}Nd#apNNV*pG@ z-xjdo%{+ishGe5b?I!r1Ui50MT8n^l3r3x?TzDZYm6CE~`z;Tk|t^1jaGSnx=V>+VF9= zz3Zx(P!N9fVes?;@o2&TU@N4%fY5rP6r6|E`@rq$UfksXq+#AVAG!f(KDoa}74`>pmw0FcCedt zRG;lH|8%!o?6QHKJef4yIguCH)l|{rTzT1=a^@7P$@k&=4y?4BBQ&)7?CHukZ+#qG zw*u%Gy3KGR94T^MDs}B{`zzKMVeuUqP7`xrr|OJH;#mYu;{Og$l&)K z7L^*;DaUXhw0AA`yc!oQ+nI=Cu0F_-7MzEH|aTf)0Na!iksXF za#X437s)i3#T^cBMeqcpk-m;7#Gwq;q1B;R)T8El_tk74XdpVyU@@qpUw2idoqnmO z&$e|yP$PV+U7CXzR4D*i=+wUT;GR!Vmejx1GYAtB%jUV8?)=Lc2;?Mz=~AB+dx;c7 z13a(m4yk!X{iwfoZawTOQgr#D`Xjfon|jNM`zcn4#ve0;)!`YzP}1OZoQ;aKE)?3CJWq8fqL2H^ozIrfxzlC z40wm7KTX=eH$>eW-1pKGIcPn*O`*1OwI>X8giCIloFs2UcD|HB+J)y7KCEOR!jr6= z%6SD>8(?y-XHP%-+g77PUR7A2N4&2nvj9=D-nMO~r=+F_T%E*P7?R@o1byc`#dcw% zi=r@SI%8Go7P7B_ZDf9}+nU%&X1_(E(+L#ceh0adtu3k1vQs9Oh9u2iNKcODybAs1#t zRTO{Xzcs7`B=2x;u{GGpe>{PoydOC=Q3&nxJ(t&Zt2;H5NxD0`Y?daJE-Vt48sdKK zyo-C4t?0Vl>3YQ&V<$pco^`4)r{f}IP)Vh459O<`-x4P>3owCb*g{O8rD=+Q*6sTcHwTbYm8k^8rHEsDXRJl&QDUN-6}0SQ zfqlE0-|_crlc&Uyqr1(<-46#}a@%EzYMKQUozF}|6zJ10SA^?0q{yp~lZ2xLc&TOXNj zP~HwCD{^cWq6GmTqjqN9BE5Xo^0z)p%aQMg5kt{0b&yANw%LD_hC{- z>~Yi$ZjOePI*BB39L!>vK_Oo>CNeE3$a}IGA=0NOAwbYu_TiYn^L~!9PFgkU$KnW6hD|7x4)KW~!%Kmo^mY54S-8Ediu^*kuZLKRlZyWQ1 z0&8L_?uz8)#`ID>j(3O($B2^L0yg1&c8PLow;{5hoK`oCsmsp^LMe-6-P5*?@at({B)HML2Zun9wPe& zvJMvjM~ zzFF&^i@BMLxtdz5YCmh&Id!UbRh@qK?ov=-K|)7DLLxvydV!4e;+c4#*BAfAFaKrg zXFu}4{7(@V z@a%j~OZ^S_4_H{bn18c%cC`gM(TcmfTe#bZa{kj<97B}ze?RJfMnXb*_u+rS|FZwO z{}0A9oL2IA*LUGq?SJflmKXnt4I0uvm+%ii2k1ZcB}z>w@pFq8=ry5eNW@5}&$r|S zQVbmu)<4z*gz_KDOs6ZaNiE1r%}K4Tr9$lubh4!8;ZpVepI%=i&&=bMTZ!Z~1k%|N zf31A2rNuCS{tc@kJzb)@+&dW>nzna5iG)K#EVk5y@jug4tmLfZ+|F$mr>d=yTAKR@LOW#|Rxk?of1AI$3|RM(S(1 zND&{5cdX_R#k%U%fpU|Yk&cC>;EXD~@=?<;Yv zs&{lA;EbQ-?;5qAKJF{!%O~+Vx%4#EA2VCBhSID=qC@YrL{P+7Vy94!9nu|`Yo?~C z1rqIPUH9*ZoVe@F0TLRTSfTmv|K5MVkvFpSO5=mF6IuzLu5UgZmUk`JFXiLU!su2y za7Sy_liMg%pf;$*`2_6FUS6%SP??E5+6#*SjH>8ef7F|ZmKU$Qq(PTUYn>?zR4%cR zuAHAPhgn1l?blQeIpv7j2iklZ6#4!papOtG)_gt>S`Jkn(^y~xWR%R4Fp>CZWOVn5 zic1w95x;4f`$*}-RF|Dt`{GwJPPSd#kqQDW`l1xwdjNhPAPbKMVch`B5nlUM*DpX$ zcS|9Kdk8(eE{C6Gq-~oD5!`7Vma;2-y3U60Ub9RB7FE}eZoQ3~DMQO_vbB|2g#U>4 z>}P|et08rSk~qjQkC=`JZX{EfCM)r82v~=gJGL(qzODte*_GfwDKygiuWT47b;3qT zFU!o7nN5kxN^BPS@UEOi)Vma%8unbP;(-N1JnDIh=i%e8SMfG)y>1AJ(5l^C6vX=5 zfOs7#^Rd#|T%K#5mjw}0%qU_ObKM{xv}!BI^4KnT(Yc!Z_rg0U8%32nxq8}CS36Md zQtyNC=7&p{E8`=NUw8Op-L4jq@qM!pY)oyO<3)ub^JWQd=In>|FoHmUzS7;?HZtaPIDN)u{XYVij>`v+pW4>StI2SXMS{Euog`BY^aWWT{ zi=xWLYqf+r4Kug6y=$Czq()gYo=Wp7#sB=^D)r;yO&te_bB;y5^J-}_h2dQ>8MoL9 z1On{OA%Nqh3;`;H-UtZsCVdmcXW0m2uy?d3S$##%>i_t~KnMn%x-HN-tL)gtud7J6 znOonYGd3u^AKYYx#Yz@(p zd(Cf-A~s;*PCK0Tjet2;CzNGUfJVml>%>Q7_{Z@O3rOl3<$&1$;#IC-!wYnwQb`>vG z8_+?AI@R)6Gu&N|@_hON{#D~hU-%^)sUFQ|P4W0cU)n?<+RB2ir*nPm+KaYjbuCW# z())K0-uSpGX&D3kb>($pxLTU(BB)()4s@pL#y3FH$6^- zj`)6d)7J;>ds}S{f0#aSK9k{t=F^^5yFVkpdy;<(FCmvTN$s-0sFIkMiVdICHvCuG z7Psr4$K%5o6R*}I)kw`*f#}0@uD?EXZs^|4CDya{Z3oMIt!lCL^|i~nXBy>WNi;;i zZ`91jSivG2tA;l4c822XC^RA2?e6tu_fCS$Ms#l(E|p}mrr1jrwpO}xj5+g&O8ntf zh|eD-O_AM z9=dm8MvV`{Rir;*q2Wz*h8T%<6XedaYd%&3nTJ@7)~EBQ7`TLODBBM0z5mCruWdew z`|5=657|Dy2D1#ejlB}6k4-Py(%_*BPu1?hg?DP^USI-6f4Cy!(wONoy%N?;#h|=) z`6kC#_9xOY-nOUvE%=?}RHyx$TOkG2UggJCZ)@1Q6->$4kAEZ8{MjFW$SG!5L4HX} zCb|XvlIjk_?>?}Yx@3P-bR(SQ5u80eSkA0oGW&uiFtZn&Zr!CuZcNg@qLk|<*CSaq_;8_oUP_C%DTy3~Hbv_@Rqc7uAdk7Whvp9o;V zJQ!_Jj_fNoA3fVR&Q3c{{@jI4xWAQ*MOM;Jr~P`^=UF+Yc9JUWxR}uDYjnH#>$i)A z!EN=mH+P*#Lvm}nGB>KxwHPa~tgehwdqi3jzO{Cb00sFYyDz=}yw$k2f>OFEF+mkYfnQmWugmY1I4z76xIK4-DQA@v^eaVAlDH0MJR zD^}aqishaJ^ZbRmuCHny^-CY1d@QRb5iv~lc%1aPV6jz*5OH#Ci%KJV$pt9!U@=_% z*5pjRe@?r+jvXiZtEjw=Nz4q76O3QB4oF88+eq70gBGrxnq_ghleqH}E2JdLZ<}DO zvI>t5vW-OMgf8C{h>zSD{e}$t%~$jwqKSE<5o(zcZte=2x(QCH#F}QNv^adTzdDN= zT&&tHSVHU)o57+>qUtC7kubqGg9}kIu?q}iR4rZT-tX{!yq4qFP{So&ZgzUIwe3}m zmEZ_qux#hs{4rKyA85Lgwfu@zUPGa3D3q_IbDa7MQ<|HUCj4l#W4MSjnQs7t;m5jW z#vXoc>ZvUrAM;~`T4Balp6xzAKF&jA4>~_m(CDc+Q_orM3EhlXCC@+VCt)LWfeh($ z6aC8y2yHnnQxu{|t%46`5HbD3${?E(XCwEpO`!Q|{hQ9^?aK&zQRU>BL*;XY@?ZUc z{-yCZ<*S4~;spaufra;AMoRWStgFz4An^{!ym$`u^7~96WAngddvovm@E_n6LARpv z&Hk34l{A?O86O$o7)#IDqS3Q(_3c|f70*hHTWM1a^ghChEJ+z!OycnI8xr8fI# z6=U(Yc!cDa7IBUJ-20<^JiYgpXHgO(2e6%wjc%=bXCr=wmMXu?fvJNp#wr9$BGk~#2b}C~d3FI;J<_yYj7Il%&hW*d;B9)tko%g;9mlr) z09Ml1BB#=yuK)|lfgYJ28XN;I*y*X#Ls6>cBgeFv@w6jJ&7bziIYSOZM_LislRCSS6;l4&= zP0UKaD|zGkTTv9T#TY=;DoMP$(#&$_@wh=ln?zK9?C|kxCdxdyc^I-@?!=p#uwVE9@b}fvbw-<3b&%GPCuWD+x5UwgvPPH>v z-sOeHdi}GuC1-o=#e-Q-9-09#Q3*En^c$Qhtt3coa_)@Vnt#_Nq`i%TAc*!TbAu zQeA@C5g?I(Xsq>ys=xJc5*S~W6?4}6xYXphhjiW6?EMB22o-RX;WbtHM-0%itLV4s z-f~2NXj2%(xGpy4%cqracSdUzu{(wMA8GM36~*YWalQ0YZ%@s7XIMTE9uNt(CY!9W z3O-!cIMuJTz^l8Y14N4ZZ)`)b!F-P)IsIeSH2}^x@w_socw}DHZ{H52n8m-s!)a>T z`)Hj7o|YkW=SjcAjz8A4PrC4OmslrJnXl>heV%Ka=x-`s3GA-j!XWw*pe>N2Ka$Eh zXt60H_IRJB4U@YkkOgm6prhb0Azoq~AFCo;hoXlhM@PNIx#T<7kX|KkANP=mH(1t0 z{YCl(#87+H6z5|$fB5EXS5xD%>~zNdTF=vVPx(7>g-U-?#|Ph+G3GB`XViZ(XxIOG zu|3u3vBU8e-8+n0KmtP$%AVSP5_=0Q>P&TL)2*rbrFH*=iH(f7urCo$vESBNF1|_e zF$Xdfw^Kp76Q5939;qyAjxPN{i7l&yt-n!6J%=l* z=@7xe?u7Z!Y`+5mE;^5P0>AE@1pP+lEu#wJe*cHSY2%=Sza`CMnt=zpxK}yEtGZ^s zA0szHMI`BGnsgO+gjs+iH)P9#nGEM_anyHGp1b%aBg4qa2|`q&N>EW{K#RutHDe&) z8zSl!p?q&9Pz$Oau$rJFPXTp4!QAiy0bO>BK@_MB6j}~gK4dH9s(p_V)!ju_?4@fk z7U`>@jt5ijT*cVsta;hjl*}HziKBRGA2<1F_RS);lnHWqaNTArZgQSj)h@|I`P}2M z)pvO=_sC5Qn!`p*QPn>YL>Z8wE@U6PZ<6?FguAaBog@E|Upif8%gAB}X@(=&Pfn#B z;IGnv!C;ygJd8a;g5Fd&i>-L7oIm_^y+e|e6_T(Fdp9eJCW+&9+}IC$xT6Ykn1LHs zJ>K8qqv^vgpOZ!^M1cO#ht-RPDM%@!VV3j{Q_m!QoW6~U_azdpNnwQCW`iltqX3v? z&2RUt@ss|~R_{8K$kHDW49Ew|c^1l$HDJnslf{n6sQ9jvc~cad^1JH#Q%(&`zUc&q z+KYmWG>;9d>C^aMHS40|RX=jjAjGd@M>Dqf7QU!AawY`%lz~V`-;HR^bv7uzNv&?< zDzZBC3Lly7k@ODpym}e5>3MwGo=Nrm8aP&5Rgv5pts$D=zt`r;>eEoe+FB!W9N-dG z)K#!4%fYGB#JAn|f&e~X!(uy~+FwpvOT##A14^N8xmj|ihO*1bz({zEwy;K>Nv>GF z&BG&PQz~u)c&C%Xoy8o0*b_SWAxjecT|NZ>@%eU(doN(zaZVR%?lOeFx$sz+M_2ll zW662dLh93Yg{4}+fM-vEt|*6#JeU!gAiOrW@;qGF zGF3?&?l{R>aYceX<#9e@j?=SIq!ilPCbNlB5MSkG1FL9`qU5XwL0l#n<>6nr&@D)@y+= zT?-kTTxvWiyEb!=lb+~KHf;sA54W%yE?5AylPb%H{SLRw^m?<4Bj<<+bYdrW9*GDj zX3lWy%@f+$S0>+46>}&+!=B@BkPMZx!m+BL(t~B-oC&}M{Atn09{wPu_aR)NUcg?{ zlxDWO+|Bum?5lj6TxhUkDFKviJRAhe*6Q2Hl#qFl`xr{)p)TN{8)n*__j$?cS$RXK z)ihBnw3=`1L_O~#EL)g{QYZBUX*;=sL}xIeT;s}vc`IFFEf`{sWOFz0pN~Ho*QG~& zUQ5Q(G|*ipG}(!S5oU${Ax_X1l>So19#)?`y=e541>>4ocR(~xwdDFivYmP_=I-TW zKyN1<3wG1@~Fc{8uNbp;aOAs&KbySb5N}zBM~>{x~+^`pT*Ij z5N|dxdyDIt&~~V*4bUF$A88W3&ad?vgO%imhh+8xjpQcsIH(Vmf@B&QFDraIdt5O^ zAZM3cyXu`B+k$57%Qe?;>`o&^XOb)G-B7!d3!2Dc5qj|e$*DEOepcQx0ngVW?XaNz zm>)Z_Z$t2Y#y!3VM_Jt0l8Q@wV-d6sjLM+@dGZuLdr@yFA656eiNHn}@zwDOa=i#? zxd!@OBD^2vh}NTV&{3a%m$8a&2eLY7IYq}ddDTAK#$>g9=9#$>4}ygRT}onN-X3!% zZ_k;dWdc;0hwHG0z{i(4)n!9+6Wznw*XY)e8dE@>&0a#WA{5Ob0{z?=zXBP#2wHgm zzSj@OW<@A%nUIm5hPsaz*YpDw57d=JGFGiHch;TaN=JbzUqe%)-sIKkXP6L;qPl#B zqHD{ZV3jzq8gmufBjr3X{b^qNvOQVBjAr!HbNb%I!36@}AAzeu)-ZnHo#AVA6&Ogt zkm4M#61dX7i6)cKKxz8%lgV$wnH0tOGRy1}o~$k*w$7r%j2R=n>rmB*vRgCp<&MkGd$ozY1r z<~R@Xf%jSq!9^2V2yN^I@0%(h;vP)?9qt5^G=9=7Jt(TH`Lv0p`8r(OJw~s>@X74u z{bso9YNs^D0O|03Z%ZOUf5V?Xh9Ks&0!Do{qOY%}?Vu;*!j&PvkXv4Fb6os2pAszK zY*!Cu#aym;qJ>{n#$D z&&6jWF>`=oO%t~)QzUUTlYe{Kk;jLM6FT*m{KD%<5^B9(LZQbaU~aXCf!th_`-SnA16%Ww zvixnJ-JL-(K9Zp(~ju8<_9;Ut|`95$thZG{4k0cHW>PLD&&XbXhdlG^g? zH$C4_#}Cv5!p~bg*HR4$3ycdfSetyCAL;C&GR-oVnf~tr0Wn77&9!cO84sl8v0=Ni z*fw#7s`!1X!Qropi)FLC2h3R6K_G_Q^&oXC1O@!VhE)D>q!|7`ndI29<22Hm$rIY; z`1OO~i=j=#haz*9nzD%qT=g#n8WHUXh0Kt`@=QgR7>{ldz+twY7qwSP^SP$e32W#N zb4|jkKa-ont_yfqf@CMjiMk7JpK9{96ZAhdMi^7h2)(+eQ2wQu=g@VzKSrB^T1K#G z9%b1PhYyri%tXCQD1X4cqHeEknsMbnvk0s%TvE&tUP&T!VK!yooA=@iI1>~{F3W5# z(*8!Kj&d@mAUMil^rcyIWHYAlA+&Z02i8SPUi96G72c-)gk0z7h^w-Nm?*&H0NY0u z;?&!WbZC>iRJ&{STzaOg-6ubghW~ts??&ljWrOuE(O`E9FhV-2Mr{mN*uNK^uW|Ue zg}`MXna4x%oI?d%Mtk?6PYw0PeS1vN3nU6uwT7?S%_TOwN+x-$^?bLEU-;}4X;s)5 zAR?pD5C70{g`G65`+CgYq7Q0Y4KdNosJ3DJEea4Ix}O~5bE{w*vZ3n1Bp}tbxf-wm zUBqQQmwB(BAPpT7sn;j4)QW!itowHm3vwPad}^_`V2E6SuG`Ku`7vG@ls#xx*7oow z78@k*W!y~S1^m;B}$a9nIY~*jpJI!aD!J7@N3`4U& zhcu1GF+Ye{r)95Hwc>h~UaNT87np|fT1h=krVs|`{t&exLjVlFsHUup_OJU`xI$|Y z!FwcmZPd(S8(#qj}^NlD2*B!HHPt+-M;i6 zdthAn1%~dbxK&I}=*L!+IWb%B+`O2}(Te5D9!1}kHrMmo3}}5~Gw2Z5CRk5bE8ktu z*}i`hWoQPA3GxmPQTtnuFmyM}g76^{krsqsIs1@&FWiGDdB!q}yBJP{DlrG~5wM4K z44)KfVLZzAuieHd$?YE0b+$6;{!Wl!SqMPqn?_7%l$JM$eb4DfukkZ3eQhP7>8#_o zy0wDJ7%c~8n_b<^W5|XTgpSA2m0bfg2OqH^Hb@IpkMr*;83gxG(sw0%+h2qDFEd&) z(zY8JTbu{e8pJSl;(b#i5=4oh@9vL;LV&e;>H64P69MnPS}w)^{3}K@Avs}Q2k*b> zkr-(V`XEsFYEJ1zm9*y5n%;U2u5?r_FTTvj{_ukmrT0T%xq+vui}IWHZ4dr;6SHq2 zY0UGi8(Nfgi{!$V-{|)%2+|J`1QN=OIP!9C!Fh~}_xIYb$yq8QFZ3Rmws|VpU|&kb zNtW^Ji_OJ++_gl5Y3r8W)k~k&Fr;4~mfro)Bf?ABTIS#brbG*~`L5)8?AdAcIcHaz zGRth@>y8$PlemEh_PbpuL^;|0mSP152~@64Et`yiI|y@rHIxOjD+MK-SSJk*sx z6vVsw9jR<)R^9&7O*`abf{!nStJK}s93$Y|*eQ^DV(zPK& zK8-OTIq?s0sj-kJ4+PXAuuq!!ksY%i0*s+J{ADP)tZ-N*T?O(cSTw3;gkIfKX@w_x z9N1pmy4=mBESa;IuWx##0g?F(*LVB~8koON>L4)MaA;C^&eRzm8V*5Te;cXPnsfGC z)Wg_Gm!KMqIapZMV8W!D~L(4p(q$iH#h$xyE-Awfkv;R)-iB7+6LM`^~c%3nX9 z?6DY?t5e;h$3dHE<&M z-(n4A*^Pc8s;jmbYSA`wvl0AwnLnKx;4K;mfNUYg0g#3xL=C|@mCXo3LUliZ`P6M* z+(qh#yis5<_|!Qo817H6=o2YE&cQC{OI*8P!a~O1ZjGXNOG`?|oKQ+0euR590J)M$pt7F@Z;GzQ)-l7uhw+;a2{<61TY@cCdFER^cZ+?RgGc2p?m}l*v#jHK+ zi^jW7V-M#NtKVjy zYHhRUe<->fWMO?XRtpHk)l0pr69g>nXfw$({`}K12{$M&2)x|s7x6Syvz06rVPF+U zxN0nhYSmeY!4m*%r$iMa^vT?PG7>{#1+TO`Hi6qmtcND>HO#K#yJanJx>TP2yJ_QY3$96h~< zmRS@6ttHL%4(`dB+a8;xZHF_HZb0b?spgw8a0421G{{ z`3HtD!xTD%x$?vK$`$ds%`1YiRc9po@{Wtpv~o>YIy*`vXma!xa#N2iazGaT@~3o{ zrjWD230?%}6=BO;LRDzeFm?I zyhH$ZNs4U5jm;t=S zJu9H>`>X_Ch|GC+oy$L8J^ugY|78SzBO1^`|M_?mDG&~E4|(-}H}-dV4Z(V({{c83 B%aQ;9 literal 0 HcmV?d00001 diff --git a/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json new file mode 100644 index 00000000..9087840d --- /dev/null +++ b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json @@ -0,0 +1,31 @@ +{ + "imageprofile": "s1-grd-sigma0-vvdvh-asc-weekly", + "collection": null, + "index_type": "vvdvh", + "max_cloud_cover": null, + "image_source": "local", + "base_imageprofile": "s1-grd-sigma0-asc-weekly", + "pixel_type": "FLOAT32", + "satellite": "s1", + "roi_bounds": [ + 161400.0, + 188000.0, + 161900.0, + 188500.0 + ], + "roi_crs": "31370", + "start_date": "2024-03-11", + "end_date_incl": "2024-03-17", + "end_date": "2024-03-18", + "period_name": "weekly", + "weeks": [ + 11 + ], + "bands": [ + "vvdvh" + ], + "time_reducer": "last", + "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-5/test_task_calc_periodic_mosaic0/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-asc-weekly/s1-grd-sigma0-vvdvh-asc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif", + "job_options": null, + "process_options": null +} \ No newline at end of file diff --git a/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.json b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.json new file mode 100644 index 00000000..3f0edb06 --- /dev/null +++ b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.json @@ -0,0 +1,31 @@ +{ + "imageprofile": "s1-grd-sigma0-vvdvh-desc-weekly", + "collection": null, + "index_type": "vvvh", + "max_cloud_cover": null, + "image_source": "local", + "base_imageprofile": "s1-grd-sigma0-desc-weekly", + "pixel_type": "FLOAT32", + "satellite": "s1", + "roi_bounds": [ + 161400.0, + 188000.0, + 161900.0, + 188500.0 + ], + "roi_crs": "31370", + "start_date": "2024-03-04", + "end_date_incl": "2024-03-10", + "end_date": "2024-03-11", + "period_name": "weekly", + "weeks": [ + 10 + ], + "bands": [ + "vvdvh" + ], + "time_reducer": "last", + "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-5/test_task_calc_periodic_mosaic0/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif", + "job_options": null, + "process_options": null +} \ No newline at end of file diff --git a/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json new file mode 100644 index 00000000..1422516f --- /dev/null +++ b/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif.json @@ -0,0 +1,31 @@ +{ + "imageprofile": "s1-grd-sigma0-vvdvh-desc-weekly", + "collection": null, + "index_type": "vvvh", + "max_cloud_cover": null, + "image_source": "local", + "base_imageprofile": "s1-grd-sigma0-desc-weekly", + "pixel_type": "FLOAT32", + "satellite": "s1", + "roi_bounds": [ + 161400.0, + 188000.0, + 161900.0, + 188500.0 + ], + "roi_crs": "31370", + "start_date": "2024-03-11", + "end_date_incl": "2024-03-17", + "end_date": "2024-03-18", + "period_name": "weekly", + "weeks": [ + 11 + ], + "bands": [ + "vvdvh" + ], + "time_reducer": "last", + "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-5/test_task_calc_periodic_mosaic0/markers/_images_periodic/roi_test/s1-grd-sigma0-vvdvh-desc-weekly/s1-grd-sigma0-vvdvh-desc-weekly_2024-03-11_2024-03-17_vvdvh_last.tif", + "job_options": null, + "process_options": null +} \ No newline at end of file diff --git a/tests/test_helper.py b/tests/test_helper.py index 10a1eaa1..973e25b5 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -37,6 +37,16 @@ class SampleData: / "s2-agri-weekly" / "s2-agri-weekly_2024-03-04_2024-03-10_B02-B03-B04-B08-B11-B12_best.tif" ) + image_s1_grd_vvdvh_asc_weekly_path = ( + image_roi_dir + / "s1-grd-sigma0-vvdvh-asc-weekly" + / "s1-grd-sigma0-vvdvh-asc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.tif" + ) + image_s1_grd_vvdvh_desc_weekly_path = ( + image_roi_dir + / "s1-grd-sigma0-vvdvh-desc-weekly" + / "s1-grd-sigma0-vvdvh-desc-weekly_2024-03-04_2024-03-10_vvdvh_last.tif.tif" + ) start_date = datetime(2024, 3, 4) end_date = datetime(2024, 3, 11) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 01345452..31742311 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -11,7 +11,8 @@ @pytest.mark.parametrize("engine", ["pyqgis", "rasterstats", "exactextract"]) -def test_zonal_stats_bulk(tmp_path, engine): +@pytest.mark.parametrize("bands, exp_results_path", [(["vvdvh"], 2), (["VV", "VH"], 8)]) +def test_zonal_stats_bulk(tmp_path, engine, bands, exp_results_path): if engine == "pyqgis" and not HAS_QGIS: pytest.skip("QGIS is not available on this system.") @@ -32,11 +33,23 @@ def test_zonal_stats_bulk(tmp_path, engine): # Make sure the s2-agri input file was copied test_image_roi_dir = test_dir / SampleData.image_dir.name / SampleData.roi_name - test_s1_asc_dir = test_image_roi_dir / SampleData.image_s1_asc_path.parent.name - test_s1_desc_dir = test_image_roi_dir / SampleData.image_s1_desc_path.parent.name + if bands == ["VV", "VH"]: + test_s1_asc_dir = test_image_roi_dir / SampleData.image_s1_asc_path.parent.name + test_s1_desc_dir = ( + test_image_roi_dir / SampleData.image_s1_desc_path.parent.name + ) + else: + test_s1_asc_dir = ( + test_image_roi_dir + / SampleData.image_s1_grd_vvdvh_asc_weekly_path.parent.name + ) + test_s1_desc_dir = ( + test_image_roi_dir + / SampleData.image_s1_grd_vvdvh_desc_weekly_path.parent.name + ) test_image_paths = list(test_s1_asc_dir.glob("*.tif")) test_image_paths.extend(test_s1_desc_dir.glob("*.tif")) - images_bands = [(path, ["VV", "VH"]) for path in test_image_paths] + images_bands = [(path, bands) for path in test_image_paths] vector_path = test_dir / SampleData.input_dir.name / "Prc_BEFL_2023_2023-07-24.gpkg" vector_info = gfo.get_layerinfo(vector_path) @@ -50,7 +63,7 @@ def test_zonal_stats_bulk(tmp_path, engine): ) result_paths = list(tmp_path.glob("*.sqlite")) - assert len(result_paths) == 8 + assert len(result_paths) == exp_results_path for result_path in result_paths: result_df = pdh.read_file(result_path) # The result should have the same number of rows as the input vector file. From f43273b6940884af99e64d20f2531577bb21b1b4 Mon Sep 17 00:00:00 2001 From: KriWay Date: Fri, 9 May 2025 10:21:55 +0200 Subject: [PATCH 13/40] refactor --- .../_zonal_stats_bulk_exactextract.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 3183a08c..525292af 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -227,14 +227,6 @@ def zonal_stats_band( dst_dir=tmp_dir, ) - # Add the operational arguments to the stats - min_coverage_frac = 0.8 - coverage_weight = "none" - operation_arguments = ( - f"(min_coverage_frac={min_coverage_frac},coverage_weight={coverage_weight})" - ) - stats = [stat + operation_arguments for stat in stats] - try: import exactextract @@ -266,6 +258,14 @@ def zonal_stats_band_tofile( # In stats replace 'std' with 'stdev' for exactextract stats = [stat.replace("std", "stdev") for stat in stats] + # Add the operational arguments to the stats + min_coverage_frac = 0.8 + coverage_weight = "none" + operation_arguments = ( + f"(min_coverage_frac={min_coverage_frac},coverage_weight={coverage_weight})" + ) + stats = [stat + operation_arguments for stat in stats] + stats_df = zonal_stats_band( vector_path=vector_path, raster_path=raster_path, From a73ec89bb42bc2ef26d6c10565b208026f123074 Mon Sep 17 00:00:00 2001 From: KriWay Date: Fri, 9 May 2025 15:28:13 +0200 Subject: [PATCH 14/40] refactor --- cropclassification/general.ini | 12 +++++++++++ .../preprocess/_timeseries_calc_openeo.py | 10 ++++++++-- cropclassification/preprocess/timeseries.py | 2 ++ .../util/zonal_stats_bulk/__init__.py | 10 ++++++---- .../_zonal_stats_bulk_exactextract.py | 11 ---------- tests/test_zonal_stats_bulk.py | 20 ++++++++++++++++--- 6 files changed, 45 insertions(+), 20 deletions(-) diff --git a/cropclassification/general.ini b/cropclassification/general.ini index 563bce67..c2f06fdc 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -138,6 +138,18 @@ on_missing_image = calculate_raise # Configuration on how/which periodic images/timeseries statistics should be generated. [timeseries] +# Engine to use for the timeseries calculation. +# Possible values are pyqgis, rasterstats and exactextract. +engine = exactextract + +# Stats to calculate for the timeseries. The following stats are available: +stats = count(min_coverage_frac=0.8,coverage_weight=none), + mean(min_coverage_frac=0.8,coverage_weight=none) , + median(min_coverage_frac=0.8,coverage_weight=none), + stdev(min_coverage_frac=0.8,coverage_weight=none), + min(min_coverage_frac=0.8,coverage_weight=none), + max(min_coverage_frac=0.8,coverage_weight=none), + # Negative buffer to apply to input parcels to account for mixels. buffer = 0 diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index fe900fe5..0ed9e75c 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -26,6 +26,8 @@ def calculate_periodic_timeseries( imageprofiles: dict[str, ImageProfile], images_periodic_dir: Path, timeseries_periodic_dir: Path, + engine: str, + stats: list[str], nb_parallel: int, on_missing_image: str, ): @@ -44,6 +46,10 @@ def calculate_periodic_timeseries( images_periodic_dir (Path): directory where the images are stored. timeseries_periodic_dir (Path): directory where the timeseries data will be saved. + engine (str): the engine to use for the calculation. Options are + "exactextract", "rasterstats" and "pyqgis". + stats (list[str]): statistics to calculate. Available statistics and + special options are dependent on the `engine` specified: nb_parallel (int): number of parallel processes to use. on_missing_image (str): what to do when an image is missing. Options are: @@ -90,7 +96,7 @@ def calculate_periodic_timeseries( id_column=conf.columns["id"], rasters_bands=images_bands, output_dir=timeseries_periodic_dir, - stats=["count", "mean", "median", "std", "min", "max"], - engine="exactextract", + stats=stats, + engine=engine, nb_parallel=nb_parallel, ) diff --git a/cropclassification/preprocess/timeseries.py b/cropclassification/preprocess/timeseries.py index 171ac0f0..cef2ceed 100644 --- a/cropclassification/preprocess/timeseries.py +++ b/cropclassification/preprocess/timeseries.py @@ -84,6 +84,8 @@ def calc_timeseries_data( imageprofiles=conf.image_profiles, images_periodic_dir=conf.paths.getpath("images_periodic_dir"), timeseries_periodic_dir=timeseries_periodic_dir, + engine=conf.timeseries.get("engine"), + stats=conf.timeseries.get("stats"), nb_parallel=conf.general.getint("nb_parallel", -1), on_missing_image=conf.images.get("on_missing_image", "calculate_raise"), ) diff --git a/cropclassification/util/zonal_stats_bulk/__init__.py b/cropclassification/util/zonal_stats_bulk/__init__.py index 00ee2c7a..30b72bd2 100644 --- a/cropclassification/util/zonal_stats_bulk/__init__.py +++ b/cropclassification/util/zonal_stats_bulk/__init__.py @@ -11,10 +11,10 @@ def zonal_stats( id_column: str, rasters_bands: list[tuple[Path, list[str]]], output_dir: Path, - stats: Union[list[str], str] = ["count", "median"], + engine: str, + stats: Union[list[str], str], cloud_filter_band: Optional[str] = None, calc_bands_parallel: bool = True, - engine: str = "rasterstats", nb_parallel: int = -1, force: bool = False, ): @@ -39,8 +39,10 @@ def zonal_stats( supported for engine "rasterstats". Defaults to None. calc_bands_parallel (bool, optional): True to calculate the bands in parallel. Only supported for engine "rasterstats". Defaults to True. - engine (str, optional): the engine to use for the calculation. Options are - "exactextract", "rasterstats" and "pyqgis". Defaults to "rasterstats". + engine (str): the engine to use for the calculation. Options are + "exactextract", "rasterstats" and "pyqgis". + stats (list[str]): statistics to calculate. Available statistics and + special options are dependent on the `engine` specified: nb_parallel (int, optional): the number of parallel processes to use. Defaults to -1: use all available processors. force (bool, optional): False to skip calculating existing output files. True to diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 525292af..8787a900 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -255,17 +255,6 @@ def zonal_stats_band_tofile( include_cols: list[str], force: bool = False, ) -> dict[str, Path]: - # In stats replace 'std' with 'stdev' for exactextract - stats = [stat.replace("std", "stdev") for stat in stats] - - # Add the operational arguments to the stats - min_coverage_frac = 0.8 - coverage_weight = "none" - operation_arguments = ( - f"(min_coverage_frac={min_coverage_frac},coverage_weight={coverage_weight})" - ) - stats = [stat + operation_arguments for stat in stats] - stats_df = zonal_stats_band( vector_path=vector_path, raster_path=raster_path, diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 31742311..b151a1d5 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -10,9 +10,23 @@ from tests.test_helper import SampleData -@pytest.mark.parametrize("engine", ["pyqgis", "rasterstats", "exactextract"]) +@pytest.mark.parametrize( + "engine, stats", + [ + ("pyqgis", ["mean", "count", "std"]), + ("rasterstats", ["mean", "count", "std"]), + ( + "exactextract", + [ + "mean(min_coverage_frac=0.8,coverage_weight=none)", + "count(min_coverage_frac=0.8,coverage_weight=none)", + "stdev(min_coverage_frac=0.8,coverage_weight=none)", + ], + ), + ], +) @pytest.mark.parametrize("bands, exp_results_path", [(["vvdvh"], 2), (["VV", "VH"], 8)]) -def test_zonal_stats_bulk(tmp_path, engine, bands, exp_results_path): +def test_zonal_stats_bulk(tmp_path, engine, stats, bands, exp_results_path): if engine == "pyqgis" and not HAS_QGIS: pytest.skip("QGIS is not available on this system.") @@ -58,7 +72,7 @@ def test_zonal_stats_bulk(tmp_path, engine, bands, exp_results_path): id_column="UID", rasters_bands=images_bands, output_dir=tmp_path, - stats=["mean", "count", "std"], + stats=stats, engine=engine, ) From fccb369b6de40b9c062115a6c3b5bfaeba4a62ee Mon Sep 17 00:00:00 2001 From: KriWay Date: Sat, 10 May 2025 08:53:01 +0200 Subject: [PATCH 15/40] refactor --- cropclassification/general.ini | 7 +---- .../util/zonal_stats_bulk/__init__.py | 6 ---- .../_zonal_stats_bulk_exactextract.py | 7 +++-- tests/test_zonal_stats_bulk.py | 29 ++++++++++++------- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/cropclassification/general.ini b/cropclassification/general.ini index c2f06fdc..124567c2 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -143,12 +143,7 @@ on_missing_image = calculate_raise engine = exactextract # Stats to calculate for the timeseries. The following stats are available: -stats = count(min_coverage_frac=0.8,coverage_weight=none), - mean(min_coverage_frac=0.8,coverage_weight=none) , - median(min_coverage_frac=0.8,coverage_weight=none), - stdev(min_coverage_frac=0.8,coverage_weight=none), - min(min_coverage_frac=0.8,coverage_weight=none), - max(min_coverage_frac=0.8,coverage_weight=none), +stats = count(min_coverage_frac=0.8,coverage_weight=none),mean(min_coverage_frac=0.8,coverage_weight=none),median(min_coverage_frac=0.8,coverage_weight=none),stdev(min_coverage_frac=0.8,coverage_weight=none),min(min_coverage_frac=0.8,coverage_weight=none),max(min_coverage_frac=0.8,coverage_weight=none) # Negative buffer to apply to input parcels to account for mixels. buffer = 0 diff --git a/cropclassification/util/zonal_stats_bulk/__init__.py b/cropclassification/util/zonal_stats_bulk/__init__.py index 30b72bd2..c02af049 100644 --- a/cropclassification/util/zonal_stats_bulk/__init__.py +++ b/cropclassification/util/zonal_stats_bulk/__init__.py @@ -29,12 +29,6 @@ def zonal_stats( output_dir (Path): directory to write the results to. stats (List[str]): statistics to calculate. Available statistics and special options are dependent on the `engine` specified: - - - "rasterstats": `rasterstats documentation `_ - - "pyqgis": "count", "sum", "mean", "median", "std", "min", "max", - "range", "minority", "majority" and "variance". - - "exactextract": `exactextract documentation `_ - cloud_filter_band (str, optional): the band to use as a cloud filter. Only supported for engine "rasterstats". Defaults to None. calc_bands_parallel (bool, optional): True to calculate the bands in parallel. diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 8787a900..7ee20894 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -238,9 +238,10 @@ def zonal_stats_band( output="pandas", include_cols=include_cols, ) - - except Exception: - raise + except Exception as ex: + message = f"Error calculating zonal stats {stats}: {ex}" + logger.error(message) + raise Exception(message) from ex return stats_df diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index b151a1d5..5f0ba2ef 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -25,7 +25,14 @@ ), ], ) -@pytest.mark.parametrize("bands, exp_results_path", [(["vvdvh"], 2), (["VV", "VH"], 8)]) +@pytest.mark.parametrize( + "bands, exp_results_path", + [ + (["vvdvh"], 2), + (["VV", "VH"], 8), + (["B02", "B03", "B04", "B08", "B11", "B12"], 18), + ], +) def test_zonal_stats_bulk(tmp_path, engine, stats, bands, exp_results_path): if engine == "pyqgis" and not HAS_QGIS: pytest.skip("QGIS is not available on this system.") @@ -48,21 +55,23 @@ def test_zonal_stats_bulk(tmp_path, engine, stats, bands, exp_results_path): # Make sure the s2-agri input file was copied test_image_roi_dir = test_dir / SampleData.image_dir.name / SampleData.roi_name if bands == ["VV", "VH"]: - test_s1_asc_dir = test_image_roi_dir / SampleData.image_s1_asc_path.parent.name - test_s1_desc_dir = ( - test_image_roi_dir / SampleData.image_s1_desc_path.parent.name - ) - else: - test_s1_asc_dir = ( + image1_dir = test_image_roi_dir / SampleData.image_s1_asc_path.parent.name + image2_dir = test_image_roi_dir / SampleData.image_s1_desc_path.parent.name + elif bands == ["vvdvh"]: + image1_dir = ( test_image_roi_dir / SampleData.image_s1_grd_vvdvh_asc_weekly_path.parent.name ) - test_s1_desc_dir = ( + image2_dir = ( test_image_roi_dir / SampleData.image_s1_grd_vvdvh_desc_weekly_path.parent.name ) - test_image_paths = list(test_s1_asc_dir.glob("*.tif")) - test_image_paths.extend(test_s1_desc_dir.glob("*.tif")) + elif bands == ["B02", "B03", "B04", "B08", "B11", "B12"]: + image1_dir = test_image_roi_dir / SampleData.image_s2_mean_path.parent.name + image2_dir = None + test_image_paths = list(image1_dir.glob("*.tif")) + if image2_dir: + test_image_paths.extend(image2_dir.glob("*.tif")) images_bands = [(path, bands) for path in test_image_paths] vector_path = test_dir / SampleData.input_dir.name / "Prc_BEFL_2023_2023-07-24.gpkg" vector_info = gfo.get_layerinfo(vector_path) From bd292e1d272bdc6147b79919458bb1482a0d973b Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 12 May 2025 08:36:25 +0200 Subject: [PATCH 16/40] refactor stats as list of strings --- cropclassification/general.ini | 10 ++++++++-- cropclassification/helpers/config_helper.py | 1 + cropclassification/preprocess/timeseries.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cropclassification/general.ini b/cropclassification/general.ini index 090e4a0d..911a0e67 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -143,8 +143,14 @@ on_missing_image = calculate_raise engine = exactextract # Stats to calculate for the timeseries. The following stats are available: -stats = count(min_coverage_frac=0.8,coverage_weight=none),mean(min_coverage_frac=0.8,coverage_weight=none),median(min_coverage_frac=0.8,coverage_weight=none),stdev(min_coverage_frac=0.8,coverage_weight=none),min(min_coverage_frac=0.8,coverage_weight=none),max(min_coverage_frac=0.8,coverage_weight=none) - +stats = [ + "count(min_coverage_frac=0.5,coverage_weight=none)", + "mean(min_coverage_frac=0.5,coverage_weight=none)", + "median(min_coverage_frac=0.5,coverage_weight=none)", + "stdev(min_coverage_frac=0.5,coverage_weight=none)", + "min(min_coverage_frac=0.5,coverage_weight=none)", + "max(min_coverage_frac=0.5,coverage_weight=none)" + ] # Negative buffer to apply to input parcels to account for mixels. buffer = 0 diff --git a/cropclassification/helpers/config_helper.py b/cropclassification/helpers/config_helper.py index a422467e..b5d088fa 100644 --- a/cropclassification/helpers/config_helper.py +++ b/cropclassification/helpers/config_helper.py @@ -143,6 +143,7 @@ def read_config( "list": lambda x: [i.strip() for i in x.split(",")], "listint": lambda x: [int(i.strip()) for i in x.split(",")], "listfloat": lambda x: [float(i.strip()) for i in x.split(",")], + "jsonlist": lambda x: None if x is None else json.loads(x), "dict": lambda x: None if x is None else json.loads(x), "path": lambda x: None if x is None else Path(x), }, diff --git a/cropclassification/preprocess/timeseries.py b/cropclassification/preprocess/timeseries.py index 6bad3c0b..032f4007 100644 --- a/cropclassification/preprocess/timeseries.py +++ b/cropclassification/preprocess/timeseries.py @@ -85,7 +85,7 @@ def calc_timeseries_data( images_periodic_dir=conf.paths.getpath("images_periodic_dir"), timeseries_periodic_dir=timeseries_periodic_dir, engine=conf.timeseries.get("engine"), - stats=conf.timeseries.get("stats"), + stats=conf.timeseries.getjsonlist("stats"), nb_parallel=conf.general.getint("nb_parallel", -1), on_missing_image=conf.images.get("on_missing_image", "calculate_raise"), ) From a1c9ec70af9325a7ccc92f50947d1b6ded6e19f6 Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 12 May 2025 11:44:44 +0200 Subject: [PATCH 17/40] new version of prc_befl_2023_2023_07_24.gpkg --- .../_inputdata/Prc_BEFL_2023_2023-07-24.gpkg | Bin 122880 -> 106496 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/markers/_inputdata/Prc_BEFL_2023_2023-07-24.gpkg b/markers/_inputdata/Prc_BEFL_2023_2023-07-24.gpkg index b792eebc213c0cde89d13be4dc94dc4725cc7b01..b3cca2d7f02f852a4f1a223cfd82dff5fa38d088 100644 GIT binary patch delta 1715 zcmZuye@s(X6u##@czyllrK?CmTZ)ru9Y5NFmV(p0V%JAUq0*w$8M3eDv6&nI0) z_$n3o4omYvVol9vc%Jm1gB9egACi(?)#cu@(7#Qx{n=7IQi1_VZg36Bx$HG200!lx z^0cB);g?^K^Rn%-3be2P7z?49dl5-Um12gjXD43-81E@DeBnap#k6aIBgyf=V_|P>ZS!+>HldJO91l%V)wXxZdd%htitE8 z+p2s<(-O$N6qz?b_pb1G1+UNMSn%ksDy32j%F*|VGEZDYC@=Tf>>lqNV2Cy!@IarU5N`O{rF7$|iYrvX*I}z1YiW1DTYmK}Y!K za+xG)s#>Lr>R^nKHVu&@PB8Es1%N)`ekW{angR#f&*W=RaNS%6if5>_X=;ZXvB5@! zPW1HjbbP}KSJu{$%@H3wdaun9*b|&wL%q#b1F4I>k(fI<-1GU_@<2)<)+ord@I>ZVcW5y2MI48))PVJw5@T3}e>d)Wy zdHNqTE5?pxHyGrGBC^j4o@I+E*M;7(T6QsS1u% zQJ-~BXjS~+e;BAabU(xjeU4j=UTR}tef{sRQ`6C-)6gSbdhpR(C6;w!WJ~vDa<&9o zwReYfj_Gp?0*fiHYyxXRk+sO_bh>Ky*45V6zF{&O%^U3Y7g|-pk!>2(7PF~VNB53R z)X~ieb%pOKzzpQiEsz$SQC8|iieF`YQYW3wO;T-(uYL3&RhsSQlG%)uCMrF>71Dq= zwvWE4L5J7Pb`lV&wkLAlw!>{$rD-4a(-E1sr%JS^CC!e+gwqcAf>~cwV5YaI(X_Ny zTaKR2Pt-c#;|L$8H|D#TuLsVWntLECeqnk!z@rF{((w0~j5dL3B~6jx^9WDizwsUX z8~z#pgsGg}2OT)z73%aLHjZth!)ZQr(E!_`9%V&2*~g z4XsG|k^Oi$B}1v$jig{f@{=;2cX8uM5%vS71uV?%wz-zM+u`l-R<>)~u9}sa`u={;a}Nxt`?jCY{<-v_XTH~c z-Pe6x*L^+rbI<+Y@_m8J_vzM#4IHP@Xh!09Fn;~;J9b@6V(d6gT+LqsDAE6j|7)(- zU(o;9PwKa^&)mK*_MNX=)@M|}B3+h$n%_z3@g1$hrQo*Q5M9{R*r1gI+dunhRL8qt zY}0n^{`yTR@TUHOp=Gh{3r>c&Z~S^xhkRmDP{)t@KaDb(Qp-)L8Hvdm$w_f3$)?*Z z+Kz|zVJu7F>9fjJ+u5|7tdsP6FJHdcaVb9NX8#W~`VTq=n%>s;887M5hlcqD1+}j` zKg{!7vE<+JlN{VMF6gYLzvhxgw=J+Jz)$;x-))*p{om_1a+$5RXXI}_t-*jP2N7Shcx{fKIkC%d4Zycoyt*DUNZ#h3h z8sWLLNiwy6d45HgddH> z$CMNoPR}bYiZMh?Ey#^9l-q7CH>RgVfvv^4w#wq1i9%>jiE7_>sZcg1voUX48QLpv z@iv-#g!xw$R$8tmwhCdA$>eLI=xVw}36RQk(5&j(%1SCrCik*XP*j;`%PFlinY}jX znpawKRbiLMR4psFlvn2Dc|%L0tfbTms~jqg<0m95Nuac-piG#gFpAYF>CbTvhw9Ji zPX|33(5XEneWQ0m z|9Pz?YUTs`2fw7XP~*8%ftK&4>>fGoY@kKbLLlL{AAY^coo9_R3(f>u$m?FvTFA|( z11+VyN3J>sW9P|aN_S+a8X4DPynG~IU(N1yYWTHj9` zP_MWCBT%XL13&rPJ;PwqSNThn21FXoUbWLi!5f6JpHcj5&fllC$o#Mt)lHvUAYhE; zqWZUjS^d|6srT*vZZDW*4ZPyD3HLN7!31VbB%wb`c>R!IZxr+304HZbr#3|yz$!c} z>Qo1qCHQT@LAcxKz;myw?azDGjgbZ`zJ~}AGr}0&LC~D9!XMQxuZSe1fqA1&0dv0J z6DF4<7_xT~y*F->H7O}6*=!@@hYLJ!=sc6R!5C7QYa8jGU`j|#NlZ^MrI;p8 zoH*H0HP=z+XfQ*ZylU#|^J7Cb_g-ze^q7oN)UG*3>0!SqZ3|Z)oEEBCnW8QSS1k;> zJ8h4eiF;mEuSYNiOzB00(Y34&K*~H@qa}mPd-n9pYGy((8_ZD5E9x!wA-b|cNnl=Snwd!x%yx1Ln77a|Fsq~r%sb(t=Jb&kS>_&yVu z=&pwbN)FwZ(kqtW8I&vs%m;IV{C-y3=e;DWi`PK7%K_(!XXhX=2dWXwaL7yX)Z+EO zVSv{^;2FVdE|^8p2M+SyxPSmSQ(7<|5B~u7&VW+eo-uKf2~~V$9j^#jeQ*lu^A>gf zddz_5?l{!(OL3A}@=jpS*|V*Dd@g2B>3fYW1KsvVc)kU>4T~O^r44YmtG2nD=Grq6 z$@4Zcx!<63=ULe}(<*{}L=hNgo)a}d>wKgogE_miz+9+V=9E7p+W!dV03U*R-3jJE z&kMdwl(|9F*jUlt54=%^p#TWz(M$+9&4&;Mv%f|#!{RI)5gZ78wujt z^}cUeHoo^Y13HBMcGg!SBJ7YI609jn2?=I0e!BK3Ub!)3&YL--1C$j_ErT2+ zT({;(3ob*}Y>R&}M7pS#b<=+W^GCw(Fg)XUC;eHgQ`Xu%Fe7{sjKEG#2Xp0cgPF?N zV6GTeFe7(>sh19>-aOIY1m+lbfSJ)7z#OMlhJYcDg@BQ+1hXRYJ_(!8W^8q&m z%)qA_)x*VLX5fh6yF~jNU`Bd3m=PTS)8KtzI`%S{)w~+a$e#vNZzY%xZvnHG77G0b zzzbw}J|6-$R71d&(nA`&3CzWB2AGk>2~Gzym3}mU{^P*R&_|cm^&k$+f%kxU%@F(` zm<~6ASxb3fnW=dR0!B9fl6t$n0%p<77W^8R5ljcOy7z*4od%}EZ-SZX8^LspOuN3! zfXGVb(9VF8E(OZUoYNp3h=c+SeGI1JhZohy!D%o(z8B2m8@)r-fA$yk`4a(V${z=F zU@MpoCW2`<4@^0knJO3hre9>$qf(&|4F#sC5KNCpfjOrW!JMKYU>02wn5ho{Gm=}u zbnL_h)xkgw252iyMU=Dl-m>Jm#relp@7Ug;{2M+aN_9HXnEokTZlXVFDY6>bK zV1!SA8OdZYi|Q`XZUu9I1z^6~EeD8y)xJ>N2u&q-ZZ3fffvtSyo1k>G|gnD7PP&Q*lff=mc{L-Uf5vU84PEFeBXpW&k6xE8{@?X3tFK zf;k0ugXu^ljv8o}4zd=We;)f(-cCm$pl3r~REsDabEKpU+dyU{9W_D6cyvI++rZTS z0L=b}!OYZAFsJUI;5|Of{!@j18j6sPU$+bURHf)3U?%iXU<9kdxEO=L?D#O417CuJ z)O!@n2u_2k{{&b`Ihc+d2D9H6VA}Z-O!;rY%+z??W$eFEhJX>oK)^_w!Hi%Um_=m< zGxd*wnUQEPQ@RDrDH{vsK))CLPvAI-1Mxn`W$A-p&h;=b2OJBgqffyhIwrq}Ibp-i z5OCn%f!Q$^Ob51r>DV+d2RtHp6_}a$7|aD{EtuEC7=Wo=f&rPC@o3;`qB2muFPEfk`$$Kyatz?8=cZV{Xz zI7+0Pn^5Xs22;NY%z>U1{2iD9x&@cxU`M9lymtT`UC{s!DcP|SOhb%-ycNvIKEdOd z*TXt>fP-SwS%i~3>T{RNYn$LCF#GeQkJmFeSz`OjJ~~utN*s>S+j=wpT@mtzZsh#511*%tB!@OE-!=!dO-=er^V% ze75VsbSM+d2>ro~FcnOPv|t7@9!z;3Fdf_a9M*q&_5=oGA+Co5%IXB>E%hLnjy(&e z;rqdiU?-R<{|%URu@_87+QH1++hE#V24*G(BZJiAPMVIp!JLv2&td&%gyY4W{3O1N zYz8w$oKp_;B$$TIfO+iz_pSkc3*{ksk|j6Qnv?52JWuE?H-=0+{?5mK>K9Yb|Bjc2 z>}b7D8qq)KQB9!VN16fd{^Ns(2VD&4^!v!~BYlQGgD)$B9`#@TpO+Oq4yL?${g+-= z1U;&~;Xf}cR#oHtBUH2ciZe`~S6?ji-{7dZQySbRG? z;OTj5qo6c9CG@)e*a>($h0dLS@<}|XdDD$V`>w~vNyGj6_fYq|5h+c&{#PC^%YFSc zv;UpP%WXF($JHy8R7gVyw+&OcZQ<3AuZxw}f?R8Oe*M+17p6Di{CfSIQ=bgNX&o24 z9_QD5e_Zi)+jFO(>{ar1y5rrD3Gp6Rv{cr=hl=NDv{dw~PMNo2_4YIVJCBpwrYgtN z>O0XsLwg*Kdf$nrO#iv&W$f&+5XUPg!T0{Xq%;rFbCP?UPxGayS9?fw*oBRr+he5_ zv0PAnPng5{9ekDMz-~i0IWP zf7taGH6kUT`FqWE&Z`B~7Xf*d&zz{pFmx1Rch|k5^3=XoasY8Nx46H=J}nj?)^4n&SLzMiblf>uJ7wB671sJk3 zv&|H$xktG_dXKz50?LOXq^W+a%yV2~Qp9VLe`U#W|M9EprvGorxqk8ay)&R>8*BgD z=U8&Qo$7bw?Yr;7uJz-j2*3V4l$3=gxIeQTfaDa+D`oJY?CzuiTMGEOiDVPIQff?C5 zU_OYN!CX%CFvwJwfI09@V9whCF#8oipY4Znu)uzyf;Y<$Fm(r!A~qC*8Cf})1GRv; zb-f-;!|#F_$yaZv`r&vYa)4uC4)_?D8JH+i{s-~IT!(gV!LWTcm{TK1p@KNakq~fh zOkhTISgY3z>Itmn6+{o%oN@M=9GAO8JL+QGvei7 zI`$!$_U{EVkoUo~^Dwwp#y|%kphsIE;DFBvehN&3o4|BvJD7S8gE_!PFb&-eW@Jn$ zGt&m9d=;1uJ^*I%Efwt>guWXr)6j1rpkX@%jP!RxVFH-nFgYNLB@)bm?g7)$a4@HY z129sqQY_kNFl(YsOp#W!`$M1O9Ti2M)QRt3;N5KyaDbsuU}{H#>2Xje{#T2?&QLc) zJ(kC=?YGUEIKi5dOU6&vd$?pFH-^mW{H0!b50{>nkj7s;%5(D67pp_XUl(vy)(&`6 zqkm9u9`NRl*4+bQoZ7$}HIa)(*EY?st*m#r?bUX-z0&2Vakv}}RgNBF&w*koHp#2x za@eb57rC5nN2On2^b7fo^?`S`q0<~qrHDQSRSm^uhi(D)vwea+0nXCUpPJh z-bp=dUBfx8AplPFRPE8v&vT?wGLO^0rwP_?*62^`59zI6Sy4Wb) z*4NW`yA2Gi-BR}=50L+-4}ZdG1?{`6D& zcGykTl}&ckv>RQzl&!@@W#y$7^fgqu>RpxfHO|VK`GyI_rM7~+BEuxxbc3;8*f{m= z&2)OK_a@$DYfmOp-1#yY7pSPJ7`v(A#+`Wkhik&)7hsaIyP zM_R(KoiPJE^G0Xzaj?F!X}+PPv|zHObh;9jrOaw5DlRt^O)V^pL5R&&?PV7S%7$Fq z1k2RIaziB7fo6w0QfYMC8>;QD>dF?EQ~7&YpRP!ErF%(}qo=&vj@z-=9lyA~4u-_) zaxKlss5Za+=v})vOZig2{`zSeNjFHN8>Abwqcu<`ug&q(X~MF?A4?XV&2_t*GUDUw zT~$?aRgLxW3u>Lsan?|_-3bBnFz(@SgywirsL<`foK4UyOC7HNnqn_gCKn;e-P zoT>C!=x{YV8ym7B65|rmOc4oW}uuYwEA7N(v z6(Z`@*&AwG?6nS9==w-G^O-He%#2q^R^(219hKe6l@j}$X1B{;<%Y29qhi_FFuyro zakR$aXhySgVH1}1#%gCnEo5CEDX(%hQy-t2o9u3StENr0lcXudnUq_lXVwY2@y5hH_hEM>R{i}S3d4~7l9-cW>|jScoXLyn`a)?qNG zrzS+i*a`~^O3K>JVdD}CZAE$I`BRGu%4bBNI9!g17;{BMMGWiIHhD%wp3C8ASmdml z8xdoQu@vPM+S*?VGsw^=$ej^kD=Ev1h)GH`CqjTyzr|KsR#03tBZ8wj?G1)Pd;J`T z%WdGOlWpbs#pvXfSsNRiRR;XOhGEqij0t5WQ4v0N)6+}|bj4C?L0?-byCUr^)s7K{ z#u`J7y{_3|XmmMioedE&=|1ii`+Ap_mVk(QSZQpiRm@yzB`Gd3B{eB6Mf{|8hdiy9 z&1-l@ljdf(0n49qyBbh*_PRPpT|`W>InGSGsR`W5Bqny-p3=)Uvw4-MX*;2z`f9e5 z(&N$;qv_pNxG^3be5-MUWw-%Jmh;Rtkjkls+2rFBAD=nC6c4Ynd?YMCX^HUt0epu~oPN98m)949SUlc~N zIVL5J6V!cI;igJ8Ra9ig^9ENI1uP3quEu#u+0ndY)m2p)qU-GLN>f(437;Ap@X4ou zWu@3u6?0ba>Qm(jS>2Umh+Yf>-Lr3qUIO-+HAA$cuCA#$%h}+rGeoyEINi-z^@iw% z#>#3eQ&?XW=e({q&&JYSot==9WXg8-?Do$`TqkG7`-;43bj4AnEPYk3#)c*J*~pgaVNbEQ10r)aH8(n|?LuGi zA=Q+SfF-(jo3~S~qfr(!UsGqV%~n>V%y@`>rHVEJ^fFxU#(KN6VX}C>EB1W>wKF-l zz%bvjBr9S{UO`!Wux7?XP4%w3duDvzwT8hap%ee~WR*t0O1}#K7HNrI zpXnDAnvQQ$lBVn5{21{LC@I#I9BWD`H=8p|ri_F{tOH56@n1q@QMpgI^6;xFo04w( EKSJuNY5)KL From bbafed8901f230512582b122abebef78f6d9e7e7 Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 12 May 2025 12:48:00 +0200 Subject: [PATCH 18/40] add zonal_stats_bulk_invalid test --- tests/test_zonal_stats_bulk.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 5f0ba2ef..9a3e27b5 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -93,3 +93,22 @@ def test_zonal_stats_bulk(tmp_path, engine, stats, bands, exp_results_path): assert len(result_df) == vector_info.featurecount # The calculates stats should not be nan for any row. assert not any(result_df["mean"].isna()) + + +def test_zonal_stats_bulk_invalid(tmp_path): + # Prepare test data + sample_dir = SampleData.markers_dir + test_dir = tmp_path / sample_dir.name + shutil.copytree(sample_dir, test_dir) + + vector_path = test_dir / SampleData.input_dir.name / "Prc_BEFL_2023_2023-07-24.gpkg" + + with pytest.raises(ValueError): + zonal_stats_bulk.zonal_stats( + vector_path=vector_path, + id_column="UID", + rasters_bands=["vvdvh"], + output_dir=tmp_path, + stats=["means"], + engine="exactextract", + ) From ca64d33bfff3f764d313886582f9660e93e2bf2a Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 12 May 2025 13:59:06 +0200 Subject: [PATCH 19/40] min_coverage_frac = 0.8 --- cropclassification/general.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cropclassification/general.ini b/cropclassification/general.ini index 911a0e67..c1cacbcc 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -144,12 +144,12 @@ engine = exactextract # Stats to calculate for the timeseries. The following stats are available: stats = [ - "count(min_coverage_frac=0.5,coverage_weight=none)", - "mean(min_coverage_frac=0.5,coverage_weight=none)", - "median(min_coverage_frac=0.5,coverage_weight=none)", - "stdev(min_coverage_frac=0.5,coverage_weight=none)", - "min(min_coverage_frac=0.5,coverage_weight=none)", - "max(min_coverage_frac=0.5,coverage_weight=none)" + "count(min_coverage_frac=0.8,coverage_weight=none)", + "mean(min_coverage_frac=0.8,coverage_weight=none)", + "median(min_coverage_frac=0.8,coverage_weight=none)", + "stdev(min_coverage_frac=0.8,coverage_weight=none)", + "min(min_coverage_frac=0.8,coverage_weight=none)", + "max(min_coverage_frac=0.8,coverage_weight=none)" ] # Negative buffer to apply to input parcels to account for mixels. buffer = 0 From 260216c1e04223dd1c755299c33eaa50b254cc02 Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 12 May 2025 16:23:22 +0200 Subject: [PATCH 20/40] refactor zonal_stats_bulk_invalid test --- .../_zonal_stats_bulk_exactextract.py | 2 +- tests/test_zonal_stats_bulk.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py index 7ee20894..6d7dbb00 100644 --- a/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py +++ b/cropclassification/util/zonal_stats_bulk/_zonal_stats_bulk_exactextract.py @@ -241,7 +241,7 @@ def zonal_stats_band( except Exception as ex: message = f"Error calculating zonal stats {stats}: {ex}" logger.error(message) - raise Exception(message) from ex + raise ValueError(message) from ex return stats_df diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 9a3e27b5..5e0ac217 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -6,6 +6,7 @@ from cropclassification.helpers import config_helper as conf from cropclassification.helpers import pandas_helper as pdh from cropclassification.util import zonal_stats_bulk +from cropclassification.util.zonal_stats_bulk import _zonal_stats_bulk_exactextract from cropclassification.util.zonal_stats_bulk._zonal_stats_bulk_pyqgis import HAS_QGIS from tests.test_helper import SampleData @@ -103,12 +104,11 @@ def test_zonal_stats_bulk_invalid(tmp_path): vector_path = test_dir / SampleData.input_dir.name / "Prc_BEFL_2023_2023-07-24.gpkg" - with pytest.raises(ValueError): - zonal_stats_bulk.zonal_stats( + with pytest.raises(ValueError, match="Error calculating zonal stats"): + _zonal_stats_bulk_exactextract.zonal_stats_band( vector_path=vector_path, - id_column="UID", - rasters_bands=["vvdvh"], - output_dir=tmp_path, + raster_path=test_dir / SampleData.image_s1_asc_path, + tmp_dir=tmp_path, stats=["means"], - engine="exactextract", + include_cols=["index", "UID", "x_ref"], ) From cfa604900884d8e754b164e4df7420701a5128e5 Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 12 May 2025 17:04:00 +0200 Subject: [PATCH 21/40] addressed review comments --- CHANGELOG.md | 2 +- cropclassification/general.ini | 3 +++ cropclassification/preprocess/_timeseries_calc_openeo.py | 6 ++++++ cropclassification/util/zonal_stats_bulk/__init__.py | 5 +++++ debug.log | 9 --------- 5 files changed, 15 insertions(+), 10 deletions(-) delete mode 100644 debug.log diff --git a/CHANGELOG.md b/CHANGELOG.md index eed44ea7..fc00c104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,7 @@ - Add support to generate sarrgb images and apply enhanced lee despeckling (#157) - Add POC (not for operational use) of a cover/bare soil marker (#168) - General small improvements, e.g. save randomforest models compressed,.. (#144) -- Use Exactextract for zonalstats calculation (#181) +- Use Exactextract as default engine for zonalstats calculation (#181) ### Bugs fixed diff --git a/cropclassification/general.ini b/cropclassification/general.ini index c1cacbcc..b311b748 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -143,6 +143,9 @@ on_missing_image = calculate_raise engine = exactextract # Stats to calculate for the timeseries. The following stats are available: +# - "rasterstats": documentation: https://pythonhosted.org/rasterstats/manual.html#statistics +# - "pyqgis": "count", "sum", "mean", "median", "std", "min", "max", "range", "minority", "majority" and "variance". +# - "exactextract": documentation: https://isciences.github.io/exactextract/operations.html stats = [ "count(min_coverage_frac=0.8,coverage_weight=none)", "mean(min_coverage_frac=0.8,coverage_weight=none)", diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index cc5e2de0..4d72b28c 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -50,6 +50,12 @@ def calculate_periodic_timeseries( "exactextract", "rasterstats" and "pyqgis". stats (list[str]): statistics to calculate. Available statistics and special options are dependent on the `engine` specified: + + - "rasterstats": `rasterstats documentation `_ + - "pyqgis": "count", "sum", "mean", "median", "std", "min", "max", + "range", "minority", "majority" and "variance". + - "exactextract": `exactextract documentation `_ + nb_parallel (int): number of parallel processes to use. on_missing_image (str): what to do when an image is missing. Options are: diff --git a/cropclassification/util/zonal_stats_bulk/__init__.py b/cropclassification/util/zonal_stats_bulk/__init__.py index c02af049..3d36ccc6 100644 --- a/cropclassification/util/zonal_stats_bulk/__init__.py +++ b/cropclassification/util/zonal_stats_bulk/__init__.py @@ -37,6 +37,11 @@ def zonal_stats( "exactextract", "rasterstats" and "pyqgis". stats (list[str]): statistics to calculate. Available statistics and special options are dependent on the `engine` specified: + - "rasterstats": `rasterstats documentation `_ + - "pyqgis": "count", "sum", "mean", "median", "std", "min", "max", + "range", "minority", "majority" and "variance". + - "exactextract": `exactextract documentation `_ + nb_parallel (int, optional): the number of parallel processes to use. Defaults to -1: use all available processors. force (bool, optional): False to skip calculating existing output files. True to diff --git a/debug.log b/debug.log deleted file mode 100644 index 11be91b6..00000000 --- a/debug.log +++ /dev/null @@ -1,9 +0,0 @@ -[0507/122314.187:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122315.996:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122323.090:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122334.963:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122456.210:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122457.711:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122502.946:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122514.849:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) -[0507/122823.970:ERROR:registration_protocol_win.cc(108)] CreateFile: Het systeem kan het opgegeven bestand niet vinden. (0x2) From af936dcbb31c50a34729070d0b67fc2a403598c8 Mon Sep 17 00:00:00 2001 From: KriWay Date: Fri, 23 May 2025 13:00:01 +0200 Subject: [PATCH 22/40] added test exactextract --- tests/test_zonal_stats_bulk.py | 90 ++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index 5e0ac217..f911d6bd 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -1,7 +1,9 @@ import shutil import geofileops as gfo +import geopandas as gpd import pytest +import shapely from cropclassification.helpers import config_helper as conf from cropclassification.helpers import pandas_helper as pdh @@ -112,3 +114,91 @@ def test_zonal_stats_bulk_invalid(tmp_path): stats=["means"], include_cols=["index", "UID", "x_ref"], ) + + +@pytest.mark.parametrize( + "min_coverage_frac, exp_count", + [ + (0.0, [49, 42, 42, 42]), + (0.2, [25, 30, 30, 30]), + (0.5, [25, 30, 30, 20]), + (0.8, [25, 20, 30, 20]), + (1.0, [25, 20, 20, 20]), + ], +) +def test_zonal_stats_bulk_exactextract(tmp_path, min_coverage_frac, exp_count): + # Prepare test data + sample_dir = SampleData.markers_dir + test_dir = tmp_path / sample_dir.name + shutil.copytree(sample_dir, test_dir) + + geoms = [ + ( + "POLYGON ((161405.210003 188506.174464, 161456.432668 188505.089414," + " 161455.476062 188453.597478, 161403.717327 188454.559035, 161405.210003" + " 188506.174464))" + ), + ( + "POLYGON ((161495.16002 188499.432483, 161546.38515 188498.480826," + " 161545.277792 188446.05759, 161493.919267 188447.011718, 161495.16002" + " 188499.432483))" + ), + ( + "POLYGON ((161433.705572 188423.843391, 161484.794828 188422.760828," + " 161483.986074 188364.860805, 161432.368184 188366.22007, 161433.705572" + " 188423.843391))" + ), + ( + "POLYGON ((161513.51231 188417.022976, 161565.006707 188416.199771," + " 161563.840308 188367.780984, 161512.222428 188369.140255, 161513.51231" + " 188417.022976))" + ), + ] + + box_geoms = shapely.from_wkt(geoms) + + gdf_geoms = gpd.GeoDataFrame(geometry=box_geoms, crs="EPSG:31370") + + # write the geofile to a file + vector_path = tmp_path / "vector.gpkg" + gfo.to_file(gdf=gdf_geoms, path=vector_path) + expression = """ + CASE + WHEN fid = 1 + THEN -0.2 + WHEN fid = 2 + THEN 0.5 + WHEN fid = 3 + THEN 0.8 + WHEN fid = 4 + THEN 0.2 + ELSE NULL + END + """ + gfo.add_column(path=vector_path, name="UID", type="TEXT", expression=expression) + + test_image_roi_dir = test_dir / SampleData.image_dir.name / SampleData.roi_name + image1_dir = test_image_roi_dir / SampleData.image_s1_asc_path.parent.name + image2_dir = test_image_roi_dir / SampleData.image_s1_desc_path.parent.name + test_image_paths = list(image1_dir.glob("*.tif")) + if image2_dir: + test_image_paths.extend(image2_dir.glob("*.tif")) + bands = ["VV", "VH"] + images_bands = [(path, bands) for path in test_image_paths] + output_path = tmp_path / f"min_cov_frac_{min_coverage_frac}" + stats = [f"count(min_coverage_frac={min_coverage_frac},coverage_weight=none)"] + + zonal_stats_bulk.zonal_stats( + vector_path=vector_path, + id_column="UID", + rasters_bands=images_bands, + output_dir=output_path, + stats=stats, + engine="exactextract", + ) + + result_paths = list(output_path.glob("*.sqlite")) + result_df = pdh.read_file(result_paths[0]) + for index in range(4): + count = result_df[result_df["index"] == index]["count"].values[0] + assert count == exp_count[index] From 7def42dbac3a836f171a5b1964c53c931f386eed Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:07:03 +0200 Subject: [PATCH 23/40] Avoid very high commited memory being reserved --- .../util/zonal_stats_bulk/_processing_util.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cropclassification/util/zonal_stats_bulk/_processing_util.py b/cropclassification/util/zonal_stats_bulk/_processing_util.py index c6432cf3..9408334a 100644 --- a/cropclassification/util/zonal_stats_bulk/_processing_util.py +++ b/cropclassification/util/zonal_stats_bulk/_processing_util.py @@ -44,6 +44,14 @@ def __exit__(self, type, value, traceback): def initialize_worker(): + """Some default inits.""" + # Reduce OpenMP threads to avoid the committed memory usage becoming huge when using + # multiprocessing. + # Should work for any numeric library used (openblas, mkl,...). + # Ref: https://stackoverflow.com/questions/77764228/pandas-scipy-high-commit-memory-usage-windows + if os.environ.get("OMP_NUM_THREADS") is None: + os.environ["OMP_NUM_THREADS"] = "1" + # We don't want the workers to block the entire system, so make them nice # if they aren't quite nice already. # Remark: on linux, depending on system settings it is not possible to From 1cafab4841a079a1cb2ecdd060ce36ceef937b90 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:08:05 +0200 Subject: [PATCH 24/40] Small improvement to logging --- cropclassification/preprocess/_timeseries_calc_openeo.py | 1 - cropclassification/util/zonal_stats_bulk/__init__.py | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cropclassification/preprocess/_timeseries_calc_openeo.py b/cropclassification/preprocess/_timeseries_calc_openeo.py index 4d72b28c..55ee2da8 100644 --- a/cropclassification/preprocess/_timeseries_calc_openeo.py +++ b/cropclassification/preprocess/_timeseries_calc_openeo.py @@ -98,7 +98,6 @@ def calculate_periodic_timeseries( if temp_dir == "None": temp_dir = Path(tempfile.gettempdir()) - logger.info(f"Calculating timeseries for {len(images_bands)} images") zonal_stats_bulk.zonal_stats( vector_path=input_parcel_path, id_column=conf.columns["id"], diff --git a/cropclassification/util/zonal_stats_bulk/__init__.py b/cropclassification/util/zonal_stats_bulk/__init__.py index 9728871e..a7fe70b3 100644 --- a/cropclassification/util/zonal_stats_bulk/__init__.py +++ b/cropclassification/util/zonal_stats_bulk/__init__.py @@ -1,5 +1,6 @@ """Calculate zonal statistics for a vector file with many raster files.""" +import logging from pathlib import Path from typing import Optional, Union @@ -10,6 +11,8 @@ ) from ._raster_helper import * # noqa: F403 +logger = logging.getLogger(__name__) + def zonal_stats( vector_path: Path, @@ -52,6 +55,10 @@ def zonal_stats( force (bool, optional): False to skip calculating existing output files. True to recalculate and overwrite existing output files. Defaults to False. """ + logger.info( + f"Calculate zonal statistics for {len(rasters_bands)} rasters ({nb_parallel=})" + ) + if isinstance(stats, str): stats = [stats] From bed1b87e8e68377f75cf20591aad546c3ff922e6 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:09:12 +0200 Subject: [PATCH 25/40] Adapt min_parcels_with_data_prc to avoid s1 being lost due to buffer=0 --- cropclassification/general.ini | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cropclassification/general.ini b/cropclassification/general.ini index 0515fca6..ec158e1c 100644 --- a/cropclassification/general.ini +++ b/cropclassification/general.ini @@ -147,13 +147,13 @@ engine = exactextract # - "pyqgis": "count", "sum", "mean", "median", "std", "min", "max", "range", "minority", "majority" and "variance". # - "exactextract": documentation: https://isciences.github.io/exactextract/operations.html stats = [ - "count(min_coverage_frac=0.8,coverage_weight=none)", - "mean(min_coverage_frac=0.8,coverage_weight=none)", - "median(min_coverage_frac=0.8,coverage_weight=none)", - "stdev(min_coverage_frac=0.8,coverage_weight=none)", - "min(min_coverage_frac=0.8,coverage_weight=none)", - "max(min_coverage_frac=0.8,coverage_weight=none)" - ] + "count(min_coverage_frac=1,coverage_weight=none)", + "mean(min_coverage_frac=1,coverage_weight=none)", + "median(min_coverage_frac=1,coverage_weight=none)", + "stdev(min_coverage_frac=1,coverage_weight=none)", + "min(min_coverage_frac=1,coverage_weight=none)", + "max(min_coverage_frac=1,coverage_weight=none)" + ] # Negative buffer to apply to input parcels to account for mixels. buffer = 0 @@ -161,7 +161,7 @@ buffer = 0 max_cloudcover_pct = 15 # The min percentage of parcels that need to have valid data for a time+sensor to use it -min_parcels_with_data_pct = 80 +min_parcels_with_data_pct = 75 # Configuration specific to the marker being calculated. [marker] From 61cccb575ee16b1bde800ca1a6872363cc5d21d3 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:09:22 +0200 Subject: [PATCH 26/40] Update calc_cropclass.py --- cropclassification/calc_cropclass.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cropclassification/calc_cropclass.py b/cropclassification/calc_cropclass.py index 2d2d006f..28253f29 100644 --- a/cropclassification/calc_cropclass.py +++ b/cropclassification/calc_cropclass.py @@ -189,6 +189,7 @@ def run_cropclass( parceldata_aggregations_to_use = conf.marker.getlist( "parceldata_aggregations_to_use" ) + ts.calc_timeseries_data( input_parcel_path=imagedata_input_parcel_path, roi_bounds=tuple(conf.roi.getlistfloat("roi_bounds")), From 306f1726dc1afbce497ce85367c69383a0e1ab31 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:09:45 +0200 Subject: [PATCH 27/40] Disable botocore.credentions info logging --- cropclassification/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cropclassification/__init__.py b/cropclassification/__init__.py index cce4f0c2..afa44509 100644 --- a/cropclassification/__init__.py +++ b/cropclassification/__init__.py @@ -1 +1,7 @@ """Package with functionalities to support agricultural parcel monitoring.""" + +import logging + +# Disable info logging pf botocore.credentials +logger_botocore_credentials = logging.getLogger("botocore.credentials") +logger_botocore_credentials.setLevel(logging.WARNING) From 268ceff294be02ce88e0cfb16cd3f763efa65598 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:10:31 +0200 Subject: [PATCH 28/40] Small improvements to tests --- tests/test_zonal_stats_bulk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index f911d6bd..bfcfcaff 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -98,7 +98,7 @@ def test_zonal_stats_bulk(tmp_path, engine, stats, bands, exp_results_path): assert not any(result_df["mean"].isna()) -def test_zonal_stats_bulk_invalid(tmp_path): +def test_exactextract_invalid_stats(tmp_path): # Prepare test data sample_dir = SampleData.markers_dir test_dir = tmp_path / sample_dir.name @@ -106,7 +106,7 @@ def test_zonal_stats_bulk_invalid(tmp_path): vector_path = test_dir / SampleData.input_dir.name / "Prc_BEFL_2023_2023-07-24.gpkg" - with pytest.raises(ValueError, match="Error calculating zonal stats"): + with pytest.raises(ValueError, match="Unsupported stat: means"): _zonal_stats_bulk_exactextract.zonal_stats_band( vector_path=vector_path, raster_path=test_dir / SampleData.image_s1_asc_path, @@ -126,7 +126,7 @@ def test_zonal_stats_bulk_invalid(tmp_path): (1.0, [25, 20, 20, 20]), ], ) -def test_zonal_stats_bulk_exactextract(tmp_path, min_coverage_frac, exp_count): +def test_exactextract_min_coverage_frac(tmp_path, min_coverage_frac, exp_count): # Prepare test data sample_dir = SampleData.markers_dir test_dir = tmp_path / sample_dir.name From fda5432c088fb785a68d0a5c42bf58ff4464af30 Mon Sep 17 00:00:00 2001 From: Pieter Roggemans Date: Sun, 24 Aug 2025 17:10:59 +0200 Subject: [PATCH 29/40] Simplify test --- tests/test_zonal_stats_bulk.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/test_zonal_stats_bulk.py b/tests/test_zonal_stats_bulk.py index bfcfcaff..9448113b 100644 --- a/tests/test_zonal_stats_bulk.py +++ b/tests/test_zonal_stats_bulk.py @@ -162,20 +162,7 @@ def test_exactextract_min_coverage_frac(tmp_path, min_coverage_frac, exp_count): # write the geofile to a file vector_path = tmp_path / "vector.gpkg" gfo.to_file(gdf=gdf_geoms, path=vector_path) - expression = """ - CASE - WHEN fid = 1 - THEN -0.2 - WHEN fid = 2 - THEN 0.5 - WHEN fid = 3 - THEN 0.8 - WHEN fid = 4 - THEN 0.2 - ELSE NULL - END - """ - gfo.add_column(path=vector_path, name="UID", type="TEXT", expression=expression) + gfo.add_column(path=vector_path, name="UID", type="TEXT", expression="fid") test_image_roi_dir = test_dir / SampleData.image_dir.name / SampleData.roi_name image1_dir = test_image_roi_dir / SampleData.image_s1_asc_path.parent.name From aa4c702c493e002382b4c0a5d1a4fccd77d4bb1f Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 27 Apr 2026 15:20:47 +0200 Subject: [PATCH 30/40] updated version --- .../_inputdata/Prc_BEFL_2023_2023-07-24.gpkg | Bin 106496 -> 131072 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/markers/_inputdata/Prc_BEFL_2023_2023-07-24.gpkg b/markers/_inputdata/Prc_BEFL_2023_2023-07-24.gpkg index b3cca2d7f02f852a4f1a223cfd82dff5fa38d088..222223acb2055e36612fbe3f1399078ed7eb9d43 100644 GIT binary patch delta 10851 zcmeHN4RBl4m3~*UCE1DnVmr2+`0qs|u^h{?B>&2>43e!#4z}z_auQ>VP^@Q1M3xju z&JTeo{tJ+FmpX#HlI#R%fX-%_fX4*~IzVPAKc%pwK*|m+Q(&OY7TB54O|m2;?78p9 zmTZ|}+Szn>26<$?@4R!)J?GqW&wcmax9_xM-#*!o3$j)b1VQoN6!_IlsH!Sf5$gUw zrveg@7vVoqNM0c?lIIs6N!z;U*+tu=d(%o%cS;Q@^`g(wW0S-AD1D@5iL{`rLN=Ba z`@^M@skc5pA)Y$**{eu$V}?ZTsEBRs-XqBO zrm{3|kZEg>RG(WQlF4Edm-CpXJ5b8hrD%E(w?h)YA~K)J=w6(WKAA?#(xlUp7bJUA z-%6FI(Bhwo9wL4szm(3E7tKaLRW9VGojUHFf4R%C8WQMu9!{%_> zjUY?)g$6^e!G5o+e;c)~!)|W1*r=P$-IQ{W*Kz(_ZiC>I((5Tx%@YnL73Nxvfp|1t zyliBVQYtSkMWbEwgO(75qJ;xQfgyr3RGdA7uHZJR)85){w0E;!7#+=&ZBd7lvURn! zslX)&VosE5F|RXrwK=I$+*4sXQi^Xu^!y%o$m1Fcc`2uPGYHKHLU3J?-9b8$KC34} z?}}9J8uWo8zW*2EEG-*m9zKe0Lj^d8ZktUfP3)7)qs-N#NE^HTO6t_nE7=I;$4-7z zf#g%a|E3sCm7TvNU5>{cBEQDjm`i@W2+3BbHi{laza{RP7~T>c={k6#TuzMcFjR`g&9eb%qo&wItlTCI|-r+KJWfqY%II={(sGRPHe;+PoI|_bp(|w-vBpHE#HB zz_>jP$g|d)zTug z&(f2Y`#|{p<3U3{;1clr2_9cd>5>}3t>7gfV8L@KQr17qzld4)J_CH@A@~5st$qp^ z$L)E*nE#Lb{hTK;9?v^?LX%J2yMq}~Aj9&ELc&3a+AsyDq=f15D3-9*hF>wQQYl7qh1VjaJOf_V+eD zxAv!y{eqrEkpeZ+7i#s|_3PKS(|ucMAMMxbG`gBm*@4$KGV|+r7ej*N@RvJ@)uVK<%S(xmXl&5PUmOEQa0zi0kfkRFwVl;0dF^?zuycPk5(Em zt^#_%nC}i=UIQ2h{4T(Fl&=PihsQ8roZTzMQK@mDzrTMB2v~3>FyL`>05JAH2QVJ( z_W;IxGQe06-^cvdMBH$B95AN;9`Iv9L;7ccal`+}%g>@H&(I76 ztl&Qp=dlAYR&*9HHpmAU51OX{V^8)2#-2|A#)cgOjO*3CfOCQ2dBE7fy?`i4oT|MoW&zN&IQa?7Qk4r2rxEi2r#ZF62RD#0l?URd>;1##)Z@X z7)P)bFm7lIkL7@I*G2$?ebJL25}bv*fPf=#hR6FkhLf)V#-82-7<=?fz*ymr0As^` z0T@@P1AwvTPXfk#V}P;2#{uI8ckul81IC7L1AI|wxb=Py5O7Gbg;?Q@fN_X=01xz! z=S$_{dy@?`(OC_1SOGLThUDL;>}^h!i<1^T(F-f%%3gG&EJZFJ zWlIlAtSO@7?+Y$KWKV8_YJ#)-dKJa3V!1eJNfaG_H?Amd7J>Omd+PW*g84$x!X18n zilQc2QA|g2@z_RghRn&y!NG(lDX89}%Gq&xkMw1D^zlf!R|3*j04%#iSmKIgfB z8PG#oJ)}o@s9^QMGa)%~PqLxT{JH@iGOjkD-!0vrb?{68p5*vE&4N75VlFj;Q91*9 zR&+yE-Q3Wc%_bFPYPXDR1UvBa3FFX1IA&{T^{=neCWOm( zjx+=Qi!)y($U~$y^VP}WQ<*BSShAKV-MJ!R>tTV-8|IfCsG?d};fH9qr($Qw8=+ky zN$ETt2?|-s+7{R4^iu zH9=0qoN*!TD)MBpJ zVr$*&r-2uC4UfMMv=^*JU)*G^V@_J@e)a`NKU--n`o?E1E*P`co&MHZ_sI%o$z7<3 z$-N8Z_be#2khTTr((X&v;_N(Y@y%aa>y9>9>)!nH4RH z{djExji;^GNef(h*_f2MdKgtQXJcqJ^YJ01VD|k4NwTYX$*#^8ya&ijuBJ9w%{Ho| ztwr4r8(b?4l+C<}y)@cdC?y61bYPI#^8i|&g0BuAM!8t)iYS(}n<;i%j8gI%RDyij zTFPu|$yiC;_rJ7VnZxgO&fTxfV)yu5{*jg%A$l(zgXon!JR7|hbDMcK1`I^c#Ju=0 zGNq69XxSLG-7*-{X;hOIkL%g@gynO@}CdM&mHjW0NADH&TkkVX~HyYcz z%nq(Apo>(L!|7`G`Zv$?O&LdW@0v&Q`n%_I#gVu^8Zm2FpGWVGguDX-bSM-F(X=ZV z>T_+OnI#OwW+sc4>*hpY9fgxZ$fi|^k@)_}q(~A3Hi1T|O4hjXDy~EJ|0J$Mq|3h3 znzHX@O=7n&=Z_%jdPRN|sig(IFk*Ku5cxc2?GaSYhF|XWhv`s+cQ|qQlboYUKfRM1 zt#2GbE7&Z%uI`Lc6;iogkM8MVhai_g{XQ4$_f)xJ4_}tWO0KQM$&2m0 zwk*o#PiIG4x240TqPkjJxNU5uz7`U@xueDG>M*V6XG+KcdbPB}IWV9JefzjbKI$b7< z;7~P?a9}Lvc3x6fgB`~-TdRYkl_)u)v8~NzGCQ0swJh^1;zfCHFB)!*-F~^0dHX0* z^QQ;tBN9qq!f zo0(Q-${||`;DjLaIvnEU#hv2JoW=Z~IK;_VL~bIGG@FoSOS313CDQ1RO(H2#U?_T| zhCfu=8i@oODk}#=eSPY_z+mO}0dH9CiFkywqQ(_19nH?}PBVq?P@P?-w$^5<#!0iYm3PKB}&-Sm%Xp{B;l zIDqeQXLE7H#>#ml%;$6a2Zr1OG-#N)#B_WV*2KHnSjnO&OZP^6m^s4!Vv2i9I1+OA zMSwVS$!hlcw}mTNL;Gnu47b@Id<)J@10Jt`0BAFpnBEr(V}7^_2i*~`+XqLZ;oV`% z4`2sfvMvxBbVm$|P1_=D09ekQ+agV!_Kx+<_SlEZi_=X;2OO4mSeosJ3i1jm8%PHH zZXac$eFHS5)z|41Dsx*~Yo{ZoEm);%GutdqYnQFn*`t7G_Ykd6X?uHnRrrL_+}@+G zz|`#D>FwL9P-#>~o2AVhd#-?r0!M30kHXyPuqag3RoW^bz#%1D20Ls$3f!94?Wfw@ zgIn+{f?I7jJFOicB+!}zes3QI|NF69K1!)`be1XN^y=$1I&6y3ZUkAg9g9NPhdlIh zD$q~$yM1Ar3cxdj*RN3NP1I@&gd$rxwb40xYw(;? zU$3sNs{cNU>u?%2J9?Vkp$Hv@WK!vRRmRP&jvfWgo37jp`GSF}!`=eJE^n1sxF7oV(x*dyVDE_51Heq=@9lXJQUhN?^aQfKrk>6atF6U zHV?XkLAa+@fT>Y<`GFq^LxWTW74V0*20}ExXEJqc?twn5QR(0diBY3f)v9p^&5kM< zRCSu(-o{Eiz}cbzWg!>}+)>?Bj(-fkzCNnl=Z?5E2E7KZ0)DuPOW-J*Y#fW$FgN=+ zy3R10IaK*BP%zv3RQYbealJ;B(>`A?Z1DOcKB|1k?~Q~FgH*Xc;POD3g8IrDCzu+( z9ZI*SNmpB~X{-#h*E6?5aD(*^-+^o4hI^c(Vc~GMFkF*%`D6zaaF_L_vAc2$?w1 z4tU%?bs#h_GrwA*EylL?Az#GX5*T!Q{q6j#8La+z0J+oN(n@WkcN-KNEUgX~U#b;N zcxIiQSQ{&Wjk9i6ys^@@UW~vc zyQZ;qwd3oq)+o2?z#w5gg9$T>8D4UBA%pM6FJoBoxJ3-c+!BUg!Nd``MS(zwcC@&x z{NhER0f}3=#1XhfOFV&Fv9J_=xq??K7z=9>exVXi;+83Klp9^0vj!!sQ7}{d3Pr%e u>Lk4kHt!BkKX(jSka)zpX1ZYlS}MNuaU8&b*jziI!3jSM2>~Y|-2)!Bsr~HQCNyhU{ z>y@AJvUj$VrgfKi1)zT8h%frz67B~nml<@KlKmbjOF&`E%x5_+(tJIOycLF&RCj|b z;EaCRr8-sk!`cV^Coii zBgiF_vyer$U4V2jOLCNET!76#+S_7|oSRumIxoPgF7uvHc%K=^G}g=+2v#AQHP5)u5~px2!kJH?<~sP6Iwt)=&1 zzMywGlGNi5@PS~pJ*ilFa=VaZmd2vfpWxtKu3)v`55xhBRD^(TOGUU6!R@H_wSObf z4c|qe%*g0u(x1@i((+WZoG#VCcGB$cW|jSUgJ-OI=BFGnX<#*Zmjx3imuToU{3#2K zY}b>6XG-+UKtWXzo1)BQv|5QDA*3~A>+D`eHgJexPM$b%;>ho0{QK*g$o6++yyNPj zmH3vNA2Ie%)!dQuu)qnjdGdVv>mBEcF^xW%<8JuvZFypK{k245UXFb5PtXyA8nh3E z`|duxXH3Q)xnezHydmdRS6)AGclv%RIjSJs_vWh4Mw8JCt2-@fOC{-6gFk&~2)U{T zHy0k5@avhL0#dqdktnin8@$2!1}1z|Q>$dUj+9h^vgQv%P0xNFmGNioR}Tf~zYVQ# z`*a`GCV8Nz-Yj=eYDNpIPEI`d(k1eg26_$G#*Fs6=aU3WL)N8J;~w1(*_?J@;$c10 z*GisAU!-q%q!?VFk}2z%w7;p&D(h*PY*Q+Gf`gL>Y4m)Vyy#q9Cg+`y4V-mwvR%(S zUl8|sj0Gk6PyOOBGzI?voqljKOmo#5uO_Wd$1r4H9h4>0cDfFRS#y1Ty*sqKHPjZ` zUs`4@D;I?7LR{^(Yv@x0I?kZuv@8Auacv;CWfnbYT4;&^IiN`fP14zS5t;0Q(p{48-0PM(BIKt(O=MI^d5Q}jnb&oh@3u3OaJmwSU+?O8r9kl9fns29fl8o>8Lmr zc2vAKPQ0hUxZ?FI6dQ9GrnWjNUYd4Pym5`Ro`O7WiNkQG&0$!lRVf|g+-w;PKtT478k^Zv35`P4?|8)T#&_rs|CAL#I=G; zpp11sk2`XBX3fS$`si#yLSbQ&*Ai3PdvHxm?Q^>%wI$?V=iz}xk}1)BCDXPW^Cm(3 zm6EaO36Slhu=;Cc7cY{%#8#*17kmNi@nN6fb@4TVbSinIC+#xh9lWbS@XuQ$9gv0o zK+wH+|09b`vlL_*YeAZ>1@wyMK2)$GVZYTA~>Nh2d+pTEB_bM5^WU zTc&9gu(V~q%S83+aBOe)=PpfD9(n6G@m((6$VtZ?*pLyc;cEMq1NAu%tAzEaiP~^gmhhUZT_IP|+9;uoxps$zgPTJWTSlZdvDTZlhv%~)Z DfpSS3 From f8d67a148a102ac8799c407381d0ebe6dee88c1a Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 30 Apr 2026 07:33:55 +0200 Subject: [PATCH 31/40] expect float16 in stead of float32 --- tests/test_raster_index_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 12e3e0ab..8f6ac23f 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,20 +236,20 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float32", np.nan), + ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), ( "ndvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["B04", "B08"], - "float32", + "float16", np.nan, ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float32", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float32", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), ], ) def test_calc_index_by_gdal_raster( From 0811a8a121561ebcd61f0f3ac324449e0da11fd3 Mon Sep 17 00:00:00 2001 From: KriWay Date: Thu, 30 Apr 2026 07:56:47 +0200 Subject: [PATCH 32/40] undo the changes --- tests/test_raster_index_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 8f6ac23f..12e3e0ab 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,20 +236,20 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), + ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float32", np.nan), ( "ndvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["B04", "B08"], - "float16", + "float32", np.nan, ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float32", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float32", np.nan), ], ) def test_calc_index_by_gdal_raster( From 6e8b0b97a9a19fa90ca6acb22b019a5121c03fc4 Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 4 May 2026 10:39:45 +0200 Subject: [PATCH 33/40] redo changes --- tests/test_raster_index_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 12e3e0ab..8f6ac23f 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,20 +236,20 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float32", np.nan), + ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), ( "ndvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["B04", "B08"], - "float32", + "float16", np.nan, ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float32", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float32", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), + ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), ], ) def test_calc_index_by_gdal_raster( From b4b70b994ea3007a4341f49d6761caf7452a295d Mon Sep 17 00:00:00 2001 From: KriWay Date: Mon, 4 May 2026 13:15:39 +0200 Subject: [PATCH 34/40] pinned version rasterio and tests with minimal python 3.12 --- .github/workflows/tests.yml | 4 ++-- ci/envs/latest.yml | 2 +- ci/envs/minimal.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e7c2268..17443b7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: matrix: os: [ubuntu-latest] dev: [false] - python: ["3.10", "3.11", "3.12"] + python: ["3.12"] env: ["latest"] # Use openblas instead of mkl saves 600 MB. Linux OK, 50% slower on Windows and OSX! extra: ["nomkl"] @@ -36,7 +36,7 @@ jobs: - env: minimal os: ubuntu-latest dev: false - python: "3.10" + python: "3.12" - env: latest os: macos-latest dev: false diff --git a/ci/envs/latest.yml b/ci/envs/latest.yml index 5cc83c7c..336634fb 100644 --- a/ci/envs/latest.yml +++ b/ci/envs/latest.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio + - rasterio >=1.5 - rasterstats - rioxarray - scikit-learn diff --git a/ci/envs/minimal.yml b/ci/envs/minimal.yml index 9d8f287a..2d5d1f75 100644 --- a/ci/envs/minimal.yml +++ b/ci/envs/minimal.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio + - rasterio =1.5 - rasterstats - rioxarray - scikit-learn From 94645468352ed7dfa057e1c3c07d62566d99f675 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:04:19 +0200 Subject: [PATCH 35/40] use python 3.13 for macos-latest test --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17443b7e..96f92e30 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: matrix: os: [ubuntu-latest] dev: [false] - python: ["3.12"] + python: ["3.10", "3.11", "3.12", "3.13"] env: ["latest"] # Use openblas instead of mkl saves 600 MB. Linux OK, 50% slower on Windows and OSX! extra: ["nomkl"] @@ -40,7 +40,7 @@ jobs: - env: latest os: macos-latest dev: false - python: "3.12" + python: "3.13" - env: latest os: windows-latest dev: false From 16c7b7402e97b648f167c6eaa5a6faff4442c514 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:05:02 +0200 Subject: [PATCH 36/40] skip tests when certain rasterio version --- environment-dev.yml | 2 +- tests/test_raster_index_util.py | 92 +++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index be31ac47..93f95dbc 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -16,7 +16,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio + - rasterio <1.5 - rasterstats - requests - rioxarray diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 8f6ac23f..197e2861 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -236,8 +236,19 @@ def test_calc_index_s2(tmp_path, index, pixel_type): [ ("ndvi", "BYTE", gdal.GDT_UInt16, 32676, ["B04", "B08", "b1"], "uint8", 255), ("ndvi", "BYTE", gdal.GDT_Float32, np.nan, ["B04", "B08"], "uint8", 255), - ("ndvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["B04", "B08"], "float16", np.nan), - ( + pytest.param( + "ndvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["B04", "B08"], + "float16", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( "ndvi", "FLOAT16", gdal.GDT_Float32, @@ -245,11 +256,84 @@ def test_calc_index_s2(tmp_path, index, pixel_type): ["B04", "B08"], "float16", np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( + "ndvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["B04", "B08"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), + ), + pytest.param( + "ndvi", + "FLOAT16", + gdal.GDT_Float32, + np.nan, + ["B04", "B08"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), ("dprvi", "BYTE", gdal.GDT_Float32, np.nan, ["VH", "VV"], "uint8", 255), - ("dprvi", "FLOAT16", gdal.GDT_UInt16, 32676, ["VH", "VV"], "float16", np.nan), - ("dprvi", "FLOAT16", gdal.GDT_Float32, np.nan, ["VH", "VV"], "float16", np.nan), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["VH", "VV"], + "float16", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_Float32, + np.nan, + ["VH", "VV"], + "float16", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ < "1.5", reason="Requires rasterio 1.5 or higher" + ), + ), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_UInt16, + 32676, + ["VH", "VV"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), + ), + pytest.param( + "dprvi", + "FLOAT16", + gdal.GDT_Float32, + np.nan, + ["VH", "VV"], + "float32", + np.nan, + marks=pytest.mark.skipif( + rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + ), + ), ], ) def test_calc_index_by_gdal_raster( From 77090f6d3e94f8219aa9666497a25880088d638d Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:45:22 +0200 Subject: [PATCH 37/40] rasterio not pinned --- ci/envs/latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/envs/latest.yml b/ci/envs/latest.yml index 336634fb..5cc83c7c 100644 --- a/ci/envs/latest.yml +++ b/ci/envs/latest.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio >=1.5 + - rasterio - rasterstats - rioxarray - scikit-learn From 0b61113902f6fa2da028bd3e3cca4ec2f5073399 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 10:52:43 +0200 Subject: [PATCH 38/40] set minimal python version to 3.10 --- .github/workflows/tests.yml | 2 +- ci/envs/minimal.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 96f92e30..b5fc907c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: - env: minimal os: ubuntu-latest dev: false - python: "3.12" + python: "3.10" - env: latest os: macos-latest dev: false diff --git a/ci/envs/minimal.yml b/ci/envs/minimal.yml index 2d5d1f75..9d8f287a 100644 --- a/ci/envs/minimal.yml +++ b/ci/envs/minimal.yml @@ -15,7 +15,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio =1.5 + - rasterio - rasterstats - rioxarray - scikit-learn From e7507089a14a49cceacfbd68f532c93b98c828f7 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 12:20:36 +0200 Subject: [PATCH 39/40] refactor yml --- .github/workflows/tests.yml | 4 +-- environment-dev.yml | 2 +- tests/test_raster_index_util.py | 45 ++++++++++++++++++++++++++++----- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5fc907c..a010a076 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,11 +40,11 @@ jobs: - env: latest os: macos-latest dev: false - python: "3.13" + python: "3.12" - env: latest os: windows-latest dev: false - python: "3.12" + python: "3.13" steps: - uses: actions/checkout@v6 diff --git a/environment-dev.yml b/environment-dev.yml index 93f95dbc..be31ac47 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -16,7 +16,7 @@ dependencies: - pandas - psutil - pyproj - - rasterio <1.5 + - rasterio - rasterstats - requests - rioxarray diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index 197e2861..fd6d8c26 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -126,11 +126,38 @@ def test_calc_index_invalid(tmp_path): "index, pixel_type, process_options, expected_bands", [ ("dprvi", "BYTE", {}, ["dprvi"]), - ("dprvi", "FLOAT16", None, ["dprvi"]), + pytest.param( + "dprvi", + "FLOAT16", + None, + ["dprvi"], + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), ("dprvi", "FLOAT32", {}, ["dprvi"]), ("rvi", "BYTE", {}, ["rvi"]), - ("vvdvh", "FLOAT16", {}, ["vvdvh"]), - ("sarrgb", "FLOAT16", {}, ["vv", "vh", "vvdvh"]), + pytest.param( + "vvdvh", + "FLOAT16", + {}, + ["vvdvh"], + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), + pytest.param( + "sarrgb", + "FLOAT16", + {}, + ["vv", "vh", "vvdvh"], + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), ("sarrgb", "FLOAT32", {"log10": True}, ["vvdb", "vhdb", "vvdvhdb"]), ("sarrgb", "BYTE", {"log10": True}, ["vvdb", "vhdb", "vvdvhdb"]), ( @@ -269,7 +296,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), pytest.param( @@ -281,7 +309,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), ("dprvi", "BYTE", gdal.GDT_UInt16, 32676, ["VH", "VV"], "uint8", 255), @@ -319,7 +348,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), pytest.param( @@ -331,7 +361,8 @@ def test_calc_index_s2(tmp_path, index, pixel_type): "float32", np.nan, marks=pytest.mark.skipif( - rasterio.__version__ >= "1.5", reason="Requires rasterio < 1.5" + rasterio.__version__ >= "1.5" or rasterio.__version__ == "1.4.4", + reason="Requires rasterio < 1.5", ), ), ], From 6f2bdc6c61b4f6ff4eb8037f5b30ec1977c39c61 Mon Sep 17 00:00:00 2001 From: KriWay Date: Tue, 5 May 2026 12:38:41 +0200 Subject: [PATCH 40/40] refactor yml --- tests/test_raster_index_util.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_raster_index_util.py b/tests/test_raster_index_util.py index fd6d8c26..b7e64022 100644 --- a/tests/test_raster_index_util.py +++ b/tests/test_raster_index_util.py @@ -238,7 +238,26 @@ def test_calc_index_s1_error( @pytest.mark.parametrize( - "index, pixel_type", [("ndvi", "BYTE"), ("ndvi", "FLOAT16"), ("bsi", "FLOAT16")] + "index, pixel_type", + [ + ("ndvi", "BYTE"), + pytest.param( + "ndvi", + "FLOAT16", + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), + pytest.param( + "bsi", + "FLOAT16", + marks=pytest.mark.skipif( + rasterio.__version__ == "1.4.4", + reason="Requires rasterio <> 1.4.4", + ), + ), + ], ) def test_calc_index_s2(tmp_path, index, pixel_type): # Prepare test data